def _preprocess_post_data(self, data): project_id = data["project_id"] info = data["info"] duplicate = task_repo.find_duplicate(project_id=project_id, info=info) if duplicate: message = { 'reason': 'DUPLICATE_TASK', 'task_id': duplicate } raise Conflict(json.dumps(message)) if 'n_answers' not in data: project = project_repo.get(project_id) data['n_answers'] = project.get_default_n_answers() user_pref = data.get('user_pref', {}) if user_pref.get('languages'): user_pref['languages'] = [s.lower() for s in user_pref.get('languages', [])] if user_pref.get('locations'): user_pref['locations'] = [s.lower() for s in user_pref.get('locations', [])] if user_pref.get('assign_user'): user_pref['assign_user'] = [s.lower() for s in user_pref.get('assign_user', [])] invalid_fields = validate_required_fields(info) if invalid_fields: raise BadRequest('Missing or incorrect required fields: {}' .format(','.join(invalid_fields))) if data.get('gold_answers'): try: gold_answers = data['gold_answers'] if type(gold_answers) is dict: data['calibration'] = 1 data['exported'] = True except Exception as e: raise BadRequest('Invalid gold_answers')
def _import_csv_tasks(self, csvreader): """Import CSV tasks.""" fields = set(['state', 'quorum', 'calibration', 'priority_0', 'n_answers', 'user_pref', 'expiration']) self._headers = [] self.field_header_index = [] row_number = 0 for row in csvreader: if not self._headers: self._headers = row self._check_no_duplicated_headers() self._check_no_empty_headers() self._check_required_headers() field_headers = set(self._headers) & fields for field in field_headers: self.field_header_index.append(self._headers.index(field)) else: row_number += 1 self._check_valid_row_length(row, row_number) # check required fields fvals = {self._headers[idx]: cell for idx, cell in enumerate(row)} invalid_fields = validate_required_fields(fvals) if invalid_fields: msg = gettext('The file you uploaded has incorrect/missing ' 'values for required header(s): {0}' .format(','.join(invalid_fields))) raise BulkImportException(msg) task_data = self._convert_row_to_task_data(row, row_number) yield task_data
def csv_validate_required_fields(self, config, callback): """Test validate_required_fields against csv data with data_access, data_source_id, data_owner.""" with patch.dict(self.flask_app.config, config): fake_csv = ('line,data_access,data_source_id,data_owner\n' 'test,"[""L4""]",123.0,456\n' 'test,"[""L4""]",123.6,456\n' 'test,"[""L4""]",abc,456\n' 'test,"[""L4""]",,456\n' 'test,"[""L4""]",123,456.0\n' 'test,"[""L4""]",123,456.6\n' 'test,"[""L4""]",123,abc\n' 'test,"[""L4""]",123,\n' 'test,,123,456\n' 'test,"[""L4""]",123,456') csvreader = csv.reader(fake_csv.splitlines()) csviterator = iter(csvreader) for index, row in enumerate(csviterator): if index == 0: # Read csv header. headers = row else: # Read csv data and check required fields. fvals = {headers[idx]: cell for idx, cell in enumerate(row)} invalid_fields = util.validate_required_fields(fvals) # Allow client to assert on result. callback(index, invalid_fields)
def test_integer_required_cast_string(self): """Test importing an integer (BBDS) with validate_required_fields.""" config = {'TASK_REQUIRED_FIELDS': { 'data_access': {'val': None, 'check_val': False}, 'data_owner': {'val': None, 'check_val': False, 'require_int': True}, 'data_source_id': {'val': None, 'check_val': False, 'require_int': True}}} # While csv imports as string values, we explicitly set an integer value. data = {'data_access': "1", 'data_owner': 5, 'data_source_id': '2'} with patch.dict(self.flask_app.config, config): invalid_fields = util.validate_required_fields(data) assert len(invalid_fields) == 0
def _preprocess_post_data(self, data): project_id = data["project_id"] info = data["info"] duplicate = task_repo.find_duplicate(project_id=project_id, info=info) if duplicate: message = {'reason': 'DUPLICATE_TASK', 'task_id': duplicate} raise Conflict(json.dumps(message)) if 'n_answers' not in data: project = Project(**get_project_data(project_id)) data['n_answers'] = project.get_default_n_answers() invalid_fields = validate_required_fields(info) if invalid_fields: raise BadRequest('Missing or incorrect required fields: {}'.format( ','.join(invalid_fields)))
def _import_csv_tasks(self, csvreader): """Import CSV tasks.""" headers = [] fields = set([ 'state', 'quorum', 'calibration', 'priority_0', 'n_answers', 'user_pref' ]) field_header_index = [] row_number = 0 for row in csvreader: if not headers: headers = self._headers = row self._check_no_duplicated_headers(headers) self._check_no_empty_headers(headers) self._check_required_headers(headers) field_headers = set(headers) & fields for field in field_headers: field_header_index.append(headers.index(field)) else: row_number += 1 self._check_valid_row_length(row, row_number, headers) # check required fields fvals = {headers[idx]: cell for idx, cell in enumerate(row)} invalid_fields = validate_required_fields(fvals) if invalid_fields: msg = gettext( 'The file you uploaded has incorrect/missing ' 'values for required header(s): {0}'.format( ','.join(invalid_fields))) raise BulkImportException(msg) task_data = {"info": {}} for idx, cell in enumerate(row): if idx in field_header_index: if headers[idx] == 'user_pref': if len(cell) > 0: task_data[headers[idx]] = json.loads( cell.lower()) else: task_data[headers[idx]] = {} else: task_data[headers[idx]] = cell else: task_data["info"][headers[idx]] = cell yield task_data
def task_generator(): row_number = 0 for row in csviterator: row_number += 1 self._check_valid_row_length(row, row_number) # check required fields fvals = { self._headers[idx]: cell for idx, cell in enumerate(row) } invalid_fields = validate_required_fields(fvals) if invalid_fields: msg = gettext( 'The file you uploaded has incorrect/missing ' 'values for required header(s): {0}'.format( ','.join(invalid_fields))) raise BulkImportException(msg) task_data = self._convert_row_to_task_data(row, row_number) yield task_data