def test_get_data_for_callback_id(self): payload = pl.Payload(payload=VALID_POST) payload.payload_to_db() cid = payload.callback_id payload_new = pl.Payload(callback_id=cid) for study in payload_new.study_obj_list: self.assertEqual(study.callback_id, cid)
def test_check_basic_content_present(self): data = {'requestEntries': [{}]} payload = pl.Payload(payload=data) self.assertTrue(payload.check_basic_content_present()) missing_data = {'requestEntries': []} payload = pl.Payload(payload=missing_data) self.assertFalse(payload.check_basic_content_present()) missing_all = {} payload = pl.Payload(payload=missing_all) self.assertFalse(payload.check_basic_content_present())
def validate_metadata_for_payload(callback_id, content): payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() payload.set_callback_id_for_studies() payload.validate_payload_metadata() response = construct_validation_response(callback_id, payload) return json.dumps(response)
def publish_and_clean_sumstats(study_list): # 1) move sumstats files to staging for publishing # 2) deactivate globus endpoint moved = 0 callback_id = None globus_endpoint_id = None for s in study_list['studyList']: study = st.Study(study_id=s['id'], file_path=s['file_path'], assembly=s['assembly'], callback_id=s['callback_id'], readme=s['readme'], entryUUID=s['entryUUID'], author_name=s['author_name'], pmid=s['pmid'], gcst=s['gcst'], raw_ss=s['rawSS']) if study.move_file_to_staging() is True: moved += 1 if callback_id is None: callback_id = s['callback_id'] if globus_endpoint_id is None: globus_endpoint_id = s['entryUUID'] if callback_id and globus_endpoint_id: payload = pl.Payload(callback_id=callback_id) payload.get_data_for_callback_id() if len(payload.study_obj_list) == moved: delete_globus_endpoint(globus_endpoint_id)
def test_get_payload__status(self): payload = pl.Payload(payload=VALID_POST) payload.payload_to_db() payload.get_data_for_callback_id() for study in payload.study_obj_list: study.set_retrieved_status(None) study.store_retrieved_status() study.set_data_valid_status(None) study.store_data_valid_status() status = payload.get_payload_status() self.assertEqual(status, 'PROCESSING') for study in payload.study_obj_list: study.set_retrieved_status(0) study.store_retrieved_status() study.set_data_valid_status(0) study.store_data_valid_status() status = payload.get_payload_status() self.assertEqual(status, 'INVALID') for study in payload.study_obj_list: study.set_retrieved_status(1) study.store_retrieved_status() study.set_data_valid_status(1) study.store_data_valid_status() status = payload.get_payload_status() self.assertEqual(status, 'VALID')
def test_parse_new_study_bad_json(self): data_missing_field = { "id": "xyz321", "md5":"b1d7e0a58d36502d59d036a17336ddf5", "assembly":"GRCh38" } payload = pl.Payload(payload=data_missing_field) study_id, file_path, md5, assembly, readme, entryUUID, rawSS = payload.parse_new_study_json(data_missing_field) self.assertIsNone(file_path)
def validate_metadata(callback_id): metadata_valid = [] payload = pl.Payload(callback_id=callback_id) payload.get_data_for_callback_id() for study in payload.study_obj_list: metadata_valid.append(study.validate_metadata()) if any(metadata_valid) == False: return False else: return True
def parse_payload(content, studyid, callback_id): payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() payload.set_callback_id_for_studies() study_meta = [s for s in payload.study_obj_list if s.study_id == studyid] if len(study_meta) != 1: print("could not find only one matching study id in payload") return False return (study_meta[0].file_path, study_meta[0].md5, study_meta[0].assembly, study_meta[0].readme, study_meta[0].entryUUID)
def validate_files_from_payload(callback_id, content, out=None, minrows=None): payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() payload.set_callback_id_for_studies() payload.validate_payload(minrows=minrows) response = construct_validation_response(callback_id, payload) if out: with open(out, 'w') as out: out.write(json.dumps(response)) return json.dumps(response)
def test_parse_new_study_json(self): data = { "id": "xyz321", "filePath": "file/path.tsv", "md5":"b1d7e0a58d36502d59d036a17336ddf5", "assembly":"GRCh38" } payload = pl.Payload(payload=data) result = payload.parse_new_study_json(data) self.assertEqual("xyz321", result[0]) self.assertEqual("file/path.tsv", result[1]) self.assertEqual("b1d7e0a58d36502d59d036a17336ddf5", result[2]) self.assertEqual("GRCh38", result[3])
def skip_validation_completely(callback_id, content): results = {"callbackID": callback_id, "validationList": []} payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() study_list = [s.study_id for s in payload.study_obj_list] for study in study_list: results['validationList'].append({ "id": study, "retrieved": 99, "dataValid": 99, "errorCode": None }) return json.dumps(results)
def test_create_study_obj_list(self): payload = pl.Payload(payload=VALID_POST) self.assertTrue(payload.create_study_obj_list()) dupe_study = { "requestEntries": [ { "id": "abc123", "filePath": "file/path.tsv", "md5":"b1d7e0a58d36502d59d036a17336ddf5", "assembly":"GRCh38" }, { "id": "abc123", "filePath": "file/path.tsv", "md5":"b1d7e0a58d36502d59d036a17336ddf5", "assembly":"GRCh38" }, ] } payload = pl.Payload(payload=dupe_study) payload.create_study_obj_list() self.assertFalse(payload.check_study_ids_valid())
def store_validation_results_in_db(validation_response): valid = True for item in json.loads(validation_response)['validationList']: study_id = item["id"] study = st.Study(study_id) study.retrieved = item["retrieved"] study.data_valid = item["dataValid"] study.error_code = item["errorCode"] study.store_validation_statuses() if study.error_code: valid = False if valid == False: callback_id = json.loads(validation_response)['callbackID'] payload = pl.Payload(callback_id=callback_id) payload.clear_validated_files()
def delete_payload_from_db(callback_id): payload = pl.Payload(callback_id=callback_id) if not payload: raise RequestedNotFound( "Couldn't find resource with callback id: {}".format(callback_id)) payload.get_data_for_callback_id() payload.remove_callback_id() status_list = [] for study in payload.study_obj_list: status_list.append({"id": study.study_id, "status": "DELETED"}) study.remove() response = { "callbackID": str(callback_id), "status": "DELETED", "statusList": status_list } return json.dumps(response)
def add_errors_if_study_missing(callback_id, content, results): if any([s['errorCode'] for s in results['validationList']]): # if we already identfied errors, there's no need to add them return results else: payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() study_list = [s.study_id for s in payload.study_obj_list] studies_with_results = [s['id'] for s in results['validationList']] for study in study_list: if study not in studies_with_results: results['validationList'].append({ "id": study, "retrieved": None, "dataValid": None, "errorCode": 10 }) return results
def update_payload(callback_id, content): payload = pl.Payload(callback_id=callback_id) payload.get_data_for_callback_id() payload.update_publication_details(content) study_list = [] for study in payload.study_obj_list: study_report = { "id": study.study_id, "gcst": study.gcst, "pmid": study.pmid, "file_path": study.file_path, "assembly": study.assembly, "callback_id": study.callback_id, "readme": study.readme, "entryUUID": study.entryUUID, "author_name": study.author_name, "rawSS": study.raw_ss } study_list.append(study_report) response = {"callbackID": str(callback_id), "studyList": study_list} return response
def construct_get_payload_response(callback_id): response = None payload = pl.Payload(callback_id=callback_id) if payload.get_data_for_callback_id(): if not payload.study_obj_list: # callback registered but studies not yet added (due to async) response = {"callbackID": str(callback_id), "status": "PROCESSING"} if payload.metadata_errors: response["metadataErrors"] = payload.metadata_errors response["status"] = "INVALID" else: payload_status = payload.get_payload_status() status_list = [] for study in payload.study_obj_list: study_report = create_study_report(study) status_list.append(study_report) response = { "callbackID": str(callback_id), "status": payload_status, "statusList": status_list } return response
def json_payload_to_db(content, callback_id=None): payload = pl.Payload(payload=content, callback_id=callback_id) payload.payload_to_db() if len(payload.metadata_errors) != 0: return False return payload.callback_id
def remove_payload_files(callback_id): payload = pl.Payload(callback_id=callback_id) payload.remove_payload_directory()
def move_to_valid(callback_id, content): payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() payload.set_callback_id_for_studies() for study in payload.study_obj_list: study.move_to_valid()
def generate_callback_id(): payload = pl.Payload() payload.generate_callback_id() return payload.callback_id
def test_generate_callback_id(self): payload = pl.Payload() payload.generate_callback_id() self.assertIsNotNone(payload.callback_id)
def results_if_failure(callback_id, content): payload = pl.Payload(callback_id=callback_id, payload=content) payload.create_study_obj_list() results = vp.construct_failure_response(callback_id, payload) return results