def test_get_data_for_callback_id(self):
payload = pl.Payload(payload=VALID_POST)
payload.payload_to_db()
cid = payload.callback_id
payload_new = pl.Payload(callback_id=cid)
for study in payload_new.study_obj_list:
self.assertEqual(study.callback_id, cid)
def test_check_basic_content_present(self):
data = {'requestEntries': [{}]}
payload = pl.Payload(payload=data)
self.assertTrue(payload.check_basic_content_present())
missing_data = {'requestEntries': []}
payload = pl.Payload(payload=missing_data)
self.assertFalse(payload.check_basic_content_present())
missing_all = {}
payload = pl.Payload(payload=missing_all)
self.assertFalse(payload.check_basic_content_present())
def validate_metadata_for_payload(callback_id, content):
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
payload.set_callback_id_for_studies()
payload.validate_payload_metadata()
response = construct_validation_response(callback_id, payload)
return json.dumps(response)
def publish_and_clean_sumstats(study_list):
# 1) move sumstats files to staging for publishing
# 2) deactivate globus endpoint
moved = 0
callback_id = None
globus_endpoint_id = None
for s in study_list['studyList']:
study = st.Study(study_id=s['id'],
file_path=s['file_path'],
assembly=s['assembly'],
callback_id=s['callback_id'],
readme=s['readme'],
entryUUID=s['entryUUID'],
author_name=s['author_name'],
pmid=s['pmid'],
gcst=s['gcst'],
raw_ss=s['rawSS'])
if study.move_file_to_staging() is True:
moved += 1
if callback_id is None:
callback_id = s['callback_id']
if globus_endpoint_id is None:
globus_endpoint_id = s['entryUUID']
if callback_id and globus_endpoint_id:
payload = pl.Payload(callback_id=callback_id)
payload.get_data_for_callback_id()
if len(payload.study_obj_list) == moved:
delete_globus_endpoint(globus_endpoint_id)
def test_get_payload__status(self):
payload = pl.Payload(payload=VALID_POST)
payload.payload_to_db()
payload.get_data_for_callback_id()
for study in payload.study_obj_list:
study.set_retrieved_status(None)
study.store_retrieved_status()
study.set_data_valid_status(None)
study.store_data_valid_status()
status = payload.get_payload_status()
self.assertEqual(status, 'PROCESSING')
for study in payload.study_obj_list:
study.set_retrieved_status(0)
study.store_retrieved_status()
study.set_data_valid_status(0)
study.store_data_valid_status()
status = payload.get_payload_status()
self.assertEqual(status, 'INVALID')
for study in payload.study_obj_list:
study.set_retrieved_status(1)
study.store_retrieved_status()
study.set_data_valid_status(1)
study.store_data_valid_status()
status = payload.get_payload_status()
self.assertEqual(status, 'VALID')
def test_parse_new_study_bad_json(self):
data_missing_field = {
"id": "xyz321",
"md5":"b1d7e0a58d36502d59d036a17336ddf5",
"assembly":"GRCh38"
}
payload = pl.Payload(payload=data_missing_field)
study_id, file_path, md5, assembly, readme, entryUUID, rawSS = payload.parse_new_study_json(data_missing_field)
self.assertIsNone(file_path)
def validate_metadata(callback_id):
metadata_valid = []
payload = pl.Payload(callback_id=callback_id)
payload.get_data_for_callback_id()
for study in payload.study_obj_list:
metadata_valid.append(study.validate_metadata())
if any(metadata_valid) == False:
return False
else:
return True
def parse_payload(content, studyid, callback_id):
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
payload.set_callback_id_for_studies()
study_meta = [s for s in payload.study_obj_list if s.study_id == studyid]
if len(study_meta) != 1:
print("could not find only one matching study id in payload")
return False
return (study_meta[0].file_path, study_meta[0].md5, study_meta[0].assembly,
study_meta[0].readme, study_meta[0].entryUUID)
def validate_files_from_payload(callback_id, content, out=None, minrows=None):
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
payload.set_callback_id_for_studies()
payload.validate_payload(minrows=minrows)
response = construct_validation_response(callback_id, payload)
if out:
with open(out, 'w') as out:
out.write(json.dumps(response))
return json.dumps(response)
def test_parse_new_study_json(self):
data = {
"id": "xyz321",
"filePath": "file/path.tsv",
"md5":"b1d7e0a58d36502d59d036a17336ddf5",
"assembly":"GRCh38"
}
payload = pl.Payload(payload=data)
result = payload.parse_new_study_json(data)
self.assertEqual("xyz321", result[0])
self.assertEqual("file/path.tsv", result[1])
self.assertEqual("b1d7e0a58d36502d59d036a17336ddf5", result[2])
self.assertEqual("GRCh38", result[3])
def skip_validation_completely(callback_id, content):
results = {"callbackID": callback_id, "validationList": []}
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
study_list = [s.study_id for s in payload.study_obj_list]
for study in study_list:
results['validationList'].append({
"id": study,
"retrieved": 99,
"dataValid": 99,
"errorCode": None
})
return json.dumps(results)
def test_create_study_obj_list(self):
payload = pl.Payload(payload=VALID_POST)
self.assertTrue(payload.create_study_obj_list())
dupe_study = {
"requestEntries": [
{
"id": "abc123",
"filePath": "file/path.tsv",
"md5":"b1d7e0a58d36502d59d036a17336ddf5",
"assembly":"GRCh38"
},
{
"id": "abc123",
"filePath": "file/path.tsv",
"md5":"b1d7e0a58d36502d59d036a17336ddf5",
"assembly":"GRCh38"
},
]
}
payload = pl.Payload(payload=dupe_study)
payload.create_study_obj_list()
self.assertFalse(payload.check_study_ids_valid())
def store_validation_results_in_db(validation_response):
valid = True
for item in json.loads(validation_response)['validationList']:
study_id = item["id"]
study = st.Study(study_id)
study.retrieved = item["retrieved"]
study.data_valid = item["dataValid"]
study.error_code = item["errorCode"]
study.store_validation_statuses()
if study.error_code:
valid = False
if valid == False:
callback_id = json.loads(validation_response)['callbackID']
payload = pl.Payload(callback_id=callback_id)
payload.clear_validated_files()
def delete_payload_from_db(callback_id):
payload = pl.Payload(callback_id=callback_id)
if not payload:
raise RequestedNotFound(
"Couldn't find resource with callback id: {}".format(callback_id))
payload.get_data_for_callback_id()
payload.remove_callback_id()
status_list = []
for study in payload.study_obj_list:
status_list.append({"id": study.study_id, "status": "DELETED"})
study.remove()
response = {
"callbackID": str(callback_id),
"status": "DELETED",
"statusList": status_list
}
return json.dumps(response)
def add_errors_if_study_missing(callback_id, content, results):
if any([s['errorCode'] for s in results['validationList']]):
# if we already identfied errors, there's no need to add them
return results
else:
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
study_list = [s.study_id for s in payload.study_obj_list]
studies_with_results = [s['id'] for s in results['validationList']]
for study in study_list:
if study not in studies_with_results:
results['validationList'].append({
"id": study,
"retrieved": None,
"dataValid": None,
"errorCode": 10
})
return results
def update_payload(callback_id, content):
payload = pl.Payload(callback_id=callback_id)
payload.get_data_for_callback_id()
payload.update_publication_details(content)
study_list = []
for study in payload.study_obj_list:
study_report = {
"id": study.study_id,
"gcst": study.gcst,
"pmid": study.pmid,
"file_path": study.file_path,
"assembly": study.assembly,
"callback_id": study.callback_id,
"readme": study.readme,
"entryUUID": study.entryUUID,
"author_name": study.author_name,
"rawSS": study.raw_ss
}
study_list.append(study_report)
response = {"callbackID": str(callback_id), "studyList": study_list}
return response
def construct_get_payload_response(callback_id):
response = None
payload = pl.Payload(callback_id=callback_id)
if payload.get_data_for_callback_id():
if not payload.study_obj_list:
# callback registered but studies not yet added (due to async)
response = {"callbackID": str(callback_id), "status": "PROCESSING"}
if payload.metadata_errors:
response["metadataErrors"] = payload.metadata_errors
response["status"] = "INVALID"
else:
payload_status = payload.get_payload_status()
status_list = []
for study in payload.study_obj_list:
study_report = create_study_report(study)
status_list.append(study_report)
response = {
"callbackID": str(callback_id),
"status": payload_status,
"statusList": status_list
}
return response
def json_payload_to_db(content, callback_id=None):
payload = pl.Payload(payload=content, callback_id=callback_id)
payload.payload_to_db()
if len(payload.metadata_errors) != 0:
return False
return payload.callback_id
def remove_payload_files(callback_id):
payload = pl.Payload(callback_id=callback_id)
payload.remove_payload_directory()
def move_to_valid(callback_id, content):
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
payload.set_callback_id_for_studies()
for study in payload.study_obj_list:
study.move_to_valid()
def generate_callback_id():
payload = pl.Payload()
payload.generate_callback_id()
return payload.callback_id
def test_generate_callback_id(self):
payload = pl.Payload()
payload.generate_callback_id()
self.assertIsNotNone(payload.callback_id)
def results_if_failure(callback_id, content):
payload = pl.Payload(callback_id=callback_id, payload=content)
payload.create_study_obj_list()
results = vp.construct_failure_response(callback_id, payload)
return results
