def test_valid_assembly(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
self.assertTrue(study.valid_assembly())
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly="FAIL")
self.assertFalse(study.valid_assembly())
def test_valid_study_id(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
self.assertTrue(study.valid_study_id())
# invalid study id
study_id = "asd1232 asd"
study = st.Study(study_id=study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
self.assertFalse(study.valid_study_id())
def publish_and_clean_sumstats(study_list):
# 1) move sumstats files to staging for publishing
# 2) deactivate globus endpoint
moved = 0
callback_id = None
globus_endpoint_id = None
for s in study_list['studyList']:
study = st.Study(study_id=s['id'],
file_path=s['file_path'],
assembly=s['assembly'],
callback_id=s['callback_id'],
readme=s['readme'],
entryUUID=s['entryUUID'],
author_name=s['author_name'],
pmid=s['pmid'],
gcst=s['gcst'],
raw_ss=s['rawSS'])
if study.move_file_to_staging() is True:
moved += 1
if callback_id is None:
callback_id = s['callback_id']
if globus_endpoint_id is None:
globus_endpoint_id = s['entryUUID']
if callback_id and globus_endpoint_id:
payload = pl.Payload(callback_id=callback_id)
payload.get_data_for_callback_id()
if len(payload.study_obj_list) == moved:
delete_globus_endpoint(globus_endpoint_id)
def validate_study(callback_id,
study_id,
filepath,
md5,
assembly,
readme,
entryUUID,
out=None,
minrows=None,
forcevalid=False):
study = st.Study(callback_id=callback_id,
study_id=study_id,
file_path=filepath,
md5=md5,
assembly=assembly,
readme=readme,
entryUUID=entryUUID)
study.validate_study(minrows=minrows, forcevalid=forcevalid)
result = {
"id": study.study_id,
"retrieved": study.retrieved,
"dataValid": study.data_valid,
"errorCode": study.error_code
}
print(result)
if out:
with open(out, 'w') as f:
f.write(json.dumps(result))
if study.data_valid != 1:
sys.exit(1)
else:
sys.exit(0)
def test_validate_study_metadata_invalid(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly="FAIL")
study.mandatory_metadata_check()
self.assertEqual(study.error_code, 5)
def test_validate_study_missing_metadata(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly="")
study.validate_study()
self.assertEqual(study.error_code, 4)
def test_mandatory_metadata_check(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
self.assertTrue(study.mandatory_metadata_check())
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly="")
self.assertFalse(study.mandatory_metadata_check())
study = st.Study(study_id=self.study_id,
file_path=None,
md5=self.md5,
assembly=self.assembly)
self.assertFalse(study.mandatory_metadata_check())
def test_validate_study_URL_invalid(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly,
callback_id="1234abcd")
study.validate_study()
self.assertEqual(study.error_code, 1)
def test_create_entry_for_study(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
study.create_entry_for_study()
check = study.get_study_from_db()
self.assertIsNotNone(check)
self.assertEqual(check['studyID'], self.study_id)
def test_validate_study_md5_invalid(self):
valid_url = "file://{}".format(
os.path.abspath("./tests/test_sumstats_file.tsv"))
study = st.Study(study_id=self.study_id,
file_path=valid_url,
md5=self.md5,
assembly=self.assembly,
callback_id="1234abcd")
study.validate_study()
self.assertEqual(study.error_code, 2)
def test_update_data_valid_status(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
study.create_entry_for_study()
check = study.get_study_from_db()
self.assertTrue('dataValid' not in check)
study.set_data_valid_status(1)
study.store_data_valid_status()
check = study.get_study_from_db()
self.assertEqual(check['dataValid'], 1)
def create_study_obj_list(self):
for item in self.payload['requestEntries']:
study_id, file_path, md5, assembly, readme, entryUUID, raw_file_path = self.parse_new_study_json(
item)
study = st.Study(study_id=study_id,
file_path=file_path,
md5=md5,
assembly=assembly,
readme=readme,
entryUUID=entryUUID,
raw_ss=raw_file_path)
self.study_obj_list.append(study)
return True
def get_data_for_callback_id(self):
mdb = mongoClient(config.MONGO_URI, config.MONGO_USER,
config.MONGO_PASSWORD, config.MONGO_DB)
data = mdb.get_data_from_callback_id(self.callback_id)
self.get_metadata_errors()
if data is None:
if mdb.check_callback_id_in_db(self.callback_id):
# callback registered but studies not yet added (due to async)
return True
else:
raise RequestedNotFound(
"Couldn't find resource with callback id: {}".format(
self.callback_id))
for study_metadata in data:
study_id = st.set_var_from_dict(study_metadata, 'studyID', None)
callback_id = st.set_var_from_dict(study_metadata, 'callbackID',
None)
file_path = st.set_var_from_dict(study_metadata, 'filePath', None)
md5 = st.set_var_from_dict(study_metadata, 'md5', None)
assembly = st.set_var_from_dict(study_metadata, 'assembly', None)
retrieved = st.set_var_from_dict(study_metadata, 'retrieved', None)
data_valid = st.set_var_from_dict(study_metadata, 'dataValid',
None)
error_code = st.set_var_from_dict(study_metadata, 'errorCode',
None)
readme = st.set_var_from_dict(study_metadata, 'readme', None)
entryUUID = st.set_var_from_dict(study_metadata, 'entryUUID', None)
author_name = st.set_var_from_dict(study_metadata, 'authorName',
None)
pmid = st.set_var_from_dict(study_metadata, 'pmid', None)
gcst = st.set_var_from_dict(study_metadata, 'gcst', None)
raw_ss = st.set_var_from_dict(study_metadata, 'rawSS', None)
study = st.Study(study_id=study_id,
callback_id=callback_id,
file_path=file_path,
md5=md5,
assembly=assembly,
retrieved=retrieved,
data_valid=data_valid,
error_code=error_code,
readme=readme,
entryUUID=entryUUID,
author_name=author_name,
pmid=pmid,
gcst=gcst,
raw_ss=raw_ss)
self.study_obj_list.append(study)
return self.study_obj_list
def store_validation_results_in_db(validation_response):
valid = True
for item in json.loads(validation_response)['validationList']:
study_id = item["id"]
study = st.Study(study_id)
study.retrieved = item["retrieved"]
study.data_valid = item["dataValid"]
study.error_code = item["errorCode"]
study.store_validation_statuses()
if study.error_code:
valid = False
if valid == False:
callback_id = json.loads(validation_response)['callbackID']
payload = pl.Payload(callback_id=callback_id)
payload.clear_validated_files()
def test_update_error(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
study.create_entry_for_study()
study.get_study_from_db()
self.assertIsNone(study.error_code)
self.assertIsNone(study.error_text)
study.set_error_code(1)
study.store_error_code()
study.get_study_from_db()
self.assertEqual(study.error_code, 1)
self.assertEqual(study.error_text,
"The summary statistics file cannot be found")
study.set_error_code(None)
study.store_error_code()
study.get_study_from_db()
self.assertEqual(study.error_code, None)
self.assertEqual(study.error_text, None)
def test_get_statuses(self):
study = st.Study(study_id=self.study_id,
file_path=self.file_path,
md5=self.md5,
assembly=self.assembly)
study.create_entry_for_study()
study.get_study_from_db()
self.assertEqual(study.get_status(), 'RETRIEVING')
study.set_retrieved_status(0)
study.store_retrieved_status()
study.get_study_from_db()
self.assertEqual(study.get_status(), 'INVALID')
study.set_retrieved_status(1)
study.store_retrieved_status()
study.get_study_from_db()
self.assertEqual(study.get_status(), 'VALIDATING')
study.set_data_valid_status(0)
study.store_data_valid_status()
study.get_study_from_db()
self.assertEqual(study.get_status(), 'INVALID')
study.set_data_valid_status(1)
study.store_data_valid_status()
study.get_study_from_db()
self.assertEqual(study.get_status(), 'VALID')
