def verified(context, collaborator, test, outpath=None):
"""Export variants which have been verified for an institute
and write them to an excel file.
Args:
collaborator(str): institute id
test(bool): True if the function is called for testing purposes
outpath(str): path to output file
Returns:
written_files(int): number of written or simulated files
"""
written_files = 0
collaborator = collaborator or 'cust000'
LOG.info('Exporting verified variants for cust {}'.format(collaborator))
adapter = context.obj['adapter']
verified_vars = adapter.verified(institute_id=collaborator)
LOG.info('FOUND {} verified variants for institute {}'.format(len(verified_vars), collaborator))
if not verified_vars:
LOG.warning('There are no verified variants for institute {} in database!'.format(collaborator))
return None
document_lines = export_verified_variants(verified_vars)
today = datetime.datetime.now().strftime('%Y-%m-%d')
document_name = '.'.join(['verified_variants', collaborator, today]) + '.xlsx'
# If this was a test and lines are created return success
if test and document_lines:
written_files +=1
LOG.info('Success. Verified variants file contains {} lines'.format(len(document_lines)))
return written_files
# create workbook and new sheet
# set up outfolder
if not outpath:
outpath = str(os.getcwd())
workbook = Workbook(os.path.join(outpath,document_name))
Report_Sheet = workbook.add_worksheet()
# Write the column header
row = 0
for col,field in enumerate(VERIFIED_VARIANTS_HEADER):
Report_Sheet.write(row,col,field)
# Write variant lines, after header (start at line 1)
for row, line in enumerate(document_lines,1): # each line becomes a row in the document
for col, field in enumerate(line): # each field in line becomes a cell
Report_Sheet.write(row,col,field)
workbook.close()
if os.path.exists(os.path.join(outpath,document_name)):
LOG.info('Success. Verified variants file of {} lines was written to disk'. format(len(document_lines)))
written_files += 1
return written_files
def test_export_verified_variants(case_obj, real_populated_database, variant_objs):
"""Test the function that creates lines with verified variants to be exported"""
adapter = real_populated_database
case_id = case_obj['_id']
# load variants to populated_database
assert adapter.variants(case_id=case_id, nr_of_variants=-1).count() == 0
nr_loaded = adapter.load_variants(case_obj=case_obj)
assert nr_loaded > 0
valid_status = ['False positive', 'True positive', 'False positive'] # for 3 vars
test_vars = list(adapter.variant_collection.find().limit(3))
assert len(test_vars) == 3
# Make sure that no variant is set as validated:
assert adapter.variant_collection.find({'validation':{'$exists':True}}).count()==0
# Set test variant as validated
for i in range(3):
# Set validation status of a variant
adapter.variant_collection.find_one_and_update(
{'_id' : test_vars[i]['_id'] },
{'$set' : {'validation' : valid_status[i] } }
)
# insert validation events
adapter.event_collection.insert_one({
'verb' : 'validate',
'institute' : test_vars[i]['institute'],
'variant_id' : test_vars[i]['variant_id'],
'case' : case_id
})
# There should be 3 validated variants now
assert adapter.variant_collection.find({'validation':{'$exists':True}}).count()==3
# Call function to get aggregated data (variant + case data):
cust = case_obj['owner']
aggregated_vars = adapter.verified(cust)
assert len(aggregated_vars) == 3 # same number of variants is returned
unique_callers = set()
for var_type, var_callers in CALLERS.items():
for caller in var_callers:
unique_callers.add(caller.get('id'))
n_individuals = len(case_obj['individuals'])
# Call function that creates document lines
document_lines = export_verified_variants(aggregated_vars, unique_callers)
assert len(document_lines) == 3 * n_individuals # one line per variant and individual
for line in document_lines:
# Make sure that document cells will be the same as in document header
assert len(line) == len(VERIFIED_VARIANTS_HEADER + list(unique_callers))
def verified_excel_file(store, institute_list, temp_excel_dir):
"""Collect all verified variants in a list on institutes and save them to file
Args:
store(adapter.MongoAdapter)
institute_list(list): a list of institute ids
temp_excel_dir(os.Path): folder where the temp excel files are written to
Returns:
written_files(int): the number of files written to temp_excel_dir
"""
document_lines = []
written_files = 0
today = datetime.datetime.now().strftime("%Y-%m-%d")
LOG.info("Creating verified variant document..")
for cust in institute_list:
verif_vars = store.verified(institute_id=cust)
LOG.info("Found {} verified variants for customer {}".format(
len(verif_vars), cust))
if not verif_vars:
continue
unique_callers = set()
for var_type, var_callers in CALLERS.items():
for caller in var_callers:
unique_callers.add(caller.get("id"))
cust_verified = export_verified_variants(verif_vars, unique_callers)
document_name = ".".join([cust, "_verified_variants", today]) + ".xlsx"
workbook = Workbook(os.path.join(temp_excel_dir, document_name))
Report_Sheet = workbook.add_worksheet()
# Write the column header
row = 0
for col, field in enumerate(VERIFIED_VARIANTS_HEADER +
list(unique_callers)):
Report_Sheet.write(row, col, field)
# Write variant lines, after header (start at line 1)
for row, line in enumerate(
cust_verified, 1): # each line becomes a row in the document
for col, field in enumerate(
line): # each field in line becomes a cell
Report_Sheet.write(row, col, field)
workbook.close()
if os.path.exists(os.path.join(temp_excel_dir, document_name)):
written_files += 1
return written_files
def test_export_verified_variants(case_obj, real_populated_database,
variant_objs):
"""Test the function that creates lines with verified variants to be exported"""
adapter = real_populated_database
case_id = case_obj["_id"]
# load variants to populated_database
assert sum(
1 for i in adapter.variants(case_id=case_id, nr_of_variants=-1)) == 0
nr_loaded = adapter.load_variants(case_obj=case_obj)
assert nr_loaded > 0
valid_status = ["False positive", "True positive",
"False positive"] # for 3 vars
test_vars = list(adapter.variant_collection.find().limit(3))
assert len(test_vars) == 3
# Make sure that no variant is set as validated:
assert (sum(1 for i in adapter.variant_collection.find(
{"validation": {
"$exists": True
}})) == 0)
# Set test variant as validated
for i in range(3):
# Set validation status of a variant
adapter.variant_collection.find_one_and_update(
{"_id": test_vars[i]["_id"]},
{"$set": {
"validation": valid_status[i]
}})
# insert validation events
adapter.event_collection.insert_one({
"verb":
"validate",
"institute":
test_vars[i]["institute"],
"variant_id":
test_vars[i]["variant_id"],
"case":
case_id,
})
# There should be 3 validated variants now
assert (sum(1 for i in adapter.variant_collection.find(
{"validation": {
"$exists": True
}})) == 3)
# Call function to get aggregated data (variant + case data):
cust = case_obj["owner"]
aggregated_vars = adapter.verified(cust)
assert len(aggregated_vars) == 3 # same number of variants is returned
unique_callers = set()
for var_type, var_callers in CALLERS.items():
for caller in var_callers:
unique_callers.add(caller.get("id"))
n_individuals = len(case_obj["individuals"])
# Call function that creates document lines
document_lines = export_verified_variants(aggregated_vars, unique_callers)
assert (len(document_lines) == 3 * n_individuals
) # one line per variant and individual
for line in document_lines:
# Make sure that document cells will be the same as in document header
assert len(line) == len(VERIFIED_VARIANTS_HEADER +
list(unique_callers))
def verified(collaborator, test, outpath=None):
"""Export variants which have been verified for an institute
and write them to an excel file.
Args:
collaborator(str): institute id
test(bool): True if the function is called for testing purposes
outpath(str): path to output file
Returns:
written_files(int): number of written or simulated files
"""
written_files = 0
collaborator = collaborator or "cust000"
LOG.info("Exporting verified variants for cust {}".format(collaborator))
adapter = store
verified_vars = adapter.verified(institute_id=collaborator)
LOG.info("FOUND {} verified variants for institute {}".format(
len(verified_vars), collaborator))
if not verified_vars:
LOG.warning(
"There are no verified variants for institute {} in database!".
format(collaborator))
return None
unique_callers = set()
for var_type, var_callers in CALLERS.items():
for caller in var_callers:
unique_callers.add(caller.get("id"))
document_lines = export_verified_variants(verified_vars, unique_callers)
today = datetime.datetime.now().strftime("%Y-%m-%d")
document_name = ".".join(["verified_variants", collaborator, today
]) + ".xlsx"
# If this was a test and lines are created return success
if test and document_lines:
written_files += 1
LOG.info("Success. Verified variants file contains {} lines".format(
len(document_lines)))
return written_files
if test:
LOG.info(
"Could not create document lines. Verified variants not found for customer {}"
.format(collaborator))
return
# create workbook and new sheet
# set up outfolder
if not outpath:
outpath = str(os.getcwd())
workbook = Workbook(os.path.join(outpath, document_name))
Report_Sheet = workbook.add_worksheet()
# Write the column header
row = 0
for col, field in enumerate(VERIFIED_VARIANTS_HEADER):
Report_Sheet.write(row, col, field)
# Write variant lines, after header (start at line 1)
for row, line in enumerate(document_lines,
1): # each line becomes a row in the document
for col, field in enumerate(line): # each field in line becomes a cell
Report_Sheet.write(row, col, field)
workbook.close()
if os.path.exists(os.path.join(outpath, document_name)):
LOG.info(
"Success. Verified variants file of {} lines was written to disk".
format(len(document_lines)))
written_files += 1
return written_files
