def verified(context, collaborator, test, outpath=None): """Export variants which have been verified for an institute and write them to an excel file. Args: collaborator(str): institute id test(bool): True if the function is called for testing purposes outpath(str): path to output file Returns: written_files(int): number of written or simulated files """ written_files = 0 collaborator = collaborator or 'cust000' LOG.info('Exporting verified variants for cust {}'.format(collaborator)) adapter = context.obj['adapter'] verified_vars = adapter.verified(institute_id=collaborator) LOG.info('FOUND {} verified variants for institute {}'.format(len(verified_vars), collaborator)) if not verified_vars: LOG.warning('There are no verified variants for institute {} in database!'.format(collaborator)) return None document_lines = export_verified_variants(verified_vars) today = datetime.datetime.now().strftime('%Y-%m-%d') document_name = '.'.join(['verified_variants', collaborator, today]) + '.xlsx' # If this was a test and lines are created return success if test and document_lines: written_files +=1 LOG.info('Success. Verified variants file contains {} lines'.format(len(document_lines))) return written_files # create workbook and new sheet # set up outfolder if not outpath: outpath = str(os.getcwd()) workbook = Workbook(os.path.join(outpath,document_name)) Report_Sheet = workbook.add_worksheet() # Write the column header row = 0 for col,field in enumerate(VERIFIED_VARIANTS_HEADER): Report_Sheet.write(row,col,field) # Write variant lines, after header (start at line 1) for row, line in enumerate(document_lines,1): # each line becomes a row in the document for col, field in enumerate(line): # each field in line becomes a cell Report_Sheet.write(row,col,field) workbook.close() if os.path.exists(os.path.join(outpath,document_name)): LOG.info('Success. Verified variants file of {} lines was written to disk'. format(len(document_lines))) written_files += 1 return written_files
def test_export_verified_variants(case_obj, real_populated_database, variant_objs): """Test the function that creates lines with verified variants to be exported""" adapter = real_populated_database case_id = case_obj['_id'] # load variants to populated_database assert adapter.variants(case_id=case_id, nr_of_variants=-1).count() == 0 nr_loaded = adapter.load_variants(case_obj=case_obj) assert nr_loaded > 0 valid_status = ['False positive', 'True positive', 'False positive'] # for 3 vars test_vars = list(adapter.variant_collection.find().limit(3)) assert len(test_vars) == 3 # Make sure that no variant is set as validated: assert adapter.variant_collection.find({'validation':{'$exists':True}}).count()==0 # Set test variant as validated for i in range(3): # Set validation status of a variant adapter.variant_collection.find_one_and_update( {'_id' : test_vars[i]['_id'] }, {'$set' : {'validation' : valid_status[i] } } ) # insert validation events adapter.event_collection.insert_one({ 'verb' : 'validate', 'institute' : test_vars[i]['institute'], 'variant_id' : test_vars[i]['variant_id'], 'case' : case_id }) # There should be 3 validated variants now assert adapter.variant_collection.find({'validation':{'$exists':True}}).count()==3 # Call function to get aggregated data (variant + case data): cust = case_obj['owner'] aggregated_vars = adapter.verified(cust) assert len(aggregated_vars) == 3 # same number of variants is returned unique_callers = set() for var_type, var_callers in CALLERS.items(): for caller in var_callers: unique_callers.add(caller.get('id')) n_individuals = len(case_obj['individuals']) # Call function that creates document lines document_lines = export_verified_variants(aggregated_vars, unique_callers) assert len(document_lines) == 3 * n_individuals # one line per variant and individual for line in document_lines: # Make sure that document cells will be the same as in document header assert len(line) == len(VERIFIED_VARIANTS_HEADER + list(unique_callers))
def verified_excel_file(store, institute_list, temp_excel_dir): """Collect all verified variants in a list on institutes and save them to file Args: store(adapter.MongoAdapter) institute_list(list): a list of institute ids temp_excel_dir(os.Path): folder where the temp excel files are written to Returns: written_files(int): the number of files written to temp_excel_dir """ document_lines = [] written_files = 0 today = datetime.datetime.now().strftime("%Y-%m-%d") LOG.info("Creating verified variant document..") for cust in institute_list: verif_vars = store.verified(institute_id=cust) LOG.info("Found {} verified variants for customer {}".format( len(verif_vars), cust)) if not verif_vars: continue unique_callers = set() for var_type, var_callers in CALLERS.items(): for caller in var_callers: unique_callers.add(caller.get("id")) cust_verified = export_verified_variants(verif_vars, unique_callers) document_name = ".".join([cust, "_verified_variants", today]) + ".xlsx" workbook = Workbook(os.path.join(temp_excel_dir, document_name)) Report_Sheet = workbook.add_worksheet() # Write the column header row = 0 for col, field in enumerate(VERIFIED_VARIANTS_HEADER + list(unique_callers)): Report_Sheet.write(row, col, field) # Write variant lines, after header (start at line 1) for row, line in enumerate( cust_verified, 1): # each line becomes a row in the document for col, field in enumerate( line): # each field in line becomes a cell Report_Sheet.write(row, col, field) workbook.close() if os.path.exists(os.path.join(temp_excel_dir, document_name)): written_files += 1 return written_files
def test_export_verified_variants(case_obj, real_populated_database, variant_objs): """Test the function that creates lines with verified variants to be exported""" adapter = real_populated_database case_id = case_obj["_id"] # load variants to populated_database assert sum( 1 for i in adapter.variants(case_id=case_id, nr_of_variants=-1)) == 0 nr_loaded = adapter.load_variants(case_obj=case_obj) assert nr_loaded > 0 valid_status = ["False positive", "True positive", "False positive"] # for 3 vars test_vars = list(adapter.variant_collection.find().limit(3)) assert len(test_vars) == 3 # Make sure that no variant is set as validated: assert (sum(1 for i in adapter.variant_collection.find( {"validation": { "$exists": True }})) == 0) # Set test variant as validated for i in range(3): # Set validation status of a variant adapter.variant_collection.find_one_and_update( {"_id": test_vars[i]["_id"]}, {"$set": { "validation": valid_status[i] }}) # insert validation events adapter.event_collection.insert_one({ "verb": "validate", "institute": test_vars[i]["institute"], "variant_id": test_vars[i]["variant_id"], "case": case_id, }) # There should be 3 validated variants now assert (sum(1 for i in adapter.variant_collection.find( {"validation": { "$exists": True }})) == 3) # Call function to get aggregated data (variant + case data): cust = case_obj["owner"] aggregated_vars = adapter.verified(cust) assert len(aggregated_vars) == 3 # same number of variants is returned unique_callers = set() for var_type, var_callers in CALLERS.items(): for caller in var_callers: unique_callers.add(caller.get("id")) n_individuals = len(case_obj["individuals"]) # Call function that creates document lines document_lines = export_verified_variants(aggregated_vars, unique_callers) assert (len(document_lines) == 3 * n_individuals ) # one line per variant and individual for line in document_lines: # Make sure that document cells will be the same as in document header assert len(line) == len(VERIFIED_VARIANTS_HEADER + list(unique_callers))
def verified(collaborator, test, outpath=None): """Export variants which have been verified for an institute and write them to an excel file. Args: collaborator(str): institute id test(bool): True if the function is called for testing purposes outpath(str): path to output file Returns: written_files(int): number of written or simulated files """ written_files = 0 collaborator = collaborator or "cust000" LOG.info("Exporting verified variants for cust {}".format(collaborator)) adapter = store verified_vars = adapter.verified(institute_id=collaborator) LOG.info("FOUND {} verified variants for institute {}".format( len(verified_vars), collaborator)) if not verified_vars: LOG.warning( "There are no verified variants for institute {} in database!". format(collaborator)) return None unique_callers = set() for var_type, var_callers in CALLERS.items(): for caller in var_callers: unique_callers.add(caller.get("id")) document_lines = export_verified_variants(verified_vars, unique_callers) today = datetime.datetime.now().strftime("%Y-%m-%d") document_name = ".".join(["verified_variants", collaborator, today ]) + ".xlsx" # If this was a test and lines are created return success if test and document_lines: written_files += 1 LOG.info("Success. Verified variants file contains {} lines".format( len(document_lines))) return written_files if test: LOG.info( "Could not create document lines. Verified variants not found for customer {}" .format(collaborator)) return # create workbook and new sheet # set up outfolder if not outpath: outpath = str(os.getcwd()) workbook = Workbook(os.path.join(outpath, document_name)) Report_Sheet = workbook.add_worksheet() # Write the column header row = 0 for col, field in enumerate(VERIFIED_VARIANTS_HEADER): Report_Sheet.write(row, col, field) # Write variant lines, after header (start at line 1) for row, line in enumerate(document_lines, 1): # each line becomes a row in the document for col, field in enumerate(line): # each field in line becomes a cell Report_Sheet.write(row, col, field) workbook.close() if os.path.exists(os.path.join(outpath, document_name)): LOG.info( "Success. Verified variants file of {} lines was written to disk". format(len(document_lines))) written_files += 1 return written_files