def test_GenomeInterface_own_handle(self): # no handle in genome genome = {'missing_genbank_handle_ref': 'hid'} origin_genome = genome.copy() self.genome_interface._own_handle(genome, 'genbank_handle_ref') self.assertItemsEqual(origin_genome, genome) # user unauthorized temp_shock_file = "/kb/module/work/tmp/shock1.txt" with open(temp_shock_file, "w") as f1: f1.write("Test Shock Handle") token2 = self.ctx2['token'] dfu2 = DataFileUtil(os.environ['SDK_CALLBACK_URL'], token=token2) shock_ret = dfu2.file_to_shock({'file_path': temp_shock_file, 'make_handle': 1}) self.nodes_to_delete.append(shock_ret['shock_id']) hid = shock_ret['handle']['hid'] genome = {'genbank_handle_ref': hid} with self.assertRaisesRegex(ValueError, 'Error getting ACLs for Shock node'): self.genome_interface._own_handle(genome, 'genbank_handle_ref') # same user shock_ret = self.dfu.file_to_shock({'file_path': temp_shock_file, 'make_handle': 1}) self.nodes_to_delete.append(shock_ret['shock_id']) hid = shock_ret['handle']['hid'] genome = {'genbank_handle_ref': hid} origin_genome = genome.copy() self.genome_interface._own_handle(genome, 'genbank_handle_ref') self.assertDictEqual(origin_genome, genome) # differet user self.wsClient.set_permissions({'workspace': self.wsName, 'new_permission': 'w', 'users': [self.ctx2['user_id']]}) token2 = self.ctx2['token'] dfu2 = DataFileUtil(os.environ['SDK_CALLBACK_URL'], token=token2) shock_ret = dfu2.file_to_shock({'file_path': temp_shock_file, 'make_handle': 1}) node = shock_ret['shock_id'] self.nodes_to_delete.append(node) hid = shock_ret['handle']['hid'] # grant user1 read access to node user1 = self.ctx['user_id'] acl = 'read' url = self.shockURL + '/node/' + node + '/acl' url += '/' + acl + '?users=' + urllib.parse.quote(user1) auth_header = {'Authorization': 'OAuth {}'.format(token2)} req = requests.put(url, headers=auth_header, allow_redirects=True) if not req.ok: err = json.loads(req.content)['error'][0] print('response error: {}'.format(err)) genome = {'genbank_handle_ref': hid} origin_genome = genome.copy() self.genome_interface._own_handle(genome, 'genbank_handle_ref') self.assertNotEqual(origin_genome['genbank_handle_ref'], genome['genbank_handle_ref'])
def create_html_report(self, callback_url, output_dir, workspace_name): ''' function for creating html report ''' dfu = DataFileUtil(callback_url) report_name = 'VariationReport' + str(uuid.uuid4()) report = KBaseReport(callback_url) report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'Variation HTML report' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def create_report(callback_url, scratch, workspace_name, result_file): html = create_html_tables(result_file) dfu = DataFileUtil(callback_url) report_name = 'fastANI_report_' + str(uuid.uuid4()) report_client = KBaseReport(callback_url) html_dir = os.path.join(scratch, report_name) os.mkdir(html_dir) with open(os.path.join(html_dir, "index.html"), 'w') as file: file.write(html) shock = dfu.file_to_shock({ 'file_path': html_dir, 'make_handle': 0, 'pack': 'zip' }) html_file = { 'shock_id': shock['shock_id'], 'name': 'index.html', 'label': 'html_files', 'description': 'FastANI HTML report' } report = report_client.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': params['workspace_name'] }) return {'report_name': report['name'], 'report_ref': report['ref']}
def _upload_report(self, report_dir, file_links, workspace_name, saved_objects): dfu = DataFileUtil(self.callback_url) upload_info = dfu.file_to_shock({ 'file_path': report_dir, 'pack': 'zip' }) shock_id = upload_info['shock_id'] report_params = { 'message': 'JGI metagenome assembly report', 'direct_html_link_index': 0, 'html_links': [{ 'shock_id': shock_id, 'name': 'index.html', 'description': 'assembly report' }], 'file_links': file_links, 'report_object_name': 'JGI_assembly_pipeline.' + str(uuid.uuid4()), 'workspace_name': workspace_name, 'objects_created': saved_objects } report_client = KBaseReport(self.callback_url) report = report_client.create_extended_report(report_params) return {'report_ref': report['ref'], 'report_name': report['name']}
def test_simple_shock_upload(self): ### Test for upload from SHOCK - upload the file to shock first print('attempting upload through shock') gbk_path = "data/e_coli/GCF_000005845.2_ASM584v2_genomic.gbff" data_file_cli = DataFileUtil(os.environ['SDK_CALLBACK_URL']) shutil.copy(gbk_path, self.cfg['scratch']) shock_id = data_file_cli.file_to_shock({ 'file_path': os.path.join(self.cfg['scratch'], gbk_path.split("/")[-1]) })['shock_id'] print("Running test") ws_obj_name2 = 'MyGenome.2' result = self.getImpl().genbank_to_genome( self.getContext(), { 'file': { 'shock_id': shock_id }, 'workspace_name': self.getWsName(), 'genome_name': ws_obj_name2, })[0] self.assertIsNotNone(result['genome_ref']) self.assertTrue( int(result['genome_info'][10]['Number of Protein Encoding Genes']) > 0)
def test_handles(self): wsName = self.generatePesudoRandomWorkspaceName() self.ws.set_permissions({'workspace': wsName, 'new_permission': 'w', 'users': [self.ctx2['user_id']]}) temp_shock_file = "/kb/module/work/tmp/shock1.txt" with open(temp_shock_file, "w") as f1: f1.write("Test Shock Handle") token1 = self.ctx['token'] dfu = DataFileUtil(os.environ['SDK_CALLBACK_URL'], token=token1) handle1 = dfu.file_to_shock({'file_path': temp_shock_file, 'make_handle': 1})['handle'] hid1 = handle1['hid'] genome_name = "Genome.1" self.impl.save_one_genome_v1(self.ctx, { 'workspace': wsName, 'name': genome_name, 'data': { 'id': "qwerty", 'scientific_name': "Qwerty", 'domain': "Bacteria", 'genetic_code': 11, 'genbank_handle_ref': hid1} }) genome = self.impl.get_genome_v1(self.ctx2, {'genomes': [{'ref': wsName + '/' + genome_name} ]})[0]['genomes'][0]['data'] self.impl.save_one_genome_v1(self.ctx2, {'workspace': wsName, 'name': genome_name, 'data': genome})[0] genome = self.impl.get_genome_v1(self.ctx2, {'genomes': [{'ref': wsName + '/' + genome_name} ]})[0]['genomes'][0]['data'] self.assertTrue('genbank_handle_ref' in genome) hid2 = genome['genbank_handle_ref'] self.assertNotEqual(hid1, hid2)
def make_fake_expression(callback_url, dummy_file, name, genome_ref, annotation_ref, alignment_ref, ws_name, ws_client): """ Makes a Fake KBaseRNASeq.RNASeqExpression object and returns a ref to it. genome_ref: reference to a genome object annotation_ref: reference to a KBaseRNASeq.GFFAnnotation alignment_ref: reference to a KBaseRNASeq.RNASeqAlignment """ dfu = DataFileUtil(callback_url) dummy_shock_info = dfu.file_to_shock({ "file_path": dummy_file, "make_handle": 1 }) exp = { "id": "fake", "type": "fake", "numerical_interpretation": "fake", "expression_levels": { "feature_1": 0, "feature_2": 1, "feature_3": 2 }, "genome_id": genome_ref, "annotation_id": annotation_ref, "mapped_rnaseq_alignment": { "id1": alignment_ref }, "condition": "", "tool_used": "none", "tool_version": "0.0.0", "file": dummy_shock_info['handle'] } return make_fake_object(exp, "KBaseRNASeq.RNASeqExpression", name, ws_name, ws_client)
def make_fake_alignment(callback_url, dummy_file, name, reads_ref, genome_ref, ws_name, ws_client): """ Makes a Fake KBaseRNASeq.RNASeqAlignment object and returns a ref to it. callback_url: needed for DataFileUtil, dummy_file: path to some dummy "alignment" file (make it small - needs to be uploaded to shock) name: the name of the object reads_ref: a reference to a valid (probably fake) reads library genome_ref: a reference to a valid (also probably fake) genome workspace_name: the name of the workspace to save this object workspace_client: a Workspace client tuned to the server of your choice """ dfu = DataFileUtil(callback_url) dummy_shock_info = dfu.file_to_shock({ "file_path": dummy_file, "make_handle": 1 }) fake_alignment = { "file": dummy_shock_info['handle'], "library_type": "fake", "read_sample_id": reads_ref, "condition": "fake", "genome_id": genome_ref } return make_fake_object(fake_alignment, "KBaseRNASeq.RNASeqAlignment", name, ws_name, ws_client)
def create_html_report(callback_url, scratch, workspace_name): ''' ''' output_dir = os.path.join(scratch, 'output') dfu = DataFileUtil(callback_url) report_name = 'METABOLIC_report_' + str(uuid.uuid4()) report = KBaseReport(callback_url) copyfile(os.path.join(os.path.dirname(__file__), 'report_template.html'), os.path.join(output_dir, 'report_template.html')) report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'report_template.html', 'label': 'report_template.html', 'description': 'HTML report for METABOLIC' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def create_html_report(self, callback_url, output_dir, workspace_name): ''' ''' dfu = DataFileUtil(callback_url) report_name = 'kb_gsea_report_' + str(uuid.uuid4()) report = KBaseReport(callback_url) #copyfile(os.path.join(os.path.dirname(__file__), 'index.html'), # os.path.join(output_dir, 'index.html')) report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'HTMLL report for GSEA' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def package_directory(callback_url, dir_path, zip_file_name, zip_file_description): ''' Simple utility for packaging a folder and saving to shock ''' dfu = DataFileUtil(callback_url) output = dfu.file_to_shock({'file_path': dir_path, 'make_handle': 0, 'pack': 'zip'}) return {'shock_id': output['shock_id'], 'name': zip_file_name, 'description': zip_file_description}
def _put_cached_index(self, assembly_info, index_files_basename, output_dir, ws_for_cache): if not ws_for_cache: print( 'WARNING: bowtie2 index cannot be cached because "ws_for_cache" field not set' ) return False try: dfu = DataFileUtil(self.callback_url) result = dfu.file_to_shock({ 'file_path': output_dir, 'make_handle': 1, 'pack': 'targz' }) bowtie2_index = { 'handle': result['handle'], 'size': result['size'], 'assembly_ref': assembly_info['ref'], 'index_files_basename': index_files_basename } ws = Workspace(self.ws_url) save_params = { 'objects': [{ 'hidden': 1, 'provenance': self.provenance, 'name': os.path.basename(output_dir), 'data': bowtie2_index, 'type': 'KBaseRNASeq.Bowtie2IndexV2' }] } if ws_for_cache.strip().isdigit(): save_params['id'] = int(ws_for_cache) else: save_params['workspace'] = ws_for_cache.strip() save_result = ws.save_objects(save_params) print('Bowtie2IndexV2 cached to: ') pprint(save_result[0]) return True except Exception: # if we fail in saving the cached object, don't worry print( 'WARNING: exception encountered when trying to cache the index files:' ) print(traceback.format_exc()) print( 'END WARNING: exception encountered when trying to cache the index files' ) return False
def read_sdf(file_path, inchi_path='/kb/module/data/Inchikey_IDs.json', mol2_file_dir=None, callback_url=None): inchi_dict = json.load(open(inchi_path)) file_name = os.path.splitext(os.path.basename(file_path))[0] sdf = AllChem.SDMolSupplier(file_path.encode('ascii', 'ignore')) compounds = [] for i, mol in enumerate(sdf): user_id = mol.GetPropsAsDict().get('id') print('Found compound ID: {}'.format(user_id)) handle_id = None if user_id and mol2_file_dir: mol2_file_path = None for root, dirs, files in os.walk(mol2_file_dir): for file in files: if os.path.splitext(file)[0] == user_id: logging.info( 'Found a matching mol2 file {} for compound {}'. format(str(file), user_id)) mol2_file_path = os.path.join(root, str(file)) if mol2_file_path: dfu = DataFileUtil(callback_url) handle_id = dfu.file_to_shock({ 'file_path': mol2_file_path, 'make_handle': True })['handle']['hid'] else: logging.warning( 'Unable to find a matching mol2 file for compound: {}'. format(user_id)) comp = _make_compound_info(mol) comp['name'] = mol.GetProp("_Name") comp['mol'] = AllChem.MolToMolBlock(mol) if comp['inchikey'] in inchi_dict: comp['kb_id'] = inchi_dict[comp['inchikey']] else: comp['kb_id'] = '%s_%s' % (file_name, i + 1) if user_id: comp['id'] = user_id else: comp['id'] = comp['kb_id'] if handle_id: comp['mol2_handle_ref'] = handle_id comp['mol2_source'] = 'user uploaded' compounds.append(comp) return compounds
def test_basic_upload_and_download(self): assemblyUtil = self.getImpl() tmp_dir = self.__class__.cfg['scratch'] file_name = "trimmed.fasta" shutil.copy(os.path.join("data", file_name), tmp_dir) fasta_path = os.path.join(tmp_dir, file_name) print('attempting upload') ws_obj_name = 'MyNewAssembly' result = assemblyUtil.save_assembly_from_fasta( self.getContext(), { 'file': { 'path': fasta_path }, 'workspace_name': self.getWsName(), 'assembly_name': ws_obj_name, 'taxon_ref': 'ReferenceTaxons/unknown_taxon', }) pprint(result) self.check_fasta_file(ws_obj_name, fasta_path) return print('attempting upload through shock') data_file_cli = DataFileUtil(os.environ['SDK_CALLBACK_URL']) shock_id = data_file_cli.file_to_shock({'file_path': fasta_path})['shock_id'] ws_obj_name2 = 'MyNewAssembly.2' result2 = assemblyUtil.save_assembly_from_fasta( self.getContext(), { 'shock_id': shock_id, 'workspace_name': self.getWsName(), 'assembly_name': ws_obj_name2 }) pprint(result2) self.check_fasta_file(ws_obj_name2, fasta_path) print('attempting upload via ftp url') ftp_url = 'ftp://ftp.ensemblgenomes.org/pub/release-29/bacteria//fasta/bacteria_8_collection/acaryochloris_marina_mbic11017/dna/Acaryochloris_marina_mbic11017.GCA_000018105.1.29.dna.genome.fa.gz' ws_obj_name3 = 'MyNewAssembly.3' result3 = assemblyUtil.save_assembly_from_fasta( self.getContext(), { 'ftp_url': ftp_url, 'workspace_name': self.getWsName(), 'assembly_name': ws_obj_name3 }) pprint(result3) # todo: add checks here on ws object ws_obj_name3 = 'MyNewAssembly.3' result4 = assemblyUtil.export_assembly_as_fasta( self.getContext(), {'input_ref': self.getWsName() + '/' + ws_obj_name3}) pprint(result4)
def make_fake_annotation(callback_url, dummy_file, name, ws_name, ws_client): dfu = DataFileUtil(callback_url) dummy_shock_info = dfu.file_to_shock({ "file_path": dummy_file, "make_handle": 1 }) annotation = { "handle": dummy_shock_info['handle'], "size": 0, "genome_id": "not_a_real_genome", "genome_scientific_name": "Genomus falsus" } return make_fake_object(annotation, "KBaseRNASeq.GFFAnnotation", name, ws_name, ws_client)
def create_html_report(self, callback_url, output_dir, workspace_name): ''' function for creating html report ''' dfu = DataFileUtil(callback_url) report_name = 'kb_gsea_report_' + str(uuid.uuid4()) report = KBaseReport(callback_url) report_dir = "localhost" #htmlstring = "<a href=" + report_dir + "/jbrowse/index.html>report link</a>" htmlstring = "<a href='./jbrowse/index.html'>report link</a>" index_file_path = output_dir + "/index.html" html_file = open(index_file_path, "wt") n = html_file.write(htmlstring) html_file.close() # Source path #source = "/kb/module/deps/jbrowse" # Destination path #destination = output_dir +"/jbrowse" #dest = shutil.copytree(source, destination) #os.system("cp -r " + source +" "+ destination) report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'HTMLL report for GSEA' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def package_folder(self, folder_path, zip_file_name, zip_file_description): ''' Simple utility for packaging a folder and saving to shock ''' if folder_path == self.scratch: raise ValueError ("cannot package scatch itself. folder path: "+folder_path) elif not folder_path.startswith(self.scratch): raise ValueError ("cannot package folder that is not a subfolder of scratch. folder path: "+folder_path) dfu = DataFileUtil(self.callback_url) if not os.path.exists(folder_path): raise ValueError ("cannot package folder that doesn't exist: "+folder_path) output = dfu.file_to_shock({'file_path': folder_path, 'make_handle': 0, 'pack': 'zip'}) return {'shock_id': output['shock_id'], 'name': zip_file_name, 'label': zip_file_description}
def create_html_report(self, callback_url, output_dir, workspace_name): ''' function for creating html report :param callback_url: :param output_dir: :param workspace_name: :return: ''' dfu = DataFileUtil(callback_url) report_name = 'kb_variant_report_' + str(uuid.uuid4()) report = KBaseReport(callback_url) index_file_path = output_dir + "/snpEff_genes.txt" htmlstring = self.create_enrichment_report("snpEff_genes.txt", output_dir) try: with open(output_dir + "/index.html", "w") as html_file: html_file.write(htmlstring + "\n") except IOError: print("Unable to write " + index_file_path + " file on disk.") report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'HTMLL report for GSEA' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def generate_product_report(callback_url, workspace_name, output_dir, product_html_loc, output_files, output_objects=None): # check params if output_objects is None: output_objects = [] # setup utils datafile_util = DataFileUtil(callback_url) report_util = KBaseReport(callback_url) # move html to main directory uploaded to shock so kbase can find it html_file = os.path.join(output_dir, 'product.html') os.rename(product_html_loc, html_file) report_shock_id = datafile_util.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_report = [{ 'shock_id': report_shock_id, 'name': os.path.basename(html_file), 'label': os.path.basename(html_file), 'description': 'DRAM product.' }] report = report_util.create_extended_report({ 'message': 'Here are the results from your DRAM run.', 'workspace_name': workspace_name, 'html_links': html_report, 'direct_html_link_index': 0, 'file_links': [ value for key, value in output_files.items() if value['path'] is not None ], 'objects_created': output_objects, }) return report
def create_html_report(self, callback_url, output_dir, workspace_name): ''' function for creating html report ''' dfu = DataFileUtil(callback_url) report_name = 'kb_gsea_report_' + str(uuid.uuid4()) report = KBaseReport(callback_url) htmlstring = "<a href='./jbrowse/index.html'>report link</a>" index_file_path = output_dir + "/index.html" try: with open(index_file_path, "wt") as html_file: n = html_file.write(htmlstring) except IOError: print("Unable to write " + index_file_path + " file on disk.") report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'HTMLL report for GSEA' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
class htmlreportutils: def __init__(self): callback_url = os.environ['SDK_CALLBACK_URL'] self.dfu = DataFileUtil(callback_url) self.report = KBaseReport(callback_url) pass def create_html_report(self, output_dir, workspace_name, objects_created): ''' function for creating html report ''' report_name = 'VariationReport' + str(uuid.uuid4()) report_shock_id = self.dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'Variation HTML report' } report_info = self.report.create_extended_report({ 'objects_created': objects_created, 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def create_html_report(self, callback_url, output_dir, workspace_name): ''' function for creating html report ''' dfu = DataFileUtil(callback_url) report_name = 'kb_gsea_report_' + str(uuid.uuid4()) report = KBaseReport(callback_url) htmlstring = self.format_files_to_html_report(output_dir) index_file_path = output_dir + "/index.html" html_file = open(index_file_path, "wt") n = html_file.write(htmlstring) html_file.close() report_shock_id = dfu.file_to_shock({ 'file_path': output_dir, 'pack': 'zip' })['shock_id'] html_file = { 'shock_id': report_shock_id, 'name': 'index.html', 'label': 'index.html', 'description': 'HTMLL report for GSEA' } report_info = report.create_extended_report({ 'direct_html_link_index': 0, 'html_links': [html_file], 'report_object_name': report_name, 'workspace_name': workspace_name }) return { 'report_name': report_info['name'], 'report_ref': report_info['ref'] }
def run_Gblocks(self, ctx, params): """ Method for trimming MSAs of either DNA or PROTEIN sequences ** ** input_type: MSA ** output_type: MSA :param params: instance of type "Gblocks_Params" (Gblocks Input Params) -> structure: parameter "workspace_name" of type "workspace_name" (** The workspace object refs are of form: ** ** objects = ws.get_objects([{'ref': params['workspace_id']+'/'+params['obj_name']}]) ** ** "ref" means the entire name combining the workspace id and the object name ** "id" is a numerical identifier of the workspace or object, and should just be used for workspace ** "name" is a string identifier of a workspace or object. This is received from Narrative.), parameter "desc" of String, parameter "input_ref" of type "data_obj_ref", parameter "output_name" of type "data_obj_name", parameter "trim_level" of Long, parameter "min_seqs_for_conserved" of Long, parameter "min_seqs_for_flank" of Long, parameter "max_pos_contig_nonconserved" of Long, parameter "min_block_len" of Long, parameter "remove_mask_positions_flag" of Long :returns: instance of type "Gblocks_Output" (Gblocks Output) -> structure: parameter "report_name" of type "data_obj_name", parameter "report_ref" of type "data_obj_ref" """ # ctx is the context object # return variables are: returnVal #BEGIN run_Gblocks console = [] invalid_msgs = [] self.log(console,'Running run_Gblocks with params=') self.log(console, "\n"+pformat(params)) report = '' # report = 'Running run_Gblocks with params=' # report += "\n"+pformat(params) #### do some basic checks # if 'workspace_name' not in params: raise ValueError('workspace_name parameter is required') if 'input_ref' not in params: raise ValueError('input_ref parameter is required') if 'output_name' not in params: raise ValueError('output_name parameter is required') #### Get the input_ref MSA object ## try: ws = workspaceService(self.workspaceURL, token=ctx['token']) objects = ws.get_objects([{'ref': params['input_ref']}]) data = objects[0]['data'] info = objects[0]['info'] input_name = info[1] input_type_name = info[2].split('.')[1].split('-')[0] except Exception as e: raise ValueError('Unable to fetch input_ref object from workspace: ' + str(e)) #to get the full stack trace: traceback.format_exc() if input_type_name == 'MSA': MSA_in = data row_order = [] default_row_labels = dict() if 'row_order' in MSA_in.keys(): row_order = MSA_in['row_order'] else: row_order = sorted(MSA_in['alignment'].keys()) if 'default_row_labels' in MSA_in.keys(): default_row_labels = MSA_in['default_row_labels'] else: for row_id in row_order: default_row_labels[row_id] = row_id if len(row_order) < 2: self.log(invalid_msgs,"must have multiple records in MSA: "+params['input_ref']) # export features to FASTA file input_MSA_file_path = os.path.join(self.scratch, input_name+".fasta") self.log(console, 'writing fasta file: '+input_MSA_file_path) records = [] for row_id in row_order: #self.log(console,"row_id: '"+row_id+"'") # DEBUG #self.log(console,"alignment: '"+MSA_in['alignment'][row_id]+"'") # DEBUG # using SeqIO makes multiline sequences. (Gblocks doesn't care, but FastTree doesn't like multiline, and I don't care enough to change code) #record = SeqRecord(Seq(MSA_in['alignment'][row_id]), id=row_id, description=default_row_labels[row_id]) #records.append(record) #SeqIO.write(records, input_MSA_file_path, "fasta") records.extend(['>'+row_id, MSA_in['alignment'][row_id] ]) with open(input_MSA_file_path,'w',0) as input_MSA_file_handle: input_MSA_file_handle.write("\n".join(records)+"\n") # Determine whether nuc or protein sequences # NUC_MSA_pattern = re.compile("^[\.\-_ACGTUXNRYSWKMBDHVacgtuxnryswkmbdhv \t\n]+$") all_seqs_nuc = True for row_id in row_order: #self.log(console, row_id+": '"+MSA_in['alignment'][row_id]+"'") if NUC_MSA_pattern.match(MSA_in['alignment'][row_id]) == None: all_seqs_nuc = False break # Missing proper input_type # else: raise ValueError('Cannot yet handle input_ref type of: '+type_name) # DEBUG: check the MSA file contents # with open(input_MSA_file_path, 'r', 0) as input_MSA_file_handle: # for line in input_MSA_file_handle: # #self.log(console,"MSA_LINE: '"+line+"'") # too big for console # self.log(invalid_msgs,"MSA_LINE: '"+line+"'") # validate input data # N_seqs = 0 L_first_seq = 0 with open(input_MSA_file_path, 'r', 0) as input_MSA_file_handle: for line in input_MSA_file_handle: if line.startswith('>'): N_seqs += 1 continue if L_first_seq == 0: for c in line: if c != '-' and c != ' ' and c != "\n": L_first_seq += 1 # min_seqs_for_conserved if 'min_seqs_for_conserved' in params and params['min_seqs_for_conserved'] != None and int(params['min_seqs_for_conserved']) != 0: if int(params['min_seqs_for_conserved']) < int(0.5*N_seqs)+1: self.log(invalid_msgs,"Min Seqs for Conserved Pos ("+str(params['min_seqs_for_conserved'])+") must be >= N/2+1 (N="+str(N_seqs)+", N/2+1="+str(int(0.5*N_seqs)+1)+")\n") if int(params['min_seqs_for_conserved']) > int(params['min_seqs_for_flank']): self.log(invalid_msgs,"Min Seqs for Conserved Pos ("+str(params['min_seqs_for_conserved'])+") must be <= Min Seqs for Flank Pos ("+str(params['min_seqs_for_flank'])+")\n") # min_seqs_for_flank if 'min_seqs_for_flank' in params and params['min_seqs_for_flank'] != None and int(params['min_seqs_for_flank']) != 0: if int(params['min_seqs_for_flank']) > N_seqs: self.log(invalid_msgs,"Min Seqs for Flank Pos ("+str(params['min_seqs_for_flank'])+") must be <= N (N="+str(N_seqs)+")\n") # max_pos_contig_nonconserved if 'max_pos_contig_nonconserved' in params and params['max_pos_contig_nonconserved'] != None and int(params['max_pos_contig_nonconserved']) != 0: if int(params['max_pos_contig_nonconserved']) < 0: self.log(invalid_msgs,"Max Num Non-Conserved Pos ("+str(params['max_pos_contig_nonconserved'])+") must be >= 0"+"\n") if int(params['max_pos_contig_nonconserved']) > L_first_seq or int(params['max_pos_contig_nonconserved']) >= 32000: self.log(invalid_msgs,"Max Num Non-Conserved Pos ("+str(params['max_pos_contig_nonconserved'])+") must be <= L first seq ("+str(L_first_seq)+") and < 32000\n") # min_block_len if 'min_block_len' in params and params['min_block_len'] != None and int(params['min_block_len']) != 0: if int(params['min_block_len']) < 2: self.log(invalid_msgs,"Min Block Len ("+str(params['min_block_len'])+") must be >= 2"+"\n") if int(params['min_block_len']) > L_first_seq or int(params['min_block_len']) >= 32000: self.log(invalid_msgs,"Min Block Len ("+str(params['min_block_len'])+") must be <= L first seq ("+str(L_first_seq)+") and < 32000\n") # trim_level if 'trim_level' in params and params['trim_level'] != None and int(params['trim_level']) != 0: if int(params['trim_level']) < 0 or int(params['trim_level']) > 2: self.log(invalid_msgs,"Trim Level ("+str(params['trim_level'])+") must be >= 0 and <= 2"+"\n") if len(invalid_msgs) > 0: # load the method provenance from the context object self.log(console,"SETTING PROVENANCE") # DEBUG provenance = [{}] if 'provenance' in ctx: provenance = ctx['provenance'] # add additional info to provenance here, in this case the input data object reference provenance[0]['input_ws_objects'] = [] provenance[0]['input_ws_objects'].append(params['input_ref']) provenance[0]['service'] = 'kb_gblocks' provenance[0]['method'] = 'run_Gblocks' # report report += "FAILURE\n\n"+"\n".join(invalid_msgs)+"\n" reportObj = { 'objects_created':[], 'text_message':report } reportName = 'gblocks_report_'+str(uuid.uuid4()) report_obj_info = ws.save_objects({ # 'id':info[6], 'workspace':params['workspace_name'], 'objects':[ { 'type':'KBaseReport.Report', 'data':reportObj, 'name':reportName, 'meta':{}, 'hidden':1, 'provenance':provenance } ] })[0] self.log(console,"BUILDING RETURN OBJECT") returnVal = { 'report_name': reportName, 'report_ref': str(report_obj_info[6]) + '/' + str(report_obj_info[0]) + '/' + str(report_obj_info[4]) # 'output_ref': None } self.log(console,"run_Gblocks DONE") return [returnVal] ### Construct the command # # e.g. # for "0.5" gaps: cat "o\n<MSA_file>\nb\n5\ng\nm\nq\n" | Gblocks # for "all" gaps: cat "o\n<MSA_file>\nb\n5\n5\ng\nm\nq\n" | Gblocks # gblocks_cmd = [self.GBLOCKS_bin] # check for necessary files if not os.path.isfile(self.GBLOCKS_bin): raise ValueError("no such file '"+self.GBLOCKS_bin+"'") if not os.path.isfile(input_MSA_file_path): raise ValueError("no such file '"+input_MSA_file_path+"'") if not os.path.getsize(input_MSA_file_path) > 0: raise ValueError("empty file '"+input_MSA_file_path+"'") # DEBUG # with open(input_MSA_file_path,'r',0) as input_MSA_file_handle: # for line in input_MSA_file_handle: # #self.log(console,"MSA LINE: '"+line+"'") # too big for console # self.log(invalid_msgs,"MSA LINE: '"+line+"'") # set the output path timestamp = int((datetime.utcnow() - datetime.utcfromtimestamp(0)).total_seconds()*1000) output_dir = os.path.join(self.scratch,'output.'+str(timestamp)) if not os.path.exists(output_dir): os.makedirs(output_dir) # Gblocks names output blocks MSA by appending "-gb" to input file #output_GBLOCKS_file_path = os.path.join(output_dir, input_name+'-gb') output_GBLOCKS_file_path = input_MSA_file_path+'-gb' output_aln_file_path = output_GBLOCKS_file_path # Gblocks is interactive and only accepts args from pipe input #if 'arg' in params and params['arg'] != None and params['arg'] != 0: # fasttree_cmd.append('-arg') # fasttree_cmd.append(val) # Run GBLOCKS, capture output as it happens # self.log(console, 'RUNNING GBLOCKS:') self.log(console, ' '+' '.join(gblocks_cmd)) # report += "\n"+'running GBLOCKS:'+"\n" # report += ' '+' '.join(gblocks_cmd)+"\n" # FastTree requires shell=True in order to see input data env = os.environ.copy() #joined_fasttree_cmd = ' '.join(fasttree_cmd) # redirect out doesn't work with subprocess unless you join command first #p = subprocess.Popen([joined_fasttree_cmd], \ p = subprocess.Popen(gblocks_cmd, \ cwd = self.scratch, \ stdin = subprocess.PIPE, \ stdout = subprocess.PIPE, \ stderr = subprocess.PIPE, \ shell = True, \ env = env) # executable = '/bin/bash' ) # write commands to process # # for "0.5" gaps: cat "o\n<MSA_file>\nb\n5\ng\nm\nq\n" | Gblocks # for "all" gaps: cat "o\n<MSA_file>\nb\n5\n5\ng\nm\nq\n" | Gblocks p.stdin.write("o"+"\n") # open MSA file p.stdin.write(input_MSA_file_path+"\n") if 'trim_level' in params and params['trim_level'] != None and int(params['trim_level']) != 0: p.stdin.write("b"+"\n") if int(params['trim_level']) >= 1: self.log (console,"changing trim level") p.stdin.write("5"+"\n") # set to "half" if int(params['trim_level']) == 2: self.log (console,"changing trim level") p.stdin.write("5"+"\n") # set to "all" elif int(params['trim_level']) > 2: raise ValueError ("trim_level ("+str(params['trim_level'])+") was not between 0-2") p.stdin.write("m"+"\n") # flank must precede conserved because it acts us upper bound for acceptable conserved values if 'min_seqs_for_flank' in params and params['min_seqs_for_flank'] != None and int(params['min_seqs_for_flank']) != 0: self.log (console,"changing min_seqs_for_flank") p.stdin.write("b"+"\n") p.stdin.write("2"+"\n") p.stdin.write(str(params['min_seqs_for_flank'])+"\n") p.stdin.write("m"+"\n") if 'min_seqs_for_conserved' in params and params['min_seqs_for_conserved'] != None and int(params['min_seqs_for_conserved']) != 0: self.log (console,"changing min_seqs_for_conserved") p.stdin.write("b"+"\n") p.stdin.write("1"+"\n") p.stdin.write(str(params['min_seqs_for_conserved'])+"\n") p.stdin.write("m"+"\n") if 'max_pos_contig_nonconserved' in params and params['max_pos_contig_nonconserved'] != None and int(params['max_pos_contig_nonconserved']) > -1: self.log (console,"changing max_pos_contig_nonconserved") p.stdin.write("b"+"\n") p.stdin.write("3"+"\n") p.stdin.write(str(params['max_pos_contig_nonconserved'])+"\n") p.stdin.write("m"+"\n") if 'min_block_len' in params and params['min_block_len'] != None and params['min_block_len'] != 0: self.log (console,"changing min_block_len") p.stdin.write("b"+"\n") p.stdin.write("4"+"\n") p.stdin.write(str(params['min_block_len'])+"\n") p.stdin.write("m"+"\n") p.stdin.write("g"+"\n") # get blocks p.stdin.write("q"+"\n") # quit p.stdin.close() p.wait() # Read output # while True: line = p.stdout.readline() #line = p.stderr.readline() if not line: break self.log(console, line.replace('\n', '')) p.stdout.close() #p.stderr.close() p.wait() self.log(console, 'return code: ' + str(p.returncode)) # if p.returncode != 0: if p.returncode != 1: raise ValueError('Error running GBLOCKS, return code: '+str(p.returncode) + '\n\n'+ '\n'.join(console)) # Check that GBLOCKS produced output # if not os.path.isfile(output_GBLOCKS_file_path): raise ValueError("failed to create GBLOCKS output: "+output_GBLOCKS_file_path) elif not os.path.getsize(output_GBLOCKS_file_path) > 0: raise ValueError("created empty file for GBLOCKS output: "+output_GBLOCKS_file_path) # load the method provenance from the context object # self.log(console,"SETTING PROVENANCE") # DEBUG provenance = [{}] if 'provenance' in ctx: provenance = ctx['provenance'] # add additional info to provenance here, in this case the input data object reference provenance[0]['input_ws_objects'] = [] provenance[0]['input_ws_objects'].append(params['input_ref']) provenance[0]['service'] = 'kb_gblocks' provenance[0]['method'] = 'run_Gblocks' # reformat output to single-line FASTA MSA and check that output not empty (often happens when param combinations don't produce viable blocks # output_fasta_buf = [] id_order = [] this_id = None ids = dict() alignment = dict() L_alignment = 0; L_alignment_set = False with open(output_GBLOCKS_file_path,'r',0) as output_GBLOCKS_file_handle: for line in output_GBLOCKS_file_handle: line = line.rstrip() if line.startswith('>'): this_id = line[1:] output_fasta_buf.append ('>'+re.sub('\s','_',default_row_labels[this_id])) id_order.append(this_id) alignment[this_id] = '' if L_alignment != 0 and not L_alignment_set: L_alignment_set = True continue output_fasta_buf.append (line) for c in line: if c != ' ' and c != "\n": alignment[this_id] += c if not L_alignment_set: L_alignment += 1 if L_alignment == 0: self.log(invalid_msgs,"params produced no blocks. Consider changing to less stringent values") else: if 'remove_mask_positions_flag' in params and params['remove_mask_positions_flag'] != None and params['remove_mask_positions_flag'] != '' and params['remove_mask_positions_flag'] == 1: self.log (console,"removing mask positions") mask = [] new_alignment = dict() for i in range(0,L_alignment): mask[i] = '+' if alignment[id_order[0]][i] == '-' \ or alignment[id_order[0]][i] == 'X' \ or alignment[id_order[0]][i] == 'x': mask[i] = '-' for row_id in id_order: new_alignment[row_id] = '' for i,c in enumerate(alignment[row_id]): if mask[i] == '+': new_alignment[row_id] += c alignment = new_alignment L_alignment = len(alignment[id_order[0]]) # write fasta with tidied ids output_MSA_file_path = os.path.join(output_dir, params['output_name']+'.fasta'); with open(output_MSA_file_path,'w',0) as output_MSA_file_handle: output_MSA_file_handle.write("\n".join(output_fasta_buf)+"\n") # Upload results # if len(invalid_msgs) == 0: self.log(console,"UPLOADING RESULTS") # DEBUG # Didn't write file # with open(output_MSA_file_path,'r',0) as output_MSA_file_handle: # output_MSA_buf = output_MSA_file_handle.read() # output_MSA_buf = output_MSA_buf.rstrip() # self.log(console,"\nMSA:\n"+output_MSA_buf+"\n") # Build output_MSA structure # first extract old info from MSA (labels, ws_refs, etc.) # MSA_out = dict() for key in MSA_in.keys(): MSA_out[key] = MSA_in[key] # then replace with new info # MSA_out['alignment'] = alignment MSA_out['name'] = params['output_name'] MSA_out['alignment_length'] = alignment_length = L_alignment MSA_name = params['output_name'] MSA_description = '' if 'desc' in params and params['desc'] != None and params['desc'] != '': MSA_out['desc'] = MSA_description = params['desc'] # Store MSA_out # new_obj_info = ws.save_objects({ 'workspace': params['workspace_name'], 'objects':[{ 'type': 'KBaseTrees.MSA', 'data': MSA_out, 'name': params['output_name'], 'meta': {}, 'provenance': provenance }] })[0] # create CLW formatted output file max_row_width = 60 id_aln_gap_width = 1 gap_chars = '' for sp_i in range(id_aln_gap_width): gap_chars += ' ' # DNA if all_seqs_nuc: strong_groups = { 'AG': True, 'CTU': True } weak_groups = None # PROTEINS else: strong_groups = { 'AST': True, 'EKNQ': True, 'HKNQ': True, 'DENQ': True, 'HKQR': True, 'ILMV': True, 'FILM': True, 'HY': True, 'FWY': True } weak_groups = { 'ACS': True, 'ATV': True, 'AGS': True, 'KNST': True, 'APST': True, 'DGNS': True, 'DEKNQS': True, 'DEHKNQ': True, 'EHKNQR': True, 'FILMV': True, 'FHY': True } clw_buf = [] clw_buf.append ('CLUSTALW format of GBLOCKS trimmed MSA '+MSA_name+': '+MSA_description) clw_buf.append ('') long_id_len = 0 aln_pos_by_id = dict() for row_id in row_order: aln_pos_by_id[row_id] = 0 row_id_disp = default_row_labels[row_id] if long_id_len < len(row_id_disp): long_id_len = len(row_id_disp) full_row_cnt = alignment_length // max_row_width if alignment_length % max_row_width == 0: full_row_cnt -= 1 for chunk_i in range (full_row_cnt + 1): for row_id in row_order: row_id_disp = re.sub('\s','_',default_row_labels[row_id]) for sp_i in range (long_id_len-len(row_id_disp)): row_id_disp += ' ' aln_chunk_upper_bound = (chunk_i+1)*max_row_width if aln_chunk_upper_bound > alignment_length: aln_chunk_upper_bound = alignment_length aln_chunk = alignment[row_id][chunk_i*max_row_width:aln_chunk_upper_bound] for c in aln_chunk: if c != '-': aln_pos_by_id[row_id] += 1 clw_buf.append (row_id_disp+gap_chars+aln_chunk+' '+str(aln_pos_by_id[row_id])) # conservation line cons_line = '' for pos_i in range(chunk_i*max_row_width, aln_chunk_upper_bound): col_chars = dict() seq_cnt = 0 for row_id in row_order: char = alignment[row_id][pos_i] if char != '-': seq_cnt += 1 col_chars[char] = True if seq_cnt <= 1: cons_char = ' ' elif len(col_chars.keys()) == 1: cons_char = '*' else: strong = False for strong_group in strong_groups.keys(): this_strong_group = True for seen_char in col_chars.keys(): if seen_char not in strong_group: this_strong_group = False break if this_strong_group: strong = True break if not strong: weak = False if weak_groups != None: for weak_group in weak_groups.keys(): this_weak_group = True for seen_char in col_chars.keys(): if seen_char not in weak_group: this_strong_group = False break if this_weak_group: weak = True if strong: cons_char = ':' elif weak: cons_char = '.' else: cons_char = ' ' cons_line += cons_char lead_space = '' for sp_i in range(long_id_len): lead_space += ' ' lead_space += gap_chars clw_buf.append(lead_space+cons_line) clw_buf.append('') # write clw to file clw_buf_str = "\n".join(clw_buf)+"\n" output_clw_file_path = os.path.join(output_dir, input_name+'-MSA.clw'); with open (output_clw_file_path, "w", 0) as output_clw_file_handle: output_clw_file_handle.write(clw_buf_str) output_clw_file_handle.close() # upload GBLOCKS FASTA output to SHOCK for file_links dfu = DFUClient(self.callbackURL) try: output_upload_ret = dfu.file_to_shock({'file_path': output_aln_file_path, # DEBUG # 'make_handle': 0, # 'pack': 'zip'}) 'make_handle': 0}) except: raise ValueError ('error loading aln_out file to shock') # upload GBLOCKS CLW output to SHOCK for file_links try: output_clw_upload_ret = dfu.file_to_shock({'file_path': output_clw_file_path, # DEBUG # 'make_handle': 0, # 'pack': 'zip'}) 'make_handle': 0}) except: raise ValueError ('error loading clw_out file to shock') # make HTML reports # # HERE # build output report object # self.log(console,"BUILDING REPORT") # DEBUG reportName = 'gblocks_report_'+str(uuid.uuid4()) reportObj = { 'objects_created':[{'ref':params['workspace_name']+'/'+params['output_name'], 'description':'GBLOCKS MSA'}], #'message': '', 'message': clw_buf_str, 'direct_html': '', #'direct_html_link_index': 0, 'file_links': [], 'html_links': [], 'workspace_name': params['workspace_name'], 'report_object_name': reportName } reportObj['file_links'] = [{'shock_id': output_upload_ret['shock_id'], 'name': params['output_name']+'-GBLOCKS.FASTA', 'label': 'GBLOCKS-trimmed MSA FASTA' }, {'shock_id': output_clw_upload_ret['shock_id'], 'name': params['output_name']+'-GBLOCKS.CLW', 'label': 'GBLOCKS-trimmed MSA CLUSTALW' }] # save report object # SERVICE_VER = 'release' reportClient = KBaseReport(self.callbackURL, token=ctx['token'], service_ver=SERVICE_VER) #report_info = report.create({'report':reportObj, 'workspace_name':params['workspace_name']}) report_info = reportClient.create_extended_report(reportObj) else: # len(invalid_msgs) > 0 reportName = 'gblocks_report_'+str(uuid.uuid4()) report += "FAILURE:\n\n"+"\n".join(invalid_msgs)+"\n" reportObj = { 'objects_created':[], 'text_message':report } ws = workspaceService(self.workspaceURL, token=ctx['token']) report_obj_info = ws.save_objects({ #'id':info[6], 'workspace':params['workspace_name'], 'objects':[ { 'type':'KBaseReport.Report', 'data':reportObj, 'name':reportName, 'meta':{}, 'hidden':1, 'provenance':provenance } ] })[0] report_info = dict() report_info['name'] = report_obj_info[1] report_info['ref'] = str(report_obj_info[6])+'/'+str(report_obj_info[0])+'/'+str(report_obj_info[4]) # done returnVal = { 'report_name': report_info['name'], 'report_ref': report_info['ref'] } self.log(console,"run_Gblocks DONE") #END run_Gblocks # At some point might do deeper type checking... if not isinstance(returnVal, dict): raise ValueError('Method run_Gblocks return value ' + 'returnVal is not type dict as required.') # return the results return [returnVal]
class BaseModule: def __init__(self, config, version, name): self.config = config if "SDK_CALLBACK_URL" in os.environ: self.callback_url = os.environ['SDK_CALLBACK_URL'] self.dfu = DataFileUtil(self.callback_url) self.version = version self.name = name self.scratch_folder = config['scratch'] logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) self.clear_context() self.report_html = None def validate_args(self, params, required, defaults): for item in required: if item not in params: raise ValueError('Required argument ' + item + ' is missing!') for key in defaults: if key not in params: params[key] = defaults[key] return params def clear_context(self): self.report_info = None self.ctx = None self.output_type = None self.output_id = None self.wsclient = None def finalize_call(self, output): if self.report_info != None: output['report_name'] = self.report_info['name'] output['report_ref'] = self.report_info['ref'] if self.workspace != None: output['workspace_name'] = self.workspace output['ws'] = self.workspace if self.output_type != None: output['type'] = self.output_type output['obj'] = self.output_id return output def initialize_call(self, ctx, workspace=None, output_type=None, output_id=None): self.clear_context() self.workspace = workspace self.ctx = ctx self.output_type = output_type self.output_id = output_id self.objects_created = [] self.wsclient = Workspace(self.config["workspace-url"], token=self.ctx['token']) def add_created_object(self, ref, description): self.objects_created.append({"ref": ref, "description": description}) def create_report(self, context, template_file=None, height=500): html_report_folder = os.path.join(self.scratch_folder, 'htmlreport') os.makedirs(html_report_folder, exist_ok=True) with open(os.path.join(html_report_folder, 'view.html'), 'w') as f: self.report_html = self.build_report(context, template_file) f.write(self.report_html) report_shock_id = "" if self.config["save_report_to_kbase"] == "1": report_shock_id = self.dfu.file_to_shock({ 'file_path': html_report_folder, 'pack': 'zip' })['shock_id'] html_output = {'name': 'view.html', 'shock_id': report_shock_id} report_params = { 'objects_created': self.objects_created, 'workspace_name': self.workspace, 'html_links': [html_output], 'direct_html_link_index': 0, 'html_window_height': height, 'report_object_name': self.name + '_report_' + str(uuid.uuid4()) } if self.config["save_report_to_kbase"] == "1": report = KBaseReport(self.callback_url, token=self.ctx['token']) self.report_info = report.create_extended_report(report_params) return self.report_html def build_report(self, context, template_file=None): if template_file == None: template_file = self.config["template_file"] # Directory this file is in array = template_file.split("/") filename = array.pop() template_dir = "/".join(array) env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir), autoescape=jinja2.select_autoescape( ['html', 'xml'])) # Return string of html return env.get_template(filename).render(context)
class BiomUtil: def _mkdir_p(self, path): """ _mkdir_p: make directory for given path """ if not path: return try: os.makedirs(path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise def _process_params(self, params): logging.info('start validating import_matrix_from_biom params') # check for required parameters for p in [ 'obj_type', 'matrix_name', 'workspace_id', 'scale', 'amplicon_type', 'sequencing_technology', 'sequencing_instrument', 'target_gene', 'target_subfragment', 'taxon_calling' ]: if p not in params: raise ValueError( '"{}" parameter is required, but missing'.format(p)) # check sequencing_technology and sequencing_instrument matching sequencing_technology = params.get('sequencing_technology') sequencing_instrument = params.get('sequencing_instrument') if sequencing_technology not in SEQ_INSTRUMENTS_MAP: raise ValueError('Unexpected sequencing technology: {}'.format( sequencing_technology)) expected_instruments = SEQ_INSTRUMENTS_MAP.get(sequencing_technology) if sequencing_instrument not in expected_instruments: raise ValueError( 'Please select sequencing instrument among {} for {}'.format( expected_instruments, sequencing_technology)) # check target_gene and target_subfragment matching target_gene = params.get('target_gene') target_subfragment = list(set(params.get('target_subfragment'))) params['target_subfragment'] = target_subfragment if target_gene not in TARGET_GENE_SUBFRAGMENT_MAP: raise ValueError('Unexpected target gene: {}'.format(target_gene)) expected_subfragments = TARGET_GENE_SUBFRAGMENT_MAP.get(target_gene) if not set(target_subfragment) <= set(expected_subfragments): raise ValueError( 'Please select target subfragments among {} for {}'.format( expected_subfragments, target_gene)) # check taxon_calling taxon_calling = params.get('taxon_calling') taxon_calling_method = list( set(taxon_calling.get('taxon_calling_method'))) params['taxon_calling_method'] = taxon_calling_method if 'denoising' in taxon_calling_method: denoise_method = taxon_calling.get('denoise_method') sequence_error_cutoff = taxon_calling.get('sequence_error_cutoff') if not (denoise_method and sequence_error_cutoff): raise ValueError( 'Please provide denoise_method and sequence_error_cutoff') params['denoise_method'] = denoise_method params['sequence_error_cutoff'] = sequence_error_cutoff if 'clustering' in taxon_calling_method: clustering_method = taxon_calling.get('clustering_method') clustering_cutoff = taxon_calling.get('clustering_cutoff') if not (clustering_method and clustering_cutoff): raise ValueError( 'Please provide clustering_method and clustering_cutoff') params['clustering_method'] = clustering_method params['clustering_cutoff'] = clustering_cutoff obj_type = params.get('obj_type') if obj_type not in self.matrix_types: raise ValueError('Unknown matrix object type: {}'.format(obj_type)) scale = params.get('scale') if scale not in SCALE_TYPES: raise ValueError('Unknown scale type: {}'.format(scale)) biom_file = None tsv_file = None fasta_file = None metadata_keys = DEFAULT_META_KEYS input_local_file = params.get('input_local_file', False) if params.get('taxonomic_abundance_tsv') and params.get( 'taxonomic_fasta'): tsv_file = params.get('taxonomic_abundance_tsv') fasta_file = params.get('taxonomic_fasta') if not (tsv_file and fasta_file): raise ValueError('missing TSV or FASTA file') if not input_local_file: tsv_file = self.dfu.download_staging_file({ 'staging_file_subdir_path': tsv_file }).get('copy_file_path') fasta_file = self.dfu.download_staging_file({ 'staging_file_subdir_path': fasta_file }).get('copy_file_path') metadata_keys_str = params.get('metadata_keys') if metadata_keys_str: metadata_keys += [ x.strip() for x in metadata_keys_str.split(',') ] mode = 'tsv_fasta' elif params.get('biom_fasta'): biom_fasta = params.get('biom_fasta') biom_file = biom_fasta.get('biom_file_biom_fasta') fasta_file = biom_fasta.get('fasta_file_biom_fasta') if not (biom_file and fasta_file): raise ValueError('missing BIOM or FASTA file') if not input_local_file: biom_file = self.dfu.download_staging_file({ 'staging_file_subdir_path': biom_file }).get('copy_file_path') fasta_file = self.dfu.download_staging_file({ 'staging_file_subdir_path': fasta_file }).get('copy_file_path') mode = 'biom_fasta' elif params.get('tsv_fasta'): tsv_fasta = params.get('tsv_fasta') tsv_file = tsv_fasta.get('tsv_file_tsv_fasta') fasta_file = tsv_fasta.get('fasta_file_tsv_fasta') if not (tsv_file and fasta_file): raise ValueError('missing TSV or FASTA file') if not input_local_file: tsv_file = self.dfu.download_staging_file({ 'staging_file_subdir_path': tsv_file }).get('copy_file_path') fasta_file = self.dfu.download_staging_file({ 'staging_file_subdir_path': fasta_file }).get('copy_file_path') metadata_keys_str = tsv_fasta.get('metadata_keys_tsv_fasta') if metadata_keys_str: metadata_keys += [ x.strip() for x in metadata_keys_str.split(',') ] mode = 'tsv_fasta' else: raise ValueError('missing valide file group type in parameters') return (biom_file, tsv_file, fasta_file, mode, list(set(metadata_keys))) def _validate_fasta_file(self, df, fasta_file): logging.info('start validating FASTA file') try: fastq_dict = SeqIO.index(fasta_file, "fasta") except Exception: raise ValueError( 'Cannot parse file. Please provide valide FASTA file') matrix_ids = df.index file_ids = fastq_dict.keys() unmatched_ids = set(matrix_ids) - set(file_ids) if unmatched_ids: raise ValueError( 'FASTA file does not have [{}] OTU id'.format(unmatched_ids)) def _file_to_amplicon_data(self, biom_file, tsv_file, fasta_file, mode, refs, matrix_name, workspace_id, scale, description, metadata_keys=None): amplicon_data = refs if mode.startswith('biom'): logging.info('start parsing BIOM file for matrix data') table = biom.load_table(biom_file) observation_metadata = table._observation_metadata sample_metadata = table._sample_metadata matrix_data = { 'row_ids': table._observation_ids.tolist(), 'col_ids': table._sample_ids.tolist(), 'values': table.matrix_data.toarray().tolist() } logging.info('start building attribute mapping object') amplicon_data.update( self.get_attribute_mapping("row", observation_metadata, matrix_data, matrix_name, refs, workspace_id)) amplicon_data.update( self.get_attribute_mapping("col", sample_metadata, matrix_data, matrix_name, refs, workspace_id)) amplicon_data['attributes'] = {} for k in ('create_date', 'generated_by'): val = getattr(table, k) if not val: continue if isinstance(val, bytes): amplicon_data['attributes'][k] = val.decode('utf-8') else: amplicon_data['attributes'][k] = str(val) elif mode.startswith('tsv'): observation_metadata = None sample_metadata = None try: logging.info('start parsing TSV file for matrix data') reader = pd.read_csv(tsv_file, sep=None, iterator=True) inferred_sep = reader._engine.data.dialect.delimiter df = pd.read_csv(tsv_file, sep=inferred_sep, index_col=0) except Exception: raise ValueError( 'Cannot parse file. Please provide valide tsv file') else: self._validate_fasta_file(df, fasta_file) metadata_df = None if metadata_keys: shared_metadata_keys = list( set(metadata_keys) & set(df.columns)) if mode == 'tsv' and 'consensus_sequence' not in shared_metadata_keys: raise ValueError( 'TSV file does not include consensus_sequence') if shared_metadata_keys: metadata_df = df[shared_metadata_keys] df.drop(columns=shared_metadata_keys, inplace=True) try: df = df.astype(float) except ValueError: err_msg = 'Found some non-float values. Matrix contains only numeric values\n' err_msg += 'Please list any non-numeric column names in Metadata Keys field' raise ValueError(err_msg) df.fillna(0, inplace=True) df.index = df.index.astype('str') df.columns = df.columns.astype('str') matrix_data = { 'row_ids': df.index.tolist(), 'col_ids': df.columns.tolist(), 'values': df.values.tolist() } logging.info('start building attribute mapping object') amplicon_data.update( self.get_attribute_mapping("row", observation_metadata, matrix_data, matrix_name, refs, workspace_id, metadata_df=metadata_df)) amplicon_data.update( self.get_attribute_mapping("col", sample_metadata, matrix_data, matrix_name, refs, workspace_id)) amplicon_data['attributes'] = {} else: raise ValueError( 'error parsing _file_to_amplicon_data, mode: {}'.format(mode)) amplicon_data.update({'data': matrix_data}) amplicon_data['search_attributes'] = [ f'{k}|{v}' for k, v in amplicon_data['attributes'].items() ] amplicon_data['scale'] = scale if description: amplicon_data['description'] = description return amplicon_data def get_attribute_mapping(self, axis, metadata, matrix_data, matrix_name, refs, workspace_id, metadata_df=None): mapping_data = {} axis_ids = matrix_data[f'{axis}_ids'] if refs.get('sample_set_ref') and axis == 'col': name = matrix_name + "_{}_attributes".format(axis) mapping_data[ f'{axis}_attributemapping_ref'] = self._sample_set_to_attribute_mapping( axis_ids, refs.get('sample_set_ref'), name, workspace_id) mapping_data[f'{axis}_mapping'] = {x: x for x in axis_ids} elif refs.get(f'{axis}_attributemapping_ref'): am_data = self.dfu.get_objects( {'object_refs': [refs[f'{axis}_attributemapping_ref']]})['data'][0]['data'] unmatched_ids = set(axis_ids) - set(am_data['instances'].keys()) if unmatched_ids: name = "Column" if axis == 'col' else "Row" raise ValueError( f"The following {name} IDs from the uploaded matrix do not match " f"the supplied {name} attribute mapping: {', '.join(unmatched_ids)}" f"\nPlease verify the input data or upload an excel file with a" f"{name} mapping tab.") else: mapping_data[f'{axis}_mapping'] = {x: x for x in axis_ids} elif metadata: name = matrix_name + "_{}_attributes".format(axis) mapping_data[ f'{axis}_attributemapping_ref'] = self._metadata_to_attribute_mapping( axis_ids, metadata, name, workspace_id) # if coming from biom file, metadata and axis IDs are guaranteed to match mapping_data[f'{axis}_mapping'] = {x: x for x in axis_ids} elif metadata_df is not None: name = matrix_name + "_{}_attributes".format(axis) mapping_data[ f'{axis}_attributemapping_ref'] = self._meta_df_to_attribute_mapping( axis_ids, metadata_df, name, workspace_id) mapping_data[f'{axis}_mapping'] = {x: x for x in axis_ids} return mapping_data def _meta_df_to_attribute_mapping(self, axis_ids, metadata_df, obj_name, ws_id): data = {'ontology_mapping_method': "TSV file", 'instances': {}} metadata_df = metadata_df.astype(str) attribute_keys = metadata_df.columns.tolist() data['attributes'] = [{ 'attribute': key, 'source': 'upload' } for key in attribute_keys] if 'taxonomy' in attribute_keys: data['attributes'].append({ 'attribute': 'parsed_user_taxonomy', 'source': 'upload' }) for axis_id in axis_ids: data['instances'][axis_id] = metadata_df.loc[axis_id].tolist() if 'taxonomy' in attribute_keys: parsed_user_taxonomy = None taxonomy_index = attribute_keys.index('taxonomy') taxonomy_str = metadata_df.loc[axis_id].tolist( )[taxonomy_index] parsed_user_taxonomy = self.taxon_util.process_taxonomic_str( taxonomy_str) data['instances'][axis_id].append(parsed_user_taxonomy) logging.info( 'start saving AttributeMapping object: {}'.format(obj_name)) info = self.dfu.save_objects({ "id": ws_id, "objects": [{ "type": "KBaseExperiments.AttributeMapping", "data": data, "name": obj_name }] })[0] return f'{info[6]}/{info[0]}/{info[4]}' def _sample_set_to_attribute_mapping(self, axis_ids, sample_set_ref, obj_name, ws_id): am_data = self.sampleservice_util.sample_set_to_attribute_mapping( sample_set_ref) unmatched_ids = set(axis_ids) - set(am_data['instances'].keys()) if unmatched_ids: name = "Column" raise ValueError( f"The following {name} IDs from the uploaded matrix do not match " f"the supplied {name} attribute mapping: {', '.join(unmatched_ids)}" f"\nPlease verify the input data or upload an excel file with a" f"{name} mapping tab.") logging.info( 'start saving AttributeMapping object: {}'.format(obj_name)) info = self.dfu.save_objects({ "id": ws_id, "objects": [{ "type": "KBaseExperiments.AttributeMapping", "data": am_data, "name": obj_name }] })[0] return f'{info[6]}/{info[0]}/{info[4]}' def _metadata_to_attribute_mapping(self, instances, metadata, obj_name, ws_id): data = {'ontology_mapping_method': "BIOM file", 'instances': {}} sample_set = metadata[0:min(len(metadata), 25)] metadata_keys = sorted( set((k for m_dict in sample_set for k in m_dict))) data['attributes'] = [{ 'attribute': key, 'source': 'upload' } for key in metadata_keys] for inst, meta in zip(instances, metadata): data['instances'][inst] = [ str(meta[attr]) for attr in metadata_keys ] logging.info( 'start saving AttributeMapping object: {}'.format(obj_name)) info = self.dfu.save_objects({ "id": ws_id, "objects": [{ "type": "KBaseExperiments.AttributeMapping", "data": data, "name": obj_name }] })[0] return f'{info[6]}/{info[0]}/{info[4]}' def _generate_visualization_content(self, output_directory, heatmap_dir, data_df, top_heatmap_dir, top_percent, display_count): row_data_summary = data_df.T.describe().round(2).to_string() col_data_summary = data_df.describe().round(2).to_string() tab_def_content = '' tab_content = '' viewer_name = 'data_summary' tab_def_content += '''\n<div class="tab">\n''' tab_def_content += '''\n<button class="tablinks" ''' tab_def_content += '''onclick="openTab(event, '{}')"'''.format( viewer_name) tab_def_content += ''' id="defaultOpen"''' tab_def_content += '''>Matrix Statistics</button>\n''' tab_content += '''\n<div id="{}" class="tabcontent" style="overflow:auto">'''.format( viewer_name) tab_content += '''\n<h5>Amplicon Matrix Size: {} x {}</h5>'''.format( len(data_df.index), len(data_df.columns)) tab_content += '''\n<h5>Row Aggregating Statistics</h5>''' html = '''\n<pre class="tab">''' + str(row_data_summary).replace( "\n", "<br>") + "</pre>" tab_content += html tab_content += '''\n<br>''' tab_content += '''\n<hr style="height:2px;border-width:0;color:gray;background-color:gray">''' tab_content += '''\n<br>''' tab_content += '''\n<h5>Column Aggregating Statistics</h5>''' html = '''\n<pre class="tab">''' + str(col_data_summary).replace( "\n", "<br>") + "</pre>" tab_content += html tab_content += '\n</div>\n' if top_heatmap_dir: viewer_name = 'TopHeatmapViewer' tab_def_content += '''\n<button class="tablinks" ''' tab_def_content += '''onclick="openTab(event, '{}')"'''.format( viewer_name) tab_def_content += '''>Top {}% ({} Rows) Heatmap</button>\n'''.format( round(top_percent, 2), display_count) heatmap_report_files = os.listdir(top_heatmap_dir) heatmap_index_page = None for heatmap_report_file in heatmap_report_files: if heatmap_report_file.endswith('.html'): heatmap_index_page = heatmap_report_file shutil.copy2( os.path.join(top_heatmap_dir, heatmap_report_file), output_directory) if heatmap_index_page: tab_content += '''\n<div id="{}" class="tabcontent">'''.format( viewer_name) msg = 'Top {} percent of matrix sorted by sum of abundance values.'.format( round(top_percent, 2)) tab_content += '''<p style="color:red;" >{}</p>'''.format(msg) tab_content += '\n<iframe height="1300px" width="100%" ' tab_content += 'src="{}" '.format(heatmap_index_page) tab_content += 'style="border:none;"></iframe>' tab_content += '\n</div>\n' else: tab_content += '''\n<div id="{}" class="tabcontent">'''.format( viewer_name) tab_content += '''\n<p style="color:red;" >''' tab_content += '''Heatmap is too large to be displayed.</p>\n''' tab_content += '\n</div>\n' viewer_name = 'MatrixHeatmapViewer' tab_def_content += '''\n<button class="tablinks" ''' tab_def_content += '''onclick="openTab(event, '{}')"'''.format( viewer_name) tab_def_content += '''>Matrix Heatmap</button>\n''' heatmap_report_files = os.listdir(heatmap_dir) heatmap_index_page = None for heatmap_report_file in heatmap_report_files: if heatmap_report_file.endswith('.html'): heatmap_index_page = heatmap_report_file shutil.copy2(os.path.join(heatmap_dir, heatmap_report_file), output_directory) if heatmap_index_page: tab_content += '''\n<div id="{}" class="tabcontent">'''.format( viewer_name) tab_content += '\n<iframe height="1300px" width="100%" ' tab_content += 'src="{}" '.format(heatmap_index_page) tab_content += 'style="border:none;"></iframe>' tab_content += '\n</div>\n' else: tab_content += '''\n<div id="{}" class="tabcontent">'''.format( viewer_name) tab_content += '''\n<p style="color:red;" >''' tab_content += '''Heatmap is too large to be displayed.</p>\n''' tab_content += '\n</div>\n' tab_def_content += '\n</div>\n' return tab_def_content + tab_content def _generate_heatmap_html_report(self, data): logging.info('Start generating heatmap report page') data_df = pd.DataFrame(data['values'], index=data['row_ids'], columns=data['col_ids']) result_directory = os.path.join(self.scratch, str(uuid.uuid4())) self._mkdir_p(result_directory) tsv_file_path = os.path.join( result_directory, 'heatmap_data_{}.tsv'.format(str(uuid.uuid4()))) data_df.to_csv(tsv_file_path) if data_df.index.size < 10000: heatmap_dir = self.report_util.build_heatmap_html({ 'tsv_file_path': tsv_file_path, 'cluster_data': True })['html_dir'] else: logging.info( 'Original matrix is too large. Skip clustering data in report.' ) heatmap_dir = self.report_util.build_heatmap_html({ 'tsv_file_path': tsv_file_path, 'cluster_data': False })['html_dir'] top_heatmap_dir = None top_percent = 100 display_count = 200 # roughly count for display items if len(data_df.index) > 1000: top_percent = min(display_count / data_df.index.size * 100, 100) top_heatmap_dir = self.report_util.build_heatmap_html({ 'tsv_file_path': tsv_file_path, 'sort_by_sum': True, 'top_percent': top_percent })['html_dir'] output_directory = os.path.join(self.scratch, str(uuid.uuid4())) logging.info( 'Start generating html report in {}'.format(output_directory)) html_report = list() self._mkdir_p(output_directory) result_file_path = os.path.join(output_directory, 'matrix_viewer_report.html') visualization_content = self._generate_visualization_content( output_directory, heatmap_dir, data_df, top_heatmap_dir, top_percent, display_count) with open(result_file_path, 'w') as result_file: with open( os.path.join(os.path.dirname(__file__), 'templates', 'matrix_template.html'), 'r') as report_template_file: report_template = report_template_file.read() report_template = report_template.replace( '<p>Visualization_Content</p>', visualization_content) result_file.write(report_template) report_shock_id = self.dfu.file_to_shock({ 'file_path': output_directory, 'pack': 'zip' })['shock_id'] html_report.append({ 'shock_id': report_shock_id, 'name': os.path.basename(result_file_path), 'label': os.path.basename(result_file_path), 'description': 'HTML summary report for Import Amplicon Matrix App' }) return html_report def _generate_report(self, matrix_obj_ref, new_row_attr_ref, new_col_attr_ref, workspace_id, data=None): """ _generate_report: generate summary report """ objects_created = [{ 'ref': matrix_obj_ref, 'description': 'Imported Amplicon Matrix' }] if new_row_attr_ref: objects_created.append({ 'ref': new_row_attr_ref, 'description': 'Imported Amplicons(Row) Attribute Mapping' }) if new_col_attr_ref: objects_created.append({ 'ref': new_col_attr_ref, 'description': 'Imported Samples(Column) Attribute Mapping' }) if data: output_html_files = self._generate_heatmap_html_report(data) report_params = { 'message': '', 'objects_created': objects_created, 'workspace_id': workspace_id, 'html_links': output_html_files, 'direct_html_link_index': 0, 'html_window_height': 1400, 'report_object_name': 'import_matrix_from_biom_' + str(uuid.uuid4()) } else: report_params = { 'message': '', 'objects_created': objects_created, 'workspace_id': workspace_id, 'report_object_name': 'import_matrix_from_biom_' + str(uuid.uuid4()) } kbase_report_client = KBaseReport(self.callback_url, token=self.token) output = kbase_report_client.create_extended_report(report_params) report_output = { 'report_name': output['name'], 'report_ref': output['ref'] } return report_output def __init__(self, config): self.callback_url = config['SDK_CALLBACK_URL'] self.scratch = config['scratch'] self.token = config['KB_AUTH_TOKEN'] self.dfu = DataFileUtil(self.callback_url) self.report_util = kb_GenericsReport(self.callback_url) self.data_util = DataUtil(config) self.sampleservice_util = SampleServiceUtil(config) self.attr_util = AttributesUtil(config) self.matrix_util = MatrixUtil(config) self.taxon_util = TaxonUtil(config) self.matrix_types = [ x.split(".")[1].split('-')[0] for x in self.data_util.list_generic_types() ] self.taxon_wsname = config['taxon-workspace-name'] self.kbse = KBaseSearchEngine(config['search-url']) self.taxon_cache = dict() def fetch_sequence(self, matrix_ref): logging.info('start to fetch consensus sequence') input_matrix_obj = self.dfu.get_objects({'object_refs': [matrix_ref]})['data'][0] input_matrix_info = input_matrix_obj['info'] matrix_name = input_matrix_info[1] matrix_type = input_matrix_info[2] matrix_data = input_matrix_obj['data'] if 'KBaseMatrices.AmpliconMatrix' not in matrix_type: raise ValueError('Unexpected data type: {}'.format(matrix_type)) handle = matrix_data.get('sequencing_file_handle') if not handle: raise ValueError( 'Missing sequencing_file_handle from the matrix object') output_directory = os.path.join(self.scratch, str(uuid.uuid4())) logging.info('Start generating consensus sequence file in {}'.format( output_directory)) self._mkdir_p(output_directory) matrix_fasta_file = self.dfu.shock_to_file({ 'handle_id': handle, 'file_path': self.scratch }).get('file_path') try: logging.info('start parsing FASTA file') fastq_dict = SeqIO.index(matrix_fasta_file, "fasta") except Exception: raise ValueError( 'Cannot parse file. Please provide valide FASTA file') row_ids = matrix_data['data']['row_ids'] fasta_file_path = os.path.join( output_directory, matrix_name + 'consensus_sequence.fasta') with open(fasta_file_path, 'w') as f: for row_id in row_ids: consensus_sequence = str(fastq_dict.get(row_id).seq) f.write('>' + str(row_id) + '\n') f.write(consensus_sequence + '\n') return fasta_file_path def import_matrix_from_biom(self, params): """ arguments: obj_type: one of ExpressionMatrix, FitnessMatrix, DifferentialExpressionMatrix matrix_name: matrix object name workspace_id: workspace id matrix object to be saved to input_shock_id: file shock id or input_file_path: absolute file path or input_staging_file_path: staging area file path optional arguments: col_attributemapping_ref: column AttributeMapping reference row_attributemapping_ref: row AttributeMapping reference genome_ref: genome reference matrix_obj_ref: Matrix reference """ (biom_file, tsv_file, fasta_file, mode, metadata_keys) = self._process_params(params) workspace_id = params.get('workspace_id') matrix_name = params.get('matrix_name') obj_type = params.get('obj_type') scale = params.get('scale') description = params.get('description') refs = {k: v for k, v in params.items() if "_ref" in k} amplicon_data = self._file_to_amplicon_data(biom_file, tsv_file, fasta_file, mode, refs, matrix_name, workspace_id, scale, description, metadata_keys) for key in [ 'amplicon_type', 'amplification', 'extraction', 'target_gene', 'target_subfragment', 'pcr_primers', 'library_kit', 'library_layout', 'library_screening_strategy', 'sequencing_center', 'sequencing_date', 'sequencing_technology', 'sequencing_instrument', 'sequencing_quality_filter_cutoff', 'read_length_cutoff', 'read_pairing', 'barcode_error_rate', 'chimera_detection_and_removal', 'taxon_calling_method', 'denoise_method', 'sequence_error_cutoff', 'clustering_method', 'clustering_cutoff', 'sample_set_ref', 'reads_set_ref' ]: if params.get(key): amplicon_data[key] = params[key] new_row_attr_ref = None if not params.get('row_attributemapping_ref'): new_row_attr_ref = amplicon_data.get('row_attributemapping_ref') new_col_attr_ref = None if not params.get('col_attributemapping_ref'): new_col_attr_ref = amplicon_data.get('col_attributemapping_ref') if fasta_file: logging.info( 'start saving consensus sequence file to shock: {}'.format( fasta_file)) handle_id = self.dfu.file_to_shock({ 'file_path': fasta_file, 'make_handle': True })['handle']['hid'] amplicon_data['sequencing_file_handle'] = handle_id logging.info('start saving Matrix object: {}'.format(matrix_name)) matrix_obj_ref = self.data_util.save_object({ 'obj_type': 'KBaseMatrices.{}'.format(obj_type), 'obj_name': matrix_name, 'data': amplicon_data, 'workspace_id': workspace_id })['obj_ref'] if params.get('sample_set_ref'): self.matrix_util._link_matrix_to_samples(matrix_obj_ref, amplicon_data, params['sample_set_ref']) returnVal = {'matrix_obj_ref': matrix_obj_ref} report_output = self._generate_report(matrix_obj_ref, new_row_attr_ref, new_col_attr_ref, workspace_id, data=amplicon_data['data']) returnVal.update(report_output) return returnVal
class DataUtil: @staticmethod def _find_between(s, start, end): """ _find_between: find string in between start and end """ return re.search('{}(.*){}'.format(start, end), s).group(1) def _find_constraints(self, obj_type): """ _find_constraints: retrieve constraints (@contains, rowsum, unique, conditionally_required) """ type_info = self.wsClient.get_type_info(obj_type) type_desc = type_info.get('description') constraints = {} for tag in ('contains', 'rowsum', 'unique', 'conditionally_required'): constraints[tag] = [line.strip().split()[1:] for line in type_desc.split("\n") if line.startswith(f'@{tag}')] return constraints def _filter_constraints(self, constraints, data): """filters out constraints with missing keys""" contains_constraints = constraints.get('contains') filtered_constraints = [] for contains_constraint in contains_constraints: in_values = contains_constraint[1:] missing_key = True for in_value in in_values: if in_value.startswith('values'): search_value = re.search('{}(.*){}'.format('\(', '\)'), in_value).group(1) unique_list = search_value.split('.') key = unique_list[0] elif ':' in in_value: key = in_value.split(':')[0] else: unique_list = in_value.split('.') key = unique_list[0] if key in data: missing_key = False break if missing_key: filtered_constraints.append(contains_constraint) for x in filtered_constraints: contains_constraints.remove(x) return constraints def _retrieve_value(self, data, value): """Parse the provided 'data' object to retrieve the item in 'value'.""" logging.info('Getting value for {}'.format(value)) retrieve_data = [] m_data = DotMap(data) if value.startswith('set('): retrieve_data = value[4:-1].split(",") elif value.startswith('values('): # TODO: nested values e.g. values(values(ids)) search_value = re.search('{}(.*){}'.format('\(', '\)'), value).group(1) unique_list = search_value.split('.') m_data_cp = m_data.copy() for attr in unique_list: m_data_cp = getattr(m_data_cp, attr) retrieve_data = list(m_data_cp.values()) elif ':' in value: obj_ref = getattr(m_data, value.split(':')[0]) if obj_ref: included = value.split(':')[1] included = '/' + included.replace('.', '/') ref_data = self.wsClient.get_objects2({'objects': [{'ref': obj_ref, 'included': [included]}]})['data'][0]['data'] m_ref_data = DotMap(ref_data) if ref_data: if '*' not in included: for key in included.split('/')[1:]: m_ref_data = getattr(m_ref_data, key) else: keys = included.split('/')[1:] m_ref_data = [x.get(keys[2]) for x in ref_data.get(keys[0])] # TODO: only works for 2 level nested data like '/features/[*]/id' retrieve_data = list(m_ref_data) else: unique_list = value.split('.') m_data_cp = m_data.copy() for attr in unique_list: m_data_cp = getattr(m_data_cp, attr) retrieve_data = list(m_data_cp) logging.info('Retrieved value (first 20):\n{}\n'.format(retrieve_data[:20])) return retrieve_data def _validate(self, constraints, data): """ _validate: validate data """ validated = True failed_constraints = defaultdict(list) unique_constraints = constraints.get('unique') for unique_constraint in unique_constraints: retrieved_value = self._retrieve_value(data, unique_constraint[0]) if len(set(retrieved_value)) != len(retrieved_value): validated = False failed_constraints['unique'].append(unique_constraint[0]) contains_constraints = constraints.get('contains') for contains_constraint in contains_constraints: value = contains_constraint[0] in_values = contains_constraint[1:] retrieved_in_values = [] for in_value in in_values: retrieved_in_values += self._retrieve_value(data, in_value) if not (set(self._retrieve_value(data, value)) <= set(retrieved_in_values)): validated = False failed_constraints['contains'].append(" ".join(contains_constraint)) conditional_constraints = constraints.get('conditionally_required') for conditional_constraint in conditional_constraints: trigger = conditional_constraint[0] required_keys = conditional_constraint[1:] if trigger in data: missing_keys = [key for key in required_keys if key not in data] if missing_keys: validated = False failed_constraints['conditionally_required'].append( (trigger, required_keys, missing_keys)) return validated, failed_constraints @staticmethod def _mkdir_p(path): """ _mkdir_p: make directory for given path """ if not path: return try: os.makedirs(path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise @staticmethod def _raise_validation_error(params, validate): """Raise a meaningful error message for failed validation""" logging.error('Data failed type checking') failed_constraints = validate.get('failed_constraints') error_msg = ['Object {} failed type checking:'.format(params.get('obj_name'))] if failed_constraints.get('unique'): unique_values = failed_constraints.get('unique') error_msg.append('Object should have unique field: {}'.format(unique_values)) if failed_constraints.get('contains'): contained_values = failed_constraints.get('contains') for contained_value in contained_values: subset_value = contained_value.split(' ')[0] super_value = ' '.join(contained_value.split(' ')[1:]) if 'col_mapping' in super_value: error_msg.append('Column attribute mapping instances should contain all ' 'column index from original data') if 'row_mapping' in super_value: error_msg.append('Row attribute mapping instances should contain all row ' 'index from original data') error_msg.append('Object field [{}] should contain field [{}]'.format( super_value, subset_value)) for failure in failed_constraints.get('conditionally_required', []): error_msg.append('If object field "{}" is present than object field(s) {} should ' 'also be present. Object is missing {}'.format(*failure)) raise ValueError('\n'.join(error_msg)) def __init__(self, config): self.ws_url = config["workspace-url"] self.callback_url = config['SDK_CALLBACK_URL'] self.token = config['KB_AUTH_TOKEN'] self.scratch = config['scratch'] self.serviceWizardURL = config['srv-wiz-url'] self.wsClient = workspaceService(self.ws_url, token=self.token) self.dfu = DataFileUtil(self.callback_url) self.generics_service = GenericsService(self.serviceWizardURL) self.ws_large_data = WsLargeDataIO(self.callback_url) def list_generic_types(self, params=None): """ *Not yet exposed in spec* list_generic_types: lists the current valid generics types arguments: none return: A list of generic types in the current environment """ returnVal = [x['type_def'] for module in GENERICS_MODULES for x in self.wsClient.get_all_type_info(module)] return returnVal def fetch_data(self, params): """ fetch_data: fetch generics data as pandas dataframe for a generics data object arguments: obj_ref: generics object reference optional arguments: generics_module: the generics data module to be retrieved from e.g. for an given data type like below: typedef structure { FloatMatrix2D data; condition_set_ref condition_set_ref; } SomeGenericsMatrix; generics_module should be {'data': 'FloatMatrix2D', 'condition_set_ref': 'condition_set_ref'} return: data_matrix: a pandas dataframe in json format """ for p in ['obj_ref']: if p not in params: raise ValueError('"{}" parameter is required, but missing'.format(p)) return self.generics_service.fetch_data(params) def validate_data(self, params): """ validate_data: validate data arguments: obj_type: obj type e.g.: 'KBaseMatrices.ExpressionMatrix-1.1' data: obj data to be validated return: validated: True or False """ constraints = self._find_constraints(params.get('obj_type')) data = params.get('data') constraints = self._filter_constraints(constraints, data) validated, failed_constraints = self._validate(constraints, data) return {'validated': validated, 'failed_constraints': failed_constraints} def save_object(self, params): """ save_object: validate data constraints and save matrix object arguments: obj_type: saving object data type obj_name: saving object name data: data to be saved workspace_name: workspace name matrix object to be saved to return: obj_ref: object reference """ logging.info('Starting validating and saving object data') obj_type = params.get('obj_type').split('-')[0] module_name = obj_type.split('.')[0] type_name = obj_type.split('.')[1] types = self.wsClient.get_module_info({'mod': module_name}).get('types') for module_type in types: if self._find_between(module_type, '\.', '\-') == type_name: obj_type = module_type break data = dict((k, v) for k, v in params.get('data').items() if v) validate = self.validate_data({'obj_type': obj_type, 'data': data}) if not validate.get('validated'): self._raise_validation_error(params, validate) # make sure users with shared object have access to the handle file upon saving handle = data.get('sequencing_file_handle') if handle: output_directory = os.path.join(self.scratch, str(uuid.uuid4())) logging.info('Downloading consensus sequence file in {}'.format(output_directory)) self._mkdir_p(output_directory) matrix_fasta_file = self.dfu.shock_to_file({ 'handle_id': handle, 'file_path': self.scratch}).get('file_path') logging.info('Saving consensus sequence file to shock: {}'.format(matrix_fasta_file)) handle_id = self.dfu.file_to_shock({'file_path': matrix_fasta_file, 'make_handle': True})['handle']['hid'] data['sequencing_file_handle'] = handle_id # cast data int_data_names = ['sequencing_quality_filter_cutoff', 'read_length_cutoff'] for data_name in int_data_names: if data_name in data: try: logging.info('Casting {} to int'.format(data_name)) data[data_name] = int(data[data_name]) except Exception as e: err_msg = 'Unexpected data type {}. '.format(data_name) err_msg += 'Data type {} requests {} to be an integer value. '.format( obj_type, data_name) err_msg += 'Provided [{}] {} instead'.format( type(data[data_name]), data[data_name]) raise ValueError(err_msg) from e float_data_names = ['barcode_error_rate', 'sequence_error_cutoff', 'clustering_cutoff'] for data_name in float_data_names: if data_name in data: try: logging.info('Casting {} to float'.format(data_name)) data[data_name] = float(data[data_name]) except Exception as e: err_msg = 'Unexpected data type {}. '.format(data_name) err_msg += 'Data type {} requests {} to be a float value. '.format( obj_type, data_name) err_msg += 'Provided [{}] {} instead'.format( type(data[data_name]), data[data_name]) raise ValueError(err_msg) from e ws_name_id = params.get('workspace_id') workspace_name = params.get('workspace_name') if not ws_name_id: if not isinstance(workspace_name, int): ws_name_id = self.dfu.ws_name_to_id(workspace_name) else: ws_name_id = workspace_name try: logging.info('Starting saving object via DataFileUtil') info = self.dfu.save_objects({ "id": ws_name_id, "objects": [{ "type": obj_type, "data": data, "name": params.get('obj_name') }] })[0] except Exception: logging.info('Saving object via DataFileUtil failed') logging.info('Starting saving object via WsLargeDataIO') data_path = os.path.join(self.scratch, params.get('obj_name') + "_" + str(uuid.uuid4()) + ".json") json.dump(data, open(data_path, 'w')) info = self.ws_large_data.save_objects({ "id": ws_name_id, "objects": [{ "type": obj_type, "data_json_file": data_path, "name": params.get('obj_name') }] })[0] return {"obj_ref": "%s/%s/%s" % (info[6], info[0], info[4])}
class ReadsAlignmentUtils: ''' Module Name: ReadsAlignmentUtils Module Description: A KBase module: ReadsAlignmentUtils This module is intended for use by Aligners and Assemblers to upload and download alignment files. The alignment may be uploaded as a sam or bam file. If a sam file is given, it is converted to the sorted bam format and saved. Upon downloading, optional parameters may be provided to get files in sam and bai formats from the downloaded bam file. This utility also generates stats from the stored alignment. ''' ######## WARNING FOR GEVENT USERS ####### noqa # Since asynchronous IO can lead to methods - even the same method - # interrupting each other, you must be *very* careful when using global # state. A method could easily clobber the state set by another while # the latter method is running. ######################################### noqa VERSION = "0.3.6" GIT_URL = "https://github.com/kbaseapps/ReadsAlignmentUtils.git" GIT_COMMIT_HASH = "75ef2c24694c056dfca71859d6f344ccff7d4725" #BEGIN_CLASS_HEADER PARAM_IN_FILE = 'file_path' PARAM_IN_SRC_REF = 'source_ref' PARAM_IN_DST_REF = 'destination_ref' PARAM_IN_CONDITION = 'condition' PARAM_IN_READ_LIB_REF = 'read_library_ref' PARAM_IN_ASM_GEN_REF = 'assembly_or_genome_ref' PARAM_IN_ALIGNED_USING = 'aligned_using' PARAM_IN_ALIGNER_VER = 'aligner_version' PARAM_IN_ALIGNER_OPTS = 'aligner_opts' PARAM_IN_REPLICATE_ID = 'replicate_id' PARAM_IN_PLATFORM = 'platform' PARAM_IN_BOWTIE2_INDEX = 'bowtie2_index' PARAM_IN_SAMPLESET_REF = 'sampleset_ref' PARAM_IN_MAPPED_SAMPLE_ID = 'mapped_sample_id' PARAM_IN_DOWNLOAD_SAM = 'downloadSAM' PARAM_IN_DOWNLOAD_BAI = 'downloadBAI' PARAM_IN_VALIDATE = 'validate' INVALID_WS_OBJ_NAME_RE = re.compile('[^\\w\\|._-]') INVALID_WS_NAME_RE = re.compile('[^\\w:._-]') def _get_file_path_info(self, file_path): """ Given a file path, returns the directory, file name, file base and file extension """ dir, file_name = os.path.split(file_path) file_base, file_ext = os.path.splitext(file_name) return dir, file_name, file_base, file_ext def _mkdir_p(self, path): """ _mkdir_p: make directory for given path """ if not path: return try: os.makedirs(path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise def _check_required_param(self, in_params, param_list): """ Checks if each of the params in the list are in the input params """ for param in param_list: if (param not in in_params or not in_params[param]): raise ValueError('{} parameter is required'.format(param)) def _proc_ws_obj_params(self, ctx, params): """ Checks the validity of workspace and object params and returns them """ dst_ref = params.get(self.PARAM_IN_DST_REF) ws_name_id, obj_name_id = os.path.split(dst_ref) if not bool(ws_name_id.strip()) or ws_name_id == '/': raise ValueError("Workspace name or id is required in " + self.PARAM_IN_DST_REF) if not bool(obj_name_id.strip()): raise ValueError("Object name or id is required in " + self.PARAM_IN_DST_REF) if not isinstance(ws_name_id, int): try: ws_name_id = self.dfu.ws_name_to_id(ws_name_id) except DFUError as se: prefix = se.message.split('.')[0] raise ValueError(prefix) self.__LOGGER.info('Obtained workspace name/id ' + str(ws_name_id)) return ws_name_id, obj_name_id def _get_ws_info(self, obj_ref): ws = Workspace(self.ws_url) try: info = ws.get_object_info_new({'objects': [{'ref': obj_ref}]})[0] except WorkspaceError as wse: self.__LOGGER.error('Logging workspace exception') self.__LOGGER.error(str(wse)) raise return info def _proc_upload_alignment_params(self, ctx, params): """ Checks the presence and validity of upload alignment params """ self._check_required_param(params, [ self.PARAM_IN_DST_REF, self.PARAM_IN_FILE, self.PARAM_IN_CONDITION, self.PARAM_IN_READ_LIB_REF, self.PARAM_IN_ASM_GEN_REF ]) ws_name_id, obj_name_id = self._proc_ws_obj_params(ctx, params) file_path = params.get(self.PARAM_IN_FILE) if not (os.path.isfile(file_path)): raise ValueError('File does not exist: ' + file_path) lib_type = self._get_ws_info(params.get(self.PARAM_IN_READ_LIB_REF))[2] if lib_type.startswith('KBaseFile.SingleEndLibrary') or \ lib_type.startswith('KBaseFile.PairedEndLibrary') or \ lib_type.startswith('KBaseAssembly.SingleEndLibrary') or \ lib_type.startswith('KBaseAssembly.PairedEndLibrary'): pass else: raise ValueError(self.PARAM_IN_READ_LIB_REF + ' parameter should be of type' + ' KBaseFile.SingleEndLibrary or' + ' KBaseFile.PairedEndLibrary or' + ' KBaseAssembly.SingleEndLibrary or' + ' KBaseAssembly.PairedEndLibrary') obj_type = self._get_ws_info(params.get(self.PARAM_IN_ASM_GEN_REF))[2] if obj_type.startswith('KBaseGenomes.Genome') or \ obj_type.startswith('KBaseGenomeAnnotations.Assembly') or \ obj_type.startswith('KBaseGenomes.ContigSet'): pass else: raise ValueError(self.PARAM_IN_ASM_GEN_REF + ' parameter should be of type' + ' KBaseGenomes.Genome or' + ' KBaseGenomeAnnotations.Assembly or' + ' KBaseGenomes.ContigSet') return ws_name_id, obj_name_id, file_path, lib_type def _get_aligner_stats(self, bam_file): """ Gets the aligner stats from BAM file How we compute this stats: For each segment (line) in SAM/BAM file: we take the first element as `reads_id` the second element as `flag` if the last bit (0x1) of flag is `1`: we treat this segment as paired end reads otherwise: we treat this segment as single end reads For single end reads: if the 3rd last bit (0x8) of flag is `1`: we increment unmapped_reads_count else: we treat this `reads_id` as mapped for all mapped `reads_ids`" if it appears only once: we treat this `reads_id` as `singletons` else: we treat this `reads_id` as `multiple_alignments` lastly, total_reads = unmapped_reads_count + identical mapped `reads_id` For paired end reads: if the 7th last bit (0x40) of flag is `1`: if the 3rd last bit (0x8) of flag is `1`: we increment unmapped_left_reads_count else: we treat this `reads_id` as mapped if the 8th last bit ( 0x80) of flag is `1`: if the 3rd last bit (0x8) of flag is `1`: we increment unmapped_right_reads_count else: we treat this `reads_id` as mapped for all mapped `reads_ids`" if it appears only once: we treat this `reads_id` as `singletons` else: we treat this `reads_id` as `multiple_alignments` lastly, total_reads = unmapped_left_reads_count + unmapped_right_reads_count + identical mapped `reads_id` """ path, file = os.path.split(bam_file) self.__LOGGER.info('Start to generate aligner stats') start_time = time.time() infile = pysam.AlignmentFile(bam_file, 'r') properly_paired = 0 unmapped_reads_count = 0 unmapped_left_reads_count = 0 unmapped_right_reads_count = 0 mapped_reads_ids = [] mapped_left_reads_ids = [] mapped_right_reads_ids = [] paired = False for alignment in infile: seg = alignment.to_string().split('\t') reads_id = seg[0] flag = "0000000" + "{0:b}".format(int(seg[1])) if flag[-1] == '1': paired = True if paired: # process paired end sequence if flag[-7] == '1': # first sequence of a pair if flag[-3] == '1': unmapped_left_reads_count += 1 else: mapped_left_reads_ids.append(reads_id) if flag[-8] == '1': # second sequence of a pair if flag[-3] == '1': unmapped_right_reads_count += 1 else: mapped_right_reads_ids.append(reads_id) if flag[-2] == '1': properly_paired += 1 else: # process single end sequence if flag[-3] == '1': unmapped_reads_count += 1 else: mapped_reads_ids.append(reads_id) if flag[-2] == '1': properly_paired += 1 infile.close() if paired: mapped_reads_ids = mapped_left_reads_ids + mapped_right_reads_ids unmapped_reads_count = unmapped_left_reads_count + unmapped_right_reads_count mapped_reads_ids_counter = Counter(mapped_reads_ids) mapped_reads_count = len(list(mapped_reads_ids_counter)) singletons = list(mapped_reads_ids_counter.values()).count(1) multiple_alignments = mapped_reads_count - singletons total_reads = unmapped_reads_count + mapped_reads_count properly_paired = properly_paired // 2 else: mapped_reads_ids_counter = Counter(mapped_reads_ids) mapped_reads_count = len(list(mapped_reads_ids_counter)) singletons = list(mapped_reads_ids_counter.values()).count(1) multiple_alignments = mapped_reads_count - singletons total_reads = unmapped_reads_count + mapped_reads_count try: alignment_rate = round( float(mapped_reads_count) / total_reads * 100, 3) except ZeroDivisionError: alignment_rate = 0 if alignment_rate > 100: alignment_rate = 100.0 elapsed_time = time.time() - start_time self.__LOGGER.info('Used: {}'.format( time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))) stats_data = { "alignment_rate": alignment_rate, "mapped_reads": mapped_reads_count, "multiple_alignments": multiple_alignments, "properly_paired": properly_paired, "singletons": singletons, "total_reads": total_reads, "unmapped_reads": unmapped_reads_count } return stats_data def _validate(self, params): samt = SamTools(self.config, self.__LOGGER) if 'ignore' in params: path, file = os.path.split(params['file_path']) rval = samt.validate(ifile=file, ipath=path, ignore=params['ignore']) else: path, file = os.path.split(params['file_path']) rval = samt.validate(ifile=file, ipath=path) return rval #END_CLASS_HEADER # config contains contents of config file in a hash or None if it couldn't # be found def __init__(self, config): #BEGIN_CONSTRUCTOR self.config = config self.__LOGGER = logging.getLogger('KBaseRNASeq') if 'log_level' in config: self.__LOGGER.setLevel(config['log_level']) else: self.__LOGGER.setLevel(logging.INFO) streamHandler = logging.StreamHandler(sys.stdout) formatter = logging.Formatter( "%(asctime)s - %(filename)s - %(lineno)d - \ %(levelname)s - %(message)s") formatter.converter = time.gmtime streamHandler.setFormatter(formatter) self.__LOGGER.addHandler(streamHandler) self.__LOGGER.info("Logger was set") script_utils.check_sys_stat(self.__LOGGER) self.scratch = config['scratch'] self.callback_url = os.environ['SDK_CALLBACK_URL'] self.ws_url = config['workspace-url'] self.dfu = DataFileUtil(self.callback_url) self.samtools = SamTools(config) #END_CONSTRUCTOR pass def validate_alignment(self, ctx, params): """ :param params: instance of type "ValidateAlignmentParams" (* Input parameters for validating a reads alignment. For validation errors to ignore, see http://broadinstitute.github.io/picard/command-line-overview.html#V alidateSamFile) -> structure: parameter "file_path" of String, parameter "ignore" of list of String :returns: instance of type "ValidateAlignmentOutput" (* Results from validate alignment *) -> structure: parameter "validated" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)) """ # ctx is the context object # return variables are: returnVal #BEGIN validate_alignment rval = self._validate(params) if rval == 0: returnVal = {'validated': True} else: returnVal = {'validated': False} #END validate_alignment # At some point might do deeper type checking... if not isinstance(returnVal, dict): raise ValueError('Method validate_alignment return value ' + 'returnVal is not type dict as required.') # return the results return [returnVal] def upload_alignment(self, ctx, params): """ Validates and uploads the reads alignment How we compute BAM stats: For each segment (line) in SAM/BAM file: we take the first element as `reads_id` the second element as `flag` if the last bit (0x1) of flag is `1`: we treat this segment as paired end reads otherwise: we treat this segment as single end reads For single end reads: if the 3rd last bit (0x8) of flag is `1`: we increment unmapped_reads_count else: we treat this `reads_id` as mapped for all mapped `reads_ids`" if it appears only once: we treat this `reads_id` as `singletons` else: we treat this `reads_id` as `multiple_alignments` lastly, total_reads = unmapped_reads_count + identical mapped `reads_id` For paired end reads: if the 7th last bit (0x40) of flag is `1`: if the 3rd last bit (0x8) of flag is `1`: we increment unmapped_left_reads_count else: we treat this `reads_id` as mapped if the 8th last bit ( 0x80) of flag is `1`: if the 3rd last bit (0x8) of flag is `1`: we increment unmapped_right_reads_count else: we treat this `reads_id` as mapped for all mapped `reads_ids`" if it appears only once: we treat this `reads_id` as `singletons` else: we treat this `reads_id` as `multiple_alignments` lastly, total_reads = unmapped_left_reads_count + unmapped_right_reads_count + identical mapped `reads_id` :param params: instance of type "UploadAlignmentParams" (* Required input parameters for uploading a reads alignment string destination_ref - object reference of alignment destination. The object ref is 'ws_name_or_id/obj_name_or_id' where ws_name_or_id is the workspace name or id and obj_name_or_id is the object name or id file_path - File with the path of the sam or bam file to be uploaded. If a sam file is provided, it will be converted to the sorted bam format before being saved read_library_ref - workspace object ref of the read sample used to make the alignment file condition - assembly_or_genome_ref - workspace object ref of genome assembly or genome object that was used to build the alignment *) -> structure: parameter "destination_ref" of String, parameter "file_path" of String, parameter "read_library_ref" of String, parameter "condition" of String, parameter "assembly_or_genome_ref" of String, parameter "aligned_using" of String, parameter "aligner_version" of String, parameter "aligner_opts" of mapping from String to String, parameter "replicate_id" of String, parameter "platform" of String, parameter "bowtie2_index" of type "ws_bowtieIndex_id", parameter "sampleset_ref" of type "ws_Sampleset_ref", parameter "mapped_sample_id" of mapping from String to mapping from String to String, parameter "validate" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "ignore" of list of String :returns: instance of type "UploadAlignmentOutput" (* Output from uploading a reads alignment *) -> structure: parameter "obj_ref" of String """ # ctx is the context object # return variables are: returnVal #BEGIN upload_alignment self.__LOGGER.info( 'Starting upload Reads Alignment, parsing parameters ') pprint(params) ws_name_id, obj_name_id, file_path, lib_type = self._proc_upload_alignment_params( ctx, params) dir, file_name, file_base, file_ext = self._get_file_path_info( file_path) if self.PARAM_IN_VALIDATE in params and params[ self.PARAM_IN_VALIDATE] is True: if self._validate(params) == 1: raise Exception('{0} failed validation'.format(file_path)) bam_file = file_path if file_ext.lower() == '.sam': bam_file = os.path.join(dir, file_base + '.bam') self.samtools.convert_sam_to_sorted_bam(ifile=file_name, ipath=dir, ofile=bam_file) uploaded_file = self.dfu.file_to_shock({ 'file_path': bam_file, 'make_handle': 1 }) file_handle = uploaded_file['handle'] file_size = uploaded_file['size'] aligner_stats = self._get_aligner_stats(file_path) aligner_data = { 'file': file_handle, 'size': file_size, 'condition': params.get(self.PARAM_IN_CONDITION), 'read_sample_id': params.get(self.PARAM_IN_READ_LIB_REF), 'library_type': lib_type, 'genome_id': params.get(self.PARAM_IN_ASM_GEN_REF), 'alignment_stats': aligner_stats } optional_params = [ self.PARAM_IN_ALIGNED_USING, self.PARAM_IN_ALIGNER_VER, self.PARAM_IN_ALIGNER_OPTS, self.PARAM_IN_REPLICATE_ID, self.PARAM_IN_PLATFORM, self.PARAM_IN_BOWTIE2_INDEX, self.PARAM_IN_SAMPLESET_REF, self.PARAM_IN_MAPPED_SAMPLE_ID ] for opt_param in optional_params: if opt_param in params and params[opt_param] is not None: aligner_data[opt_param] = params[opt_param] self.__LOGGER.info('========= Adding extra_provenance_refs') self.__LOGGER.info(params.get(self.PARAM_IN_READ_LIB_REF)) self.__LOGGER.info(params.get(self.PARAM_IN_ASM_GEN_REF)) self.__LOGGER.info('=======================================') res = self.dfu.save_objects({ "id": ws_name_id, "objects": [{ "type": "KBaseRNASeq.RNASeqAlignment", "data": aligner_data, "name": obj_name_id, "extra_provenance_input_refs": [ params.get(self.PARAM_IN_READ_LIB_REF), params.get(self.PARAM_IN_ASM_GEN_REF) ] }] })[0] self.__LOGGER.info('save complete') returnVal = { 'obj_ref': str(res[6]) + '/' + str(res[0]) + '/' + str(res[4]) } self.__LOGGER.info('Uploaded object: ') self.__LOGGER.info(returnVal) #END upload_alignment # At some point might do deeper type checking... if not isinstance(returnVal, dict): raise ValueError('Method upload_alignment return value ' + 'returnVal is not type dict as required.') # return the results return [returnVal] def download_alignment(self, ctx, params): """ Downloads alignment files in .bam, .sam and .bai formats. Also downloads alignment stats * :param params: instance of type "DownloadAlignmentParams" (* Required input parameters for downloading a reads alignment string source_ref - object reference of alignment source. The object ref is 'ws_name_or_id/obj_name_or_id' where ws_name_or_id is the workspace name or id and obj_name_or_id is the object name or id *) -> structure: parameter "source_ref" of String, parameter "downloadSAM" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "downloadBAI" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "validate" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "ignore" of list of String :returns: instance of type "DownloadAlignmentOutput" (* The output of the download method. *) -> structure: parameter "destination_dir" of String, parameter "stats" of type "AlignmentStats" -> structure: parameter "properly_paired" of Long, parameter "multiple_alignments" of Long, parameter "singletons" of Long, parameter "alignment_rate" of Double, parameter "unmapped_reads" of Long, parameter "mapped_reads" of Long, parameter "total_reads" of Long """ # ctx is the context object # return variables are: returnVal #BEGIN download_alignment self.__LOGGER.info('Running download_alignment with params:\n' + pformat(params)) inref = params.get(self.PARAM_IN_SRC_REF) if not inref: raise ValueError('{} parameter is required'.format( self.PARAM_IN_SRC_REF)) try: alignment = self.dfu.get_objects({'object_refs': [inref]})['data'] except DFUError as e: self.__LOGGER.error( 'Logging stacktrace from workspace exception:\n' + e.data) raise # set the output dir uuid_str = str(uuid.uuid4()) output_dir = os.path.join(self.scratch, 'download_' + uuid_str) self._mkdir_p(output_dir) file_ret = self.dfu.shock_to_file({ 'shock_id': alignment[0]['data']['file']['id'], 'file_path': output_dir }) if zipfile.is_zipfile(file_ret.get('file_path')): with zipfile.ZipFile(file_ret.get('file_path')) as z: z.extractall(output_dir) for f in glob.glob(output_dir + '/*.zip'): os.remove(f) bam_files = glob.glob(output_dir + '/*.bam') if len(bam_files) == 0: raise ValueError("Alignment object does not contain a bam file") for bam_file_path in bam_files: dir, file_name, file_base, file_ext = self._get_file_path_info( bam_file_path) if params.get(self.PARAM_IN_VALIDATE, False): validate_params = {'file_path': bam_file_path} if self._validate(validate_params) == 1: raise Exception( '{0} failed validation'.format(bam_file_path)) if params.get(self.PARAM_IN_DOWNLOAD_BAI, False): bai_file = file_base + '.bai' bai_file_path = os.path.join(output_dir, bai_file) self.samtools.create_bai_from_bam(ifile=file_name, ipath=output_dir, ofile=bai_file) if not os.path.isfile(bai_file_path): raise ValueError('Error creating {}'.format(bai_file_path)) if params.get(self.PARAM_IN_DOWNLOAD_SAM, False): sam_file = file_base + '.sam' sam_file_path = os.path.join(output_dir, sam_file) self.samtools.convert_bam_to_sam(ifile=file_name, ipath=output_dir, ofile=sam_file) if not os.path.isfile(sam_file_path): raise ValueError('Error creating {}'.format(sam_file_path)) returnVal = { 'destination_dir': output_dir, 'stats': alignment[0]['data']['alignment_stats'] } #END download_alignment # At some point might do deeper type checking... if not isinstance(returnVal, dict): raise ValueError('Method download_alignment return value ' + 'returnVal is not type dict as required.') # return the results return [returnVal] def export_alignment(self, ctx, params): """ Wrapper function for use by in-narrative downloaders to download alignments from shock * :param params: instance of type "ExportParams" (* Required input parameters for exporting a reads alignment string source_ref - object reference of alignment source. The object ref is 'ws_name_or_id/obj_name_or_id' where ws_name_or_id is the workspace name or id and obj_name_or_id is the object name or id *) -> structure: parameter "source_ref" of String, parameter "exportSAM" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "exportBAI" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "validate" of type "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)), parameter "ignore" of list of String :returns: instance of type "ExportOutput" -> structure: parameter "shock_id" of String """ # ctx is the context object # return variables are: output #BEGIN export_alignment inref = params.get(self.PARAM_IN_SRC_REF) if not inref: raise ValueError('{} parameter is required'.format( self.PARAM_IN_SRC_REF)) if params.get(self.PARAM_IN_VALIDATE, False) or \ params.get('exportBAI', False) or \ params.get('exportSAM', False): """ Need to validate or convert files. Use download_alignment """ download_params = {} for key, val in params.items(): download_params[key.replace('export', 'download')] = val download_retVal = self.download_alignment(ctx, download_params)[0] export_dir = download_retVal['destination_dir'] # package and load to shock ret = self.dfu.package_for_download({ 'file_path': export_dir, 'ws_refs': [inref] }) output = {'shock_id': ret['shock_id']} else: """ return shock id from the object """ try: alignment = self.dfu.get_objects({'object_refs': [inref]})['data'] except DFUError as e: self.__LOGGER.error( 'Logging stacktrace from workspace exception:\n' + e.data) raise output = {'shock_id': alignment[0]['data']['file']['id']} #END export_alignment # At some point might do deeper type checking... if not isinstance(output, dict): raise ValueError('Method export_alignment return value ' + 'output is not type dict as required.') # return the results return [output] def status(self, ctx): #BEGIN_STATUS returnVal = { 'state': "OK", 'message': "", 'version': self.VERSION, 'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH } #END_STATUS return [returnVal]
class ProkkaUtils: def __init__(self, config): self.scratch = config["scratch"] self.ctx = config['ctx'] self.callback_url = config["SDK_CALLBACK_URL"] self.ws_client = workspaceService(config["workspace-url"]) self.gfu = GenomeFileUtil(self.callback_url) self.au = AssemblyUtil(self.callback_url) self.kbr = KBaseReport(self.callback_url) self.dfu = DataFileUtil(self.callback_url) self.genome_api = GenomeAnnotationAPI(self.callback_url) self.sso_ref = None self.sso_event = None self.ec_to_sso = {} self.output_workspace = None @staticmethod def _get_input_value(params, key): """Get value of key after checking for its existence :param params: Params dictionary haystack :param key: Key to search in Params :return: Parameter Value :raises ValueError: raises an exception if the key doesn"t exist """ if not key in params: raise ValueError("Parameter " + key + " should be set in input parameters") return params[key] @staticmethod def _get_qualifier_value(qualifier): """Get first qualifier from the list of qualifiers :param qualifier: list contents of the qualifier from BCBio GFF Tools :return: first element in the list """ return qualifier[0] if (qualifier and len(qualifier) > 0) else None def download_seed_data(self): """Download Seed Data Ontology, and set the gene_ontology reference (sso_ref) and the create a table from ec numbers to sso (ec_to_sso) :return: None """ # Download Seed Reference Data sso_ret = self.ws_client.get_objects([{ "ref": "KBaseOntology/seed_subsystem_ontology" }])[0] sso = sso_ret["data"] for sso_id in sso["term_hash"]: sso_name = sso["term_hash"][sso_id]["name"] if "(EC " in sso_name and sso_name.endswith(")"): ec = sso_name[sso_name.index("(EC ") + 4:-1].strip() sso_list = self.ec_to_sso.get(ec, None) if not sso_list: sso_list = [] self.ec_to_sso[ec] = sso_list sso_list.append(sso["term_hash"][sso_id]) print("EC found in SSO: " + str(len(self.ec_to_sso))) sso_info = sso_ret["info"] sso_ref = str(sso_info[6]) + "/" + str(sso_info[0]) + "/" + str( sso_info[4]) with open("/kb/module/work/seed_so.json", "w") as outfile: json.dump(sso, outfile, sort_keys=True, indent=4) self.sso_ref = sso_ref def inspect_assembly(self, assembly_meta, assembly_ref): """Check to see if assembly has too many contigs and might not be a metagenome or non prokaryotic dataset :param assembly_meta: information about the assembly reference :param assembly_ref: the assembly reference number :return: a tuple containing gc_content and dna_size """ gc_content = float(assembly_meta.get("GC content")) dna_size = int(assembly_meta.get("Size")) n_contigs = 0 if "N Contigs" in assembly_meta: n_contigs = int(assembly_meta.get("N Contigs")) else: contig = self.ws_client.get_objects([{"ref": assembly_ref}])[0] n_contigs = len(contig["data"]["contigs"]) if n_contigs >= 30000: message = """ Hmmm. There are over 30,000 contigs in this Assembly. It looks like you are trying to run Prokka on a metagenome or non-prokaryotic data set. If this is a metagenome data set we recommend using an App like MaxBin to first bin the contigs into genome-like bins. These bins can then be individually annotated as a single genome using Prokka. If this data comes from a Eukaryotic sample, KBase does not currently have an annotation app designed for Eukaryotes. Alternatively, you can try reducing the number of contigs using a filter app.") raise ValueError("Too many contigs for Prokka. See logs for details and suggestions """ print(message) #raise ValueError("Too many contigs for Prokka. See logs for details and suggestions") assembly_info = namedtuple("assembly_info", "gc_content dna_size") return assembly_info(gc_content, dna_size) @staticmethod def create_renamed_assembly(assembly_fasta_filepath): """Rename records to be in the format of contig_N and output a new fasta file :param assembly_fasta_filepath: :return: A tuple with The path to the fasta file with renamed contigs the number of contigs, the mapping from old ids to new ids, and the contigs as SeqRecords """ records = [] new_ids_to_old = {} contig_counter = 0 for record in SeqIO.parse(assembly_fasta_filepath, "fasta"): contig_counter += 1 old_id = record.id new_id = "contig_" + str(contig_counter) sequence = record.seq # it has type "Seq" record = SeqRecord(sequence, id=new_id, description="(" + old_id + ")") records.append(record) new_ids_to_old[new_id] = old_id renamed_assembly_fasta_filepath = assembly_fasta_filepath + "_renamed.fna" SeqIO.write(records, renamed_assembly_fasta_filepath, "fasta") renamed_assembly = namedtuple( "renamed_assembly", "filepath contig_counter new_ids_to_old records") return renamed_assembly(renamed_assembly_fasta_filepath, contig_counter, new_ids_to_old, records) def run_prokka(self, params, subject_fasta_filepath): """Run Prokka :param params: Prokka parameters :param subject_fasta_filepath: The contigs or genes to run prokka against :return: The directory with all of the prokka output files """ output_dir = "/kb/module/work/tmp/temp_" + str(uuid.uuid4()) # --kingdom [X] Annotation mode: Archaea|Bacteria|Mitochondria|Viruses (default "Bacteria") kingdom = "Bacteria" if "kingdom" in params and params["kingdom"]: kingdom = params["kingdom"] prokka_cmd_list = [ "perl", "/kb/prokka/bin/prokka", "--metagenome", "--outdir", output_dir, "--prefix", "mygenome", "--kingdom", kingdom ] # --genus [X] Genus name (triggers to use --usegenus) if "genus" in params and params["genus"]: prokka_cmd_list.extend( ["--genus", str(params["genus"]), "--usegenus"]) # --gcode [N] Genetic code / Translation table (set if --kingdom is set) (default "0") if "gcode" in params and params["gcode"]: prokka_cmd_list.extend(["--gcode", str(params["gcode"])]) else: prokka_cmd_list.extend(["--gcode", "0"]) # --gram [X] Gram: -/neg +/pos (default "") if "gram" in params and params["gram"]: raise ValueError( "gram parameter is not supported in current Prokka installation" ) # --metagenome Improve gene predictions for highly fragmented genomes (default OFF) if "metagenome" in params and params["metagenome"] == 1: prokka_cmd_list.append("--metagenome") # --rawproduct Do not clean up /product annotation (default OFF) if "rawproduct" in params and params["rawproduct"] == 1: prokka_cmd_list.append("--rawproduct") # --fast Fast mode - skip CDS /product searching (default OFF) if "fast" in params and params["fast"] == 1: prokka_cmd_list.append("--fast") # --mincontiglen [N] Minimum contig size [NCBI needs 200] (default "1") if "mincontiglen" in params and params["mincontiglen"]: prokka_cmd_list.extend( ["--mincontiglen", str(params["mincontiglen"])]) # --evalue [n.n] Similarity e-value cut-off (default "1e-06") if "evalue" in params and params["evalue"]: prokka_cmd_list.extend(["--evalue", str(params["evalue"])]) # --rfam Enable searching for ncRNAs with Infernal+Rfam (SLOW!) (default "0") if "rfam" in params and params["rfam"] == 1: prokka_cmd_list.append("--rfam") # --norrna Don"t run rRNA search (default OFF) if "norrna" in params and params["norrna"] == 1: prokka_cmd_list.append("--norrna") # --notrna Don"t run tRNA search (default OFF) if "notrna" in params and params["notrna"] == 1: prokka_cmd_list.append("--notrna") prokka_cmd_list.append(subject_fasta_filepath) print("Prokka command line: " + str(prokka_cmd_list)) #tbl2asn or some other non essential prokka binary will fail, so supress that try: check_output(prokka_cmd_list, cwd=self.scratch) except CalledProcessError as e: pprint(e) return output_dir @staticmethod def retrieve_prokka_results(output_dir): """ Gather up the relevant prokka results, load the records from the results files :param output_dir: :return: A tuple containing Sequences from the .faa .ffn files and the gff_filepath """ faa_file = output_dir + "/mygenome.faa" cds_to_prot = {} for record in SeqIO.parse(faa_file, "fasta"): cds_to_prot[record.id] = str(record.seq) ffn_file = output_dir + "/mygenome.ffn" cds_to_dna = {} for record in SeqIO.parse(ffn_file, "fasta"): cds_to_dna[record.id] = str(record.seq) gff_file = output_dir + "/mygenome.gff" if not os.path.isfile(gff_file): raise ValueError("PROKKA output GFF file is not found") prokka_results = namedtuple("prokka_results", "cds_to_prot cds_to_dna gff_filepath") return prokka_results(cds_to_prot, cds_to_dna, gff_file) def parse_prokka_results(self, **prokka_parse_parameters): """ Go through the prokka results from the input contigs and then create the features, mrnas and cdss components of the KbaseGenome.Genome object for genome annotation only. :param prokka_parse_parameters: gff_filepath, mappings :return: A tuple with Genome:features Genome:cdss Genome:mrnas report_message of genes discovered """ gff_filepath = prokka_parse_parameters["gff_filepath"] cds_to_dna = prokka_parse_parameters["cds_to_dna"] cds_to_prot = prokka_parse_parameters["cds_to_prot"] new_ids_to_old = prokka_parse_parameters["new_ids_to_old"] evidence = self.make_annotation_evidence() cdss = [] mrnas = [] features = [] non_hypothetical = 0 genes_with_ec = 0 genes_with_sso = 0 prot_lengths = [] with open(gff_filepath, "r") as f1: for rec in GFF.parse(f1): contig_id = new_ids_to_old[str(rec.id)] for ft in rec.features: loc = ft.location min_pos = int(loc.start) + 1 max_pos = int(loc.end) strand = "+" if loc.strand == 1 else "-" flen = max_pos - min_pos + 1 start = min_pos if strand == "+" else max_pos location = [[contig_id, start, strand, flen]] qualifiers = ft.qualifiers generated_id = self._get_qualifier_value( qualifiers.get("ID")) if not generated_id: # Skipping feature with no ID (mostly repeat regions) continue dna = cds_to_dna.get(generated_id) if not dna: # Skipping feature with no DNA (mostly repeat regions) continue name = self._get_qualifier_value(qualifiers.get("Name")) ec = self._get_qualifier_value(qualifiers.get("eC_number")) gene = self._get_qualifier_value(qualifiers.get("gene")) product = self._get_qualifier_value( qualifiers.get("product")) fid = generated_id aliases = [] if name: aliases.append(name) if gene: aliases.append(gene) if ec: aliases.append(ec) genes_with_ec += 1 md5 = hashlib.md5(dna).hexdigest() feature = { "id": fid, "location": location, "type": "gene", "aliases": aliases, "md5": md5, "dna_sequence": dna, "dna_sequence_length": len(dna), } if product: feature["function"] = product if product != "hypothetical protein": non_hypothetical += 1 if ec and ec in self.ec_to_sso: sso_list = self.ec_to_sso[ec] sso_terms = {} for sso_item in sso_list: sso_terms[sso_item["id"]] = { "id": sso_item["id"], "evidence": [evidence], "term_name": sso_item["name"], "ontology_ref": self.sso_ref, "term_lineage": [] } feature["ontology_terms"] = {"SSO": sso_terms} genes_with_sso += 1 cds = None mrna = None prot = cds_to_prot.get(generated_id) if prot: cds_id = fid + "_CDS" mrna_id = fid + "_mRNA" prot_len = len(prot) prot_lengths.append(prot_len) feature["protein_translation"] = prot feature["protein_translation_length"] = prot_len feature["cdss"] = [cds_id] feature["mrnas"] = [mrna_id] cds = { "id": cds_id, "location": location, "md5": md5, "parent_gene": fid, "parent_mrna": mrna_id, "function": (product if product else ""), "ontology_terms": {}, "protein_translation": prot, "protein_translation_length": prot_len, "aliases": aliases } mrna = { "id": mrna_id, "location": location, "md5": md5, "parent_gene": fid, "cds": cds_id } features.append(feature) if cds: cdss.append(cds) if mrna: mrnas.append(mrna) # Prepare report report = "" report += "Number of genes predicted: " + str(len(features)) + "\n" report += "Number of protein coding genes: " + str( len(prot_lengths)) + "\n" report += "Number of genes with non-hypothetical function: " + str( non_hypothetical) + "\n" report += "Number of genes with EC-number: " + str( genes_with_ec) + "\n" report += "Number of genes with Seed Subsystem Ontology: " + str( genes_with_sso) + "\n" report += "Average protein length: " + str( int(sum(prot_lengths) / float(len(prot_lengths)))) + " aa.\n" annotated_assembly = namedtuple("annotated_assembly", "features cdss mrnas report_message") return annotated_assembly(features, cdss, mrnas, report) def get_new_annotations(self, gff_filepath): """ :param gff_filepath: A dictionary of ids with products and ec numbers :return: """ evidence = self.make_annotation_evidence() genome = {} with open(gff_filepath, "r") as f: for rec in GFF.parse(f): gid = rec.id gene_features = {"id": id} for feature in rec.features: qualifiers = feature.qualifiers if "product" in qualifiers: gene_features["function"] = " ".join( qualifiers["product"]) if "eC_number" in qualifiers: ec_numbers = qualifiers["eC_number"] sso_terms = dict() for ec in ec_numbers: sso_list = self.ec_to_sso.get(ec, []) for sso_item in sso_list: sso_terms[sso_item["id"]] = { "id": sso_item["id"], "evidence": [evidence], "term_name": sso_item["name"], "ontology_ref": self.sso_ref, "term_lineage": [] } gene_features["ontology_terms"] = sso_terms genome[gid] = gene_features return genome def write_genome_to_fasta(self, genome_data): """ :param genome_data: :return: """ fasta_for_prokka_filepath = os.path.join( self.scratch, "features_" + str(uuid.uuid4()) + ".fasta") count = 0 with open(fasta_for_prokka_filepath, "w") as f: for item in genome_data["data"]["features"]: if "id" not in item or "dna_sequence" not in item: print("This feature does not have a valid dna sequence.") else: f.write(">" + item["id"] + "\n" + item["dna_sequence"] + "\n") count += 1 print("Finished printing to" + fasta_for_prokka_filepath) if os.stat(fasta_for_prokka_filepath).st_size == 0: raise Exception( "This genome does not contain features with DNA_SEQUENCES. Fasta file is empty." ) return fasta_for_prokka_filepath def make_sso_ontology_event(self): """ :param sso_ref: Reference to the annotation library set :return: Ontology_event to be appended to the list of genome ontology events """ time_string = str( datetime.datetime.fromtimestamp( time.time()).strftime('%Y_%m_%d_%H_%M_%S')) yml_text = open('/kb/module/kbase.yml').read() version = re.search("module-version:\n\W+(.+)\n", yml_text).group(1) return { "method": "Prokka Annotation", "method_version": version, "timestamp": time_string, "id": "SSO", "ontology_ref": self.sso_ref } def make_annotation_evidence(self): """ Create a dict for the evidence field for the genome :param sso_ref: Reference to the annotation library set :return: Ontology_event to be appended to the list of genome ontology events """ time_string = str( datetime.datetime.fromtimestamp( time.time()).strftime('%Y_%m_%d_%H_%M_%S')) yml_text = open('/kb/module/kbase.yml').read() version = re.search("module-version:\n\W+(.+)\n", yml_text).group(1) return { "method": "Prokka Annotation (Evidence)", "method_version": version, "timestamp": time_string, } def create_genome_ontology_fields(self, genome_data): """ Create ontology event fields for a genome object :param genome_data: A genome object's data filed :return: a named tuple containg the modified genome object and a new ontology event index """ # Make sure ontologies_events exist sso_event = self.make_sso_ontology_event() ontology_event_index = 0 if 'ontology_events' in genome_data['data']: genome_data['data']['ontology_events'].append(sso_event) ontology_event_index += len( genome_data['data']['ontology_events']) - 1 else: genome_data['data']['ontology_events'] = [sso_event] genome_obj_modified = namedtuple('genome_obj_modified', 'genome_data ontology_event_index') return genome_obj_modified(genome_data, ontology_event_index) @staticmethod def old_genome_ontologies(feature, new_ontology): """ Update the feature's ontologies for an old genome :param feature: Feature to update :param new_ontology: New Ontology to update with :return: The feature with the ontology updated, in the old style """ if "ontology_terms" not in feature: feature["ontology_terms"] = {"SSO": {}} if "SSO" not in feature["ontology_terms"]: feature["ontology_terms"]["SSO"] = {} for key in new_ontology.keys(): feature["ontology_terms"]["SSO"][key] = new_ontology[key] return feature @staticmethod def new_genome_ontologies(feature, new_ontology, ontology_event_index): """ Update the feature's ontologies for a new genome :param feature: Feature to update :param new_ontology: New Ontology to update with :param ontology_event_index: Ontology index to update the feature with :return: the updated feature """ if "ontology_terms" not in feature: feature["ontology_terms"] = {"SSO": {}} if "SSO" not in feature["ontology_terms"]: feature["ontology_terms"]["SSO"] = {} for key in new_ontology.keys(): id = new_ontology[key]["id"] if id in feature["ontology_terms"]["SSO"]: feature["ontology_terms"]["SSO"][id].append( ontology_event_index) else: feature["ontology_terms"]["SSO"][id] = [ontology_event_index] return feature def annotate_genome_with_new_annotations(self, **annotation_args): """ Annotate the genome with new annotations for Genome ReAnnotation :param annotation_args: genome_data from the genome obj, new_annotations from prokka, and the output_genome_name :return: A tuple containg the genome_ref, filepaths for the function and ontology summary, and stats about the annotations """ genome_data = annotation_args["genome_data"] new_annotations = annotation_args["new_annotations"] new_genome = False if 'feature_counts' in genome_data['data']: new_genome = True genome_obj_modified = self.create_genome_ontology_fields( genome_data) genome_data = genome_obj_modified.genome_data ontology_event_index = genome_obj_modified.ontology_event_index stats = { "current_functions": len(genome_data["data"]["features"]), "new_functions": 0, "found_functions": 0, "new_ontologies": 0 } function_summary_fp = os.path.join(self.scratch, "ontology_report") ontology_summary_fp = os.path.join(self.scratch, "function_report") onto_r = open(function_summary_fp, "w") func_r = open(ontology_summary_fp, "w") func_r.write("function_id current_function new_function\n") onto_r.write("function_id current_ontology new_ontology\n") ontologies_present = {"SSO": {}} for i, feature in enumerate(genome_data["data"]["features"]): fid = feature["id"] current_function = feature.get("function", "") current_functions = feature.get("functions", []) current_ontology = feature.get("ontology_terms", None) new_function = "" new_ontology = dict() if fid in new_annotations: # Set Function new_function = new_annotations[fid].get("function", "") if new_function and "hypothetical protein" not in new_function: if (new_function != current_function and new_function not in current_functions): stats['new_functions'] += 1 genome_data["data"]["features"][i][ "function"] = new_function genome_data["data"]["features"][i]["functions"] = [ new_function ] stats['found_functions'] += 1 # Set Ontologies new_ontology = new_annotations[fid].get("ontology_terms", None) if new_ontology: stats['new_ontologies'] += 1 if new_genome: # New style genome_data["data"]["features"][i] = self. \ new_genome_ontologies(feature, new_ontology, ontology_event_index) # Add to ontologies Present for key in new_ontology.keys(): oid = new_ontology[key]["id"] name = new_ontology[key].get("name", "Unknown") ontologies_present["SSO"][oid] = name else: genome_data["data"]["features"][i] = self. \ old_genome_ontologies(feature, new_ontology) if current_function: func_r.write( json.dumps([fid, [current_function], [new_function]]) + "\n") else: func_r.write( json.dumps([fid, current_functions, [new_function]]) + "\n") onto_r.write( json.dumps([fid, current_ontology, new_ontology]) + "\n") func_r.close() onto_r.close() if ontologies_present: if "ontologies_present" in genome_data["data"]: if "SSO" in genome_data["data"]["ontologies_present"]: for key, value in ontologies_present["SSO"].items(): genome_data["data"]["ontologies_present"]["SSO"][ key] = value else: genome_data["data"][ "ontologies_present"] = ontologies_present["SSO"] else: genome_data["data"]["ontologies_present"] = ontologies_present info = self.gfu.save_one_genome({ "workspace": self.output_workspace, "name": annotation_args["output_genome_name"], "data": genome_data["data"], "provenance": self.ctx.provenance() })["info"] genome_ref = str(info[6]) + "/" + str(info[0]) + "/" + str(info[4]) annotated_genome = namedtuple( "annotated_genome", "genome_ref function_summary_filepath ontology_summary_filepath stats" ) return annotated_genome(genome_ref, function_summary_fp, ontology_summary_fp, stats) def upload_file(self, filepath, message="Annotation report generated by kb_prokka"): """ Upload a file to shock :param filepath: File to upload :param message: Optional Upload Message :return: """ output_file_shock_id = self.dfu.file_to_shock({"file_path": filepath})["shock_id"] print("Uploaded filepath" + filepath + "to shock and got id" + output_file_shock_id) return { "shock_id": output_file_shock_id, "name": os.path.basename(filepath), "label": os.path.basename(filepath), "description": message } def report_annotated_genome(self, genome): """ Create report output with newly reannotated genome, and some stats :param genome: Reannotated Genome Reference, Report Files and Stats :return: Reference to Report Object """ genome_ref = genome.genome_ref stats = genome.stats file_links = [ self.upload_file(genome.ontology_summary_filepath), self.upload_file(genome.function_summary_filepath) ] report_message = ("Genome Ref:{0}\n" "Number of features sent into prokka:{1}\n" "New functions found:{2}\n" "Ontology terms found:{3}\n").format( genome_ref, stats["current_functions"], stats["new_functions"], stats["new_ontologies"]) report_info = self.kbr.create_extended_report({ "message": report_message, "objects_created": [{ "ref": genome_ref, "description": "Annotated genome" }], "file_links": file_links, "report_object_name": "kb_prokka_report_" + str(uuid.uuid4()), "workspace_name": self.output_workspace }) return { "output_genome_ref": genome_ref, "report_name": report_info["name"], "report_ref": report_info["ref"] } def annotate_genome(self, params): """ User input an existing genome to re-annotate. :param params: Reference to the genome, Output File Name, UI Parameters :return: Report with Reannotated Genome and Stats about it """ self.download_seed_data() self.output_workspace = params["output_workspace"] genome_ref = self._get_input_value(params, "object_ref") output_name = self._get_input_value(params, "output_genome_name") # genome_data = self.dfu.get_objects({"object_refs": [genome_ref]})["data"][0] genome_data = \ self.genome_api.get_genome_v1({"genomes": [{"ref": genome_ref}], 'downgrade': 0})[ "genomes"][0] fasta_for_prokka_filepath = self.write_genome_to_fasta(genome_data) output_dir = self.run_prokka(params, fasta_for_prokka_filepath) prokka_results = self.retrieve_prokka_results(output_dir) new_annotations = self.get_new_annotations(prokka_results.gff_filepath) annotated_genome = self.annotate_genome_with_new_annotations( genome_data=genome_data, new_annotations=new_annotations, output_genome_name=output_name) return self.report_annotated_genome(annotated_genome) def annotate_assembly(self, params, assembly_info): """ Annotate an assembly with Prokka. The steps include to download the assembly as a fasta file, rename the contigs, run prokka against the contigs, parse the results, and finally, create and upload a genome object. :param params: object reference, output_genome_name and output_workspace :param assembly_info: Information used to determine if the assembly is too big :return: Report with newly annotated assembly as a genome, and stats about it """ self.download_seed_data() output_workspace = params["output_workspace"] assembly_ref = self._get_input_value(params, "object_ref") output_genome_name = self._get_input_value(params, "output_genome_name") output_workspace = self._get_input_value(params, "output_workspace") assembly_info = self.inspect_assembly(assembly_info[10], assembly_ref) orig_fasta_file = self.au.get_assembly_as_fasta({"ref": assembly_ref})["path"] # Rename Assembly and Keep Track of Old Contigs renamed_assembly = self.create_renamed_assembly(orig_fasta_file) # Run Prokka with the modified, renamed fasta file output_dir = self.run_prokka(params, renamed_assembly.filepath) # Prokka_results prokka_results = self.retrieve_prokka_results(output_dir) # Parse Results annotated_assembly = self.parse_prokka_results( gff_filepath=prokka_results.gff_filepath, cds_to_dna=prokka_results.cds_to_dna, cds_to_prot=prokka_results.cds_to_prot, new_ids_to_old=renamed_assembly.new_ids_to_old) # Force defaults for optional parameters that may be set to None scientific_name = 'Unknown' if 'scientific_name' in params and params['scientific_name']: scientific_name = params['scientific_name'] domain = "Bacteria" if 'kingdom' in params and params['kingdom']: domain = params['kingdom'] gcode = 0 if 'gcode' in params and params['gcode']: gcode = params['gcode'] genome = { "id": "Unknown", "features": annotated_assembly.features, "scientific_name": scientific_name, "domain": domain, "genetic_code": gcode, "assembly_ref": assembly_ref, "cdss": annotated_assembly.cdss, "mrnas": annotated_assembly.mrnas, "source": "PROKKA annotation pipeline", "gc_content": assembly_info.gc_content, "dna_size": assembly_info.dna_size, "reference_annotation": 0 } info = self.gfu.save_one_genome({ "workspace": output_workspace, "name": output_genome_name, "data": genome, "provenance": self.ctx.provenance() })["info"] genome_ref = str(info[6]) + "/" + str(info[0]) + "/" + str(info[4]) report_message = "Genome saved to: " + output_workspace + "/" + \ output_genome_name + "\n" + annotated_assembly.report_message report_info = self.kbr.create_extended_report({ "message": report_message, "objects_created": [{ "ref": genome_ref, "description": "Annotated genome" }], "report_object_name": "kb_prokka_report_" + str(uuid.uuid4()), "workspace_name": output_workspace }) return { "output_genome_ref": genome_ref, "report_name": report_info["name"], "report_ref": report_info["ref"] }
class ImportSRAUtil: SRA_TOOLKIT_PATH = '/kb/deployment/bin/fastq-dump' def _run_command(self, command): """ _run_command: run command and print result """ log('Start executing command:\n{}'.format(command)) pipe = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) output = pipe.communicate()[0] exitCode = pipe.returncode if (exitCode == 0): log('Executed command:\n{}\n'.format(command) + 'Exit Code: {}\nOutput:\n{}'.format(exitCode, output)) else: error_msg = 'Error running command:\n{}\n'.format(command) error_msg += 'Exit Code: {}\nOutput:\n{}'.format(exitCode, output) raise ValueError(error_msg) def _check_fastq_dump_result(self, tmp_dir, sra_name): """ _check_fastq_dump_result: check fastq_dump result is PE or SE """ return os.path.exists(tmp_dir + '/' + sra_name + '/1') def _sra_to_fastq(self, scratch_sra_file_path, params): """ _sra_to_fastq: convert SRA file to FASTQ file(s) """ tmp_dir = os.path.join(self.scratch, str(uuid.uuid4())) handler_utils._mkdir_p(tmp_dir) command = self.SRA_TOOLKIT_PATH + ' --split-3 -T -O ' command += tmp_dir + ' ' + scratch_sra_file_path self._run_command(command) sra_name = os.path.basename(scratch_sra_file_path).partition('.')[0] paired_end = self._check_fastq_dump_result(tmp_dir, sra_name) if paired_end: self._validate_paired_end_advanced_params(params) fwd_file = os.path.join(tmp_dir, sra_name, '1', 'fastq') os.rename(fwd_file, fwd_file + '.fastq') fwd_file = fwd_file + '.fastq' rev_file = os.path.join(tmp_dir, sra_name, '2', 'fastq') os.rename(rev_file, rev_file + '.fastq') rev_file = rev_file + '.fastq' else: self._validate_single_end_advanced_params(params) fwd_file = os.path.join(tmp_dir, sra_name, 'fastq') os.rename(fwd_file, fwd_file + '.fastq') fwd_file = fwd_file + '.fastq' rev_file = None fastq_file_path = { 'fwd_file': fwd_file, 'rev_file': rev_file } return fastq_file_path def _validate_single_end_advanced_params(self, params): """ _validate_single_end_advanced_params: validate advanced params for single end reads """ if (params.get('insert_size_mean') or params.get('insert_size_std_dev') or params.get('read_orientation_outward')): error_msg = 'Advanced params "Mean Insert Size", "St. Dev. of Insert Size" or ' error_msg += '"Reads Orientation Outward" is Paried End Reads specific' raise ValueError(error_msg) if 'interleaved' in params: del params['interleaved'] def _validate_paired_end_advanced_params(self, params): """ _validate_paired_end_advanced_params: validate advanced params for paired end reads """ sequencing_tech = params.get('sequencing_tech') if sequencing_tech in ['PacBio CCS', 'PacBio CLR']: error_msg = 'Sequencing Technology: "PacBio CCS" or "PacBio CLR" ' error_msg += 'is Single End Reads specific' raise ValueError(error_msg) def _validate_upload_staging_file_availability(self, staging_file_subdir_path): """ _validate_upload_file_path_availability: validates file availability in user's staging area """ pass # TODO ftp_server needs to be fixed for subdir # list = ftp_service(self.callback_url).list_files() # if staging_file_subdir_path not in list: # error_msg = 'Target file: {} is NOT available.\n'.format( # staging_file_subdir_path.rpartition('/')[-1]) # error_msg += 'Available files:\n {}'.format("\n".join(list)) # raise ValueError(error_msg) def __init__(self, config): self.callback_url = config['SDK_CALLBACK_URL'] self.token = config['KB_AUTH_TOKEN'] self.scratch = os.path.join(config['scratch'], 'import_SRA_' + str(uuid.uuid4())) handler_utils._mkdir_p(self.scratch) self.dfu = DataFileUtil(self.callback_url) self.ru = ReadsUtils(self.callback_url) self.uploader_utils = UploaderUtil(config) def import_sra_from_staging(self, params): ''' import_sra_from_staging: wrapper method for GenomeFileUtil.genbank_to_genome required params: staging_file_subdir_path: subdirectory file path e.g. for file: /data/bulk/user_name/file_name staging_file_subdir_path is file_name for file: /data/bulk/user_name/subdir_1/subdir_2/file_name staging_file_subdir_path is subdir_1/subdir_2/file_name sequencing_tech: sequencing technology name: output reads file name workspace_name: workspace name/ID of the object Optional Params: single_genome: whether the reads are from a single genome or a metagenome. insert_size_mean: mean (average) insert length insert_size_std_dev: standard deviation of insert lengths read_orientation_outward: whether reads in a pair point outward return: obj_ref: return object reference ''' log('--->\nrunning ImportSRAUtil.import_sra_from_staging\n' + 'params:\n{}'.format(json.dumps(params, indent=1))) self.validate_import_sra_from_staging_params(params) download_staging_file_params = { 'staging_file_subdir_path': params.get('staging_file_subdir_path') } scratch_sra_file_path = self.dfu.download_staging_file( download_staging_file_params).get('copy_file_path') log('Downloaded staging file to: {}'.format(scratch_sra_file_path)) fastq_file_path = self._sra_to_fastq(scratch_sra_file_path, params) import_sra_reads_params = params import_sra_reads_params.update(fastq_file_path) workspace_name_or_id = params.get('workspace_name') if str(workspace_name_or_id).isdigit(): import_sra_reads_params['wsid'] = int(workspace_name_or_id) else: import_sra_reads_params['wsname'] = str(workspace_name_or_id) log('--->\nrunning ReadsUtils.upload_reads\nparams:\n{}'.format( json.dumps(import_sra_reads_params, indent=1))) returnVal = self.ru.upload_reads(import_sra_reads_params) """ Update the workspace object related meta-data for staged file """ self.uploader_utils.update_staging_service(params.get('staging_file_subdir_path'), returnVal['obj_ref']) return returnVal def import_sra_from_web(self, params): ''' import_sra_from_web: wrapper method for GenomeFileUtil.genbank_to_genome required params: download_type: download type for web source fastq file ('Direct Download', 'FTP', 'DropBox', 'Google Drive') workspace_name: workspace name/ID of the object sra_urls_to_add: dict of SRA file URLs required params: file_url: SRA file URL sequencing_tech: sequencing technology name: output reads file name Optional Params: single_genome: whether the reads are from a single genome or a metagenome. insert_size_mean: mean (average) insert length insert_size_std_dev: standard deviation of insert lengths read_orientation_outward: whether reads in a pair point outward return: obj_ref: return object reference ''' log('--->\nrunning ImportSRAUtil.import_sra_from_web\n' + 'params:\n{}'.format(json.dumps(params, indent=1))) self.validate_import_sra_from_web_params(params) download_type = params.get('download_type') workspace_name = params.get('workspace_name') obj_refs = [] uploaded_files = [] for sra_url_to_add in params.get('sra_urls_to_add'): download_web_file_params = { 'download_type': download_type, 'file_url': sra_url_to_add.get('file_url') } scratch_sra_file_path = self.dfu.download_web_file( download_web_file_params).get('copy_file_path') log('Downloaded web file to: {}'.format(scratch_sra_file_path)) fastq_file_path = self._sra_to_fastq(scratch_sra_file_path, sra_url_to_add) import_sra_reads_params = sra_url_to_add import_sra_reads_params.update(fastq_file_path) workspace_name_or_id = workspace_name if str(workspace_name_or_id).isdigit(): import_sra_reads_params['wsid'] = int(workspace_name_or_id) else: import_sra_reads_params['wsname'] = str(workspace_name_or_id) log('--->\nrunning ReadsUtils.upload_reads\nparams:\n{}'.format( json.dumps(import_sra_reads_params, indent=1))) obj_ref = self.ru.upload_reads(import_sra_reads_params).get('obj_ref') obj_refs.append(obj_ref) uploaded_files.append(sra_url_to_add.get('file_url')) return {'obj_refs': obj_refs, 'uploaded_files': uploaded_files} def validate_import_sra_from_staging_params(self, params): """ validate_import_genbank_from_staging_params: validates params passed to import_genbank_from_staging method """ # check for required parameters for p in ['staging_file_subdir_path', 'sequencing_tech', 'name', 'workspace_name']: if p not in params: raise ValueError('"' + p + '" parameter is required, but missing') self._validate_upload_staging_file_availability(params.get('staging_file_subdir_path')) def validate_import_sra_from_web_params(self, params): """ validate_import_genbank_from_staging_params: validates params passed to import_genbank_from_staging method """ # check for required parameters for p in ['download_type', 'workspace_name', 'sra_urls_to_add']: if p not in params: raise ValueError('"{}" parameter is required, but missing'.format(p)) if not isinstance(params.get('sra_urls_to_add'), list): raise ValueError('sra_urls_to_add is not type list as required') for sra_url_to_add in params.get('sra_urls_to_add'): for p in ['file_url', 'sequencing_tech', 'name']: if p not in sra_url_to_add: raise ValueError('"{}" parameter is required, but missing'.format(p)) def generate_report(self, obj_refs_list, params): """ generate_report: generate summary report obj_refs: generated workspace object references. (return of import_sra_from_staging/web) params: staging_file_subdir_path: subdirectory file path e.g. for file: /data/bulk/user_name/file_name staging_file_subdir_path is file_name for file: /data/bulk/user_name/subdir_1/subdir_2/file_name staging_file_subdir_path is subdir_1/subdir_2/file_name workspace_name: workspace name/ID that reads will be stored to """ uuid_string = str(uuid.uuid4()) objects_created = list() objects_data = list() for obj_ref in obj_refs_list: get_objects_params = { 'object_refs': [obj_ref], 'ignore_errors': False } objects_data.append(self.dfu.get_objects(get_objects_params)) objects_created.append({'ref': obj_ref, 'description': 'Imported Reads'}) output_html_files = self.generate_html_report(objects_data, params, uuid_string) report_params = { 'message': '', 'workspace_name': params.get('workspace_name'), 'objects_created': objects_created, 'html_links': output_html_files, 'direct_html_link_index': 0, 'html_window_height': 460, 'report_object_name': 'kb_sra_upload_report_' + uuid_string} kbase_report_client = KBaseReport(self.callback_url, token=self.token) output = kbase_report_client.create_extended_report(report_params) report_output = {'report_name': output['name'], 'report_ref': output['ref']} return report_output def generate_html_report(self, reads_objs, params, uuid_string): """ _generate_html_report: generate html summary report """ log('Start generating html report') pprint(params) tmp_dir = os.path.join(self.scratch, uuid_string) handler_utils._mkdir_p(tmp_dir) result_file_path = os.path.join(tmp_dir, 'report.html') html_report = list() objects_content = '' for index, reads_obj in enumerate(reads_objs): idx = str(index) reads_data = reads_obj.get('data')[0].get('data') reads_info = reads_obj.get('data')[0].get('info') reads_ref = str(reads_info[6]) + '/' + str(reads_info[0]) + '/' + str(reads_info[4]) reads_obj_name = str(reads_info[1]) with open(os.path.join(os.path.dirname(__file__), 'report_template_sra/table_panel.html'), 'r') as object_content_file: report_template = object_content_file.read() report_template = report_template.replace('_NUM', str(idx)) report_template = report_template.replace('OBJECT_NAME', reads_obj_name) if index == 0: report_template = report_template.replace('panel-collapse collapse', 'panel-collapse collapse in') objects_content += report_template base_percentages = '' for key, val in reads_data.get('base_percentages').items(): base_percentages += '{}({}%) '.format(key, val) reads_overview_data = collections.OrderedDict() reads_overview_data['Name'] = '{} ({})'.format(reads_obj_name, reads_ref) reads_overview_data['Uploaded File'] = params.get('uploaded_files')[index] reads_overview_data['Date Uploaded'] = time.strftime("%c") reads_overview_data['Number of Reads'] = '{:,}'.format(reads_data.get('read_count')) reads_type = reads_info[2].lower() if 'single' in reads_type: reads_overview_data['Type'] = 'Single End' elif 'paired' in reads_type: reads_overview_data['Type'] = 'Paired End' else: reads_overview_data['Type'] = 'Unknown' reads_overview_data['Platform'] = reads_data.get('sequencing_tech', 'Unknown') reads_single_genome = str(reads_data.get('single_genome', 'Unknown')) if '0' in reads_single_genome: reads_overview_data['Single Genome'] = 'No' elif '1' in reads_single_genome: reads_overview_data['Single Genome'] = 'Yes' else: reads_overview_data['Single Genome'] = 'Unknown' insert_size_mean = params.get('insert_size_mean', 'Not Specified') if insert_size_mean is not None: reads_overview_data['Insert Size Mean'] = str(insert_size_mean) else: reads_overview_data['Insert Size Mean'] = 'Not Specified' insert_size_std_dev = params.get('insert_size_std_dev', 'Not Specified') if insert_size_std_dev is not None: reads_overview_data['Insert Size Std Dev'] = str(insert_size_std_dev) else: reads_overview_data['Insert Size Std Dev'] = 'Not Specified' reads_outward_orientation = str(reads_data.get('read_orientation_outward', 'Unknown')) if '0' in reads_outward_orientation: reads_overview_data['Outward Read Orientation'] = 'No' elif '1' in reads_outward_orientation: reads_overview_data['Outward Read Orientation'] = 'Yes' else: reads_overview_data['Outward Read Orientation'] = 'Unknown' reads_stats_data = collections.OrderedDict() reads_stats_data['Number of Reads'] = '{:,}'.format(reads_data.get('read_count')) reads_stats_data['Total Number of Bases'] = '{:,}'.format(reads_data.get('total_bases')) reads_stats_data['Mean Read Length'] = str(reads_data.get('read_length_mean')) reads_stats_data['Read Length Std Dev'] = str(reads_data.get('read_length_stdev')) dup_reads_percent = '{:.2f}'.format(float(reads_data.get('number_of_duplicates') * 100) / \ reads_data.get('read_count')) reads_stats_data['Number of Duplicate Reads(%)'] = '{} ({}%)' \ .format(str(reads_data.get('number_of_duplicates')), dup_reads_percent) reads_stats_data['Phred Type'] = str(reads_data.get('phred_type')) reads_stats_data['Quality Score Mean'] = '{0:.2f}'.format(reads_data.get('qual_mean')) reads_stats_data['Quality Score (Min/Max)'] = '{}/{}'.format(str(reads_data.get('qual_min')), str(reads_data.get('qual_max'))) reads_stats_data['GC Percentage'] = str(round(reads_data.get('gc_content') * 100, 2)) + '%' reads_stats_data['Base Percentages'] = base_percentages overview_content = '' for key, val in reads_overview_data.items(): overview_content += '<tr><td><b>{}</b></td>'.format(key) overview_content += '<td>{}</td>'.format(val) overview_content += '</tr>' stats_content = '' for key, val in reads_stats_data.items(): stats_content += '<tr><td><b>{}</b></td>'.format(key) stats_content += '<td>{}</td>'.format(val) stats_content += '</tr>' objects_content = objects_content.replace('###OVERVIEW_CONTENT###', overview_content) objects_content = objects_content.replace('###STATS_CONTENT###', stats_content) with open(result_file_path, 'w') as result_file: with open(os.path.join(os.path.dirname(__file__), 'report_template_sra/report_head.html'), 'r') as report_template_file: report_template = report_template_file.read() report_template = report_template.replace('###TABLE_PANELS_CONTENT###', objects_content) result_file.write(report_template) result_file.close() shutil.copytree(os.path.join(os.path.dirname(__file__), 'report_template_sra/bootstrap-3.3.7'), os.path.join(tmp_dir, 'bootstrap-3.3.7')) shutil.copy(os.path.join(os.path.dirname(__file__), 'report_template_sra/jquery-3.2.1.min.js'), os.path.join(tmp_dir, 'jquery-3.2.1.min.js')) matched_files = [] for root, dirnames, filenames in os.walk(tmp_dir): for filename in fnmatch.filter(filenames, '*.gz'): matched_files.append(os.path.join(root, filename)) for gz_file in matched_files: print(('Removing ' + gz_file)) os.remove(gz_file) report_shock_id = self.dfu.file_to_shock({'file_path': tmp_dir, 'pack': 'zip'})['shock_id'] html_report.append({'shock_id': report_shock_id, 'name': os.path.basename(result_file_path), 'label': os.path.basename(result_file_path), 'description': 'HTML summary report for Imported Assembly'}) return html_report
class ImportAssemblyUtil: def __init__(self, config): self.callback_url = config['SDK_CALLBACK_URL'] self.scratch = os.path.join(config['scratch'], 'import_assembly_' + str(uuid.uuid4())) handler_utils._mkdir_p(self.scratch) self.token = config['KB_AUTH_TOKEN'] self.dfu = DataFileUtil(self.callback_url) self.au = AssemblyUtil(self.callback_url) self.uploader_utils = UploaderUtil(config) self.max_contigs_for_report = 200 def import_fasta_as_assembly_from_staging(self, params): """ import_fasta_as_assembly_from_staging: wrapper method for AssemblyUtil.save_assembly_from_fasta required params: staging_file_subdir_path - subdirectory file path e.g. for file: /data/bulk/user_name/file_name staging_file_subdir_path is file_name for file: /data/bulk/user_name/subdir_1/subdir_2/file_name staging_file_subdir_path is subdir_1/subdir_2/file_name assembly_name - output Assembly file name workspace_name - the name of the workspace it gets saved to. return: obj_ref: return object reference """ logging.info( '--->\nrunning ImportAssemblyUtil.import_fasta_as_assembly_from_staging\n' f'params:\n{json.dumps(params, indent=1)}') self.validate_import_fasta_as_assembly_from_staging(params) download_staging_file_params = { 'staging_file_subdir_path': params.get('staging_file_subdir_path') } scratch_file_path = self.dfu.download_staging_file( download_staging_file_params).get('copy_file_path') file = {'path': scratch_file_path} import_assembly_params = params import_assembly_params['file'] = file ref = self.au.save_assembly_from_fasta(import_assembly_params) """ Update the workspace object related meta-data for staged file """ # self.uploader_utils.update_staging_service(params.get('staging_file_subdir_path'), ref) returnVal = {'obj_ref': ref} return returnVal def validate_import_fasta_as_assembly_from_staging(self, params): """ validate_import_fasta_as_assembly_from_staging: validates params passed to import_fasta_as_assembly_from_staging method """ # check for required parameters for p in [ 'staging_file_subdir_path', 'workspace_name', 'assembly_name' ]: if p not in params: raise ValueError(f'"{p}" parameter is required, but missing') def generate_html_report(self, assembly_ref, assembly_object, params): """ _generate_html_report: generate html summary report """ logging.info('start generating html report') html_report = list() assembly_data = assembly_object.get('data')[0].get('data') assembly_info = assembly_object.get('data')[0].get('info') tmp_dir = os.path.join(self.scratch, str(uuid.uuid4())) handler_utils._mkdir_p(tmp_dir) result_file_path = os.path.join(tmp_dir, 'report.html') assembly_name = str(assembly_info[1]) assembly_file = params.get('staging_file_subdir_path') dna_size = assembly_data.get('dna_size') num_contigs = assembly_data.get('num_contigs') assembly_overview_data = collections.OrderedDict() assembly_overview_data['Name'] = '{} ({})'.format( assembly_name, assembly_ref) assembly_overview_data['Uploaded File'] = assembly_file assembly_overview_data['Date Uploaded'] = time.strftime("%c") assembly_overview_data['DNA Size'] = dna_size assembly_overview_data['Number of Contigs'] = num_contigs overview_content = ['<br/><table>\n'] for key, val in assembly_overview_data.items(): overview_content.append(f'<tr><td><b>{key}</b></td>') overview_content.append(f'<td>{val}</td></tr>\n') overview_content.append('</table>') contig_data = assembly_data.get('contigs').values() contig_content = str([str(e['contig_id']), e['length']] for e in contig_data) with open(result_file_path, 'w') as result_file: with open( os.path.join(os.path.dirname(__file__), 'report_template', 'report_template_assembly.html'), 'r') as report_template_file: report_template = report_template_file.read() report_template = report_template.replace( '<p>*Overview_Content*</p>', ''.join(overview_content)) report_template = report_template.replace( '*CONTIG_DATA*', contig_content) result_file.write(report_template) result_file.close() report_shock_id = self.dfu.file_to_shock({ 'file_path': tmp_dir, 'pack': 'zip' })['shock_id'] html_report.append({ 'shock_id': report_shock_id, 'name': os.path.basename(result_file_path), 'label': os.path.basename(result_file_path), 'description': 'HTML summary report for Imported Assembly' }) return html_report def generate_report(self, obj_ref, params): """ generate_report: generate summary report obj_ref: generated workspace object references. (return of import_fasta_as_assembly_from_staging) params: staging_file_subdir_path: subdirectory file path e.g. for file: /data/bulk/user_name/file_name staging_file_subdir_path is file_name for file: /data/bulk/user_name/subdir_1/subdir_2/file_name staging_file_subdir_path is subdir_1/subdir_2/file_name workspace_name: workspace name/ID that reads will be stored to """ object_data = self.dfu.get_objects({'object_refs': [obj_ref]}) report_params = { 'workspace_name': params.get('workspace_name'), 'objects_created': [{ 'ref': obj_ref, 'description': 'Imported Assembly' }], 'report_object_name': f'kb_upload_assembly_report_{uuid.uuid4()}' } num_contigs = object_data['data'][0]['data']['num_contigs'] if num_contigs > self.max_contigs_for_report: report_params['message'] = ( "The uploaded assembly has too many contigs to display " "here. Click on the object for a dedicated viewer") else: output_html_files = self.generate_html_report( obj_ref, object_data, params) report_params.update({ 'html_links': output_html_files, 'direct_html_link_index': 0, 'html_window_height': 375, }) kbase_report_client = KBaseReport(self.callback_url, token=self.token) output = kbase_report_client.create_extended_report(report_params) report_output = { 'report_name': output['name'], 'report_ref': output['ref'] } return report_output
def create_fake_reads(self, ctx, params): """ :param params: instance of type "CreateFakeReadsParams" (ws_id/ws_name - two alternative ways to set target workspace, obj_names - list of names for target workspace objects (of type 'KBaseFile.SingleEndLibrary'), metadata - optional metadata.) -> structure: parameter "ws_id" of Long, parameter "ws_name" of String, parameter "obj_names" of list of String, parameter "metadata" of mapping from String to String :returns: instance of list of type "object_info" (Information about an object, including user provided metadata. obj_id objid - the numerical id of the object. obj_name name - the name of the object. type_string type - the type of the object. timestamp save_date - the save date of the object. obj_ver ver - the version of the object. username saved_by - the user that saved or copied the object. ws_id wsid - the workspace containing the object. ws_name workspace - the workspace containing the object. string chsum - the md5 checksum of the object. int size - the size of the object in bytes. usermeta meta - arbitrary user-supplied metadata about the object.) -> tuple of size 11: parameter "objid" of type "obj_id" (The unique, permanent numerical ID of an object.), parameter "name" of type "obj_name" (A string used as a name for an object. Any string consisting of alphanumeric characters and the characters |._- that is not an integer is acceptable.), parameter "type" of type "type_string" (A type string. Specifies the type and its version in a single string in the format [module].[typename]-[major].[minor]: module - a string. The module name of the typespec containing the type. typename - a string. The name of the type as assigned by the typedef statement. major - an integer. The major version of the type. A change in the major version implies the type has changed in a non-backwards compatible way. minor - an integer. The minor version of the type. A change in the minor version implies that the type has changed in a way that is backwards compatible with previous type definitions. In many cases, the major and minor versions are optional, and if not provided the most recent version will be used. Example: MyModule.MyType-3.1), parameter "save_date" of type "timestamp" (A time in the format YYYY-MM-DDThh:mm:ssZ, where Z is either the character Z (representing the UTC timezone) or the difference in time to UTC in the format +/-HHMM, eg: 2012-12-17T23:24:06-0500 (EST time) 2013-04-03T08:56:32+0000 (UTC time) 2013-04-03T08:56:32Z (UTC time)), parameter "version" of Long, parameter "saved_by" of type "username" (Login name of a KBase user account.), parameter "wsid" of type "ws_id" (The unique, permanent numerical ID of a workspace.), parameter "workspace" of type "ws_name" (A string used as a name for a workspace. Any string consisting of alphanumeric characters and "_", ".", or "-" that is not an integer is acceptable. The name may optionally be prefixed with the workspace owner's user name and a colon, e.g. kbasetest:my_workspace.), parameter "chsum" of String, parameter "size" of Long, parameter "meta" of type "usermeta" (User provided metadata about an object. Arbitrary key-value pairs provided by the user.) -> mapping from String to String """ # ctx is the context object # return variables are: returnVal #BEGIN create_fake_reads metadata = params.get('metadata') objects = [] dfu = DataFileUtil(os.environ['SDK_CALLBACK_URL']) path_to_temp_file = "/kb/module/work/tmp/temp_" + str( time.time()) + ".fq" with open(path_to_temp_file, 'w') as f: f.write(' ') uploadedfile = dfu.file_to_shock({ 'file_path': path_to_temp_file, 'make_handle': 1, 'pack': 'gzip' }) fhandle = uploadedfile['handle'] os.remove(path_to_temp_file) data = { 'lib': { 'encoding': "ascii", 'file': fhandle, 'size': 1, 'type': "fq" }, 'sequencing_tech': "Illumina", 'single_genome': 1 } for obj_name in params['obj_names']: objects.append({ 'type': 'KBaseFile.SingleEndLibrary', 'data': data, 'name': obj_name, 'meta': metadata }) so_params = {'objects': objects} if 'ws_id' in params: so_params['id'] = params['ws_id'] elif 'ws_name' in params: so_params['workspace'] = params['ws_name'] returnVal = self.ws(ctx).save_objects(so_params) #END create_fake_reads # At some point might do deeper type checking... if not isinstance(returnVal, list): raise ValueError('Method create_fake_reads return value ' + 'returnVal is not type list as required.') # return the results return [returnVal]