def parse_reference_output(self): """Reads and processes DIAMOND tabular output of the first DIAMOND search. Note: this function finds query sequences similar to reference proteins. Since a query sequence may have more than one areas of similarity (for instance, in fusion proteins of two subunits or in multi-domain proteins), it will try to find as many such areas as possible. DIAMOND hits are filtered by two parameters: length of alignment and amino acid identity %. This function does not return anything. Instead, it populates 'reads' dictionary with AnnotatedRead objects. """ tsvfile = os.path.join( self.options.get_project_dir(self.sample.sample_id), self.sample.sample_id + '_' + self.end + '_' + self.options.ref_output_name ) current_sequence_read_id = '' hit_list = DiamondHitList(current_sequence_read_id) # TODO: cleanup identity_cutoff = self.config.get_identity_cutoff(self.collection) length_cutoff = self.config.get_length_cutoff(self.collection) print('Length cutoff:', length_cutoff) with open(tsvfile, 'r', newline='') as infile: tsvin = csv.reader(infile, delimiter='\t') for row in tsvin: hit = DiamondHit() (row[0], _) = parse_fastq_seqid(row[0]) hit.create_hit(row) # filtering by length if hit.length < length_cutoff: continue # go to next hit if hit.query_id != current_sequence_read_id: # when new query ID reached, process collected hits, # then start over with new query identifier # filtering: remove overlapping hits hit_list.filter_list(self.config.get_overlap_cutoff(self.collection)) # if any hits left, assign function to hits and populate reads dictionary hit_list.annotate_hits(self.ref_data) hit_list.filter_list_by_identity(self.ref_data) if hit_list.hits_number != 0: read = AnnotatedRead(current_sequence_read_id) read.hit_list = hit_list self.reads[current_sequence_read_id] = read # start over current_sequence_read_id = hit.query_id hit_list = DiamondHitList(current_sequence_read_id) hit_list.add_hit(hit) # when EOF reached, process collected hits hit_list.filter_list(self.config.get_overlap_cutoff(self.collection)) hit_list.annotate_hits(self.ref_data) hit_list.filter_list_by_identity(self.ref_data) if hit_list.hits_number != 0: read = AnnotatedRead(current_sequence_read_id) read.hit_list = hit_list self.reads[current_sequence_read_id] = read
def test_3_protein_taxonomy(self): self.project.import_reads_json(sample, ENDS) protein = 'D16-4706_contig_11213_7' print('D16-4706_contig_11213_7 taxonomy') print(self.project.samples[sample].reads[end][protein].taxonomy) parser = DiamondParser(config=self.project.config, options=self.project.options, taxonomy_data=self.project.taxonomy_data, ref_data=self.project.ref_data, sample=self.project.samples[sample], end=end) parser.parse_reference_output() print(str(parser.reads[protein])) # parse_background_output(parser) hit_line = 'D16-4706_contig_11213_7|4|257 fig|408672.3.peg.2637 63.0 254 94 256 1 254 2 255 1.1e-97 362.1' hit = DiamondHit() hit.create_hit(tabular_output_fields=hit_line.split('\t')) hit_list = DiamondHitList('D16-4706_contig_11213_7|4|257') hit_list.add_hit(hit) hit_list.annotate_hits(self.project.ref_data) hit_list.filter_list_by_identity(self.project.ref_data) print('hit_list') print(hit_list) compare_protein_hits_lca(parser.reads[protein], 4, 257, hit_list, 0.03, 1.0, 1.0, self.project.taxonomy_data, self.project.ref_data) print(parser.reads[protein].taxonomy) self.assertEqual(parser.reads[protein].taxonomy, '408672')
def parse_background_output(self): """Reads and processes DIAMOND tabular output of the classification DIAMOND search. Note: this function takes existing list of hits and compares each of them with results of new similarity serach (against classification DB). For the comparison, it calls compare_hits_lca function. """ tsvfile = os.path.join(self.assembly_dir, 'all_contigs_' + self.project.options.background_output_name) current_query_id = None hit_list = None length_cutoff = self.project.config.get_length_cutoff( self.project.options.get_collection()) biscore_range_cutoff = self.project.config.get_biscore_range_cutoff( self.project.options.get_collection()) print('Relative bit-score cutoff: ', biscore_range_cutoff, ', Length cutoff: ', length_cutoff) average_coverage = self.assembly.calculate_average_coverage() with open(tsvfile, 'r', newline='') as infile: tsvin = csv.reader(infile, delimiter='\t') function_id = '' contig_id = '' gene_id = '' coverage = '' for row in tsvin: if current_query_id is None: current_query_id = row[0] hit_list = DiamondHitList(current_query_id) hit = DiamondHit() hit.create_hit(row) # filtering by identity and length if hit.length < length_cutoff: continue # skip this hit if hit.query_id != current_query_id: hit_list.annotate_hits(self.project.ref_data) hit_list.filter_list_by_identity(self.project.ref_data) # compare list of hits from search in background DB with existing # hit from search in reference DB current_query_id_tokens = current_query_id.split('|') function_id = current_query_id_tokens[0] contig_id = '_'.join(current_query_id_tokens[1].split('_')[:-1]) gene_id = '|'.join(current_query_id_tokens[:-2]) coverage = self.assembly.contigs[function_id][contig_id].get_coverage() try: compare_protein_hits_lca( self.assembly.contigs[function_id][contig_id].genes[gene_id], int(current_query_id_tokens[-2]), # hit_start int(current_query_id_tokens[-1]), # hit_end hit_list, biscore_range_cutoff, coverage, average_coverage, self.project.taxonomy_data, self.project.ref_data ) except KeyError: print(' '.join(['Gene not found:', gene_id, 'in', function_id, contig_id])) current_query_id = hit.query_id hit_list = DiamondHitList(current_query_id) hit_list.add_hit(hit) hit_list.annotate_hits(self.project.ref_data) hit_list.filter_list_by_identity(self.project.ref_data) current_query_id_tokens = current_query_id.split('|') function_id = current_query_id_tokens[0] contig_id = '_'.join(current_query_id_tokens[1].split('_')[:-1]) gene_id = '|'.join(current_query_id_tokens[:-2]) coverage = self.assembly.contigs[function_id][contig_id].get_coverage() try: compare_protein_hits_lca( self.assembly.contigs[function_id][contig_id].genes[gene_id], int(current_query_id_tokens[-2]), # hit_start int(current_query_id_tokens[-1]), # hit_end hit_list, biscore_range_cutoff, coverage, average_coverage, self.project.taxonomy_data, self.project.ref_data ) except KeyError: print(' '.join(['Gene not found:', gene_id, 'in', function_id, contig_id]))
def parse_background_output(self): """Reads and processes DIAMOND tabular output of the second DIAMOND search. Note: this function takes existing list of hits and compares each of them with results of other similarity serach (against larger DB). For the comparison, it calls compare_hits_erpk_lca function, which in turn updates entries in the 'reads' dictionary. Raises: KeyError if read identifier not found in the 'reads' dictionary """ if not self.reads: # Something went wrong and 'reads' dictionary is empty. # Let's try to import list of reads from file. self.reads = self.import_hit_list() tsvfile = os.path.join( self.sample.work_directory, self.sample.sample_id + '_' + self.end + '_' + self.options.background_output_name ) average_read_length = self.sample.get_avg_read_length(self.end) current_query_id = None hit_list = None length_cutoff = self.config.get_length_cutoff(self.collection) bitscore_range_cutoff = self.config.get_biscore_range_cutoff(self.collection) print('Relative bit-score cutoff:', bitscore_range_cutoff, ', Length cutoff:', length_cutoff ) with open(tsvfile, 'r', newline='') as infile: tsvin = csv.reader(infile, delimiter='\t') for row in tsvin: if current_query_id is None: current_query_id = row[0] hit_list = DiamondHitList(current_query_id) hit = DiamondHit() hit.create_hit(row) # filtering by length if hit.length < length_cutoff: continue # skip this line # when new query ID reached, process collected hits, # then start over with new query identifier if hit.query_id != current_query_id: # assign functions to selected hits hit_list.annotate_hits(self.ref_data) hit_list.filter_list_by_identity(self.ref_data) # extract initial read identifier from identifier of the hit current_query_id_tokens = current_query_id.split('|') read_id = '|'.join(current_query_id_tokens[:-2]) # compare list of hits from search in background DB # with existing hit from the first similarity search try: compare_hits_erpk_lca( self.reads[read_id], int(current_query_id_tokens[-2]), # hit_start int(current_query_id_tokens[-1]), # hit_end hit_list, bitscore_range_cutoff, length_cutoff, average_read_length, self.taxonomy_data, self.ref_data ) except KeyError: print('Read not found: ', read_id) # starting over current_query_id = hit.query_id hit_list = DiamondHitList(current_query_id) hit_list.add_hit(hit) # when EOF reached, process collected hits # assign functions to selected hits hit_list.annotate_hits(self.ref_data) hit_list.filter_list_by_identity(self.ref_data) # extract initial read identifier from identifier of the hit current_query_id_tokens = current_query_id.split('|') read_id = '|'.join(current_query_id_tokens[:-2]) # compare list of hits from search in background DB with # existing hit from the first similarity search try: compare_hits_erpk_lca( self.reads[read_id], int(current_query_id_tokens[-2]), # hit_start int(current_query_id_tokens[-1]), # hit_end hit_list, bitscore_range_cutoff, length_cutoff, average_read_length, self.taxonomy_data, self.ref_data ) except KeyError: print('Read not found: ', read_id)
def parse_background_output(parser): """Reads and processes DIAMOND tabular output of the second DIAMOND search. Args: parser (:obj:DiamondParser): parser object Note: this function takes existing list of hits and compares each of them with results of other similarity serach (against larger DB). For the comparison, it calls compare_hits_lca function, which in turn updates entries in the 'reads' dictionary. Raises: KeyError if read identifier not found in the 'reads' dictionary """ tsvfile = os.path.join( parser.sample.work_directory, parser.sample.sample_id + '_' + parser.end + '_' + parser.options.background_output_name) coverage_data = load_coverage_data(parser) total_coverage = 0.0 if coverage_data: for contig_id in coverage_data.keys(): total_coverage += coverage_data[contig_id] average_coverage = total_coverage / len(coverage_data) else: average_coverage = 1.0 current_query_id = None _hit_list = None length_cutoff = parser.config.get_length_cutoff(parser.collection) biscore_range_cutoff = parser.config.get_biscore_range_cutoff( parser.collection) print('Relative bitscore cutoff: ', biscore_range_cutoff, ', Length cutoff: ', length_cutoff) with open(tsvfile, 'r', newline='') as infile: tsvin = csv.reader(infile, delimiter='\t') for row in tsvin: if current_query_id is None: current_query_id = row[0] _hit_list = DiamondHitList(current_query_id) hit = DiamondHit() hit.create_hit(row) # filtering by length if hit.length < length_cutoff: continue if hit.query_id != current_query_id: _hit_list.annotate_hits(parser.ref_data) _hit_list.filter_list_by_identity(parser.ref_data) current_query_id_tokens = current_query_id.split('|') protein_id = '|'.join(current_query_id_tokens[:-2]) hit_start = int(current_query_id_tokens[-2]) hit_end = int(current_query_id_tokens[-1]) # Coverage data can be used only if protein ID contains contig ID contig_id = '_'.join( protein_id.split(' # ')[0].split('_')[:-1])[1:] coverage = 1.0 if coverage_data is not None and contig_id in coverage_data: coverage = coverage_data[contig_id] try: compare_protein_hits_lca(parser.reads[protein_id], hit_start, hit_end, _hit_list, biscore_range_cutoff, coverage, average_coverage, parser.taxonomy_data, parser.ref_data) except KeyError: print('Protein not found: ', protein_id) current_query_id = hit.query_id _hit_list = DiamondHitList(current_query_id) _hit_list.add_hit(hit) _hit_list.annotate_hits(parser.ref_data) _hit_list.filter_list_by_identity(parser.ref_data) current_query_id_tokens = current_query_id.split('|') protein_id = '|'.join(current_query_id_tokens[:-2]) hit_start = int(current_query_id_tokens[-2]) hit_end = int(current_query_id_tokens[-1]) try: # Coverage data can be used only if protein ID contains contig ID contig_id = '_'.join( protein_id.split(' # ')[0].split('_')[:-1])[1:] coverage = 1.0 if coverage_data is not None and contig_id in coverage_data: coverage = coverage_data[contig_id] compare_protein_hits_lca(parser.reads[protein_id], hit_start, hit_end, _hit_list, biscore_range_cutoff, coverage, average_coverage, parser.taxonomy_data, parser.ref_data) except KeyError: print('Protein not found: ', protein_id)