def process_gene_call(self,
bam_file_object,
gene_call,
contig_sequence,
codons_to_profile=None):
if gene_call['partial']:
return None
contig_name = gene_call['contig']
# here we will create a dictionary to translate codons in a gene to nucleotide positions in the context
# of the contig. thanks to this dict, we will be able to profile only a small number of codons from a
# gene, if they are specified in `codons_to_profile` variable. for instance, this function is called
# during profiling only with codon positions in genes that possess nucleotide variation.
codon_order_to_nt_positions = utils.get_codon_order_to_nt_positions_dict(
gene_call)
# here we generate the actual 'linkmers' information.
d = {}
linkmers = LinkMersData()
linkmers.quiet = True
for codon_order in codon_order_to_nt_positions:
if codons_to_profile and codon_order not in codons_to_profile:
continue
nt_positions = codon_order_to_nt_positions[codon_order]
reference_codon_sequence = contig_sequence[
nt_positions[0]:nt_positions[2] + 1]
# if concensus sequence contains shitty characters, we will not continue
if reference_codon_sequence not in codon_to_AA:
continue
linkmers.data = []
linkmers.append(bam_file_object,
'sample_id',
None,
contig_name,
nt_positions,
only_complete_links=True)
data = linkmers.data[0][2]
hash_to_oligotype = {}
unique_hashes = set([datum.read_unique_id for datum in data])
for unique_hash in unique_hashes:
hash_to_oligotype[unique_hash] = []
for datum in data:
hash_to_oligotype[datum.read_unique_id].append(
(datum.pos_in_contig, datum.base), )
for unique_hash in unique_hashes:
hash_to_oligotype[unique_hash] = ''.join(
[e[1] for e in sorted(hash_to_oligotype[unique_hash])])
nt_frequencies = Counter(hash_to_oligotype.values())
aa_frequencies = Counter({})
# if the gene is reverse, we want to use the dict for reverse complementary conversions for DNA to AA
conv_dict = codon_to_AA_RC if gene_call[
'direction'] == 'r' else codon_to_AA
reference_codon_AA = conv_dict[reference_codon_sequence]
for nt in nt_frequencies:
if conv_dict[
nt]: # <-- this check here eliminates any codon that contains anything but [A, T, C, G].
aa_frequencies[conv_dict[nt]] += nt_frequencies[nt]
coverage = sum(aa_frequencies.values())
if not coverage:
# FIXME: there was at least one case where the coverage here in this context was 0,
# which crashed the profiling. we never went after this issue, and it is
# important to understand how often this happens, and why.
continue
# here we quantify the ratio of frequencies of non-reference-aas observed in this codon
# to the overall overage, and that is our `departure_from_reference`:
total_frequency_of_all_codons_but_the_conensus = sum([
aa_frequencies[aa] for aa in aa_frequencies
if aa != reference_codon_AA
])
departure_from_reference = total_frequency_of_all_codons_but_the_conensus / coverage
d[codon_order] = {
'reference': reference_codon_AA,
'coverage': coverage,
'frequencies': aa_frequencies,
'departure_from_reference': departure_from_reference
}
return d
def recover_base_frequencies_for_all_samples(self):
self.progress.new('Recovering AA frequencies for all')
samples_wanted = self.samples_of_interest if self.samples_of_interest else self.sample_ids
splits_wanted = self.splits_of_interest if self.splits_of_interest else set(
self.splits_basic_info.keys())
next_available_entry_id = max(self.data.keys()) + 1
unique_pos_identifier_str_to_consenus_codon = {}
unique_pos_identifier_str_to_unique_pos_identifier = {}
for e in self.data.values():
upi = e['unique_pos_identifier_str']
unique_pos_identifier_str_to_consenus_codon[upi] = e['consensus']
unique_pos_identifier_str_to_unique_pos_identifier[upi] = e[
'unique_pos_identifier']
self.progress.update(
'creating a dict to track missing AA frequencies for each sample / split / pos'
)
splits_to_consider = {}
for split_name in splits_wanted:
splits_to_consider[split_name] = {}
self.progress.update(
'populating the dict to track missing AA frequencies for each sample / split / pos'
)
for entry_id in self.data:
v = self.data[entry_id]
gene_codon_key = '%d_%d' % (v['corresponding_gene_call'],
v['codon_order_in_gene'])
d = splits_to_consider[v['split_name']]
if d.has_key(gene_codon_key):
d[gene_codon_key].remove(v['sample_id'])
else:
d[gene_codon_key] = copy.deepcopy(samples_wanted)
d[gene_codon_key].remove(v['sample_id'])
counter = 0
for split_name in splits_to_consider:
counter += 1
self.progress.update(
'accessing split coverages and updating variable positions dict :: %s'
% pp(counter))
split_coverage_across_samples = self.merged_split_coverage_values.get(
split_name)
split_info = self.splits_basic_info[split_name]
contig_name = split_info['parent']
for gene_codon_key in splits_to_consider[split_name]:
corresponding_gene_call, codon_order_in_gene = [
int(k) for k in gene_codon_key.split('_')
]
for sample_name in splits_to_consider[split_name][
gene_codon_key]:
unique_pos_identifier_str = '_'.join([
split_name,
str(corresponding_gene_call),
str(codon_order_in_gene)
])
consensus_codon = unique_pos_identifier_str_to_consenus_codon[
unique_pos_identifier_str]
self.data[next_available_entry_id] = {
'unique_pos_identifier_str':
unique_pos_identifier_str,
'unique_pos_identifier':
unique_pos_identifier_str_to_unique_pos_identifier[
unique_pos_identifier_str],
'sample_id':
sample_name,
'split_name':
split_name,
'contig_name':
contig_name,
'corresponding_gene_call':
corresponding_gene_call,
'codon_order_in_gene':
codon_order_in_gene,
'departure_from_consensus':
0,
'coverage':
None,
'consensus':
consensus_codon
}
# DEALING WITH COVERAGE ##################################################################
# some very cool but expensive shit is going on here, let me break it down for poor souls of the future.
# what we want to do is to learn the coverage of this codon in the sample. all we have is the corresponding
# gene call id, and the order of this codon in the gene. so here how it goes:
#
# learn the gene call
gene_call = self.genes_in_contigs_dict[
corresponding_gene_call]
# the following dict converts codon orders into nt positions in contig for a geven gene call
codon_order_to_nt_positions_in_contig = utils.get_codon_order_to_nt_positions_dict(
gene_call)
# so the nucleotide positions for this codon in the contig is the following:
nt_positions_for_codon_in_contig = codon_order_to_nt_positions_in_contig[
codon_order_in_gene]
# but we need to convert those positions to the context of this split. so here is the start pos:
split_start = self.splits_basic_info[split_name]['start']
# here we map nt positions from the contig context to split context using the start position
nt_positions_for_codon_in_split = [
p - split_start
for p in nt_positions_for_codon_in_contig
]
# we acquire coverages that match to these positions
coverages = split_coverage_across_samples[sample_name][
nt_positions_for_codon_in_split]
coverage = int(round(sum(coverages) / 3))
# and finally update the data table
self.data[next_available_entry_id]['coverage'] = coverage
# DEALING WITH AAs ##################################################################
# here we need to put all the codons into the data table for this sample
for codon in set(codon_to_AA.values()):
self.data[next_available_entry_id][codon] = 0
# and finally update the frequency of the consensus codon with the coverage (WHICH IS VERY BAD,
# WE HAVE NO CLUE WHAT IS THE ACTUAL COVERAGE OF TRIPLICATE LINKMERS):
self.data[next_available_entry_id][
consensus_codon] = coverage
next_available_entry_id += 1
self.progress.end()
def recover_base_frequencies_for_all_samples(self):
self.progress.new('Recovering AA frequencies for all')
samples_wanted = self.samples_of_interest if self.samples_of_interest else self.sample_ids
splits_wanted = self.splits_of_interest if self.splits_of_interest else set(self.splits_basic_info.keys())
next_available_entry_id = max(self.data.keys()) + 1
unique_pos_identifier_str_to_consenus_codon = {}
unique_pos_identifier_str_to_unique_pos_identifier = {}
for e in self.data.values():
upi = e['unique_pos_identifier_str']
unique_pos_identifier_str_to_consenus_codon[upi] = e['reference']
unique_pos_identifier_str_to_unique_pos_identifier[upi] = e['unique_pos_identifier']
self.progress.update('creating a dict to track missing AA frequencies for each sample / split / pos')
splits_to_consider = {}
for split_name in splits_wanted:
splits_to_consider[split_name] = {}
self.progress.update('populating the dict to track missing AA frequencies for each sample / split / pos')
for entry_id in self.data:
v = self.data[entry_id]
gene_codon_key = '%d_%d' % (v['corresponding_gene_call'], v['codon_order_in_gene'])
d = splits_to_consider[v['split_name']]
if gene_codon_key in d:
d[gene_codon_key].remove(v['sample_id'])
else:
d[gene_codon_key] = copy.deepcopy(samples_wanted)
d[gene_codon_key].remove(v['sample_id'])
counter = 0
for split_name in splits_to_consider:
counter += 1
self.progress.update('accessing split coverages and updating variable positions dict :: %s' % pp(counter))
split_coverage_across_samples = self.merged_split_coverage_values.get(split_name)
split_info = self.splits_basic_info[split_name]
contig_name = split_info['parent']
for gene_codon_key in splits_to_consider[split_name]:
corresponding_gene_call, codon_order_in_gene = [int(k) for k in gene_codon_key.split('_')]
for sample_name in splits_to_consider[split_name][gene_codon_key]:
unique_pos_identifier_str = '_'.join([split_name, str(corresponding_gene_call), str(codon_order_in_gene)])
reference_codon = unique_pos_identifier_str_to_consenus_codon[unique_pos_identifier_str]
self.data[next_available_entry_id] = {'unique_pos_identifier_str': unique_pos_identifier_str,
'unique_pos_identifier': unique_pos_identifier_str_to_unique_pos_identifier[unique_pos_identifier_str],
'sample_id': sample_name,
'split_name': split_name,
'contig_name': contig_name,
'corresponding_gene_call': corresponding_gene_call,
'codon_order_in_gene': codon_order_in_gene,
'departure_from_reference': 0,
'coverage': None,
'reference': reference_codon}
# DEALING WITH COVERAGE ##################################################################
# some very cool but expensive shit is going on here, let me break it down for poor souls of the future.
# what we want to do is to learn the coverage of this codon in the sample. all we have is the corresponding
# gene call id, and the order of this codon in the gene. so here how it goes:
#
# learn the gene call
gene_call = self.genes_in_contigs_dict[corresponding_gene_call]
# the following dict converts codon orders into nt positions in contig for a geven gene call
codon_order_to_nt_positions_in_contig = utils.get_codon_order_to_nt_positions_dict(gene_call)
# so the nucleotide positions for this codon in the contig is the following:
nt_positions_for_codon_in_contig = codon_order_to_nt_positions_in_contig[codon_order_in_gene]
# but we need to convert those positions to the context of this split. so here is the start pos:
split_start = self.splits_basic_info[split_name]['start']
# here we map nt positions from the contig context to split context using the start position
nt_positions_for_codon_in_split = [p - split_start for p in nt_positions_for_codon_in_contig]
# we acquire coverages that match to these positions
coverages = split_coverage_across_samples[sample_name][nt_positions_for_codon_in_split]
coverage = int(round(sum(coverages) / 3))
# and finally update the data table
self.data[next_available_entry_id]['coverage'] = coverage
# DEALING WITH AAs ##################################################################
# here we need to put all the codons into the data table for this sample
for codon in set(codon_to_AA.values()):
self.data[next_available_entry_id][codon] = 0
# and finally update the frequency of the reference codon with the coverage (WHICH IS VERY BAD,
# WE HAVE NO CLUE WHAT IS THE ACTUAL COVERAGE OF TRIPLICATE LINKMERS):
self.data[next_available_entry_id][reference_codon] = coverage
# insert additional fields for this newly added data point
self.insert_additional_fields([next_available_entry_id])
next_available_entry_id += 1
self.progress.end()
def process_gene_call(self, bam_file_object, gene_call, contig_sequence, codons_to_profile=None, return_AA_frequencies_instead=False):
if gene_call['partial']:
return None
contig_name = gene_call['contig']
# here we will create a dictionary to translate codons in a gene to nucleotide positions in the context
# of the contig. thanks to this dict, we will be able to profile only a small number of codons from a
# gene, if they are specified in `codons_to_profile` variable. for instance, this function is called
# during profiling only with codon positions in genes that possess nucleotide variation.
codon_order_to_nt_positions = utils.get_codon_order_to_nt_positions_dict(gene_call)
# here we generate the actual 'linkmers' information.
d = {}
linkmers = LinkMersData()
linkmers.quiet = True
for codon_order in codon_order_to_nt_positions:
if codons_to_profile and codon_order not in codons_to_profile:
continue
nt_positions = codon_order_to_nt_positions[codon_order]
reference_codon_sequence = contig_sequence[nt_positions[0]:nt_positions[2] + 1]
# if consensus sequence contains shitty characters, we will not continue
if reference_codon_sequence not in constants.codon_to_AA:
continue
linkmers.data = []
linkmers.append(bam_file_object, 'sample_id', None, contig_name, nt_positions, only_complete_links=True)
data = linkmers.data[0][2]
hash_to_oligotype = {}
unique_hashes = set([datum.read_unique_id for datum in data])
for unique_hash in unique_hashes:
hash_to_oligotype[unique_hash] = []
for datum in data:
hash_to_oligotype[datum.read_unique_id].append((datum.pos_in_contig, datum.base),)
for unique_hash in unique_hashes:
hash_to_oligotype[unique_hash] = ''.join([e[1] for e in sorted(hash_to_oligotype[unique_hash])])
codon_frequencies = Counter(list(hash_to_oligotype.values()))
# depending on what we want to return item frequencies will contain frequencies for amino acids or
# codons.
item_frequencies = Counter({})
# our conversion dicts will differ if the user is asking for codons or AAs, and if the gene is
# reverse or forward.
conv_dict = None
gene_is_reverse = gene_call['direction'] == 'r'
if return_AA_frequencies_instead:
if gene_is_reverse:
conv_dict = constants.codon_to_AA_RC
else:
conv_dict = constants.codon_to_AA
else:
if gene_is_reverse:
conv_dict = constants.codon_to_codon_RC
else:
# no conversion is necessary, so this is a mock dictionary that
# resturns the key.
conv_dict = dict(zip(constants.codons, constants.codons))
# the magic happens here:
for codon in codon_frequencies:
if codon in conv_dict:
item_frequencies[conv_dict[codon]] += codon_frequencies[codon]
else:
# so there is a way the programmer can learn that some weird
# stuff did not get reported
self.not_reported_items[codon] += 1
reference_item = conv_dict[reference_codon_sequence]
coverage = sum(item_frequencies.values())
if not coverage:
# FIXME: there was at least one case where the coverage here in this context was 0,
# which crashed the profiling. we never went after this issue, and it is
# important to understand how often this happens, and why.
continue
# here we quantify the ratio of frequencies of non-reference-aas observed in this codon
# to the overall overage, and that is our `departure_from_reference`:
total_frequency_of_all_items_but_the_conensus = sum([item_frequencies[item] for item in item_frequencies if item != reference_item])
departure_from_reference = total_frequency_of_all_items_but_the_conensus / coverage
d[codon_order] = {'reference': reference_item,
'coverage': coverage,
'frequencies': item_frequencies,
'departure_from_reference': departure_from_reference}
return d
def process_gene_call(self, bam_file_object, gene_call, contig_sequence, codons_to_profile=None):
if gene_call['partial']:
return None
contig_name = gene_call['contig']
# here we will create a dictionary to translate codons in a gene to nucleotide positions in the context
# of the contig. thanks to this dict, we will be able to profile only a small number of codons from a
# gene, if they are specified in `codons_to_profile` variable. for instance, this function is called
# during profiling only with codon positions in genes that possess nucleotide variation.
codon_order_to_nt_positions = utils.get_codon_order_to_nt_positions_dict(gene_call)
# here we generate the actual 'linkmers' information.
d = {}
linkmers = LinkMersData()
linkmers.quiet = True
for codon_order in codon_order_to_nt_positions:
if codons_to_profile and codon_order not in codons_to_profile:
continue
nt_positions = codon_order_to_nt_positions[codon_order]
reference_codon_sequence = contig_sequence[nt_positions[0]:nt_positions[2] + 1]
# if concensus sequence contains shitty characters, we will not continue
if reference_codon_sequence not in codon_to_AA:
continue
linkmers.data = []
linkmers.append(bam_file_object, 'sample_id', None, contig_name, nt_positions, only_complete_links=True)
data = linkmers.data[0][2]
hash_to_oligotype = {}
unique_hashes = set([datum.read_unique_id for datum in data])
for unique_hash in unique_hashes:
hash_to_oligotype[unique_hash] = []
for datum in data:
hash_to_oligotype[datum.read_unique_id].append((datum.pos_in_contig, datum.base),)
for unique_hash in unique_hashes:
hash_to_oligotype[unique_hash] = ''.join([e[1] for e in sorted(hash_to_oligotype[unique_hash])])
nt_frequencies = Counter(hash_to_oligotype.values())
aa_frequencies = Counter({})
# if the gene is reverse, we want to use the dict for reverse complementary conversions for DNA to AA
conv_dict = codon_to_AA_RC if gene_call['direction'] == 'r' else codon_to_AA
reference_codon_AA = conv_dict[reference_codon_sequence]
for nt in nt_frequencies:
if conv_dict[nt]: # <-- this check here eliminates any codon that contains anything but [A, T, C, G].
aa_frequencies[conv_dict[nt]] += nt_frequencies[nt]
coverage = sum(aa_frequencies.values())
if not coverage:
# FIXME: there was at least one case where the coverage here in this context was 0,
# which crashed the profiling. we never went after this issue, and it is
# important to understand how often this happens, and why.
continue
# here we quantify the ratio of frequencies of non-reference-aas observed in this codon
# to the overall overage, and that is our `departure_from_reference`:
total_frequency_of_all_codons_but_the_conensus = sum([aa_frequencies[aa] for aa in aa_frequencies if aa != reference_codon_AA])
departure_from_reference = total_frequency_of_all_codons_but_the_conensus / coverage
d[codon_order] = {'reference': reference_codon_AA,
'coverage': coverage,
'frequencies': aa_frequencies,
'departure_from_reference': departure_from_reference}
return d