def generate_variabile_nts_table(self):
if self.skip_SNV_profiling:
return
variable_nts_table = TableForVariability(self.profile_db_path,
progress=self.progress)
for contig in self.contigs:
for split in contig.splits:
for column_profile in list(split.column_profiles.values()):
# let's figure out more about this particular variable position
pos_in_contig = column_profile['pos_in_contig']
column_profile['in_partial_gene_call'], \
column_profile['in_complete_gene_call'],\
column_profile['base_pos_in_codon'] = self.get_nt_position_info(contig.name, pos_in_contig)
column_profile['sample_id'] = self.sample_id
column_profile[
'corresponding_gene_call'] = -1 # this means there is no gene call that corresponds to this
# nt position, which will be updated in the following lines.
# yeah, we use '-1', because genecaller ids start from 0 :/
column_profile['codon_order_in_gene'] = -1
# if this particular position (`pos_in_contig`) falls within a COMPLETE gene call,
# we would like to find out which unique gene caller id(s) match to this position.
if column_profile['in_complete_gene_call']:
corresponding_gene_caller_ids = self.get_corresponding_gene_caller_ids_for_base_position(
contig.name, pos_in_contig)
# if there are more than one corresponding gene call, this usually indicates an assembly error
# just to be on the safe side, we will not report a corresopnding unique gene callers id for this
# position
if len(corresponding_gene_caller_ids) == 1:
# if we are here, it means this nucleotide position is in a complete gene call. we will do two things here.
# first, we will store the gene_callers_id that corresponds to this nt position, and then we will store the
# order of the corresponding codon in the gene for this nt position.
gene_callers_id = corresponding_gene_caller_ids[0]
column_profile[
'corresponding_gene_call'] = gene_callers_id
column_profile[
'codon_order_in_gene'] = self.get_corresponding_codon_order_in_gene(
gene_callers_id, contig.name,
pos_in_contig)
# save this information for later use
self.codons_in_genes_to_profile_SCVs.add(
(gene_callers_id,
column_profile['codon_order_in_gene']), )
variable_nts_table.append(column_profile)
variable_nts_table.store()
self.layer_additional_data[
'num_SNVs_reported'] = variable_nts_table.num_entries
self.layer_additional_keys.append('num_SNVs_reported')
def generate_variabile_nts_table(self):
if self.skip_SNV_profiling:
return
variable_nts_table = TableForVariability(self.profile_db_path, progress=null_progress)
for contig in self.contigs:
for split in contig.splits:
for column_profile in list(split.column_profiles.values()):
variable_nts_table.append(column_profile)
variable_nts_table.store()
def merge_variable_nts_tables(self):
variable_nts_table = TableForVariability(self.merged_profile_db_path, progress=self.progress)
for input_profile_db_path in self.profile_dbs_info_dict:
sample_profile_db = dbops.ProfileDatabase(input_profile_db_path, quiet=True)
sample_variable_nts_table = sample_profile_db.db.get_table_as_list_of_tuples(tables.variable_nts_table_name, tables.variable_nts_table_structure)
sample_profile_db.disconnect()
for tpl in sample_variable_nts_table:
entry = tuple([variable_nts_table.next_id(tables.variable_nts_table_name)] + list(tpl[1:]))
variable_nts_table.db_entries.append(entry)
variable_nts_table.store()
def profile(self):
manager = multiprocessing.Manager()
available_index_queue = manager.Queue()
output_queue = manager.Queue(self.queue_size)
# put contig indices into the queue to be read from within
# the worker
for i in range(0, self.num_contigs):
available_index_queue.put(i)
processes = []
for i in range(0, self.num_threads):
processes.append(
multiprocessing.Process(
target=BAMProfiler.profile_contig_worker,
args=(self, available_index_queue, output_queue)))
for proc in processes:
proc.start()
recieved_contigs = 0
discarded_contigs = 0
memory_usage = None
self.progress.new('Profiling w/' + str(self.num_threads) +
' thread%s' % ('s' if self.num_threads > 1 else ''),
progress_total_items=self.num_contigs)
self.progress.update('initializing threads ...')
# FIXME: memory usage should be generalized.
last_memory_update = int(time.time())
self.progress.update('contigs are being processed ...')
self.progress.increment(recieved_contigs)
while recieved_contigs < self.num_contigs:
try:
contig = output_queue.get()
# if we have a contig back, it means we are good to go with it,
# otherwise it is garbage.
if contig:
self.contigs.append(contig)
else:
discarded_contigs += 1
recieved_contigs += 1
if (int(time.time()) - last_memory_update) > 5:
memory_usage = utils.get_total_memory_usage()
last_memory_update = int(time.time())
self.progress.update('%d of %d contigs ⚙ / MEM ☠️ %s' % \
(recieved_contigs, self.num_contigs, memory_usage or '??'))
# here you're about to witness the poor side of Python (or our use of it).
# the problem we run into here was the lack of action from the garbage
# collector on the processed objects. although we couldn't find any refs to
# these objects, garbage collecter kept them in the memory, and `del` statement
# on the `split` object did not yield any improvement either. so here we are
# accessing to the atomic data structures in our split objects to try to relieve
# the memory by encouraging the garbage collector to realize what's up
# explicitly.
if self.write_buffer_size > 0 and len(
self.contigs) % self.write_buffer_size == 0:
self.store_contigs_buffer()
for c in self.contigs:
for split in c.splits:
del split.coverage
del split.auxiliary
del split
del c.splits[:]
del c.coverage
del c
del self.contigs[:]
except KeyboardInterrupt:
self.run.info_single(
"Anvi'o profiler recieved SIGINT, terminating all processes...",
nl_before=2)
break
for proc in processes:
proc.terminate()
self.store_contigs_buffer()
self.auxiliary_db.close()
self.progress.end()
# FIXME: this needs to be checked:
if discarded_contigs > 0:
self.run.info('contigs_after_C',
pp(recieved_contigs - discarded_contigs))
overall_mean_coverage = 1
if self.total_length_of_all_contigs != 0:
overall_mean_coverage = self.total_coverage_values_for_all_contigs / self.total_length_of_all_contigs
# FIXME: We know this is ugly. You can keep your opinion to yourself.
if overall_mean_coverage > 0.0:
# avoid dividing by zero
dbops.ProfileDatabase(self.profile_db_path).db._exec(
"UPDATE atomic_data_splits SET abundance = abundance / " +
str(overall_mean_coverage) + " * 1.0;")
dbops.ProfileDatabase(self.profile_db_path).db._exec(
"UPDATE atomic_data_contigs SET abundance = abundance / " +
str(overall_mean_coverage) + " * 1.0;")
if not self.skip_SNV_profiling:
self.layer_additional_data[
'num_SNVs_reported'] = TableForVariability(
self.profile_db_path, progress=null_progress).num_entries
self.layer_additional_keys.append('num_SNVs_reported')
self.check_contigs(num_contigs=recieved_contigs - discarded_contigs)