def do(ref_fpath, aligned_contigs_fpaths, output_dirpath, features_dict, operons_fpaths,
detailed_contigs_reports_dirpath, genome_stats_dirpath):
coords_dirpath = os.path.join(detailed_contigs_reports_dirpath, qconfig.minimap_output_dirname)
from quast_libs import search_references_meta
if search_references_meta.is_quast_first_run:
coords_dirpath = os.path.join(coords_dirpath, 'raw')
logger.print_timestamp()
logger.main_info('Running Genome analyzer...')
if not os.path.isdir(genome_stats_dirpath):
os.mkdir(genome_stats_dirpath)
genome_size, reference_chromosomes, ns_by_chromosomes = fastaparser.get_genome_stats(ref_fpath)
# reading genome size
# genome_size = fastaparser.get_lengths_from_fastafile(reference)[0]
# reading reference name
# >gi|48994873|gb|U00096.2| Escherichia coli str. K-12 substr. MG1655, complete genome
# ref_file = open(reference, 'r')
# reference_name = ref_file.readline().split()[0][1:]
# ref_file.close()
# RESULTS file
result_fpath = os.path.join(genome_stats_dirpath, 'genome_info.txt')
res_file = open(result_fpath, 'w')
containers = []
for feature, feature_fpath in features_dict.items():
containers.append(FeatureContainer([feature_fpath], feature))
if not features_dict:
logger.notice('No file with genomic features were provided. '
'Use the --features option if you want to specify it.\n', indent=' ')
if operons_fpaths:
containers.append(FeatureContainer(operons_fpaths, 'operon'))
else:
logger.notice('No file with operons were provided. '
'Use the -O option if you want to specify it.', indent=' ')
for container in containers:
if not container.fpaths:
continue
for fpath in container.fpaths:
container.region_list += genes_parser.get_genes_from_file(fpath, container.kind)
if len(container.region_list) == 0:
logger.warning('No genomic features of type "' + container.kind + '" were loaded.', indent=' ')
res_file.write('Genomic features of type "' + container.kind + '" loaded: ' + 'None' + '\n')
else:
logger.info(' Loaded ' + str(len(container.region_list)) + ' genomic features of type "' + container.kind + '"')
res_file.write('Genomic features of type "' + container.kind + '" loaded: ' + str(len(container.region_list)) + '\n')
container.chr_names_dict = chromosomes_names_dict(container.kind, container.region_list, list(reference_chromosomes.keys()))
ref_genes_num, ref_operons_num = None, None
for contigs_fpath in aligned_contigs_fpaths:
report = reporting.get(contigs_fpath)
genomic_features = 0
for container in containers:
if container.kind == 'operon':
ref_operons_num = len(container.region_list)
report.add_field(reporting.Fields.REF_OPERONS, len(container.region_list))
else:
genomic_features += len(container.region_list)
if genomic_features:
ref_genes_num = genomic_features
report.add_field(reporting.Fields.REF_GENES, genomic_features)
# for cumulative plots:
files_features_in_contigs = {} # "filename" : [ genes in sorted contigs (see below) ]
files_unsorted_features_in_contigs = {} # "filename" : [ genes in sorted contigs (see below) ]
files_operons_in_contigs = {}
files_unsorted_operons_in_contigs = {}
# for histograms
genome_mapped = []
full_found_genes = []
full_found_operons = []
# process all contig files
num_nf_errors = logger._num_nf_errors
n_jobs = min(len(aligned_contigs_fpaths), qconfig.max_threads)
parallel_run_args = [(contigs_fpath, index, coords_dirpath, genome_stats_dirpath,
reference_chromosomes, ns_by_chromosomes, containers)
for index, contigs_fpath in enumerate(aligned_contigs_fpaths)]
ref_lengths, results_genes_operons_tuples = run_parallel(process_single_file, parallel_run_args, n_jobs, filter_results=True)
num_nf_errors += len(aligned_contigs_fpaths) - len(ref_lengths)
logger._num_nf_errors = num_nf_errors
if not ref_lengths:
logger.main_info('Genome analyzer failed for all the assemblies.')
res_file.close()
return
for ref in reference_chromosomes:
ref_lengths_by_contigs[ref] = [ref_lengths[i][ref] for i in range(len(ref_lengths))]
res_file.write('reference chromosomes:\n')
for chr_name, chr_len in reference_chromosomes.items():
aligned_len = max(ref_lengths_by_contigs[chr_name])
res_file.write('\t' + chr_name + ' (total length: ' + str(chr_len) + ' bp, ' +
'total length without N\'s: ' + str(chr_len - len(ns_by_chromosomes[chr_name])) +
' bp, maximal covered length: ' + str(aligned_len) + ' bp)\n')
res_file.write('\n')
res_file.write('total genome size: ' + str(genome_size) + '\n\n')
res_file.write('gap min size: ' + str(qconfig.min_gap_size) + '\n')
res_file.write('partial gene/operon min size: ' + str(qconfig.min_gene_overlap) + '\n\n')
# header
# header
res_file.write('\n\n')
res_file.write('%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n'
% ('assembly', 'genome', 'duplication', 'gaps', 'genes', 'partial', 'operons', 'partial'))
res_file.write('%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n'
% ('', 'fraction', 'ratio', 'number', '', 'genes', '', 'operons'))
res_file.write('=' * 120 + '\n')
for contigs_fpath, (results, unsorted_features_in_contigs, features_in_contigs, unsorted_operons_in_contigs, operons_in_contigs)\
in zip(aligned_contigs_fpaths, results_genes_operons_tuples):
assembly_name = qutils.name_from_fpath(contigs_fpath)
files_features_in_contigs[contigs_fpath] = features_in_contigs
files_unsorted_features_in_contigs[contigs_fpath] = unsorted_features_in_contigs
files_operons_in_contigs[contigs_fpath] = operons_in_contigs
files_unsorted_operons_in_contigs[contigs_fpath] = unsorted_operons_in_contigs
full_found_genes.append(sum(features_in_contigs))
full_found_operons.append(sum(operons_in_contigs))
gaps_count = results["gaps_count"]
genes_full = results[reporting.Fields.GENES + "_full"]
genes_part = results[reporting.Fields.GENES + "_partial"]
operons_full = results[reporting.Fields.OPERONS + "_full"]
operons_part = results[reporting.Fields.OPERONS + "_partial"]
report = reporting.get(contigs_fpath)
res_file.write('%-25s| %-10s| %-12s| %-10s|'
% (assembly_name[:24], report.get_field(reporting.Fields.MAPPEDGENOME), report.get_field(reporting.Fields.DUPLICATION_RATIO), gaps_count))
genome_mapped.append(float(report.get_field(reporting.Fields.MAPPEDGENOME)))
for (field, full, part) in [(reporting.Fields.GENES, genes_full, genes_part),
(reporting.Fields.OPERONS, operons_full, operons_part)]:
if full is None and part is None:
res_file.write(' %-10s| %-10s|' % ('-', '-'))
else:
res_file.write(' %-10s| %-10s|' % (full, part))
report.add_field(field, '%s + %s part' % (full, part))
res_file.write('\n')
res_file.close()
if qconfig.html_report:
from quast_libs.html_saver import html_saver
if ref_genes_num:
html_saver.save_features_in_contigs(output_dirpath, aligned_contigs_fpaths, 'features', files_features_in_contigs, ref_genes_num)
if ref_operons_num:
html_saver.save_features_in_contigs(output_dirpath, aligned_contigs_fpaths, 'operons', files_operons_in_contigs, ref_operons_num)
if qconfig.draw_plots:
# cumulative plots:
from . import plotter
from quast_libs.ca_utils.misc import contigs_aligned_lengths
if ref_genes_num:
plotter.genes_operons_plot(ref_genes_num, aligned_contigs_fpaths, files_features_in_contigs,
genome_stats_dirpath + '/features_cumulative_plot', 'genomic features')
plotter.frc_plot(output_dirpath, ref_fpath, aligned_contigs_fpaths, contigs_aligned_lengths, files_unsorted_features_in_contigs,
genome_stats_dirpath + '/features_frcurve_plot', 'genomic features')
plotter.histogram(aligned_contigs_fpaths, full_found_genes, genome_stats_dirpath + '/complete_features_histogram',
'# complete genomic features')
if ref_operons_num:
plotter.genes_operons_plot(ref_operons_num, aligned_contigs_fpaths, files_operons_in_contigs,
genome_stats_dirpath + '/operons_cumulative_plot', 'operons')
plotter.frc_plot(output_dirpath, ref_fpath, aligned_contigs_fpaths, contigs_aligned_lengths, files_unsorted_operons_in_contigs,
genome_stats_dirpath + '/operons_frcurve_plot', 'operons')
plotter.histogram(aligned_contigs_fpaths, full_found_operons, genome_stats_dirpath + '/complete_operons_histogram',
'# complete operons')
plotter.histogram(aligned_contigs_fpaths, genome_mapped, genome_stats_dirpath + '/genome_fraction_histogram',
'Genome fraction, %', top_value=100)
logger.main_info('Done.')
return containers
def do(ref_fpath, aligned_contigs_fpaths, output_dirpath, json_output_dirpath,
genes_fpaths, operons_fpaths, detailed_contigs_reports_dirpath, genome_stats_dirpath):
nucmer_path_dirpath = os.path.join(detailed_contigs_reports_dirpath, 'nucmer_output')
from quast_libs import search_references_meta
if search_references_meta.is_quast_first_run:
nucmer_path_dirpath = os.path.join(nucmer_path_dirpath, 'raw')
logger.print_timestamp()
logger.main_info('Running Genome analyzer...')
if not os.path.isdir(genome_stats_dirpath):
os.mkdir(genome_stats_dirpath)
reference_chromosomes = {}
genome_size = 0
for name, seq in fastaparser.read_fasta(ref_fpath):
chr_name = name.split()[0]
chr_len = len(seq)
genome_size += chr_len
reference_chromosomes[chr_name] = chr_len
# reading genome size
# genome_size = fastaparser.get_lengths_from_fastafile(reference)[0]
# reading reference name
# >gi|48994873|gb|U00096.2| Escherichia coli str. K-12 substr. MG1655, complete genome
# ref_file = open(reference, 'r')
# reference_name = ref_file.readline().split()[0][1:]
# ref_file.close()
# RESULTS file
result_fpath = genome_stats_dirpath + '/genome_info.txt'
res_file = open(result_fpath, 'w')
genes_container = FeatureContainer(genes_fpaths, 'gene')
operons_container = FeatureContainer(operons_fpaths, 'operon')
for container in [genes_container, operons_container]:
if not container.fpaths:
logger.notice('No file with ' + container.kind + 's provided. '
'Use the -' + container.kind[0].capitalize() + ' option '
'if you want to specify it.', indent=' ')
continue
for fpath in container.fpaths:
container.region_list += genes_parser.get_genes_from_file(fpath, container.kind)
if len(container.region_list) == 0:
logger.warning('No ' + container.kind + 's were loaded.', indent=' ')
res_file.write(container.kind + 's loaded: ' + 'None' + '\n')
else:
logger.info(' Loaded ' + str(len(container.region_list)) + ' ' + container.kind + 's')
res_file.write(container.kind + 's loaded: ' + str(len(container.region_list)) + '\n')
container.chr_names_dict = chromosomes_names_dict(container.kind, container.region_list, list(reference_chromosomes.keys()))
for contigs_fpath in aligned_contigs_fpaths:
report = reporting.get(contigs_fpath)
if genes_container.fpaths:
report.add_field(reporting.Fields.REF_GENES, len(genes_container.region_list))
if operons_container.fpaths:
report.add_field(reporting.Fields.REF_OPERONS, len(operons_container.region_list))
# for cumulative plots:
files_genes_in_contigs = {} # "filename" : [ genes in sorted contigs (see below) ]
files_operons_in_contigs = {}
# for histograms
genome_mapped = []
full_found_genes = []
full_found_operons = []
# process all contig files
num_nf_errors = logger._num_nf_errors
n_jobs = min(len(aligned_contigs_fpaths), qconfig.max_threads)
if is_python2():
from joblib import Parallel, delayed
else:
from joblib3 import Parallel, delayed
process_results = Parallel(n_jobs=n_jobs)(delayed(process_single_file)(
contigs_fpath, index, nucmer_path_dirpath, genome_stats_dirpath,
reference_chromosomes, genes_container, operons_container)
for index, contigs_fpath in enumerate(aligned_contigs_fpaths))
num_nf_errors += len([res for res in process_results if res is None])
logger._num_nf_errors = num_nf_errors
process_results = [res for res in process_results if res]
if not process_results:
logger.main_info('Genome analyzer failed for all the assemblies.')
res_file.close()
return
ref_lengths = [process_results[i][0] for i in range(len(process_results))]
results_genes_operons_tuples = [process_results[i][1] for i in range(len(process_results))]
for ref in reference_chromosomes:
ref_lengths_by_contigs[ref] = [ref_lengths[i][ref] for i in range(len(ref_lengths))]
res_file.write('reference chromosomes:\n')
for chr_name, chr_len in reference_chromosomes.items():
aligned_len = max(ref_lengths_by_contigs[chr_name])
res_file.write('\t' + chr_name + ' (total length: ' + str(chr_len) + ' bp, maximal covered length: ' + str(aligned_len) + ' bp)\n')
res_file.write('\n')
res_file.write('total genome size: ' + str(genome_size) + '\n\n')
res_file.write('gap min size: ' + str(qconfig.min_gap_size) + '\n')
res_file.write('partial gene/operon min size: ' + str(qconfig.min_gene_overlap) + '\n\n')
# header
# header
res_file.write('\n\n')
res_file.write('%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n'
% ('assembly', 'genome', 'duplication', 'gaps', 'genes', 'partial', 'operons', 'partial'))
res_file.write('%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n'
% ('', 'fraction', 'ratio', 'number', '', 'genes', '', 'operons'))
res_file.write('================================================================================================================\n')
for contigs_fpath, (results, genes_in_contigs, operons_in_contigs) in zip(aligned_contigs_fpaths, results_genes_operons_tuples):
assembly_name = qutils.name_from_fpath(contigs_fpath)
files_genes_in_contigs[contigs_fpath] = genes_in_contigs
files_operons_in_contigs[contigs_fpath] = operons_in_contigs
full_found_genes.append(sum(genes_in_contigs))
full_found_operons.append(sum(operons_in_contigs))
covered_bp = results["covered_bp"]
gaps_count = results["gaps_count"]
genes_full = results[reporting.Fields.GENES + "_full"]
genes_part = results[reporting.Fields.GENES + "_partial"]
operons_full = results[reporting.Fields.OPERONS + "_full"]
operons_part = results[reporting.Fields.OPERONS + "_partial"]
report = reporting.get(contigs_fpath)
genome_fraction = float(covered_bp) * 100 / float(genome_size)
duplication_ratio = (report.get_field(reporting.Fields.TOTALLEN) +
report.get_field(reporting.Fields.MISINTERNALOVERLAP) +
report.get_field(reporting.Fields.AMBIGUOUSEXTRABASES) -
report.get_field(reporting.Fields.UNALIGNEDBASES)) /\
((genome_fraction / 100.0) * float(genome_size))
res_file.write('%-25s| %-10s| %-12s| %-10s|'
% (assembly_name[:24], '%3.5f%%' % genome_fraction, '%1.5f' % duplication_ratio, gaps_count))
report.add_field(reporting.Fields.MAPPEDGENOME, '%.3f' % genome_fraction)
report.add_field(reporting.Fields.DUPLICATION_RATIO, '%.3f' % duplication_ratio)
genome_mapped.append(genome_fraction)
for (field, full, part) in [(reporting.Fields.GENES, genes_full, genes_part),
(reporting.Fields.OPERONS, operons_full, operons_part)]:
if full is None and part is None:
res_file.write(' %-10s| %-10s|' % ('-', '-'))
else:
res_file.write(' %-10s| %-10s|' % (full, part))
report.add_field(field, '%s + %s part' % (full, part))
res_file.write('\n')
res_file.close()
if genes_container.region_list:
ref_genes_num = len(genes_container.region_list)
else:
ref_genes_num = None
if operons_container.region_list:
ref_operons_num = len(operons_container.region_list)
else:
ref_operons_num = None
# saving json
if json_output_dirpath:
if genes_container.region_list:
json_saver.save_features_in_contigs(json_output_dirpath, aligned_contigs_fpaths, 'genes', files_genes_in_contigs, ref_genes_num)
if operons_container.region_list:
json_saver.save_features_in_contigs(json_output_dirpath, aligned_contigs_fpaths, 'operons', files_operons_in_contigs, ref_operons_num)
if qconfig.html_report:
from quast_libs.html_saver import html_saver
if genes_container.region_list:
html_saver.save_features_in_contigs(output_dirpath, aligned_contigs_fpaths, 'genes', files_genes_in_contigs, ref_genes_num)
if operons_container.region_list:
html_saver.save_features_in_contigs(output_dirpath, aligned_contigs_fpaths, 'operons', files_operons_in_contigs, ref_operons_num)
if qconfig.draw_plots:
# cumulative plots:
from . import plotter
if genes_container.region_list:
plotter.genes_operons_plot(len(genes_container.region_list), aligned_contigs_fpaths, files_genes_in_contigs,
genome_stats_dirpath + '/genes_cumulative_plot', 'genes')
plotter.histogram(aligned_contigs_fpaths, full_found_genes, genome_stats_dirpath + '/complete_genes_histogram',
'# complete genes')
if operons_container.region_list:
plotter.genes_operons_plot(len(operons_container.region_list), aligned_contigs_fpaths, files_operons_in_contigs,
genome_stats_dirpath + '/operons_cumulative_plot', 'operons')
plotter.histogram(aligned_contigs_fpaths, full_found_operons, genome_stats_dirpath + '/complete_operons_histogram',
'# complete operons')
plotter.histogram(aligned_contigs_fpaths, genome_mapped, genome_stats_dirpath + '/genome_fraction_histogram',
'Genome fraction, %', top_value=100)
logger.main_info('Done.')
return [genes_container, operons_container]
def do(ref_fpath, aligned_contigs_fpaths, output_dirpath, features_dict,
operons_fpaths, detailed_contigs_reports_dirpath, genome_stats_dirpath):
coords_dirpath = os.path.join(detailed_contigs_reports_dirpath,
qconfig.minimap_output_dirname)
from quast_libs import search_references_meta
if search_references_meta.is_quast_first_run:
coords_dirpath = os.path.join(coords_dirpath, 'raw')
logger.print_timestamp()
logger.main_info('Running Genome analyzer...')
if not os.path.isdir(genome_stats_dirpath):
os.mkdir(genome_stats_dirpath)
genome_size, reference_chromosomes, ns_by_chromosomes = fastaparser.get_genome_stats(
ref_fpath)
# reading genome size
# genome_size = fastaparser.get_lengths_from_fastafile(reference)[0]
# reading reference name
# >gi|48994873|gb|U00096.2| Escherichia coli str. K-12 substr. MG1655, complete genome
# ref_file = open(reference, 'r')
# reference_name = ref_file.readline().split()[0][1:]
# ref_file.close()
# RESULTS file
result_fpath = os.path.join(genome_stats_dirpath, 'genome_info.txt')
res_file = open(result_fpath, 'w')
containers = []
for feature, feature_fpath in features_dict.items():
containers.append(FeatureContainer([feature_fpath], feature))
if not features_dict:
logger.notice(
'No file with genomic features were provided. '
'Use the --features option if you want to specify it.\n',
indent=' ')
if operons_fpaths:
containers.append(FeatureContainer(operons_fpaths, 'operon'))
else:
logger.notice(
'No file with operons were provided. '
'Use the -O option if you want to specify it.',
indent=' ')
for container in containers:
if not container.fpaths:
continue
for fpath in container.fpaths:
container.region_list += genes_parser.get_genes_from_file(
fpath, container.kind)
if len(container.region_list) == 0:
logger.warning('No genomic features of type "' + container.kind +
'" were loaded.',
indent=' ')
res_file.write('Genomic features of type "' + container.kind +
'" loaded: ' + 'None' + '\n')
else:
logger.info(' Loaded ' + str(len(container.region_list)) +
' genomic features of type "' + container.kind + '"')
res_file.write('Genomic features of type "' + container.kind +
'" loaded: ' + str(len(container.region_list)) +
'\n')
container.chr_names_dict = chromosomes_names_dict(
container.kind, container.region_list,
list(reference_chromosomes.keys()))
ref_genes_num, ref_operons_num = None, None
for contigs_fpath in aligned_contigs_fpaths:
report = reporting.get(contigs_fpath)
genomic_features = 0
for container in containers:
if container.kind == 'operon':
ref_operons_num = len(container.region_list)
report.add_field(reporting.Fields.REF_OPERONS,
len(container.region_list))
else:
genomic_features += len(container.region_list)
if genomic_features:
ref_genes_num = genomic_features
report.add_field(reporting.Fields.REF_GENES, genomic_features)
# for cumulative plots:
files_features_in_contigs = {
} # "filename" : [ genes in sorted contigs (see below) ]
files_unsorted_features_in_contigs = {
} # "filename" : [ genes in sorted contigs (see below) ]
files_operons_in_contigs = {}
files_unsorted_operons_in_contigs = {}
# for histograms
genome_mapped = []
full_found_genes = []
full_found_operons = []
# process all contig files
num_nf_errors = logger._num_nf_errors
n_jobs = min(len(aligned_contigs_fpaths), qconfig.max_threads)
if is_python2():
from joblib2 import Parallel, delayed
else:
from joblib3 import Parallel, delayed
if not qconfig.memory_efficient:
process_results = Parallel(n_jobs=n_jobs)(
delayed(process_single_file)(
contigs_fpath, index, coords_dirpath, genome_stats_dirpath,
reference_chromosomes, ns_by_chromosomes, containers)
for index, contigs_fpath in enumerate(aligned_contigs_fpaths))
else:
process_results = [
process_single_file(contigs_fpath, index, coords_dirpath,
genome_stats_dirpath, reference_chromosomes,
ns_by_chromosomes, containers)
for index, contigs_fpath in enumerate(aligned_contigs_fpaths)
]
num_nf_errors += len([res for res in process_results if res is None])
logger._num_nf_errors = num_nf_errors
process_results = [res for res in process_results if res]
if not process_results:
logger.main_info('Genome analyzer failed for all the assemblies.')
res_file.close()
return
ref_lengths = [process_results[i][0] for i in range(len(process_results))]
results_genes_operons_tuples = [
process_results[i][1] for i in range(len(process_results))
]
for ref in reference_chromosomes:
ref_lengths_by_contigs[ref] = [
ref_lengths[i][ref] for i in range(len(ref_lengths))
]
res_file.write('reference chromosomes:\n')
for chr_name, chr_len in reference_chromosomes.items():
aligned_len = max(ref_lengths_by_contigs[chr_name])
res_file.write('\t' + chr_name + ' (total length: ' + str(chr_len) +
' bp, ' + 'total length without N\'s: ' +
str(chr_len - len(ns_by_chromosomes[chr_name])) +
' bp, maximal covered length: ' + str(aligned_len) +
' bp)\n')
res_file.write('\n')
res_file.write('total genome size: ' + str(genome_size) + '\n\n')
res_file.write('gap min size: ' + str(qconfig.min_gap_size) + '\n')
res_file.write('partial gene/operon min size: ' +
str(qconfig.min_gene_overlap) + '\n\n')
# header
# header
res_file.write('\n\n')
res_file.write(
'%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n' %
('assembly', 'genome', 'duplication', 'gaps', 'genes', 'partial',
'operons', 'partial'))
res_file.write(
'%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n' %
('', 'fraction', 'ratio', 'number', '', 'genes', '', 'operons'))
res_file.write('=' * 120 + '\n')
for contigs_fpath, (results, unsorted_features_in_contigs, features_in_contigs, unsorted_operons_in_contigs, operons_in_contigs)\
in zip(aligned_contigs_fpaths, results_genes_operons_tuples):
assembly_name = qutils.name_from_fpath(contigs_fpath)
files_features_in_contigs[contigs_fpath] = features_in_contigs
files_unsorted_features_in_contigs[
contigs_fpath] = unsorted_features_in_contigs
files_operons_in_contigs[contigs_fpath] = operons_in_contigs
files_unsorted_operons_in_contigs[
contigs_fpath] = unsorted_operons_in_contigs
full_found_genes.append(sum(features_in_contigs))
full_found_operons.append(sum(operons_in_contigs))
gaps_count = results["gaps_count"]
genes_full = results[reporting.Fields.GENES + "_full"]
genes_part = results[reporting.Fields.GENES + "_partial"]
operons_full = results[reporting.Fields.OPERONS + "_full"]
operons_part = results[reporting.Fields.OPERONS + "_partial"]
report = reporting.get(contigs_fpath)
res_file.write(
'%-25s| %-10s| %-12s| %-10s|' %
(assembly_name[:24], report.get_field(
reporting.Fields.MAPPEDGENOME),
report.get_field(reporting.Fields.DUPLICATION_RATIO), gaps_count))
genome_mapped.append(
float(report.get_field(reporting.Fields.MAPPEDGENOME)))
for (field, full,
part) in [(reporting.Fields.GENES, genes_full, genes_part),
(reporting.Fields.OPERONS, operons_full, operons_part)]:
if full is None and part is None:
res_file.write(' %-10s| %-10s|' % ('-', '-'))
else:
res_file.write(' %-10s| %-10s|' % (full, part))
report.add_field(field, '%s + %s part' % (full, part))
res_file.write('\n')
res_file.close()
if qconfig.html_report:
from quast_libs.html_saver import html_saver
if ref_genes_num:
html_saver.save_features_in_contigs(output_dirpath,
aligned_contigs_fpaths,
'features',
files_features_in_contigs,
ref_genes_num)
if ref_operons_num:
html_saver.save_features_in_contigs(output_dirpath,
aligned_contigs_fpaths,
'operons',
files_operons_in_contigs,
ref_operons_num)
if qconfig.draw_plots:
# cumulative plots:
from . import plotter
from quast_libs.ca_utils.misc import contigs_aligned_lengths
if ref_genes_num:
plotter.genes_operons_plot(
ref_genes_num, aligned_contigs_fpaths,
files_features_in_contigs,
genome_stats_dirpath + '/features_cumulative_plot',
'genomic features')
plotter.frc_plot(output_dirpath, ref_fpath, aligned_contigs_fpaths,
contigs_aligned_lengths,
files_unsorted_features_in_contigs,
genome_stats_dirpath + '/features_frcurve_plot',
'genomic features')
plotter.histogram(
aligned_contigs_fpaths, full_found_genes,
genome_stats_dirpath + '/complete_features_histogram',
'# complete genomic features')
if ref_operons_num:
plotter.genes_operons_plot(
ref_operons_num, aligned_contigs_fpaths,
files_operons_in_contigs,
genome_stats_dirpath + '/operons_cumulative_plot', 'operons')
plotter.frc_plot(output_dirpath, ref_fpath, aligned_contigs_fpaths,
contigs_aligned_lengths,
files_unsorted_operons_in_contigs,
genome_stats_dirpath + '/operons_frcurve_plot',
'operons')
plotter.histogram(
aligned_contigs_fpaths, full_found_operons,
genome_stats_dirpath + '/complete_operons_histogram',
'# complete operons')
plotter.histogram(aligned_contigs_fpaths,
genome_mapped,
genome_stats_dirpath + '/genome_fraction_histogram',
'Genome fraction, %',
top_value=100)
logger.main_info('Done.')
return containers
def do(ref_fpath, aligned_contigs_fpaths, output_dirpath, genes_fpaths,
operons_fpaths, detailed_contigs_reports_dirpath, genome_stats_dirpath):
nucmer_path_dirpath = os.path.join(detailed_contigs_reports_dirpath,
'nucmer_output')
from quast_libs import search_references_meta
if search_references_meta.is_quast_first_run:
nucmer_path_dirpath = os.path.join(nucmer_path_dirpath, 'raw')
logger.print_timestamp()
logger.main_info('Running Genome analyzer...')
if not os.path.isdir(genome_stats_dirpath):
os.mkdir(genome_stats_dirpath)
reference_chromosomes = {}
genome_size = 0
for name, seq in fastaparser.read_fasta(ref_fpath):
chr_name = name.split()[0]
chr_len = len(seq)
genome_size += chr_len
reference_chromosomes[chr_name] = chr_len
# reading genome size
# genome_size = fastaparser.get_lengths_from_fastafile(reference)[0]
# reading reference name
# >gi|48994873|gb|U00096.2| Escherichia coli str. K-12 substr. MG1655, complete genome
# ref_file = open(reference, 'r')
# reference_name = ref_file.readline().split()[0][1:]
# ref_file.close()
# RESULTS file
result_fpath = genome_stats_dirpath + '/genome_info.txt'
res_file = open(result_fpath, 'w')
genes_container = FeatureContainer(genes_fpaths, 'gene')
operons_container = FeatureContainer(operons_fpaths, 'operon')
for container in [genes_container, operons_container]:
if not container.fpaths:
logger.notice('No file with ' + container.kind + 's provided. '
'Use the -' + container.kind[0].capitalize() +
' option '
'if you want to specify it.',
indent=' ')
continue
for fpath in container.fpaths:
container.region_list += genes_parser.get_genes_from_file(
fpath, container.kind)
if len(container.region_list) == 0:
logger.warning('No ' + container.kind + 's were loaded.',
indent=' ')
res_file.write(container.kind + 's loaded: ' + 'None' + '\n')
else:
logger.info(' Loaded ' + str(len(container.region_list)) + ' ' +
container.kind + 's')
res_file.write(container.kind + 's loaded: ' +
str(len(container.region_list)) + '\n')
container.chr_names_dict = chromosomes_names_dict(
container.kind, container.region_list,
list(reference_chromosomes.keys()))
for contigs_fpath in aligned_contigs_fpaths:
report = reporting.get(contigs_fpath)
if genes_container.fpaths:
report.add_field(reporting.Fields.REF_GENES,
len(genes_container.region_list))
if operons_container.fpaths:
report.add_field(reporting.Fields.REF_OPERONS,
len(operons_container.region_list))
# for cumulative plots:
files_genes_in_contigs = {
} # "filename" : [ genes in sorted contigs (see below) ]
files_unsorted_genes_in_contigs = {
} # "filename" : [ genes in sorted contigs (see below) ]
files_operons_in_contigs = {}
files_unsorted_operons_in_contigs = {}
# for histograms
genome_mapped = []
full_found_genes = []
full_found_operons = []
# process all contig files
num_nf_errors = logger._num_nf_errors
n_jobs = min(len(aligned_contigs_fpaths), qconfig.max_threads)
if is_python2():
from joblib import Parallel, delayed
else:
from joblib3 import Parallel, delayed
if not qconfig.memory_efficient:
process_results = Parallel(n_jobs=n_jobs)(
delayed(process_single_file)
(contigs_fpath, index, nucmer_path_dirpath, genome_stats_dirpath,
reference_chromosomes, genes_container, operons_container)
for index, contigs_fpath in enumerate(aligned_contigs_fpaths))
else:
process_results = [
process_single_file(contigs_fpath, index, nucmer_path_dirpath,
genome_stats_dirpath, reference_chromosomes,
genes_container, operons_container)
for index, contigs_fpath in enumerate(aligned_contigs_fpaths)
]
num_nf_errors += len([res for res in process_results if res is None])
logger._num_nf_errors = num_nf_errors
process_results = [res for res in process_results if res]
if not process_results:
logger.main_info('Genome analyzer failed for all the assemblies.')
res_file.close()
return
ref_lengths = [process_results[i][0] for i in range(len(process_results))]
results_genes_operons_tuples = [
process_results[i][1] for i in range(len(process_results))
]
for ref in reference_chromosomes:
ref_lengths_by_contigs[ref] = [
ref_lengths[i][ref] for i in range(len(ref_lengths))
]
res_file.write('reference chromosomes:\n')
for chr_name, chr_len in reference_chromosomes.items():
aligned_len = max(ref_lengths_by_contigs[chr_name])
res_file.write('\t' + chr_name + ' (total length: ' + str(chr_len) +
' bp, maximal covered length: ' + str(aligned_len) +
' bp)\n')
res_file.write('\n')
res_file.write('total genome size: ' + str(genome_size) + '\n\n')
res_file.write('gap min size: ' + str(qconfig.min_gap_size) + '\n')
res_file.write('partial gene/operon min size: ' +
str(qconfig.min_gene_overlap) + '\n\n')
# header
# header
res_file.write('\n\n')
res_file.write(
'%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n' %
('assembly', 'genome', 'duplication', 'gaps', 'genes', 'partial',
'operons', 'partial'))
res_file.write(
'%-25s| %-10s| %-12s| %-10s| %-10s| %-10s| %-10s| %-10s|\n' %
('', 'fraction', 'ratio', 'number', '', 'genes', '', 'operons'))
res_file.write('=' * 120 + '\n')
for contigs_fpath, (results, unsorted_genes_in_contigs, genes_in_contigs, unsorted_operons_in_contigs, operons_in_contigs)\
in zip(aligned_contigs_fpaths, results_genes_operons_tuples):
assembly_name = qutils.name_from_fpath(contigs_fpath)
files_genes_in_contigs[contigs_fpath] = genes_in_contigs
files_unsorted_genes_in_contigs[
contigs_fpath] = unsorted_genes_in_contigs
files_operons_in_contigs[contigs_fpath] = operons_in_contigs
files_unsorted_operons_in_contigs[
contigs_fpath] = unsorted_operons_in_contigs
full_found_genes.append(sum(genes_in_contigs))
full_found_operons.append(sum(operons_in_contigs))
covered_bp = results["covered_bp"]
gaps_count = results["gaps_count"]
genes_full = results[reporting.Fields.GENES + "_full"]
genes_part = results[reporting.Fields.GENES + "_partial"]
operons_full = results[reporting.Fields.OPERONS + "_full"]
operons_part = results[reporting.Fields.OPERONS + "_partial"]
report = reporting.get(contigs_fpath)
genome_fraction = float(covered_bp) * 100 / float(genome_size)
duplication_ratio = (report.get_field(reporting.Fields.TOTALLEN) +
report.get_field(reporting.Fields.MISINTERNALOVERLAP) +
report.get_field(reporting.Fields.AMBIGUOUSEXTRABASES) -
report.get_field(reporting.Fields.UNALIGNEDBASES)) /\
((genome_fraction / 100.0) * float(genome_size))
res_file.write('%-25s| %-10s| %-12s| %-10s|' %
(assembly_name[:24], '%3.5f%%' % genome_fraction,
'%1.5f' % duplication_ratio, gaps_count))
report.add_field(reporting.Fields.MAPPEDGENOME,
'%.3f' % genome_fraction)
report.add_field(reporting.Fields.DUPLICATION_RATIO,
'%.3f' % duplication_ratio)
genome_mapped.append(genome_fraction)
for (field, full,
part) in [(reporting.Fields.GENES, genes_full, genes_part),
(reporting.Fields.OPERONS, operons_full, operons_part)]:
if full is None and part is None:
res_file.write(' %-10s| %-10s|' % ('-', '-'))
else:
res_file.write(' %-10s| %-10s|' % (full, part))
report.add_field(field, '%s + %s part' % (full, part))
res_file.write('\n')
res_file.close()
if genes_container.region_list:
ref_genes_num = len(genes_container.region_list)
else:
ref_genes_num = None
if operons_container.region_list:
ref_operons_num = len(operons_container.region_list)
else:
ref_operons_num = None
if qconfig.html_report:
from quast_libs.html_saver import html_saver
if genes_container.region_list:
html_saver.save_features_in_contigs(output_dirpath,
aligned_contigs_fpaths,
'genes',
files_genes_in_contigs,
ref_genes_num)
if operons_container.region_list:
html_saver.save_features_in_contigs(output_dirpath,
aligned_contigs_fpaths,
'operons',
files_operons_in_contigs,
ref_operons_num)
if qconfig.draw_plots:
# cumulative plots:
from . import plotter
from quast_libs.ca_utils.misc import contigs_aligned_lengths
if genes_container.region_list:
plotter.genes_operons_plot(
len(genes_container.region_list), aligned_contigs_fpaths,
files_genes_in_contigs,
genome_stats_dirpath + '/genes_cumulative_plot', 'genes')
plotter.frc_plot(output_dirpath, ref_fpath, aligned_contigs_fpaths,
contigs_aligned_lengths,
files_unsorted_genes_in_contigs,
genome_stats_dirpath + '/genes_frcurve_plot',
'genes')
plotter.histogram(
aligned_contigs_fpaths, full_found_genes,
genome_stats_dirpath + '/complete_genes_histogram',
'# complete genes')
if operons_container.region_list:
plotter.genes_operons_plot(
len(operons_container.region_list), aligned_contigs_fpaths,
files_operons_in_contigs,
genome_stats_dirpath + '/operons_cumulative_plot', 'operons')
plotter.frc_plot(output_dirpath, ref_fpath, aligned_contigs_fpaths,
contigs_aligned_lengths,
files_unsorted_operons_in_contigs,
genome_stats_dirpath + '/operons_frcurve_plot',
'operons')
plotter.histogram(
aligned_contigs_fpaths, full_found_operons,
genome_stats_dirpath + '/complete_operons_histogram',
'# complete operons')
plotter.histogram(aligned_contigs_fpaths,
genome_mapped,
genome_stats_dirpath + '/genome_fraction_histogram',
'Genome fraction, %',
top_value=100)
logger.main_info('Done.')
return [genes_container, operons_container]