def main(args):
check_dirpath(
qconfig.QUAST_HOME,
'You are trying to run it from ' + str(qconfig.QUAST_HOME) + '.\n' +
'Please, put QUAST in a different directory, then try again.\n',
exit_code=3)
if not args:
qconfig.usage(meta=True)
sys.exit(0)
metaquast_path = [os.path.realpath(__file__)]
quast_py_args, contigs_fpaths = parse_options(logger,
metaquast_path + args,
is_metaquast=True)
output_dirpath, ref_fpaths, labels = qconfig.output_dirpath, qconfig.reference, qconfig.labels
html_report = qconfig.html_report
test_mode = qconfig.test
# Directories
output_dirpath, _, _ = qutils.set_up_output_dir(output_dirpath,
None,
not output_dirpath,
save_json=False)
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
qconfig.set_max_threads(logger)
qutils.logger = logger
########################################################################
from quast_libs import reporting
try:
import imp
imp.reload(reporting)
except:
reload(reporting)
from quast_libs import plotter
if os.path.isdir(corrected_dirpath):
shutil.rmtree(corrected_dirpath)
os.mkdir(corrected_dirpath)
# PROCESSING REFERENCES
if ref_fpaths:
logger.main_info()
logger.main_info('Reference(s):')
corrected_ref_fpaths, combined_ref_fpath, chromosomes_by_refs, ref_names =\
correct_meta_references(ref_fpaths, corrected_dirpath)
# PROCESSING CONTIGS
logger.main_info()
logger.main_info('Contigs:')
qconfig.no_check_meta = True
assemblies, labels = correct_assemblies(contigs_fpaths, output_dirpath,
labels)
if not assemblies:
logger.error(
"None of the assembly files contains correct contigs. "
"Please, provide different files or decrease --min-contig threshold."
)
return 4
# Running QUAST(s)
if qconfig.gene_finding:
quast_py_args += ['--mgm']
downloaded_refs = False
# SEARCHING REFERENCES
if not ref_fpaths:
logger.main_info()
if qconfig.max_references == 0:
logger.notice(
"Maximum number of references (--max-ref-number) is set to 0, search in SILVA 16S rRNA database is disabled"
)
else:
if qconfig.references_txt:
logger.main_info(
"List of references was provided, starting to download reference genomes from NCBI..."
)
else:
logger.main_info(
"No references are provided, starting to search for reference genomes in SILVA 16S rRNA database "
"and to download them from NCBI...")
downloaded_dirpath = os.path.join(output_dirpath,
qconfig.downloaded_dirname)
if not os.path.isdir(downloaded_dirpath):
os.mkdir(downloaded_dirpath)
corrected_dirpath = os.path.join(output_dirpath,
qconfig.corrected_dirname)
ref_fpaths = search_references_meta.do(assemblies, labels,
downloaded_dirpath,
corrected_dirpath,
qconfig.references_txt)
if ref_fpaths:
search_references_meta.is_quast_first_run = True
if not qconfig.references_txt:
downloaded_refs = True
logger.main_info()
logger.main_info('Downloaded reference(s):')
corrected_ref_fpaths, combined_ref_fpath, chromosomes_by_refs, ref_names =\
correct_meta_references(ref_fpaths, corrected_dirpath, downloaded_refs=True)
elif test_mode and not ref_fpaths:
logger.error(
'Failed to download or setup SILVA 16S rRNA database for working without '
'references on metagenome datasets!',
to_stderr=True,
exit_with_code=4)
if not ref_fpaths:
# No references, running regular quast with MetaGenemark gene finder
logger.main_info()
logger.notice(
'No references are provided, starting regular QUAST with MetaGeneMark gene finder'
)
_start_quast_main(quast_py_args,
assemblies=assemblies,
output_dirpath=output_dirpath,
run_regular_quast=True)
exit(0)
# Running combined reference
combined_output_dirpath = os.path.join(output_dirpath,
qconfig.combined_output_name)
reads_fpaths = []
if qconfig.forward_reads:
reads_fpaths.append(qconfig.forward_reads)
if qconfig.reverse_reads:
reads_fpaths.append(qconfig.reverse_reads)
cov_fpath = qconfig.cov_fpath
physical_cov_fpath = qconfig.phys_cov_fpath
if (reads_fpaths or qconfig.sam or qconfig.bam) and ref_fpaths:
bed_fpath, cov_fpath, physical_cov_fpath = reads_analyzer.do(
combined_ref_fpath,
contigs_fpaths,
reads_fpaths,
corrected_ref_fpaths,
os.path.join(combined_output_dirpath, qconfig.variation_dirname),
external_logger=logger,
sam_fpath=qconfig.sam,
bam_fpath=qconfig.bam,
bed_fpath=qconfig.bed)
qconfig.bed = bed_fpath
if qconfig.bed:
quast_py_args += ['--sv-bed']
quast_py_args += [qconfig.bed]
if cov_fpath:
quast_py_args += ['--cov']
quast_py_args += [cov_fpath]
if physical_cov_fpath:
quast_py_args += ['--phys-cov']
quast_py_args += [physical_cov_fpath]
if qconfig.sam:
quast_py_args += ['--sam']
quast_py_args += [qconfig.sam]
if qconfig.bam:
quast_py_args += ['--bam']
quast_py_args += [qconfig.bam]
quast_py_args += ['--combined-ref']
if qconfig.draw_plots or qconfig.html_report:
if plotter_data.dict_color_and_ls:
colors_and_ls = [
plotter_data.dict_color_and_ls[asm.label] for asm in assemblies
]
quast_py_args += ['--colors']
quast_py_args += [','.join([style[0] for style in colors_and_ls])]
quast_py_args += ['--ls']
quast_py_args += [','.join([style[1] for style in colors_and_ls])]
run_name = 'for the combined reference'
logger.main_info()
logger.main_info('Starting quast.py ' + run_name + '...')
total_num_notices = 0
total_num_warnings = 0
total_num_nf_errors = 0
total_num_notifications = (total_num_notices, total_num_warnings,
total_num_nf_errors)
if qconfig.html_report:
from quast_libs.html_saver import json_saver
json_texts = []
else:
json_texts = None
if qconfig.unique_mapping:
ambiguity_opts = []
else:
ambiguity_opts = ["--ambiguity-usage", 'all']
return_code, total_num_notifications = \
_start_quast_main(quast_py_args + ambiguity_opts,
labels=labels,
assemblies=assemblies,
reference_fpath=combined_ref_fpath,
output_dirpath=combined_output_dirpath,
num_notifications_tuple=total_num_notifications,
is_combined_ref=True)
if json_texts is not None:
json_texts.append(json_saver.json_text)
search_references_meta.is_quast_first_run = False
genome_info_dirpath = os.path.join(output_dirpath,
qconfig.combined_output_name,
'genome_stats')
genome_info_fpath = os.path.join(genome_info_dirpath, 'genome_info.txt')
if not os.path.exists(genome_info_fpath):
logger.main_info('')
if not downloaded_refs:
msg = 'Try to restart MetaQUAST with another references.'
else:
msg = 'Try to use option --max-ref-number to change maximum number of references (per each assembly) to download.'
logger.main_info(
'Failed aligning the contigs for all the references. ' + msg)
logger.main_info('')
cleanup(corrected_dirpath)
logger.main_info('MetaQUAST finished.')
return logger.finish_up(numbers=tuple(total_num_notifications),
check_test=test_mode)
if downloaded_refs:
logger.main_info()
logger.main_info(
'Excluding downloaded references with low genome fraction from further analysis..'
)
corr_ref_fpaths = get_downloaded_refs_with_alignments(
genome_info_fpath, ref_fpaths, chromosomes_by_refs)
if corr_ref_fpaths and corr_ref_fpaths != ref_fpaths:
logger.main_info()
logger.main_info('Filtered reference(s):')
os.remove(combined_ref_fpath)
contigs_analyzer.ref_labels_by_chromosomes = OrderedDict()
corrected_ref_fpaths, combined_ref_fpath, chromosomes_by_refs, ref_names = \
correct_meta_references(corr_ref_fpaths, corrected_dirpath)
assemblies, labels = correct_assemblies(contigs_fpaths,
output_dirpath, labels)
run_name = 'for the corrected combined reference'
logger.main_info()
logger.main_info('Starting quast.py ' + run_name + '...')
return_code, total_num_notifications = \
_start_quast_main(quast_py_args + ambiguity_opts,
labels=labels,
assemblies=assemblies,
reference_fpath=combined_ref_fpath,
output_dirpath=combined_output_dirpath,
num_notifications_tuple=total_num_notifications,
is_combined_ref=True)
if json_texts is not None:
json_texts = json_texts[:-1]
json_texts.append(json_saver.json_text)
elif corr_ref_fpaths == ref_fpaths:
logger.main_info(
'All downloaded references have genome fraction more than 10%. Nothing was excluded.'
)
else:
logger.main_info(
'All downloaded references have low genome fraction. Nothing was excluded for now.'
)
if qconfig.calculate_read_support:
calculate_ave_read_support(combined_output_dirpath, assemblies)
for arg in args:
if arg in ('-s', "--scaffolds"):
quast_py_args.remove(arg)
quast_py_args += ['--no-check-meta']
qconfig.contig_thresholds = ','.join([
str(threshold) for threshold in qconfig.contig_thresholds
if threshold >= qconfig.min_contig
])
if not qconfig.contig_thresholds:
qconfig.contig_thresholds = 'None'
quast_py_args = remove_from_quast_py_args(quast_py_args,
'--contig-thresholds',
qconfig.contig_thresholds)
quast_py_args += ['--contig-thresholds']
quast_py_args += [qconfig.contig_thresholds]
quast_py_args.remove('--combined-ref')
logger.main_info()
logger.main_info(
'Partitioning contigs into bins aligned to each reference..')
assemblies_by_reference, not_aligned_assemblies = partition_contigs(
assemblies, corrected_ref_fpaths, corrected_dirpath,
os.path.join(combined_output_dirpath, 'contigs_reports',
'alignments_%s.tsv'), labels)
output_dirpath_per_ref = os.path.join(output_dirpath,
qconfig.per_ref_dirname)
if not qconfig.memory_efficient and \
len(assemblies_by_reference) > len(assemblies) and len(assemblies) < qconfig.max_threads:
logger.main_info()
logger.main_info('Run QUAST on different references in parallel..')
threads_per_ref = max(
1, qconfig.max_threads // len(assemblies_by_reference))
quast_py_args += ['--memory-efficient']
quast_py_args += ['-t', str(threads_per_ref)]
num_notifications = (0, 0, 0)
parallel_run_args = [
(quast_py_args, output_dirpath_per_ref, ref_fpath, ref_assemblies,
num_notifications, True)
for ref_fpath, ref_assemblies in assemblies_by_reference
]
ref_names, ref_json_texts, ref_notifications = \
run_parallel(_run_quast_per_ref, parallel_run_args, qconfig.max_threads, filter_results=True)
per_ref_num_notifications = map(sum, zip(*ref_notifications))
total_num_notifications = map(
sum, zip(total_num_notifications, per_ref_num_notifications))
if json_texts is not None:
json_texts.extend(ref_json_texts)
quast_py_args.remove('--memory-efficient')
quast_py_args = remove_from_quast_py_args(quast_py_args, '-t',
str(threads_per_ref))
else:
ref_names = []
for ref_fpath, ref_assemblies in assemblies_by_reference:
ref_name, json_text, total_num_notifications = \
_run_quast_per_ref(quast_py_args, output_dirpath_per_ref, ref_fpath, ref_assemblies, total_num_notifications)
if not ref_name:
continue
ref_names.append(ref_name)
if json_texts is not None:
json_texts.append(json_text)
# Finally running for the contigs that has not been aligned to any reference
no_unaligned_contigs = True
for assembly in not_aligned_assemblies:
if os.path.isfile(
assembly.fpath) and os.stat(assembly.fpath).st_size != 0:
no_unaligned_contigs = False
break
run_name = 'for the contigs not aligned anywhere'
logger.main_info()
if no_unaligned_contigs:
logger.main_info('Skipping quast.py ' + run_name +
' (everything is aligned!)')
else:
logger.main_info('Starting quast.py ' + run_name + '... (logging to ' +
os.path.join(output_dirpath, qconfig.not_aligned_name,
qconfig.LOGGER_DEFAULT_NAME + '.log)'))
return_code, total_num_notifications = _start_quast_main(
quast_py_args,
assemblies=not_aligned_assemblies,
output_dirpath=os.path.join(output_dirpath,
qconfig.not_aligned_name),
num_notifications_tuple=total_num_notifications)
if return_code not in [0, 4]:
logger.error(
'Error running quast.py for the contigs not aligned anywhere')
elif return_code == 4: # no unaligned contigs, i.e. everything aligned
no_unaligned_contigs = True
if not no_unaligned_contigs:
if json_texts is not None:
json_texts.append(json_saver.json_text)
if ref_names:
logger.print_timestamp()
logger.main_info("Summarizing results...")
summary_output_dirpath = os.path.join(output_dirpath,
qconfig.meta_summary_dir)
if not os.path.isdir(summary_output_dirpath):
os.makedirs(summary_output_dirpath)
if html_report and json_texts:
from quast_libs.html_saver import html_saver
html_summary_report_fpath = html_saver.init_meta_report(
output_dirpath)
else:
html_summary_report_fpath = None
from quast_libs import create_meta_summary
metrics_for_plots = reporting.Fields.main_metrics
misassembly_metrics = [
reporting.Fields.MIS_RELOCATION,
reporting.Fields.MIS_TRANSLOCATION, reporting.Fields.MIS_INVERTION,
reporting.Fields.MIS_ISTRANSLOCATIONS
]
if no_unaligned_contigs:
full_ref_names = [
qutils.name_from_fpath(ref_fpath)
for ref_fpath in corrected_ref_fpaths
]
else:
full_ref_names = [
qutils.name_from_fpath(ref_fpath)
for ref_fpath in corrected_ref_fpaths
] + [qconfig.not_aligned_name]
create_meta_summary.do(html_summary_report_fpath,
summary_output_dirpath, combined_output_dirpath,
output_dirpath_per_ref, metrics_for_plots,
misassembly_metrics, full_ref_names)
if html_report and json_texts:
html_saver.save_colors(output_dirpath,
contigs_fpaths,
plotter_data.dict_color_and_ls,
meta=True)
if qconfig.create_icarus_html:
icarus_html_fpath = html_saver.create_meta_icarus(
output_dirpath, ref_names)
logger.main_info(' Icarus (contig browser) is saved to %s' %
icarus_html_fpath)
html_saver.create_meta_report(output_dirpath, json_texts)
cleanup(corrected_dirpath)
logger.main_info('')
logger.main_info('MetaQUAST finished.')
return logger.finish_up(numbers=tuple(total_num_notifications),
check_test=test_mode)
def main(args):
check_dirpath(qconfig.QUAST_HOME, 'You are trying to run it from ' + str(qconfig.QUAST_HOME) + '.\n' +
'Please, put QUAST in a different directory, then try again.\n', exit_code=3)
if not args:
qconfig.usage(meta=True)
sys.exit(0)
metaquast_path = [os.path.realpath(__file__)]
quast_py_args, contigs_fpaths = parse_options(logger, metaquast_path + args, is_metaquast=True)
output_dirpath, ref_fpaths, labels = qconfig.output_dirpath, qconfig.reference, qconfig.labels
html_report = qconfig.html_report
test_mode = qconfig.test
# Directories
output_dirpath, _, _ = qutils.set_up_output_dir(
output_dirpath, None, not output_dirpath,
save_json=False)
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
qconfig.set_max_threads(logger)
qutils.logger = logger
########################################################################
from quast_libs import reporting
try:
import imp
imp.reload(reporting)
except:
reload(reporting)
from quast_libs import plotter
if os.path.isdir(corrected_dirpath):
shutil.rmtree(corrected_dirpath)
os.mkdir(corrected_dirpath)
# PROCESSING REFERENCES
if ref_fpaths:
logger.main_info()
logger.main_info('Reference(s):')
corrected_ref_fpaths, combined_ref_fpath, chromosomes_by_refs, ref_names =\
correct_meta_references(ref_fpaths, corrected_dirpath)
# PROCESSING CONTIGS
logger.main_info()
logger.main_info('Contigs:')
qconfig.no_check_meta = True
assemblies, labels = correct_assemblies(contigs_fpaths, output_dirpath, labels)
if not assemblies:
logger.error("None of the assembly files contains correct contigs. "
"Please, provide different files or decrease --min-contig threshold.")
return 4
# Running QUAST(s)
quast_py_args += ['--meta']
downloaded_refs = False
# SEARCHING REFERENCES
if not ref_fpaths:
logger.main_info()
if qconfig.max_references == 0:
logger.notice("Maximum number of references (--max-ref-number) is set to 0, search in SILVA 16S rRNA database is disabled")
else:
if qconfig.references_txt:
logger.main_info("List of references was provided, starting to download reference genomes from NCBI...")
else:
logger.main_info("No references are provided, starting to search for reference genomes in SILVA 16S rRNA database "
"and to download them from NCBI...")
downloaded_dirpath = os.path.join(output_dirpath, qconfig.downloaded_dirname)
if not os.path.isdir(downloaded_dirpath):
os.mkdir(downloaded_dirpath)
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
ref_fpaths = search_references_meta.do(assemblies, labels, downloaded_dirpath, corrected_dirpath, qconfig.references_txt)
if ref_fpaths:
search_references_meta.is_quast_first_run = True
if not qconfig.references_txt:
downloaded_refs = True
logger.main_info()
logger.main_info('Downloaded reference(s):')
corrected_ref_fpaths, combined_ref_fpath, chromosomes_by_refs, ref_names =\
correct_meta_references(ref_fpaths, corrected_dirpath)
elif test_mode and not ref_fpaths:
logger.error('Failed to download or setup SILVA 16S rRNA database for working without '
'references on metagenome datasets!', to_stderr=True, exit_with_code=4)
if not ref_fpaths:
# No references, running regular quast with MetaGenemark gene finder
logger.main_info()
logger.notice('No references are provided, starting regular QUAST with MetaGeneMark gene finder')
_start_quast_main(quast_py_args, assemblies=assemblies, output_dirpath=output_dirpath, run_regular_quast=True)
exit(0)
# Running combined reference
combined_output_dirpath = os.path.join(output_dirpath, qconfig.combined_output_name)
reads_fpaths = []
if qconfig.forward_reads:
reads_fpaths.append(qconfig.forward_reads)
if qconfig.reverse_reads:
reads_fpaths.append(qconfig.reverse_reads)
cov_fpath = qconfig.cov_fpath
physical_cov_fpath = qconfig.phys_cov_fpath
if (reads_fpaths or qconfig.sam or qconfig.bam) and ref_fpaths:
bed_fpath, cov_fpath, physical_cov_fpath = reads_analyzer.do(combined_ref_fpath, contigs_fpaths, reads_fpaths, corrected_ref_fpaths,
os.path.join(combined_output_dirpath, qconfig.variation_dirname),
external_logger=logger, sam_fpath=qconfig.sam, bam_fpath=qconfig.bam, bed_fpath=qconfig.bed)
qconfig.bed = bed_fpath
if qconfig.bed:
quast_py_args += ['--sv-bed']
quast_py_args += [qconfig.bed]
if cov_fpath:
quast_py_args += ['--cov']
quast_py_args += [cov_fpath]
if physical_cov_fpath:
quast_py_args += ['--phys-cov']
quast_py_args += [physical_cov_fpath]
if qconfig.sam:
quast_py_args += ['--sam']
quast_py_args += [qconfig.sam]
if qconfig.bam:
quast_py_args += ['--bam']
quast_py_args += [qconfig.bam]
quast_py_args += ['--combined-ref']
if qconfig.draw_plots or qconfig.html_report:
if plotter.dict_color_and_ls:
colors_and_ls = [plotter.dict_color_and_ls[asm.label] for asm in assemblies]
quast_py_args += ['--colors']
quast_py_args += [','.join([style[0] for style in colors_and_ls])]
quast_py_args += ['--ls']
quast_py_args += [','.join([style[1] for style in colors_and_ls])]
run_name = 'for the combined reference'
logger.main_info()
logger.main_info('Starting quast.py ' + run_name + '...')
total_num_notices = 0
total_num_warnings = 0
total_num_nf_errors = 0
total_num_notifications = (total_num_notices, total_num_warnings, total_num_nf_errors)
if qconfig.html_report:
from quast_libs.html_saver import json_saver
json_texts = []
else:
json_texts = None
if qconfig.unique_mapping:
ambiguity_opts = []
else:
ambiguity_opts = ["--ambiguity-usage", 'all']
return_code, total_num_notifications, assemblies, labels = \
_start_quast_main(quast_py_args + ambiguity_opts,
assemblies=assemblies,
reference_fpath=combined_ref_fpath,
output_dirpath=combined_output_dirpath,
num_notifications_tuple=total_num_notifications, is_first_run=True)
if json_texts is not None:
json_texts.append(json_saver.json_text)
search_references_meta.is_quast_first_run = False
genome_info_dirpath = os.path.join(output_dirpath, qconfig.combined_output_name, 'genome_stats')
genome_info_fpath = os.path.join(genome_info_dirpath, 'genome_info.txt')
if not os.path.exists(genome_info_fpath):
logger.main_info('')
if not downloaded_refs:
msg = 'Try to restart MetaQUAST with another references.'
else:
msg = 'Try to use option --max-ref-number to change maximum number of references (per each assembly) to download.'
logger.main_info('Failed aligning the contigs for all the references. ' + msg)
logger.main_info('')
cleanup(corrected_dirpath)
logger.main_info('MetaQUAST finished.')
return logger.finish_up(numbers=tuple(total_num_notifications), check_test=test_mode)
if downloaded_refs:
logger.main_info()
logger.main_info('Excluding downloaded references with low genome fraction from further analysis..')
corr_ref_fpaths = get_downloaded_refs_with_alignments(genome_info_fpath, ref_fpaths, chromosomes_by_refs)
if corr_ref_fpaths and corr_ref_fpaths != ref_fpaths:
logger.main_info()
logger.main_info('Filtered reference(s):')
os.remove(combined_ref_fpath)
contigs_analyzer.ref_labels_by_chromosomes = {}
corrected_ref_fpaths, combined_ref_fpath, chromosomes_by_refs, ref_names = \
correct_meta_references(corr_ref_fpaths, corrected_dirpath)
run_name = 'for the corrected combined reference'
logger.main_info()
logger.main_info('Starting quast.py ' + run_name + '...')
return_code, total_num_notifications, assemblies, labels = \
_start_quast_main(quast_py_args + ambiguity_opts,
assemblies=assemblies,
reference_fpath=combined_ref_fpath,
output_dirpath=combined_output_dirpath,
num_notifications_tuple=total_num_notifications, is_first_run=True)
if json_texts is not None:
json_texts = json_texts[:-1]
json_texts.append(json_saver.json_text)
elif corr_ref_fpaths == ref_fpaths:
logger.main_info('All downloaded references have genome fraction more than 10%. Nothing was excluded.')
else:
logger.main_info('All downloaded references have low genome fraction. Nothing was excluded for now.')
if qconfig.calculate_read_support:
calculate_ave_read_support(combined_output_dirpath, assemblies)
for arg in args:
if arg in ('-s', "--scaffolds"):
quast_py_args.remove(arg)
quast_py_args += ['--no-check-meta']
qconfig.contig_thresholds = ','.join([str(threshold) for threshold in qconfig.contig_thresholds if threshold > qconfig.min_contig])
if not qconfig.contig_thresholds:
qconfig.contig_thresholds = 'None'
quast_py_args = remove_from_quast_py_args(quast_py_args, '--contig-thresholds', qconfig.contig_thresholds)
quast_py_args += ['--contig-thresholds']
quast_py_args += [qconfig.contig_thresholds]
quast_py_args.remove('--combined-ref')
logger.main_info()
logger.main_info('Partitioning contigs into bins aligned to each reference..')
assemblies_by_reference, not_aligned_assemblies = partition_contigs(
assemblies, corrected_ref_fpaths, corrected_dirpath,
os.path.join(combined_output_dirpath, 'contigs_reports', 'alignments_%s.tsv'), labels)
ref_names = []
output_dirpath_per_ref = os.path.join(output_dirpath, qconfig.per_ref_dirname)
for ref_fpath, ref_assemblies in assemblies_by_reference:
ref_name = qutils.name_from_fpath(ref_fpath)
logger.main_info('')
if not ref_assemblies:
logger.main_info('No contigs were aligned to the reference ' + ref_name + ', skipping..')
else:
ref_names.append(ref_name)
run_name = 'for the contigs aligned to ' + ref_name
logger.main_info('Starting quast.py ' + run_name)
return_code, total_num_notifications = _start_quast_main(quast_py_args,
assemblies=ref_assemblies,
reference_fpath=ref_fpath,
output_dirpath=os.path.join(output_dirpath_per_ref, ref_name),
num_notifications_tuple=total_num_notifications)
if json_texts is not None:
json_texts.append(json_saver.json_text)
# Finally running for the contigs that has not been aligned to any reference
no_unaligned_contigs = True
for assembly in not_aligned_assemblies:
if os.path.isfile(assembly.fpath) and os.stat(assembly.fpath).st_size != 0:
no_unaligned_contigs = False
break
run_name = 'for the contigs not aligned anywhere'
logger.main_info()
if no_unaligned_contigs:
logger.main_info('Skipping quast.py ' + run_name + ' (everything is aligned!)')
else:
logger.main_info('Starting quast.py ' + run_name + '...')
return_code, total_num_notifications = _start_quast_main(quast_py_args,
assemblies=not_aligned_assemblies,
output_dirpath=os.path.join(output_dirpath, qconfig.not_aligned_name),
num_notifications_tuple=total_num_notifications)
if return_code not in [0, 4]:
logger.error('Error running quast.py for the contigs not aligned anywhere')
elif return_code == 4: # no unaligned contigs, i.e. everything aligned
no_unaligned_contigs = True
if not no_unaligned_contigs:
if json_texts is not None:
json_texts.append(json_saver.json_text)
if ref_names:
logger.print_timestamp()
logger.main_info("Summarizing results...")
summary_output_dirpath = os.path.join(output_dirpath, qconfig.meta_summary_dir)
if not os.path.isdir(summary_output_dirpath):
os.makedirs(summary_output_dirpath)
if html_report and json_texts:
from quast_libs.html_saver import html_saver
html_summary_report_fpath = html_saver.init_meta_report(output_dirpath)
else:
html_summary_report_fpath = None
from quast_libs import create_meta_summary
metrics_for_plots = reporting.Fields.main_metrics
misassembl_metrics = [reporting.Fields.MIS_RELOCATION, reporting.Fields.MIS_TRANSLOCATION, reporting.Fields.MIS_INVERTION,
reporting.Fields.MIS_ISTRANSLOCATIONS]
if no_unaligned_contigs:
full_ref_names = ref_names
else:
full_ref_names = ref_names + [qconfig.not_aligned_name]
create_meta_summary.do(html_summary_report_fpath, summary_output_dirpath, combined_output_dirpath,
output_dirpath_per_ref, metrics_for_plots, misassembl_metrics, full_ref_names)
if html_report and json_texts:
html_saver.save_colors(output_dirpath, contigs_fpaths, plotter.dict_color_and_ls, meta=True)
if qconfig.create_icarus_html:
icarus_html_fpath = html_saver.create_meta_icarus(output_dirpath, ref_names)
logger.main_info(' Icarus (contig browser) is saved to %s' % icarus_html_fpath)
html_saver.create_meta_report(output_dirpath, json_texts)
cleanup(corrected_dirpath)
logger.main_info('')
logger.main_info('MetaQUAST finished.')
return logger.finish_up(numbers=tuple(total_num_notifications), check_test=test_mode)
def main(args):
check_dirpath(
qconfig.QUAST_HOME,
'You are trying to run it from ' + str(qconfig.QUAST_HOME) + '\n.' +
'Please, put QUAST in a different directory, then try again.\n',
exit_code=3)
if not args:
qconfig.usage(stream=sys.stderr)
sys.exit(1)
try:
import imp
imp.reload(qconfig)
imp.reload(qutils)
except:
reload(qconfig)
reload(qutils)
try:
locale.setlocale(locale.LC_ALL, 'en_US.utf8')
except Exception:
try:
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
except Exception:
logger.warning('Python locale settings can\'t be changed')
quast_path = [os.path.realpath(__file__)]
quast_py_args, contigs_fpaths = parse_options(logger, quast_path + args)
output_dirpath, ref_fpath, labels = qconfig.output_dirpath, qconfig.reference, qconfig.labels
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
logger.main_info()
logger.print_params()
########################################################################
from quast_libs import reporting
reports = reporting.reports
try:
import imp
imp.reload(reporting)
except:
reload(reporting)
reporting.reports = reports
reporting.assembly_fpaths = []
from quast_libs import plotter # Do not remove this line! It would lead to a warning in matplotlib.
if qconfig.is_combined_ref:
corrected_dirpath = os.path.join(output_dirpath, '..',
qconfig.corrected_dirname)
else:
if os.path.isdir(corrected_dirpath):
shutil.rmtree(corrected_dirpath)
os.mkdir(corrected_dirpath)
qconfig.set_max_threads(logger)
check_reads_fpaths(logger)
# PROCESSING REFERENCE
if ref_fpath:
logger.main_info()
logger.main_info('Reference:')
original_ref_fpath = ref_fpath
ref_fpath = qutils.correct_reference(ref_fpath, corrected_dirpath)
if qconfig.optimal_assembly:
if not qconfig.pacbio_reads and not qconfig.nanopore_reads and not qconfig.mate_pairs:
logger.warning(
'Optimal assembly cannot be created. It requires mate-pairs or long reads (Pacbio SMRT or Oxford Nanopore).'
)
else:
optimal_assembly_fpath = optimal_assembly.do(
ref_fpath, original_ref_fpath,
os.path.join(output_dirpath,
qconfig.optimal_assembly_basename))
if optimal_assembly_fpath is not None:
contigs_fpaths.insert(0, optimal_assembly_fpath)
labels.insert(0, 'Optimal')
labels = qutils.process_labels(contigs_fpaths, labels)
else:
ref_fpath = ''
# PROCESSING CONTIGS
logger.main_info()
logger.main_info('Contigs:')
contigs_fpaths, old_contigs_fpaths = qutils.correct_contigs(
contigs_fpaths, corrected_dirpath, labels, reporting)
for contigs_fpath in contigs_fpaths:
report = reporting.get(contigs_fpath)
report.add_field(reporting.Fields.NAME,
qutils.label_from_fpath(contigs_fpath))
qconfig.assemblies_num = len(contigs_fpaths)
cov_fpath = qconfig.cov_fpath
physical_cov_fpath = qconfig.phys_cov_fpath
if qconfig.reads_fpaths or qconfig.reference_sam or qconfig.reference_sam or qconfig.sam_fpaths or qconfig.bam_fpaths:
bed_fpath, cov_fpath, physical_cov_fpath = reads_analyzer.do(
ref_fpath,
contigs_fpaths,
os.path.join(output_dirpath, qconfig.reads_stats_dirname),
external_logger=logger)
qconfig.bed = bed_fpath
if not contigs_fpaths:
logger.error(
"None of the assembly files contains correct contigs. "
"Please, provide different files or decrease --min-contig threshold.",
fake_if_nested_run=True)
return 4
if qconfig.used_colors and qconfig.used_ls:
for i, label in enumerate(labels):
plotter_data.dict_color_and_ls[label] = (qconfig.used_colors[i],
qconfig.used_ls[i])
qconfig.assemblies_fpaths = contigs_fpaths
# Where all pdfs will be saved
all_pdf_fpath = None
if qconfig.draw_plots and plotter.can_draw_plots:
all_pdf_fpath = os.path.join(output_dirpath, qconfig.plots_fname)
if qconfig.json_output_dirpath:
from quast_libs.html_saver import json_saver
if json_saver.simplejson_error:
qconfig.json_output_dirpath = None
########################################################################
### Stats and plots
########################################################################
from quast_libs import basic_stats
icarus_gc_fpath, circos_gc_fpath = basic_stats.do(
ref_fpath, contigs_fpaths, os.path.join(output_dirpath, 'basic_stats'),
output_dirpath)
if qconfig.large_genome and ref_fpath:
unique_kmers.do(os.path.join(output_dirpath, 'basic_stats'), ref_fpath,
contigs_fpaths, logger)
aligned_contigs_fpaths = []
aligned_lengths_lists = []
contig_alignment_plot_fpath = None
icarus_html_fpath = None
circos_png_fpath = None
if ref_fpath:
########################################################################
### former PLANTAKOLYA, PLANTAGORA
########################################################################
from quast_libs import contigs_analyzer
is_cyclic = qconfig.prokaryote and not qconfig.check_for_fragmented_ref
aligner_statuses, aligned_lengths_per_fpath = contigs_analyzer.do(
ref_fpath, contigs_fpaths, is_cyclic,
os.path.join(output_dirpath, 'contigs_reports'),
old_contigs_fpaths, qconfig.bed)
for contigs_fpath in contigs_fpaths:
if aligner_statuses[
contigs_fpath] == contigs_analyzer.AlignerStatus.OK:
aligned_contigs_fpaths.append(contigs_fpath)
aligned_lengths_lists.append(
aligned_lengths_per_fpath[contigs_fpath])
# Before continue evaluating, check if aligner didn't skip all of the contigs files.
detailed_contigs_reports_dirpath = None
features_containers = None
if len(aligned_contigs_fpaths) and ref_fpath:
detailed_contigs_reports_dirpath = os.path.join(
output_dirpath, 'contigs_reports')
########################################################################
### NAx and NGAx ("aligned Nx and NGx")
########################################################################
from quast_libs import aligned_stats
aligned_stats.do(ref_fpath, aligned_contigs_fpaths, output_dirpath,
aligned_lengths_lists,
os.path.join(output_dirpath, 'aligned_stats'))
########################################################################
### GENOME_ANALYZER
########################################################################
from quast_libs import genome_analyzer
features_containers = genome_analyzer.do(
ref_fpath, aligned_contigs_fpaths, output_dirpath,
qconfig.features, qconfig.operons,
detailed_contigs_reports_dirpath,
os.path.join(output_dirpath, 'genome_stats'))
genes_by_labels = None
if qconfig.gene_finding:
if qconfig.glimmer:
########################################################################
### Glimmer
########################################################################
from quast_libs import glimmer
genes_by_labels = glimmer.do(
contigs_fpaths, qconfig.genes_lengths,
os.path.join(output_dirpath, 'predicted_genes'))
if not qconfig.glimmer or qconfig.test:
########################################################################
### GeneMark
########################################################################
from quast_libs import genemark
genes_by_labels = genemark.do(
contigs_fpaths, qconfig.genes_lengths,
os.path.join(output_dirpath, 'predicted_genes'),
qconfig.prokaryote, qconfig.metagenemark)
else:
logger.main_info("")
logger.notice(
"Genes are not predicted by default. Use --gene-finding option to enable it."
)
if qconfig.rna_gene_finding:
run_barrnap.do(contigs_fpaths,
os.path.join(output_dirpath, 'predicted_genes'), logger)
if qconfig.run_busco and not qconfig.is_combined_ref:
if qconfig.platform_name == 'macosx':
logger.main_info("")
logger.warning("BUSCO can be run on Linux only")
elif sys.version[0:3] == '2.5':
logger.main_info("")
logger.warning(
"BUSCO does not support Python versions older than 2.6.")
else:
from quast_libs import run_busco
run_busco.do(contigs_fpaths,
os.path.join(output_dirpath, qconfig.busco_dirname),
logger)
########################################################################
reports_fpaths, transposed_reports_fpaths = reporting.save_total(
output_dirpath)
########################################################################
### LARGE DRAWING TASKS
########################################################################
if qconfig.draw_plots or qconfig.create_icarus_html:
logger.print_timestamp()
logger.main_info('Creating large visual summaries...')
logger.main_info(
'This may take a while: press Ctrl-C to skip this step..')
try:
if detailed_contigs_reports_dirpath:
report_for_icarus_fpath_pattern = os.path.join(
detailed_contigs_reports_dirpath,
qconfig.icarus_report_fname_pattern)
stdout_pattern = os.path.join(
detailed_contigs_reports_dirpath,
qconfig.contig_report_fname_pattern)
else:
report_for_icarus_fpath_pattern = None
stdout_pattern = None
draw_alignment_plots = qconfig.draw_svg or qconfig.create_icarus_html
draw_circos_plot = qconfig.draw_plots and ref_fpath and len(
aligned_contigs_fpaths) and not qconfig.space_efficient
number_of_steps = sum([
int(bool(value)) for value in
[draw_alignment_plots, draw_circos_plot, all_pdf_fpath]
])
if draw_alignment_plots:
########################################################################
### VISUALIZE CONTIG ALIGNMENT
########################################################################
logger.main_info(' 1 of %d: Creating Icarus viewers...' %
number_of_steps)
from quast_libs import icarus
icarus_html_fpath, contig_alignment_plot_fpath = icarus.do(
contigs_fpaths,
report_for_icarus_fpath_pattern,
output_dirpath,
ref_fpath,
stdout_pattern=stdout_pattern,
features=features_containers,
cov_fpath=cov_fpath,
physical_cov_fpath=physical_cov_fpath,
gc_fpath=icarus_gc_fpath,
json_output_dir=qconfig.json_output_dirpath,
genes_by_labels=genes_by_labels)
if draw_circos_plot:
logger.main_info(
' %d of %d: Creating Circos plots...' %
(2 if draw_alignment_plots else 1, number_of_steps))
from quast_libs import circos
circos_png_fpath, circos_legend_fpath = circos.do(
ref_fpath, contigs_fpaths, report_for_icarus_fpath_pattern,
circos_gc_fpath, features_containers, cov_fpath,
os.path.join(output_dirpath, 'circos'), logger)
if all_pdf_fpath:
# full report in PDF format: all tables and plots
logger.main_info(
' %d of %d: Creating PDF with all tables and plots...' %
(number_of_steps, number_of_steps))
plotter.fill_all_pdf_file(all_pdf_fpath)
logger.main_info('Done')
except KeyboardInterrupt:
logger.main_info('..step skipped!')
if all_pdf_fpath and os.path.isfile(all_pdf_fpath):
os.remove(all_pdf_fpath)
########################################################################
### TOTAL REPORT
########################################################################
logger.print_timestamp()
logger.main_info('RESULTS:')
logger.main_info(' Text versions of total report are saved to ' +
reports_fpaths)
logger.main_info(
' Text versions of transposed total report are saved to ' +
transposed_reports_fpaths)
if qconfig.html_report:
from quast_libs.html_saver import html_saver
html_saver.save_colors(output_dirpath, contigs_fpaths,
plotter_data.dict_color_and_ls)
html_saver.save_total_report(output_dirpath, qconfig.min_contig,
ref_fpath)
if all_pdf_fpath and os.path.isfile(all_pdf_fpath):
logger.main_info(' PDF version (tables and plots) is saved to ' +
all_pdf_fpath)
if circos_png_fpath:
logger.main_info(
' Circos plot is saved to %s (the annotation is in %s). Circos configuration file is saved to %s'
% (circos_png_fpath, circos_legend_fpath,
circos_png_fpath.replace('.png', '.conf')))
if icarus_html_fpath:
logger.main_info(' Icarus (contig browser) is saved to %s' %
icarus_html_fpath)
if qconfig.draw_svg and contig_alignment_plot_fpath:
logger.main_info(' Contig alignment plot is saved to %s' %
contig_alignment_plot_fpath)
cleanup(corrected_dirpath)
return logger.finish_up(check_test=qconfig.test)
def main(args):
check_dirpath(qconfig.QUAST_HOME, 'You are trying to run it from ' + str(qconfig.QUAST_HOME) + '\n.' +
'Please, put QUAST in a different directory, then try again.\n', exit_code=3)
if not args:
qconfig.usage(stream=sys.stderr)
sys.exit(1)
try:
import imp
imp.reload(qconfig)
imp.reload(qutils)
except:
reload(qconfig)
reload(qutils)
try:
locale.setlocale(locale.LC_ALL, 'en_US.utf8')
except Exception:
try:
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
except Exception:
logger.warning('Python locale settings can\'t be changed')
quast_path = [os.path.realpath(__file__)]
quast_py_args, contigs_fpaths = parse_options(logger, quast_path + args)
output_dirpath, ref_fpath, labels = qconfig.output_dirpath, qconfig.reference, qconfig.labels
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
logger.main_info()
logger.print_params()
########################################################################
from quast_libs import reporting
reports = reporting.reports
try:
import imp
imp.reload(reporting)
except:
reload(reporting)
reporting.reports = reports
reporting.assembly_fpaths = []
from quast_libs import plotter # Do not remove this line! It would lead to a warning in matplotlib.
if qconfig.is_combined_ref:
corrected_dirpath = os.path.join(output_dirpath, '..', qconfig.corrected_dirname)
else:
if os.path.isdir(corrected_dirpath):
shutil.rmtree(corrected_dirpath)
os.mkdir(corrected_dirpath)
qconfig.set_max_threads(logger)
check_reads_fpaths(logger)
# PROCESSING REFERENCE
if ref_fpath:
logger.main_info()
logger.main_info('Reference:')
original_ref_fpath = ref_fpath
ref_fpath = qutils.correct_reference(ref_fpath, corrected_dirpath)
if qconfig.ideal_assembly:
ideal_assembly_fpath = ideal_assembly.do(ref_fpath, original_ref_fpath,
os.path.join(output_dirpath, qconfig.ideal_assembly_basename))
if ideal_assembly_fpath is not None:
contigs_fpaths.insert(0, ideal_assembly_fpath)
labels.insert(0, 'IDEAL ASSEMBLY')
labels = qutils.process_labels(contigs_fpaths, labels)
else:
ref_fpath = ''
# PROCESSING CONTIGS
logger.main_info()
logger.main_info('Contigs:')
contigs_fpaths, old_contigs_fpaths = qutils.correct_contigs(contigs_fpaths, corrected_dirpath, labels, reporting)
for contigs_fpath in contigs_fpaths:
report = reporting.get(contigs_fpath)
report.add_field(reporting.Fields.NAME, qutils.label_from_fpath(contigs_fpath))
qconfig.assemblies_num = len(contigs_fpaths)
cov_fpath = qconfig.cov_fpath
physical_cov_fpath = qconfig.phys_cov_fpath
if qconfig.reads_fpaths or qconfig.reference_sam or qconfig.reference_sam or qconfig.sam_fpaths or qconfig.bam_fpaths:
bed_fpath, cov_fpath, physical_cov_fpath = reads_analyzer.do(ref_fpath, contigs_fpaths,
os.path.join(output_dirpath, qconfig.reads_stats_dirname),
external_logger=logger)
qconfig.bed = bed_fpath
if not contigs_fpaths:
logger.error("None of the assembly files contains correct contigs. "
"Please, provide different files or decrease --min-contig threshold.",
fake_if_nested_run=True)
return 4
if qconfig.used_colors and qconfig.used_ls:
for i, label in enumerate(labels):
plotter_data.dict_color_and_ls[label] = (qconfig.used_colors[i], qconfig.used_ls[i])
qconfig.assemblies_fpaths = contigs_fpaths
# Where all pdfs will be saved
all_pdf_fpath = None
if qconfig.draw_plots and plotter.can_draw_plots:
all_pdf_fpath = os.path.join(output_dirpath, qconfig.plots_fname)
if qconfig.json_output_dirpath:
from quast_libs.html_saver import json_saver
if json_saver.simplejson_error:
qconfig.json_output_dirpath = None
########################################################################
### Stats and plots
########################################################################
from quast_libs import basic_stats
icarus_gc_fpath, circos_gc_fpath = basic_stats.do(ref_fpath, contigs_fpaths, os.path.join(output_dirpath, 'basic_stats'), output_dirpath)
if qconfig.large_genome and ref_fpath:
unique_kmers.do(os.path.join(output_dirpath, 'basic_stats'), ref_fpath, contigs_fpaths, logger)
aligned_contigs_fpaths = []
aligned_lengths_lists = []
contig_alignment_plot_fpath = None
icarus_html_fpath = None
circos_png_fpath = None
if ref_fpath:
########################################################################
### former PLANTAKOLYA, PLANTAGORA
########################################################################
from quast_libs import contigs_analyzer
is_cyclic = qconfig.prokaryote and not qconfig.check_for_fragmented_ref
nucmer_statuses, aligned_lengths_per_fpath = contigs_analyzer.do(
ref_fpath, contigs_fpaths, is_cyclic, os.path.join(output_dirpath, 'contigs_reports'),
old_contigs_fpaths, qconfig.bed)
for contigs_fpath in contigs_fpaths:
if nucmer_statuses[contigs_fpath] == contigs_analyzer.NucmerStatus.OK:
aligned_contigs_fpaths.append(contigs_fpath)
aligned_lengths_lists.append(aligned_lengths_per_fpath[contigs_fpath])
# Before continue evaluating, check if nucmer didn't skip all of the contigs files.
detailed_contigs_reports_dirpath = None
features_containers = None
if len(aligned_contigs_fpaths) and ref_fpath:
detailed_contigs_reports_dirpath = os.path.join(output_dirpath, 'contigs_reports')
########################################################################
### NAx and NGAx ("aligned Nx and NGx")
########################################################################
from quast_libs import aligned_stats
aligned_stats.do(
ref_fpath, aligned_contigs_fpaths, output_dirpath,
aligned_lengths_lists, os.path.join(output_dirpath, 'aligned_stats'))
########################################################################
### GENOME_ANALYZER
########################################################################
from quast_libs import genome_analyzer
features_containers = genome_analyzer.do(
ref_fpath, aligned_contigs_fpaths, output_dirpath,
qconfig.genes, qconfig.operons, detailed_contigs_reports_dirpath,
os.path.join(output_dirpath, 'genome_stats'))
genes_by_labels = None
if qconfig.gene_finding:
if qconfig.glimmer:
########################################################################
### Glimmer
########################################################################
from quast_libs import glimmer
genes_by_labels = glimmer.do(contigs_fpaths, qconfig.genes_lengths, os.path.join(output_dirpath, 'predicted_genes'))
if not qconfig.glimmer or qconfig.test:
########################################################################
### GeneMark
########################################################################
from quast_libs import genemark
genes_by_labels = genemark.do(contigs_fpaths, qconfig.genes_lengths, os.path.join(output_dirpath, 'predicted_genes'),
qconfig.prokaryote, qconfig.metagenemark)
else:
logger.main_info("")
logger.notice("Genes are not predicted by default. Use --gene-finding option to enable it.")
if qconfig.rna_gene_finding:
run_barrnap.do(contigs_fpaths, os.path.join(output_dirpath, 'predicted_genes'), logger)
if qconfig.run_busco and not qconfig.is_combined_ref:
if qconfig.platform_name == 'macosx':
logger.main_info("")
logger.warning("BUSCO can be run on Linux only")
elif sys.version[0:3] == '2.5':
logger.main_info("")
logger.warning("BUSCO does not support Python versions older than 2.6.")
else:
from quast_libs import run_busco
run_busco.do(contigs_fpaths, os.path.join(output_dirpath, qconfig.busco_dirname), logger)
########################################################################
reports_fpaths, transposed_reports_fpaths = reporting.save_total(output_dirpath)
########################################################################
### LARGE DRAWING TASKS
########################################################################
if qconfig.draw_plots or qconfig.create_icarus_html:
logger.print_timestamp()
logger.main_info('Creating large visual summaries...')
logger.main_info('This may take a while: press Ctrl-C to skip this step..')
try:
if detailed_contigs_reports_dirpath:
report_for_icarus_fpath_pattern = os.path.join(detailed_contigs_reports_dirpath, qconfig.icarus_report_fname_pattern)
stdout_pattern = os.path.join(detailed_contigs_reports_dirpath, qconfig.contig_report_fname_pattern)
else:
report_for_icarus_fpath_pattern = None
stdout_pattern = None
draw_alignment_plots = qconfig.draw_svg or qconfig.create_icarus_html
draw_circos_plot = qconfig.draw_plots and ref_fpath and len(aligned_contigs_fpaths) and not qconfig.space_efficient
number_of_steps = sum([int(bool(value)) for value in [draw_alignment_plots, draw_circos_plot, all_pdf_fpath]])
if draw_alignment_plots:
########################################################################
### VISUALIZE CONTIG ALIGNMENT
########################################################################
logger.main_info(' 1 of %d: Creating Icarus viewers...' % number_of_steps)
from quast_libs import icarus
icarus_html_fpath, contig_alignment_plot_fpath = icarus.do(
contigs_fpaths, report_for_icarus_fpath_pattern, output_dirpath, ref_fpath,
stdout_pattern=stdout_pattern, features=features_containers,
cov_fpath=cov_fpath, physical_cov_fpath=physical_cov_fpath, gc_fpath=icarus_gc_fpath,
json_output_dir=qconfig.json_output_dirpath, genes_by_labels=genes_by_labels)
if draw_circos_plot:
logger.main_info(' %d of %d: Creating Circos plots...' % (2 if draw_alignment_plots else 1, number_of_steps))
from quast_libs import circos
circos_png_fpath, circos_legend_fpath = circos.do(ref_fpath, contigs_fpaths, report_for_icarus_fpath_pattern, circos_gc_fpath,
features_containers, cov_fpath, os.path.join(output_dirpath, 'circos'), logger)
if all_pdf_fpath:
# full report in PDF format: all tables and plots
logger.main_info(' %d of %d: Creating PDF with all tables and plots...' % (number_of_steps, number_of_steps))
plotter.fill_all_pdf_file(all_pdf_fpath)
logger.main_info('Done')
except KeyboardInterrupt:
logger.main_info('..step skipped!')
if all_pdf_fpath and os.path.isfile(all_pdf_fpath):
os.remove(all_pdf_fpath)
########################################################################
### TOTAL REPORT
########################################################################
logger.print_timestamp()
logger.main_info('RESULTS:')
logger.main_info(' Text versions of total report are saved to ' + reports_fpaths)
logger.main_info(' Text versions of transposed total report are saved to ' + transposed_reports_fpaths)
if qconfig.html_report:
from quast_libs.html_saver import html_saver
html_saver.save_colors(output_dirpath, contigs_fpaths, plotter_data.dict_color_and_ls)
html_saver.save_total_report(output_dirpath, qconfig.min_contig, ref_fpath)
if all_pdf_fpath and os.path.isfile(all_pdf_fpath):
logger.main_info(' PDF version (tables and plots) is saved to ' + all_pdf_fpath)
if circos_png_fpath:
logger.main_info(' Circos plot is saved to %s (the annotation is in %s). Circos configuration file is saved to %s' %
(circos_png_fpath, circos_legend_fpath, circos_png_fpath.replace('.png', '.conf')))
if icarus_html_fpath:
logger.main_info(' Icarus (contig browser) is saved to %s' % icarus_html_fpath)
if qconfig.draw_svg and contig_alignment_plot_fpath:
logger.main_info(' Contig alignment plot is saved to %s' % contig_alignment_plot_fpath)
cleanup(corrected_dirpath)
return logger.finish_up(check_test=qconfig.test)
def main(args):
if ' ' in qconfig.QUAST_HOME:
logger.error('QUAST does not support spaces in paths. \n'
'You are trying to run it from ' + str(qconfig.QUAST_HOME) + '\n'
'Please, put QUAST in a different directory, then try again.\n',
to_stderr=True,
exit_with_code=3)
if not args:
qconfig.usage()
sys.exit(0)
reload(qconfig)
quast_path = [os.path.realpath(__file__)]
quast_py_args, contigs_fpaths = parse_options(logger, quast_path + args)
output_dirpath, ref_fpath, labels = qconfig.output_dirpath, qconfig.reference, qconfig.labels
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
logger.main_info()
logger.print_params()
########################################################################
from quast_libs import reporting
reports = reporting.reports
reload(reporting)
reporting.reports = reports
reporting.assembly_fpaths = []
from quast_libs import plotter # Do not remove this line! It would lead to a warning in matplotlib.
if qconfig.is_combined_ref:
corrected_dirpath = os.path.join(output_dirpath, '..', qconfig.corrected_dirname)
else:
if os.path.isdir(corrected_dirpath):
shutil.rmtree(corrected_dirpath)
os.mkdir(corrected_dirpath)
# PROCESSING REFERENCE
if ref_fpath:
logger.main_info()
logger.main_info('Reference:')
ref_fpath = qutils.correct_reference(ref_fpath, corrected_dirpath)
else:
ref_fpath = ''
# PROCESSING CONTIGS
logger.main_info()
logger.main_info('Contigs:')
contigs_fpaths, old_contigs_fpaths = qutils.correct_contigs(contigs_fpaths, corrected_dirpath, labels, reporting)
for contigs_fpath in contigs_fpaths:
report = reporting.get(contigs_fpath)
report.add_field(reporting.Fields.NAME, qutils.label_from_fpath(contigs_fpath))
qconfig.assemblies_num = len(contigs_fpaths)
reads_fpaths = []
cov_fpath = []
physical_cov_fpath = []
if qconfig.forward_reads:
reads_fpaths.append(qconfig.forward_reads)
if qconfig.reverse_reads:
reads_fpaths.append(qconfig.reverse_reads)
if (reads_fpaths or qconfig.sam or qconfig.bam) and ref_fpath:
bed_fpath, cov_fpath, physical_cov_fpath = reads_analyzer.do(ref_fpath, contigs_fpaths, reads_fpaths, None,
os.path.join(output_dirpath, qconfig.variation_dirname),
external_logger=logger, sam_fpath=qconfig.sam, bam_fpath=qconfig.bam, bed_fpath=qconfig.bed)
qconfig.bed = bed_fpath
if not contigs_fpaths:
logger.error("None of the assembly files contains correct contigs. "
"Please, provide different files or decrease --min-contig threshold.",
fake_if_nested_run=True)
return 4
if qconfig.used_colors and qconfig.used_ls:
for i, label in enumerate(labels):
plotter.dict_color_and_ls[label] = (qconfig.used_colors[i], qconfig.used_ls[i])
qconfig.assemblies_fpaths = contigs_fpaths
if qconfig.with_gage:
########################################################################
### GAGE
########################################################################
if not ref_fpath:
logger.warning("GAGE can't be run without a reference and will be skipped.")
else:
from quast_libs import gage
gage.do(ref_fpath, contigs_fpaths, output_dirpath)
# Where all pdfs will be saved
all_pdf_fpath = os.path.join(output_dirpath, qconfig.plots_fname)
all_pdf_file = None
if qconfig.draw_plots and plotter.can_draw_plots:
try:
from matplotlib.backends.backend_pdf import PdfPages
all_pdf_file = PdfPages(all_pdf_fpath)
except:
all_pdf_file = None
if qconfig.json_output_dirpath:
from quast_libs.html_saver import json_saver
if json_saver.simplejson_error:
json_output_dirpath = None
########################################################################
### Stats and plots
########################################################################
from quast_libs import basic_stats
basic_stats.do(ref_fpath, contigs_fpaths, os.path.join(output_dirpath, 'basic_stats'),
qconfig.json_output_dirpath, output_dirpath)
aligned_contigs_fpaths = []
aligned_lengths_lists = []
contig_alignment_plot_fpath = None
icarus_html_fpath = None
if ref_fpath:
########################################################################
### former PLANTAKOLYA, PLANTAGORA
########################################################################
from quast_libs import contigs_analyzer
nucmer_statuses, aligned_lengths_per_fpath = contigs_analyzer.do(
ref_fpath, contigs_fpaths, qconfig.prokaryote, os.path.join(output_dirpath, 'contigs_reports'),
old_contigs_fpaths, qconfig.bed)
for contigs_fpath in contigs_fpaths:
if nucmer_statuses[contigs_fpath] == contigs_analyzer.NucmerStatus.OK:
aligned_contigs_fpaths.append(contigs_fpath)
aligned_lengths_lists.append(aligned_lengths_per_fpath[contigs_fpath])
# Before continue evaluating, check if nucmer didn't skip all of the contigs files.
detailed_contigs_reports_dirpath = None
features_containers = None
if len(aligned_contigs_fpaths) and ref_fpath:
detailed_contigs_reports_dirpath = os.path.join(output_dirpath, 'contigs_reports')
########################################################################
### NAx and NGAx ("aligned Nx and NGx")
########################################################################
from quast_libs import aligned_stats
aligned_stats.do(
ref_fpath, aligned_contigs_fpaths, output_dirpath, qconfig.json_output_dirpath,
aligned_lengths_lists, os.path.join(output_dirpath, 'aligned_stats'))
########################################################################
### GENOME_ANALYZER
########################################################################
from quast_libs import genome_analyzer
features_containers = genome_analyzer.do(
ref_fpath, aligned_contigs_fpaths, output_dirpath, qconfig.json_output_dirpath,
qconfig.genes, qconfig.operons, detailed_contigs_reports_dirpath,
os.path.join(output_dirpath, 'genome_stats'))
genes_by_labels = None
if qconfig.gene_finding:
if qconfig.glimmer:
########################################################################
### Glimmer
########################################################################
from quast_libs import glimmer
genes_by_labels = glimmer.do(contigs_fpaths, qconfig.genes_lengths, os.path.join(output_dirpath, 'predicted_genes'))
else:
########################################################################
### GeneMark
########################################################################
from quast_libs import genemark
genes_by_labels = genemark.do(contigs_fpaths, qconfig.genes_lengths, os.path.join(output_dirpath, 'predicted_genes'),
qconfig.prokaryote, qconfig.meta)
else:
logger.main_info("")
logger.notice("Genes are not predicted by default. Use --gene-finding option to enable it.")
########################################################################
reports_fpaths, transposed_reports_fpaths = reporting.save_total(output_dirpath)
########################################################################
### LARGE DRAWING TASKS
########################################################################
if qconfig.draw_plots or qconfig.create_icarus_html:
logger.print_timestamp()
logger.main_info('Creating large visual summaries...')
logger.main_info('This may take a while: press Ctrl-C to skip this step..')
try:
if detailed_contigs_reports_dirpath:
report_for_icarus_fpath_pattern = os.path.join(detailed_contigs_reports_dirpath, qconfig.icarus_report_fname_pattern)
stdout_pattern = os.path.join(detailed_contigs_reports_dirpath, qconfig.contig_report_fname_pattern)
else:
report_for_icarus_fpath_pattern = None
stdout_pattern = None
draw_alignment_plots = qconfig.draw_svg or qconfig.create_icarus_html
number_of_steps = sum([int(bool(value)) for value in [draw_alignment_plots, all_pdf_file]])
if draw_alignment_plots:
########################################################################
### VISUALIZE CONTIG ALIGNMENT
########################################################################
logger.main_info(' 1 of %d: Creating Icarus viewers...' % number_of_steps)
from quast_libs import icarus
icarus_html_fpath, contig_alignment_plot_fpath = icarus.do(
contigs_fpaths, report_for_icarus_fpath_pattern, output_dirpath, ref_fpath,
stdout_pattern=stdout_pattern, features=features_containers, cov_fpath=cov_fpath,
physical_cov_fpath=physical_cov_fpath, json_output_dir=qconfig.json_output_dirpath,
genes_by_labels=genes_by_labels)
if all_pdf_file:
# full report in PDF format: all tables and plots
logger.main_info(' %d of %d: Creating PDF with all tables and plots...' % (number_of_steps, number_of_steps))
plotter.fill_all_pdf_file(all_pdf_file)
logger.main_info('Done')
except KeyboardInterrupt:
logger.main_info('..step skipped!')
os.remove(all_pdf_fpath)
########################################################################
### TOTAL REPORT
########################################################################
logger.print_timestamp()
logger.main_info('RESULTS:')
logger.main_info(' Text versions of total report are saved to ' + reports_fpaths)
logger.main_info(' Text versions of transposed total report are saved to ' + transposed_reports_fpaths)
if qconfig.json_output_dirpath:
json_saver.save_total_report(qconfig.json_output_dirpath, qconfig.min_contig, ref_fpath)
if qconfig.html_report:
from quast_libs.html_saver import html_saver
html_saver.save_colors(output_dirpath, contigs_fpaths, plotter.dict_color_and_ls)
html_saver.save_total_report(output_dirpath, qconfig.min_contig, ref_fpath)
if os.path.isfile(all_pdf_fpath):
logger.main_info(' PDF version (tables and plots) is saved to ' + all_pdf_fpath)
if icarus_html_fpath:
logger.main_info(' Icarus (contig browser) is saved to %s' % icarus_html_fpath)
if qconfig.draw_svg and contig_alignment_plot_fpath:
logger.main_info(' Contig alignment plot is saved to %s' % contig_alignment_plot_fpath)
cleanup(corrected_dirpath)
return logger.finish_up(check_test=qconfig.test)
def main(args):
check_dirpath(
qconfig.QUAST_HOME,
"You are trying to run it from "
+ str(qconfig.QUAST_HOME)
+ "\n."
+ "Please, put QUAST in a different directory, then try again.\n",
exit_code=3,
)
if not args:
qconfig.usage()
sys.exit(0)
try:
import imp
imp.reload(qconfig)
except:
reload(qconfig)
try:
locale.setlocale(locale.LC_ALL, "en_US.utf8")
except Exception:
try:
locale.setlocale(locale.LC_ALL, "en_US.UTF-8")
except Exception:
logger.warning("Python locale settings can't be changed")
quast_path = [os.path.realpath(__file__)]
quast_py_args, contigs_fpaths = parse_options(logger, quast_path + args)
output_dirpath, ref_fpath, labels = qconfig.output_dirpath, qconfig.reference, qconfig.labels
corrected_dirpath = os.path.join(output_dirpath, qconfig.corrected_dirname)
logger.main_info()
logger.print_params()
########################################################################
from quast_libs import reporting
reports = reporting.reports
try:
import imp
imp.reload(reporting)
except:
reload(reporting)
reporting.reports = reports
reporting.assembly_fpaths = []
from quast_libs import plotter # Do not remove this line! It would lead to a warning in matplotlib.
if qconfig.is_combined_ref:
corrected_dirpath = os.path.join(output_dirpath, "..", qconfig.corrected_dirname)
else:
if os.path.isdir(corrected_dirpath):
shutil.rmtree(corrected_dirpath)
os.mkdir(corrected_dirpath)
qconfig.set_max_threads(logger)
# PROCESSING REFERENCE
if ref_fpath:
logger.main_info()
logger.main_info("Reference:")
ref_fpath = qutils.correct_reference(ref_fpath, corrected_dirpath)
else:
ref_fpath = ""
# PROCESSING CONTIGS
logger.main_info()
logger.main_info("Contigs:")
contigs_fpaths, old_contigs_fpaths = qutils.correct_contigs(contigs_fpaths, corrected_dirpath, labels, reporting)
for contigs_fpath in contigs_fpaths:
report = reporting.get(contigs_fpath)
report.add_field(reporting.Fields.NAME, qutils.label_from_fpath(contigs_fpath))
qconfig.assemblies_num = len(contigs_fpaths)
reads_fpaths = []
cov_fpath = qconfig.cov_fpath
physical_cov_fpath = qconfig.phys_cov_fpath
if qconfig.forward_reads:
reads_fpaths.append(qconfig.forward_reads)
if qconfig.reverse_reads:
reads_fpaths.append(qconfig.reverse_reads)
if (reads_fpaths or qconfig.sam or qconfig.bam) and ref_fpath:
bed_fpath, cov_fpath, physical_cov_fpath = reads_analyzer.do(
ref_fpath,
contigs_fpaths,
reads_fpaths,
None,
os.path.join(output_dirpath, qconfig.variation_dirname),
external_logger=logger,
sam_fpath=qconfig.sam,
bam_fpath=qconfig.bam,
bed_fpath=qconfig.bed,
)
qconfig.bed = bed_fpath
if not contigs_fpaths:
logger.error(
"None of the assembly files contains correct contigs. "
"Please, provide different files or decrease --min-contig threshold.",
fake_if_nested_run=True,
)
return 4
if qconfig.used_colors and qconfig.used_ls:
for i, label in enumerate(labels):
plotter.dict_color_and_ls[label] = (qconfig.used_colors[i], qconfig.used_ls[i])
qconfig.assemblies_fpaths = contigs_fpaths
# Where all pdfs will be saved
all_pdf_fpath = os.path.join(output_dirpath, qconfig.plots_fname)
all_pdf_file = None
if qconfig.draw_plots and plotter.can_draw_plots:
try:
from matplotlib.backends.backend_pdf import PdfPages
all_pdf_file = PdfPages(all_pdf_fpath)
except:
all_pdf_file = None
if qconfig.json_output_dirpath:
from quast_libs.html_saver import json_saver
if json_saver.simplejson_error:
json_output_dirpath = None
########################################################################
### Stats and plots
########################################################################
from quast_libs import basic_stats
basic_stats.do(
ref_fpath,
contigs_fpaths,
os.path.join(output_dirpath, "basic_stats"),
qconfig.json_output_dirpath,
output_dirpath,
)
aligned_contigs_fpaths = []
aligned_lengths_lists = []
contig_alignment_plot_fpath = None
icarus_html_fpath = None
if ref_fpath:
########################################################################
### former PLANTAKOLYA, PLANTAGORA
########################################################################
from quast_libs import contigs_analyzer
is_cyclic = qconfig.prokaryote and not qconfig.check_for_fragmented_ref
nucmer_statuses, aligned_lengths_per_fpath = contigs_analyzer.do(
ref_fpath,
contigs_fpaths,
is_cyclic,
os.path.join(output_dirpath, "contigs_reports"),
old_contigs_fpaths,
qconfig.bed,
)
for contigs_fpath in contigs_fpaths:
if nucmer_statuses[contigs_fpath] == contigs_analyzer.NucmerStatus.OK:
aligned_contigs_fpaths.append(contigs_fpath)
aligned_lengths_lists.append(aligned_lengths_per_fpath[contigs_fpath])
# Before continue evaluating, check if nucmer didn't skip all of the contigs files.
detailed_contigs_reports_dirpath = None
features_containers = None
if len(aligned_contigs_fpaths) and ref_fpath:
detailed_contigs_reports_dirpath = os.path.join(output_dirpath, "contigs_reports")
########################################################################
### NAx and NGAx ("aligned Nx and NGx")
########################################################################
from quast_libs import aligned_stats
aligned_stats.do(
ref_fpath,
aligned_contigs_fpaths,
output_dirpath,
qconfig.json_output_dirpath,
aligned_lengths_lists,
os.path.join(output_dirpath, "aligned_stats"),
)
########################################################################
### GENOME_ANALYZER
########################################################################
from quast_libs import genome_analyzer
features_containers = genome_analyzer.do(
ref_fpath,
aligned_contigs_fpaths,
output_dirpath,
qconfig.json_output_dirpath,
qconfig.genes,
qconfig.operons,
detailed_contigs_reports_dirpath,
os.path.join(output_dirpath, "genome_stats"),
)
if qconfig.with_gage:
########################################################################
### GAGE
########################################################################
if not ref_fpath:
logger.warning("GAGE can't be run without a reference and will be skipped.")
else:
from quast_libs import gage
gage.do(ref_fpath, contigs_fpaths, output_dirpath)
genes_by_labels = None
if qconfig.gene_finding:
if qconfig.glimmer:
########################################################################
### Glimmer
########################################################################
from quast_libs import glimmer
genes_by_labels = glimmer.do(
contigs_fpaths, qconfig.genes_lengths, os.path.join(output_dirpath, "predicted_genes")
)
else:
########################################################################
### GeneMark
########################################################################
from quast_libs import genemark
genes_by_labels = genemark.do(
contigs_fpaths,
qconfig.genes_lengths,
os.path.join(output_dirpath, "predicted_genes"),
qconfig.prokaryote,
qconfig.meta,
)
else:
logger.main_info("")
logger.notice("Genes are not predicted by default. Use --gene-finding option to enable it.")
########################################################################
reports_fpaths, transposed_reports_fpaths = reporting.save_total(output_dirpath)
########################################################################
### LARGE DRAWING TASKS
########################################################################
if qconfig.draw_plots or qconfig.create_icarus_html:
logger.print_timestamp()
logger.main_info("Creating large visual summaries...")
logger.main_info("This may take a while: press Ctrl-C to skip this step..")
try:
if detailed_contigs_reports_dirpath:
report_for_icarus_fpath_pattern = os.path.join(
detailed_contigs_reports_dirpath, qconfig.icarus_report_fname_pattern
)
stdout_pattern = os.path.join(detailed_contigs_reports_dirpath, qconfig.contig_report_fname_pattern)
else:
report_for_icarus_fpath_pattern = None
stdout_pattern = None
draw_alignment_plots = qconfig.draw_svg or qconfig.create_icarus_html
number_of_steps = sum([int(bool(value)) for value in [draw_alignment_plots, all_pdf_file]])
if draw_alignment_plots:
########################################################################
### VISUALIZE CONTIG ALIGNMENT
########################################################################
logger.main_info(" 1 of %d: Creating Icarus viewers..." % number_of_steps)
from quast_libs import icarus
icarus_html_fpath, contig_alignment_plot_fpath = icarus.do(
contigs_fpaths,
report_for_icarus_fpath_pattern,
output_dirpath,
ref_fpath,
stdout_pattern=stdout_pattern,
features=features_containers,
cov_fpath=cov_fpath,
physical_cov_fpath=physical_cov_fpath,
json_output_dir=qconfig.json_output_dirpath,
genes_by_labels=genes_by_labels,
)
if all_pdf_file:
# full report in PDF format: all tables and plots
logger.main_info(
" %d of %d: Creating PDF with all tables and plots..." % (number_of_steps, number_of_steps)
)
plotter.fill_all_pdf_file(all_pdf_file)
logger.main_info("Done")
except KeyboardInterrupt:
logger.main_info("..step skipped!")
os.remove(all_pdf_fpath)
########################################################################
### TOTAL REPORT
########################################################################
logger.print_timestamp()
logger.main_info("RESULTS:")
logger.main_info(" Text versions of total report are saved to " + reports_fpaths)
logger.main_info(" Text versions of transposed total report are saved to " + transposed_reports_fpaths)
if qconfig.json_output_dirpath:
json_saver.save_total_report(qconfig.json_output_dirpath, qconfig.min_contig, ref_fpath)
if qconfig.html_report:
from quast_libs.html_saver import html_saver
html_saver.save_colors(output_dirpath, contigs_fpaths, plotter.dict_color_and_ls)
html_saver.save_total_report(output_dirpath, qconfig.min_contig, ref_fpath)
if os.path.isfile(all_pdf_fpath):
logger.main_info(" PDF version (tables and plots) is saved to " + all_pdf_fpath)
if icarus_html_fpath:
logger.main_info(" Icarus (contig browser) is saved to %s" % icarus_html_fpath)
if qconfig.draw_svg and contig_alignment_plot_fpath:
logger.main_info(" Contig alignment plot is saved to %s" % contig_alignment_plot_fpath)
cleanup(corrected_dirpath)
return logger.finish_up(check_test=qconfig.test)