def contigs(args): step_list = [] (contigs_fasta, database_folder, taxonomy_folder, r, one_minus_r, f, out_prefix, predicted_proteins_fasta, diamond_file, path_to_prodigal, path_to_diamond, no_stars, force, quiet, no_log, nproc, sensitive, block_size, index_chunks, tmpdir, top) = check.convert_arguments(args) if no_log: log_file = None else: # Check out_prefix already as the log file needs to be written to a # valid location. error = check.check_out_prefix(out_prefix, None, quiet) if error: sys.exit(1) log_file = '{0}.log'.format(out_prefix) with open(log_file, 'w') as outf1: pass message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, log_file, quiet, show_time=False) # Check at which state to start. if predicted_proteins_fasta is None and diamond_file is None: message = ('\n' 'CAT is running. Protein prediction, alignment, and contig ' 'classification are carried out.\n' 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Contigs fasta: {1}\n' 'Taxonomy folder: {2}/\n' 'Database folder: {3}/\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format(' '.join(sys.argv), contigs_fasta, taxonomy_folder, database_folder, args.r, args.f, log_file)) shared.give_user_feedback(message, log_file, quiet, show_time=False) step_list.append('run_prodigal') step_list.append('run_diamond') elif (predicted_proteins_fasta is not None and diamond_file is None): message = ('\n' 'CAT is running. Since a predicted protein fasta is ' 'supplied, only alignment and contig classification are ' 'carried out.\n' 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Contigs fasta: {1}\n' 'Taxonomy folder: {2}/\n' 'Database folder: {3}/\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format(' '.join(sys.argv), contigs_fasta, taxonomy_folder, database_folder, args.r, args.f, log_file)) shared.give_user_feedback(message, log_file, quiet, show_time=False) step_list.append('run_diamond') elif (predicted_proteins_fasta is not None and diamond_file is not None): message = ('\n' 'CAT is running. Since a predicted protein fasta and ' 'DIAMOND alignment file are supplied, only contig ' 'classification is carried out.\n' 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Contigs fasta: {1}\n' 'Taxonomy folder: {2}/\n' 'Database folder: {3}/\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format(' '.join(sys.argv), contigs_fasta, taxonomy_folder, database_folder, args.r, args.f, log_file)) shared.give_user_feedback(message, log_file, quiet, show_time=False) elif (predicted_proteins_fasta is None and diamond_file is not None): message = ('ERROR: if you want CAT to directly do the classification, ' 'you should not only supply a DIAMOND alignment table but ' 'also a predicted protein fasta file with argument ' '[-p / --proteins].') shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) # Check binaries, output files, taxonomy folder and database folder, and # set parameters. message = 'Doing some pre-flight checks first.' shared.give_user_feedback(message, log_file, quiet, show_time=False) errors = [] errors.append(check.check_out_prefix(out_prefix, log_file, quiet)) if 'run_prodigal' in step_list: errors.append( check.check_prodigal_binaries(path_to_prodigal, log_file, quiet)) predicted_proteins_fasta = ('{0}.predicted_proteins.faa' ''.format(out_prefix)) predicted_proteins_gff = ('{0}.predicted_proteins.gff' ''.format(out_prefix)) if not force: errors.append( check.check_output_file(predicted_proteins_fasta, log_file, quiet)) errors.append( check.check_output_file(predicted_proteins_gff, log_file, quiet)) if 'run_diamond' in step_list: errors.append( check.check_diamond_binaries(path_to_diamond, log_file, quiet)) diamond_file = '{0}.alignment.diamond'.format(out_prefix) if not force: errors.append( check.check_output_file(diamond_file, log_file, quiet)) else: diamond_file = diamond_file errors.append( check.check_folders_for_run(taxonomy_folder, database_folder, step_list, log_file, quiet)) contig2classification_output_file = ('{0}.contig2classification.txt' ''.format(out_prefix)) ORF2LCA_output_file = '{0}.ORF2LCA.txt'.format(out_prefix) if not force: errors.append( check.check_output_file(contig2classification_output_file, log_file, quiet)) errors.append( check.check_output_file(ORF2LCA_output_file, log_file, quiet)) if 'run_prodigal' not in step_list: if not check.check_whether_file_is_fasta(predicted_proteins_fasta): message = ('ERROR: {0} is not a fasta file.' ''.format(predicted_proteins_fasta)) shared.give_user_feedback(message, log_file, quiet, error=True) errors.append(True) errors.append(check.check_top(top, r, log_file, quiet)) if True in errors: sys.exit(1) (nodes_dmp, names_dmp, prot_accession2taxid_file ) = check.inspect_taxonomy_folder(taxonomy_folder) (nr_file, diamond_database, fastaid2LCAtaxid_file, taxids_with_multiple_offspring_file ) = check.inspect_database_folder(database_folder) message = 'Ready to fly!\n\n-----------------\n' shared.give_user_feedback(message, log_file, quiet, show_time=False) # Start CAT. contig_names = shared.import_contig_names(contigs_fasta, log_file, quiet) if 'run_prodigal' in step_list: shared.run_prodigal(path_to_prodigal, contigs_fasta, predicted_proteins_fasta, predicted_proteins_gff, log_file, quiet) contig2ORFs = shared.import_ORFs(predicted_proteins_fasta, log_file, quiet) check.check_whether_ORFs_are_based_on_contigs(contig_names, contig2ORFs, log_file, quiet) if 'run_diamond' in step_list: shared.run_diamond(path_to_diamond, diamond_database, predicted_proteins_fasta, diamond_file, nproc, sensitive, block_size, index_chunks, tmpdir, top, log_file, quiet) (ORF2hits, all_hits) = shared.parse_diamond_file(diamond_file, one_minus_r, log_file, quiet) (taxid2parent, taxid2rank) = tax.import_nodes(nodes_dmp, log_file, quiet) fastaid2LCAtaxid = tax.import_fastaid2LCAtaxid(fastaid2LCAtaxid_file, all_hits, log_file, quiet) taxids_with_multiple_offspring = tax.import_taxids_with_multiple_offspring( taxids_with_multiple_offspring_file, log_file, quiet) message = ('CAT is spinning! Files {0} and {1} are created.' ''.format(contig2classification_output_file, ORF2LCA_output_file)) shared.give_user_feedback(message, log_file, quiet) number_of_classified_contigs = 0 with open(contig2classification_output_file, 'w') as outf1, open(ORF2LCA_output_file, 'w') as outf2: outf1.write('# contig\tclassification\treason\tlineage\t' 'lineage scores\n') outf2.write('# ORF\tlineage\tbit-score\n') for contig in sorted(contig_names): if contig not in contig2ORFs: outf1.write('{0}\tunclassified\tno ORFs found\n' ''.format(contig)) continue LCAs_ORFs = [] for ORF in contig2ORFs[contig]: if ORF not in ORF2hits: outf2.write('{0}\tORF has no hit to database\n' ''.format(ORF)) continue (taxid, top_bitscore) = tax.find_LCA_for_ORF(ORF2hits[ORF], fastaid2LCAtaxid, taxid2parent) if taxid.startswith('no taxid found'): outf2.write('{0}\t{1}\t{2}\n'.format( ORF, taxid, top_bitscore)) else: lineage = tax.find_lineage(taxid, taxid2parent) if not no_stars: lineage = tax.star_lineage( lineage, taxids_with_multiple_offspring) outf2.write('{0}\t{1}\t{2}\n' ''.format(ORF, ';'.join(lineage[::-1]), top_bitscore)) LCAs_ORFs.append((taxid, top_bitscore), ) if len(LCAs_ORFs) == 0: outf1.write('{0}\tunclassified\tno hits to database\n' ''.format(contig)) continue (lineages, lineages_scores, based_on_number_of_ORFs) = tax.find_weighted_LCA( LCAs_ORFs, taxid2parent, f) if lineages == 'no ORFs with taxids found.': outf1.write('{0}\tunclassified\t' 'hits not found in taxonomy files\n' ''.format(contig)) continue if lineages == 'no lineage whitelisted.': outf1.write('{0}\tunclassified\t' 'no lineage reached minimum bit-score support\n' ''.format(contig)) continue # The contig has a valid classification. number_of_classified_contigs += 1 for (i, lineage) in enumerate(lineages): if not no_stars: lineage = tax.star_lineage(lineage, taxids_with_multiple_offspring) scores = [ '{0:.2f}'.format(score) for score in lineages_scores[i] ] if len(lineages) == 1: # There is only one classification. outf1.write('{0}\tclassified\t' 'based on {1}/{2} ORFs\t{3}\t{4}\n' ''.format(contig, based_on_number_of_ORFs, len(contig2ORFs[contig]), ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) else: # There are multiple classifications. outf1.write('{0}\tclassified ({1}/{2})\t' 'based on {3}/{4} ORFs\t{5}\t{6}\n' ''.format(contig, i + 1, len(lineages), based_on_number_of_ORFs, len(contig2ORFs[contig]), ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) message = ('\n-----------------\n\n' '[{0}] CAT is done! {1}/{2} contigs classified.' ''.format(datetime.datetime.now(), number_of_classified_contigs, len(contig_names))) shared.give_user_feedback(message, log_file, quiet, show_time=False) if f < 0.5: message = ('\nWARNING: since f is set to smaller than 0.5, one ' 'contig may have multiple classifications.') shared.give_user_feedback(message, log_file, quiet, show_time=False)
def run(): args = parse_arguments() message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) # Check at which state to start. step_list = [] if not args.proteins_fasta and not args.alignment_file: message = ( '\n' 'BAT is running. Protein prediction, alignment, and bin ' 'classification are carried out.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) step_list.append('predict_proteins') step_list.append('align') elif args.proteins_fasta and not args.alignment_file: message = ( '\n' 'BAT is running. Since a predicted protein fasta is supplied, ' 'only alignment and bin classification are carried out.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) step_list.append('align') elif args.proteins_fasta and args.alignment_file: message = ( '\n' 'BAT is running. Since a predicted protein fasta and ' 'alignment file are supplied, only bin classification is ' 'carried out.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) elif not args.proteins_fasta and args.alignment_file: message = ( 'if you want BAT to directly classify a set of bins, you ' 'should not only supply a DIAMOND alignment table but also a ' 'concatenated predicted protein fasta file with argument ' '[-p / --proteins].') shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) step_list.append('classify') # Print variables. message = ( 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Bin folder: {1}\n' 'Taxonomy folder: {2}\n' 'Database folder: {3}\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format( ' '.join(sys.argv), args.bin_folder, args.taxonomy_folder, args.database_folder, int(args.r), float(args.f), args.log_file)) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) # Check binaries, output files, taxonomy folder and database folder, and # set variables. message = 'Doing some pre-flight checks first.' shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) errors = [] errors.append( check.check_bin_folder( args.bin_folder, args.bin_suffix, args.log_file, args.quiet)) errors.append( check.check_out_prefix(args.out_prefix, args.log_file, args.quiet)) if 'predict_proteins' in step_list: errors.append( check.check_prodigal_binaries( args.path_to_prodigal, args.log_file, args.quiet)) setattr(args, 'concatenated_fasta', '{0}.concatenated.fasta'.format(args.out_prefix)) setattr(args, 'proteins_fasta', '{0}.concatenated.predicted_proteins.faa'.format( args.out_prefix)) setattr(args, 'proteins_gff', '{0}.concatenated.predicted_proteins.gff'.format( args.out_prefix)) if not args.force: errors.append( check.check_output_file( args.concatenated_fasta, args.log_file, args.quiet)) errors.append( check.check_output_file( args.proteins_fasta, args.log_file, args.quiet)) errors.append( check.check_output_file( args.proteins_gff, args.log_file, args.quiet)) if 'align' in step_list: errors.append( check.check_diamond_binaries( args.path_to_diamond, args.log_file, args.quiet)) setattr(args, 'alignment_file', '{0}.concatenated.alignment.diamond'.format(args.out_prefix)) if not args.force: errors.append( check.check_output_file( args.alignment_file, args.log_file, args.quiet)) errors.append( check.check_folders_for_run( args.taxonomy_folder, args.nodes_dmp, args.names_dmp, args.database_folder, args.diamond_database, args.fastaid2LCAtaxid_file, args.taxids_with_multiple_offspring_file, step_list, args.log_file, args.quiet)) setattr(args, 'bin2classification_output_file', '{0}.bin2classification.txt'.format(args.out_prefix)) setattr(args, 'ORF2LCA_output_file', '{0}.ORF2LCA.txt'.format(args.out_prefix)) if not args.force: errors.append( check.check_output_file( args.bin2classification_output_file, args.log_file, args.quiet)) errors.append( check.check_output_file( args.ORF2LCA_output_file, args.log_file, args.quiet)) if 'predict_proteins' not in step_list: errors.append( check.check_fasta( args.proteins_fasta, args.log_file, args.quiet)) if 'align' in step_list: errors.append( check.check_top(args.top, args.r, args.log_file, args.quiet)) # Print all variables. shared.print_variables(args, step_list) if True in errors: sys.exit(1) message = 'Ready to fly!\n\n-----------------\n' shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) # Start BAT. (bin2contigs, contig_names) = import_bins( args.bin_folder, args.bin_suffix, args.log_file, args.quiet) if 'predict_proteins' in step_list: make_concatenated_fasta( args.concatenated_fasta, bin2contigs, args.bin_folder, args.log_file, args.quiet) shared.run_prodigal( args.path_to_prodigal, args.concatenated_fasta, args.proteins_fasta, args.proteins_gff, args.log_file, args.quiet) contig2ORFs = shared.import_ORFs( args.proteins_fasta, args.log_file, args.quiet) check.check_whether_ORFs_are_based_on_contigs( contig_names, contig2ORFs, args.log_file, args.quiet) if 'align' in step_list: shared.run_diamond(args) (ORF2hits, all_hits) = shared.parse_tabular_alignment( args.alignment_file, args.one_minus_r, args.log_file, args.quiet) (taxid2parent, taxid2rank) = tax.import_nodes( args.nodes_dmp, args.log_file, args.quiet) fastaid2LCAtaxid = tax.import_fastaid2LCAtaxid( args.fastaid2LCAtaxid_file, all_hits, args.log_file, args.quiet) taxids_with_multiple_offspring = tax.import_taxids_with_multiple_offspring( args.taxids_with_multiple_offspring_file, args.log_file, args.quiet) message = 'BAT is flying! Files {0} and {1} are created.'.format( args.bin2classification_output_file, args.ORF2LCA_output_file) shared.give_user_feedback(message, args.log_file, args.quiet) n_classified_bins = 0 with open(args.bin2classification_output_file, 'w') as outf1, open(args.ORF2LCA_output_file, 'w') as outf2: outf1.write('# bin\tclassification\treason\tlineage\tlineage scores\n') outf2.write('# ORF\tbin\tnumber of hits\tlineage\ttop bit-score\n') for bin_ in sorted(bin2contigs): LCAs_ORFs = [] for contig in sorted(bin2contigs[bin_]): if contig not in contig2ORFs: continue for ORF in contig2ORFs[contig]: if ORF not in ORF2hits: outf2.write('{0}\t{1}\tORF has no hit to database\n' ''.format(ORF, bin_)) continue n_hits = len(ORF2hits[ORF]) (taxid, top_bitscore) = tax.find_LCA_for_ORF( ORF2hits[ORF], fastaid2LCAtaxid, taxid2parent) if taxid.startswith('no taxid found'): outf2.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( ORF, bin_, n_hits, taxid, top_bitscore)) else: lineage = tax.find_lineage(taxid, taxid2parent) if not args.no_stars: lineage = tax.star_lineage( lineage, taxids_with_multiple_offspring) outf2.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( ORF, bin_, n_hits, ';'.join(lineage[::-1]), top_bitscore)) LCAs_ORFs.append((taxid, top_bitscore),) if len(LCAs_ORFs) == 0: outf1.write('{0}\tno taxid assigned\tno hits to database\n' ''.format(bin_)) continue (lineages, lineages_scores, based_on_n_ORFs) = tax.find_weighted_LCA( LCAs_ORFs, taxid2parent, args.f) if lineages == 'no ORFs with taxids found.': outf1.write('{0}\tno taxid assigned\t' 'hits not found in taxonomy files\n'.format(bin_)) continue if lineages == 'no lineage whitelisted.': outf1.write( '{0}\tno taxid assigned\t' 'no lineage reached minimum bit-score support\n' ''.format(bin_)) continue # The bin has a valid classification. n_classified_bins += 1 total_n_ORFs = sum([len(contig2ORFs[contig]) for contig in bin2contigs[bin_] if contig in contig2ORFs]) for (i, lineage) in enumerate(lineages): if not args.no_stars: lineage = tax.star_lineage( lineage, taxids_with_multiple_offspring) scores = ['{0:.2f}'.format(score) for score in lineages_scores[i]] if len(lineages) == 1: # There is only one classification. outf1.write( '{0}\t' 'taxid assigned\t' 'based on {1}/{2} ORFs\t' '{3}\t' '{4}\n'.format( bin_, based_on_n_ORFs, total_n_ORFs, ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) else: # There are multiple classifications. outf1.write( '{0}\t' 'taxid assigned ({1}/{2})\t' 'based on {3}/{4} ORFs\t' '{5}\t' '{6}\n'.format( bin_, i + 1, len(lineages), based_on_n_ORFs, total_n_ORFs, ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) message = ('\n-----------------\n\n' '{0} BAT is done! {1:,d}/{2:,d} bins have taxonomy assigned.' ''.format(shared.timestamp(), n_classified_bins, len(bin2contigs))) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) if args.f < 0.5: message = ('\nWARNING: since f is set to smaller than 0.5, one bin ' 'may have multiple classifications.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) return