def contigs(args): step_list = [] (contigs_fasta, database_folder, taxonomy_folder, r, one_minus_r, f, out_prefix, predicted_proteins_fasta, diamond_file, path_to_prodigal, path_to_diamond, no_stars, force, quiet, no_log, nproc, sensitive, block_size, index_chunks, tmpdir, top) = check.convert_arguments(args) if no_log: log_file = None else: # Check out_prefix already as the log file needs to be written to a # valid location. error = check.check_out_prefix(out_prefix, None, quiet) if error: sys.exit(1) log_file = '{0}.log'.format(out_prefix) with open(log_file, 'w') as outf1: pass message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, log_file, quiet, show_time=False) # Check at which state to start. if predicted_proteins_fasta is None and diamond_file is None: message = ('\n' 'CAT is running. Protein prediction, alignment, and contig ' 'classification are carried out.\n' 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Contigs fasta: {1}\n' 'Taxonomy folder: {2}/\n' 'Database folder: {3}/\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format(' '.join(sys.argv), contigs_fasta, taxonomy_folder, database_folder, args.r, args.f, log_file)) shared.give_user_feedback(message, log_file, quiet, show_time=False) step_list.append('run_prodigal') step_list.append('run_diamond') elif (predicted_proteins_fasta is not None and diamond_file is None): message = ('\n' 'CAT is running. Since a predicted protein fasta is ' 'supplied, only alignment and contig classification are ' 'carried out.\n' 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Contigs fasta: {1}\n' 'Taxonomy folder: {2}/\n' 'Database folder: {3}/\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format(' '.join(sys.argv), contigs_fasta, taxonomy_folder, database_folder, args.r, args.f, log_file)) shared.give_user_feedback(message, log_file, quiet, show_time=False) step_list.append('run_diamond') elif (predicted_proteins_fasta is not None and diamond_file is not None): message = ('\n' 'CAT is running. Since a predicted protein fasta and ' 'DIAMOND alignment file are supplied, only contig ' 'classification is carried out.\n' 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Contigs fasta: {1}\n' 'Taxonomy folder: {2}/\n' 'Database folder: {3}/\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format(' '.join(sys.argv), contigs_fasta, taxonomy_folder, database_folder, args.r, args.f, log_file)) shared.give_user_feedback(message, log_file, quiet, show_time=False) elif (predicted_proteins_fasta is None and diamond_file is not None): message = ('ERROR: if you want CAT to directly do the classification, ' 'you should not only supply a DIAMOND alignment table but ' 'also a predicted protein fasta file with argument ' '[-p / --proteins].') shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) # Check binaries, output files, taxonomy folder and database folder, and # set parameters. message = 'Doing some pre-flight checks first.' shared.give_user_feedback(message, log_file, quiet, show_time=False) errors = [] errors.append(check.check_out_prefix(out_prefix, log_file, quiet)) if 'run_prodigal' in step_list: errors.append( check.check_prodigal_binaries(path_to_prodigal, log_file, quiet)) predicted_proteins_fasta = ('{0}.predicted_proteins.faa' ''.format(out_prefix)) predicted_proteins_gff = ('{0}.predicted_proteins.gff' ''.format(out_prefix)) if not force: errors.append( check.check_output_file(predicted_proteins_fasta, log_file, quiet)) errors.append( check.check_output_file(predicted_proteins_gff, log_file, quiet)) if 'run_diamond' in step_list: errors.append( check.check_diamond_binaries(path_to_diamond, log_file, quiet)) diamond_file = '{0}.alignment.diamond'.format(out_prefix) if not force: errors.append( check.check_output_file(diamond_file, log_file, quiet)) else: diamond_file = diamond_file errors.append( check.check_folders_for_run(taxonomy_folder, database_folder, step_list, log_file, quiet)) contig2classification_output_file = ('{0}.contig2classification.txt' ''.format(out_prefix)) ORF2LCA_output_file = '{0}.ORF2LCA.txt'.format(out_prefix) if not force: errors.append( check.check_output_file(contig2classification_output_file, log_file, quiet)) errors.append( check.check_output_file(ORF2LCA_output_file, log_file, quiet)) if 'run_prodigal' not in step_list: if not check.check_whether_file_is_fasta(predicted_proteins_fasta): message = ('ERROR: {0} is not a fasta file.' ''.format(predicted_proteins_fasta)) shared.give_user_feedback(message, log_file, quiet, error=True) errors.append(True) errors.append(check.check_top(top, r, log_file, quiet)) if True in errors: sys.exit(1) (nodes_dmp, names_dmp, prot_accession2taxid_file ) = check.inspect_taxonomy_folder(taxonomy_folder) (nr_file, diamond_database, fastaid2LCAtaxid_file, taxids_with_multiple_offspring_file ) = check.inspect_database_folder(database_folder) message = 'Ready to fly!\n\n-----------------\n' shared.give_user_feedback(message, log_file, quiet, show_time=False) # Start CAT. contig_names = shared.import_contig_names(contigs_fasta, log_file, quiet) if 'run_prodigal' in step_list: shared.run_prodigal(path_to_prodigal, contigs_fasta, predicted_proteins_fasta, predicted_proteins_gff, log_file, quiet) contig2ORFs = shared.import_ORFs(predicted_proteins_fasta, log_file, quiet) check.check_whether_ORFs_are_based_on_contigs(contig_names, contig2ORFs, log_file, quiet) if 'run_diamond' in step_list: shared.run_diamond(path_to_diamond, diamond_database, predicted_proteins_fasta, diamond_file, nproc, sensitive, block_size, index_chunks, tmpdir, top, log_file, quiet) (ORF2hits, all_hits) = shared.parse_diamond_file(diamond_file, one_minus_r, log_file, quiet) (taxid2parent, taxid2rank) = tax.import_nodes(nodes_dmp, log_file, quiet) fastaid2LCAtaxid = tax.import_fastaid2LCAtaxid(fastaid2LCAtaxid_file, all_hits, log_file, quiet) taxids_with_multiple_offspring = tax.import_taxids_with_multiple_offspring( taxids_with_multiple_offspring_file, log_file, quiet) message = ('CAT is spinning! Files {0} and {1} are created.' ''.format(contig2classification_output_file, ORF2LCA_output_file)) shared.give_user_feedback(message, log_file, quiet) number_of_classified_contigs = 0 with open(contig2classification_output_file, 'w') as outf1, open(ORF2LCA_output_file, 'w') as outf2: outf1.write('# contig\tclassification\treason\tlineage\t' 'lineage scores\n') outf2.write('# ORF\tlineage\tbit-score\n') for contig in sorted(contig_names): if contig not in contig2ORFs: outf1.write('{0}\tunclassified\tno ORFs found\n' ''.format(contig)) continue LCAs_ORFs = [] for ORF in contig2ORFs[contig]: if ORF not in ORF2hits: outf2.write('{0}\tORF has no hit to database\n' ''.format(ORF)) continue (taxid, top_bitscore) = tax.find_LCA_for_ORF(ORF2hits[ORF], fastaid2LCAtaxid, taxid2parent) if taxid.startswith('no taxid found'): outf2.write('{0}\t{1}\t{2}\n'.format( ORF, taxid, top_bitscore)) else: lineage = tax.find_lineage(taxid, taxid2parent) if not no_stars: lineage = tax.star_lineage( lineage, taxids_with_multiple_offspring) outf2.write('{0}\t{1}\t{2}\n' ''.format(ORF, ';'.join(lineage[::-1]), top_bitscore)) LCAs_ORFs.append((taxid, top_bitscore), ) if len(LCAs_ORFs) == 0: outf1.write('{0}\tunclassified\tno hits to database\n' ''.format(contig)) continue (lineages, lineages_scores, based_on_number_of_ORFs) = tax.find_weighted_LCA( LCAs_ORFs, taxid2parent, f) if lineages == 'no ORFs with taxids found.': outf1.write('{0}\tunclassified\t' 'hits not found in taxonomy files\n' ''.format(contig)) continue if lineages == 'no lineage whitelisted.': outf1.write('{0}\tunclassified\t' 'no lineage reached minimum bit-score support\n' ''.format(contig)) continue # The contig has a valid classification. number_of_classified_contigs += 1 for (i, lineage) in enumerate(lineages): if not no_stars: lineage = tax.star_lineage(lineage, taxids_with_multiple_offspring) scores = [ '{0:.2f}'.format(score) for score in lineages_scores[i] ] if len(lineages) == 1: # There is only one classification. outf1.write('{0}\tclassified\t' 'based on {1}/{2} ORFs\t{3}\t{4}\n' ''.format(contig, based_on_number_of_ORFs, len(contig2ORFs[contig]), ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) else: # There are multiple classifications. outf1.write('{0}\tclassified ({1}/{2})\t' 'based on {3}/{4} ORFs\t{5}\t{6}\n' ''.format(contig, i + 1, len(lineages), based_on_number_of_ORFs, len(contig2ORFs[contig]), ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) message = ('\n-----------------\n\n' '[{0}] CAT is done! {1}/{2} contigs classified.' ''.format(datetime.datetime.now(), number_of_classified_contigs, len(contig_names))) shared.give_user_feedback(message, log_file, quiet, show_time=False) if f < 0.5: message = ('\nWARNING: since f is set to smaller than 0.5, one ' 'contig may have multiple classifications.') shared.give_user_feedback(message, log_file, quiet, show_time=False)
def summarise_contigs(args): message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) errors = [] errors.append( check.check_input_file(args.input_file, args.log_file, args.quiet)) if not args.force: errors.append( check.check_output_file(args.output_file, args.log_file, args.quiet)) if True in errors: sys.exit(1) contig2length = import_contig_lengths(args.contigs_fasta, args.log_file, args.quiet) message = 'Summarising...' shared.give_user_feedback(message, args.log_file, args.quiet) with open(args.input_file, 'r') as f1: for line in f1: if line.startswith('#'): line = line.split('\t') if line[0] != '# contig': message = '{0} is not a CAT classification file.'.format( args.input_file) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) if line[0] == '# bin': message = ( '{0} appears to be a BAT classification file. ' 'If you want to summarise bin ' 'classifications, simply don\'t supply a ' 'contigs fasta and everything should be fine.' ''.format(args.input_file)) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) try: superkingdom_index = line.index('superkingdom') except: message = ( 'official ranks not found in header of {0}. Make ' 'sure that the CAT classification file is named ' 'with official ranks with \'CAT add_names ' '--only_official\'.'.format(args.input_file)) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) break else: message = 'input file does not have a recognisable header.' shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) length = {} length['no taxid assigned'] = [] ORFs = {} official_ranks = [ 'superkingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species' ] for rank in official_ranks: length[rank] = {} ORFs[rank] = {} n = 0 contig_trace = set() doubles = set() with open(args.input_file, 'r') as f1: for line in f1: line = line.rstrip() if line.startswith('#'): continue n += 1 line = line.split('\t') contig = line[0] if contig in contig_trace: doubles.add(contig) contig_trace.add(contig) if contig not in contig2length: message = ( 'contig {0} in CAT classification file is not found ' 'in supplied contigs fasta file. Are you sure the CAT ' 'classification file is based on the contigs fasta?' ''.format(contig)) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) if line[1] == 'no taxid assigned': length['no taxid assigned'].append(contig2length[contig]) continue for (i, classification) in enumerate(line[superkingdom_index:]): classification = classification.rsplit(': ', 1)[0].rstrip('*') rank = official_ranks[i] if classification not in length[rank]: length[rank][classification] = [] ORFs[rank][classification] = [] length[rank][classification].append(contig2length[contig]) # NOTE that the total number of ORFs on a contig is reproted, # not only the number of ORFs a classification is based on. ORFs_on_contig = int(line[2].split('/')[1].split(' ')[0]) ORFs[rank][classification].append(ORFs_on_contig) if len(doubles) != 0: message = ('some contigs have multiple classifications. CAT summarise ' 'currently does not allow for this. Contigs with multiple ' 'classifications: {0}.'.format(', '.join(list(doubles)))) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) if n != len(contig2length): message = ('the number of classified contigs is not the same as the ' 'number of contigs in contigs fasta. Are you sure the CAT ' 'classification file is based on the contigs fasta?') shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) with open(args.output_file, 'w') as outf1: n_contigs = len(contig2length) total_length = sum(contig2length.values()) n_classified_contigs = n_contigs - len(length['no taxid assigned']) total_classified_length = total_length - sum( length['no taxid assigned']) outf1.write('# total number of contigs in {0} is {1:,d} representing ' '{2:,d} positions.\n'.format(args.contigs_fasta, n_contigs, total_length)) outf1.write( '# {0:,d} contigs have taxonomy assigned ({1:.2f}%) ' 'representing {2:,d} positions ({3:.2f}%) in {4}.\n'.format( n_classified_contigs, n_classified_contigs / n_contigs * 100, total_classified_length, total_classified_length / total_length * 100, args.input_file)) outf1.write('#\n') outf1.write('# rank\t' 'clade\t' 'number of contigs\t' 'number of ORFs\t' 'number of positions\n') for rank in official_ranks: for clade in sorted(length[rank], key=lambda x: sum(length[rank][x]), reverse=True): outf1.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( rank, clade, len(length[rank][clade]), sum(ORFs[rank][clade]), sum(length[rank][clade]))) message = '{0} is created!'.format(args.output_file) shared.give_user_feedback(message, args.log_file, args.quiet) return
def summarise_bins(args): message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) errors = [] errors.append( check.check_input_file(args.input_file, args.log_file, args.quiet)) if not args.force: errors.append( check.check_output_file(args.output_file, args.log_file, args.quiet)) if True in errors: sys.exit(1) message = 'Summarising...' shared.give_user_feedback(message, args.log_file, args.quiet) with open(args.input_file, 'r') as f1: for line in f1: if line.startswith('#'): line = line.split('\t') if line[0] != '# bin': message = '{0} is not a BAT classification file.'.format( args.input_file) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) if line[0] == '# contig': message = ( '{0} appears to be a CAT classification file. ' 'If you want to summarise contig ' 'classifications, supply a contigs fasta with ' 'argument [-c / --contigs_fasta].'.format( args.input_file)) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) try: superkingdom_index = line.index('superkingdom') except: message = ( 'official ranks not found in header of {0}. Make ' 'sure that the BAT classification file is named ' 'with official ranks with \'CAT add_names ' '--only_official\'.'.format(args.input_file)) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) break else: message = 'input file does not have a recognisable header.' shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) n_bins = {} n_bins['no taxid assigned'] = 0 official_ranks = [ 'superkingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species' ] for rank in official_ranks: n_bins[rank] = {} n = 0 bin_trace = set() doubles = set() with open(args.input_file, 'r') as f1: for line in f1: line = line.rstrip() if line.startswith('#'): continue n += 1 line = line.split('\t') bin_ = line[0] if bin_ in bin_trace: doubles.add(bin_) bin_trace.add(bin_) if line[1] == 'no taxid assigned': n_bins['no taxid assigned'] += 1 continue for (i, classification) in enumerate(line[superkingdom_index:]): classification = classification.rsplit(': ', 1)[0].rstrip('*') rank = official_ranks[i] if classification not in n_bins[rank]: n_bins[rank][classification] = 0 n_bins[rank][classification] += 1 if len(doubles) != 0: message = ('some bins have multiple classifications. CAT summarise ' 'currently does not allow for this. Bins with multiple ' 'classifications: {0}.'.format(', '.join(list(doubles)))) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) n_classified_bins = n - n_bins['no taxid assigned'] with open(args.output_file, 'w') as outf1: outf1.write('# total number of bins is {0:,d}, of which {1:,d} ' '({2:.2f}%) have taxonomy assigned.\n'.format( n, n_classified_bins, n_classified_bins / n * 100)) outf1.write('#\n') outf1.write('# rank\tclade\tnumber of bins\n') for rank in official_ranks: for clade in sorted(n_bins[rank], key=lambda x: n_bins[rank][x], reverse=True): outf1.write('{0}\t{1}\t{2}\n'.format(rank, clade, n_bins[rank][clade])) message = '{0} is created!'.format(args.output_file) shared.give_user_feedback(message, args.log_file, args.quiet) return
def run(): args = parse_arguments() message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) errors = [] errors.append( check.check_input_file(args.input_file, args.log_file, args.quiet)) if not args.force: errors.append( check.check_output_file(args.output_file, args.log_file, args.quiet)) if True in errors: sys.exit(1) (taxid2parent, taxid2rank) = tax.import_nodes(args.nodes_dmp, args.log_file, args.quiet) taxid2name = tax.import_names(args.names_dmp, args.log_file, args.quiet) message = 'Appending names...' shared.give_user_feedback(message, args.log_file, args.quiet) with open(args.input_file, 'r') as f1: for line in f1: if line.startswith('#'): line = line.rstrip().split('\t') if 'lineage' in line: lineage_index = line.index('lineage') else: message = ('{0} is not a supported classification file.' ''.format(input_file)) shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) try: scores_index = line.index('lineage scores') except: scores_index = None full_length = len(line) break else: message = ('{0} is not a supported classification file.'.format( args.input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) with open(args.input_file, 'r') as f1, open(args.output_file, 'w') as outf1: for line in f1: line = line.rstrip() if line.startswith('#'): if args.only_official: outf1.write('{0}\tsuperkingdom\tphylum\tclass\torder\t' 'family\tgenus\tspecies\n'.format(line)) else: outf1.write('{0}\tfull lineage names\n'.format(line)) continue line = line.split('\t') if len(line) != full_length: # Entry does not have a full annotation. outf1.write('{0}\n'.format('\t'.join(line))) continue if (line[1].startswith('no taxid found') or line[2].startswith('no taxid found')): # ORF has database hits but the accession number is not found # in the taxonomy files. outf1.write('{0}\n'.format('\t'.join(line))) continue lineage = line[lineage_index].split(';') if scores_index is not None and not args.exclude_scores: scores = line[scores_index].split(';') else: scores = None if args.only_official: names = tax.convert_to_official_names(lineage, taxid2rank, taxid2name, scores) else: names = tax.convert_to_names(lineage, taxid2rank, taxid2name, scores) outf1.write('{0}\t{1}\n'.format('\t'.join(line), '\t'.join(names))) message = 'Names written to {0}!'.format(args.output_file) shared.give_user_feedback(message, args.log_file, args.quiet) return
def summarise_bins(input_file, output_file, force, quiet): # Currently summarise does not a allow for a log file. log_file = None message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, log_file, quiet, show_time=False) errors = [] errors.append(check.check_input_file(input_file, log_file, quiet)) if not force: errors.append(check.check_output_file(output_file, log_file, quiet)) if True in errors: sys.exit(1) message = 'Summarising...' shared.give_user_feedback(message, log_file, quiet) with shared.open_maybe_gzip(input_file, 'rt') as f1: for line in f1: if line.startswith('#'): line = line.split('\t') if line[0] != '# bin': message = ('ERROR: {0} is not a BAT classification file.' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) if line[0] == '# contig': message = ('ERROR: {0} appears to be a CAT ' 'classification file. If you want to ' 'summarise contig classifications, please ' 'supply a contigs fasta.' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) try: superkingdom_index = line.index('superkingdom') except: message = ('ERROR: official ranks not found in header of ' '{0}. Make sure that the BAT classification ' 'file is named with official ranks with \'CAT ' 'add_names --only_official\'.' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) break else: message = 'ERROR: input file does not have a recognisable header.' shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) number_of_bins = {} number_of_bins['unclassified'] = 0 official_ranks = [ 'superkingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species' ] for rank in official_ranks: number_of_bins[rank] = {} n = 0 bin_trace = set() doubles = set() with open(input_file, 'r') as f1: for line in f1: line = line.rstrip() if line.startswith('#'): continue n += 1 line = line.split('\t') bin_ = line[0] if bin_ in bin_trace: doubles.add(bin_) bin_trace.add(bin_) if line[1] == 'unclassified': number_of_bins['unclassified'] += 1 continue for (i, classification) in enumerate(line[superkingdom_index:]): classification = classification.rsplit(': ', 1)[0].rstrip('*') rank = official_ranks[i] if classification not in number_of_bins[rank]: number_of_bins[rank][classification] = 0 number_of_bins[rank][classification] += 1 if len(doubles) != 0: message = ('ERROR: some bins have multiple classifications. CAT ' 'summarise currently does not allow for this. Bins with ' 'multiple classifications: {0}.' ''.format(', '.join(list(doubles)))) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) number_of_classified_bins = n - number_of_bins['unclassified'] with shared.open_maybe_gzip(output_file, 'wt') as outf1: outf1.write('# total number of bins is {0}, of which {1} ({2:.2f}%) ' 'are classified.\n' ''.format(n, number_of_classified_bins, number_of_classified_bins / n * 100)) outf1.write('#\n') outf1.write('# rank\tclade\tnumber of bins\n') for rank in official_ranks: for clade in sorted(number_of_bins[rank], key=lambda x: number_of_bins[rank][x], reverse=True): outf1.write('{0}\t{1}\t{2}\n' ''.format(rank, clade, number_of_bins[rank][clade])) message = '{0} is created!'.format(output_file) shared.give_user_feedback(message, log_file, quiet)
def add_names(args): (input_file, output_file, taxonomy_folder, only_official, exclude_scores, force, quiet) = check.convert_arguments(args) # Currently add_names does not allow for a log file. log_file = None message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, log_file, quiet, show_time=False) errors = [] errors.append(check.check_input_file(input_file, log_file, quiet)) if not force: errors.append(check.check_output_file(output_file, log_file, quiet)) if True in errors: sys.exit(1) (nodes_dmp, names_dmp, prot_accession2taxid_file ) = check.inspect_taxonomy_folder(taxonomy_folder) (taxid2parent, taxid2rank) = tax.import_nodes(nodes_dmp, log_file, quiet) taxid2name = tax.import_names(names_dmp, log_file, quiet) message = 'Appending names...' shared.give_user_feedback(message, log_file, quiet) with shared.open_maybe_gzip(input_file, 'rt') as f1: for line in f1: if line.startswith('#'): line = line.rstrip().split('\t') try: lineage_index = line.index('lineage') except: message = ('ERROR: {0} is not a supported classification ' 'file.'.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) try: scores_index = line.index('lineage scores') except: scores_index = None full_length = len(line) break else: message = ('ERROR: {0} is not a supported classification file.' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) with shared.open_maybe_gzip(input_file, 'rt') as f1, shared.open_maybe_gzip( output_file, 'wt') as outf1: for line in f1: line = line.rstrip() if line.startswith('#'): if only_official: outf1.write('{0}\tsuperkingdom\tphylum\tclass\torder\t' 'family\tgenus\tspecies\n'.format(line)) else: outf1.write('{0}\tfull lineage names\n'.format(line)) continue line = line.split('\t') if len(line) != full_length: # Entry does not have a full annotation. outf1.write('{0}\n'.format('\t'.join(line))) continue if (line[1].startswith('no taxid found') or line[2].startswith('no taxid found')): # ORF has database hits but the accession number is not found # in the taxonomy files. outf1.write('{0}\n'.format('\t'.join(line))) continue lineage = line[lineage_index].split(';') if scores_index and not exclude_scores: scores = line[scores_index].split(';') else: scores = None if only_official: names = tax.convert_to_official_names(lineage, taxid2rank, taxid2name, scores) else: names = tax.convert_to_names(lineage, taxid2rank, taxid2name, scores) outf1.write('{0}\t{1}\n'.format('\t'.join(line), '\t'.join(names))) message = 'Names written to {0}!'.format(output_file) shared.give_user_feedback(message, log_file, quiet)
def run(): args = parse_arguments() message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) # Check at which state to start. step_list = [] if not args.proteins_fasta and not args.alignment_file: message = ( '\n' 'BAT is running. Protein prediction, alignment, and bin ' 'classification are carried out.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) step_list.append('predict_proteins') step_list.append('align') elif args.proteins_fasta and not args.alignment_file: message = ( '\n' 'BAT is running. Since a predicted protein fasta is supplied, ' 'only alignment and bin classification are carried out.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) step_list.append('align') elif args.proteins_fasta and args.alignment_file: message = ( '\n' 'BAT is running. Since a predicted protein fasta and ' 'alignment file are supplied, only bin classification is ' 'carried out.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) elif not args.proteins_fasta and args.alignment_file: message = ( 'if you want BAT to directly classify a set of bins, you ' 'should not only supply a DIAMOND alignment table but also a ' 'concatenated predicted protein fasta file with argument ' '[-p / --proteins].') shared.give_user_feedback(message, args.log_file, args.quiet, error=True) sys.exit(1) step_list.append('classify') # Print variables. message = ( 'Rarw!\n\n' 'Supplied command: {0}\n\n' 'Bin folder: {1}\n' 'Taxonomy folder: {2}\n' 'Database folder: {3}\n' 'Parameter r: {4}\n' 'Parameter f: {5}\n' 'Log file: {6}\n\n' '-----------------\n'.format( ' '.join(sys.argv), args.bin_folder, args.taxonomy_folder, args.database_folder, int(args.r), float(args.f), args.log_file)) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) # Check binaries, output files, taxonomy folder and database folder, and # set variables. message = 'Doing some pre-flight checks first.' shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) errors = [] errors.append( check.check_bin_folder( args.bin_folder, args.bin_suffix, args.log_file, args.quiet)) errors.append( check.check_out_prefix(args.out_prefix, args.log_file, args.quiet)) if 'predict_proteins' in step_list: errors.append( check.check_prodigal_binaries( args.path_to_prodigal, args.log_file, args.quiet)) setattr(args, 'concatenated_fasta', '{0}.concatenated.fasta'.format(args.out_prefix)) setattr(args, 'proteins_fasta', '{0}.concatenated.predicted_proteins.faa'.format( args.out_prefix)) setattr(args, 'proteins_gff', '{0}.concatenated.predicted_proteins.gff'.format( args.out_prefix)) if not args.force: errors.append( check.check_output_file( args.concatenated_fasta, args.log_file, args.quiet)) errors.append( check.check_output_file( args.proteins_fasta, args.log_file, args.quiet)) errors.append( check.check_output_file( args.proteins_gff, args.log_file, args.quiet)) if 'align' in step_list: errors.append( check.check_diamond_binaries( args.path_to_diamond, args.log_file, args.quiet)) setattr(args, 'alignment_file', '{0}.concatenated.alignment.diamond'.format(args.out_prefix)) if not args.force: errors.append( check.check_output_file( args.alignment_file, args.log_file, args.quiet)) errors.append( check.check_folders_for_run( args.taxonomy_folder, args.nodes_dmp, args.names_dmp, args.database_folder, args.diamond_database, args.fastaid2LCAtaxid_file, args.taxids_with_multiple_offspring_file, step_list, args.log_file, args.quiet)) setattr(args, 'bin2classification_output_file', '{0}.bin2classification.txt'.format(args.out_prefix)) setattr(args, 'ORF2LCA_output_file', '{0}.ORF2LCA.txt'.format(args.out_prefix)) if not args.force: errors.append( check.check_output_file( args.bin2classification_output_file, args.log_file, args.quiet)) errors.append( check.check_output_file( args.ORF2LCA_output_file, args.log_file, args.quiet)) if 'predict_proteins' not in step_list: errors.append( check.check_fasta( args.proteins_fasta, args.log_file, args.quiet)) if 'align' in step_list: errors.append( check.check_top(args.top, args.r, args.log_file, args.quiet)) # Print all variables. shared.print_variables(args, step_list) if True in errors: sys.exit(1) message = 'Ready to fly!\n\n-----------------\n' shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) # Start BAT. (bin2contigs, contig_names) = import_bins( args.bin_folder, args.bin_suffix, args.log_file, args.quiet) if 'predict_proteins' in step_list: make_concatenated_fasta( args.concatenated_fasta, bin2contigs, args.bin_folder, args.log_file, args.quiet) shared.run_prodigal( args.path_to_prodigal, args.concatenated_fasta, args.proteins_fasta, args.proteins_gff, args.log_file, args.quiet) contig2ORFs = shared.import_ORFs( args.proteins_fasta, args.log_file, args.quiet) check.check_whether_ORFs_are_based_on_contigs( contig_names, contig2ORFs, args.log_file, args.quiet) if 'align' in step_list: shared.run_diamond(args) (ORF2hits, all_hits) = shared.parse_tabular_alignment( args.alignment_file, args.one_minus_r, args.log_file, args.quiet) (taxid2parent, taxid2rank) = tax.import_nodes( args.nodes_dmp, args.log_file, args.quiet) fastaid2LCAtaxid = tax.import_fastaid2LCAtaxid( args.fastaid2LCAtaxid_file, all_hits, args.log_file, args.quiet) taxids_with_multiple_offspring = tax.import_taxids_with_multiple_offspring( args.taxids_with_multiple_offspring_file, args.log_file, args.quiet) message = 'BAT is flying! Files {0} and {1} are created.'.format( args.bin2classification_output_file, args.ORF2LCA_output_file) shared.give_user_feedback(message, args.log_file, args.quiet) n_classified_bins = 0 with open(args.bin2classification_output_file, 'w') as outf1, open(args.ORF2LCA_output_file, 'w') as outf2: outf1.write('# bin\tclassification\treason\tlineage\tlineage scores\n') outf2.write('# ORF\tbin\tnumber of hits\tlineage\ttop bit-score\n') for bin_ in sorted(bin2contigs): LCAs_ORFs = [] for contig in sorted(bin2contigs[bin_]): if contig not in contig2ORFs: continue for ORF in contig2ORFs[contig]: if ORF not in ORF2hits: outf2.write('{0}\t{1}\tORF has no hit to database\n' ''.format(ORF, bin_)) continue n_hits = len(ORF2hits[ORF]) (taxid, top_bitscore) = tax.find_LCA_for_ORF( ORF2hits[ORF], fastaid2LCAtaxid, taxid2parent) if taxid.startswith('no taxid found'): outf2.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( ORF, bin_, n_hits, taxid, top_bitscore)) else: lineage = tax.find_lineage(taxid, taxid2parent) if not args.no_stars: lineage = tax.star_lineage( lineage, taxids_with_multiple_offspring) outf2.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( ORF, bin_, n_hits, ';'.join(lineage[::-1]), top_bitscore)) LCAs_ORFs.append((taxid, top_bitscore),) if len(LCAs_ORFs) == 0: outf1.write('{0}\tno taxid assigned\tno hits to database\n' ''.format(bin_)) continue (lineages, lineages_scores, based_on_n_ORFs) = tax.find_weighted_LCA( LCAs_ORFs, taxid2parent, args.f) if lineages == 'no ORFs with taxids found.': outf1.write('{0}\tno taxid assigned\t' 'hits not found in taxonomy files\n'.format(bin_)) continue if lineages == 'no lineage whitelisted.': outf1.write( '{0}\tno taxid assigned\t' 'no lineage reached minimum bit-score support\n' ''.format(bin_)) continue # The bin has a valid classification. n_classified_bins += 1 total_n_ORFs = sum([len(contig2ORFs[contig]) for contig in bin2contigs[bin_] if contig in contig2ORFs]) for (i, lineage) in enumerate(lineages): if not args.no_stars: lineage = tax.star_lineage( lineage, taxids_with_multiple_offspring) scores = ['{0:.2f}'.format(score) for score in lineages_scores[i]] if len(lineages) == 1: # There is only one classification. outf1.write( '{0}\t' 'taxid assigned\t' 'based on {1}/{2} ORFs\t' '{3}\t' '{4}\n'.format( bin_, based_on_n_ORFs, total_n_ORFs, ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) else: # There are multiple classifications. outf1.write( '{0}\t' 'taxid assigned ({1}/{2})\t' 'based on {3}/{4} ORFs\t' '{5}\t' '{6}\n'.format( bin_, i + 1, len(lineages), based_on_n_ORFs, total_n_ORFs, ';'.join(lineage[::-1]), ';'.join(scores[::-1]))) message = ('\n-----------------\n\n' '{0} BAT is done! {1:,d}/{2:,d} bins have taxonomy assigned.' ''.format(shared.timestamp(), n_classified_bins, len(bin2contigs))) shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) if args.f < 0.5: message = ('\nWARNING: since f is set to smaller than 0.5, one bin ' 'may have multiple classifications.') shared.give_user_feedback(message, args.log_file, args.quiet, show_time=False) return
def summarise_contigs(input_file, output_file, contigs_fasta, force, quiet): # Currently summarise does not a allow for a log file. log_file = None message = '# CAT v{0}.'.format(about.__version__) shared.give_user_feedback(message, log_file, quiet, show_time=False) errors = [] errors.append(check.check_input_file(input_file, log_file, quiet)) if not force: errors.append(check.check_output_file(output_file, log_file, quiet)) if True in errors: sys.exit(1) contig2length = import_contig_lengths(contigs_fasta, log_file, quiet) message = 'Summarising...' shared.give_user_feedback(message, log_file, quiet) with open(input_file, 'r') as f1: for line in f1: if line.startswith('#'): line = line.split('\t') if line[0] != '# contig': message = ('ERROR: {0} is not a CAT classification file.' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) if line[0] == '# bin': message = ('ERROR: {0} appears to be a BAT ' 'classification file. If you want to ' 'summarise bin classifications, just ' 'don\'t supply a contigs fasta and ' 'everything should be fine!' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) try: superkingdom_index = line.index('superkingdom') except: message = ('ERROR: official ranks not found in header of ' '{0}. Make sure that the CAT classification ' 'file is named with official ranks with \'CAT ' 'add_names --only_official\'.' ''.format(input_file)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) break else: message = 'ERROR: input file does not have a recognisable header.' shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) length = {} length['unclassified'] = [] ORFs = {} official_ranks = ['superkingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species'] for rank in official_ranks: length[rank] = {} ORFs[rank] = {} n = 0 contig_trace = set() doubles = set() with open(input_file, 'r') as f1: for line in f1: line = line.rstrip() if line.startswith('#'): continue n += 1 line = line.split('\t') contig = line[0] if contig in contig_trace: doubles.add(contig) contig_trace.add(contig) if contig not in contig2length: message = ('ERROR: contig {0} in CAT classification file is ' 'not found in supplied contigs fasta file. Are you ' 'sure the CAT classification file is based on the ' 'contigs fasta?'.format(contig)) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) if line[1].startswith('unclassified'): length['unclassified'].append(contig2length[contig]) continue for (i, classification) in enumerate(line[superkingdom_index:]): classification = classification.rsplit(': ', 1)[0].rstrip('*') rank = official_ranks[i] if classification not in length[rank]: length[rank][classification] = [] ORFs[rank][classification] = [] length[rank][classification].append(contig2length[contig]) ORFs[rank][classification].append(int(line[2])) if len(doubles) != 0: message = ('ERROR: some contigs have multiple classifications. CAT ' 'summarise currently does not allow for this. Contigs with ' 'multiple classifications: {0}.' ''.format(', '.join(list(doubles)))) shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) if n != len(contig2length): message = ('ERROR: the number of classified contigs is not the same ' 'as the number of contigs in contigs fasta. Are you sure ' 'the CAT classification file is based on the contigs ' 'fasta?') shared.give_user_feedback(message, log_file, quiet, error=True) sys.exit(1) with open(output_file, 'w') as outf1: number_of_contigs = len(contig2length) total_length = sum(contig2length.values()) number_of_classified_contigs = number_of_contigs - len(length['unclassified']) total_classified_length = total_length - sum(length['unclassified']) outf1.write('# total number of contigs in {0} is {1} representing {2} ' 'positions.\n' ''.format(contigs_fasta, number_of_contigs, total_length)) outf1.write('# {0} contigs are classified ({1:.2f}%) representing {2} ' 'positions ({3:.2f}%) in {4}.\n' ''.format(number_of_classified_contigs, number_of_classified_contigs / number_of_contigs * 100, total_classified_length, total_classified_length / total_length * 100, input_file)) outf1.write('#\n') outf1.write('# rank\t' 'clade\t' 'number of contigs\t' 'number of ORFs\t' 'number of positions\n') for rank in official_ranks: for clade in sorted(length[rank], key=lambda x: sum(length[rank][x]), reverse=True): outf1.write('{0}\t{1}\t{2}\t{3}\t{4}\n' ''.format(rank, clade, len(length[rank][clade]), sum(ORFs[rank][clade]), sum(length[rank][clade]))) message = '{0} is created!'.format(output_file) shared.give_user_feedback(message, log_file, quiet)