Пример #1
0
def contigs(args):
    step_list = []

    (contigs_fasta, database_folder, taxonomy_folder, r, one_minus_r, f,
     out_prefix, predicted_proteins_fasta, diamond_file, path_to_prodigal,
     path_to_diamond, no_stars, force, quiet, no_log, nproc, sensitive,
     block_size, index_chunks, tmpdir, top) = check.convert_arguments(args)

    if no_log:
        log_file = None
    else:
        # Check out_prefix already as the log file needs to be written to a
        # valid location.
        error = check.check_out_prefix(out_prefix, None, quiet)
        if error:
            sys.exit(1)

        log_file = '{0}.log'.format(out_prefix)
        with open(log_file, 'w') as outf1:
            pass

    message = '# CAT v{0}.'.format(about.__version__)
    shared.give_user_feedback(message, log_file, quiet, show_time=False)

    # Check at which state to start.
    if predicted_proteins_fasta is None and diamond_file is None:
        message = ('\n'
                   'CAT is running. Protein prediction, alignment, and contig '
                   'classification are carried out.\n'
                   'Rarw!\n\n'
                   'Supplied command: {0}\n\n'
                   'Contigs fasta: {1}\n'
                   'Taxonomy folder: {2}/\n'
                   'Database folder: {3}/\n'
                   'Parameter r: {4}\n'
                   'Parameter f: {5}\n'
                   'Log file: {6}\n\n'
                   '-----------------\n'.format(' '.join(sys.argv),
                                                contigs_fasta, taxonomy_folder,
                                                database_folder, args.r,
                                                args.f, log_file))
        shared.give_user_feedback(message, log_file, quiet, show_time=False)

        step_list.append('run_prodigal')
        step_list.append('run_diamond')
    elif (predicted_proteins_fasta is not None and diamond_file is None):
        message = ('\n'
                   'CAT is running. Since a predicted protein fasta is '
                   'supplied, only alignment and contig classification are '
                   'carried out.\n'
                   'Rarw!\n\n'
                   'Supplied command: {0}\n\n'
                   'Contigs fasta: {1}\n'
                   'Taxonomy folder: {2}/\n'
                   'Database folder: {3}/\n'
                   'Parameter r: {4}\n'
                   'Parameter f: {5}\n'
                   'Log file: {6}\n\n'
                   '-----------------\n'.format(' '.join(sys.argv),
                                                contigs_fasta, taxonomy_folder,
                                                database_folder, args.r,
                                                args.f, log_file))
        shared.give_user_feedback(message, log_file, quiet, show_time=False)

        step_list.append('run_diamond')
    elif (predicted_proteins_fasta is not None and diamond_file is not None):
        message = ('\n'
                   'CAT is running. Since a predicted protein fasta and '
                   'DIAMOND alignment file are supplied, only contig '
                   'classification is carried out.\n'
                   'Rarw!\n\n'
                   'Supplied command: {0}\n\n'
                   'Contigs fasta: {1}\n'
                   'Taxonomy folder: {2}/\n'
                   'Database folder: {3}/\n'
                   'Parameter r: {4}\n'
                   'Parameter f: {5}\n'
                   'Log file: {6}\n\n'
                   '-----------------\n'.format(' '.join(sys.argv),
                                                contigs_fasta, taxonomy_folder,
                                                database_folder, args.r,
                                                args.f, log_file))
        shared.give_user_feedback(message, log_file, quiet, show_time=False)
    elif (predicted_proteins_fasta is None and diamond_file is not None):
        message = ('ERROR: if you want CAT to directly do the classification, '
                   'you should not only supply a DIAMOND alignment table but '
                   'also a predicted protein fasta file with argument '
                   '[-p / --proteins].')
        shared.give_user_feedback(message, log_file, quiet, error=True)

        sys.exit(1)

    # Check binaries, output files, taxonomy folder and database folder, and
    # set parameters.
    message = 'Doing some pre-flight checks first.'
    shared.give_user_feedback(message, log_file, quiet, show_time=False)

    errors = []

    errors.append(check.check_out_prefix(out_prefix, log_file, quiet))

    if 'run_prodigal' in step_list:
        errors.append(
            check.check_prodigal_binaries(path_to_prodigal, log_file, quiet))

        predicted_proteins_fasta = ('{0}.predicted_proteins.faa'
                                    ''.format(out_prefix))
        predicted_proteins_gff = ('{0}.predicted_proteins.gff'
                                  ''.format(out_prefix))

        if not force:
            errors.append(
                check.check_output_file(predicted_proteins_fasta, log_file,
                                        quiet))
            errors.append(
                check.check_output_file(predicted_proteins_gff, log_file,
                                        quiet))

    if 'run_diamond' in step_list:
        errors.append(
            check.check_diamond_binaries(path_to_diamond, log_file, quiet))

        diamond_file = '{0}.alignment.diamond'.format(out_prefix)

        if not force:
            errors.append(
                check.check_output_file(diamond_file, log_file, quiet))
    else:
        diamond_file = diamond_file

    errors.append(
        check.check_folders_for_run(taxonomy_folder, database_folder,
                                    step_list, log_file, quiet))

    contig2classification_output_file = ('{0}.contig2classification.txt'
                                         ''.format(out_prefix))
    ORF2LCA_output_file = '{0}.ORF2LCA.txt'.format(out_prefix)

    if not force:
        errors.append(
            check.check_output_file(contig2classification_output_file,
                                    log_file, quiet))
        errors.append(
            check.check_output_file(ORF2LCA_output_file, log_file, quiet))

    if 'run_prodigal' not in step_list:
        if not check.check_whether_file_is_fasta(predicted_proteins_fasta):
            message = ('ERROR: {0} is not a fasta file.'
                       ''.format(predicted_proteins_fasta))
            shared.give_user_feedback(message, log_file, quiet, error=True)

            errors.append(True)

    errors.append(check.check_top(top, r, log_file, quiet))

    if True in errors:
        sys.exit(1)

    (nodes_dmp, names_dmp, prot_accession2taxid_file
     ) = check.inspect_taxonomy_folder(taxonomy_folder)
    (nr_file, diamond_database, fastaid2LCAtaxid_file,
     taxids_with_multiple_offspring_file
     ) = check.inspect_database_folder(database_folder)

    message = 'Ready to fly!\n\n-----------------\n'
    shared.give_user_feedback(message, log_file, quiet, show_time=False)

    # Start CAT.
    contig_names = shared.import_contig_names(contigs_fasta, log_file, quiet)

    if 'run_prodigal' in step_list:
        shared.run_prodigal(path_to_prodigal, contigs_fasta,
                            predicted_proteins_fasta, predicted_proteins_gff,
                            log_file, quiet)

    contig2ORFs = shared.import_ORFs(predicted_proteins_fasta, log_file, quiet)

    check.check_whether_ORFs_are_based_on_contigs(contig_names, contig2ORFs,
                                                  log_file, quiet)

    if 'run_diamond' in step_list:
        shared.run_diamond(path_to_diamond, diamond_database,
                           predicted_proteins_fasta, diamond_file, nproc,
                           sensitive, block_size, index_chunks, tmpdir, top,
                           log_file, quiet)

    (ORF2hits, all_hits) = shared.parse_diamond_file(diamond_file, one_minus_r,
                                                     log_file, quiet)

    (taxid2parent, taxid2rank) = tax.import_nodes(nodes_dmp, log_file, quiet)
    fastaid2LCAtaxid = tax.import_fastaid2LCAtaxid(fastaid2LCAtaxid_file,
                                                   all_hits, log_file, quiet)
    taxids_with_multiple_offspring = tax.import_taxids_with_multiple_offspring(
        taxids_with_multiple_offspring_file, log_file, quiet)

    message = ('CAT is spinning! Files {0} and {1} are created.'
               ''.format(contig2classification_output_file,
                         ORF2LCA_output_file))
    shared.give_user_feedback(message, log_file, quiet)

    number_of_classified_contigs = 0

    with open(contig2classification_output_file,
              'w') as outf1, open(ORF2LCA_output_file, 'w') as outf2:
        outf1.write('# contig\tclassification\treason\tlineage\t'
                    'lineage scores\n')
        outf2.write('# ORF\tlineage\tbit-score\n')

        for contig in sorted(contig_names):
            if contig not in contig2ORFs:
                outf1.write('{0}\tunclassified\tno ORFs found\n'
                            ''.format(contig))

                continue

            LCAs_ORFs = []

            for ORF in contig2ORFs[contig]:
                if ORF not in ORF2hits:
                    outf2.write('{0}\tORF has no hit to database\n'
                                ''.format(ORF))

                    continue

                (taxid,
                 top_bitscore) = tax.find_LCA_for_ORF(ORF2hits[ORF],
                                                      fastaid2LCAtaxid,
                                                      taxid2parent)

                if taxid.startswith('no taxid found'):
                    outf2.write('{0}\t{1}\t{2}\n'.format(
                        ORF, taxid, top_bitscore))
                else:
                    lineage = tax.find_lineage(taxid, taxid2parent)

                    if not no_stars:
                        lineage = tax.star_lineage(
                            lineage, taxids_with_multiple_offspring)

                    outf2.write('{0}\t{1}\t{2}\n'
                                ''.format(ORF, ';'.join(lineage[::-1]),
                                          top_bitscore))

                LCAs_ORFs.append((taxid, top_bitscore), )

            if len(LCAs_ORFs) == 0:
                outf1.write('{0}\tunclassified\tno hits to database\n'
                            ''.format(contig))

                continue

            (lineages, lineages_scores,
             based_on_number_of_ORFs) = tax.find_weighted_LCA(
                 LCAs_ORFs, taxid2parent, f)

            if lineages == 'no ORFs with taxids found.':
                outf1.write('{0}\tunclassified\t'
                            'hits not found in taxonomy files\n'
                            ''.format(contig))

                continue

            if lineages == 'no lineage whitelisted.':
                outf1.write('{0}\tunclassified\t'
                            'no lineage reached minimum bit-score support\n'
                            ''.format(contig))

                continue

            # The contig has a valid classification.
            number_of_classified_contigs += 1

            for (i, lineage) in enumerate(lineages):
                if not no_stars:
                    lineage = tax.star_lineage(lineage,
                                               taxids_with_multiple_offspring)
                scores = [
                    '{0:.2f}'.format(score) for score in lineages_scores[i]
                ]

                if len(lineages) == 1:
                    # There is only one classification.
                    outf1.write('{0}\tclassified\t'
                                'based on {1}/{2} ORFs\t{3}\t{4}\n'
                                ''.format(contig, based_on_number_of_ORFs,
                                          len(contig2ORFs[contig]),
                                          ';'.join(lineage[::-1]),
                                          ';'.join(scores[::-1])))
                else:
                    # There are multiple classifications.
                    outf1.write('{0}\tclassified ({1}/{2})\t'
                                'based on {3}/{4} ORFs\t{5}\t{6}\n'
                                ''.format(contig, i + 1, len(lineages),
                                          based_on_number_of_ORFs,
                                          len(contig2ORFs[contig]),
                                          ';'.join(lineage[::-1]),
                                          ';'.join(scores[::-1])))

    message = ('\n-----------------\n\n'
               '[{0}] CAT is done! {1}/{2} contigs classified.'
               ''.format(datetime.datetime.now(), number_of_classified_contigs,
                         len(contig_names)))
    shared.give_user_feedback(message, log_file, quiet, show_time=False)

    if f < 0.5:
        message = ('\nWARNING: since f is set to smaller than 0.5, one '
                   'contig may have multiple classifications.')
        shared.give_user_feedback(message, log_file, quiet, show_time=False)
Пример #2
0
def run():
    args = parse_arguments()

    message = '# CAT v{0}.'.format(about.__version__)
    shared.give_user_feedback(message, args.log_file, args.quiet,
            show_time=False)
    
    # Check at which state to start.
    step_list = []
    if not args.proteins_fasta and not args.alignment_file:
        message = (
                '\n'
                'BAT is running. Protein prediction, alignment, and bin '
                'classification are carried out.')
        shared.give_user_feedback(message, args.log_file, args.quiet,
                show_time=False)

        step_list.append('predict_proteins')
        step_list.append('align')
    elif args.proteins_fasta and not args.alignment_file:
        message = (
                '\n'
                'BAT is running. Since a predicted protein fasta is supplied, '
                'only alignment and bin classification are carried out.')
        shared.give_user_feedback(message, args.log_file, args.quiet,
                show_time=False)

        step_list.append('align')
    elif args.proteins_fasta and args.alignment_file:
        message = (
                '\n'
                'BAT is running. Since a predicted protein fasta and '
                'alignment file are supplied, only bin classification is '
                'carried out.')
        shared.give_user_feedback(message, args.log_file, args.quiet,
                show_time=False)
    elif not args.proteins_fasta and args.alignment_file:
        message = (
                'if you want BAT to directly classify a set of bins, you '
                'should not only supply a DIAMOND alignment table but also a '
                'concatenated predicted protein fasta file with argument '
                '[-p / --proteins].')
        shared.give_user_feedback(message, args.log_file, args.quiet,
                error=True)

        sys.exit(1)

    step_list.append('classify')

    # Print variables.
    message = (
            'Rarw!\n\n'
            'Supplied command: {0}\n\n'
            'Bin folder: {1}\n'
            'Taxonomy folder: {2}\n'
            'Database folder: {3}\n'
            'Parameter r: {4}\n'
            'Parameter f: {5}\n'
            'Log file: {6}\n\n'
            '-----------------\n'.format(
                ' '.join(sys.argv),
                args.bin_folder,
                args.taxonomy_folder,
                args.database_folder,
                int(args.r),
                float(args.f),
                args.log_file))
    shared.give_user_feedback(message, args.log_file, args.quiet,
            show_time=False)

    # Check binaries, output files, taxonomy folder and database folder, and
    # set variables.
    message = 'Doing some pre-flight checks first.'
    shared.give_user_feedback(message, args.log_file, args.quiet,
            show_time=False)

    errors = []

    errors.append(
            check.check_bin_folder(
                args.bin_folder, args.bin_suffix, args.log_file, args.quiet))
    
    errors.append(
            check.check_out_prefix(args.out_prefix, args.log_file, args.quiet))
    
    if 'predict_proteins' in step_list:
        errors.append(
                check.check_prodigal_binaries(
                    args.path_to_prodigal, args.log_file, args.quiet))

        setattr(args,
                'concatenated_fasta',
                '{0}.concatenated.fasta'.format(args.out_prefix))
        setattr(args,
                'proteins_fasta',
                '{0}.concatenated.predicted_proteins.faa'.format(
                    args.out_prefix))
        setattr(args,
                'proteins_gff',
                '{0}.concatenated.predicted_proteins.gff'.format(
                    args.out_prefix))

        if not args.force:
            errors.append(
                    check.check_output_file(
                        args.concatenated_fasta, args.log_file, args.quiet))
            errors.append(
                    check.check_output_file(
                        args.proteins_fasta, args.log_file, args.quiet))
            errors.append(
                    check.check_output_file(
                        args.proteins_gff, args.log_file, args.quiet))
            
    if 'align' in step_list:
        errors.append(
                check.check_diamond_binaries(
                    args.path_to_diamond, args.log_file, args.quiet))

        setattr(args,
                'alignment_file',
                '{0}.concatenated.alignment.diamond'.format(args.out_prefix))

        if not args.force:
            errors.append(
                    check.check_output_file(
                        args.alignment_file, args.log_file, args.quiet))

    errors.append(
            check.check_folders_for_run(
                args.taxonomy_folder,
                args.nodes_dmp,
                args.names_dmp,
                args.database_folder,
                args.diamond_database,
                args.fastaid2LCAtaxid_file,
                args.taxids_with_multiple_offspring_file,
                step_list,
                args.log_file,
                args.quiet))

    setattr(args,
            'bin2classification_output_file',
            '{0}.bin2classification.txt'.format(args.out_prefix))
    setattr(args,
            'ORF2LCA_output_file',
            '{0}.ORF2LCA.txt'.format(args.out_prefix))

    if not args.force:
        errors.append(
                check.check_output_file(
                    args.bin2classification_output_file,
                    args.log_file,
                    args.quiet))
        errors.append(
                check.check_output_file(
                    args.ORF2LCA_output_file, args.log_file, args.quiet))
        
    if 'predict_proteins' not in step_list:
        errors.append(
                check.check_fasta(
                    args.proteins_fasta, args.log_file, args.quiet))

    if 'align' in step_list:
        errors.append(
                check.check_top(args.top, args.r, args.log_file, args.quiet))

    # Print all variables.
    shared.print_variables(args, step_list)

    if True in errors:
        sys.exit(1)

    message = 'Ready to fly!\n\n-----------------\n'
    shared.give_user_feedback(message, args.log_file, args.quiet,
            show_time=False)
    
    # Start BAT.
    (bin2contigs, contig_names) = import_bins(
            args.bin_folder, args.bin_suffix, args.log_file, args.quiet)

    if 'predict_proteins' in step_list:
        make_concatenated_fasta(
                args.concatenated_fasta,
                bin2contigs,
                args.bin_folder,
                args.log_file,
                args.quiet)

        shared.run_prodigal(
                args.path_to_prodigal,
                args.concatenated_fasta,
                args.proteins_fasta,
                args.proteins_gff,
                args.log_file,
                args.quiet)
        
    contig2ORFs = shared.import_ORFs(
            args.proteins_fasta, args.log_file, args.quiet)
    
    check.check_whether_ORFs_are_based_on_contigs(
            contig_names, contig2ORFs, args.log_file, args.quiet)
    
    if 'align' in step_list:
        shared.run_diamond(args)

    (ORF2hits,
            all_hits) = shared.parse_tabular_alignment(
                    args.alignment_file,
                    args.one_minus_r,
                    args.log_file,
                    args.quiet)

    (taxid2parent,
            taxid2rank) = tax.import_nodes(
            args.nodes_dmp, args.log_file, args.quiet)
    fastaid2LCAtaxid = tax.import_fastaid2LCAtaxid(
            args.fastaid2LCAtaxid_file, all_hits, args.log_file, args.quiet)
    taxids_with_multiple_offspring = tax.import_taxids_with_multiple_offspring(
            args.taxids_with_multiple_offspring_file,
            args.log_file,
            args.quiet)
    
    message = 'BAT is flying! Files {0} and {1} are created.'.format(
        args.bin2classification_output_file, args.ORF2LCA_output_file)
    shared.give_user_feedback(message, args.log_file, args.quiet)

    n_classified_bins = 0

    with open(args.bin2classification_output_file, 'w') as outf1, open(args.ORF2LCA_output_file, 'w') as outf2:
        outf1.write('# bin\tclassification\treason\tlineage\tlineage scores\n')

        outf2.write('# ORF\tbin\tnumber of hits\tlineage\ttop bit-score\n')
        
        for bin_ in sorted(bin2contigs):
            LCAs_ORFs = []

            for contig in sorted(bin2contigs[bin_]):
                if contig not in contig2ORFs:
                    continue

                for ORF in contig2ORFs[contig]:
                    if ORF not in ORF2hits:
                        outf2.write('{0}\t{1}\tORF has no hit to database\n'
                                ''.format(ORF, bin_))

                        continue

                    n_hits = len(ORF2hits[ORF])

                    (taxid,
                            top_bitscore) = tax.find_LCA_for_ORF(
                                    ORF2hits[ORF],
                                    fastaid2LCAtaxid,
                                    taxid2parent)
                     
                    if taxid.startswith('no taxid found'):
                        outf2.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(
                            ORF, bin_, n_hits, taxid, top_bitscore))
                    else:
                        lineage = tax.find_lineage(taxid, taxid2parent)

                        if not args.no_stars:
                            lineage = tax.star_lineage(
                                    lineage, taxids_with_multiple_offspring)

                        outf2.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(
                            ORF,
                            bin_,
                            n_hits,
                            ';'.join(lineage[::-1]),
                            top_bitscore))
                                       
                    LCAs_ORFs.append((taxid, top_bitscore),)
                    
            if len(LCAs_ORFs) == 0:
                outf1.write('{0}\tno taxid assigned\tno hits to database\n'
                        ''.format(bin_))

                continue

            (lineages,
                    lineages_scores,
                    based_on_n_ORFs) = tax.find_weighted_LCA(
                            LCAs_ORFs, taxid2parent, args.f)

            if lineages == 'no ORFs with taxids found.':
                outf1.write('{0}\tno taxid assigned\t'
                        'hits not found in taxonomy files\n'.format(bin_))

                continue

            if lineages == 'no lineage whitelisted.':
                outf1.write(
                        '{0}\tno taxid assigned\t'
                        'no lineage reached minimum bit-score support\n'
                        ''.format(bin_))

                continue
            
            # The bin has a valid classification.
            n_classified_bins += 1

            total_n_ORFs = sum([len(contig2ORFs[contig]) for
                contig in bin2contigs[bin_] if contig in contig2ORFs])
            
            for (i, lineage) in enumerate(lineages):
                if not args.no_stars:
                    lineage = tax.star_lineage(
                            lineage, taxids_with_multiple_offspring)
                
                scores = ['{0:.2f}'.format(score) for
                        score in lineages_scores[i]]
                
                if len(lineages) == 1:
                    # There is only one classification.
                    outf1.write(
                            '{0}\t'
                            'taxid assigned\t'
                            'based on {1}/{2} ORFs\t'
                            '{3}\t'
                            '{4}\n'.format(
                                bin_,
                                based_on_n_ORFs,
                                total_n_ORFs,
                                ';'.join(lineage[::-1]),
                                ';'.join(scores[::-1])))
                else:
                    # There are multiple classifications.
                    outf1.write(
                            '{0}\t'
                            'taxid assigned ({1}/{2})\t'
                            'based on {3}/{4} ORFs\t'
                            '{5}\t'
                            '{6}\n'.format(
                                bin_,
                                i + 1,
                                len(lineages),
                                based_on_n_ORFs,
                                total_n_ORFs,
                                ';'.join(lineage[::-1]),
                                ';'.join(scores[::-1])))
                                   
    message = ('\n-----------------\n\n'
            '{0} BAT is done! {1:,d}/{2:,d} bins have taxonomy assigned.'
            ''.format(shared.timestamp(), n_classified_bins, len(bin2contigs)))
    shared.give_user_feedback(message, args.log_file, args.quiet,
            show_time=False)
  
    if args.f < 0.5:
        message = ('\nWARNING: since f is set to smaller than 0.5, one bin '
                'may have multiple classifications.')
        shared.give_user_feedback(message, args.log_file, args.quiet,
                show_time=False)

    return