Exemplo n.º 1
0
 def test_blastn_commands(self):
     """generate both BLASTN+ and legacy BLASTN commands."""
     # BLAST+
     cmds = anib.generate_blastn_commands(self.blastdbfnames,
                                          self.outdir,
                                          mode="ANIb")
     assert_equal(cmds, self.blastntgt)
     cmds = anib.generate_blastn_commands(self.blastdbfnames,
                                          self.outdir,
                                          mode="ANIblastall")
     assert_equal(cmds, self.blastalltgt)
Exemplo n.º 2
0
 def test_blastn_commands(self):
     """generate BLASTN+ commands."""
     # BLAST+
     cmds = anib.generate_blastn_commands(self.blastdbfnames,
                                          self.outdir,
                                          mode="ANIb")
     assert_equal(cmds, self.blastntgt)
Exemplo n.º 3
0
def test_anib_concordance():
    """Test concordance of ANIb method with JSpecies output.

    This may take some time. Please be patient.
    """
    # Make/check output directory
    mode = "ANIb"
    outdirname = make_outdir(mode)

    # Get dataframes of JSpecies output
    anib_jspecies = parse_table(JSPECIES_OUTFILE, 'ANIb')

    # Identify our input files, and the total lengths of each organism seq
    infiles = pyani_files.get_fasta_files(INDIRNAME)
    org_lengths = pyani_files.get_sequence_lengths(infiles)

    # Test ANIb concordance:
    # Make fragments
    fragfiles, fraglengths = anib.fragment_FASTA_files(infiles, outdirname,
                                                       pyani_config.FRAGSIZE)
    # Build databases
    cmdlist = anib.generate_blastdb_commands(infiles, outdirname,
                                             pyani_config.MAKEBLASTDB_DEFAULT,
                                             mode="ANIb")
    multiprocessing_run(cmdlist)
    # Run pairwise BLASTN
    cmdlist = anib.generate_blastn_commands(fragfiles, outdirname,
                                            pyani_config.BLASTN_DEFAULT,
                                            mode="ANIb")
    multiprocessing_run(cmdlist, verbose=False)
    # Process BLAST; the pid data is in anib_data[1]
    anib_data = anib.process_blast(outdirname, org_lengths, fraglengths,
                                   mode="ANIb")
    anib_pid = anib_data[1].sort(axis=0).sort(axis=1) * 100.

    index, columns = anib_pid.index, anib_pid.columns
    diffmat = anib_pid.as_matrix() - anib_jspecies.as_matrix()
    anib_diff = pd.DataFrame(diffmat, index=index, columns=columns)

    # Write dataframes to file, for reference
    anib_pid.to_csv(os.path.join(outdirname,
                                'ANIb_pid.tab'),
                   sep='\t')
    anib_jspecies.to_csv(os.path.join(outdirname,
                                      'ANIb_jspecies.tab'),
                         sep='\t')
    anib_diff.to_csv(os.path.join(outdirname,
                                  'ANIb_diff.tab'),
                     sep='\t')
    print "ANIb concordance test output placed in %s" % outdirname
    print anib_pid, anib_jspecies, anib_diff

    # We'd like the absolute difference reported to be < ANIB_THRESHOLD
    max_diff = anib_diff.abs().values.max()
    print "Maximum difference for ANIb: %e" % max_diff
    assert_less(max_diff, ANIB_THRESHOLD)
Exemplo n.º 4
0
def test_anib_concordance():
    """Test concordance of ANIb method with JSpecies output.

    This may take some time. Please be patient.
    """
    # Make/check output directory
    mode = "ANIb"
    outdirname = make_outdir(mode)

    # Get dataframes of JSpecies output
    anib_jspecies = parse_table(JSPECIES_OUTFILE, 'ANIb')

    # Identify our input files, and the total lengths of each organism seq
    infiles = pyani_files.get_fasta_files(INDIRNAME)
    org_lengths = pyani_files.get_sequence_lengths(infiles)

    # Test ANIb concordance:
    # Make fragments
    fragfiles, fraglengths = anib.fragment_FASTA_files(infiles, outdirname,
                                                       pyani_config.FRAGSIZE)
    # Build databases
    cmdlist = anib.generate_blastdb_commands(infiles,
                                             outdirname,
                                             pyani_config.MAKEBLASTDB_DEFAULT,
                                             mode="ANIb")
    multiprocessing_run(cmdlist)
    # Run pairwise BLASTN
    cmdlist = anib.generate_blastn_commands(fragfiles,
                                            outdirname,
                                            pyani_config.BLASTN_DEFAULT,
                                            mode="ANIb")
    multiprocessing_run(cmdlist, verbose=False)
    # Process BLAST; the pid data is in anib_data[1]
    anib_data = anib.process_blast(outdirname,
                                   org_lengths,
                                   fraglengths,
                                   mode="ANIb")
    anib_pid = anib_data[1].sort(axis=0).sort(axis=1) * 100.

    index, columns = anib_pid.index, anib_pid.columns
    diffmat = anib_pid.as_matrix() - anib_jspecies.as_matrix()
    anib_diff = pd.DataFrame(diffmat, index=index, columns=columns)

    # Write dataframes to file, for reference
    anib_pid.to_csv(os.path.join(outdirname, 'ANIb_pid.tab'), sep='\t')
    anib_jspecies.to_csv(os.path.join(outdirname, 'ANIb_jspecies.tab'),
                         sep='\t')
    anib_diff.to_csv(os.path.join(outdirname, 'ANIb_diff.tab'), sep='\t')
    print "ANIb concordance test output placed in %s" % outdirname
    print anib_pid, anib_jspecies, anib_diff

    # We'd like the absolute difference reported to be < ANIB_THRESHOLD
    max_diff = anib_diff.abs().values.max()
    print "Maximum difference for ANIb: %e" % max_diff
    assert_less(max_diff, ANIB_THRESHOLD)
Exemplo n.º 5
0
def test_blastall_multiple(path_fna_two, tmp_path):
    """Generate legacy BLASTN commands."""
    cmds = anib.generate_blastn_commands(path_fna_two,
                                         tmp_path,
                                         mode="ANIblastall")
    expected = [
        ("blastall -p blastn -o "
         f"{tmp_path / str(path_fna_two[0].stem + '_vs_' + path_fna_two[1].stem + '.blast_tab')} "
         f"-i {path_fna_two[0]} "
         f"-d {path_fna_two[1]} "
         "-X 150 -q -1 -F F -e 1e-15 -b 1 -v 1 -m 8"),
        ("blastall -p blastn -o "
         f"{tmp_path / str(path_fna_two[1].stem + '_vs_' + path_fna_two[0].stem + '.blast_tab')} "
         f"-i {path_fna_two[1]} "
         f"-d {path_fna_two[0]} "
         "-X 150 -q -1 -F F -e 1e-15 -b 1 -v 1 -m 8"),
    ]
    assert cmds == expected
Exemplo n.º 6
0
def test_blastn_multiple(path_fna_two, tmp_path):
    """Generate BLASTN+ commands."""
    # BLAST+
    cmds = anib.generate_blastn_commands(path_fna_two, tmp_path, mode="ANIb")
    expected = [
        (f"blastn -out {tmp_path / str(path_fna_two[0].stem + '_vs_' + path_fna_two[1].stem + '.blast_tab')} "
         f"-query {path_fna_two[0]} "
         f"-db {path_fna_two[1]} "
         "-xdrop_gap_final 150 -dust no -evalue 1e-15 "
         "-max_target_seqs 1 -outfmt '6 qseqid sseqid "
         "length mismatch pident nident qlen slen qstart "
         "qend sstart send positive ppos gaps' -task blastn"),
        (f"blastn -out {tmp_path / str(path_fna_two[1].stem + '_vs_' + path_fna_two[0].stem + '.blast_tab')} "
         f"-query {path_fna_two[1]} "
         f"-db {path_fna_two[0]} "
         "-xdrop_gap_final 150 -dust no -evalue 1e-15 "
         "-max_target_seqs 1 -outfmt '6 qseqid sseqid length "
         "mismatch pident nident qlen slen qstart qend "
         "sstart send positive ppos gaps' -task blastn"),
    ]
    assert cmds == expected
Exemplo n.º 7
0
 def test_legacy_blastn_commands(self):
     """generate legacy BLASTN commands."""
     cmds = anib.generate_blastn_commands(self.blastdbfnames,
                                          self.outdir,
                                          mode="ANIblastall")
     assert_equal(cmds, self.blastalltgt)
Exemplo n.º 8
0
def unified_anib(infiles, org_lengths):
    """Calculate ANIb for files in input directory.

    - infiles - paths to each input file
    - org_lengths - dictionary of input sequence lengths, keyed by sequence

    Calculates ANI by the ANIb method, as described in Goris et al. (2007)
    Int J Syst Evol Micr 57: 81-91. doi:10.1099/ijs.0.64483-0. There are
    some minor differences depending on whether BLAST+ or legacy BLAST
    (BLASTALL) methods are used.

    All FASTA format files (selected by suffix) in the input directory are
    used to construct BLAST databases, placed in the output directory.
    Each file's contents are also split into sequence fragments of length
    options.fragsize, and the multiple FASTA file that results written to
    the output directory. These are BLASTNed, pairwise, against the
    databases.

    The BLAST output is interrogated for all fragment matches that cover
    at least 70% of the query sequence, with at least 30% nucleotide
    identity over the full length of the query sequence. This is an odd
    choice and doesn't correspond to the twilight zone limit as implied by
    Goris et al. We persist with their definition, however.  Only these
    qualifying matches contribute to the total aligned length, and total
    aligned sequence identity used to calculate ANI.

    The results are processed to give matrices of aligned sequence length
    (aln_lengths.tab), similarity error counts (sim_errors.tab), ANIs
    (perc_ids.tab), and minimum aligned percentage (perc_aln.tab) of
    each genome, for each pairwise comparison. These are written to the
    output directory in plain text tab-separated format.
    """
    logger.info("Running %s" % args.method)
    # Build BLAST databases and run pairwise BLASTN
    if not args.skip_blastn:
        # Make sequence fragments
        logger.info("Fragmenting input files, and writing to %s" %
                    args.outdirname)
        # Fraglengths does not get reused with BLASTN
        fragfiles, fraglengths = anib.fragment_FASTA_files(infiles,
                                                           args.outdirname,
                                                           args.fragsize)
        # Export fragment lengths as JSON, in case we re-run BLASTALL with
        # --skip_blastn
        if args.method == "ANIblastall":
            with open(os.path.join(args.outdirname,
                                   'fraglengths.json'), 'w') as outfile:
                json.dump(fraglengths, outfile)

        # Which executables are we using?
        if args.method == "ANIblastall":
            blastdb_exe = args.formatdb_exe
            blastn_exe = args.blastall_exe
        else:
            blastdb_exe = args.makeblastdb_exe
            blastn_exe = args.blastn_exe

        # Build BLASTN databases
        logger.info("Constructing %s BLAST databases" % args.method)
        cmdlist = anib.generate_blastdb_commands(infiles, args.outdirname,
                                                 blastdb_exe=blastdb_exe,
                                                 mode=args.method)
        logger.info("Generated commands:\n%s" % '\n'.join(cmdlist))
        if args.scheduler == 'multiprocessing':
            logger.info("Running jobs with multiprocessing")
            cumval = multiprocessing_run(cmdlist, verbose=args.verbose)
            if 0 < cumval:
                logger.warning("At least one makeblastdb run failed. " +
                               "%s may fail." % args.method)
            else:
                logger.info("All multiprocessing jobs complete.")
        else:
            logger.info("Running jobs with SGE")
            raise NotImplementedError

        # Run pairwise BLASTN
        logger.info("Running %s BLASTN jobs" % args.method)
        cmdlist = anib.generate_blastn_commands(fragfiles, args.outdirname,
                                                blastn_exe, mode=args.method)
        logger.info("Generated commands:\n%s" % '\n'.join(cmdlist))
        if args.scheduler == 'multiprocessing':
            logger.info("Running jobs with multiprocessing")
            cumval = multiprocessing_run(cmdlist, verbose=args.verbose)
            logger.info("Cumulative return value: %d" % cumval)
            if 0 < cumval:
                logger.warning("At least one BLASTN comparison failed. " +
                               "%s may fail." % args.method)
            else:
                logger.info("All multiprocessing jobs complete.")
        else:
            logger.info("Running jobs with SGE")
            raise NotImplementedError
    else:
        # Import fragment lengths from JSON
        if args.method == "ANIblastall":
            with open(os.path.join(args.outdirname, 'fraglengths.json'),
                      'rU') as infile:
                fraglengths = json.load(infile)
        else:
            fraglengths = None
        logger.warning("Skipping BLASTN runs (as instructed)!")

    # Process pairwise BLASTN output
    logger.info("Processing pairwise %s BLAST output." % args.method)
    try:
        data = anib.process_blast(args.outdirname, org_lengths,
                                  fraglengths=fraglengths, mode=args.method)
    except ZeroDivisionError:
        logger.error("One or more BLAST output files has a problem.")
        if not args.skip_blastn:
            if 0 < cumval:
                logger.error("This is possibly due to BLASTN run failure, " +
                             "please investigate")
            else:
                logger.error("This is possibly due to a BLASTN comparison " +
                             "being too distant for use.")
        logger.error(last_exception())
    return data
Exemplo n.º 9
0
def unified_anib(infiles, org_lengths):
    """Calculate ANIb for files in input directory.

    - infiles - paths to each input file
    - org_lengths - dictionary of input sequence lengths, keyed by sequence

    Calculates ANI by the ANIb method, as described in Goris et al. (2007)
    Int J Syst Evol Micr 57: 81-91. doi:10.1099/ijs.0.64483-0. There are
    some minor differences depending on whether BLAST+ or legacy BLAST
    (BLASTALL) methods are used.

    All FASTA format files (selected by suffix) in the input directory are
    used to construct BLAST databases, placed in the output directory.
    Each file's contents are also split into sequence fragments of length
    options.fragsize, and the multiple FASTA file that results written to
    the output directory. These are BLASTNed, pairwise, against the
    databases.

    The BLAST output is interrogated for all fragment matches that cover
    at least 70% of the query sequence, with at least 30% nucleotide
    identity over the full length of the query sequence. This is an odd
    choice and doesn't correspond to the twilight zone limit as implied by
    Goris et al. We persist with their definition, however.  Only these
    qualifying matches contribute to the total aligned length, and total
    aligned sequence identity used to calculate ANI.

    The results are processed to give matrices of aligned sequence length
    (aln_lengths.tab), similarity error counts (sim_errors.tab), ANIs
    (perc_ids.tab), and minimum aligned percentage (perc_aln.tab) of
    each genome, for each pairwise comparison. These are written to the
    output directory in plain text tab-separated format.
    """
    logger.info("Running %s" % args.method)
    # Build BLAST databases and run pairwise BLASTN
    if not args.skip_blastn:
        # Make sequence fragments
        logger.info("Fragmenting input files, and writing to %s" %
                    args.outdirname)
        # Fraglengths does not get reused with BLASTN
        fragfiles, fraglengths = anib.fragment_FASTA_files(
            infiles, args.outdirname, args.fragsize)
        # Export fragment lengths as JSON, in case we re-run BLASTALL with
        # --skip_blastn
        if args.method == "ANIblastall":
            with open(os.path.join(args.outdirname, 'fraglengths.json'),
                      'w') as outfile:
                json.dump(fraglengths, outfile)

        # Which executables are we using?
        if args.method == "ANIblastall":
            blastdb_exe = args.formatdb_exe
            blastn_exe = args.blastall_exe
        else:
            blastdb_exe = args.makeblastdb_exe
            blastn_exe = args.blastn_exe

        # Build BLASTN databases
        logger.info("Constructing %s BLAST databases" % args.method)
        cmdlist = anib.generate_blastdb_commands(infiles,
                                                 args.outdirname,
                                                 blastdb_exe=blastdb_exe,
                                                 mode=args.method)
        logger.info("Generated commands:\n%s" % '\n'.join(cmdlist))
        if args.scheduler == 'multiprocessing':
            logger.info("Running jobs with multiprocessing")
            cumval = multiprocessing_run(cmdlist, verbose=args.verbose)
            if 0 < cumval:
                logger.warning("At least one makeblastdb run failed. " +
                               "%s may fail." % args.method)
            else:
                logger.info("All multiprocessing jobs complete.")
        else:
            logger.info("Running jobs with SGE")
            raise NotImplementedError

        # Run pairwise BLASTN
        logger.info("Running %s BLASTN jobs" % args.method)
        cmdlist = anib.generate_blastn_commands(fragfiles,
                                                args.outdirname,
                                                blastn_exe,
                                                mode=args.method)
        logger.info("Generated commands:\n%s" % '\n'.join(cmdlist))
        if args.scheduler == 'multiprocessing':
            logger.info("Running jobs with multiprocessing")
            cumval = multiprocessing_run(cmdlist, verbose=args.verbose)
            logger.info("Cumulative return value: %d" % cumval)
            if 0 < cumval:
                logger.warning("At least one BLASTN comparison failed. " +
                               "%s may fail." % args.method)
            else:
                logger.info("All multiprocessing jobs complete.")
        else:
            logger.info("Running jobs with SGE")
            raise NotImplementedError
    else:
        # Import fragment lengths from JSON
        if args.method == "ANIblastall":
            with open(os.path.join(args.outdirname, 'fraglengths.json'),
                      'rU') as infile:
                fraglengths = json.load(infile)
        else:
            fraglengths = None
        logger.warning("Skipping BLASTN runs (as instructed)!")

    # Process pairwise BLASTN output
    logger.info("Processing pairwise %s BLAST output." % args.method)
    try:
        data = anib.process_blast(args.outdirname,
                                  org_lengths,
                                  fraglengths=fraglengths,
                                  mode=args.method)
    except ZeroDivisionError:
        logger.error("One or more BLAST output files has a problem.")
        if not args.skip_blastn:
            if 0 < cumval:
                logger.error("This is possibly due to BLASTN run failure, " +
                             "please investigate")
            else:
                logger.error("This is possibly due to a BLASTN comparison " +
                             "being too distant for use.")
        logger.error(last_exception())
    return data