示例#1
0
    def test_anim_concordance(self):
        """ANIm results concordant with JSpecies."""
        # Perform ANIm on the input directory contents
        # We have to separate nucmer/delta-filter command generation
        # because Travis-CI doesn't play nicely with changes we made
        # for local SGE/OGE integration.
        # This might be avoidable with a scheduler flag passed to
        # jobgroup generation in the anim.py module. That's a TODO.
        ncmds, fcmds = anim.generate_nucmer_commands(self.infiles, self.outdir)
        run_mp.multiprocessing_run(ncmds)

        # delta-filter commands need to be treated with care for
        # Travis-CI. Our cluster won't take redirection or semicolon
        # separation in individual commands, but the wrapper we wrote
        # for this (delta_filter_wrapper.py) can't be called under
        # Travis-CI. So we must deconstruct the commands below
        dfcmds = [
            " > ".join([" ".join(fcmd.split()[1:-1]), fcmd.split()[-1]])
            for fcmd in fcmds
        ]
        run_mp.multiprocessing_run(dfcmds)

        results = anim.process_deltadir(self.deltadir, self.orglengths)
        result_pid = results.percentage_identity
        result_pid.to_csv(os.path.join(self.outdir, "pyani_anim.tab"), sep="\t")

        # Compare JSpecies output to results
        result_pid = result_pid.sort_index(axis=0).sort_index(axis=1) * 100.0
        diffmat = result_pid.values - self.target["ANIm"].values
        anim_diff = pd.DataFrame(
            diffmat, index=result_pid.index, columns=result_pid.columns
        )
        anim_diff.to_csv(os.path.join(self.outdir, "pyani_anim_diff.tab"), sep="\t")
        assert_less(anim_diff.abs().values.max(), self.tolerance["ANIm"])
示例#2
0
    def test_anim_concordance(self):
        """Check ANIm results are concordant with JSpecies."""
        # Perform ANIm on the input directory contents
        # We have to separate nucmer/delta-filter command generation
        # because Travis-CI doesn't play nicely with changes we made
        # for local SGE/OGE integration.
        # This might be avoidable with a scheduler flag passed to
        # jobgroup generation in the anim.py module. That's a TODO.
        ncmds, fcmds = anim.generate_nucmer_commands(self.infiles, self.outdir)
        run_mp.multiprocessing_run(ncmds)

        # delta-filter commands need to be treated with care for
        # Travis-CI. Our cluster won't take redirection or semicolon
        # separation in individual commands, but the wrapper we wrote
        # for this (delta_filter_wrapper.py) can't be called under
        # Travis-CI. So we must deconstruct the commands below
        dfcmds = [
            " > ".join([" ".join(fcmd.split()[1:-1]),
                        fcmd.split()[-1]]) for fcmd in fcmds
        ]
        run_mp.multiprocessing_run(dfcmds)

        results = anim.process_deltadir(self.deltadir, self.orglengths)
        result_pid = results.percentage_identity
        result_pid.to_csv(self.outdir / "pyani_anim.tab", sep="\t")

        # Compare JSpecies output to results
        result_pid = result_pid.sort_index(axis=0).sort_index(axis=1) * 100.0
        diffmat = result_pid.values - self.target["ANIm"].values
        anim_diff = pd.DataFrame(diffmat,
                                 index=result_pid.index,
                                 columns=result_pid.columns)
        anim_diff.to_csv(self.outdir / "pyani_anim_diff.tab", sep="\t")
        self.assertLess(anim_diff.abs().values.max(), self.tolerance["ANIm"])
示例#3
0
文件: test_anim.py 项目: zwets/pyani
    def test_multi_cmd_generation(self):
        """generate multiple abstract NUCmer/delta-filter command-lines.

        Tests that all the input files are correctly-paired
        """
        cmds = anim.generate_nucmer_commands(self.files)
        assert_equal(cmds, (self.ncmdlist, self.fcmdlist))
示例#4
0
def test_anim_concordance(
    paths_concordance_fna, path_concordance_jspecies, tolerance_anim, tmp_path
):
    """Check ANIm results are concordant with JSpecies."""
    # Perform ANIm on the input directory contents
    # We have to separate nucmer/delta-filter command generation
    # because Travis-CI doesn't play nicely with changes we made
    # for local SGE/OGE integration.
    # This might be avoidable with a scheduler flag passed to
    # jobgroup generation in the anim.py module. That's a TODO.
    ncmds, fcmds = anim.generate_nucmer_commands(paths_concordance_fna, tmp_path)
    (tmp_path / "nucmer_output").mkdir(exist_ok=True, parents=True)
    run_mp.multiprocessing_run(ncmds)

    # delta-filter commands need to be treated with care for
    # Travis-CI. Our cluster won't take redirection or semicolon
    # separation in individual commands, but the wrapper we wrote
    # for this (delta_filter_wrapper.py) can't be called under
    # Travis-CI. So we must deconstruct the commands below
    dfcmds = [
        " > ".join([" ".join(fcmd.split()[1:-1]), fcmd.split()[-1]]) for fcmd in fcmds
    ]
    run_mp.multiprocessing_run(dfcmds)

    orglengths = pyani_files.get_sequence_lengths(paths_concordance_fna)

    results = anim.process_deltadir(tmp_path / "nucmer_output", orglengths)
    result_pid = results.percentage_identity
    result_pid.to_csv(tmp_path / "pyani_anim.tab", sep="\t")

    # Compare JSpecies output to results
    result_pid = (result_pid.sort_index(axis=0).sort_index(axis=1) * 100.0).values
    tgt_pid = parse_jspecies(path_concordance_jspecies)["ANIm"].values

    assert result_pid - tgt_pid == pytest.approx(0, abs=tolerance_anim)
示例#5
0
    def test_multi_cmd_generation(self):
        """generate multiple abstract NUCmer/delta-filter command-lines.

        Tests that all the input files are correctly-paired
        """
        cmds = anim.generate_nucmer_commands(self.files)
        assert_equal(cmds, (self.ncmdlist, self.fcmdlist))
示例#6
0
def test_mummer_multiple(mummer_cmds_four):
    """Generate multiple abstract NUCmer/delta-filter command-lines.

    Tests that all the input files are correctly-paired
    """
    cmds = anim.generate_nucmer_commands(mummer_cmds_four.infiles)
    print(f"\n{cmds}")
    print((mummer_cmds_four.ncmds, mummer_cmds_four.fcmds))
    assert cmds == (mummer_cmds_four.ncmds, mummer_cmds_four.fcmds)
示例#7
0
def test_anim_collection():
    """Test generation of list of NUCmer comparison commands.
    """
    files = ["file1", "file2", "file3", "file4"]
    cmdlist = anim.generate_nucmer_commands(files)
    assert_equal(cmdlist, ['nucmer -mum -p ./file1_vs_file2 file1 file2',
                           'nucmer -mum -p ./file1_vs_file3 file1 file3',
                           'nucmer -mum -p ./file1_vs_file4 file1 file4',
                           'nucmer -mum -p ./file2_vs_file3 file2 file3',
                           'nucmer -mum -p ./file2_vs_file4 file2 file4',
                           'nucmer -mum -p ./file3_vs_file4 file3 file4'])
    print(cmdlist)
示例#8
0
def test_anim_collection():
    """Test generation of list of NUCmer comparison commands.
    """
    files = ["file1", "file2", "file3", "file4"]
    cmdlist = anim.generate_nucmer_commands(files)
    assert_equal(cmdlist, [
        'nucmer -mum -p ./file1_vs_file2 file1 file2',
        'nucmer -mum -p ./file1_vs_file3 file1 file3',
        'nucmer -mum -p ./file1_vs_file4 file1 file4',
        'nucmer -mum -p ./file2_vs_file3 file2 file3',
        'nucmer -mum -p ./file2_vs_file4 file2 file4',
        'nucmer -mum -p ./file3_vs_file4 file3 file4'
    ])
    print(cmdlist)
示例#9
0
def test_anim_concordance():
    """Test concordance of ANIm method with JSpecies output."""
    # Make/check output directory
    mode = "ANIm"
    outdirname = delete_and_remake_outdir(mode)
    nucmername = os.path.join(outdirname, 'nucmer_output')
    os.makedirs(nucmername, exist_ok=True)

    # Get dataframes of JSpecies output
    anim_jspecies = parse_table(JSPECIES_OUTFILE, 'ANIm')

    # Identify our input files, and the total lengths of each organism seq
    infiles = pyani_files.get_fasta_files(INDIRNAME)
    org_lengths = pyani_files.get_sequence_lengths(infiles)

    # Test ANIm concordance:
    # Run pairwise NUCmer
    cmdlist = anim.generate_nucmer_commands(infiles, outdirname,
                                            pyani_config.NUCMER_DEFAULT)
    print('\n'.join(cmdlist))
    multiprocessing_run(cmdlist)
    # Process .delta files
    results = anim.process_deltadir(nucmername, org_lengths)
    anim_pid = \
        results.percentage_identity.sort_index(axis=0).sort_index(axis=1) * 100.

    print("ANIm data\n", results)

    index, columns = anim_pid.index, anim_pid.columns
    diffmat = anim_pid.as_matrix() - anim_jspecies.as_matrix()
    anim_diff = pd.DataFrame(diffmat, index=index, columns=columns)

    # Write dataframes to file, for reference
    anim_pid.to_csv(os.path.join(outdirname, 'ANIm_pid.tab'), sep='\t')
    anim_jspecies.to_csv(os.path.join(outdirname, 'ANIm_jspecies.tab'),
                         sep='\t')
    anim_diff.to_csv(os.path.join(outdirname, 'ANIm_diff.tab'), sep='\t')
    print("ANIm concordance test output placed in %s" % outdirname)
    print("ANIm PID\n", anim_pid)
    print("ANIm JSpecies\n", anim_jspecies)
    print("ANIm diff\n", anim_diff)

    # We'd like the absolute difference reported to be < ANIB_THRESHOLD
    max_diff = anim_diff.abs().values.max()
    print("Maximum difference for ANIm: %e" % max_diff)
    assert_less(max_diff, ANIM_THRESHOLD)
示例#10
0
def test_anim_concordance():
    """Test concordance of ANIm method with JSpecies output."""
    # Make/check output directory
    mode = "ANIm"
    outdirname = make_outdir(mode)

    # Get dataframes of JSpecies output
    anim_jspecies = parse_table(JSPECIES_OUTFILE, 'ANIm')

    # Identify our input files, and the total lengths of each organism seq
    infiles = pyani_files.get_fasta_files(INDIRNAME)
    org_lengths = pyani_files.get_sequence_lengths(infiles)

    # Test ANIm concordance:
    # Run pairwise NUCmer
    cmdlist = anim.generate_nucmer_commands(infiles, outdirname,
                                            pyani_config.NUCMER_DEFAULT)
    multiprocessing_run(cmdlist, verbose=False)
    # Process .delta files
    anim_data = anim.process_deltadir(outdirname, org_lengths)
    anim_pid = anim_data[1].sort(axis=0).sort(axis=1) * 100.

    print anim_data

    index, columns = anim_pid.index, anim_pid.columns
    diffmat = anim_pid.as_matrix() - anim_jspecies.as_matrix()
    anim_diff = pd.DataFrame(diffmat, index=index, columns=columns)

    # Write dataframes to file, for reference
    anim_pid.to_csv(os.path.join(outdirname,
                                'ANIm_pid.tab'),
                   sep='\t')
    anim_jspecies.to_csv(os.path.join(outdirname,
                                      'ANIm_jspecies.tab'),
                         sep='\t')
    anim_diff.to_csv(os.path.join(outdirname,
                                  'ANIm_diff.tab'),
                     sep='\t')
    print "ANIm concordance test output placed in %s" % outdirname
    print anim_pid, anim_jspecies, anim_diff

    # We'd like the absolute difference reported to be < ANIB_THRESHOLD
    max_diff = anim_diff.abs().values.max()
    print "Maximum difference for ANIm: %e" % max_diff
    assert_less(max_diff, ANIM_THRESHOLD)
示例#11
0
 def test_anim_collection(self):
     """Test generation of list of NUCmer comparison commands."""
     files = [Path("file1"), Path("file2"), Path("file3"), Path("file4")]
     cmds_nucmer, cmds_filter = anim.generate_nucmer_commands(files)
     tgts_nucmer = [
         "nucmer --mum -p nucmer_output/file1_vs_file2 file1 file2",
         "nucmer --mum -p nucmer_output/file1_vs_file3 file1 file3",
         "nucmer --mum -p nucmer_output/file1_vs_file4 file1 file4",
         "nucmer --mum -p nucmer_output/file2_vs_file3 file2 file3",
         "nucmer --mum -p nucmer_output/file2_vs_file4 file2 file4",
         "nucmer --mum -p nucmer_output/file3_vs_file4 file3 file4",
     ]
     tgts_filter = [
         "delta_filter_wrapper.py delta-filter -1 nucmer_output/file1_vs_file2.delta nucmer_output/file1_vs_file2.filter",
         "delta_filter_wrapper.py delta-filter -1 nucmer_output/file1_vs_file3.delta nucmer_output/file1_vs_file3.filter",
         "delta_filter_wrapper.py delta-filter -1 nucmer_output/file1_vs_file4.delta nucmer_output/file1_vs_file4.filter",
         "delta_filter_wrapper.py delta-filter -1 nucmer_output/file2_vs_file3.delta nucmer_output/file2_vs_file3.filter",
         "delta_filter_wrapper.py delta-filter -1 nucmer_output/file2_vs_file4.delta nucmer_output/file2_vs_file4.filter",
         "delta_filter_wrapper.py delta-filter -1 nucmer_output/file3_vs_file4.delta nucmer_output/file3_vs_file4.filter",
     ]
     self.assertEqual(cmds_nucmer, tgts_nucmer)
     self.assertEqual(cmds_filter, tgts_filter)
示例#12
0
def calculate_anim(infiles, org_lengths):
    """Returns ANIm result dataframes for files in input directory.

    - infiles - paths to each input file
    - org_lengths - dictionary of input sequence lengths, keyed by sequence

    Finds ANI by the ANIm method, as described in Richter et al (2009)
    Proc Natl Acad Sci USA 106: 19126-19131 doi:10.1073/pnas.0906412106.

    All FASTA format files (selected by suffix) in the input directory
    are compared against each other, pairwise, using NUCmer (which must
    be in the path). NUCmer output is stored in the output directory.

    The NUCmer .delta file output is parsed to obtain an alignment length
    and similarity error count for every unique region alignment between
    the two organisms, as represented by the sequences in the FASTA files.

    These are processed to give matrices of aligned sequence lengths,
    average nucleotide identity (ANI) percentages, coverage (aligned
    percentage of whole genome), and similarity error cound for each pairwise
    comparison.
    """
    logger.info("Running ANIm")
    logger.info("Generating NUCmer command-lines")
    # Schedule NUCmer runs
    if not args.skip_nucmer:
        cmdlist = anim.generate_nucmer_commands(infiles, args.outdirname,
                                                nucmer_exe=args.nucmer_exe,
                                                maxmatch=args.maxmatch)
        logger.info("NUCmer commands:\n" + os.linesep.join(cmdlist))
        if args.scheduler == 'multiprocessing':
            logger.info("Running jobs with multiprocessing")
            cumval = multiprocessing_run(cmdlist, verbose=args.verbose)
            logger.info("Cumulative return value: %d" % cumval)
            if 0 < cumval:
                logger.warning("At least one NUCmer comparison failed. " +
                               "ANIm may fail.")
            else:
                logger.info("All multiprocessing jobs complete.")
        else:
            logger.info("Running jobs with SGE")
            raise NotImplementedError
    else:
        logger.warning("Skipping NUCmer run (as instructed)!")

    # Process resulting .delta files
    logger.info("Processing NUCmer .delta files.")
    try:
        data = anim.process_deltadir(args.outdirname, org_lengths)
    except ZeroDivisionError:
        logger.error("One or more NUCmer output files has a problem.")
        if not args.skip_nucmer:
            if 0 < cumval:
                logger.error("This is possibly due to NUCmer run failure, " +
                             "please investigate")
            else:
                logger.error("This is possibly due to a NUCmer comparison " +
                             "being too distant for use. Please consider " +
                             "using the --maxmatch option.")
        logger.error(last_exception())
    return data
示例#13
0
def calculate_anim(infiles, org_lengths):
    """Returns ANIm result dataframes for files in input directory.

    - infiles - paths to each input file
    - org_lengths - dictionary of input sequence lengths, keyed by sequence

    Finds ANI by the ANIm method, as described in Richter et al (2009)
    Proc Natl Acad Sci USA 106: 19126-19131 doi:10.1073/pnas.0906412106.

    All FASTA format files (selected by suffix) in the input directory
    are compared against each other, pairwise, using NUCmer (which must
    be in the path). NUCmer output is stored in the output directory.

    The NUCmer .delta file output is parsed to obtain an alignment length
    and similarity error count for every unique region alignment between
    the two organisms, as represented by the sequences in the FASTA files.

    These are processed to give matrices of aligned sequence lengths,
    average nucleotide identity (ANI) percentages, coverage (aligned
    percentage of whole genome), and similarity error cound for each pairwise
    comparison.
    """
    logger.info("Running ANIm")
    logger.info("Generating NUCmer command-lines")
    # Schedule NUCmer runs
    if not args.skip_nucmer:
        cmdlist = anim.generate_nucmer_commands(infiles,
                                                args.outdirname,
                                                nucmer_exe=args.nucmer_exe,
                                                maxmatch=args.maxmatch)
        logger.info("NUCmer commands:\n" + os.linesep.join(cmdlist))
        if args.scheduler == 'multiprocessing':
            logger.info("Running jobs with multiprocessing")
            cumval = multiprocessing_run(cmdlist, verbose=args.verbose)
            logger.info("Cumulative return value: %d" % cumval)
            if 0 < cumval:
                logger.warning("At least one NUCmer comparison failed. " +
                               "ANIm may fail.")
            else:
                logger.info("All multiprocessing jobs complete.")
        else:
            logger.info("Running jobs with SGE")
            raise NotImplementedError
    else:
        logger.warning("Skipping NUCmer run (as instructed)!")

    # Process resulting .delta files
    logger.info("Processing NUCmer .delta files.")
    try:
        data = anim.process_deltadir(args.outdirname, org_lengths)
    except ZeroDivisionError:
        logger.error("One or more NUCmer output files has a problem.")
        if not args.skip_nucmer:
            if 0 < cumval:
                logger.error("This is possibly due to NUCmer run failure, " +
                             "please investigate")
            else:
                logger.error("This is possibly due to a NUCmer comparison " +
                             "being too distant for use. Please consider " +
                             "using the --maxmatch option.")
        logger.error(last_exception())
    return data