Example #1
0
def test_atomiccmd__paths_non_str(temp_folder):
    cmd = AtomicCmd(("touch", 1234),
                    OUT_FOO="1234",
                    set_cwd=True)
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert os.path.exists(os.path.join(temp_folder, "1234"))
Example #2
0
def test_atomiccmd__paths__key(temp_folder):
    cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"),
                    OUT_STDOUT=AtomicCmd.PIPE)
    cmd.run(temp_folder)
    path = cmd._proc.stdout.read()
    assert os.path.samefile(temp_folder, path), (temp_folder, path)
    assert_equal(cmd.join(), [0])
Example #3
0
def test_atomiccmd__commit_before_join(temp_folder):
    cmd = AtomicCmd(("sleep", "0.1"))
    cmd.run(temp_folder)
    while cmd._proc.poll() is None:
        pass
    assert_raises(CmdError, cmd.commit, temp_folder)
    cmd.join()
Example #4
0
 def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
     cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
     cmd.run(temp_folder)
     assert_equal(cmd.join(), [0])
     result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd))))
     result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd))))
     assert_equal(result_out, "STDOUT!")
     assert_equal(result_err, "STDERR!")
Example #5
0
def test_pformat__simple__done__set_cwd(temp_folder):
    cmd = AtomicCmd("true", set_cwd=True)
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert_equal(pformat(cmd), ("<Command = ['true']\n"
                                " Status  = Exited with return-code 0\n"
                                " STDOUT* = 'pipe_true_{id}.stdout'\n"
                                " STDERR* = 'pipe_true_{id}.stderr'\n"
                                " CWD     = '{temp_dir}'>").format(id=id(cmd),
                                                                   temp_dir=temp_folder))
Example #6
0
def test_atomiccmd__commit_missing_files(temp_folder):
    destination, temp_folder = _setup_for_commit(temp_folder, False)
    cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
                    OUT_FOO=os.path.join(destination, "1234"),
                    OUT_BAR=os.path.join(destination, "4567"))
    cmd.run(temp_folder)
    cmd.join()
    before = set(os.listdir(temp_folder))
    assert_raises(CmdError, cmd.commit, temp_folder)
    assert_equal(before, set(os.listdir(temp_folder)))
Example #7
0
def test_atomiccmd__pipes_stdin(temp_folder):
    fname = test_file("fasta_file.fasta")
    cmd = AtomicCmd("cat",
                    IN_STDIN=fname,
                    OUT_STDOUT="result.txt")
    assert_equal(cmd.input_files, frozenset([fname]))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    result = get_file_contents(os.path.join(temp_folder, "result.txt"))
    assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
Example #8
0
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
    cmd = AtomicCmd("cat",
                    TEMP_IN_STDIN="infile.fasta",
                    OUT_STDOUT="result.txt")
    assert_equal(cmd.input_files, frozenset())
    set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    result = get_file_contents(os.path.join(temp_folder, "result.txt"))
    assert_equal(result, "a\nbc\nd")
Example #9
0
    def _test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
        cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
                        TEMP_OUT_STDOUT="result.txt",
                        set_cwd=set_cwd,
                        **kwargs)
        cmd.run(temp_folder)
        assert_equal(cmd.join(), [0])

        expected = os.path.join("" if set_cwd else temp_folder, "test_file")
        result = get_file_contents(os.path.join(temp_folder, "result.txt"))
        assert_equal(os.path.abspath(expected), os.path.abspath(result))
Example #10
0
def test_pformat__simple__done(temp_folder):
    cmd = AtomicCmd("true")
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert_equal(pformat(cmd),
                 ("Command = true\n"
                  "Status  = Exited with return-code 0\n"
                  "STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
                  "CWD     = '{cwd}'").format(id=id(cmd),
                                              cwd=os.getcwd(),
                                              temp_dir=temp_folder))
Example #11
0
    def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd):
        cwd = os.getcwd()
        cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"),
                        TEMP_OUT_STDOUT="result.txt",
                        set_cwd=set_cwd)
        cmd.run(temp_folder)
        assert_equal(cmd.join(), [0])
        assert_equal(cwd, os.getcwd())

        expected = temp_folder if set_cwd else cwd
        result = get_file_contents(os.path.join(temp_folder, "result.txt"))
        assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
Example #12
0
    def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
        assert_equal(paleomix.atomiccmd.command._PROCS, set())
        cmd = AtomicCmd("ls")
        cmd.run(temp_folder)
        ref = iter(paleomix.atomiccmd.command._PROCS).next()
        assert ref
        assert_equal(ref(), cmd._proc)

        assert_equal(cmd.join(), [0])
        cmd = func(cmd, temp_folder)

        assert ref not in paleomix.atomiccmd.command._PROCS
Example #13
0
def test_commandnode_teardown__missing_optional_files(temp_folder):
    destination, temp_folder = _setup_temp_folders(temp_folder)

    cmd = AtomicCmd(("echo", "-n", "1 2 3"),
                    IN_DUMMY=_EMPTY_FILE,
                    TEMP_OUT_BAR="bar.txt",
                    OUT_STDOUT=os.path.join(destination, "foo.txt"))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    node = CommandNode(cmd)
    node._teardown(None, temp_folder)
    assert_equal(os.listdir(temp_folder), [])
    assert_equal(os.listdir(destination), ["foo.txt"])
Example #14
0
def test_pformat__simple__temp_root_in_arguments(temp_folder):
    cmd = AtomicCmd(("echo", "${TEMP_DIR}"))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert_equal(pformat(cmd),
                 ("Command = echo '{temp_dir}'\n"
                  "Status  = Exited with return-code 0\n"
                  "STDOUT* = '{temp_dir}/pipe_echo_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_echo_{id}.stderr'\n"
                  "CWD     = '{cwd}'")
                 .format(id=id(cmd),
                         temp_dir=temp_folder,
                         cwd=os.getcwd()))
Example #15
0
def test_atomiccmd__piping_is_only_allowed_once(temp_folder):
    cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"],
                      OUT_STDOUT=AtomicCmd.PIPE)
    cmd_2a = AtomicCmd(["grep", "foo"],
                       IN_STDIN=cmd_1)
    cmd_2b = AtomicCmd(["grep", "bar"],
                       IN_STDIN=cmd_1)
    cmd_1.run(temp_folder)
    cmd_2a.run(temp_folder)
    assert_raises(CmdError, cmd_2b.run, temp_folder)
    assert_equal(cmd_1.join(), [0])
    assert_equal(cmd_2a.join(), [0])
    assert_equal(cmd_2b.join(), [None])
Example #16
0
def test_atomiccmd__ready(tmp_path):
    cmd = AtomicCmd("ls")
    assert cmd.join() == [None]
    assert not cmd.ready()
    cmd.run(tmp_path)
    assert cmd.join() == [0]
    assert cmd.ready()
Example #17
0
def test_atomiccmd__ready(temp_folder):
    cmd = AtomicCmd("ls")
    assert_equal(cmd.join(), [None])
    assert not cmd.ready()
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert cmd.ready()
Example #18
0
def test_commandnode_teardown(temp_folder):
    destination, temp_folder = _setup_temp_folders(temp_folder)

    cmd = AtomicCmd(("echo", "-n", "1 2 3"),
                    IN_DUMMY=_EMPTY_FILE,
                    OUT_STDOUT=os.path.join(destination, "foo.txt"))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    node = CommandNode(cmd)
    assert os.path.exists(os.path.join(temp_folder, "foo.txt"))
    assert not os.path.exists(os.path.join(destination, "foo.txt"))
    node._teardown(None, temp_folder)
    assert not os.path.exists(os.path.join(temp_folder, "foo.txt"))
    assert os.path.exists(os.path.join(destination, "foo.txt"))
Example #19
0
def test_pformat__simple__done__before_join(temp_folder):
    cmd = AtomicCmd("true")
    cmd.run(temp_folder)
    # pylint: disable=protected-access
    cmd._proc.wait()
    assert_equal(pformat(cmd),
                 ("Command = true\n"
                  "Status  = Exited with return-code 0\n"
                  "STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
                  "CWD     = '{cwd}'").format(id=id(cmd),
                                              cwd=os.getcwd(),
                                              temp_dir=temp_folder))
    assert_equal(cmd.join(), [0])
Example #20
0
def test_pformat__simple__killed_by_signal(temp_folder):
    cmd = AtomicCmd(("sleep", "10"))
    cmd.run(temp_folder)
    # pylint: disable=protected-access
    os.killpg(cmd._proc.pid, signal.SIGTERM)
    assert_equal(cmd.join(), ["SIGTERM"])
    assert_equal(pformat(cmd),
                 ("Command = sleep 10\n"
                  "Status  = Terminated with signal SIGTERM\n"
                  "STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
                  "CWD     = '{cwd}'").format(id=id(cmd),
                                              temp_dir=temp_folder,
                                              cwd=os.getcwd()))
Example #21
0
def test_pformat__simple__killed_by_signal(temp_folder):
    cmd = AtomicCmd(("sleep", "10"))
    cmd.run(temp_folder)
    # pylint: disable=protected-access
    os.killpg(cmd._proc.pid, signal.SIGTERM)
    assert_equal(cmd.join(), ["SIGTERM"])
    assert_equal(pformat(cmd),
                 ("Command = sleep 10\n"
                  "Status  = Terminated with signal SIGTERM\n"
                  "STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
                  "CWD     = '{cwd}'").format(id=id(cmd),
                                              temp_dir=temp_folder,
                                              cwd=os.getcwd()))
Example #22
0
    def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
        cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
        cmd.run(temp_folder)
        assert_equal(cmd.join(), [0])

        expected_files = []
        for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
            if tmpl is not None:
                fname = tmpl.format(id(cmd))
                result = get_file_contents(os.path.join(temp_folder, fname))
                assert_equal(result, text)
                expected_files.append(fname)

        assert_equal(set(os.listdir(temp_folder)), set(expected_files))
Example #23
0
def test_pformat__simple__done__before_join(temp_folder):
    cmd = AtomicCmd("true")
    cmd.run(temp_folder)
    # pylint: disable=protected-access
    cmd._proc.wait()
    assert_equal(pformat(cmd),
                 ("Command = true\n"
                  "Status  = Exited with return-code 0\n"
                  "STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
                  "CWD     = '{cwd}'").format(id=id(cmd),
                                              cwd=os.getcwd(),
                                              temp_dir=temp_folder))
    assert_equal(cmd.join(), [0])
Example #24
0
def _setup_for_commit(tmp_path, create_cmd=True):
    destination = tmp_path / "out"
    tmp_path = tmp_path / "tmp"
    os.makedirs(destination)
    os.makedirs(tmp_path)

    if not create_cmd:
        return destination, tmp_path

    cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
                    OUT_FOO=os.path.join(destination, "1234"))
    cmd.run(tmp_path)
    assert cmd.join() == [0]

    return destination, tmp_path, cmd
Example #25
0
def test_commandnode_teardown__missing_optional_files(tmp_path):
    destination, tmp_path = _setup_temp_folders(tmp_path)

    cmd = AtomicCmd(
        ("echo", "-n", "1 2 3"),
        IN_DUMMY=_EMPTY_FILE,
        TEMP_OUT_BAR="bar.txt",
        OUT_STDOUT=os.path.join(destination, "foo.txt"),
    )
    cmd.run(tmp_path)
    assert cmd.join() == [0]
    node = CommandNode(cmd)
    node._teardown(None, tmp_path)
    assert os.listdir(tmp_path) == []
    assert os.listdir(destination) == ["foo.txt"]
Example #26
0
def _setup_for_commit(temp_folder, create_cmd=True):
    destination = os.path.join(temp_folder, "out")
    temp_folder = os.path.join(temp_folder, "tmp")
    os.makedirs(destination)
    os.makedirs(temp_folder)

    if not create_cmd:
        return destination, temp_folder

    cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
                    OUT_FOO=os.path.join(destination, "1234"))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])

    return destination, temp_folder, cmd
Example #27
0
def test_atomiccmd__commit_before_join(tmp_path):
    cmd = AtomicCmd(("sleep", "0.1"))
    cmd.run(tmp_path)
    while cmd._proc.poll() is None:
        pass
    with pytest.raises(CmdError):
        cmd.commit(tmp_path)
    cmd.join()
Example #28
0
def _setup_for_commit(temp_folder, create_cmd=True):
    destination = os.path.join(temp_folder, "out")
    temp_folder = os.path.join(temp_folder, "tmp")
    os.makedirs(destination)
    os.makedirs(temp_folder)

    if not create_cmd:
        return destination, temp_folder

    cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
                    OUT_FOO=os.path.join(destination, "1234"))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])

    return destination, temp_folder, cmd
Example #29
0
def test_pformat__atomiccmd__simple_with_stdin():
    cmd = AtomicCmd("gzip", IN_STDIN="/etc/fstab")
    assert pformat(cmd) == ("Command = gzip\n"
                            "STDIN   = '/etc/fstab'\n"
                            "STDOUT* = '${TEMP_DIR}/pipe_gzip_%i.stdout'\n"
                            "STDERR* = '${TEMP_DIR}/pipe_gzip_%i.stderr'") % (
                                id(cmd), id(cmd))
Example #30
0
def test_commandnode_teardown__extra_files_in_temp(temp_folder):
    destination, temp_folder = _setup_temp_folders(temp_folder)

    cmd = AtomicCmd(("echo", "-n", "1 2 3"),
                    IN_DUMMY=_EMPTY_FILE,
                    OUT_STDOUT=os.path.join(destination, "foo.txt"))
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    node = CommandNode(cmd)
    set_file_contents(os.path.join(temp_folder, "bar.txt"), "1 2 3")
    temp_files_before = set(os.listdir(temp_folder))
    dest_files_before = set(os.listdir(destination))

    assert_raises(CmdNodeError, node._teardown, None, temp_folder)
    assert_equal(temp_files_before, set(os.listdir(temp_folder)))
    assert_equal(dest_files_before, set(os.listdir(destination)))
Example #31
0
def test_pformat__simple():
    cmd = AtomicCmd(("touch", "something"))
    assert_equal(pformat(cmd),
                 ("Command = touch something\n"
                  "STDOUT* = '${TEMP_DIR}/pipe_touch_%i.stdout'\n"
                  "STDERR* = '${TEMP_DIR}/pipe_touch_%i.stderr'") %
                 (id(cmd), id(cmd)))
Example #32
0
def test_commandnode_teardown(tmp_path):
    destination, tmp_path = _setup_temp_folders(tmp_path)

    cmd = AtomicCmd(
        ("echo", "-n", "1 2 3"),
        IN_DUMMY=_EMPTY_FILE,
        OUT_STDOUT=os.path.join(destination, "foo.txt"),
    )
    cmd.run(tmp_path)
    assert cmd.join() == [0]
    node = CommandNode(cmd)
    assert os.path.exists(os.path.join(tmp_path, "foo.txt"))
    assert not os.path.exists(os.path.join(destination, "foo.txt"))
    node._teardown(None, tmp_path)
    assert not os.path.exists(os.path.join(tmp_path, "foo.txt"))
    assert os.path.exists(os.path.join(destination, "foo.txt"))
Example #33
0
    def finalize(self):
        """Creates an AtomicCmd object based on the AtomicParam object. Once
        finalized, the AtomicCmdBuilder cannot be modified further."""
        if not self._object:
            self._object = AtomicCmd(self.call, **self.kwargs)

        return self._object
Example #34
0
def test_pformat__atomiccmd__simple_with_temp_outfile():
    cmd = AtomicCmd(("touch", "%(TEMP_OUT_RC)s"), TEMP_OUT_RC="bashrc")

    assert pformat(cmd) == ("Command = touch '${TEMP_DIR}/bashrc'\n"
                            "STDOUT* = '${TEMP_DIR}/pipe_touch_%i.stdout'\n"
                            "STDERR* = '${TEMP_DIR}/pipe_touch_%i.stderr'") % (
                                id(cmd), id(cmd))
Example #35
0
def test_pformat__atomiccmd__simple_with_temp_infile():
    cmd = AtomicCmd(("cat", "%(TEMP_IN_FILE)s"), TEMP_IN_FILE="infile.txt")

    assert pformat(cmd) == ("Command = cat '${TEMP_DIR}/infile.txt'\n"
                            "STDOUT* = '${TEMP_DIR}/pipe_cat_%i.stdout'\n"
                            "STDERR* = '${TEMP_DIR}/pipe_cat_%i.stderr'") % (
                                id(cmd), id(cmd))
Example #36
0
def test_pformat__atomiccmd__simple_with_temp_stdin__set_cwd():
    cmd = AtomicCmd("gzip", TEMP_IN_STDIN="stabstabstab", set_cwd=True)
    assert pformat(cmd) == ("Command = gzip\n"
                            "STDIN*  = 'stabstabstab'\n"
                            "STDOUT* = 'pipe_gzip_%i.stdout'\n"
                            "STDERR* = 'pipe_gzip_%i.stderr'\n"
                            "CWD     = '${TEMP_DIR}'") % (id(cmd), id(cmd))
Example #37
0
    def __init__(self, input_prefix, output_prefix, nchroms, dependencies=()):
        self._input_prefix = input_prefix
        self._output_prefix = output_prefix
        self._nchroms = nchroms

        cmd = AtomicCmd(
            ("smartpca", "-p", "%(TEMP_OUT_PARAMS)s"),
            TEMP_OUT_PARAMS="parameters.txt",
            IN_FILE_BED=input_prefix + ".bed",
            IN_FILE_BIM=input_prefix + ".bim",
            IN_FILE_FAM=input_prefix + ".fam",
            OUT_STDOUT=output_prefix + ".log",
            OUT_EVEC=output_prefix + ".evec",
            OUT_EVAL=output_prefix + ".eval",
            OUT_SNPS=output_prefix + ".deleted_snps",
            CHECK_VERSION=SMARTPCA_VERSION,
            set_cwd=True,
        )

        CommandNode.__init__(
            self,
            description="<SmartPCA -> '%s.*>" % (output_prefix, ),
            command=cmd,
            dependencies=dependencies,
        )
Example #38
0
    def __init__(self,
                 samples,
                 treefile,
                 bootstraps,
                 output_prefix,
                 dependencies=()):
        rscript = rtools.rscript("zonkey", "tinytree.r")

        cmd = AtomicCmd(("Rscript", rscript, "%(TEMP_OUT_FILE)s",
                         "%(IN_SAMPLES)s", "%(TEMP_OUT_PREFIX)s"),
                        AUX_RSCRIPT=rscript,
                        IN_SAMPLES=samples,
                        IN_FILE=treefile,
                        IN_BOOTSTRAPS=bootstraps,
                        TEMP_OUT_FILE="rerooted.newick",
                        TEMP_OUT_PREFIX=os.path.basename(output_prefix),
                        OUT_TREE_PDF=output_prefix + ".pdf",
                        OUT_TREE_PNG=output_prefix + ".png",
                        CHECK_RSCRIPT=RSCRIPT_VERSION,
                        CHECK_RSCRIPT_APE=rtools.requirement("ape"),
                        CHECK_RSCRIPT_GGPLOT2=rtools.requirement("ggplot2"),
                        CHECK_RSCRIPT_GRID=rtools.requirement("grid"))

        self._treefile = treefile
        self._bootstraps = bootstraps

        CommandNode.__init__(self,
                             description="<DrawPhylogeny -> '%s.*'>" %
                             (output_prefix, ),
                             command=cmd,
                             dependencies=dependencies)
Example #39
0
    def __init__(self, config, reference, intervals, infiles, outfile,
                 dependencies=()):
        self._basename = os.path.basename(outfile)

        infiles = safe_coerce_to_tuple(infiles)
        jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar")
        command = AtomicJavaCmdBuilder(jar_file,
                                       jre_options=config.jre_options)
        command.set_option("-T", "IndelRealigner")
        command.set_option("-R", "%(IN_REFERENCE)s")
        command.set_option("-targetIntervals", "%(IN_INTERVALS)s")
        command.set_option("-o", "%(OUT_BAMFILE)s")
        command.set_option("--bam_compression", 0)
        command.set_option("--disable_bam_indexing")
        _set_input_files(command, infiles)

        command.set_kwargs(IN_REFERENCE=reference,
                           IN_REF_DICT=fileutils.swap_ext(reference, ".dict"),
                           IN_INTERVALS=intervals,
                           OUT_BAMFILE=outfile,
                           CHECK_GATK=_get_gatk_version_check(config))

        calmd = AtomicCmd(["samtools", "calmd", "-b",
                           "%(TEMP_IN_BAM)s", "%(IN_REF)s"],
                          TEMP_IN_BAM=self._basename,
                          IN_REF=reference,
                          TEMP_OUT_STDOUT=self._basename + ".calmd",
                          CHECK_VERSION=SAMTOOLS_VERSION)

        description = "<GATK Indel Realigner (aligning): %s -> %r>" \
            % (describe_files(infiles), outfile)
        CommandNode.__init__(self,
                             description=description,
                             command=ParallelCmds([command.finalize(), calmd]),
                             dependencies=dependencies)
Example #40
0
    def __init__(self,
                 contigs,
                 mapping,
                 input_file,
                 output_prefix,
                 dependencies=()):
        self._contigs = contigs
        self._mapping = dict(zip(mapping.values(), mapping))
        self._input_file = input_file

        script = rtools.rscript("zonkey", "coverage.r")
        cmd = AtomicCmd(
            ("Rscript", script, "%(TEMP_OUT_TABLE)s", "%(TEMP_OUT_PREFIX)s"),
            AUX_RSCRIPT=script,
            IN_FILE=input_file,
            TEMP_OUT_TABLE="contigs.table",
            OUT_PDF=output_prefix + ".pdf",
            OUT_PNG=output_prefix + ".png",
            TEMP_OUT_PREFIX=os.path.basename(output_prefix),
            CHECK_R=RSCRIPT_VERSION,
            CHECK_R_GGPLOT2=rtools.requirement("ggplot2"),
            set_cwd=True,
        )

        CommandNode.__init__(
            self,
            description="<CoveragePlot -> '%s.*'>" % (output_prefix, ),
            command=cmd,
            dependencies=dependencies,
        )
Example #41
0
    def __init__(self,
                 output_prefix,
                 tfam,
                 tped,
                 indep_filter=None,
                 indep_parameters=None,
                 plink_parameters=None,
                 dependencies=()):
        temp_prefix = os.path.basename(output_prefix)

        plink_cmd = [
            "plink", "--make-bed", "--noweb", "--tped", "%(IN_TPED)s",
            "--tfam", "%(IN_TFAM)s", "--out", "%(TEMP_OUT_PREFIX)s"
        ]

        plink_cmd.extend(self._parse_parameters(plink_parameters))

        command = AtomicCmd(plink_cmd,
                            IN_TPED=tped,
                            IN_TFAM=tfam,
                            TEMP_OUT_PREFIX=temp_prefix,
                            OUT_BED=output_prefix + ".bed",
                            OUT_BIM=output_prefix + ".bim",
                            OUT_FAM=output_prefix + ".fam",
                            OUT_LOG=output_prefix + ".log",
                            TEMP_OUT_NOSEX=temp_prefix + ".nosex",
                            TEMP_OUT_NOF=temp_prefix + ".nof",
                            CHECK_VERSION=PLINK_VERSION,
                            set_cwd=True)

        CommandNode.__init__(self,
                             description="<BuildBEDFiles -> '%s.*'>" %
                             (output_prefix, ),
                             command=command,
                             dependencies=dependencies)
Example #42
0
    def __init__(self, infile, index_format=".bai", dependencies=()):
        if index_format == ".bai":
            samtools_call = ["samtools", "index", "%(IN_BAM)s", "%(OUT_IDX)s"]
        elif index_format == ".csi":
            samtools_call = [
                "samtools", "index", "-c", "%(IN_BAM)s", "%(OUT_IDX)s"
            ]
        else:
            raise ValueError("Unknown format type %r; expected .bai or .csi" %
                             (index_format, ))

        command = AtomicCmd(
            samtools_call,
            IN_BAM=infile,
            OUT_IDX=infile + index_format,
            CHECK_SAM=SAMTOOLS_VERSION,
        )

        CommandNode.__init__(
            self,
            description="<BAMIndex (%s): '%s'>" %
            (index_format[1:].upper(), infile),
            command=command,
            dependencies=dependencies,
        )
Example #43
0
    def __init__(self,
                 input_file,
                 output_prefix,
                 order,
                 samples,
                 dependencies=()):
        self._samples = samples
        self._order = tuple(order) + ("Sample", )

        script = rtools.rscript("zonkey", "admixture.r")

        cmd = AtomicCmd(("Rscript", script, "%(IN_FILE)s",
                         "%(TEMP_OUT_NAMES)s", "%(TEMP_OUT_PREFIX)s"),
                        AUX_RSCRIPT=script,
                        IN_FILE=input_file,
                        IN_SAMPLES=samples,
                        OUT_PDF=output_prefix + ".pdf",
                        OUT_PNG=output_prefix + ".png",
                        TEMP_OUT_NAMES="samples.txt",
                        TEMP_OUT_PREFIX=os.path.basename(output_prefix),
                        CHECK_R=RSCRIPT_VERSION,
                        CHECK_R_GGPLOT2=rtools.requirement("ggplot2"),
                        CHECK_R_RESHAPE2=rtools.requirement("reshape2"),
                        set_cwd=True)

        CommandNode.__init__(self,
                             description="<AdmixturePlot -> '%s.*'>" %
                             (output_prefix, ),
                             command=cmd,
                             dependencies=dependencies)
Example #44
0
def test_atomiccmd__commit_failure_cleanup(temp_folder):
    counter = []
    move_file = fileutils.move_file

    def _monkey_move_file(source, destination):
        if counter:
            raise OSError("ARRRGHHH!")
        counter.append(destination)

        return move_file(source, destination)

    destination, temp_folder = _setup_for_commit(temp_folder, False)
    command = AtomicCmd(
        ("touch", "%(OUT_FILE_1)s", "%(OUT_FILE_2)s", "%(OUT_FILE_3)s"),
        OUT_FILE_1=os.path.join(destination, "file_1"),
        OUT_FILE_2=os.path.join(destination, "file_2"),
        OUT_FILE_3=os.path.join(destination, "file_3"))

    try:
        fileutils.move_file = _monkey_move_file
        command.run(temp_folder)
        assert_equal(command.join(), [0])
        assert_raises(OSError, command.commit, temp_folder)

        assert_equal(tuple(os.listdir(destination)), ())
    finally:
        fileutils.move_file = move_file
Example #45
0
    def __init__(self, samples, prefix, output_prefix, dependencies=()):
        abs_prefix = os.path.abspath(prefix)

        script = rtools.rscript("zonkey", "pca.r")
        call = [
            "Rscript", script, abs_prefix, "%(IN_SAMPLES)s",
            "%(TEMP_OUT_PREFIX)s"
        ]

        cmd = AtomicCmd(call,
                        AUX_SCRIPT=script,
                        IN_FILE_EVAL=prefix + ".eval",
                        IN_FILE_EVEC=prefix + ".evec",
                        IN_SAMPLES=samples,
                        TEMP_OUT_PREFIX=os.path.basename(output_prefix),
                        OUT_PDF=output_prefix + ".pdf",
                        OUT_PNG=output_prefix + ".png",
                        CHECK_R=RSCRIPT_VERSION,
                        CHECK_R_GGPLOT2=rtools.requirement("ggplot2"),
                        CHECK_R_LABELS=rtools.requirement("ggrepel"),
                        set_cwd=True)

        CommandNode.__init__(self,
                             description="<PlotPCA -> '%s.*'>" %
                             (output_prefix, ),
                             command=cmd,
                             dependencies=dependencies)
Example #46
0
def test_atomiccmd__commit_failure_cleanup(tmp_path):
    counter = []
    move_file = fileutils.move_file

    def _monkey_move_file(source, destination):
        if counter:
            raise OSError("ARRRGHHH!")
        counter.append(destination)

        return move_file(source, destination)

    destination, tmp_path = _setup_for_commit(tmp_path, False)
    command = AtomicCmd(
        ("touch", "%(OUT_FILE_1)s", "%(OUT_FILE_2)s", "%(OUT_FILE_3)s"),
        OUT_FILE_1=os.path.join(destination, "file_1"),
        OUT_FILE_2=os.path.join(destination, "file_2"),
        OUT_FILE_3=os.path.join(destination, "file_3"),
    )

    try:
        fileutils.move_file = _monkey_move_file
        command.run(tmp_path)
        assert command.join() == [0]
        with pytest.raises(OSError):
            command.commit(tmp_path)

        assert tuple(os.listdir(destination)) == ()
    finally:
        fileutils.move_file = move_file
Example #47
0
def test_atomiccmd__paths():
    cmd = AtomicCmd(
        "ls",
        IN_AAA="/a/b/c",
        IN_AAB="/x/y/z",
        TEMP_IN_ABB="tmp_in",
        OUT_AAA="/out/foo",
        OUT_BBC="foo/bar",
        TEMP_OUT_A="xyb",
        EXEC_OTHER="true",
        AUX_WAT="wat/wat",
        CHECK_FUNC=bool,
        OUT_STDERR="/var/log/pipe.stderr",
        TEMP_OUT_STDOUT="pipe.stdout",
    )

    assert cmd.executables == frozenset(["ls", "true"])
    assert cmd.requirements == frozenset([bool])
    assert cmd.input_files == frozenset(["/a/b/c", "/x/y/z"])
    assert cmd.output_files == frozenset(
        ["/out/foo", "foo/bar", "/var/log/pipe.stderr"])
    assert cmd.auxiliary_files == frozenset(["wat/wat"])
    assert cmd.expected_temp_files == frozenset(["foo", "bar", "pipe.stderr"])
    assert "xyb" in cmd.optional_temp_files
    assert "pipe.stdout" in cmd.optional_temp_files
Example #48
0
def test_parallel_commands__join_before_run():
    mock = Mock()
    cmd_1 = AtomicCmd(["ls"])
    cmd_1.join = mock.join_1
    cmd_2 = AtomicCmd(["ls"])
    cmd_2.join = mock.join_2
    cmd_3 = AtomicCmd(["ls"])
    cmd_3.join = mock.join_3

    cmds = ParallelCmds((cmd_3, cmd_2, cmd_1))
    assert cmds.join() == [None, None, None]

    assert mock.mock_calls == []
Example #49
0
def test_pformat__atomiccmd__simple_with_temp_stdout__set_cwd():
    cmd = AtomicCmd(("echo", "Room service. Room service."),
                    TEMP_OUT_STDOUT="pv",
                    set_cwd=True)
    assert pformat(cmd) == ("Command = echo 'Room service. Room service.'\n"
                            "STDOUT* = 'pv'\n"
                            "STDERR* = 'pipe_echo_%i.stderr'\n"
                            "CWD     = '${TEMP_DIR}'") % (id(cmd), )
Example #50
0
def test_atomiccmd__pipes_out(tmp_path, stdout, stderr, kwargs):
    call = ("bash", "-c",
            "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';")

    cmd = AtomicCmd(call, **kwargs)
    cmd.run(tmp_path)
    assert cmd.join() == [0]

    expected_files = []
    for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
        if tmpl is not None:
            fname = tmpl.format(id(cmd))
            result = (tmp_path / fname).read_text()
            assert result == text
            expected_files.append(fname)

    assert set(os.listdir(tmp_path)) == set(expected_files)
Example #51
0
    def _test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
        call = ("bash", "-c",
                "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';")

        cmd = AtomicCmd(call, **kwargs)
        cmd.run(temp_folder)
        assert_equal(cmd.join(), [0])

        expected_files = []
        for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
            if tmpl is not None:
                fname = tmpl.format(id(cmd))
                result = get_file_contents(os.path.join(temp_folder, fname))
                assert_equal(result, text)
                expected_files.append(fname)

        assert_equal(set(os.listdir(temp_folder)), set(expected_files))
Example #52
0
def test_atomiccmd__ready(temp_folder):
    cmd = AtomicCmd("ls")
    assert_equal(cmd.join(), [None])
    assert not cmd.ready()
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert cmd.ready()
Example #53
0
def test_atomiccmd__terminate_race_condition(temp_folder):
    cmd = AtomicCmd("true")
    cmd.run(temp_folder)
    while cmd._proc.poll() is None:
        pass
    cmd.terminate()
    assert_equal(cmd.join(), [0])
Example #54
0
def test_atomiccmd__piping_temp(temp_folder):
    cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"],
                      TEMP_OUT_STDOUT=AtomicCmd.PIPE)
    assert_equal(cmd_1.output_files, frozenset())
    cmd_2 = AtomicCmd(["cat"],
                      TEMP_IN_STDIN=cmd_1,
                      OUT_STDOUT="piped.txt")
    assert_equal(cmd_2.input_files, frozenset())
    cmd_1.run(temp_folder)
    cmd_2.run(temp_folder)
    assert_equal(cmd_1.join(), [0])
    assert_equal(cmd_2.join(), [0])
    result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
    assert_equal(result, "#@!$^")
Example #55
0
def test_pformat__simple__running__set_cwd(temp_folder):
    cmd = AtomicCmd(("sleep", "10"), set_cwd=True)
    cmd.run(temp_folder)
    assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
                                " Status  = Running ...\n"
                                " STDOUT* = 'pipe_sleep_{id}.stdout'\n"
                                " STDERR* = 'pipe_sleep_{id}.stderr'\n"
                                " CWD     = '{temp_dir}'>").format(id=id(cmd),
                                                                   temp_dir=temp_folder))
    cmd.terminate()
    cmd.join()
Example #56
0
def test_atomiccmd__commit_temp_only(temp_folder):
    cmd = AtomicCmd(("echo", "foo"),
                    TEMP_OUT_STDOUT="bar.txt")
    cmd.run(temp_folder)
    assert_equal(cmd.join(), [0])
    assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
    cmd.commit(temp_folder)
    assert_equal(os.listdir(temp_folder), [])
Example #57
0
def test_pformat__simple__running(temp_folder):
    cmd = AtomicCmd(("sleep", "10"))
    cmd.run(temp_folder)
    assert_equal(pformat(cmd),
                 ("Command = sleep 10\n"
                  "Status  = Running ...\n"
                  "STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
                  "STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
                  "CWD     = '{cwd}'").format(id=id(cmd),
                                              cwd=os.getcwd(),
                                              temp_dir=temp_folder))
    cmd.terminate()
    cmd.join()
Example #58
0
    def _do_test_atomiccmd__terminate(temp_folder, raise_on_terminate):
        cmd = AtomicCmd(("sleep", "10"))
        cmd.run(temp_folder)

        killpg_was_called = []

        def _wrap_killpg(pid, sig):
            assert_equal(pid, cmd._proc.pid)
            assert_equal(sig, signal.SIGTERM)
            killpg_was_called.append(True)
            if raise_on_terminate:
                raise OSError("KABOOM!")

        with Monkeypatch("os.killpg", _wrap_killpg):
            cmd.terminate()
        cmd.terminate()
        assert_equal(cmd.join(), ["SIGTERM"])
        assert killpg_was_called
Example #59
0
def test_atomiccmd__commit_temp_out(temp_folder):
    dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
    cmd = AtomicCmd(("echo", "foo"),
                    OUT_STDOUT=os.path.join(dest, "foo.txt"),
                    TEMP_OUT_FOO="bar.txt")
    cmd.run(temp)
    assert_equal(cmd.join(), [0])
    set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
    cmd.commit(temp)
    assert_equal(os.listdir(temp), [])
    assert_equal(os.listdir(dest), ["foo.txt"])
Example #60
0
def test_pformat__simple__killed(temp_folder):
    cmd = AtomicCmd(("sleep", "10"))
    cmd.run(temp_folder)
    cmd.terminate()
    assert_equal(cmd.join(), ["SIGTERM"])
    assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
                                " Status  = Terminated with signal SIGTERM\n"
                                " STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
                                " STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
                                " CWD     = '{cwd}'>").format(id=id(cmd),
                                                              temp_dir=temp_folder,
                                                              cwd=os.getcwd()))