Example #1
0
def test_nested_pipes_stream_setup_stream_multiplex():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    p.run(tool, input=["Makefile", "README.rst", "setup.py"],
          output="${input}.dat")
    p.expand(validate=False)

    # 2 nodes 1 edge
    assert len(p) == 6
    assert len(p.edges) == 3
    t1_0 = p.get("TestJob1.0")
    t2_0 = p.get("TestJob2.0")
    assert t1_0.has_outgoing(t2_0, ('output', 'input'), True)
    t1_1 = p.get("TestJob1.1")
    t2_1 = p.get("TestJob2.1")
    assert t1_1.has_outgoing(t2_1, ('output', 'input'), True)
    t1_2 = p.get("TestJob1.2")
    t2_2 = p.get("TestJob2.2")
    assert t1_2.has_outgoing(t2_2, ('output', 'input'), True)

    # test option values
    cwd = os.getcwd()
    join = os.path.join
    assert t1_0.input == join(cwd, 'Makefile')
    assert t1_1.input == join(cwd, 'README.rst')
    assert t1_2.input == join(cwd, 'setup.py')
    assert t2_0.output == join(cwd, 'Makefile.dat')
    assert t2_1.output == join(cwd, 'README.rst.dat')
    assert t2_2.output == join(cwd, 'setup.py.dat')
Example #2
0
def test_tool_class_decorator_options_for_functions():
    data = []
    validated = []

    @jip.pytool()
    class mytool(object):
        """
        usage:
            tool <name>
        """
        def validate(self):
            validated.append(True)

        def run(self):
            # in call implementation, we are not a tool instance
            assert isinstance(self, mytool)
            # but we have the helpers
            assert hasattr(self, 'args')
            assert hasattr(self, 'options')
            assert hasattr(self, 'tool_instance')
            assert hasattr(self, 'check_file')
            assert hasattr(self, 'ensure')
            assert hasattr(self, 'validation_error')
            data.append(self.options['name'].get())

    t = find('mytool')
    assert t
    assert t.options['name'] is not None
    t.options['name'] = 'data.txt'
    t.validate()
    t.run()
    assert data[0] == 'data.txt'
    assert validated[0]
Example #3
0
def test_bash_tool_options():
    bash = find("bash")
    assert bash.options is not None
    assert len(bash.options) == 5
    assert bash.options['cmd'] is not None
    assert bash.options['input'] is not None
    assert bash.options['output'] is not None
Example #4
0
    def testPipelineStructure(self):
        # load the pipeline
        tool = jip.find("examples/bwa/pileup.jip")
        assert tool is not None

        # create a pipeline
        p = jip.Pipeline()
        # create a new pipeline node and configure id
        p.run(tool, input="setup.py", reference="Makefile", output="out.txt")

        # expand the pipeline such that the internal pipeline is resolved
        p.expand(validate=False)

        # after expansion with this setuo, the pipeline should have 7 nodes
        assert len(p) == 7
        # the graph should consist of 6 edges
        assert len(p.edges) == 6
        # get out the nodes. we have to use indexes here
        # because the names might have changed after expansion
        ref = p.get("ref")
        align = p.get("align")
        sam = p.get("sam")
        bam = p.get("bam")
        dups = p.get("dups")
        index = p.get("index")
        pileup = p.get("pileup")
        # check the connections
        assert not ref.has_incoming()
        assert align.has_incoming(ref)
        assert sam.has_incoming(align)
        assert bam.has_incoming(sam)
        assert dups.has_incoming(bam)
        assert index.has_incoming(dups)
        assert pileup.has_incoming(index)
        assert not pileup.has_outgoing()
Example #5
0
def test_tool_hidden_option():
    @jip.tool("test")
    class TestTool(object):
        """\Test tool

        Usage:
            test [-a <opt_a>] [-b <opt_b>] [-c] -i <input> -o <output>

        Inputs:
            -i, --input <input>  The input

        Outputs:
            -o, --output <output>  The output

        Options:
            -a <opt_a>  Option a
            -b <opt_b>  Option b
            -c          Options c (a flag)
        """
        def setup(self):
            self.opts['output'].hidden = True

        def get_command(self):
            return '${options()} ${output|arg(">")}'

    tool = find('test')
    tool.parse_args(['-i', 'input.txt', '-a', 'A', '-c', '-o', 'output.txt'])
    assert tool.get_command() == ('bash', '-a A -c -i input.txt -o output.txt >output.txt')
Example #6
0
def test_nested_pipes_stream_setup_stream_multiplex():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    p.run(tool,
          input=["Makefile", "README.rst", "setup.py"],
          output="${input}.dat")
    p.expand(validate=False)

    # 2 nodes 1 edge
    assert len(p) == 6
    assert len(p.edges) == 3
    t1_0 = p.get("TestJob1.0")
    t2_0 = p.get("TestJob2.0")
    assert t1_0.has_outgoing(t2_0, ('output', 'input'), True)
    t1_1 = p.get("TestJob1.1")
    t2_1 = p.get("TestJob2.1")
    assert t1_1.has_outgoing(t2_1, ('output', 'input'), True)
    t1_2 = p.get("TestJob1.2")
    t2_2 = p.get("TestJob2.2")
    assert t1_2.has_outgoing(t2_2, ('output', 'input'), True)

    # test option values
    cwd = os.getcwd()
    join = os.path.join
    assert t1_0.input == join(cwd, 'Makefile')
    assert t1_1.input == join(cwd, 'README.rst')
    assert t1_2.input == join(cwd, 'setup.py')
    assert t2_0.output == join(cwd, 'Makefile.dat')
    assert t2_1.output == join(cwd, 'README.rst.dat')
    assert t2_2.output == join(cwd, 'setup.py.dat')
Example #7
0
def test_tool_decorator_options_for_functions():
    data = []

    @jip.pytool()
    def mytool(self):
        """
        usage:
            tool -i <input>
        options:
            -i, --input <input>  The input
        """
        assert isinstance(self, jip.tools.Tool)
        # but we have the helpers
        assert hasattr(self, 'args')
        assert hasattr(self, 'options')
        #assert hasattr(self, 'tool_instance')
        assert hasattr(self, 'check_file')
        assert hasattr(self, 'ensure')
        assert hasattr(self, 'validation_error')
        data.append(self.options['input'].get())

    t = find('mytool')
    assert t
    assert t.options['input'] is not None
    t.options['input'] = 'data.txt'
    t.run()
    assert data[0] == 'data.txt'
def test_bash_tool_options():
    bash = find("bash")
    assert bash.options is not None
    assert len(bash.options) == 5
    assert bash.options['cmd'] is not None
    assert bash.options['input'] is not None
    assert bash.options['output'] is not None
Example #9
0
def test_tool_decorator_delegate():
    @jip.tool("test_delegates")
    class MyTool(object):
        """\
        usage: delegate [-i <input>]
        """
        def validate(self):
            return "delegated_validate:%s" % self.options['i'].get()

        def is_done(self):
            return "delegated_is_done:%s" % self.options['i'].get()

        def pipeline(self):
            return "delegated_pipeline:%s" % self.options['i'].get()

        def cleanup(self):
            return "delegated_cleanup:%s" % self.options['i'].get()

        def help(self):
            return "delegated_help:%s" % self.options['i'].get()

        def get_command(self):
            return 'inter', "delegated_cmd:%s" % self.options['i'].get()

    test_tool = find("test_delegates")
    test_tool.parse_args(["-i", "infile.txt"])
    assert test_tool is not None
    assert test_tool.validate() == "delegated_validate:infile.txt"
    assert test_tool.is_done() == "delegated_is_done:infile.txt"
    assert test_tool.pipeline() == "delegated_pipeline:infile.txt"
    assert test_tool.cleanup() == "delegated_cleanup:infile.txt"
    assert test_tool.help() == "delegated_help:infile.txt"
    assert test_tool.get_command() == ('inter', "delegated_cmd:infile.txt")
Example #10
0
    def testPipelineStructure(self):
        # load the pipeline
        tool = jip.find("examples/bwa/pileup.jip")
        assert tool is not None

        # create a pipeline
        p = jip.Pipeline()
        # create a new pipeline node and configure id
        p.run(tool, input="setup.py", reference="Makefile", output="out.txt")

        # expand the pipeline such that the internal pipeline is resolved
        p.expand(validate=False)

        # after expansion with this setuo, the pipeline should have 7 nodes
        assert len(p) == 7
        # the graph should consist of 6 edges
        assert len(p.edges) == 6
        # get out the nodes. we have to use indexes here
        # because the names might have changed after expansion
        ref = p.get("ref")
        align = p.get("align")
        sam = p.get("sam")
        bam = p.get("bam")
        dups = p.get("dups")
        index = p.get("index")
        pileup = p.get("pileup")
        # check the connections
        assert not ref.has_incoming()
        assert align.has_incoming(ref)
        assert sam.has_incoming(align)
        assert bam.has_incoming(sam)
        assert dups.has_incoming(bam)
        assert index.has_incoming(dups)
        assert pileup.has_incoming(index)
        assert not pileup.has_outgoing()
Example #11
0
    def testPipelineStructureMultiplexed(self):
        # load the pipeline
        tool = jip.find("examples/bwa/pileup.jip")
        assert tool is not None

        # create a pipeline
        p = jip.Pipeline()
        # create a new pipeline node and configure id
        p.run(tool,
              input=["setup.py", "README.rst"],
              reference="Makefile",
              output="${input|ext}_out.txt")

        # expand the pipeline such that the internal pipeline is resolved
        # this will also validate all nodes and raise an exception
        # if one of the nodes validations failed
        p.expand(validate=False)

        # after expansion with this setuo, the pipeline should have 7 nodes
        assert len(p) == 13
        # the graph should consist of 6 edges
        assert len(p.edges) == 12
        # get out the nodes. we have to use indexes here
        # because the names might have changed after expansion
        ref = p.get("ref")
        align = p.get("align.0")
        sam = p.get("sam.0")
        bam = p.get("bam.0")
        dups = p.get("dups.0")
        index = p.get("index.0")
        pileup = p.get("pileup.0")
        # check the connections
        assert not ref.has_incoming()
        assert align.has_incoming(ref)
        assert sam.has_incoming(align)
        assert bam.has_incoming(sam)
        assert dups.has_incoming(bam)
        assert index.has_incoming(dups)
        assert pileup.has_incoming(index)
        assert not pileup.has_outgoing()
        # test second set
        ref = p.get("ref")
        align = p.get("align.1")
        sam = p.get("sam.1")
        bam = p.get("bam.1")
        dups = p.get("dups.1")
        index = p.get("index.1")
        pileup = p.get("pileup.1")
        # check the connections
        assert not ref.has_incoming()
        assert align.has_incoming(ref)
        assert sam.has_incoming(align)
        assert bam.has_incoming(sam)
        assert dups.has_incoming(bam)
        assert index.has_incoming(dups)
        assert pileup.has_incoming(index)
        assert not pileup.has_outgoing()
Example #12
0
def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file)
    except LookupError, e:
        print >>sys.stderr, str(e)
        sys.exit(1)
Example #13
0
def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file)
    except LookupError, e:
        print >> sys.stderr, str(e)
        sys.exit(1)
Example #14
0
    def testPipelineStructureMultiplexed(self):
        # load the pipeline
        tool = jip.find("examples/bwa/pileup.jip")
        assert tool is not None

        # create a pipeline
        p = jip.Pipeline()
        # create a new pipeline node and configure id
        p.run(tool, input=["setup.py", "README.rst"],
              reference="Makefile",
              output="${input|ext}_out.txt")

        # expand the pipeline such that the internal pipeline is resolved
        # this will also validate all nodes and raise an exception
        # if one of the nodes validations failed
        p.expand(validate=False)

        # after expansion with this setuo, the pipeline should have 7 nodes
        assert len(p) == 13
        # the graph should consist of 6 edges
        assert len(p.edges) == 12
        # get out the nodes. we have to use indexes here
        # because the names might have changed after expansion
        ref = p.get("ref")
        align = p.get("align.0")
        sam = p.get("sam.0")
        bam = p.get("bam.0")
        dups = p.get("dups.0")
        index = p.get("index.0")
        pileup = p.get("pileup.0")
        # check the connections
        assert not ref.has_incoming()
        assert align.has_incoming(ref)
        assert sam.has_incoming(align)
        assert bam.has_incoming(sam)
        assert dups.has_incoming(bam)
        assert index.has_incoming(dups)
        assert pileup.has_incoming(index)
        assert not pileup.has_outgoing()
        # test second set
        ref = p.get("ref")
        align = p.get("align.1")
        sam = p.get("sam.1")
        bam = p.get("bam.1")
        dups = p.get("dups.1")
        index = p.get("index.1")
        pileup = p.get("pileup.1")
        # check the connections
        assert not ref.has_incoming()
        assert align.has_incoming(ref)
        assert sam.has_incoming(align)
        assert bam.has_incoming(sam)
        assert dups.has_incoming(bam)
        assert index.has_incoming(dups)
        assert pileup.has_incoming(index)
        assert not pileup.has_outgoing()
Example #15
0
def test_tool_setup_called_on_init():
    @jip.tool()
    class setup_tool(object):
        def init(self):
            self.add_output('output')

        def get_comand(self):
            return "true"

    t = jip.find('setup_tool')
    assert t.options['output'] is not None
Example #16
0
def test_nested_pipes_stream_setup_intermediate():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    p.run(tool, input="Makefile", output="out.txt", inter="inter.out")
    p.expand()

    # 2 nodes 1 edge
    assert len(p) == 2
    assert len(p.edges) == 1
    t1 = p.get("TestJob1")
    t2 = p.get("TestJob2")
    assert t1.has_outgoing(t2, ('output', 'input'), False)
Example #17
0
def test_nested_pipes_stream_setup_intermediate():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    p.run(tool, input="Makefile", output="out.txt", inter="inter.out")
    p.expand()

    # 2 nodes 1 edge
    assert len(p) == 2
    assert len(p.edges) == 1
    t1 = p.get("TestJob1")
    t2 = p.get("TestJob2")
    assert t1.has_outgoing(t2, ('output', 'input'), False)
Example #18
0
def test_nested_pipes_stream_setup_stream_jobs():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    p.run(tool, input="Makefile", output="out.txt")
    jobs = jip.create_jobs(p)
    groups = jip.create_groups(jobs)

    cwd = os.getcwd()
    join = os.path.join
    assert len(groups) == 1
    assert len(jobs) == 2
    assert jobs[0].configuration['input'].get() == join(cwd, 'Makefile')
    assert jobs[1].configuration['output'].get() == join(cwd, 'out.txt')
Example #19
0
def test_nested_pipes_stream_setup_stream_jobs():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    p.run(tool, input="Makefile", output="out.txt")
    jobs = jip.create_jobs(p)
    groups = jip.create_groups(jobs)

    cwd = os.getcwd()
    join = os.path.join
    assert len(groups) == 1
    assert len(jobs) == 2
    assert jobs[0].configuration['input'].get() == join(cwd, 'Makefile')
    assert jobs[1].configuration['output'].get() == join(cwd, 'out.txt')
Example #20
0
def test_render_boolean_option():
    @jip.tool("simple")
    class SimpleTool(object):
        """\
        Test tool
        Usage:
            tool [-t]
        Options:
            -t   A bool option
        """
        pass
    tool = jip.find("simple")
    assert render_template("${t|arg}", tool=tool, test="1") == ""
    tool.parse_args(['-t'])
    assert render_template("${t|arg}", tool=tool, test="1") == "-t"
Example #21
0
def test_tool_decorator_options_for_templates():
    @jip.tool()
    def mytool():
        """
        usage:
            tool -i <input>
        options:
            -i, --input <input>  The input
        """
        return "run on ${input}"

    t = find('mytool')
    assert t
    assert t.options['input'] is not None
    t.options['input'] = 'data.txt'
    assert t.get_command()[1] == 'run on data.txt'
Example #22
0
def test_render_boolean_option():
    @jip.tool("simple")
    class SimpleTool(object):
        """\
        Test tool
        Usage:
            tool [-t]
        Options:
            -t   A bool option
        """
        pass

    tool = jip.find("simple")
    assert render_template("${t|arg}", tool=tool, test="1") == ""
    tool.parse_args(['-t'])
    assert render_template("${t|arg}", tool=tool, test="1") == "-t"
Example #23
0
def test_render_value_option():
    @jip.tool("simple")
    class SimpleTool(object):
        """\
        Test tool
        Usage:
            tool [-t <in>]
        Options:
            -t <in>  A value option
        """
        pass
    tool = jip.find("simple")
    tool.parse_args(["-t", "infile"])
    assert render_template("${t|arg}", tool=tool, test="1") == "-t infile"
    assert render_template("${t|arg('-o ')}",
                           tool=tool, test="1") == "-o infile"
Example #24
0
def test_render_value_option():
    @jip.tool("simple")
    class SimpleTool(object):
        """\
        Test tool
        Usage:
            tool [-t <in>]
        Options:
            -t <in>  A value option
        """
        pass

    tool = jip.find("simple")
    tool.parse_args(["-t", "infile"])
    assert render_template("${t|arg}", tool=tool, test="1") == "-t infile"
    assert render_template("${t|arg('-o ')}", tool=tool,
                           test="1") == "-o infile"
Example #25
0
def test_node_options_with_assignment():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    n = p.run(tool)
    n.input = "Makefile"
    n.output = "out.txt"
    n.inter = "inter.out"
    assert n._tool.options['input'].get() == "Makefile"
    p.expand()

    # 2 nodes 1 edge
    assert len(p) == 2
    assert len(p.edges) == 1
    t1 = p.get("TestJob1")
    t2 = p.get("TestJob2")
    assert t1.has_outgoing(t2, ('output', 'input'), False)
Example #26
0
def test_node_options_with_assignment():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    n = p.run(tool)
    n.input = "Makefile"
    n.output = "out.txt"
    n.inter = "inter.out"
    assert n._tool.options['input'].get() == "Makefile"
    p.expand()

    # 2 nodes 1 edge
    assert len(p) == 2
    assert len(p.edges) == 1
    t1 = p.get("TestJob1")
    t2 = p.get("TestJob2")
    assert t1.has_outgoing(t2, ('output', 'input'), False)
Example #27
0
File: db.py Project: heathsc/pyjip
 def tool(self):
     """Get the tool instance that is associated with this job. If
     the tool is not set, it will be loaded using the :py:func:`jip.find`
     function
     """
     if not self._tool:
         try:
             from jip import find
             self._tool = find(self.tool_name if self.path is None
                               else self.path)
             for opt in self.configuration:
                 if opt.name in self._tool.options:
                     self._tool.options[opt.name]._value = opt._value
                 else:
                     self._tool.options[opt.name] = opt
         except:
             log.error("Unable to reload tool: %s", self.tool_name,
                       exc_info=True)
     return self._tool
Example #28
0
 def tool(self):
     """Get the tool instance that is associated with this job. If
     the tool is not set, it will be loaded using the :py:func:`jip.find`
     function
     """
     if not self._tool:
         try:
             from jip import find
             self._tool = find(self.tool_name if self.path is None
                               else self.path)
             for opt in self.configuration:
                 if opt.name in self._tool.options:
                     self._tool.options[opt.name]._value = opt._value
                 else:
                     self._tool.options[opt.name] = opt
         except:
             log.error("Unable to reload tool: %s", self.tool_name,
                       exc_info=True)
     return self._tool
Example #29
0
def test_tool_property_names():
    @jip.pytool()
    class mytool(object):
        """
        usage:
            mytool --input --no-hash

        """

        def validate(self):
            assert hasattr(self, 'input')
            assert hasattr(self, 'no_hash')

        def run(self):
            pass

    t = find('mytool')
    assert t.options["input"] is not None
    assert t.options["no_hash"] is not None
    t.validate()
Example #30
0
def test_tool_decorator_delegate_validate_to_external_method():

    def my_validate(self):
        return "delegated_validate:%s" % self.options['i'].get()

    def my_is_done(self):
        return "delegated_is_done:%s" % self.options['i'].get()

    def my_pipeline(self):
        return "delegated_pipeline:%s" % self.options['i'].get()

    def my_cleanup(self):
        return "delegated_cleanup:%s" % self.options['i'].get()

    def my_help(self):
        return "delegated_help:%s" % self.options['i'].get()

    def my_get_command(self):
        return 'inter', "delegated_cmd:%s" % self.options['i'].get()

    @jip.tool("test_delegates",
              validate=my_validate,
              pipeline=my_pipeline,
              get_command=my_get_command,
              cleanup=my_cleanup,
              is_done=my_is_done,
              help=my_help)
    class MyTool(object):
        """\
        usage: delegate [-i <input>]
        """

    test_tool = find("test_delegates")
    test_tool.parse_args(["-i", "infile.txt"])
    assert test_tool is not None
    assert test_tool.validate() == "delegated_validate:infile.txt"
    assert test_tool.is_done() == "delegated_is_done:infile.txt"
    assert test_tool.pipeline() == "delegated_pipeline:infile.txt"
    assert test_tool.cleanup() == "delegated_cleanup:infile.txt"
    assert test_tool.help() == "delegated_help:infile.txt"
    assert test_tool.get_command() == ('inter', "delegated_cmd:infile.txt")
Example #31
0
def test_node_options_with_assignment_jobs():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    n = p.run(tool)
    n.input = "Makefile"
    n.output = "out.txt"
    n.inter = "inter.out"

    jobs = jip.create_jobs(p)
    groups = jip.create_groups(jobs)
    cwd = os.getcwd()

    join = os.path.join
    assert len(groups) == 2
    assert len(jobs) == 2

    assert jobs[0].configuration['input'].get() == join(cwd, 'Makefile')
    assert jobs[0].configuration['output'].get() == join(cwd, 'inter.out')
    assert jobs[1].configuration['input'].get() == join(cwd, 'inter.out')
    assert jobs[1].configuration['output'].get() == join(cwd, 'out.txt')
Example #32
0
def test_node_options_with_assignment_jobs():
    tool = jip.find('joined_pipeline')
    assert tool is not None
    p = jip.Pipeline()
    n = p.run(tool)
    n.input = "Makefile"
    n.output = "out.txt"
    n.inter = "inter.out"

    jobs = jip.create_jobs(p)
    groups = jip.create_groups(jobs)
    cwd = os.getcwd()

    join = os.path.join
    assert len(groups) == 2
    assert len(jobs) == 2

    assert jobs[0].configuration['input'].get() == join(cwd, 'Makefile')
    assert jobs[0].configuration['output'].get() == join(cwd, 'inter.out')
    assert jobs[1].configuration['input'].get() == join(cwd, 'inter.out')
    assert jobs[1].configuration['output'].get() == join(cwd, 'out.txt')
Example #33
0
def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file)
    except LookupError as e:
        print(str(e), file=sys.stderr)
        sys.exit(1)

    # disable required checks
    jip.options._check_required = False
    profile = jip.profiles.get(name='default')
    jobs = jip.jobs.create_jobs(script, args=script_args, profile=profile)
    specs = {}
    default_env = os.environ
    env_exckludes = [
        "JIP_MODULES", "JIP_LOGLEVEL", "JIP_PATH", "JIP_DB_LOGLEVEL"
    ]
    for j in jobs:
        job_env = {}
        for k, v in iteritems(j.env):
            if not k in env_exckludes and v and v != default_env.get(k, None):
                job_env[k] = v
        spec = sorted_dict({
            "threads": j.threads,
            "mem": j.max_memory,
            "queue": j.queue,
            "priority": j.priority,
            "time": j.max_time,
            "account": j.account,
            "extra": j.extra,
            "env": job_env
        })
        specs[j.name] = spec

    print(json.dumps({"jobs": specs}, indent=4, sort_keys=False))
Example #34
0
def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file)
    except LookupError as e:
        print(str(e), file=sys.stderr)
        sys.exit(1)

    # load profile
    profile = jip.profiles.get(
        name='default' if not args['--profile'] else args['--profile'])
    profile.tool_name = script.name
    if args['--spec']:
        spec_prof = jip.profiles.Profile.from_file(args['--spec'])
        spec_prof.update(profile)
        profile = spec_prof

    profile.load_args(args)
    log.info("Profile: %s", profile)

    if args['--dry'] or args['--show']:
        # we handle --dry and --show separatly,
        # create the jobs and call the show commands
        jobs = jip.jobs.create_jobs(script, args=script_args, profile=profile)
        error = None
        try:
            jip.jobs.check_output_files(jobs)
            jip.jobs.check_queued_jobs(jobs)
        except Exception as err:
            error = err
        if args['--dry']:
            show_dry(jobs, options=script.options, profiles=True)
        if args['--show']:
            show_commands(jobs)
        if error:
            print("%s\n" % colorize("Validation error!", RED), file=sys.stderr)
            print(str(error), file=sys.stderr)
            sys.exit(1)
        return

    #####################################################
    # prepare jobs for submission
    #####################################################
    force = args['--force']
    jobs = jip.jobs.create_jobs(script,
                                args=script_args,
                                keep=args['--keep'],
                                profile=profile,
                                profiler=args['--with-profiler'])
    if len(jobs) == 0:
        return
    if args['--hold']:
        #####################################################
        # Only save the jobs and let them stay on hold
        #####################################################
        jip.db.save(jobs)
        print("Jobs stored and put on hold")
    else:
        try:
            #####################################################
            # Iterate the executions and submit
            #####################################################
            for exe in jip.jobs.create_executions(jobs,
                                                  save=True,
                                                  check_outputs=not force,
                                                  check_queued=not force):
                if exe.completed and not force:
                    print(colorize("Skipping %s" % exe.name, YELLOW))
                else:
                    if jip.jobs.submit_job(exe.job, force=force):
                        print("Submitted %s with remote id %s" %
                              (exe.job.id, exe.job.job_id))
        except Exception as err:
            log.debug("Submission error: %s", err, exc_info=True)
            print(colorize("Error while submitting job:", RED),
                  colorize(str(err), RED),
                  file=sys.stderr)
            ##################################################
            # delete all submitted jobs
            ##################################################
            jip.jobs.delete(jobs, clean_logs=True)
Example #35
0
def main():
    args = parse_args(__doc__, options_first=True)

    print "Tools scripts"
    print "-------------"
    print "Please note that there might be more. Here, we search only for"
    print "files with the .jip extension!"
    print ""
    print "Search paths:"
    print "Current directory: %s" % getcwd()
    print "Jip configuration: %s" % jip.config.get("jip_path", "")
    print "JIP_PATH variable: %s" % getenv("JIP_PATH", "")
    print ""
    rows = []
    for name, path in jip.scanner.scan_files().iteritems():
        rows.append((name, path))
    print render_table(["Name", "Path"], rows)
    print ""

    print "Tools implemented in Python modules"
    print "-----------------------------------"
    print "The modules must be available in PYTHONPATH and must be specified"
    print "in the jip configuration or in the JIP_MODULES environment"
    print "variable. Please note that pipeline scripts that contain"
    print "python blocks are allowed to load modules that contain tool"
    print "implementation. These tools might not be found by this scan!"
    print ""
    print "Jip configuration: %s" % jip.config.get("jip_modules", "")
    print "JIP_MODULES variable: %s" % getenv("JIP_MODULES", "")
    print ""
    rows = []
    jip.scanner.scan_modules()
    for name, cls in jip.scanner.registry.iteritems():
        help = cls.help()
        description = "-"
        if help is not None:
            description = help.split("\n")[0]
        if len(description) > 60:
            description = "%s ..." % description[:46]
        rows.append((name, description))
    print render_table(["Tool", "Description"], rows)

    print ""
    print "All Tools detected"
    print "------------------"
    print ""
    covered = set([])
    rows = []
    for name, p in jip.scanner.scan().iteritems():
        if name in covered:
            continue
        covered.add(name)
        cls = jip.find(name)
        help = cls.help()
        description = "-"
        if help is not None:
            description = help.split("\n")[0]
        if len(description) > 60:
            description = "%s ..." % description[:46]
        rows.append((cls.name, description))
    print render_table(["Tool", "Description"], rows)
Example #36
0
def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file, is_pipeline=args['--pipeline'])
    except LookupError as e:
        print(str(e), file=sys.stderr)
        sys.exit(1)

    if args['--dry'] or args['--show']:
        dry(script, script_args, dry=args['--dry'], show=args['--show'])
        return

    keep = args['--keep']
    force = args['--force']
    profiler = args['--with-profiler']
    silent = not args['--status']
    try:
        profile = jip.profiles.Profile()
        if args['--spec']:
            spec_prof = jip.profiles.Profile.from_file(args['--spec'])
            spec_prof.update(profile)
            profile = spec_prof
        profile.load_args(args)

        jobs = jip.jobs.create_jobs(script,
                                    args=script_args,
                                    keep=keep,
                                    profile=profile)
        # assign job ids
        for i, j in enumerate(jobs):
            j.id = i + 1

        for exe in jip.jobs.create_executions(jobs):
            if exe.completed and not force:
                if not silent:
                    print(colorize("Skipping", YELLOW),
                          exe.name,
                          file=sys.stderr)
            else:
                if not silent:
                    print(colorize("Running", YELLOW),
                          " {name:30} ".format(name=colorize(exe.name, BLUE)),
                          file=sys.stderr)
                    sys.stderr.flush()
                start = datetime.now()
                success = jip.jobs.run_job(exe.job, profiler=profiler)
                end = timedelta(seconds=(datetime.now() - start).seconds)
                if success:
                    if not silent:
                        print(colorize(exe.job.state, GREEN),
                              "[%s]" % end,
                              file=sys.stderr)
                else:
                    if not silent:
                        print(colorize(exe.job.state, RED), file=sys.stderr)
                    sys.exit(1)
    except (jip.ValidationError, jip.ParserException) as va:
        print(str(va), file=sys.stderr)
        sys.exit(1)
    except Exception as va:
        raise
Example #37
0
def main():
    args = parse_args(__doc__, options_first=True)

    print "Tools scripts"
    print "-------------"
    print "Please note that there might be more. Here, we search only for"
    print "files with the .jip extension!"
    print ""
    print "Search paths:"
    print "Current directory: %s" % getcwd()
    print "Jip configuration: %s" % jip.config.get("jip_path", "")
    print "JIP_PATH variable: %s" % getenv("JIP_PATH", "")
    print ""
    rows = []
    for name, path in jip.scanner.scan_files().iteritems():
        rows.append((name, path))
    print render_table(["Name", "Path"], rows)
    print ""

    print "Tools implemented in Python modules"
    print "-----------------------------------"
    print "The modules must be available in PYTHONPATH and must be specified"
    print "in the jip configuration or in the JIP_MODULES environment"
    print "variable. Please note that pipeline scripts that contain"
    print "python blocks are allowed to load modules that contain tool"
    print "implementation. These tools might not be found by this scan!"
    print ""
    print "Jip configuration: %s" % jip.config.get("jip_modules", "")
    print "JIP_MODULES variable: %s" % getenv("JIP_MODULES", "")
    print ""
    rows = []
    jip.scanner.scan_modules()
    for name, cls in jip.scanner.registry.iteritems():
        help = cls.help()
        description = "-"
        if help is not None:
            description = help.split("\n")[0]
        if len(description) > 60:
            description = "%s ..." % description[:46]
        rows.append((name, description))
    print render_table(["Tool", "Description"], rows)

    print ""
    print "All Tools detected"
    print "------------------"
    print ""
    covered = set([])
    rows = []
    for name, p in jip.scanner.scan().iteritems():
        if name in covered:
            continue
        covered.add(name)
        cls = jip.find(name)
        help = cls.help()
        description = "-"
        if help is not None:
            description = help.split("\n")[0]
        if len(description) > 60:
            description = "%s ..." % description[:46]
        rows.append((cls.name, description))
    print render_table(["Tool", "Description"], rows)