def run(argv=()):
    if "--help" in argv:
        print __doc__
        return 0
    logging.basicConfig(level=logging.INFO)
    t_start = time.time()
    for file_name in os.listdir(DIR_NAME):
        tc_file = op.join(DIR_NAME, file_name)
        tc = load_tool_contract_from(tc_file)
        if tc.task.task_id.split(".")[0] in SKIP_NS:
            logging.warn("Skipping %s", file_name)
            continue
        exe = tc.driver.driver_exe.strip().split()
        cmd = list(exe[:-1])
        if exe[-1] == "--resolved-tool-contract":
            cmd.append("--emit-tool-contract")
        elif exe[-1] == "run-rtc":
            cmd.extend(["emit-tool-contract", tc.task.task_id])
        else:
            logging.warn("Don't know how to translate command '%s'",
                         tc.driver.driver_exe)
            continue
        logging.info("Running command '%s'", " ".join(cmd))
        json_str = subprocess.check_output(cmd)
        with open(tc_file, "w") as json_out:
            json_out.write(json_str)
    t_end = time.time()
    logging.info("Tool contracts processed in %.0fs", t_end-t_start)
    return 0
Example #2
0
 def test_sanity(self):
     d = get_temp_dir("resolved-tool-contract")
     tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
     tool_options = {}
     rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)
     self.assertIsInstance(rtc, ResolvedToolContract)
     self.assertIsInstance(rtc.task, ResolvedToolContractTask)
     self.assertEqual(rtc.task.is_distributed, False)
     self.assertEqual(rtc.task.options[self.ALPHA], 25)
     self.assertEqual(rtc.task.options[self.BETA], 1.234)
     self.assertEqual(rtc.task.options[self.GAMMA], True)
     self.assertEqual(rtc.task.options[self.PLOIDY], "haploid")
     self.assertEqual(rtc.task.options[self.DELTA], 1)
     self.assertEqual(rtc.task.options[self.EPS], 0.1)
     self.assertEqual(rtc.task.options[self.COMMENTS], "asdf")
     # non-defaults
     tool_options = {self.ALPHA: 15, self.BETA: 2.5, self.GAMMA: False, self.PLOIDY: "diploid", self.DELTA: 2, self.EPS: 1.0, self.COMMENTS: "Hello, world!"}
     rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)
     self.assertEqual(rtc.task.options[self.ALPHA], 15)
     self.assertEqual(rtc.task.options[self.BETA], 2.5)
     self.assertEqual(rtc.task.options[self.GAMMA], False)
     self.assertEqual(rtc.task.options[self.PLOIDY], "diploid")
     self.assertEqual(rtc.task.options[self.DELTA], 2)
     self.assertEqual(rtc.task.options[self.EPS], 1.0)
     self.assertEqual(rtc.task.options[self.COMMENTS], "Hello, world!")
Example #3
0
def run(argv=()):
    if "--help" in argv:
        print __doc__
        return 0
    logging.basicConfig(level=logging.INFO)
    t_start = time.time()
    for file_name in os.listdir(DIR_NAME):
        tc_file = op.join(DIR_NAME, file_name)
        tc = load_tool_contract_from(tc_file)
        if tc.task.task_id.split(".")[0] in SKIP_NS:
            logging.warn("Skipping %s", file_name)
            continue
        exe = tc.driver.driver_exe.strip().split()
        cmd = list(exe[:-1])
        if exe[-1] == "--resolved-tool-contract":
            cmd.append("--emit-tool-contract")
        elif exe[-1] == "run-rtc":
            cmd.extend(["emit-tool-contract", tc.task.task_id])
        else:
            logging.warn("Don't know how to translate command '%s'",
                         tc.driver.driver_exe)
            continue
        logging.info("Running command '%s'", " ".join(cmd))
        json_str = subprocess.check_output(cmd)
        with open(tc_file, "w") as json_out:
            json_out.write(json_str)
    t_end = time.time()
    logging.info("Tool contracts processed in %.0fs", t_end - t_start)
    return 0
Example #4
0
 def test_sanity(self):
     d = get_temp_dir("resolved-tool-contract")
     tc = load_tool_contract_from(get_data_file(self.FILE_NAME))
     rtc = resolve_gather_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, self.TOOL_OPTIONS, self.CHUNK_KEY)
     self.assertIsInstance(rtc, ResolvedToolContract)
     self.assertIsInstance(rtc.task, ResolvedGatherToolContractTask)
     self.assertEqual(rtc.task.chunk_key, self.CHUNK_KEY)
Example #5
0
    def test_run_e2e(self):
        # hack to skip running the base Test class (which is the nose default behavior)
        if self.__class__.__name__ in ('PbTestApp', 'PbTestScatterApp', 'PbTestGatherApp'):
            return

        if self.REQUIRES_PBCORE:
            if not HAS_PBCORE:
                self.assertTrue(True, pbcore_skip_msg("Skipping running e2e for {d}".format(d=self.DRIVER_EMIT)))
                return

        output_dir = get_temp_dir(suffix="rtc-test")
        tmp_dir = get_temp_dir(suffix="rtc-temp")

        log.debug("Driver {e}".format(e=self.DRIVER_EMIT))
        log.debug("input files {i}".format(i=self.INPUT_FILES))
        log.debug("running in {p}".format(p=output_dir))

        output_tc = get_temp_file("-{n}-tool_contract.json".format(n=self.__class__.__name__), output_dir)
        emit_tc_exe = "{e} > {o}".format(e=self.DRIVER_EMIT, o=output_tc)
        rcode = subprocess.call([emit_tc_exe], shell=True)

        self.assertEquals(rcode, 0, "Emitting tool contract failed for '{e}'".format(e=emit_tc_exe))

        # sanity marshall-unmashalling
        log.debug("Loading tool-contract from {p}".format(p=output_tc))
        tc = load_tool_contract_from(output_tc)

        log.info("Resolving tool contract to RTC")

        rtc = self._to_rtc(tc, output_dir, tmp_dir)

        output_json_rtc = get_temp_file("resolved_tool_contract.json", output_dir)
        write_resolved_tool_contract(rtc, output_json_rtc)

        # sanity
        loaded_rtc = load_resolved_tool_contract_from(output_json_rtc)
        self.assertIsInstance(loaded_rtc, ResolvedToolContract)

        # Test Resolved options if specified.
        for opt, resolved_value in self.RESOLVED_TASK_OPTIONS.iteritems():
            self.assertTrue(opt in rtc.task.options, "Resolved option {x} not in RTC options.".format(x=opt))
            # this needs to support polymorphic equals (i.e., almostEquals
            if not isinstance(resolved_value, float):
                emsg = "Resolved option {o} are not equal. Expected {a}, got {b}".format(o=opt, b=rtc.task.options[opt], a=resolved_value)
                self.assertEquals(rtc.task.options[opt], resolved_value, emsg)

        # Resolved NPROC
        self.assertEquals(rtc.task.nproc, self.RESOLVED_NPROC)

        log.info("running resolved contract {r}".format(r=output_json_rtc))

        exe = "{d} {p}".format(p=output_json_rtc, d=self.DRIVER_RESOLVE)
        log.info("Running exe '{e}'".format(e=exe))
        rcode = subprocess.call([exe], shell=True)
        self.assertEqual(rcode, 0, "Running from resolved tool contract failed from {e}".format(e=exe))
        log.info("Successfully completed running e2e for {d}".format(d=self.DRIVER_EMIT))

        self._test_outputs_exists(rtc)

        self.run_after(rtc, output_dir)
Example #6
0
 def test_sanity(self):
     d = get_temp_dir("resolved-tool-contract")
     tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
     rtc = resolve_gather_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, self.TOOL_OPTIONS, self.CHUNK_KEY, False)
     self.assertIsInstance(rtc, ResolvedToolContract)
     self.assertIsInstance(rtc.task, ResolvedGatherToolContractTask)
     self.assertEqual(rtc.task.chunk_key, self.CHUNK_KEY)
     self.assertEqual(rtc.task.is_distributed, False)
 def test_sanity(self):
     d = get_temp_dir("resolved-tool-contract")
     tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
     rtc = resolve_scatter_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, self.TOOL_OPTIONS, self.MAX_NCHUNKS, self.CHUNK_KEYS, False)
     self.assertIsInstance(rtc, ResolvedToolContract)
     self.assertIsInstance(rtc.task, ResolvedScatteredToolContractTask)
     self.assertEqual(rtc.task.max_nchunks, 7)
     self.assertEqual(rtc.task.is_distributed, False)
Example #8
0
 def test_sanity(self):
     d = get_temp_dir("resolved-tool-contract")
     tc = load_tool_contract_from(get_data_file(self.FILE_NAME))
     rtc = resolve_scatter_tool_contract(tc, self.INPUT_FILES, d, d,
                                         self.MAX_NPROC, self.TOOL_OPTIONS,
                                         self.MAX_NCHUNKS, self.CHUNK_KEYS)
     self.assertIsInstance(rtc, ResolvedToolContract)
     self.assertIsInstance(rtc.task, ResolvedScatteredToolContractTask)
     self.assertEqual(rtc.task.max_nchunks, 7)
Example #9
0
 def test_failure_modes(self):
     d = get_temp_dir("resolved-tool-contract")
     tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
     tool_options = {self.PLOIDY: "other"}
     self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
     tool_options = {self.ALPHA:2.5}
     self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
     tool_options = {self.ALPHA:"abcdef"}
     self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
     tool_options = {self.BETA:"asdf"}
     self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
     tool_options = {self.GAMMA:1.0}
     self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
     tool_options = {self.GAMMA:""}
     self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
Example #10
0
def _load_all_tool_contracts_from(dir_name):
    # the old MetaTask model is making this
    # a bit convoluted
    import pbsmrtpipe.pb_io as IO
    mtasks = {}
    for file_name in os.listdir(dir_name):
        if file_name.endswith('.json'):
            f = os.path.join(dir_name, file_name)
            # sanity check using pbcommand
            _ = load_tool_contract_from(f)
            # Old layer to use MetaTask
            mtask = IO.tool_contract_to_meta_task_from_file(f)
            mtasks[mtask.task_id] = mtask

    return mtasks
Example #11
0
def _load_all_tool_contracts_from(dir_name):
    # the old MetaTask model is making this
    # a bit convoluted
    import pbsmrtpipe.pb_io as IO
    mtasks = {}
    for file_name in os.listdir(dir_name):
        if file_name.endswith('.json'):
            f = os.path.join(dir_name, file_name)
            # sanity check using pbcommand
            _ = load_tool_contract_from(f)
            # Old layer to use MetaTask
            mtask = IO.tool_contract_to_meta_task_from_file(f)
            mtasks[mtask.task_id] = mtask

    return mtasks
def _run_main(args):
    return run_main(load_tool_contract_from(args.tc_path))
Example #13
0
    def test_run_e2e(self):
        # hack to skip running the base Test class (which is the nose default behavior)
        if self.__class__.__name__ in ('PbTestApp', 'PbTestScatterApp',
                                       'PbTestGatherApp'):
            return

        if self.REQUIRES_PBCORE:
            if not HAS_PBCORE:
                self.assertTrue(
                    True,
                    pbcore_skip_msg("Skipping running e2e for {d}".format(
                        d=self.DRIVER_EMIT)))
                return

        output_dir = get_temp_dir(suffix="rtc-test")
        tmp_dir = get_temp_dir(suffix="rtc-temp")

        log.debug("Driver {e}".format(e=self.DRIVER_EMIT))
        log.debug("input files {i}".format(i=self.INPUT_FILES))
        log.debug("running in {p}".format(p=output_dir))

        output_tc = get_temp_file(
            "-{n}-tool_contract.json".format(n=self.__class__.__name__),
            output_dir)
        emit_tc_exe = "{e} > {o}".format(e=self.DRIVER_EMIT, o=output_tc)
        rcode = subprocess.call([emit_tc_exe], shell=True)
        self.assertEquals(
            rcode, 0,
            "Emitting tool contract failed for '{e}'".format(e=emit_tc_exe))

        # sanity marshall-unmashalling
        log.debug("Loading tool-contract from {p}".format(p=output_tc))
        tc = load_tool_contract_from(output_tc)

        log.info("Resolving tool contract to RTC")

        rtc = self._to_rtc(tc, output_dir, tmp_dir)

        output_json_rtc = get_temp_file("resolved_tool_contract.json",
                                        output_dir)
        write_resolved_tool_contract(rtc, output_json_rtc)

        # sanity
        loaded_rtc = load_resolved_tool_contract_from(output_json_rtc)
        self.assertIsInstance(loaded_rtc, ResolvedToolContract)

        # Test Resolved options if specified.
        for opt, resolved_value in self.RESOLVED_TASK_OPTIONS.iteritems():
            self.assertTrue(
                opt in rtc.task.options,
                "Resolved option {x} not in RTC options.".format(x=opt))
            # this needs to support polymorphic equals (i.e., almostEquals
            if not isinstance(resolved_value, float):
                emsg = "Resolved option {o} are not equal. Expected {a}, got {b}".format(
                    o=opt, b=rtc.task.options[opt], a=resolved_value)
                self.assertEquals(rtc.task.options[opt], resolved_value, emsg)

        # Resolved NPROC
        self.assertEquals(rtc.task.nproc, self.RESOLVED_NPROC)
        self.assertEquals(rtc.task.is_distributed,
                          self.RESOLVED_IS_DISTRIBUTED)

        log.info("running resolved contract {r}".format(r=output_json_rtc))

        exe = "{d} {p}".format(p=output_json_rtc, d=self.DRIVER_RESOLVE)
        log.info("Running exe '{e}'".format(e=exe))
        with tempfile.TemporaryFile() as stdout:
            rcode = subprocess.call([exe], shell=True, stdout=stdout)
            self.assertEquals(
                rcode, 0,
                "Running from resolved tool contract failed from {x}".format(
                    x=exe))
        log.info("Successfully completed running e2e for {d}".format(
            d=self.DRIVER_EMIT))

        self._test_outputs_exists(rtc)
        self.run_after(rtc, output_dir)
 def test_validate_tool_contracts(self):
     for path in walker(DATA_DIR_TC, json_filter):
         f = _to_assertion(path, validate_tc)
         f(self)
         self.assertIsInstance(load_tool_contract_from(path), ToolContract)
Example #15
0
 def test_validate_tool_contracts(self):
     for path in walker(DATA_DIR_TC, json_filter):
         f = _to_assertion(path, validate_tc)
         f(self)
         self.assertIsInstance(load_tool_contract_from(path), ToolContract)
 def test_validate_tool_contracts(self):
     for path in _get_all_from(DATA_DIR, _filter_tc):
         f = _to_assertion(path, validate_tc)
         f(self)
         self.assertIsInstance(load_tool_contract_from(path), ToolContract)
 def test_validate_tool_contracts(self):
     for path in _get_all_from(DATA_DIR, _filter_tc):
         f = _to_assertion(path, validate_tc)
         f(self)
         self.assertIsInstance(load_tool_contract_from(path), ToolContract)