def run(argv=()):
if "--help" in argv:
print __doc__
return 0
logging.basicConfig(level=logging.INFO)
t_start = time.time()
for file_name in os.listdir(DIR_NAME):
tc_file = op.join(DIR_NAME, file_name)
tc = load_tool_contract_from(tc_file)
if tc.task.task_id.split(".")[0] in SKIP_NS:
logging.warn("Skipping %s", file_name)
continue
exe = tc.driver.driver_exe.strip().split()
cmd = list(exe[:-1])
if exe[-1] == "--resolved-tool-contract":
cmd.append("--emit-tool-contract")
elif exe[-1] == "run-rtc":
cmd.extend(["emit-tool-contract", tc.task.task_id])
else:
logging.warn("Don't know how to translate command '%s'",
tc.driver.driver_exe)
continue
logging.info("Running command '%s'", " ".join(cmd))
json_str = subprocess.check_output(cmd)
with open(tc_file, "w") as json_out:
json_out.write(json_str)
t_end = time.time()
logging.info("Tool contracts processed in %.0fs", t_end-t_start)
return 0
def test_sanity(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
tool_options = {}
rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)
self.assertIsInstance(rtc, ResolvedToolContract)
self.assertIsInstance(rtc.task, ResolvedToolContractTask)
self.assertEqual(rtc.task.is_distributed, False)
self.assertEqual(rtc.task.options[self.ALPHA], 25)
self.assertEqual(rtc.task.options[self.BETA], 1.234)
self.assertEqual(rtc.task.options[self.GAMMA], True)
self.assertEqual(rtc.task.options[self.PLOIDY], "haploid")
self.assertEqual(rtc.task.options[self.DELTA], 1)
self.assertEqual(rtc.task.options[self.EPS], 0.1)
self.assertEqual(rtc.task.options[self.COMMENTS], "asdf")
# non-defaults
tool_options = {self.ALPHA: 15, self.BETA: 2.5, self.GAMMA: False, self.PLOIDY: "diploid", self.DELTA: 2, self.EPS: 1.0, self.COMMENTS: "Hello, world!"}
rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)
self.assertEqual(rtc.task.options[self.ALPHA], 15)
self.assertEqual(rtc.task.options[self.BETA], 2.5)
self.assertEqual(rtc.task.options[self.GAMMA], False)
self.assertEqual(rtc.task.options[self.PLOIDY], "diploid")
self.assertEqual(rtc.task.options[self.DELTA], 2)
self.assertEqual(rtc.task.options[self.EPS], 1.0)
self.assertEqual(rtc.task.options[self.COMMENTS], "Hello, world!")
def run(argv=()):
if "--help" in argv:
print __doc__
return 0
logging.basicConfig(level=logging.INFO)
t_start = time.time()
for file_name in os.listdir(DIR_NAME):
tc_file = op.join(DIR_NAME, file_name)
tc = load_tool_contract_from(tc_file)
if tc.task.task_id.split(".")[0] in SKIP_NS:
logging.warn("Skipping %s", file_name)
continue
exe = tc.driver.driver_exe.strip().split()
cmd = list(exe[:-1])
if exe[-1] == "--resolved-tool-contract":
cmd.append("--emit-tool-contract")
elif exe[-1] == "run-rtc":
cmd.extend(["emit-tool-contract", tc.task.task_id])
else:
logging.warn("Don't know how to translate command '%s'",
tc.driver.driver_exe)
continue
logging.info("Running command '%s'", " ".join(cmd))
json_str = subprocess.check_output(cmd)
with open(tc_file, "w") as json_out:
json_out.write(json_str)
t_end = time.time()
logging.info("Tool contracts processed in %.0fs", t_end - t_start)
return 0
def test_sanity(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_data_file(self.FILE_NAME))
rtc = resolve_gather_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, self.TOOL_OPTIONS, self.CHUNK_KEY)
self.assertIsInstance(rtc, ResolvedToolContract)
self.assertIsInstance(rtc.task, ResolvedGatherToolContractTask)
self.assertEqual(rtc.task.chunk_key, self.CHUNK_KEY)
def test_run_e2e(self):
# hack to skip running the base Test class (which is the nose default behavior)
if self.__class__.__name__ in ('PbTestApp', 'PbTestScatterApp', 'PbTestGatherApp'):
return
if self.REQUIRES_PBCORE:
if not HAS_PBCORE:
self.assertTrue(True, pbcore_skip_msg("Skipping running e2e for {d}".format(d=self.DRIVER_EMIT)))
return
output_dir = get_temp_dir(suffix="rtc-test")
tmp_dir = get_temp_dir(suffix="rtc-temp")
log.debug("Driver {e}".format(e=self.DRIVER_EMIT))
log.debug("input files {i}".format(i=self.INPUT_FILES))
log.debug("running in {p}".format(p=output_dir))
output_tc = get_temp_file("-{n}-tool_contract.json".format(n=self.__class__.__name__), output_dir)
emit_tc_exe = "{e} > {o}".format(e=self.DRIVER_EMIT, o=output_tc)
rcode = subprocess.call([emit_tc_exe], shell=True)
self.assertEquals(rcode, 0, "Emitting tool contract failed for '{e}'".format(e=emit_tc_exe))
# sanity marshall-unmashalling
log.debug("Loading tool-contract from {p}".format(p=output_tc))
tc = load_tool_contract_from(output_tc)
log.info("Resolving tool contract to RTC")
rtc = self._to_rtc(tc, output_dir, tmp_dir)
output_json_rtc = get_temp_file("resolved_tool_contract.json", output_dir)
write_resolved_tool_contract(rtc, output_json_rtc)
# sanity
loaded_rtc = load_resolved_tool_contract_from(output_json_rtc)
self.assertIsInstance(loaded_rtc, ResolvedToolContract)
# Test Resolved options if specified.
for opt, resolved_value in self.RESOLVED_TASK_OPTIONS.iteritems():
self.assertTrue(opt in rtc.task.options, "Resolved option {x} not in RTC options.".format(x=opt))
# this needs to support polymorphic equals (i.e., almostEquals
if not isinstance(resolved_value, float):
emsg = "Resolved option {o} are not equal. Expected {a}, got {b}".format(o=opt, b=rtc.task.options[opt], a=resolved_value)
self.assertEquals(rtc.task.options[opt], resolved_value, emsg)
# Resolved NPROC
self.assertEquals(rtc.task.nproc, self.RESOLVED_NPROC)
log.info("running resolved contract {r}".format(r=output_json_rtc))
exe = "{d} {p}".format(p=output_json_rtc, d=self.DRIVER_RESOLVE)
log.info("Running exe '{e}'".format(e=exe))
rcode = subprocess.call([exe], shell=True)
self.assertEqual(rcode, 0, "Running from resolved tool contract failed from {e}".format(e=exe))
log.info("Successfully completed running e2e for {d}".format(d=self.DRIVER_EMIT))
self._test_outputs_exists(rtc)
self.run_after(rtc, output_dir)
def test_sanity(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
rtc = resolve_gather_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, self.TOOL_OPTIONS, self.CHUNK_KEY, False)
self.assertIsInstance(rtc, ResolvedToolContract)
self.assertIsInstance(rtc.task, ResolvedGatherToolContractTask)
self.assertEqual(rtc.task.chunk_key, self.CHUNK_KEY)
self.assertEqual(rtc.task.is_distributed, False)
def test_sanity(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
rtc = resolve_scatter_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, self.TOOL_OPTIONS, self.MAX_NCHUNKS, self.CHUNK_KEYS, False)
self.assertIsInstance(rtc, ResolvedToolContract)
self.assertIsInstance(rtc.task, ResolvedScatteredToolContractTask)
self.assertEqual(rtc.task.max_nchunks, 7)
self.assertEqual(rtc.task.is_distributed, False)
def test_sanity(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_data_file(self.FILE_NAME))
rtc = resolve_scatter_tool_contract(tc, self.INPUT_FILES, d, d,
self.MAX_NPROC, self.TOOL_OPTIONS,
self.MAX_NCHUNKS, self.CHUNK_KEYS)
self.assertIsInstance(rtc, ResolvedToolContract)
self.assertIsInstance(rtc.task, ResolvedScatteredToolContractTask)
self.assertEqual(rtc.task.max_nchunks, 7)
def test_failure_modes(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
tool_options = {self.PLOIDY: "other"}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.ALPHA:2.5}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.ALPHA:"abcdef"}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.BETA:"asdf"}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.GAMMA:1.0}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.GAMMA:""}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
def _load_all_tool_contracts_from(dir_name):
# the old MetaTask model is making this
# a bit convoluted
import pbsmrtpipe.pb_io as IO
mtasks = {}
for file_name in os.listdir(dir_name):
if file_name.endswith('.json'):
f = os.path.join(dir_name, file_name)
# sanity check using pbcommand
_ = load_tool_contract_from(f)
# Old layer to use MetaTask
mtask = IO.tool_contract_to_meta_task_from_file(f)
mtasks[mtask.task_id] = mtask
return mtasks
def _load_all_tool_contracts_from(dir_name):
# the old MetaTask model is making this
# a bit convoluted
import pbsmrtpipe.pb_io as IO
mtasks = {}
for file_name in os.listdir(dir_name):
if file_name.endswith('.json'):
f = os.path.join(dir_name, file_name)
# sanity check using pbcommand
_ = load_tool_contract_from(f)
# Old layer to use MetaTask
mtask = IO.tool_contract_to_meta_task_from_file(f)
mtasks[mtask.task_id] = mtask
return mtasks
def _run_main(args):
return run_main(load_tool_contract_from(args.tc_path))
def test_run_e2e(self):
# hack to skip running the base Test class (which is the nose default behavior)
if self.__class__.__name__ in ('PbTestApp', 'PbTestScatterApp',
'PbTestGatherApp'):
return
if self.REQUIRES_PBCORE:
if not HAS_PBCORE:
self.assertTrue(
True,
pbcore_skip_msg("Skipping running e2e for {d}".format(
d=self.DRIVER_EMIT)))
return
output_dir = get_temp_dir(suffix="rtc-test")
tmp_dir = get_temp_dir(suffix="rtc-temp")
log.debug("Driver {e}".format(e=self.DRIVER_EMIT))
log.debug("input files {i}".format(i=self.INPUT_FILES))
log.debug("running in {p}".format(p=output_dir))
output_tc = get_temp_file(
"-{n}-tool_contract.json".format(n=self.__class__.__name__),
output_dir)
emit_tc_exe = "{e} > {o}".format(e=self.DRIVER_EMIT, o=output_tc)
rcode = subprocess.call([emit_tc_exe], shell=True)
self.assertEquals(
rcode, 0,
"Emitting tool contract failed for '{e}'".format(e=emit_tc_exe))
# sanity marshall-unmashalling
log.debug("Loading tool-contract from {p}".format(p=output_tc))
tc = load_tool_contract_from(output_tc)
log.info("Resolving tool contract to RTC")
rtc = self._to_rtc(tc, output_dir, tmp_dir)
output_json_rtc = get_temp_file("resolved_tool_contract.json",
output_dir)
write_resolved_tool_contract(rtc, output_json_rtc)
# sanity
loaded_rtc = load_resolved_tool_contract_from(output_json_rtc)
self.assertIsInstance(loaded_rtc, ResolvedToolContract)
# Test Resolved options if specified.
for opt, resolved_value in self.RESOLVED_TASK_OPTIONS.iteritems():
self.assertTrue(
opt in rtc.task.options,
"Resolved option {x} not in RTC options.".format(x=opt))
# this needs to support polymorphic equals (i.e., almostEquals
if not isinstance(resolved_value, float):
emsg = "Resolved option {o} are not equal. Expected {a}, got {b}".format(
o=opt, b=rtc.task.options[opt], a=resolved_value)
self.assertEquals(rtc.task.options[opt], resolved_value, emsg)
# Resolved NPROC
self.assertEquals(rtc.task.nproc, self.RESOLVED_NPROC)
self.assertEquals(rtc.task.is_distributed,
self.RESOLVED_IS_DISTRIBUTED)
log.info("running resolved contract {r}".format(r=output_json_rtc))
exe = "{d} {p}".format(p=output_json_rtc, d=self.DRIVER_RESOLVE)
log.info("Running exe '{e}'".format(e=exe))
with tempfile.TemporaryFile() as stdout:
rcode = subprocess.call([exe], shell=True, stdout=stdout)
self.assertEquals(
rcode, 0,
"Running from resolved tool contract failed from {x}".format(
x=exe))
log.info("Successfully completed running e2e for {d}".format(
d=self.DRIVER_EMIT))
self._test_outputs_exists(rtc)
self.run_after(rtc, output_dir)
def test_validate_tool_contracts(self):
for path in walker(DATA_DIR_TC, json_filter):
f = _to_assertion(path, validate_tc)
f(self)
self.assertIsInstance(load_tool_contract_from(path), ToolContract)
def test_validate_tool_contracts(self):
for path in walker(DATA_DIR_TC, json_filter):
f = _to_assertion(path, validate_tc)
f(self)
self.assertIsInstance(load_tool_contract_from(path), ToolContract)
def test_validate_tool_contracts(self):
for path in _get_all_from(DATA_DIR, _filter_tc):
f = _to_assertion(path, validate_tc)
f(self)
self.assertIsInstance(load_tool_contract_from(path), ToolContract)
def test_validate_tool_contracts(self):
for path in _get_all_from(DATA_DIR, _filter_tc):
f = _to_assertion(path, validate_tc)
f(self)
self.assertIsInstance(load_tool_contract_from(path), ToolContract)
