def test_sanity(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
tool_options = {}
rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)
self.assertIsInstance(rtc, ResolvedToolContract)
self.assertIsInstance(rtc.task, ResolvedToolContractTask)
self.assertEqual(rtc.task.is_distributed, False)
self.assertEqual(rtc.task.options[self.ALPHA], 25)
self.assertEqual(rtc.task.options[self.BETA], 1.234)
self.assertEqual(rtc.task.options[self.GAMMA], True)
self.assertEqual(rtc.task.options[self.PLOIDY], "haploid")
self.assertEqual(rtc.task.options[self.DELTA], 1)
self.assertEqual(rtc.task.options[self.EPS], 0.1)
self.assertEqual(rtc.task.options[self.COMMENTS], "asdf")
# non-defaults
tool_options = {self.ALPHA: 15, self.BETA: 2.5, self.GAMMA: False, self.PLOIDY: "diploid", self.DELTA: 2, self.EPS: 1.0, self.COMMENTS: "Hello, world!"}
rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)
self.assertEqual(rtc.task.options[self.ALPHA], 15)
self.assertEqual(rtc.task.options[self.BETA], 2.5)
self.assertEqual(rtc.task.options[self.GAMMA], False)
self.assertEqual(rtc.task.options[self.PLOIDY], "diploid")
self.assertEqual(rtc.task.options[self.DELTA], 2)
self.assertEqual(rtc.task.options[self.EPS], 1.0)
self.assertEqual(rtc.task.options[self.COMMENTS], "Hello, world!")
def run_main(tc):
""":type tc: ToolContract"""
print "Loaded tc {c}".format(c=tc)
h = History()
if tc.task.nproc == SymbolTypes.MAX_NPROC:
nproc = get_input('Enter max nproc: ')
else:
# not quite right
nproc = 1
output_dir = get_input('Output Directory: ',
enable_system_bindings=Always())
output_dir = os.path.abspath(output_dir)
input_files = []
for i, input_type in enumerate(tc.task.input_file_types):
in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type))
if not os.path.exists(in_path):
warnings.warn("Unable to find {p}".format(p=in_path))
input_files.append(in_path)
tool_options = {}
rtc = resolve_tool_contract(tc, input_files, output_dir, '/tmp',
int(nproc), tool_options)
print rtc
print "writing RTC"
file_name = tc.task.task_id + "_resolved_tool_contract.json"
rtc_path = os.path.join(output_dir, file_name)
write_resolved_tool_contract(rtc, rtc_path)
return rtc
def static_meta_task_to_rtc(static_meta_task, task, task_options, task_dir,
tmp_dir, max_nproc, is_distributed):
"""
Shim layer to converts a static metatask to ResolvedToolContract
:type static_meta_task: ToolContractMetaTask
:type task: MetaTask
:param static_meta_task:
:return: dict representation of driver manifest
"""
rtc = resolve_tool_contract(static_meta_task.tool_contract,
task.input_files,
task_dir,
tmp_dir,
max_nproc,
task_options,
is_distributable=is_distributed)
# this is a hack because the 'resolving' is done in meta_task_to_task
# for python defined tasks. This code path will be deleted shortly
rtc.task.output_files = task.output_files
rtc.task.nproc = task.nproc
task.resources = task.resources
return rtc
def run_main(tc):
""":type tc: ToolContract"""
print "Loaded tc {c}".format(c=tc)
if tc.task.nproc == SymbolTypes.MAX_NPROC:
nproc = get_input("Enter max nproc: ")
else:
# not quite right
nproc = 1
output_dir = get_input("Output Directory: ", enable_system_bindings=Always())
output_dir = os.path.abspath(output_dir)
input_files = []
for i, input_type in enumerate(tc.task.input_file_types):
in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type))
if not os.path.exists(in_path):
warnings.warn("Unable to find {p}".format(p=in_path))
input_files.append(in_path)
tool_options = {}
rtc = resolve_tool_contract(tc, input_files, output_dir, "/tmp", int(nproc), tool_options)
print rtc
file_name = tc.task.task_id + "_resolved_tool_contract.json"
rtc_path = os.path.join(output_dir, file_name)
print "writing RTC to {f}".format(f=rtc_path)
write_resolved_tool_contract(rtc, rtc_path)
return rtc
def run_main(tc):
""":type tc: ToolContract"""
print("Loaded tc {c}".format(c=tc))
if tc.task.nproc == SymbolTypes.MAX_NPROC:
nproc = get_input('Enter max nproc: ')
else:
# not quite right
nproc = 1
output_dir = get_input('Output Directory: ',
enable_system_bindings=Always())
output_dir = os.path.abspath(output_dir)
input_files = []
for i, input_type in enumerate(tc.task.input_file_types):
in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type))
if not os.path.exists(in_path):
warnings.warn("Unable to find {p}".format(p=in_path))
# Make sure all inputs are abspaths
p = in_path if os.path.isabs(in_path) else os.path.abspath(in_path)
input_files.append(p)
tool_options = {}
rtc = resolve_tool_contract(tc,
input_files,
output_dir,
'/tmp',
int(nproc),
tool_options,
is_distributable=False)
print(rtc)
def to_n(ext):
return "resolved_tool_contract." + ext
def to_f(ext):
return "_".join([tc.task.task_id, to_n(ext)])
def to_p(ext):
return os.path.join(output_dir, to_f(ext))
rtc_path = to_p("json")
print("writing RTC to {f}".format(f=rtc_path))
# Always write the JSON RTC file
write_resolved_tool_contract(rtc, rtc_path)
if rtc.driver.serialization.lower() == "avro":
avro_rtc_path = to_p("avro")
print("writing AVRO RTC to {f}".format(f=avro_rtc_path))
write_resolved_tool_contract_avro(rtc, avro_rtc_path)
return rtc
def test_01(self):
name = "dev_example_tool_contract.json"
p = get_data_file(name)
tc = load_tool_contract_from(p)
input_files = ["/tmp/file.txt"]
root_output_dir = "/tmp"
root_tmp_dir = root_output_dir
max_nproc = 2
rtc = resolve_tool_contract(tc, input_files, root_output_dir, root_tmp_dir, max_nproc, {})
log.info(pprint.pformat(rtc))
self.assertIsNotNone(rtc)
self.assertEqual(os.path.basename(rtc.task.output_files[0]),
"output.txt")
def test_01(self):
name = "dev_example_tool_contract.json"
p = get_data_file(name)
tc = load_tool_contract_from(p)
input_files = ["/tmp/file.txt"]
root_output_dir = "/tmp"
root_tmp_dir = root_output_dir
max_nproc = 2
rtc = resolve_tool_contract(tc, input_files, root_output_dir,
root_tmp_dir, max_nproc, {})
log.info(pprint.pformat(rtc))
self.assertIsNotNone(rtc)
self.assertEqual(os.path.basename(rtc.task.output_files[0]),
"output.txt")
def run_main(tc):
""":type tc: ToolContract"""
print "Loaded tc {c}".format(c=tc)
if tc.task.nproc == SymbolTypes.MAX_NPROC:
nproc = get_input('Enter max nproc: ')
else:
# not quite right
nproc = 1
output_dir = get_input('Output Directory: ', enable_system_bindings=Always())
output_dir = os.path.abspath(output_dir)
input_files = []
for i, input_type in enumerate(tc.task.input_file_types):
in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type))
if not os.path.exists(in_path):
warnings.warn("Unable to find {p}".format(p=in_path))
# Make sure all inputs are abspaths
p = in_path if os.path.isabs(in_path) else os.path.abspath(in_path)
input_files.append(p)
tool_options = {}
rtc = resolve_tool_contract(tc, input_files, output_dir, '/tmp', int(nproc), tool_options, is_distributable=False)
print rtc
def to_n(ext):
return "resolved_tool_contract." + ext
def to_f(ext):
return "_".join([tc.task.task_id, to_n(ext)])
def to_p(ext):
return os.path.join(output_dir, to_f(ext))
rtc_path = to_p("json")
print "writing RTC to {f}".format(f=rtc_path)
# Always write the JSON RTC file
write_resolved_tool_contract(rtc, rtc_path)
if rtc.driver.serialization.lower() == "avro":
avro_rtc_path = to_p("avro")
print "writing AVRO RTC to {f}".format(f=avro_rtc_path)
write_resolved_tool_contract_avro(rtc, avro_rtc_path)
return rtc
def static_meta_task_to_rtc(static_meta_task, task, task_options, task_dir, tmp_dir, max_nproc, is_distributed):
"""
Shim layer to converts a static metatask to ResolvedToolContract
:type static_meta_task: ToolContractMetaTask
:type task: MetaTask
:param static_meta_task:
:return: dict representation of driver manifest
"""
rtc = resolve_tool_contract(static_meta_task.tool_contract, task.input_files, task_dir, tmp_dir, max_nproc, task_options, is_distributable=is_distributed)
# this is a hack because the 'resolving' is done in meta_task_to_task
# for python defined tasks. This code path will be deleted shortly
rtc.task.output_files = task.output_files
rtc.task.nproc = task.nproc
task.resources = task.resources
return rtc
def test_01(self):
name = "dev_example_dev_txt_app_tool_contract.json"
p = get_data_file(name)
tc = load_tool_contract_from(p)
input_files = ["/tmp/file.txt"]
root_output_dir = "/tmp"
root_tmp_dir = root_output_dir
tmp_file = tempfile.NamedTemporaryFile().name
max_nproc = 2
tool_options = {}
rtc = resolve_tool_contract(tc, input_files, root_output_dir, root_tmp_dir, max_nproc, tool_options, False)
log.info(pprint.pformat(rtc))
self.assertIsNotNone(rtc)
self.assertEqual(os.path.basename(rtc.task.output_files[0]),
"output.txt")
# Validate Resolved Resource Types
log.debug("Resources {t}".format(t=rtc.task.resources))
self.assertEqual(len(rtc.task.tmpdir_resources), 1)
self.assertEqual(len(rtc.task.tmpfile_resources), 2)
def test_failure_modes(self):
d = get_temp_dir("resolved-tool-contract")
tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME))
tool_options = {self.PLOIDY: "other"}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.ALPHA:2.5}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.ALPHA:"abcdef"}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.BETA:"asdf"}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.GAMMA:1.0}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
tool_options = {self.GAMMA:""}
self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
def _to_rtc(self, tc, output_dir, tmp_dir):
# handled the polymorphism in subclasses by overriding
return resolve_tool_contract(tc, self.INPUT_FILES, output_dir, tmp_dir,
self.MAX_NPROC, self.TASK_OPTIONS,
self.IS_DISTRIBUTED)
def _to_rtc(self, tc, output_dir, tmp_dir):
# handled the polymorphism in subclasses by overriding
return resolve_tool_contract(tc, self.INPUT_FILES, output_dir, tmp_dir, self.MAX_NPROC, self.TASK_OPTIONS)
