def test_sanity(self): d = get_temp_dir("resolved-tool-contract") tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME)) tool_options = {} rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False) self.assertIsInstance(rtc, ResolvedToolContract) self.assertIsInstance(rtc.task, ResolvedToolContractTask) self.assertEqual(rtc.task.is_distributed, False) self.assertEqual(rtc.task.options[self.ALPHA], 25) self.assertEqual(rtc.task.options[self.BETA], 1.234) self.assertEqual(rtc.task.options[self.GAMMA], True) self.assertEqual(rtc.task.options[self.PLOIDY], "haploid") self.assertEqual(rtc.task.options[self.DELTA], 1) self.assertEqual(rtc.task.options[self.EPS], 0.1) self.assertEqual(rtc.task.options[self.COMMENTS], "asdf") # non-defaults tool_options = {self.ALPHA: 15, self.BETA: 2.5, self.GAMMA: False, self.PLOIDY: "diploid", self.DELTA: 2, self.EPS: 1.0, self.COMMENTS: "Hello, world!"} rtc = resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False) self.assertEqual(rtc.task.options[self.ALPHA], 15) self.assertEqual(rtc.task.options[self.BETA], 2.5) self.assertEqual(rtc.task.options[self.GAMMA], False) self.assertEqual(rtc.task.options[self.PLOIDY], "diploid") self.assertEqual(rtc.task.options[self.DELTA], 2) self.assertEqual(rtc.task.options[self.EPS], 1.0) self.assertEqual(rtc.task.options[self.COMMENTS], "Hello, world!")
def run_main(tc): """:type tc: ToolContract""" print "Loaded tc {c}".format(c=tc) h = History() if tc.task.nproc == SymbolTypes.MAX_NPROC: nproc = get_input('Enter max nproc: ') else: # not quite right nproc = 1 output_dir = get_input('Output Directory: ', enable_system_bindings=Always()) output_dir = os.path.abspath(output_dir) input_files = [] for i, input_type in enumerate(tc.task.input_file_types): in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type)) if not os.path.exists(in_path): warnings.warn("Unable to find {p}".format(p=in_path)) input_files.append(in_path) tool_options = {} rtc = resolve_tool_contract(tc, input_files, output_dir, '/tmp', int(nproc), tool_options) print rtc print "writing RTC" file_name = tc.task.task_id + "_resolved_tool_contract.json" rtc_path = os.path.join(output_dir, file_name) write_resolved_tool_contract(rtc, rtc_path) return rtc
def static_meta_task_to_rtc(static_meta_task, task, task_options, task_dir, tmp_dir, max_nproc, is_distributed): """ Shim layer to converts a static metatask to ResolvedToolContract :type static_meta_task: ToolContractMetaTask :type task: MetaTask :param static_meta_task: :return: dict representation of driver manifest """ rtc = resolve_tool_contract(static_meta_task.tool_contract, task.input_files, task_dir, tmp_dir, max_nproc, task_options, is_distributable=is_distributed) # this is a hack because the 'resolving' is done in meta_task_to_task # for python defined tasks. This code path will be deleted shortly rtc.task.output_files = task.output_files rtc.task.nproc = task.nproc task.resources = task.resources return rtc
def run_main(tc): """:type tc: ToolContract""" print "Loaded tc {c}".format(c=tc) if tc.task.nproc == SymbolTypes.MAX_NPROC: nproc = get_input("Enter max nproc: ") else: # not quite right nproc = 1 output_dir = get_input("Output Directory: ", enable_system_bindings=Always()) output_dir = os.path.abspath(output_dir) input_files = [] for i, input_type in enumerate(tc.task.input_file_types): in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type)) if not os.path.exists(in_path): warnings.warn("Unable to find {p}".format(p=in_path)) input_files.append(in_path) tool_options = {} rtc = resolve_tool_contract(tc, input_files, output_dir, "/tmp", int(nproc), tool_options) print rtc file_name = tc.task.task_id + "_resolved_tool_contract.json" rtc_path = os.path.join(output_dir, file_name) print "writing RTC to {f}".format(f=rtc_path) write_resolved_tool_contract(rtc, rtc_path) return rtc
def run_main(tc): """:type tc: ToolContract""" print("Loaded tc {c}".format(c=tc)) if tc.task.nproc == SymbolTypes.MAX_NPROC: nproc = get_input('Enter max nproc: ') else: # not quite right nproc = 1 output_dir = get_input('Output Directory: ', enable_system_bindings=Always()) output_dir = os.path.abspath(output_dir) input_files = [] for i, input_type in enumerate(tc.task.input_file_types): in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type)) if not os.path.exists(in_path): warnings.warn("Unable to find {p}".format(p=in_path)) # Make sure all inputs are abspaths p = in_path if os.path.isabs(in_path) else os.path.abspath(in_path) input_files.append(p) tool_options = {} rtc = resolve_tool_contract(tc, input_files, output_dir, '/tmp', int(nproc), tool_options, is_distributable=False) print(rtc) def to_n(ext): return "resolved_tool_contract." + ext def to_f(ext): return "_".join([tc.task.task_id, to_n(ext)]) def to_p(ext): return os.path.join(output_dir, to_f(ext)) rtc_path = to_p("json") print("writing RTC to {f}".format(f=rtc_path)) # Always write the JSON RTC file write_resolved_tool_contract(rtc, rtc_path) if rtc.driver.serialization.lower() == "avro": avro_rtc_path = to_p("avro") print("writing AVRO RTC to {f}".format(f=avro_rtc_path)) write_resolved_tool_contract_avro(rtc, avro_rtc_path) return rtc
def test_01(self): name = "dev_example_tool_contract.json" p = get_data_file(name) tc = load_tool_contract_from(p) input_files = ["/tmp/file.txt"] root_output_dir = "/tmp" root_tmp_dir = root_output_dir max_nproc = 2 rtc = resolve_tool_contract(tc, input_files, root_output_dir, root_tmp_dir, max_nproc, {}) log.info(pprint.pformat(rtc)) self.assertIsNotNone(rtc) self.assertEqual(os.path.basename(rtc.task.output_files[0]), "output.txt")
def run_main(tc): """:type tc: ToolContract""" print "Loaded tc {c}".format(c=tc) if tc.task.nproc == SymbolTypes.MAX_NPROC: nproc = get_input('Enter max nproc: ') else: # not quite right nproc = 1 output_dir = get_input('Output Directory: ', enable_system_bindings=Always()) output_dir = os.path.abspath(output_dir) input_files = [] for i, input_type in enumerate(tc.task.input_file_types): in_path = get_input(" {i} file {p} path :".format(i=i, p=input_type)) if not os.path.exists(in_path): warnings.warn("Unable to find {p}".format(p=in_path)) # Make sure all inputs are abspaths p = in_path if os.path.isabs(in_path) else os.path.abspath(in_path) input_files.append(p) tool_options = {} rtc = resolve_tool_contract(tc, input_files, output_dir, '/tmp', int(nproc), tool_options, is_distributable=False) print rtc def to_n(ext): return "resolved_tool_contract." + ext def to_f(ext): return "_".join([tc.task.task_id, to_n(ext)]) def to_p(ext): return os.path.join(output_dir, to_f(ext)) rtc_path = to_p("json") print "writing RTC to {f}".format(f=rtc_path) # Always write the JSON RTC file write_resolved_tool_contract(rtc, rtc_path) if rtc.driver.serialization.lower() == "avro": avro_rtc_path = to_p("avro") print "writing AVRO RTC to {f}".format(f=avro_rtc_path) write_resolved_tool_contract_avro(rtc, avro_rtc_path) return rtc
def test_01(self): name = "dev_example_dev_txt_app_tool_contract.json" p = get_data_file(name) tc = load_tool_contract_from(p) input_files = ["/tmp/file.txt"] root_output_dir = "/tmp" root_tmp_dir = root_output_dir tmp_file = tempfile.NamedTemporaryFile().name max_nproc = 2 tool_options = {} rtc = resolve_tool_contract(tc, input_files, root_output_dir, root_tmp_dir, max_nproc, tool_options, False) log.info(pprint.pformat(rtc)) self.assertIsNotNone(rtc) self.assertEqual(os.path.basename(rtc.task.output_files[0]), "output.txt") # Validate Resolved Resource Types log.debug("Resources {t}".format(t=rtc.task.resources)) self.assertEqual(len(rtc.task.tmpdir_resources), 1) self.assertEqual(len(rtc.task.tmpfile_resources), 2)
def test_failure_modes(self): d = get_temp_dir("resolved-tool-contract") tc = load_tool_contract_from(get_tool_contract(self.FILE_NAME)) tool_options = {self.PLOIDY: "other"} self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)) tool_options = {self.ALPHA:2.5} self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)) tool_options = {self.ALPHA:"abcdef"} self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)) tool_options = {self.BETA:"asdf"} self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)) tool_options = {self.GAMMA:1.0} self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False)) tool_options = {self.GAMMA:""} self.assertRaises(ToolContractError, lambda: resolve_tool_contract(tc, self.INPUT_FILES, d, d, self.MAX_NPROC, tool_options, False))
def _to_rtc(self, tc, output_dir, tmp_dir): # handled the polymorphism in subclasses by overriding return resolve_tool_contract(tc, self.INPUT_FILES, output_dir, tmp_dir, self.MAX_NPROC, self.TASK_OPTIONS, self.IS_DISTRIBUTED)
def _to_rtc(self, tc, output_dir, tmp_dir): # handled the polymorphism in subclasses by overriding return resolve_tool_contract(tc, self.INPUT_FILES, output_dir, tmp_dir, self.MAX_NPROC, self.TASK_OPTIONS)