def setUp(self): super(TestStager, self).setUp() from .test_utils import get_test_tool self.tool = get_test_tool() self.client = MockClient(self.temp_directory, self.tool) inputs = self.__setup_inputs() self.client_job_description = ClientJobDescription( tool=self.tool, command_line="run_test.exe", config_files=[], input_files=inputs, client_outputs=ClientOutputs("/galaxy/database/working_directory/1", []), working_directory="/galaxy/database/working_directory/1", dependencies_description=DependenciesDescription(requirements=[TEST_REQUIREMENT_1, TEST_REQUIREMENT_2]), env=[TEST_ENV_1], rewrite_paths=False, ) self.job_config = dict( configs_directory="/pulsar/staging/1/configs", working_directory="/pulsar/staging/1/working", outputs_directory="/pulsar/staging/1/outputs", system_properties=dict( separator="\\", ), )
def __client_outputs(self, client, job_wrapper): work_dir_outputs = self.get_work_dir_outputs(job_wrapper) output_files = self.get_output_files(job_wrapper) client_outputs = ClientOutputs( working_directory=job_wrapper.working_directory, work_dir_outputs=work_dir_outputs, output_files=output_files, version_file=job_wrapper.get_version_string_path(), ) return client_outputs
def run_and_check_submission(self): job_id = "0" galaxy_working = temp_directory_persist() output_name = "dataset_1211231231231231231.dat" galaxy_output = os.path.join(galaxy_working, output_name) pulsar_output = os.path.join(self.staging_directory, job_id, "outputs", output_name) pulsar_input = os.path.join(self.staging_directory, job_id, "inputs", "cow") with files_server("/") as test_files_server: files_endpoint = test_files_server.application_url action = {"name": "cow", "type": "input", "action": {"action_type": "message", "contents": "cow file contents\n"}} client_outputs = ClientOutputs( working_directory=galaxy_working, output_files=[os.path.join(galaxy_working, output_name)], ) launch_params = dict( command_line="cat '%s' > '%s'" % (pulsar_input, pulsar_output), job_id=job_id, setup_params=dict( job_id=job_id, ), setup=True, remote_staging={ "setup": [action], "action_mapper": { "default_action": "remote_transfer", "files_endpoint": files_endpoint, }, "client_outputs": client_outputs.to_dict(), }, ) base64 = to_base64_json(launch_params) assert not os.path.exists(galaxy_output) submit.main(["--base64", base64] + self._encode_application()) assert os.path.exists(galaxy_output) out_contents = open(galaxy_output, "r").read() assert out_contents == "cow file contents\n", out_contents
def __client_outputs(self, client, job_wrapper): work_dir_outputs = self.get_work_dir_outputs(job_wrapper) output_files = self.get_output_files(job_wrapper) if self.app.config.metadata_strategy == "legacy": # Drop this branch in 19.09. metadata_directory = job_wrapper.working_directory else: metadata_directory = os.path.join(job_wrapper.working_directory, "metadata") client_outputs = ClientOutputs( working_directory=job_wrapper.tool_working_directory, metadata_directory=metadata_directory, work_dir_outputs=work_dir_outputs, output_files=output_files, version_file=job_wrapper.get_version_string_path(), ) return client_outputs
def _run_client_for_job(args): if args.job_id is None: args.job_id = str(uuid.uuid4()) output_patterns = [] output_patterns.extend(args.output_pattern) for output in args.output: output_patterns.append(fnmatch.translate(output)) client_options = extract_client_options(args) client, client_manager = client_info(args, client_options) try: working_directory = args.working_directory client_outputs = ClientOutputs( working_directory=working_directory, dynamic_outputs=output_patterns, ) job_description = ClientJobDescription( command_line=args.command, working_directory=working_directory, client_outputs=client_outputs, ) submit_job(client, job_description) waiter = Waiter(client, client_manager) result_status = waiter.wait() pulsar_outputs = PulsarOutputs.from_status_response(result_status) if args.result_json: open(args.result_json, "w").write(json_dumps(result_status)) finish_args = dict( client=client, job_completed_normally=True, cleanup_job=args.cleanup, client_outputs=client_outputs, pulsar_outputs=pulsar_outputs, ) failed = finish_job(**finish_args) return failed finally: client_manager.shutdown()
def run(options): waiter = None try: temp_directory = tempfile.mkdtemp() temp_index_dir = os.path.join(temp_directory, "idx", "bwa") temp_index_dir_sibbling = os.path.join(temp_directory, "idx", "seq") temp_shared_dir = os.path.join(temp_directory, "shared", "test1") temp_work_dir = os.path.join(temp_directory, "w") temp_tool_dir = os.path.join(temp_directory, "t") __makedirs([temp_tool_dir, temp_work_dir, temp_index_dir, temp_index_dir_sibbling, temp_shared_dir]) temp_input_path = os.path.join(temp_directory, "dataset_0.dat") temp_input_extra_path = os.path.join(temp_directory, "dataset_0_files", "input_subdir", "extra") temp_index_path = os.path.join(temp_index_dir, "human.fa") temp_config_path = os.path.join(temp_work_dir, "config.txt") temp_tool_path = os.path.join(temp_directory, "t", "script.py") temp_output_path = os.path.join(temp_directory, "dataset_1.dat") temp_output2_path = os.path.join(temp_directory, "dataset_2.dat") temp_output3_path = os.path.join(temp_directory, "dataset_3.dat") temp_output4_path = os.path.join(temp_directory, "dataset_4.dat") temp_version_output_path = os.path.join(temp_directory, "GALAXY_VERSION_1234") temp_output_workdir_destination = os.path.join(temp_directory, "dataset_77.dat") temp_output_workdir = os.path.join(temp_work_dir, "env_test") temp_output_workdir_destination2 = os.path.join(temp_directory, "dataset_78.dat") temp_output_workdir2 = os.path.join(temp_work_dir, "rewrite_action_test") __write_to_file(temp_input_path, b"Hello world input!!@!") __write_to_file(temp_input_extra_path, b"INPUT_EXTRA_CONTENTS") __write_to_file(temp_config_path, EXPECTED_OUTPUT) __write_to_file(temp_tool_path, TEST_SCRIPT) __write_to_file(temp_index_path, b"AGTC") # Implicit files that should also get transferred since depth > 0 __write_to_file("%s.fai" % temp_index_path, b"AGTC") __write_to_file(os.path.join(temp_index_dir_sibbling, "human_full_seqs"), b"AGTC") empty_input = u"/foo/bar/x" test_unicode = getattr(options, "test_unicode", False) # TODO Switch this in integration tests cmd_text = EXAMPLE_UNICODE_TEXT if test_unicode else "Hello World" command_line_params = ( temp_tool_path, temp_config_path, temp_input_path, temp_output_path, empty_input, temp_output2_path, cmd_text, temp_output3_path, temp_input_extra_path, temp_version_output_path, temp_index_path, temp_output4_path, temp_shared_dir, ) assert os.path.exists(temp_index_path) command_line = u'python %s "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s"' % command_line_params config_files = [temp_config_path] input_files = [temp_input_path, empty_input] output_files = [ temp_output_path, temp_output2_path, temp_output3_path, temp_output4_path, temp_output_workdir_destination, temp_output_workdir_destination2 ] client, client_manager = __client(temp_directory, options) waiter = Waiter(client, client_manager) client_outputs = ClientOutputs( working_directory=temp_work_dir, work_dir_outputs=[ (temp_output_workdir, temp_output_workdir_destination), (temp_output_workdir2, temp_output_workdir_destination2), ], output_files=output_files, version_file=temp_version_output_path, ) job_description = ClientJobDescription( command_line=command_line, tool=MockTool(temp_tool_dir), config_files=config_files, input_files=input_files, client_outputs=client_outputs, working_directory=temp_work_dir, **__extra_job_description_kwargs(options) ) submit_job(client, job_description) result_status = waiter.wait() __finish(options, client, client_outputs, result_status) __assert_contents(temp_output_path, EXPECTED_OUTPUT, result_status) __assert_contents(temp_output2_path, cmd_text, result_status) __assert_contents(os.path.join(temp_work_dir, "galaxy.json"), b"GALAXY_JSON", result_status) __assert_contents(os.path.join(temp_directory, "dataset_1_files", "extra"), b"EXTRA_OUTPUT_CONTENTS", result_status) if getattr(options, "test_rewrite_action", False): __assert_contents(temp_output_workdir_destination2, os.path.join(temp_directory, "shared2", "test1"), result_status) if job_description.env: __assert_contents(temp_output_workdir_destination, b"TEST_ENV_VALUE", result_status) __assert_contents(temp_version_output_path, b"1.0.1", result_status) if job_description.dependencies_description: __assert_contents(temp_output3_path, "moo_override", result_status) else: __assert_contents(temp_output3_path, "moo_default", result_status) if client.default_file_action != "none": rewritten_index_path = open(temp_output4_path, 'r', encoding='utf-8').read() # Path written to this file will differ between Windows and Linux. assert re.search(r"123456[/\\]unstructured[/\\]\w+[/\\]bwa[/\\]human.fa", rewritten_index_path) is not None __exercise_errors(options, client, temp_output_path, temp_directory) client_manager.shutdown() except BaseException: if not options.suppress_output: traceback.print_exc() raise finally: if waiter is not None: waiter.shutdown() shutil.rmtree(temp_directory)
def run(options): waiter = None try: temp_directory = tempfile.mkdtemp() temp_index_dir = os.path.join(temp_directory, "idx", "bwa") temp_index_dir_sibbling = os.path.join(temp_directory, "idx", "seq") temp_shared_dir = os.path.join(temp_directory, "shared", "test1") temp_work_dir = os.path.join(temp_directory, "w") temp_metadata_dir = os.path.join(temp_directory, "m") temp_false_working_dir = os.path.join(temp_metadata_dir, "working") temp_tool_dir = os.path.join(temp_directory, "t") __makedirs([ temp_tool_dir, temp_work_dir, temp_index_dir, temp_index_dir_sibbling, temp_shared_dir, temp_metadata_dir, temp_false_working_dir, ]) temp_input_path = os.path.join(temp_directory, "dataset_0.dat") temp_input_extra_path = os.path.join(temp_directory, "dataset_0_files", "input_subdir", "extra") temp_input_metadata_path = os.path.join(temp_directory, "metadata", "12312231231231.dat") temp_index_path = os.path.join(temp_index_dir, "human.fa") temp_config_path = os.path.join(temp_work_dir, "config.txt") temp_tool_path = os.path.join(temp_directory, "t", "script.py") temp_output_path = os.path.join(temp_directory, "dataset_1.dat") temp_output2_path = os.path.join(temp_directory, "dataset_2.dat") temp_output3_path = os.path.join(temp_directory, "dataset_3.dat") temp_output4_path = os.path.join(temp_directory, "dataset_4.dat") temp_version_output_path = os.path.join(temp_directory, "GALAXY_VERSION_1234") temp_output_workdir_destination = os.path.join(temp_directory, "dataset_77.dat") temp_output_workdir = os.path.join(temp_work_dir, "env_test") temp_metadata_path = os.path.join(temp_metadata_dir, "metadata_test123") temp_output_workdir_destination2 = os.path.join( temp_directory, "dataset_78.dat") temp_output_workdir2 = os.path.join(temp_work_dir, "rewrite_action_test") __write_to_file(temp_input_path, b"Hello world input!!@!") __write_to_file(temp_input_extra_path, b"INPUT_EXTRA_CONTENTS") __write_to_file(temp_input_metadata_path, b"INPUT METADATA CONTENTS...") __write_to_file(temp_config_path, EXPECTED_OUTPUT) __write_to_file(temp_metadata_path, "meta input") __write_to_file(temp_tool_path, TEST_SCRIPT) __write_to_file(temp_index_path, b"AGTC") # Implicit files that should also get transferred since depth > 0 __write_to_file("%s.fai" % temp_index_path, b"AGTC") __write_to_file( os.path.join(temp_index_dir_sibbling, "human_full_seqs"), b"AGTC") empty_input = u"/foo/bar/x" test_unicode = getattr(options, "test_unicode", False) # TODO Switch this in integration tests cmd_text = EXAMPLE_UNICODE_TEXT if test_unicode else "Hello World" command_line_params = ( temp_tool_path, temp_config_path, temp_input_path, temp_output_path, empty_input, temp_output2_path, cmd_text, temp_output3_path, temp_input_extra_path, temp_version_output_path, temp_index_path, temp_output4_path, temp_shared_dir, temp_metadata_path, temp_input_metadata_path, ) assert os.path.exists(temp_index_path) command_line = u'python %s "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s"' % command_line_params config_files = [temp_config_path] client_inputs = [] client_inputs.append( ClientInput(temp_input_path, CLIENT_INPUT_PATH_TYPES.INPUT_PATH)) client_inputs.append( ClientInput(temp_input_path, CLIENT_INPUT_PATH_TYPES.INPUT_PATH)) # Reverting empty input handling added in: # https://github.com/galaxyproject/pulsar/commit/2fb36ba979cf047a595c53cdef833cae79cbb380 # Seems like it really should cause a failure. # client_inputs.append(ClientInput(empty_input, CLIENT_INPUT_PATH_TYPES.INPUT_PATH)) client_inputs.append( ClientInput(os.path.join(temp_directory, "dataset_0_files"), CLIENT_INPUT_PATH_TYPES.INPUT_EXTRA_FILES_PATH)) client_inputs.append( ClientInput(temp_input_metadata_path, CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH)) output_files = [ temp_output_path, temp_output2_path, temp_output3_path, temp_output4_path, temp_output_workdir_destination, temp_output_workdir_destination2 ] client, client_manager = __client(temp_directory, options) waiter = Waiter(client, client_manager) client_outputs = ClientOutputs( working_directory=temp_work_dir, metadata_directory=temp_metadata_dir, work_dir_outputs=[ (temp_output_workdir, temp_output_workdir_destination), (temp_output_workdir2, temp_output_workdir_destination2), ], output_files=output_files, version_file=temp_version_output_path, ) job_description = ClientJobDescription( command_line=command_line, tool=MockTool(temp_tool_dir), config_files=config_files, client_inputs=ClientInputs(client_inputs), client_outputs=client_outputs, working_directory=temp_work_dir, metadata_directory=temp_metadata_dir, **__extra_job_description_kwargs(options)) submit_job(client, job_description) result_status = waiter.wait() stdout = result_status["stdout"].strip() assert "stdout output".startswith(stdout) if hasattr(options, "maximum_stream_size"): assert len(stdout) == options.maximum_stream_size __finish(options, client, client_outputs, result_status) __assert_contents(temp_output_path, EXPECTED_OUTPUT, result_status) __assert_contents(temp_output2_path, cmd_text, result_status) __assert_contents(os.path.join(temp_work_dir, "galaxy.json"), b"GALAXY_JSON", result_status) __assert_contents( os.path.join(temp_directory, "dataset_1_files", "extra"), b"EXTRA_OUTPUT_CONTENTS", result_status) __assert_contents(os.path.join(temp_metadata_dir, "metadata_output"), b"meta output", result_status) if getattr(options, "test_rewrite_action", False): __assert_contents(temp_output_workdir_destination2, os.path.join(temp_directory, "shared2", "test1"), result_status) if job_description.env: __assert_contents(temp_output_workdir_destination, b"TEST_ENV_VALUE", result_status) __assert_contents(temp_version_output_path, b"1.0.1", result_status) if job_description.dependencies_description: __assert_contents(temp_output3_path, "moo_override", result_status) else: __assert_contents(temp_output3_path, "moo_default", result_status) __assert_has_rewritten_bwa_path(client, temp_output4_path) __exercise_errors(options, client, temp_output_path, temp_directory) client_manager.shutdown() except BaseException: if not options.suppress_output: traceback.print_exc() raise finally: if waiter is not None: waiter.shutdown() shutil.rmtree(temp_directory)