def test(self): # TODO: test unstaging, would actually require files server and some # sort MQ listening. with files_server("/"): # as test_files_server: config_directory = os.path.join(self.temp_directory, "config") staging_directory = os.path.join(self.temp_directory, "staging") os.makedirs(config_directory) app_conf = dict( staging_directory=staging_directory, message_queue_url="memory://submittest" ) app_conf_path = os.path.join(config_directory, "app.yml") with open(app_conf_path, "w") as f: f.write(yaml.dump(app_conf)) job_id = "43" output_path = os.path.join(staging_directory, job_id, "out") launch_params = dict( command_line="echo 'moo' > '%s'" % output_path, job_id=job_id, setup_params=dict( job_id=job_id, ) ) base64 = to_base64_json(launch_params) submit.main(["--base64", base64, "--app_conf_path", app_conf_path]) out_contents = open(output_path, "r").read() assert out_contents == "moo\n", out_contents
def _populate_task_data_for_job(self, task, job): if "env" not in job: job["env"] = [] job["env"].extend(self._mesos_env_vars()) # In case job itself wants to utilize Mesos # populate environment variables. task_data = dict(job=job, manager=self.manager_options) task.data = to_base64_json(task_data)
def _populate_task_data_for_job(self, task, job): if "env" not in job: job["env"] = [] job["env"].extend(self._mesos_env_vars()) # In case job itself wants to utilize Mesos # populate environment variables. task_data = dict( job=job, manager=self.manager_options ) task.data = to_base64_json( task_data )
def run_and_check_submission(self): job_id = "0" galaxy_working = temp_directory_persist() output_name = "dataset_1211231231231231231.dat" galaxy_output = os.path.join(galaxy_working, output_name) pulsar_output = os.path.join(self.staging_directory, job_id, "outputs", output_name) pulsar_input = os.path.join(self.staging_directory, job_id, "inputs", "cow") with files_server("/") as test_files_server: files_endpoint = test_files_server.application_url action = {"name": "cow", "type": "input", "action": {"action_type": "message", "contents": "cow file contents\n"}} client_outputs = ClientOutputs( working_directory=galaxy_working, output_files=[os.path.join(galaxy_working, output_name)], ) launch_params = dict( command_line="cat '%s' > '%s'" % (pulsar_input, pulsar_output), job_id=job_id, setup_params=dict( job_id=job_id, ), setup=True, remote_staging={ "setup": [action], "action_mapper": { "default_action": "remote_transfer", "files_endpoint": files_endpoint, }, "client_outputs": client_outputs.to_dict(), }, ) base64 = to_base64_json(launch_params) assert not os.path.exists(galaxy_output) submit.main(["--base64", base64] + self._encode_application()) assert os.path.exists(galaxy_output) out_contents = open(galaxy_output, "r").read() assert out_contents == "cow file contents\n", out_contents
def _encode_application(self): app_conf = dict( staging_directory=self.staging_directory, message_queue_url="memory://submittest" ) return ["--app_conf_base64", to_base64_json(app_conf)]