def test(self):
# TODO: test unstaging, would actually require files server and some
# sort MQ listening.
with files_server("/"): # as test_files_server:
config_directory = os.path.join(self.temp_directory, "config")
staging_directory = os.path.join(self.temp_directory, "staging")
os.makedirs(config_directory)
app_conf = dict(
staging_directory=staging_directory,
message_queue_url="memory://submittest"
)
app_conf_path = os.path.join(config_directory, "app.yml")
with open(app_conf_path, "w") as f:
f.write(yaml.dump(app_conf))
job_id = "43"
output_path = os.path.join(staging_directory, job_id, "out")
launch_params = dict(
command_line="echo 'moo' > '%s'" % output_path,
job_id=job_id,
setup_params=dict(
job_id=job_id,
)
)
base64 = to_base64_json(launch_params)
submit.main(["--base64", base64, "--app_conf_path", app_conf_path])
out_contents = open(output_path, "r").read()
assert out_contents == "moo\n", out_contents
def _populate_task_data_for_job(self, task, job):
if "env" not in job:
job["env"] = []
job["env"].extend(self._mesos_env_vars())
# In case job itself wants to utilize Mesos
# populate environment variables.
task_data = dict(job=job, manager=self.manager_options)
task.data = to_base64_json(task_data)
def _populate_task_data_for_job(self, task, job):
if "env" not in job:
job["env"] = []
job["env"].extend(self._mesos_env_vars())
# In case job itself wants to utilize Mesos
# populate environment variables.
task_data = dict(
job=job,
manager=self.manager_options
)
task.data = to_base64_json(
task_data
)
def run_and_check_submission(self):
job_id = "0"
galaxy_working = temp_directory_persist()
output_name = "dataset_1211231231231231231.dat"
galaxy_output = os.path.join(galaxy_working, output_name)
pulsar_output = os.path.join(self.staging_directory, job_id, "outputs", output_name)
pulsar_input = os.path.join(self.staging_directory, job_id, "inputs", "cow")
with files_server("/") as test_files_server:
files_endpoint = test_files_server.application_url
action = {"name": "cow", "type": "input", "action": {"action_type": "message", "contents": "cow file contents\n"}}
client_outputs = ClientOutputs(
working_directory=galaxy_working,
output_files=[os.path.join(galaxy_working, output_name)],
)
launch_params = dict(
command_line="cat '%s' > '%s'" % (pulsar_input, pulsar_output),
job_id=job_id,
setup_params=dict(
job_id=job_id,
),
setup=True,
remote_staging={
"setup": [action],
"action_mapper": {
"default_action": "remote_transfer",
"files_endpoint": files_endpoint,
},
"client_outputs": client_outputs.to_dict(),
},
)
base64 = to_base64_json(launch_params)
assert not os.path.exists(galaxy_output)
submit.main(["--base64", base64] + self._encode_application())
assert os.path.exists(galaxy_output)
out_contents = open(galaxy_output, "r").read()
assert out_contents == "cow file contents\n", out_contents
def _encode_application(self):
app_conf = dict(
staging_directory=self.staging_directory,
message_queue_url="memory://submittest"
)
return ["--app_conf_base64", to_base64_json(app_conf)]