def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception(
f"({job_wrapper.get_id_tag()}) failure writing job script")
return
# Construction of Kubernetes objects follow: https://kubernetes.io/docs/concepts/workloads/controllers/job/
if self.__has_guest_ports(job_wrapper):
try:
self.__configure_port_routing(ajs)
except HTTPError:
log.exception(
"Kubernetes failed to expose tool ports as services, HTTP exception encountered"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to export tool ports as services."
self.mark_as_failed(ajs)
return
k8s_job_prefix = self.__produce_k8s_job_prefix()
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix,
self.__get_k8s_job_spec(ajs))
job = Job(self._pykube_api, k8s_job_obj)
try:
job.create()
except HTTPError:
log.exception(
"Kubernetes failed to create job, HTTP exception encountered")
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
if not job.name:
log.exception(
f"Kubernetes failed to create job, empty name encountered: [{job.obj}]"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
job_id = job.name
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = job_id
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(job_id)
self.monitor_queue.put(ajs)
def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception(
f"({job_wrapper.get_id_tag()}) failure writing job script")
return
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
k8s_job_prefix = self.__produce_k8s_job_prefix()
guest_ports = ajs.job_wrapper.guest_ports
ports_dict = {}
for guest_port in guest_ports:
ports_dict[str(guest_port)] = dict(host='manual',
port=guest_port,
protocol="https")
eps = None
if ajs.job_wrapper.guest_ports:
k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix,
ajs.job_wrapper)
log.debug(
f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}'
)
k8s_service_obj = service_object_dict(
self.runner_params, k8s_job_name,
self.__get_k8s_service_spec(ajs))
eps = self.app.interactivetool_manager.configure_entry_points(
ajs.job_wrapper.get_job(), ports_dict)
k8s_ingress_obj = ingress_object_dict(
self.runner_params, k8s_job_name,
self.__get_k8s_ingress_spec(ajs, eps))
service = Service(self._pykube_api, k8s_service_obj)
service.create()
ingress = Ingress(self._pykube_api, k8s_ingress_obj)
ingress.create()
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix,
self.__get_k8s_job_spec(ajs, eps))
job = Job(self._pykube_api, k8s_job_obj)
try:
job.create()
except HTTPError:
log.exception(
"Kubernetes failed to create job, HTTP exception encountered")
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
if not job.name:
log.exception(
f"Kubernetes failed to create job, empty name encountered: [{job.obj}]"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
job_id = job.name
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = job_id
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(job_id)
self.monitor_queue.put(ajs)
