def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug("Starting queue_job for job " + job_wrapper.get_id_tag())
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception("(%s) failure writing job script" %
job_wrapper.get_id_tag())
return
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
k8s_job_prefix = self.__produce_k8s_job_prefix()
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix,
self.__get_k8s_job_spec(ajs))
job = Job(self._pykube_api, k8s_job_obj)
job.create()
job_id = job.metadata['name']
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = job_id
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(job_id)
self.monitor_queue.put(ajs)
def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug("Starting queue_job for job " + job_wrapper.get_id_tag())
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception("(%s) failure writing job script" %
job_wrapper.get_id_tag())
return
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
k8s_job_name = self.__produce_unique_k8s_job_name(
job_wrapper.get_id_tag())
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_name,
self.__get_k8s_job_spec(ajs))
# Checks if job exists and is trusted, or if it needs re-creation.
job = Job(self._pykube_api, k8s_job_obj)
if job.exists() and not self._galaxy_instance_id:
# if galaxy instance id is not set, then we don't trust matching jobs and we simply delete and
# re-create the job
log.debug(
"Matching job exists, but Job is not trusted, so it will be deleted and a new one created."
)
job.delete()
elapsed_seconds = 0
while job.exists():
sleep(3)
elapsed_seconds += 3
if elapsed_seconds > self.runner_params[
'k8s_timeout_seconds_job_deletion']:
log.debug(
"Timed out before k8s could delete existing untrusted job "
+ k8s_job_name +
", not queuing associated Galaxy job.")
return
log.debug("Waiting for job to be deleted " + k8s_job_name)
Job(self._pykube_api, k8s_job_obj).create()
elif job.exists() and self._galaxy_instance_id:
# The job exists and we trust the identifier.
log.debug(
"Matching job exists, but Job is trusted, so we simply use the existing one for "
+ k8s_job_name)
# We simply leave the k8s job to be handled later on by the check watched-items.
else:
# Creates the Kubernetes Job if it doesn't exist.
job.create()
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = k8s_job_name
# store runner information for tracking if Galaxy restarts
job_wrapper.set_job_destination(job_wrapper.job_destination,
k8s_job_name)
self.monitor_queue.put(ajs)
def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception(
f"({job_wrapper.get_id_tag()}) failure writing job script")
return
# Construction of Kubernetes objects follow: https://kubernetes.io/docs/concepts/workloads/controllers/job/
if self.__has_guest_ports(job_wrapper):
try:
self.__configure_port_routing(ajs)
except HTTPError:
log.exception(
"Kubernetes failed to expose tool ports as services, HTTP exception encountered"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to export tool ports as services."
self.mark_as_failed(ajs)
return
k8s_job_prefix = self.__produce_k8s_job_prefix()
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix,
self.__get_k8s_job_spec(ajs))
job = Job(self._pykube_api, k8s_job_obj)
try:
job.create()
except HTTPError:
log.exception(
"Kubernetes failed to create job, HTTP exception encountered")
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
if not job.name:
log.exception(
f"Kubernetes failed to create job, empty name encountered: [{job.obj}]"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
job_id = job.name
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = job_id
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(job_id)
self.monitor_queue.put(ajs)
def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception(
f"({job_wrapper.get_id_tag()}) failure writing job script")
return
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
k8s_job_prefix = self.__produce_k8s_job_prefix()
guest_ports = ajs.job_wrapper.guest_ports
ports_dict = {}
for guest_port in guest_ports:
ports_dict[str(guest_port)] = dict(host='manual',
port=guest_port,
protocol="https")
eps = None
if ajs.job_wrapper.guest_ports:
k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix,
ajs.job_wrapper)
log.debug(
f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}'
)
k8s_service_obj = service_object_dict(
self.runner_params, k8s_job_name,
self.__get_k8s_service_spec(ajs))
eps = self.app.interactivetool_manager.configure_entry_points(
ajs.job_wrapper.get_job(), ports_dict)
k8s_ingress_obj = ingress_object_dict(
self.runner_params, k8s_job_name,
self.__get_k8s_ingress_spec(ajs, eps))
service = Service(self._pykube_api, k8s_service_obj)
service.create()
ingress = Ingress(self._pykube_api, k8s_ingress_obj)
ingress.create()
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix,
self.__get_k8s_job_spec(ajs, eps))
job = Job(self._pykube_api, k8s_job_obj)
try:
job.create()
except HTTPError:
log.exception(
"Kubernetes failed to create job, HTTP exception encountered")
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
if not job.name:
log.exception(
f"Kubernetes failed to create job, empty name encountered: [{job.obj}]"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
job_id = job.name
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = job_id
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(job_id)
self.monitor_queue.put(ajs)