def __configure_port_routing(self, ajs):
# Configure interactive tool entry points first
guest_ports = ajs.job_wrapper.guest_ports
ports_dict = {}
for guest_port in guest_ports:
ports_dict[str(guest_port)] = dict(host='manual',
port=guest_port,
protocol="https")
self.app.interactivetool_manager.configure_entry_points(
ajs.job_wrapper.get_job(), ports_dict)
# Configure additional k8s service and ingress for tools with guest ports
k8s_job_prefix = self.__produce_k8s_job_prefix()
k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix, ajs.job_wrapper)
log.debug(
f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}'
)
k8s_service_obj = service_object_dict(self.runner_params, k8s_job_name,
self.__get_k8s_service_spec(ajs))
k8s_ingress_obj = ingress_object_dict(self.runner_params, k8s_job_name,
self.__get_k8s_ingress_spec(ajs))
service = Service(self._pykube_api, k8s_service_obj)
service.create()
ingress = Ingress(self._pykube_api, k8s_ingress_obj)
ingress.create()
def __cleanup_k8s_guest_ports(self, job_wrapper, k8s_job):
k8s_job_prefix = self.__produce_k8s_job_prefix()
k8s_job_name = f"{k8s_job_prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}"
log.debug(f'Deleting service/ingress for job with ID {job_wrapper.get_id_tag()}')
job_failed = (k8s_job.obj['status']['failed'] > 0
if 'failed' in k8s_job.obj['status'] else False)
ingress_to_delete = find_ingress_object_by_name(self._pykube_api, k8s_job_name, self.runner_params['k8s_namespace'])
if ingress_to_delete and len(ingress_to_delete.response['items']) > 0:
k8s_ingress = Ingress(self._pykube_api, ingress_to_delete.response['items'][0])
self.__cleanup_k8s_ingress(k8s_ingress, job_failed)
service_to_delete = find_service_object_by_name(self._pykube_api, k8s_job_name, self.runner_params['k8s_namespace'])
if service_to_delete and len(service_to_delete.response['items']) > 0:
k8s_service = Service(self._pykube_api, service_to_delete.response['items'][0])
self.__cleanup_k8s_service(k8s_service, job_failed)
def queue_job(self, job_wrapper):
"""Create job script and submit it to Kubernetes cluster"""
# prepare the job
# We currently don't need to include_metadata or include_work_dir_outputs, as working directory is the same
# where galaxy will expect results.
log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}")
ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory,
job_wrapper=job_wrapper,
job_destination=job_wrapper.job_destination)
if not self.prepare_job(job_wrapper,
include_metadata=False,
modify_command_for_container=False,
stdout_file=ajs.output_file,
stderr_file=ajs.error_file):
return
script = self.get_job_file(job_wrapper,
exit_code_path=ajs.exit_code_file,
shell=job_wrapper.shell,
galaxy_virtual_env=None)
try:
self.write_executable_script(ajs.job_file, script)
except Exception:
job_wrapper.fail("failure preparing job script", exception=True)
log.exception(
f"({job_wrapper.get_id_tag()}) failure writing job script")
return
# Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
k8s_job_prefix = self.__produce_k8s_job_prefix()
guest_ports = ajs.job_wrapper.guest_ports
ports_dict = {}
for guest_port in guest_ports:
ports_dict[str(guest_port)] = dict(host='manual',
port=guest_port,
protocol="https")
eps = None
if ajs.job_wrapper.guest_ports:
k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix,
ajs.job_wrapper)
log.debug(
f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}'
)
k8s_service_obj = service_object_dict(
self.runner_params, k8s_job_name,
self.__get_k8s_service_spec(ajs))
eps = self.app.interactivetool_manager.configure_entry_points(
ajs.job_wrapper.get_job(), ports_dict)
k8s_ingress_obj = ingress_object_dict(
self.runner_params, k8s_job_name,
self.__get_k8s_ingress_spec(ajs, eps))
service = Service(self._pykube_api, k8s_service_obj)
service.create()
ingress = Ingress(self._pykube_api, k8s_ingress_obj)
ingress.create()
k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix,
self.__get_k8s_job_spec(ajs, eps))
job = Job(self._pykube_api, k8s_job_obj)
try:
job.create()
except HTTPError:
log.exception(
"Kubernetes failed to create job, HTTP exception encountered")
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
if not job.name:
log.exception(
f"Kubernetes failed to create job, empty name encountered: [{job.obj}]"
)
ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
ajs.fail_message = "Kubernetes failed to create job."
self.mark_as_failed(ajs)
return
job_id = job.name
# define job attributes in the AsyncronousJobState for follow-up
ajs.job_id = job_id
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(job_id)
self.monitor_queue.put(ajs)