def _create_docker_task(self, job_exe): '''Creates and returns a docker task for this job execution :param job_exe: The JobExecution that we are creating a task for :type job_exe: :class:`job.models.JobExecution` returns: The Docker Mesos task rtype: :class:`mesos_pb2.TaskInfo` ''' node_work_dir = settings.NODE_WORK_DIR input_dir = get_job_exe_input_dir(self.job_exe_id, node_work_dir) output_dir = get_job_exe_output_dir(self.job_exe_id, node_work_dir) task_name = 'Job Execution %i (%s)' % (self.job_exe_id, self._cached_job_type_name) task = self._create_base_task(task_name) task.container.type = mesos_pb2.ContainerInfo.DOCKER docker_image = job_exe.get_docker_image() command = job_exe.get_job_interface().get_command() command_arguments = job_exe.command_arguments assert docker_image is not None task.container.docker.image = docker_image # If the docker container is to run in privileged mode, # set the 'privileged' boolean attribute. if job_exe.is_docker_privileged(): task.container.docker.privileged = True # TODO: Determine whether or not there is an entry point within # the docker image in order to pass in the docker container # command arguments correctly. # Right now we assume an entry point task.command.shell = False # parse through the docker arguments and add them # to the CommandInfo 'arguments' list arguments = command_arguments.split(" ") for argument in arguments: task.command.arguments.append(argument) input_vol = task.container.docker.parameters.add() input_vol.key = "volume" input_vol.value = "%s:%s:ro" % (input_dir, input_dir) output_vol = task.container.docker.parameters.add() output_vol.key = "volume" output_vol.value = "%s:%s:rw" % (output_dir, output_dir) task.container.docker.network = mesos_pb2.ContainerInfo.DockerInfo.Network.Value('BRIDGE') logger.info("about to launch docker (assuming an entry point) with:") logger.info("arguments:%s", task.command.arguments) logger.info("input_vol:%s", input_vol.value) logger.info("output_vol:%s", output_vol.value) return task
def handle(self, **options): '''See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. ''' exe_id = options.get('job_exe_id') logger.info('Command starting: scale_pre_steps - Job Execution ID: %i', exe_id) try: node_work_dir = settings.NODE_WORK_DIR job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(exe_id) job_dir = file_system.get_job_exe_dir(exe_id, node_work_dir) input_dir = file_system.get_job_exe_input_dir(exe_id, node_work_dir) output_dir = file_system.get_job_exe_output_dir(exe_id, node_work_dir) job_dirs = [job_dir, input_dir, output_dir] for target_dir in job_dirs: self._create_job_dir(exe_id, target_dir) job_interface = job_exe.get_job_interface() job_data = job_exe.job.get_job_data() job_environment = job_exe.get_job_environment() job_interface.perform_pre_steps(job_data, job_environment, exe_id) command_args = job_interface.fully_populate_command_argument(job_data, job_environment, exe_id) # This shouldn't be necessary once we have user namespaces in docker self._chmod_job_dir(file_system.get_job_exe_output_data_dir(exe_id)) # Perform a force pull for docker jobs to get the latest version of the image before running # TODO: Remove this hack in favor of the feature in Mesos 0.22.x, see MESOS-1886 for details docker_image = job_exe.job.job_type.docker_image if docker_image: logger.info('Pulling latest docker image: %s', docker_image) try: subprocess.check_call(['sudo', 'docker', 'pull', docker_image]) except subprocess.CalledProcessError: logger.exception('Docker pull returned unexpected exit code.') except OSError: logger.exception('OS unable to run docker pull command.') logger.info('Executing job: %i -> %s', exe_id, ' '.join(command_args)) JobExecution.objects.pre_steps_command_arguments(exe_id, command_args) except Exception as e: logger.exception('Job Execution %i: Error performing pre-job steps', exe_id) exit_code = -1 if isinstance(e, DatabaseError): exit_code = DB_EXIT_CODE elif isinstance(e, NfsError): exit_code = NFS_EXIT_CODE elif isinstance(e, IOError): exit_code = IO_EXIT_CODE sys.exit(exit_code) logger.info('Command completed: scale_pre_steps')
def handle(self, **options): '''See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. ''' exe_id = options.get('job_exe_id') logger.info('Command starting: scale_post_steps - Job Execution ID: %i', exe_id) try: node_work_dir = settings.NODE_WORK_DIR output_dir = file_system.get_job_exe_output_dir(exe_id, node_work_dir) # TODO: remove when we can # This shouldn't be necessary once we have user namespaces in docker if output_dir: subprocess.call(['sudo', 'chmod', '-R', '777', output_dir]) # Get the pre-loaded job_exe for efficiency job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(exe_id) job_interface = job_exe.get_job_interface() job_data = job_exe.job.get_job_data() stdout_and_stderr = (job_exe.stdout or '') + '\n' + (job_exe.stderr or '') job_results, results_manifest = job_interface.perform_post_steps(job_exe, job_data, stdout_and_stderr) JobExecution.objects.post_steps_results(exe_id, job_results, results_manifest) if not settings.settings.SKIP_CLEANUP_JOB_DIR: self._cleanup(exe_id) except Exception as ex: logger.exception('Job Execution %i: Error performing post-job steps', exe_id) if not settings.settings.SKIP_CLEANUP_JOB_DIR: self._cleanup(exe_id) exit_code = -1 if isinstance(ex, DatabaseError): exit_code = DB_EXIT_CODE elif isinstance(ex, NfsError): exit_code = NFS_EXIT_CODE elif isinstance(ex, IOError): exit_code = IO_EXIT_CODE sys.exit(exit_code) logger.info('Command completed: scale_post_steps')
def _create_docker_task(task): """Creates and returns a Dockerized Mesos task from a Scale task :param task: The task :type task: :class:`job.execution.running.tasks.base_task.Task` returns: The Dockerized Mesos task rtype: :class:`mesos_pb2.TaskInfo` """ mesos_task = _create_base_task(task) mesos_task.container.type = mesos_pb2.ContainerInfo.DOCKER mesos_task.container.docker.image = task.docker_image # add parameters for k, v in task.docker_params: mesos_task.container.docker.parameters.add(key=k, value=v) if task.is_docker_privileged: mesos_task.container.docker.privileged = True # TODO: Determine whether or not there is an entry point within # the docker image in order to pass in the docker container # command arguments correctly. # Right now we assume an entry point mesos_task.command.shell = False # parse through the docker arguments and add them # to the CommandInfo 'arguments' list arguments = task.command_arguments.split(" ") for argument in arguments: mesos_task.command.arguments.append(argument) input_dir = get_job_exe_input_dir(task.job_exe_id) output_dir = get_job_exe_output_dir(task.job_exe_id) input_vol = mesos_task.container.docker.parameters.add() input_vol.key = "volume" input_vol.value = "%s:%s:ro" % (input_dir, input_dir) output_vol = mesos_task.container.docker.parameters.add() output_vol.key = "volume" output_vol.value = "%s:%s:rw" % (output_dir, output_dir) mesos_task.container.docker.network = mesos_pb2.ContainerInfo.DockerInfo.Network.Value( 'BRIDGE') return mesos_task
def _create_docker_task(task): """Creates and returns a Dockerized Mesos task from a Scale task :param task: The task :type task: :class:`job.execution.running.tasks.base_task.Task` returns: The Dockerized Mesos task rtype: :class:`mesos_pb2.TaskInfo` """ mesos_task = _create_base_task(task) mesos_task.container.type = mesos_pb2.ContainerInfo.DOCKER mesos_task.container.docker.image = task.docker_image # add parameters for k, v in task.docker_params: mesos_task.container.docker.parameters.add(key=k, value=v) if task.is_docker_privileged: mesos_task.container.docker.privileged = True # TODO: Determine whether or not there is an entry point within # the docker image in order to pass in the docker container # command arguments correctly. # Right now we assume an entry point mesos_task.command.shell = False # parse through the docker arguments and add them # to the CommandInfo 'arguments' list arguments = task.command_arguments.split(" ") for argument in arguments: mesos_task.command.arguments.append(argument) input_dir = get_job_exe_input_dir(task.job_exe_id) output_dir = get_job_exe_output_dir(task.job_exe_id) input_vol = mesos_task.container.docker.parameters.add() input_vol.key = "volume" input_vol.value = "%s:%s:ro" % (input_dir, input_dir) output_vol = mesos_task.container.docker.parameters.add() output_vol.key = "volume" output_vol.value = "%s:%s:rw" % (output_dir, output_dir) mesos_task.container.docker.network = mesos_pb2.ContainerInfo.DockerInfo.Network.Value('BRIDGE') return mesos_task
def handle(self, **options): '''See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. ''' exe_id = options.get('job_exe_id') logger.info('Command starting: scale_post_steps - Job Execution ID: %i', exe_id) try: output_dir = file_system.get_job_exe_output_dir(exe_id) # TODO: remove when we can # This shouldn't be necessary once we have user namespaces in docker if output_dir: subprocess.call(['sudo', 'chmod', '-R', '777', output_dir]) # Get the pre-loaded job_exe for efficiency job_exe = self._get_job_exe(exe_id) self._perform_post_steps(job_exe) if not settings.settings.SKIP_CLEANUP_JOB_DIR: self._cleanup(exe_id) except Exception as ex: logger.exception('Job Execution %i: Error performing post-job steps', exe_id) if not settings.settings.SKIP_CLEANUP_JOB_DIR: self._cleanup(exe_id) exit_code = GENERAL_FAIL_EXIT_CODE if isinstance(ex, OperationalError): exit_code = DB_OP_EXIT_CODE elif isinstance(ex, DatabaseError): exit_code = DB_EXIT_CODE elif isinstance(ex, NfsError): exit_code = NFS_EXIT_CODE elif isinstance(ex, IOError): exit_code = IO_EXIT_CODE sys.exit(exit_code) logger.info('Command completed: scale_post_steps')
def handle(self, **options): '''See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. ''' exe_id = options.get('job_exe_id') logger.info('Command starting: scale_post_steps - Job Execution ID: %i', exe_id) try: output_dir = file_system.get_job_exe_output_dir(exe_id) # TODO: remove when we can # This shouldn't be necessary once we have user namespaces in docker if output_dir: subprocess.call(['sudo', 'chmod', '-R', '777', output_dir]) # Get the pre-loaded job_exe for efficiency job_exe = self._get_job_exe(exe_id) self._perform_post_steps(job_exe) if not settings.settings.SKIP_CLEANUP_JOB_DIR: self._cleanup(exe_id) except Exception as ex: logger.exception('Job Execution %i: Error performing post-job steps', exe_id) if not settings.settings.SKIP_CLEANUP_JOB_DIR: self._cleanup(exe_id) exit_code = -1 if isinstance(ex, DatabaseError): exit_code = DB_EXIT_CODE elif isinstance(ex, NfsError): exit_code = NFS_EXIT_CODE elif isinstance(ex, IOError): exit_code = IO_EXIT_CODE sys.exit(exit_code) logger.info('Command completed: scale_post_steps')