def __command(self, **extra_kwds): kwds = dict(runner=self.runner, job_wrapper=self.job_wrapper, include_metadata=self.include_metadata, include_work_dir_outputs=self.include_work_dir_outputs, **extra_kwds) return build_command(**kwds)
def __prepare_job(self, job_wrapper, job_destination): """ Build command-line and Pulsar client for this job. """ command_line = None client = None remote_job_config = None compute_environment = None try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool remote_job_config = client.setup(tool.id, tool.version) rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client) prepare_kwds = {} if rewrite_parameters: compute_environment = PulsarComputeEnvironment( client, job_wrapper, remote_job_config) prepare_kwds['compute_environment'] = compute_environment job_wrapper.prepare(**prepare_kwds) self.__prepare_input_files_locally(job_wrapper) remote_metadata = PulsarJobRunner.__remote_metadata(client) dependency_resolution = PulsarJobRunner.__dependency_resolution( client) metadata_kwds = self.__build_metadata_configuration( client, job_wrapper, remote_metadata, remote_job_config) remote_command_params = dict( working_directory=remote_job_config['working_directory'], metadata_kwds=metadata_kwds, dependency_resolution=dependency_resolution, ) remote_working_directory = remote_job_config['working_directory'] # TODO: Following defs work for Pulsar, always worked for Pulsar but should be # calculated at some other level. remote_job_directory = os.path.abspath( os.path.join(remote_working_directory, os.path.pardir)) remote_tool_directory = os.path.abspath( os.path.join(remote_job_directory, "tool_files")) container = self._find_container( job_wrapper, compute_working_directory=remote_working_directory, compute_tool_directory=remote_tool_directory, compute_job_directory=remote_job_directory, ) job_wrapper.disable_commands_in_new_shell() command_line = build_command( self, job_wrapper=job_wrapper, container=container, include_metadata=remote_metadata, include_work_dir_outputs=False, remote_command_params=remote_command_params, ) except Exception: job_wrapper.fail("failure preparing job", exception=True) log.exception("failure running job %d" % job_wrapper.job_id) # If we were able to get a command line, run the job if not command_line: job_wrapper.finish('', '') return command_line, client, remote_job_config, compute_environment
def build_command_line(self, job_wrapper, include_metadata=False, include_work_dir_outputs=True): container = self._find_container(job_wrapper) return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, container=container, )
def __command(self, **extra_kwds): kwds = dict( runner=self.runner, job_wrapper=self.job_wrapper, include_metadata=self.include_metadata, include_work_dir_outputs=self.include_work_dir_outputs, **extra_kwds ) return build_command(**kwds)
def build_command_line(self, job_wrapper, include_metadata=False, include_work_dir_outputs=True): container = self._find_container(job_wrapper) return build_command(self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, container=container)
def __prepare_job(self, job_wrapper, job_destination): """ Build command-line and Pulsar client for this job. """ command_line = None client = None remote_job_config = None compute_environment = None try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool remote_job_config = client.setup(tool.id, tool.version) rewrite_parameters = PulsarJobRunner.__rewrite_parameters( client ) prepare_kwds = {} if rewrite_parameters: compute_environment = PulsarComputeEnvironment( client, job_wrapper, remote_job_config ) prepare_kwds[ 'compute_environment' ] = compute_environment job_wrapper.prepare( **prepare_kwds ) self.__prepare_input_files_locally(job_wrapper) remote_metadata = PulsarJobRunner.__remote_metadata( client ) dependency_resolution = PulsarJobRunner.__dependency_resolution( client ) metadata_kwds = self.__build_metadata_configuration(client, job_wrapper, remote_metadata, remote_job_config) remote_command_params = dict( working_directory=remote_job_config['working_directory'], metadata_kwds=metadata_kwds, dependency_resolution=dependency_resolution, ) remote_working_directory = remote_job_config['working_directory'] # TODO: Following defs work for Pulsar, always worked for Pulsar but should be # calculated at some other level. remote_job_directory = os.path.abspath(os.path.join(remote_working_directory, os.path.pardir)) remote_tool_directory = os.path.abspath(os.path.join(remote_job_directory, "tool_files")) container = self._find_container( job_wrapper, compute_working_directory=remote_working_directory, compute_tool_directory=remote_tool_directory, compute_job_directory=remote_job_directory, ) job_wrapper.disable_commands_in_new_shell() command_line = build_command( self, job_wrapper=job_wrapper, container=container, include_metadata=remote_metadata, include_work_dir_outputs=False, remote_command_params=remote_command_params, ) except Exception: job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) # If we were able to get a command line, run the job if not command_line: job_wrapper.finish( '', '' ) return command_line, client, remote_job_config, compute_environment
def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): # TODO: Eliminate extra kwds no longer used (since LWR skips # abstraction and calls build_command directly). container = self._find_container( job_wrapper ) return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, container=container )
def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=True ): container = self._find_container( job_wrapper ) return build_command( self, job_wrapper, this_app=self.app, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, modify_command_for_container=modify_command_for_container, container=container )
def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=True ): container = self._find_container( job_wrapper ) if not container and job_wrapper.requires_containerization: raise Exception("Failed to find a container when required, contact Galaxy admin.") return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, modify_command_for_container=modify_command_for_container, container=container )
def build_command_line(self, job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=True): container = self._find_container(job_wrapper) if not container and job_wrapper.requires_containerization: raise Exception("Failed to find a container when required, contact Galaxy admin.") return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, modify_command_for_container=modify_command_for_container, container=container )
def build_command_line(self, job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=True): container = self._find_container(job_wrapper) return build_command( self, job_wrapper, this_app=self.app, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs, modify_command_for_container=modify_command_for_container, container=container)
def __prepare_job(self, job_wrapper, job_destination): """ Build command-line and LWR client for this job. """ command_line = None client = None remote_job_config = None try: job_wrapper.prepare() self.__prepare_input_files_locally(job_wrapper) client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool remote_job_config = client.setup(tool.id, tool.version) remote_metadata = LwrJobRunner.__remote_metadata( client ) remote_work_dir_copy = LwrJobRunner.__remote_work_dir_copy( client ) dependency_resolution = LwrJobRunner.__dependency_resolution( client ) metadata_kwds = self.__build_metadata_configuration(client, job_wrapper, remote_metadata, remote_job_config) remote_command_params = dict( working_directory=remote_job_config['working_directory'], metadata_kwds=metadata_kwds, dependency_resolution=dependency_resolution, ) command_line = build_command( self, job_wrapper=job_wrapper, include_metadata=remote_metadata, include_work_dir_outputs=remote_work_dir_copy, remote_command_params=remote_command_params, ) except Exception: job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) # If we were able to get a command line, run the job if not command_line: job_wrapper.finish( '', '' ) return command_line, client, remote_job_config
def __prepare_job(self, job_wrapper, job_destination): """Build command-line and Pulsar client for this job.""" command_line = None client = None remote_job_config = None compute_environment = None fail_or_resubmit = False try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool remote_job_config = client.setup( tool.id, tool.version, tool.requires_galaxy_python_environment) needed_features = self.__needed_features(client) PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features) rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client) prepare_kwds = {} if rewrite_parameters: compute_environment = PulsarComputeEnvironment( client, job_wrapper, remote_job_config) prepare_kwds['compute_environment'] = compute_environment job_wrapper.prepare(**prepare_kwds) self.__prepare_input_files_locally(job_wrapper) remote_metadata = PulsarJobRunner.__remote_metadata(client) dependency_resolution = PulsarJobRunner.__dependency_resolution( client) metadata_kwds = self.__build_metadata_configuration( client, job_wrapper, remote_metadata, remote_job_config) remote_working_directory = remote_job_config['working_directory'] remote_job_directory = os.path.abspath( os.path.join(remote_working_directory, os.path.pardir)) remote_tool_directory = os.path.abspath( os.path.join(remote_job_directory, "tool_files")) # This should be remote_job_directory ideally, this patch using configs is a workaround for # older Pulsar versions that didn't support writing stuff to the job directory natively. script_directory = os.path.join(remote_job_directory, "configs") remote_command_params = dict( working_directory=remote_job_config['metadata_directory'], script_directory=script_directory, metadata_kwds=metadata_kwds, dependency_resolution=dependency_resolution, ) # TODO: Following directories work for Pulsar, always worked for Pulsar - but should be # calculated at some other level. container = self._find_container( job_wrapper, compute_working_directory=remote_working_directory, compute_tool_directory=remote_tool_directory, compute_job_directory=remote_job_directory, ) job_wrapper.disable_commands_in_new_shell() # Pulsar handles ``create_tool_working_directory`` and # ``include_work_dir_outputs`` details. command_line = build_command( self, job_wrapper=job_wrapper, container=container, include_metadata=remote_metadata, create_tool_working_directory=False, include_work_dir_outputs=False, remote_command_params=remote_command_params, remote_job_directory=remote_job_directory, ) except UnsupportedPulsarException: log.exception("failure running job %d, unsupported Pulsar target", job_wrapper.job_id) fail_or_resubmit = True except PulsarClientTransportError: log.exception("failure running job %d, Pulsar connection failed", job_wrapper.job_id) fail_or_resubmit = True except Exception: log.exception("failure running job %d", job_wrapper.job_id) fail_or_resubmit = True # If we were unable to get a command line, there was problem fail_or_resubmit = fail_or_resubmit or not command_line if fail_or_resubmit: job_state = self._job_state(job_wrapper.get_job(), job_wrapper) self.work_queue.put((self.fail_job, job_state)) return command_line, client, remote_job_config, compute_environment
def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs )