def _write_wms_id_list(self, gc_id_jobnum_list): try: job_fd, job_fn = tempfile.mkstemp('.jobids') safe_write(os.fdopen(job_fd, 'w'), str.join('\n', self._iter_wms_ids(gc_id_jobnum_list))) except Exception: raise BackendError('Could not write wms ids to %s.' % job_fn) return job_fn
def _prepareSubmit(self, task, jobnum_list, queryArguments): localJdlFilePath = os.path.join(self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' % (self.parentPool.wms_name,md5_hex(self.getURI()))) readyJobNumList = self._stageSubmitFiles(task, jobnum_list) safe_write(open(localJdlFilePath, 'w'), lmap(lambda line: line + '\n', self._getJDLData(task, readyJobNumList, queryArguments))) raise NotImplementedError('JDL must get moved to remote') return jdlFilePath
def _write_job_config(self, job_config_fn, jobnum, task, extras): try: job_env_dict = dict_union(task.get_job_dict(jobnum), extras) job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip() content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n') safe_write(open(job_config_fn, 'w'), content) except Exception: raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _prepareSubmit(self, task, jobnum_list, queryArguments): jdlFilePath = os.path.join( self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' % (self.parentPool.wms_name, md5_hex(self.getURI()))) safe_write( open(jdlFilePath, 'w'), lmap(lambda line: line + '\n', self._getJDLData(task, jobnum_list, queryArguments))) return jdlFilePath
def _write_jdl(self, jobnum_list, task): # construct a temporary JDL for this batch of jobs jdl_fd, jdl_fn = tempfile.mkstemp(suffix='.jdl') try: data = self._get_jdl_str_list(jobnum_list, task) safe_write(os.fdopen(jdl_fd, 'w'), data) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) return jdl_fn
def _prepareSubmit(self, task, jobnum_list, queryArguments): localJdlFilePath = os.path.join( self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' % (self.parentPool.wms_name, md5_hex(self.getURI()))) readyJobNumList = self._stageSubmitFiles(task, jobnum_list) safe_write( open(localJdlFilePath, 'w'), lmap(lambda line: line + '\n', self._getJDLData(task, readyJobNumList, queryArguments))) raise NotImplementedError('JDL must get moved to remote') return jdlFilePath
def _submit_job(self, jobnum, task): # Submit job and yield (jobnum, WMS ID, other data) jdl_fd, jdl_fn = tempfile.mkstemp('.jdl') try: jdl_line_list = self._make_jdl(jobnum, task) safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) try: submit_arg_list = [] for key_value in filter_dict(self._submit_args_dict, value_filter=identity).items(): submit_arg_list.extend(key_value) submit_arg_list.append(jdl_fn) activity = Activity('submitting job %d' % jobnum) proc = LocalProcess(self._submit_exec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submit_arg_list) wms_id = None stripped_stdout_iter = imap(str.strip, proc.stdout.iter(timeout=60)) for line in ifilter(lambda x: x.startswith('http'), stripped_stdout_iter): wms_id = line exit_code = proc.status(timeout=0, terminate=True) activity.finish() if (exit_code != 0) or (wms_id is None): if self._explain_error(proc, exit_code): pass else: self._log.log_process( proc, files={'jdl': SafeFile(jdl_fn).read()}) finally: remove_files([jdl_fn]) job_data = {'jdl': str.join('', jdl_line_list)} return (jobnum, self._create_gc_id(wms_id), job_data)
def _submit_job(self, jobnum, task): # Submit job and yield (jobnum, WMS ID, other data) jdl_fd, jdl_fn = tempfile.mkstemp('.jdl') try: jdl_line_list = self._make_jdl(jobnum, task) safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) try: submit_arg_list = [] for key_value in filter_dict(self._submit_args_dict, value_filter=identity).items(): submit_arg_list.extend(key_value) submit_arg_list.append(jdl_fn) activity = Activity('submitting job %d' % jobnum) proc = LocalProcess(self._submit_exec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submit_arg_list) wms_id = None stripped_stdout_iter = imap(str.strip, proc.stdout.iter(timeout=60)) for line in ifilter(lambda x: x.startswith('http'), stripped_stdout_iter): wms_id = line exit_code = proc.status(timeout=0, terminate=True) activity.finish() if (exit_code != 0) or (wms_id is None): if self._explain_error(proc, exit_code): pass else: self._log.log_process(proc, files={'jdl': SafeFile(jdl_fn).read()}) finally: remove_files([jdl_fn]) job_data = {'jdl': str.join('', jdl_line_list)} return (jobnum, self._create_gc_id(wms_id), job_data)
def _prepareSubmit(self, task, jobnum_list, queryArguments): jdlFilePath = os.path.join(self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' % (self.parentPool.wms_name,md5_hex(self.getURI()))) safe_write(open(jdlFilePath, 'w'), lmap(lambda line: line + '\n', self._getJDLData(task, jobnum_list, queryArguments))) return jdlFilePath