def _write_wms_id_list(self, gc_id_jobnum_list):
try:
job_fd, job_fn = tempfile.mkstemp('.jobids')
safe_write(os.fdopen(job_fd, 'w'), str.join('\n', self._iter_wms_ids(gc_id_jobnum_list)))
except Exception:
raise BackendError('Could not write wms ids to %s.' % job_fn)
return job_fn
def _prepareSubmit(self, task, jobnum_list, queryArguments):
localJdlFilePath = os.path.join(self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' % (self.parentPool.wms_name,md5_hex(self.getURI())))
readyJobNumList = self._stageSubmitFiles(task, jobnum_list)
safe_write(open(localJdlFilePath, 'w'),
lmap(lambda line: line + '\n', self._getJDLData(task, readyJobNumList, queryArguments)))
raise NotImplementedError('JDL must get moved to remote')
return jdlFilePath
def _write_wms_id_list(self, gc_id_jobnum_list):
try:
job_fd, job_fn = tempfile.mkstemp('.jobids')
safe_write(os.fdopen(job_fd, 'w'),
str.join('\n', self._iter_wms_ids(gc_id_jobnum_list)))
except Exception:
raise BackendError('Could not write wms ids to %s.' % job_fn)
return job_fn
def _write_job_config(self, job_config_fn, jobnum, task, extras):
try:
job_env_dict = dict_union(task.get_job_dict(jobnum), extras)
job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip()
content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n')
safe_write(open(job_config_fn, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _write_job_config(self, job_config_fn, jobnum, task, extras):
try:
job_env_dict = dict_union(task.get_job_dict(jobnum), extras)
job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip()
content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n')
safe_write(open(job_config_fn, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _prepareSubmit(self, task, jobnum_list, queryArguments):
jdlFilePath = os.path.join(
self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' %
(self.parentPool.wms_name, md5_hex(self.getURI())))
safe_write(
open(jdlFilePath, 'w'),
lmap(lambda line: line + '\n',
self._getJDLData(task, jobnum_list, queryArguments)))
return jdlFilePath
def _write_jdl(self, jobnum_list, task):
# construct a temporary JDL for this batch of jobs
jdl_fd, jdl_fn = tempfile.mkstemp(suffix='.jdl')
try:
data = self._get_jdl_str_list(jobnum_list, task)
safe_write(os.fdopen(jdl_fd, 'w'), data)
except Exception:
remove_files([jdl_fn])
raise BackendError('Could not write jdl data to %s.' % jdl_fn)
return jdl_fn
def _write_jdl(self, jobnum_list, task):
# construct a temporary JDL for this batch of jobs
jdl_fd, jdl_fn = tempfile.mkstemp(suffix='.jdl')
try:
data = self._get_jdl_str_list(jobnum_list, task)
safe_write(os.fdopen(jdl_fd, 'w'), data)
except Exception:
remove_files([jdl_fn])
raise BackendError('Could not write jdl data to %s.' % jdl_fn)
return jdl_fn
def _prepareSubmit(self, task, jobnum_list, queryArguments):
localJdlFilePath = os.path.join(
self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' %
(self.parentPool.wms_name, md5_hex(self.getURI())))
readyJobNumList = self._stageSubmitFiles(task, jobnum_list)
safe_write(
open(localJdlFilePath, 'w'),
lmap(lambda line: line + '\n',
self._getJDLData(task, readyJobNumList, queryArguments)))
raise NotImplementedError('JDL must get moved to remote')
return jdlFilePath
def _submit_job(self, jobnum, task):
# Submit job and yield (jobnum, WMS ID, other data)
jdl_fd, jdl_fn = tempfile.mkstemp('.jdl')
try:
jdl_line_list = self._make_jdl(jobnum, task)
safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list)
except Exception:
remove_files([jdl_fn])
raise BackendError('Could not write jdl data to %s.' % jdl_fn)
try:
submit_arg_list = []
for key_value in filter_dict(self._submit_args_dict,
value_filter=identity).items():
submit_arg_list.extend(key_value)
submit_arg_list.append(jdl_fn)
activity = Activity('submitting job %d' % jobnum)
proc = LocalProcess(self._submit_exec, '--nomsg', '--noint',
'--logfile', '/dev/stderr', *submit_arg_list)
wms_id = None
stripped_stdout_iter = imap(str.strip,
proc.stdout.iter(timeout=60))
for line in ifilter(lambda x: x.startswith('http'),
stripped_stdout_iter):
wms_id = line
exit_code = proc.status(timeout=0, terminate=True)
activity.finish()
if (exit_code != 0) or (wms_id is None):
if self._explain_error(proc, exit_code):
pass
else:
self._log.log_process(
proc, files={'jdl': SafeFile(jdl_fn).read()})
finally:
remove_files([jdl_fn])
job_data = {'jdl': str.join('', jdl_line_list)}
return (jobnum, self._create_gc_id(wms_id), job_data)
def _submit_job(self, jobnum, task):
# Submit job and yield (jobnum, WMS ID, other data)
jdl_fd, jdl_fn = tempfile.mkstemp('.jdl')
try:
jdl_line_list = self._make_jdl(jobnum, task)
safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list)
except Exception:
remove_files([jdl_fn])
raise BackendError('Could not write jdl data to %s.' % jdl_fn)
try:
submit_arg_list = []
for key_value in filter_dict(self._submit_args_dict, value_filter=identity).items():
submit_arg_list.extend(key_value)
submit_arg_list.append(jdl_fn)
activity = Activity('submitting job %d' % jobnum)
proc = LocalProcess(self._submit_exec, '--nomsg', '--noint',
'--logfile', '/dev/stderr', *submit_arg_list)
wms_id = None
stripped_stdout_iter = imap(str.strip, proc.stdout.iter(timeout=60))
for line in ifilter(lambda x: x.startswith('http'), stripped_stdout_iter):
wms_id = line
exit_code = proc.status(timeout=0, terminate=True)
activity.finish()
if (exit_code != 0) or (wms_id is None):
if self._explain_error(proc, exit_code):
pass
else:
self._log.log_process(proc, files={'jdl': SafeFile(jdl_fn).read()})
finally:
remove_files([jdl_fn])
job_data = {'jdl': str.join('', jdl_line_list)}
return (jobnum, self._create_gc_id(wms_id), job_data)
def _prepareSubmit(self, task, jobnum_list, queryArguments): jdlFilePath = os.path.join(self.parentPool.getSandboxPath(), 'htc-%s.schedd-%s.jdl' % (self.parentPool.wms_name,md5_hex(self.getURI()))) safe_write(open(jdlFilePath, 'w'), lmap(lambda line: line + '\n', self._getJDLData(task, jobnum_list, queryArguments))) return jdlFilePath
