def substitute_variables(self,
name,
inp,
jobnum=None,
additional_var_dict=None,
check=True):
additional_var_dict = additional_var_dict or {}
merged_var_dict = dict_union(additional_var_dict,
self._get_const_job_env())
if jobnum is not None:
merged_var_dict.update(self.get_job_dict(jobnum))
def _do_subst(value):
return replace_with_dict(
value, merged_var_dict,
ichain([
self.get_var_alias_map().items(),
izip(additional_var_dict, additional_var_dict)
]))
result = _do_subst(_do_subst(str(inp)))
if check and self._var_checker.check(result):
raise ConfigError('%s references unknown variables: %s' %
(name, result))
return result
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def get_job_dict(self, jobnum): # Get job dependent environment variables
job_psp = self._source.get_job_content(jobnum)
job_env_dict = dict(
imap(lambda key: (key.value, job_psp.get(key.value, '')),
self._source.get_job_metadata()))
job_env_dict['GC_ARGS'] = self.get_job_arguments(jobnum)
return dict_union(job_env_dict, self._get_const_job_env())
def get_task_dict(
self): # OLD API: Get environment variables for gc_config.sh
task_base_dict = {
# Storage element
'SE_MINFILESIZE':
self._se_min_size,
# Sandbox
'SB_OUTPUT_FILES':
str.join(' ', self.get_sb_out_fn_list()),
'SB_INPUT_FILES':
str.join(' ', imap(lambda x: x.path_rel,
self.get_sb_in_fpi_list())),
# Runtime
'GC_JOBTIMEOUT':
self._job_timeout,
'GC_RUNTIME':
self.get_command(),
# Seeds and substitutions
'SUBST_FILES':
str.join(' ', imap(os.path.basename, self._get_subst_fn_list())),
'GC_SUBST_OLD_STYLE':
str('__' in self._var_checker.markers).lower(),
# Task infos
'GC_TASK_CONF':
self._task_config_name,
'GC_TASK_DATE':
self._task_date,
'GC_TASK_TIME':
self._task_time,
'GC_TASK_ID':
self._task_id,
'GC_VERSION':
get_version(),
}
return dict_union(task_base_dict, self._task_var_dict)
def _publish(self, job_obj, jobnum, task_id, usermsg):
(_, backend, wms_id) = job_obj.gc_id.split('.', 2)
dash_id = '%s_%s' % (jobnum, wms_id)
if 'http' not in job_obj.gc_id:
dash_id = '%s_https://%s:/%s' % (jobnum, backend, wms_id)
msg = dict_union({'taskId': task_id, 'jobId': dash_id, 'sid': wms_id}, *usermsg)
DashboardAPI(task_id, dash_id).publish(**filter_dict(msg, value_filter=identity))
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def _write_job_config(self, job_config_fn, jobnum, task, extras):
try:
job_env_dict = dict_union(task.get_job_dict(jobnum), extras)
job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip()
content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n')
safe_write(open(job_config_fn, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _write_job_config(self, job_config_fn, jobnum, task, extras):
try:
job_env_dict = dict_union(task.get_job_dict(jobnum), extras)
job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip()
content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n')
safe_write(open(job_config_fn, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _publish(self, job_obj, jobnum, task_id, usermsg):
(_, backend, wms_id) = job_obj.gc_id.split('.', 2)
dash_id = '%s_%s' % (jobnum, wms_id)
if 'http' not in job_obj.gc_id:
dash_id = '%s_https://%s:/%s' % (jobnum, backend, wms_id)
msg = dict_union({
'taskId': task_id,
'jobId': dash_id,
'sid': wms_id
}, *usermsg)
DashboardAPI(
task_id,
dash_id).publish(**filter_dict(msg, value_filter=identity))
def substitute_variables(self, name, inp, jobnum=None, additional_var_dict=None, check=True):
additional_var_dict = additional_var_dict or {}
merged_var_dict = dict_union(additional_var_dict, self._get_const_job_env())
if jobnum is not None:
merged_var_dict.update(self.get_job_dict(jobnum))
def _do_subst(value):
return replace_with_dict(value, merged_var_dict,
ichain([self.get_var_alias_map().items(), izip(additional_var_dict, additional_var_dict)]))
result = _do_subst(_do_subst(str(inp)))
if check and self._var_checker.check(result):
raise ConfigError('%s references unknown variables: %s' % (name, result))
return result
def get_task_dict(self): # OLD API: Get environment variables for gc_config.sh
task_base_dict = {
# Storage element
'SE_MINFILESIZE': self._se_min_size,
# Sandbox
'SB_OUTPUT_FILES': str.join(' ', self.get_sb_out_fn_list()),
'SB_INPUT_FILES': str.join(' ', imap(lambda x: x.path_rel, self.get_sb_in_fpi_list())),
# Runtime
'GC_JOBTIMEOUT': self._job_timeout,
'GC_RUNTIME': self.get_command(),
# Seeds and substitutions
'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())),
'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(),
# Task infos
'GC_TASK_CONF': self._task_config_name,
'GC_TASK_DATE': self._task_date,
'GC_TASK_TIME': self._task_time,
'GC_TASK_ID': self._task_id,
'GC_VERSION': get_version(),
}
return dict_union(task_base_dict, self._task_var_dict)
def get_task_dict(self):
return dict_union(UserTask.get_task_dict(self), {'GC_ROOTSYS': self._root_dn})
def get_var_alias_map(self):
if self._has_dataset: # create alias NICK for DATASETNICK
return dict_union(TaskModule.get_var_alias_map(self),
{'NICK': 'DATASETNICK'})
return TaskModule.get_var_alias_map(self)
def get_mon_env_dict(self):
tmp = RemoteEventHandler.get_mon_env_dict(self)
return dict_union(
*(lmap(lambda m: m.get_mon_env_dict(), self._handlers) + [tmp]))
def get_mon_env_dict(self): tmp = RemoteEventHandler.get_mon_env_dict(self) return dict_union(*(lmap(lambda m: m.get_mon_env_dict(), self._handlers) + [tmp]))
def get_job_dict(self, jobnum): # Get job dependent environment variables job_psp = self._source.get_job_content(jobnum) job_env_dict = dict(imap(lambda key: (key.value, job_psp.get(key.value, '')), self._source.get_job_metadata())) job_env_dict['GC_ARGS'] = self.get_job_arguments(jobnum) return dict_union(job_env_dict, self._get_const_job_env())
def get_task_dict(self):
return dict_union(UserTask.get_task_dict(self), {'GC_ROOTSYS': self._root_dn})
def get_var_alias_map(self):
if self._has_dataset: # create alias NICK for DATASETNICK
return dict_union(TaskModule.get_var_alias_map(self), {'NICK': 'DATASETNICK'})
return TaskModule.get_var_alias_map(self)