def substitute_variables(self, name, inp, jobnum=None, additional_var_dict=None, check=True): additional_var_dict = additional_var_dict or {} merged_var_dict = dict_union(additional_var_dict, self._get_const_job_env()) if jobnum is not None: merged_var_dict.update(self.get_job_dict(jobnum)) def _do_subst(value): return replace_with_dict( value, merged_var_dict, ichain([ self.get_var_alias_map().items(), izip(additional_var_dict, additional_var_dict) ])) result = _do_subst(_do_subst(str(inp))) if check and self._var_checker.check(result): raise ConfigError('%s references unknown variables: %s' % (name, result)) return result
def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def get_job_dict(self, jobnum): # Get job dependent environment variables job_psp = self._source.get_job_content(jobnum) job_env_dict = dict( imap(lambda key: (key.value, job_psp.get(key.value, '')), self._source.get_job_metadata())) job_env_dict['GC_ARGS'] = self.get_job_arguments(jobnum) return dict_union(job_env_dict, self._get_const_job_env())
def get_task_dict( self): # OLD API: Get environment variables for gc_config.sh task_base_dict = { # Storage element 'SE_MINFILESIZE': self._se_min_size, # Sandbox 'SB_OUTPUT_FILES': str.join(' ', self.get_sb_out_fn_list()), 'SB_INPUT_FILES': str.join(' ', imap(lambda x: x.path_rel, self.get_sb_in_fpi_list())), # Runtime 'GC_JOBTIMEOUT': self._job_timeout, 'GC_RUNTIME': self.get_command(), # Seeds and substitutions 'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())), 'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(), # Task infos 'GC_TASK_CONF': self._task_config_name, 'GC_TASK_DATE': self._task_date, 'GC_TASK_TIME': self._task_time, 'GC_TASK_ID': self._task_id, 'GC_VERSION': get_version(), } return dict_union(task_base_dict, self._task_var_dict)
def _publish(self, job_obj, jobnum, task_id, usermsg): (_, backend, wms_id) = job_obj.gc_id.split('.', 2) dash_id = '%s_%s' % (jobnum, wms_id) if 'http' not in job_obj.gc_id: dash_id = '%s_https://%s:/%s' % (jobnum, backend, wms_id) msg = dict_union({'taskId': task_id, 'jobId': dash_id, 'sid': wms_id}, *usermsg) DashboardAPI(task_id, dash_id).publish(**filter_dict(msg, value_filter=identity))
def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def _write_job_config(self, job_config_fn, jobnum, task, extras): try: job_env_dict = dict_union(task.get_job_dict(jobnum), extras) job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip() content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n') safe_write(open(job_config_fn, 'w'), content) except Exception: raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _write_job_config(self, job_config_fn, jobnum, task, extras): try: job_env_dict = dict_union(task.get_job_dict(jobnum), extras) job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip() content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n') safe_write(open(job_config_fn, 'w'), content) except Exception: raise BackendError('Could not write job config data to %s.' % job_config_fn)
def _publish(self, job_obj, jobnum, task_id, usermsg): (_, backend, wms_id) = job_obj.gc_id.split('.', 2) dash_id = '%s_%s' % (jobnum, wms_id) if 'http' not in job_obj.gc_id: dash_id = '%s_https://%s:/%s' % (jobnum, backend, wms_id) msg = dict_union({ 'taskId': task_id, 'jobId': dash_id, 'sid': wms_id }, *usermsg) DashboardAPI( task_id, dash_id).publish(**filter_dict(msg, value_filter=identity))
def substitute_variables(self, name, inp, jobnum=None, additional_var_dict=None, check=True): additional_var_dict = additional_var_dict or {} merged_var_dict = dict_union(additional_var_dict, self._get_const_job_env()) if jobnum is not None: merged_var_dict.update(self.get_job_dict(jobnum)) def _do_subst(value): return replace_with_dict(value, merged_var_dict, ichain([self.get_var_alias_map().items(), izip(additional_var_dict, additional_var_dict)])) result = _do_subst(_do_subst(str(inp))) if check and self._var_checker.check(result): raise ConfigError('%s references unknown variables: %s' % (name, result)) return result
def get_task_dict(self): # OLD API: Get environment variables for gc_config.sh task_base_dict = { # Storage element 'SE_MINFILESIZE': self._se_min_size, # Sandbox 'SB_OUTPUT_FILES': str.join(' ', self.get_sb_out_fn_list()), 'SB_INPUT_FILES': str.join(' ', imap(lambda x: x.path_rel, self.get_sb_in_fpi_list())), # Runtime 'GC_JOBTIMEOUT': self._job_timeout, 'GC_RUNTIME': self.get_command(), # Seeds and substitutions 'SUBST_FILES': str.join(' ', imap(os.path.basename, self._get_subst_fn_list())), 'GC_SUBST_OLD_STYLE': str('__' in self._var_checker.markers).lower(), # Task infos 'GC_TASK_CONF': self._task_config_name, 'GC_TASK_DATE': self._task_date, 'GC_TASK_TIME': self._task_time, 'GC_TASK_ID': self._task_id, 'GC_VERSION': get_version(), } return dict_union(task_base_dict, self._task_var_dict)
def get_task_dict(self): return dict_union(UserTask.get_task_dict(self), {'GC_ROOTSYS': self._root_dn})
def get_var_alias_map(self): if self._has_dataset: # create alias NICK for DATASETNICK return dict_union(TaskModule.get_var_alias_map(self), {'NICK': 'DATASETNICK'}) return TaskModule.get_var_alias_map(self)
def get_mon_env_dict(self): tmp = RemoteEventHandler.get_mon_env_dict(self) return dict_union( *(lmap(lambda m: m.get_mon_env_dict(), self._handlers) + [tmp]))
def get_mon_env_dict(self): tmp = RemoteEventHandler.get_mon_env_dict(self) return dict_union(*(lmap(lambda m: m.get_mon_env_dict(), self._handlers) + [tmp]))
def get_job_dict(self, jobnum): # Get job dependent environment variables job_psp = self._source.get_job_content(jobnum) job_env_dict = dict(imap(lambda key: (key.value, job_psp.get(key.value, '')), self._source.get_job_metadata())) job_env_dict['GC_ARGS'] = self.get_job_arguments(jobnum) return dict_union(job_env_dict, self._get_const_job_env())
def get_task_dict(self): return dict_union(UserTask.get_task_dict(self), {'GC_ROOTSYS': self._root_dn})
def get_var_alias_map(self): if self._has_dataset: # create alias NICK for DATASETNICK return dict_union(TaskModule.get_var_alias_map(self), {'NICK': 'DATASETNICK'}) return TaskModule.get_var_alias_map(self)