def _do_subst(value): return replace_with_dict( value, merged_var_dict, ichain([ self.get_var_alias_map().items(), izip(additional_var_dict, additional_var_dict) ]))
def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def __init__(self, config, name): self._name = name # needed for changeView calls before the constructor head = [('DATASETNICK', 'Nickname')] # Mapping between nickname and config files: self._nmCfg = config.getLookup('nickname config', {}, defaultMatcher = 'regex', parser = lambda x: lmap(str.strip, x.split(',')), strfun = lambda x: str.join(',', x)) if not self._nmCfg.empty(): allConfigFiles = sorted(set(ichain(self._nmCfg.get_values()))) config.set('config file', str.join('\n', allConfigFiles)) head.append((1, 'Config file')) elif config.get('config file', ''): raise ConfigError("Please use 'nickname config' instead of 'config file'") # Mapping between nickname and constants - only display - work is handled by the 'normal' parameter factory nmCName = config.getList('nickname constants', [], onChange = None) param_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['parameters']) param_config.set('constants', str.join(' ', nmCName), '+=') for cName in nmCName: param_config.set(cName + ' matcher', 'regex') param_config.set(cName + ' lookup', 'DATASETNICK') head.append((cName, cName)) # Mapping between nickname and lumi filter - only display - work is handled by the 'normal' lumi filter config.set('lumi filter matcher', 'regex') if 'nickname lumi filter' in config.getOptions(): config.set('lumi filter', strDictLong(config.getDict('nickname lumi filter', {}, onChange = None))) self._nmLumi = config.getLookup('lumi filter', {}, parser = parseLumiFilter, strfun = strLumi, onChange = None) if not self._nmLumi.empty(): head.append((2, 'Lumi filter')) CMSSW.__init__(self, config, name) self._displaySetup(config.getWorkPath('datacache.dat'), head)
def process(self, pNum, splitInfo, result): if self.enabled(): lumi_filter = self._lumi_filter.lookup(splitInfo[DataSplitter.Nickname], is_selector = False) if lumi_filter: idxRuns = splitInfo[DataSplitter.MetadataHeader].index("Runs") iterRuns = ichain(imap(lambda m: m[idxRuns], splitInfo[DataSplitter.Metadata])) short_lumi_filter = filterLumiFilter(list(iterRuns), lumi_filter) result['LUMI_RANGE'] = str.join(',', imap(lambda lr: '"%s"' % lr, formatLumi(short_lumi_filter)))
def process(self, pnum, partition, result): if self.enabled(): lumi_filter = self._lumi_filter.lookup(partition[DataSplitter.Nickname], is_selector=False) if lumi_filter: idx_runs = partition[DataSplitter.MetadataHeader].index('Runs') iter_run = ichain(imap(lambda m: m[idx_runs], partition[DataSplitter.Metadata])) short_lumi_filter = filter_lumi_filter(list(iter_run), lumi_filter) iter_lumi_range_str = imap(lambda lr: '"%s"' % lr, format_lumi(short_lumi_filter)) result['LUMI_RANGE'] = str.join(',', iter_lumi_range_str)
def substVars(self, name, inp, jobNum = None, addDict = None, check = True): addDict = addDict or {} allVars = utils.mergeDicts([addDict, self.getTaskConfig()]) if jobNum is not None: allVars.update(self.getJobConfig(jobNum)) subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)])) result = subst(subst(str(inp))) if check and self._varCheck.check(result): raise ConfigError('%s references unknown variables: %s' % (name, result)) return result
def __init__(self, config, name): self._name = name # needed for change_view calls before the constructor head = [('DATASETNICK', 'Nickname')] # Mapping between nickname and config files: self._nm_cfg = config.get_lookup( 'nickname config', {}, default_matcher='RegExMatcher', parser=lambda x: lmap(str.strip, x.split(',')), strfun=lambda x: str.join(',', x)) if not self._nm_cfg.empty(): all_config_fn_list = sorted(set(ichain(self._nm_cfg.get_values()))) config.set('config file', str.join('\n', all_config_fn_list)) head.append((1, 'Config file')) elif config.get('config file', ''): raise ConfigError( "Please use 'nickname config' instead of 'config file'") # Mapping between nickname and constants - configure and display # work is handled by the 'normal' parameter factory nm_const_vn_list = config.get_list('nickname constants', [], on_change=None) param_config = config.change_view(view_class='TaggedConfigView', set_classes=None, set_names=None, add_sections=['parameters']) param_config.set('constants', str.join(' ', nm_const_vn_list), '+=') for const_vn in nm_const_vn_list: param_config.set(const_vn + ' matcher', 'RegexMatcher') param_config.set(const_vn + ' lookup', 'DATASETNICK') head.append((const_vn, const_vn)) # Mapping between nickname and lumi filter - configure and display # work is handled by the 'normal' lumi filter config.set('lumi filter matcher', 'RegexMatcher') if 'nickname lumi filter' in config.get_option_list(): config.set( 'lumi filter', str_dict_cfg( config.get_dict('nickname lumi filter', {}, on_change=None))) self._nm_lumi = config.get_lookup('lumi filter', {}, parser=parse_lumi_filter, strfun=str_lumi, on_change=None) if not self._nm_lumi.empty(): head.append((2, 'Lumi filter')) CMSSW.__init__(self, config, name) self._display_setup(config.get_work_path('datacache.dat'), head)
def __init__(self, config, name): self._name = name # needed for changeView calls before the constructor head = [('DATASETNICK', 'Nickname')] # Mapping between nickname and config files: self._nmCfg = config.getLookup( 'nickname config', {}, defaultMatcher='regex', parser=lambda x: lmap(str.strip, x.split(',')), strfun=lambda x: str.join(',', x)) if not self._nmCfg.empty(): allConfigFiles = sorted(set(ichain(self._nmCfg.get_values()))) config.set('config file', str.join('\n', allConfigFiles)) head.append((1, 'Config file')) elif config.get('config file', ''): raise ConfigError( "Please use 'nickname config' instead of 'config file'") # Mapping between nickname and constants - only display - work is handled by the 'normal' parameter factory nmCName = config.getList('nickname constants', [], onChange=None) param_config = config.changeView(viewClass='TaggedConfigView', setClasses=None, setNames=None, addSections=['parameters']) param_config.set('constants', str.join(' ', nmCName), '+=') for cName in nmCName: param_config.set(cName + ' matcher', 'regex') param_config.set(cName + ' lookup', 'DATASETNICK') head.append((cName, cName)) # Mapping between nickname and lumi filter - only display - work is handled by the 'normal' lumi filter config.set('lumi filter matcher', 'regex') if 'nickname lumi filter' in config.getOptions(): config.set( 'lumi filter', strDictLong( config.getDict('nickname lumi filter', {}, onChange=None))) self._nmLumi = config.getLookup('lumi filter', {}, parser=parseLumiFilter, strfun=strLumi, onChange=None) if not self._nmLumi.empty(): head.append((2, 'Lumi filter')) CMSSW.__init__(self, config, name) self._displaySetup(config.getWorkPath('datacache.dat'), head)
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set( ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg), os.listdir(utils.pathPKG())) depFiles = lmap( lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts( imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({ 'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self._name }) taskConfig = sorted( utils.DictFormat(escapeString=True).format( taskEnv, format='export %s%s%s\n')) varMappingDict = dict( izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted( utils.DictFormat(delimeter=' ').format(varMappingDict, format='%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([ monitor.getFiles(), depFiles, getTaskFiles(), [ VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping) ] ])
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG())) depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self._name}) taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n')) varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([monitor.getFiles(), depFiles, getTaskFiles(), [VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def iter_config_entries(self, option, filter_fun): source_list = [ self._content.get(option, []), self._content_default.get(option, {}).values() ] return ifilter(filter_fun, ichain(source_list))
def show(self): result = ParameterSource.show(self) result.extend(imap(lambda x: '\t' + x, ichain(imap(lambda ps: ps.show(), self._psourceList)))) return result
def _collect(value): return list(ichain(imap(parse_value, value)))
def get_options(self): return sorted(set(ichain([self._content.keys(), self._content_default.keys()])))
def iter_config_entries(self, option, filter_fun): source_list = [self._content.get(option, []), self._content_default.get(option, {}).values()] return ifilter(filter_fun, ichain(source_list))
def getEntries(self, option, filterExpr): entryChain = ichain([self._content.get(option, []), self._content_default.get(option, {}).values()]) return ifilter(filterExpr, entryChain)
def getKeys(self): return sorted(set(ichain([self._content.keys(), self._content_default.keys()])))
def _do_subst(value): return replace_with_dict(value, merged_var_dict, ichain([self.get_var_alias_map().items(), izip(additional_var_dict, additional_var_dict)]))
def collect(value): return list(ichain(imap(parseValue, value)))
def validateVariables(self): example_vars = dict.fromkeys(self.getVarNames(), '') example_vars.update(dict.fromkeys(['X', 'XBASE', 'XEXT', 'GC_DATE', 'GC_TIMESTAMP', 'GC_GUID', 'RANDOM'], '')) for name, value in ichain([self.getTaskConfig().items(), example_vars.items()]): self.substVars(name, value, None, example_vars)
def show_psrc(self): return ParameterSource.show_psrc(self) + lmap(lambda x: '\t' + x, ichain(imap(lambda ps: ps.show_psrc(), self._psrc_list)))
def show_psrc(self): return ParameterSource.show_psrc(self) + lmap( lambda x: '\t' + x, ichain(imap(lambda ps: ps.show_psrc(), self._psrc_list)))