def _do_subst(value):
return replace_with_dict(
value, merged_var_dict,
ichain([
self.get_var_alias_map().items(),
izip(additional_var_dict, additional_var_dict)
]))
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def __init__(self, config, name):
self._name = name # needed for changeView calls before the constructor
head = [('DATASETNICK', 'Nickname')]
# Mapping between nickname and config files:
self._nmCfg = config.getLookup('nickname config', {}, defaultMatcher = 'regex',
parser = lambda x: lmap(str.strip, x.split(',')), strfun = lambda x: str.join(',', x))
if not self._nmCfg.empty():
allConfigFiles = sorted(set(ichain(self._nmCfg.get_values())))
config.set('config file', str.join('\n', allConfigFiles))
head.append((1, 'Config file'))
elif config.get('config file', ''):
raise ConfigError("Please use 'nickname config' instead of 'config file'")
# Mapping between nickname and constants - only display - work is handled by the 'normal' parameter factory
nmCName = config.getList('nickname constants', [], onChange = None)
param_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['parameters'])
param_config.set('constants', str.join(' ', nmCName), '+=')
for cName in nmCName:
param_config.set(cName + ' matcher', 'regex')
param_config.set(cName + ' lookup', 'DATASETNICK')
head.append((cName, cName))
# Mapping between nickname and lumi filter - only display - work is handled by the 'normal' lumi filter
config.set('lumi filter matcher', 'regex')
if 'nickname lumi filter' in config.getOptions():
config.set('lumi filter', strDictLong(config.getDict('nickname lumi filter', {}, onChange = None)))
self._nmLumi = config.getLookup('lumi filter', {}, parser = parseLumiFilter, strfun = strLumi, onChange = None)
if not self._nmLumi.empty():
head.append((2, 'Lumi filter'))
CMSSW.__init__(self, config, name)
self._displaySetup(config.getWorkPath('datacache.dat'), head)
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def process(self, pNum, splitInfo, result):
if self.enabled():
lumi_filter = self._lumi_filter.lookup(splitInfo[DataSplitter.Nickname], is_selector = False)
if lumi_filter:
idxRuns = splitInfo[DataSplitter.MetadataHeader].index("Runs")
iterRuns = ichain(imap(lambda m: m[idxRuns], splitInfo[DataSplitter.Metadata]))
short_lumi_filter = filterLumiFilter(list(iterRuns), lumi_filter)
result['LUMI_RANGE'] = str.join(',', imap(lambda lr: '"%s"' % lr, formatLumi(short_lumi_filter)))
def process(self, pnum, partition, result):
if self.enabled():
lumi_filter = self._lumi_filter.lookup(partition[DataSplitter.Nickname], is_selector=False)
if lumi_filter:
idx_runs = partition[DataSplitter.MetadataHeader].index('Runs')
iter_run = ichain(imap(lambda m: m[idx_runs], partition[DataSplitter.Metadata]))
short_lumi_filter = filter_lumi_filter(list(iter_run), lumi_filter)
iter_lumi_range_str = imap(lambda lr: '"%s"' % lr, format_lumi(short_lumi_filter))
result['LUMI_RANGE'] = str.join(',', iter_lumi_range_str)
def process(self, pnum, partition, result):
if self.enabled():
lumi_filter = self._lumi_filter.lookup(partition[DataSplitter.Nickname], is_selector=False)
if lumi_filter:
idx_runs = partition[DataSplitter.MetadataHeader].index('Runs')
iter_run = ichain(imap(lambda m: m[idx_runs], partition[DataSplitter.Metadata]))
short_lumi_filter = filter_lumi_filter(list(iter_run), lumi_filter)
iter_lumi_range_str = imap(lambda lr: '"%s"' % lr, format_lumi(short_lumi_filter))
result['LUMI_RANGE'] = str.join(',', iter_lumi_range_str)
def substVars(self, name, inp, jobNum = None, addDict = None, check = True):
addDict = addDict or {}
allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
if jobNum is not None:
allVars.update(self.getJobConfig(jobNum))
subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)]))
result = subst(subst(str(inp)))
if check and self._varCheck.check(result):
raise ConfigError('%s references unknown variables: %s' % (name, result))
return result
def __init__(self, config, name):
self._name = name # needed for change_view calls before the constructor
head = [('DATASETNICK', 'Nickname')]
# Mapping between nickname and config files:
self._nm_cfg = config.get_lookup(
'nickname config', {},
default_matcher='RegExMatcher',
parser=lambda x: lmap(str.strip, x.split(',')),
strfun=lambda x: str.join(',', x))
if not self._nm_cfg.empty():
all_config_fn_list = sorted(set(ichain(self._nm_cfg.get_values())))
config.set('config file', str.join('\n', all_config_fn_list))
head.append((1, 'Config file'))
elif config.get('config file', ''):
raise ConfigError(
"Please use 'nickname config' instead of 'config file'")
# Mapping between nickname and constants - configure and display
# work is handled by the 'normal' parameter factory
nm_const_vn_list = config.get_list('nickname constants', [],
on_change=None)
param_config = config.change_view(view_class='TaggedConfigView',
set_classes=None,
set_names=None,
add_sections=['parameters'])
param_config.set('constants', str.join(' ', nm_const_vn_list), '+=')
for const_vn in nm_const_vn_list:
param_config.set(const_vn + ' matcher', 'RegexMatcher')
param_config.set(const_vn + ' lookup', 'DATASETNICK')
head.append((const_vn, const_vn))
# Mapping between nickname and lumi filter - configure and display
# work is handled by the 'normal' lumi filter
config.set('lumi filter matcher', 'RegexMatcher')
if 'nickname lumi filter' in config.get_option_list():
config.set(
'lumi filter',
str_dict_cfg(
config.get_dict('nickname lumi filter', {},
on_change=None)))
self._nm_lumi = config.get_lookup('lumi filter', {},
parser=parse_lumi_filter,
strfun=str_lumi,
on_change=None)
if not self._nm_lumi.empty():
head.append((2, 'Lumi filter'))
CMSSW.__init__(self, config, name)
self._display_setup(config.get_work_path('datacache.dat'), head)
def __init__(self, config, name):
self._name = name # needed for changeView calls before the constructor
head = [('DATASETNICK', 'Nickname')]
# Mapping between nickname and config files:
self._nmCfg = config.getLookup(
'nickname config', {},
defaultMatcher='regex',
parser=lambda x: lmap(str.strip, x.split(',')),
strfun=lambda x: str.join(',', x))
if not self._nmCfg.empty():
allConfigFiles = sorted(set(ichain(self._nmCfg.get_values())))
config.set('config file', str.join('\n', allConfigFiles))
head.append((1, 'Config file'))
elif config.get('config file', ''):
raise ConfigError(
"Please use 'nickname config' instead of 'config file'")
# Mapping between nickname and constants - only display - work is handled by the 'normal' parameter factory
nmCName = config.getList('nickname constants', [], onChange=None)
param_config = config.changeView(viewClass='TaggedConfigView',
setClasses=None,
setNames=None,
addSections=['parameters'])
param_config.set('constants', str.join(' ', nmCName), '+=')
for cName in nmCName:
param_config.set(cName + ' matcher', 'regex')
param_config.set(cName + ' lookup', 'DATASETNICK')
head.append((cName, cName))
# Mapping between nickname and lumi filter - only display - work is handled by the 'normal' lumi filter
config.set('lumi filter matcher', 'regex')
if 'nickname lumi filter' in config.getOptions():
config.set(
'lumi filter',
strDictLong(
config.getDict('nickname lumi filter', {}, onChange=None)))
self._nmLumi = config.getLookup('lumi filter', {},
parser=parseLumiFilter,
strfun=strLumi,
onChange=None)
if not self._nmLumi.empty():
head.append((2, 'Lumi filter'))
CMSSW.__init__(self, config, name)
self._displaySetup(config.getWorkPath('datacache.dat'), head)
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(
ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
os.listdir(utils.pathPKG()))
depFiles = lmap(
lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
depList)
taskEnv = utils.mergeDicts(
imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({
'GC_DEPFILES': str.join(' ', depList),
'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self._name
})
taskConfig = sorted(
utils.DictFormat(escapeString=True).format(
taskEnv, format='export %s%s%s\n'))
varMappingDict = dict(
izip(monitor.getTaskConfig().keys(),
monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(
utils.DictFormat(delimeter=' ').format(varMappingDict,
format='%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([
monitor.getFiles(), depFiles,
getTaskFiles(),
[
VirtualFile('_config.sh', taskConfig),
VirtualFile('_varmap.dat', varMapping)
]
])
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self._name})
taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def iter_config_entries(self, option, filter_fun):
source_list = [
self._content.get(option, []),
self._content_default.get(option, {}).values()
]
return ifilter(filter_fun, ichain(source_list))
def show(self): result = ParameterSource.show(self) result.extend(imap(lambda x: '\t' + x, ichain(imap(lambda ps: ps.show(), self._psourceList)))) return result
def _collect(value): return list(ichain(imap(parse_value, value)))
def get_options(self): return sorted(set(ichain([self._content.keys(), self._content_default.keys()])))
def iter_config_entries(self, option, filter_fun):
source_list = [self._content.get(option, []), self._content_default.get(option, {}).values()]
return ifilter(filter_fun, ichain(source_list))
def getEntries(self, option, filterExpr):
entryChain = ichain([self._content.get(option, []), self._content_default.get(option, {}).values()])
return ifilter(filterExpr, entryChain)
def getKeys(self): return sorted(set(ichain([self._content.keys(), self._content_default.keys()])))
def _do_subst(value): return replace_with_dict(value, merged_var_dict, ichain([self.get_var_alias_map().items(), izip(additional_var_dict, additional_var_dict)]))
def collect(value):
return list(ichain(imap(parseValue, value)))
def validateVariables(self): example_vars = dict.fromkeys(self.getVarNames(), '') example_vars.update(dict.fromkeys(['X', 'XBASE', 'XEXT', 'GC_DATE', 'GC_TIMESTAMP', 'GC_GUID', 'RANDOM'], '')) for name, value in ichain([self.getTaskConfig().items(), example_vars.items()]): self.substVars(name, value, None, example_vars)
def show_psrc(self): return ParameterSource.show_psrc(self) + lmap(lambda x: '\t' + x, ichain(imap(lambda ps: ps.show_psrc(), self._psrc_list)))
def show_psrc(self):
return ParameterSource.show_psrc(self) + lmap(
lambda x: '\t' + x,
ichain(imap(lambda ps: ps.show_psrc(), self._psrc_list)))
