def _parseParameterTuple(self, varName, tupleValue, tupleType, varType,
varIndex):
if tupleType == 'tuple':
tupleDelimeter = self.get(self._getParameterOption(varName),
'delimeter', ',')
tupleStrings = lmap(
str.strip,
utils.split_advanced(tupleValue, lambda tok: tok in ' \n',
lambda tok: False))
tupleList = lmap(lambda t: parseTuple(t, tupleDelimeter),
tupleStrings)
elif tupleType == 'binning':
tupleList = lzip(tupleValue.split(), tupleValue.split()[1:])
result = []
for tupleEntry in tupleList:
try:
tmp = self._parseParameter(varName, tupleEntry[varIndex],
varType)
except Exception:
raise ConfigError('Unable to parse %r' % repr(
(tupleEntry, tupleStrings)))
if isinstance(tmp, list):
if len(tmp) != 1:
raise ConfigError(
'[Variable: %s] Tuple entry (%s) expands to multiple variable entries (%s)!'
% (varName, tupleEntry[varIndex], tmp))
result.append(tmp[0])
else:
result.append(tmp)
return result
def __init__(self, remoteType="", **kwargs):
self._log = logging.getLogger('backend.condor')
self.cmd=False
# pick requested remote connection
try:
self.remoteType = self.RPHType.str2enum(remoteType)
self.cmd = self.RPHTemplate[self.remoteType]["command"]
self.copy = self.RPHTemplate[self.remoteType]["copy"]
self.path = self.RPHTemplate[self.remoteType]["path"]
self.argFormat = self.RPHTemplate[self.remoteType]["argFormat"]
except Exception:
raise ConfigError("Request to initialize RemoteProcessHandler of unknown type: %s" % remoteType)
# destination should be of type: [user@]host
if self.remoteType==self.RPHType.SSH or self.remoteType==self.RPHType.GSISSH:
try:
self.cmd = self.cmd % { "rhost" : kwargs["host"] }
self.copy = self.copy % { "rhost" : kwargs["host"] }
self.host = kwargs["host"]
except Exception:
raise ConfigError("Request to initialize RemoteProcessHandler of type %s without remote host." % self.RPHType.enum2str(self.remoteType))
# add default arguments for all commands
self.cmd = self.cmd % { "cmdargs" : kwargs.get("cmdargs",""), "args" : kwargs.get("args","") }
self.copy = self.copy % { "cpargs" : kwargs.get("cpargs",""), "args" : kwargs.get("args","") }
# test connection once
proc = LoggedProcess(self.cmd % { "cmd" : "exit"})
ret = proc.getAll()[0]
if ret != 0:
raise CondorProcessError('Validation of remote connection failed!', proc)
self._log.log(logging.INFO2, 'Remote interface initialized:\n\tCmd: %s\n\tCp : %s', self.cmd, self.copy)
def _processConfigFiles(self, config, cfgFiles, fragment_path, autoPrepare,
mustPrepare):
# process list of uninitialized config files
for (cfg, cfg_new,
doPrepare) in self._cfgFindUninitialized(config, cfgFiles,
autoPrepare,
mustPrepare):
if doPrepare and (autoPrepare or utils.getUserBool(
'Do you want to prepare %s for running over the dataset?' %
cfg, True)):
self._cfgStore(cfg, cfg_new, fragment_path)
else:
self._cfgStore(cfg, cfg_new)
result = []
for cfg in cfgFiles:
cfg_new = config.getWorkPath(os.path.basename(cfg))
if not os.path.exists(cfg_new):
raise ConfigError(
'Config file %r was not copied to the work directory!' %
cfg)
isInstrumented = self._cfgIsInstrumented(cfg_new)
if mustPrepare and not isInstrumented:
raise ConfigError(
'Config file %r must use %s to work properly!' %
(cfg,
str.join(', ',
imap(lambda x: '@%s@' % x, self.neededVars()))))
if autoPrepare and not isInstrumented:
self._log.warning('Config file %r was not instrumented!', cfg)
result.append(cfg_new)
return result
def _process_config_file_list(self, config, config_file_list,
fragment_path, auto_prepare, must_prepare):
# process list of uninitialized config files
iter_uninitialized_config_files = self._config_find_uninitialized(
config, config_file_list, auto_prepare, must_prepare)
for (cfg, cfg_new, do_prepare) in iter_uninitialized_config_files:
ask_user_msg = 'Do you want to prepare %s for running over the dataset?' % cfg
if do_prepare and (auto_prepare
or self._uii.prompt_bool(ask_user_msg, True)):
self._config_store_backup(cfg, cfg_new, fragment_path)
else:
self._config_store_backup(cfg, cfg_new)
result = []
for cfg in config_file_list:
cfg_new = config.get_work_path(os.path.basename(cfg))
if not os.path.exists(cfg_new):
raise ConfigError(
'Config file %r was not copied to the work directory!' %
cfg)
is_instrumented = self._config_is_instrumented(cfg_new)
if must_prepare and not is_instrumented:
raise ConfigError(
'Config file %r must use %s to work properly!' %
(cfg,
str.join(
', ',
imap(lambda x: '@%s@' % x, sorted(
self._needed_vn_set)))))
if auto_prepare and not is_instrumented:
self._log.warning('Config file %r was not instrumented!', cfg)
result.append(cfg_new)
return result
def parseLumiFilter(lumiexpr):
if lumiexpr == '':
return None
lumis = []
from grid_control.config import ConfigError
for token in imap(str.strip, lumiexpr.split(',')):
token = lmap(str.strip, token.split('|'))
if True in imap(str.isalpha,
token[0].lower().replace('min', '').replace('max',
'')):
if len(token) == 1:
token.append('')
try:
json_fn = os.path.normpath(
os.path.expandvars(os.path.expanduser(token[0].strip())))
json_fp = open(json_fn)
lumis.extend(parseLumiFromJSON(json_fp.read(), token[1]))
json_fp.close()
except Exception:
raise ConfigError(
'Could not process lumi filter file: %r (filter: %r)' %
tuple(token))
else:
try:
lumis.append(parseLumiFromString(token[0]))
except Exception:
raise ConfigError(
'Could not process lumi filter expression:\n\t%s' %
token[0])
return mergeLumi(lumis)
def lookupConfigParser(pconfig, outputKey, lookupKeys):
def collectKeys(src):
result = []
src.fillParameterKeys(result)
return result
outputKey = collectKeys(outputKey)[0]
if lookupKeys is None:
lookupKeys = [pconfig.get('default lookup')]
else:
lookupKeys = collectKeys(lookupKeys)
if not lookupKeys or lookupKeys == ['']:
raise ConfigError('Lookup parameter not defined!')
defaultMatcher = pconfig.get('', 'default matcher', 'equal')
matchstrList = pconfig.get(outputKey.lstrip('!'), 'matcher',
defaultMatcher).lower().splitlines()
if len(matchstrList) != len(lookupKeys):
if len(matchstrList) == 1:
matchstrList = matchstrList * len(lookupKeys)
else:
raise ConfigError(
'Match-functions (length %d) and match-keys (length %d) do not match!'
% (len(matchstrList), len(lookupKeys)))
matchfun = []
for matcherName in matchstrList:
matchfun.append(Matcher.createInstance(matcherName, pconfig,
outputKey))
(content, order) = pconfig.getParameter(outputKey.lstrip('!'))
if not pconfig.getBool(outputKey.lstrip('!'), 'empty set', False):
for k in content:
if len(content[k]) == 0:
content[k].append('')
return (outputKey, lookupKeys, matchfun, (content, order))
def parse_lumi_filter(lumi_str):
if lumi_str == '':
return None
run_lumi_range_list = []
from grid_control.config import ConfigError
for token in imap(str.strip, lumi_str.split(',')):
token = lmap(str.strip, token.split('|'))
if True in imap(str.isalpha,
token[0].lower().replace('min', '').replace('max',
'')):
if len(token) == 1:
token.append('')
try:
json_fn = os.path.normpath(
os.path.expandvars(os.path.expanduser(token[0].strip())))
run_lumi_range_list.extend(
parse_lumi_from_json(
SafeFile(json_fn).read_close(), token[1]))
except Exception:
raise ConfigError(
'Could not process lumi filter file: %r (filter: %r)' %
tuple(token))
else:
try:
run_lumi_range_list.append(parse_lumi_from_str(token[0]))
except Exception:
raise ConfigError(
'Could not process lumi filter expression:\n\t%s' %
token[0])
return merge_lumi_list(run_lumi_range_list)
def frange(start, end = None, num = None, steps = None, format = '%g'):
if (end is None) and (num is None):
raise ConfigError('frange: No exit condition!')
if (end is not None) and (num is not None) and (steps is not None):
raise ConfigError('frange: Overdetermined parameters!')
if (end is not None) and (num is not None) and (steps is None):
steps = float(end - start) / (num - 1)
if (end is not None) and (num is None):
steps = steps or 1
num = int(1 + (end - start) / steps)
return lmap(lambda x: format % x, imap(lambda i: start + (steps or 1) * i, irange(num)))
def __init__(self, config, name):
DataTask.__init__(self, config, name)
config.set('area files matcher mode', 'ShellStyleMatcher')
# SCRAM settings
scram_arch_default = unspecified
scram_project = config.get_list('scram project', [])
if scram_project: # manual scram setup
if len(scram_project) != 2:
raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
self._project_area = None
self._project_area_selector_list = None
self._scram_project = scram_project[0]
self._scram_project_version = scram_project[1]
# ensure project area is not used
if 'project area' in config.get_option_list():
raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))
else: # scram setup used from project area
self._project_area = config.get_dn('project area')
self._always_matcher = Matcher.create_instance('AlwaysMatcher', config, [''])
self._project_area_base_fn = config.get_bool('area files basename', True,
on_change=TriggerInit('sandbox'))
self._project_area_matcher = config.get_matcher('area files',
'-.* -config bin lib python module data *.xml *.sql *.db *.cfi *.cff *.py -CVS -work.* *.pcm',
default_matcher='BlackWhiteMatcher', on_change=TriggerInit('sandbox'))
self._log.info('Project area found in: %s', self._project_area)
# try to determine scram settings from environment settings
scram_path = os.path.join(self._project_area, '.SCRAM')
scram_env = self._parse_scram_file(os.path.join(scram_path, 'Environment'))
try:
self._scram_project = scram_env['SCRAM_PROJECTNAME']
self._scram_project_version = scram_env['SCRAM_PROJECTVERSION']
except:
raise ConfigError('Installed program in project area not recognized.')
def filter_arch_dir(dn):
return os.path.isdir(os.path.join(scram_path, dn))
for arch_dir in sorted(ifilter(filter_arch_dir, os.listdir(scram_path))):
scram_arch_default = arch_dir
self._scram_version = config.get('scram version', 'scramv1')
self._scram_arch = config.get('scram arch', scram_arch_default)
self._scram_req_list = []
if config.get_bool('scram arch requirements', True, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_arch))
if config.get_bool('scram project requirements', False, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project))
if config.get_bool('scram project version requirements', False, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project_version))
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('dataset processor', 'LumiDataProcessor', '+=')
config.set('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor ' +
'LFNPartitionProcessor LumiPartitionProcessor CMSSWPartitionProcessor')
dash_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dashboard'])
dash_config.set('application', 'cmsRun')
SCRAMTask.__init__(self, config, name)
if self._scramProject != 'CMSSW':
raise ConfigError('Project area contains no CMSSW project')
self._oldReleaseTop = None
if self._projectArea:
self._oldReleaseTop = self._parse_scram_file(os.path.join(self._projectArea, '.SCRAM', self._scramArch, 'Environment')).get('RELEASETOP', None)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
self._projectAreaTarballSE = config.getBool(['se runtime', 'se project area'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self._dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0') # this can be a variable like @USER_EVENTS@!
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self._dataSplitter is not None))
# Create project area tarball
if self._projectArea and not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self._projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self._projectArea, self._projectAreaPattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def __init__(self, config, name):
self._name = name # needed for changeView calls before the constructor
head = [('DATASETNICK', 'Nickname')]
# Mapping between nickname and config files:
self._nmCfg = config.getLookup(
'nickname config', {},
defaultMatcher='regex',
parser=lambda x: lmap(str.strip, x.split(',')),
strfun=lambda x: str.join(',', x))
if not self._nmCfg.empty():
if 'config file' in config.getOptions():
raise ConfigError(
"Please use 'nickname config' instead of 'config file'")
allConfigFiles = sorted(
set(utils.flatten(self._nmCfg.get_values())))
config.set('config file', str.join('\n', allConfigFiles))
head.append((1, 'Config file'))
# Mapping between nickname and constants - only display - work is handled by the 'normal' parameter factory
nmCName = config.getList('nickname constants', [], onChange=None)
param_config = config.changeView(viewClass='TaggedConfigView',
setClasses=None,
setNames=None,
addSections=['parameters'])
param_config.set('constants', str.join(' ', nmCName), '+=')
for cName in nmCName:
param_config.set(cName + ' matcher', 'regex')
param_config.set(cName + ' lookup', 'DATASETNICK')
head.append((cName, cName))
# Mapping between nickname and lumi filter - only display - work is handled by the 'normal' lumi filter
if ('lumi filter' in config.getOptions()) and ('nickname lumi filter'
in config.getOptions()):
raise ConfigError('Please use "lumi filter" exclusively')
config.set('lumi filter matcher', 'regex')
config.set(
'lumi filter',
strDictLong(
config.getDict('nickname lumi filter', {}, onChange=None)))
self._nmLumi = config.getLookup('lumi filter', {},
parser=parseLumiFilter,
strfun=strLumi,
onChange=None)
if not self._nmLumi.empty():
head.append((2, 'Lumi filter'))
CMSSW.__init__(self, config, name)
self._displaySetup(config.getWorkPath('datacache.dat'), head)
def do_transfer(self, desc_source_target_list):
for (desc, source, target) in desc_source_target_list:
if not self._storage_paths:
raise ConfigError(
"%s can't be transferred because '%s path wasn't set" %
(desc, self._storage_channel))
for idx, se_path in enumerate(set(self._storage_paths)):
activity = Activity('Copy %s to SE %d ' % (desc, idx + 1))
proc = se_copy(source, os.path.join(se_path, target),
self._storage_force)
proc.status(timeout=5 * 60, terminate=True)
activity.finish()
if proc.status(timeout=0) == 0:
self._log.info('Copy %s to SE %d finished', desc, idx + 1)
else:
self._log.info('Copy %s to SE %d failed', desc, idx + 1)
self._log.log_process(proc)
self._log.critical(
'Unable to copy %s! You can try to copy it manually.',
desc)
msg = 'Is %s (%s) available on SE %s?' % (desc, source,
se_path)
if not UserInputInterface().prompt_bool(msg, False):
raise StorageError('%s is missing on SE %s!' %
(desc, se_path))
def __init__(self, **kwargs):
ProcessHandler.__init__(self, **kwargs)
ssh_default_args = ' -vvv -o BatchMode=yes -o ForwardX11=no'
self._shell_cmd = resolve_install_path('ssh') + ssh_default_args
self._copy_cmd = resolve_install_path('scp') + ssh_default_args + ' -r'
self._ssh_link_id = 0
self._ssh_link_args = ''
self._ssh_link_timestamp = 0
self._ssh_link_fail_count = 0
self._ssh_link_master_proc = None
try:
self._remote_host = kwargs['remote_host']
except Exception:
raise ConfigError(
'Request to initialize SSH-Type RemoteProcessHandler without remote host.'
)
try:
self._ssh_link_base = os.path.abspath(kwargs['sshLink'])
# older ssh/gsissh puts a maximum length limit on control paths, use a different one
if len(self._ssh_link_base) >= 107:
self._ssh_link_base = os.path.expanduser(
'~/.ssh/%s' % os.path.basename(self._ssh_link_base))
self._ssh_link = self._ssh_link_base
_ssh_link_secure(self._ssh_link, init_dn=True)
self._get_ssh_link()
except KeyError:
clear_current_exception()
self._ssh_link = False
# test connection once
proc_test = self.logged_execute('exit')
if proc_test.wait() != 0:
raise CondorProcessError('Failed to validate remote connection.',
proc_test)
def doTransfer(self, listDescSourceTarget):
for (desc, source, target) in listDescSourceTarget:
if not self.smPaths:
raise ConfigError(
"%s can't be transferred because '%s path wasn't set" %
(desc, self.smOptPrefix))
for idx, sePath in enumerate(set(self.smPaths)):
utils.vprint('Copy %s to SE %d ' % (desc, idx + 1),
-1,
newline=False)
sys.stdout.flush()
proc = se_copy(source, os.path.join(sePath, target),
self.smForce)
if proc.status(timeout=5 * 60, terminate=True) == 0:
utils.vprint('finished', -1)
else:
utils.vprint('failed', -1)
utils.eprint(proc.stderr.read(timeout=0))
utils.eprint(
'Unable to copy %s! You can try to copy it manually.' %
desc)
if not utils.getUserBool(
'Is %s (%s) available on SE %s?' %
(desc, source, sePath), False):
raise StorageError('%s is missing on SE %s!' %
(desc, sePath))
def __init__(self, **kwargs):
ProcessHandler.__init__(self, **kwargs)
self.__initcommands(**kwargs)
self.defaultArgs="-vvv -o BatchMode=yes -o ForwardX11=no " + kwargs.get("defaultArgs","")
self.socketArgs=""
self.socketEnforce=kwargs.get("sshLinkEnforce",True)
try:
self.remoteHost = kwargs["remoteHost"]
except Exception:
raise ConfigError("Request to initialize SSH-Type RemoteProcessHandler without remote host.")
try:
self.sshLinkBase=os.path.abspath(kwargs["sshLink"])
# older ssh/gsissh puts a maximum length limit on control paths, use a different one
if ( len(self.sshLinkBase)>= 107):
self.sshLinkBase=os.path.expanduser("~/.ssh/%s"%os.path.basename(self.sshLinkBase))
self.sshLink=self.sshLinkBase
self._secureSSHLink(initDirectory=True)
self._socketHandler()
except KeyError:
self.sshLink=False
# test connection once
testProcess = self.LoggedExecute( "exit" )
if testProcess.wait() != 0:
testProcess.getError()
raise CondorProcessError('Failed to validate remote connection.', testProcess)
def substitute_variables(self,
name,
inp,
jobnum=None,
additional_var_dict=None,
check=True):
additional_var_dict = additional_var_dict or {}
merged_var_dict = dict_union(additional_var_dict,
self._get_const_job_env())
if jobnum is not None:
merged_var_dict.update(self.get_job_dict(jobnum))
def _do_subst(value):
return replace_with_dict(
value, merged_var_dict,
ichain([
self.get_var_alias_map().items(),
izip(additional_var_dict, additional_var_dict)
]))
result = _do_subst(_do_subst(str(inp)))
if check and self._var_checker.check(result):
raise ConfigError('%s references unknown variables: %s' %
(name, result))
return result
def __init__(self, config, datasetExpr, datasetNick = None):
DataProvider.__init__(self, config, datasetExpr, datasetNick)
(self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@')
self._selist = parseList(selist, ',') or None
if not (self._path and self._events):
raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def doTransfer(self, listDescSourceTarget):
for (desc, source, target) in listDescSourceTarget:
if not self.smPaths:
raise ConfigError(
"%s can't be transferred because '%s path wasn't set" %
(desc, self.smOptPrefix))
for idx, sePath in enumerate(set(self.smPaths)):
activity = Activity('Copy %s to SE %d ' % (desc, idx + 1))
proc = se_copy(source, os.path.join(sePath, target),
self.smForce)
proc.status(timeout=5 * 60, terminate=True)
activity.finish()
if proc.status(timeout=0) == 0:
self._log.info('Copy %s to SE %d finished', desc, idx + 1)
else:
self._log.info('Copy %s to SE %d failed', desc, idx + 1)
self._log.critical(proc.stderr.read(timeout=0))
self._log.critical(
'Unable to copy %s! You can try to copy it manually.',
desc)
if not utils.getUserBool(
'Is %s (%s) available on SE %s?' %
(desc, source, sePath), False):
raise StorageError('%s is missing on SE %s!' %
(desc, sePath))
def getSubmitArguments(self, jobNum, jobName, reqs, sandbox, stdout,
stderr):
timeStr = lambda s: '%02d:%02d:%02d' % (s / 3600,
(s / 60) % 60, s % 60)
reqMap = {
WMS.MEMORY: ('h_vmem', lambda m: '%dM' % m),
WMS.WALLTIME: ('s_rt', timeStr),
WMS.CPUTIME: ('h_cpu', timeStr)
}
# Restart jobs = no
params = ' -r n -notify'
if self._project:
params += ' -P %s' % self._project
# Job requirements
(queue, nodes) = (reqs.get(WMS.QUEUES, [''])[0], reqs.get(WMS.SITES))
if not nodes and queue:
params += ' -q %s' % queue
elif nodes and queue:
params += ' -q %s' % str.join(
',', imap(lambda node: '%s@%s' % (queue, node), nodes))
elif nodes:
raise ConfigError(
'Please also specify queue when selecting nodes!')
return params + PBSGECommon.getCommonSubmitArguments(
self, jobNum, jobName, reqs, sandbox, stdout, stderr, reqMap)
def __init__(self, config, name):
# Determine ROOT path from previous settings / environment / config file
self._rootpath = config.get('root path',
os.environ.get('ROOTSYS', ''),
persistent=True,
onChange=changeInitNeeded('sandbox'))
if not self._rootpath:
raise ConfigError(
'Either set environment variable "ROOTSYS" or set option "root path"!'
)
utils.vprint('Using the following ROOT path: %s' % self._rootpath, -1)
# Special handling for executables bundled with ROOT
self._executable = config.get('executable',
onChange=changeInitNeeded('sandbox'))
exeFull = os.path.join(self._rootpath, 'bin',
self._executable.lstrip('/'))
self.builtIn = os.path.exists(exeFull)
if self.builtIn:
config.set('send executable', 'False')
# store resolved built-in executable path?
# Apply default handling from UserTask
UserTask.__init__(self, config, name)
self.updateErrorDict(utils.pathShare('gc-run.root.sh'))
# Collect lib files needed by executable
self.libFiles = []
def fill_parameter_content(self, pnum, result):
output_tuple = self._helper.lookup(result)
if output_tuple is None:
return
elif len(output_tuple) != 1:
raise ConfigError("%s can't handle multiple lookup parameter sets!" % self.__class__.__name__)
elif output_tuple[0] is not None:
result[self._output_vn] = output_tuple[0]
def __init__(self, config, name):
DataTask.__init__(self, config, name)
# SCRAM settings
scramArchDefault = noDefault
scramProject = config.getList('scram project', [])
if scramProject: # manual scram setup
if len(scramProject) != 2:
raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
self._projectArea = None
self._projectAreaPattern = None
self._scramProject = scramProject[0]
self._scramProjectVersion = scramProject[1]
# ensure project area is not used
if 'project area' in config.getOptions():
raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))
else: # scram setup used from project area
self._projectArea = config.getPath('project area')
self._projectAreaPattern = config.getList('area files', ['-.*', '-config', 'bin', 'lib', 'python', 'module',
'*/data', '*.xml', '*.sql', '*.db', '*.cf[if]', '*.py', '-*/.git', '-*/.svn', '-*/CVS', '-*/work.*'])
logging.getLogger('user').info('Project area found in: %s', self._projectArea)
# try to determine scram settings from environment settings
scramPath = os.path.join(self._projectArea, '.SCRAM')
scramEnv = self._parse_scram_file(os.path.join(scramPath, 'Environment'))
try:
self._scramProject = scramEnv['SCRAM_PROJECTNAME']
self._scramProjectVersion = scramEnv['SCRAM_PROJECTVERSION']
except:
raise ConfigError('Installed program in project area not recognized.')
for arch_dir in sorted(ifilter(lambda dn: os.path.isdir(os.path.join(scramPath, dn)), os.listdir(scramPath))):
scramArchDefault = arch_dir
self._scramVersion = config.get('scram version', 'scramv1')
self._scramArch = config.get('scram arch', scramArchDefault)
self._scramReqs = []
if config.getBool('scram arch requirements', True, onChange = None):
self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramArch))
if config.getBool('scram project requirements', False, onChange = None):
self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramProject))
if config.getBool('scram project version requirements', False, onChange = None):
self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramProjectVersion))
def _get_lookup_args(pconfig, output_user, lookup_user_list):
# Transform output and lookup input: eg. key('A', 'B') -> ['A', 'B']
def _keys_to_vn_list(src):
result = []
src.fill_parameter_metadata(result)
return lmap(lambda meta: meta.value, result)
if isinstance(output_user, str):
output_vn = output_user
else:
output_vn = _keys_to_vn_list(output_user)[0]
if isinstance(lookup_user_list, str):
lookup_vn_list = lookup_user_list.split()
elif lookup_user_list is not None:
lookup_vn_list = _keys_to_vn_list(lookup_user_list)
else: # no lookup information given - query config for default lookup variable
lookup_vn_list = [pconfig.get('default lookup')]
if not lookup_vn_list or lookup_vn_list == ['']:
raise ConfigError('Lookup parameter not defined!')
# configure lookup matcher
name_matcher_default = pconfig.get('', 'default matcher', 'equal')
name_matcher_raw = pconfig.get(output_vn, 'matcher', name_matcher_default)
name_matcher_list = name_matcher_raw.lower().splitlines()
if len(
name_matcher_list
) == 1: # single matcher given - extend to same length as lookup_list
name_matcher_list = name_matcher_list * len(lookup_vn_list)
elif len(name_matcher_list) != len(lookup_vn_list):
raise ConfigError(
'Match-functions (length %d) and match-keys (length %d) do not match!'
% (len(name_matcher_list), len(lookup_vn_list)))
lookup_matcher_list = []
for name_matcher in name_matcher_list:
lookup_matcher_list.append(
Matcher.create_instance(name_matcher, pconfig, output_vn))
# configure lookup dictionary
(lookup_dict, lookup_order) = pconfig.get_parameter(output_vn)
if not pconfig.get_bool(output_vn, 'empty set', False):
for lookup_key in lookup_dict:
if len(lookup_dict[lookup_key]) == 0:
lookup_dict[lookup_key].append('')
return (output_vn, lookup_vn_list, lookup_matcher_list, lookup_dict,
lookup_order)
def _parse_scram_file(self, fn):
try:
fp = open(fn, 'r')
try:
return utils.DictFormat().parse(fp, keyParser = {None: str})
finally:
fp.close()
except Exception:
raise ConfigError('Project area file %s cannot be parsed!' % fn)
def _getConfigFiles(self, config):
cfgDefault = utils.QM(self.prolog.isActive() or self.epilog.isActive(),
[], noDefault)
for cfgFile in config.getPaths('config file',
cfgDefault,
mustExist=False):
if not os.path.exists(cfgFile):
raise ConfigError('Config file %r not found.' % cfgFile)
yield cfgFile
def fillParameterInfo(self, pNum, result):
lookupResult = self._matcher.lookup(result)
if lookupResult is None:
return
elif len(lookupResult) != 1:
raise ConfigError(
"%s can't handle multiple lookup parameter sets!" %
self.__class__.__name__)
elif lookupResult[0] is not None:
result[self._key] = lookupResult[0]
def __init__(self, config):
config.set('jobs', 'monitor', 'dashboard', override=False)
config.set('grid', 'sites', '-samtest -cmsprodhi', append=True)
site_db = CRIC()
token = AccessToken.create_instance('VomsProxy', create_config(),
'token')
self._hn_name = site_db.dn_to_username(token.get_fq_user_name())
if not self._hn_name:
raise ConfigError('Unable to map grid certificate to hn name!')
def substVars(self, inp, jobNum = None, addDict = None, check = True):
addDict = addDict or {}
allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
if jobNum is not None:
allVars.update(self.getJobConfig(jobNum))
subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)]))
result = subst(subst(str(inp)))
if check and self._varCheck.check(result):
raise ConfigError("'%s' contains invalid variable specifiers: '%s'" % (inp, result))
return result
def _setup(self, setup_key, setup_mod):
if setup_key:
(delim, ds, de) = utils.optSplit(setup_key, '::')
modifier = identity
if setup_mod and (setup_mod.strip() != 'value'):
try:
modifier = eval('lambda value: ' + setup_mod) # pylint:disable=eval-used
except Exception:
raise ConfigError('Unable to parse delimeter modifier %r' %
setup_mod)
return (delim, parseStr(ds, int), parseStr(de, int), modifier)
def __init__(self, jobDB, task, jobs = None, configString = ''):
Report.__init__(self, jobDB, task, jobs, configString)
self._levelMap = {'wms': 2, 'endpoint': 3, 'site': 4, 'queue': 5}
self._useHistory = ('history' in configString)
configString = configString.replace('history', '')
self._idxList = lmap(lambda x: self._levelMap[x.lower()], configString.split())
self._idxList.reverse()
if not self._idxList:
raise ConfigError('Backend report was not configured!')
self._stateMap = [(None, 'WAITING'), (Job.RUNNING, 'RUNNING'),
(Job.FAILED, 'FAILED'), (Job.SUCCESS, 'SUCCESS')]