def __call__(self, config, old_obj, cur_obj, cur_entry, obj2str):
log = logging.getLogger('config.onchange.%s' % self._option.lower())
config = config.changeView(setSections=['interactive'])
interaction_def = config.getBool('default', True, onChange=None)
interaction_opt = config.getBool(self._option,
interaction_def,
onChange=None)
if interaction_opt:
msg = 'The option "%s" was changed from the old value:' % cur_entry.format_opt(
)
if utils.getUserBool(
msg +
('\n\t> %s\nto the new value:' % obj2str(old_obj).lstrip()) +
('\n\t> %s\nDo you want to abort?' %
obj2str(cur_obj).lstrip()), False):
raise ConfigError('Abort due to unintentional config change!')
if not utils.getUserBool(
'A partial reinitialization (same as --reinit %s) is needed to apply this change! Do you want to continue?'
% self._option, True):
log.log(logging.INFO1, 'Using stored value %s for option %s',
obj2str(old_obj), cur_entry.format_opt())
return old_obj
config.setState(True, 'init', detail=self._option)
config.setState(
True, 'init',
detail='config') # This will trigger a write of the new options
return cur_obj
def cancel(self, wms, jobs, interactive, showJobs):
if len(jobs) == 0:
return
if showJobs:
self._reportClass(self.jobDB, self._task, jobs).display()
if interactive and not utils.getUserBool(
'Do you really want to cancel these jobs?', True):
return
def mark_cancelled(jobNum):
jobObj = self.jobDB.get(jobNum)
if jobObj is None:
return
self._update(jobObj, jobNum, Job.CANCELLED)
self._eventhandler.onJobUpdate(wms, jobObj, jobNum,
{'reason': 'cancelled'})
jobs.reverse()
for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)):
# Remove deleted job from todo list and mark as cancelled
assert (self.jobDB.get(jobNum).wmsId == wmsId)
jobs.remove(jobNum)
mark_cancelled(jobNum)
if len(jobs) > 0:
self._log_user.warning(
'There was a problem with cancelling the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if (interactive and utils.getUserBool(
'Do you want to mark them as cancelled?',
True)) or not interactive:
lmap(mark_cancelled, jobs)
if interactive:
utils.wait(2)
def cancel(self, wms, jobs, interactive = False, showJobs = True):
if len(jobs) == 0:
return
if showJobs:
self._reportClass(self.jobDB, self._task, jobs).display()
if interactive and not utils.getUserBool('Do you really want to cancel these jobs?', True):
return
def mark_cancelled(jobNum):
jobObj = self.jobDB.get(jobNum)
if jobObj is None:
return
self._update(jobObj, jobNum, Job.CANCELLED)
self._eventhandler.onJobUpdate(wms, jobObj, jobNum, {'reason': 'cancelled'})
jobs.reverse()
for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)):
# Remove deleted job from todo list and mark as cancelled
assert(self.jobDB.get(jobNum).wmsId == wmsId)
jobs.remove(jobNum)
mark_cancelled(jobNum)
if len(jobs) > 0:
self._log_user.warning('There was a problem with cancelling the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if (interactive and utils.getUserBool('Do you want to mark them as cancelled?', True)) or not interactive:
lmap(mark_cancelled, jobs)
if interactive:
utils.wait(2)
def __call__(self, config, old_obj, cur_obj, cur_entry, obj2str):
log = logging.getLogger('config.onchange.%s' % self._option.lower())
if config.isInteractive(self._option, default = True):
msg = 'The option "%s" was changed from the old value:' % cur_entry.format_opt()
if utils.getUserBool(msg + ('\n\t> %s\nto the new value:' % obj2str(old_obj).lstrip()) +
('\n\t> %s\nDo you want to abort?' % obj2str(cur_obj).lstrip()), False):
raise ConfigError('Abort due to unintentional config change!')
if not utils.getUserBool('A partial reinitialization (same as --reinit %s) is needed to apply this change! Do you want to continue?' % self._option, True):
log.log(logging.INFO1, 'Using stored value %s for option %s', obj2str(old_obj), cur_entry.format_opt())
return old_obj
config.setState(True, 'init', detail = self._option)
config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options
return cur_obj
def __call__(self, config, old_obj, cur_obj, cur_entry, obj2str):
log = logging.getLogger('config.onChange.%s' % self._option)
config = config.changeView(setSections = ['interactive'])
interaction_def = config.getBool('default', True, onChange = None)
interaction_opt = config.getBool(self._option, interaction_def, onChange = None)
if interaction_opt:
if utils.getUserBool('The option "%s" was changed from the old value:' % cur_entry.format_opt() +
'\n\t%s\nto the new value:\n\t%s\nDo you want to abort?' % (obj2str(old_obj), obj2str(cur_obj)), False):
raise ConfigError('Abort due to unintentional config change!')
if not utils.getUserBool('A partial reinitialization (same as --reinit %s) is needed to apply this change! Do you want to continue?' % self._option, True):
log.log(logging.INFO1, 'Using stored value %s for option %s' % (obj2str(old_obj), cur_entry.format_opt()))
return old_obj
config.setState(True, 'init', detail = self._option)
config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options
return cur_obj
def makeJDL(self, jobNum, module):
cfgPath = os.path.join(self._jobPath, 'job_%d.var' % jobNum)
sbIn = lmap(lambda d_s_t: d_s_t[1], self._getSandboxFilesIn(module))
sbOut = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(module))
wcList = lfilter(lambda x: '*' in x, sbOut)
if len(wcList):
self._writeJobConfig(cfgPath, jobNum, module, {'GC_WC': str.join(' ', wcList)})
sandboxOutJDL = lfilter(lambda x: x not in wcList, sbOut) + ['GC_WC.tar.gz']
else:
self._writeJobConfig(cfgPath, jobNum, module, {})
sandboxOutJDL = sbOut
# Warn about too large sandboxes
sbSizes = lmap(os.path.getsize, sbIn)
if sbSizes and (self._warnSBSize > 0) and (sum(sbSizes) > self._warnSBSize * 1024 * 1024):
if not utils.getUserBool('Sandbox is very large (%d bytes) and can cause issues with the WMS! Do you want to continue?' % sum(sbSizes), False):
sys.exit(os.EX_OK)
self._warnSBSize = 0
reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum), WMS.SITES)
formatStrList = lambda strList: '{ %s }' % str.join(', ', imap(lambda x: '"%s"' % x, strList))
contents = {
'Executable': '"gc-run.sh"',
'Arguments': '"%d"' % jobNum,
'StdOutput': '"gc.stdout"',
'StdError': '"gc.stderr"',
'InputSandbox': formatStrList(sbIn + [cfgPath]),
'OutputSandbox': formatStrList(sandboxOutJDL),
'VirtualOrganisation': '"%s"' % self.vo,
'Rank': '-other.GlueCEStateEstimatedResponseTime',
'RetryCount': 2
}
return self._jdl_writer.format(reqs, contents)
def _findCollision(self,
tName,
nameDict,
varDict,
hashKeys,
keyFmt,
nameFmt=identity):
dupesDict = {}
for (key, name) in nameDict.items():
dupesDict.setdefault(nameFmt(name), []).append(keyFmt(name, key))
ask = True
for name, key_list in sorted(dupesDict.items()):
if len(key_list) > 1:
self._log.warn('Multiple %s keys are mapped to the name %s!',
tName, repr(name))
for key in sorted(key_list):
self._log.warn('\t%s hash %s using:', tName,
str.join('#', key))
for var, value in self._getFilteredVarDict(
varDict, key, hashKeys).items():
self._log.warn('\t\t%s = %s', var, value)
if ask and not utils.getUserBool('Do you want to continue?',
False):
sys.exit(os.EX_OK)
ask = False
def doTransfer(self, listDescSourceTarget):
for (desc, source, target) in listDescSourceTarget:
if not self.smPaths:
raise ConfigError(
"%s can't be transferred because '%s path wasn't set" %
(desc, self.smOptPrefix))
for idx, sePath in enumerate(set(self.smPaths)):
utils.vprint('Copy %s to SE %d ' % (desc, idx + 1),
-1,
newline=False)
sys.stdout.flush()
proc = se_copy(source, os.path.join(sePath, target),
self.smForce)
if proc.status(timeout=5 * 60, terminate=True) == 0:
utils.vprint('finished', -1)
else:
utils.vprint('failed', -1)
utils.eprint(proc.stderr.read(timeout=0))
utils.eprint(
'Unable to copy %s! You can try to copy it manually.' %
desc)
if not utils.getUserBool(
'Is %s (%s) available on SE %s?' %
(desc, source, sePath), False):
raise StorageError('%s is missing on SE %s!' %
(desc, sePath))
def doTransfer(self, listDescSourceTarget):
for (desc, source, target) in listDescSourceTarget:
if not self.smPaths:
raise ConfigError(
"%s can't be transferred because '%s path wasn't set" %
(desc, self.smOptPrefix))
for idx, sePath in enumerate(set(self.smPaths)):
activity = Activity('Copy %s to SE %d ' % (desc, idx + 1))
proc = se_copy(source, os.path.join(sePath, target),
self.smForce)
proc.status(timeout=5 * 60, terminate=True)
activity.finish()
if proc.status(timeout=0) == 0:
self._log.info('Copy %s to SE %d finished', desc, idx + 1)
else:
self._log.info('Copy %s to SE %d failed', desc, idx + 1)
self._log.critical(proc.stderr.read(timeout=0))
self._log.critical(
'Unable to copy %s! You can try to copy it manually.',
desc)
if not utils.getUserBool(
'Is %s (%s) available on SE %s?' %
(desc, source, sePath), False):
raise StorageError('%s is missing on SE %s!' %
(desc, sePath))
def _processConfigFiles(self, config, cfgFiles, fragment_path, autoPrepare,
mustPrepare):
# process list of uninitialized config files
for (cfg, cfg_new,
doPrepare) in self._cfgFindUninitialized(config, cfgFiles,
autoPrepare,
mustPrepare):
if doPrepare and (autoPrepare or utils.getUserBool(
'Do you want to prepare %s for running over the dataset?' %
cfg, True)):
self._cfgStore(cfg, cfg_new, fragment_path)
else:
self._cfgStore(cfg, cfg_new)
result = []
for cfg in cfgFiles:
cfg_new = config.getWorkPath(os.path.basename(cfg))
if not os.path.exists(cfg_new):
raise ConfigError(
'Config file %r was not copied to the work directory!' %
cfg)
isInstrumented = self._cfgIsInstrumented(cfg_new)
if mustPrepare and not isInstrumented:
raise ConfigError(
'Config file %r must use %s to work properly!' %
(cfg,
str.join(', ',
imap(lambda x: '@%s@' % x, self.neededVars()))))
if autoPrepare and not isInstrumented:
self._log.warning('Config file %r was not instrumented!', cfg)
result.append(cfg_new)
return result
def resyncMapping(self, newSplitPath, oldBlocks, newBlocks):
log = utils.ActivityLog('Performing resynchronization of dataset')
(blocksAdded, blocksMissing,
blocksMatching) = DataProvider.resyncSources(oldBlocks, newBlocks)
for rmBlock in blocksMissing: # Files in matching blocks are already sorted
sort_inplace(rmBlock[DataProvider.FileList],
key=lambda x: x[DataProvider.URL])
log.finish()
# User overview and setup starts here
resultRedo = []
resultDisable = []
newSplitPathTMP = newSplitPath + '.tmp'
resyncIter = self._resyncIterator(resultRedo, resultDisable,
blocksAdded, blocksMissing,
blocksMatching)
self.savePartitions(
newSplitPathTMP,
resyncIter,
sourceLen=self.getMaxJobs(),
message=
'Performing resynchronization of dataset map (progress is estimated)'
)
if self._interactive:
# TODO: print info and ask
if not utils.getUserBool(
'Do you want to use the new dataset partition?', False):
return None
os.rename(newSplitPathTMP, newSplitPath)
return (resultRedo, resultDisable)
def instrumentCfgQueue(self, cfgFiles, fragment, mustPrepare = False):
def isInstrumented(cfgName):
cfg = open(cfgName, 'r').read()
for tag in self.neededVars():
if (not '__%s__' % tag in cfg) and (not '@%s@' % tag in cfg):
return False
return True
def doInstrument(cfgName):
if not isInstrumented(cfgName) or 'customise_for_gc' not in open(cfgName, 'r').read():
utils.vprint('Instrumenting...', os.path.basename(cfgName), -1)
open(cfgName, 'a').write(open(fragment, 'r').read())
else:
utils.vprint('%s already contains customise_for_gc and all needed variables' % os.path.basename(cfgName), -1)
cfgStatus = []
comPath = os.path.dirname(os.path.commonprefix(cfgFiles))
for cfg in cfgFiles:
cfgStatus.append({0: cfg.split(comPath, 1)[1].lstrip('/'), 1: str(isInstrumented(cfg)), 2: cfg})
utils.printTabular([(0, 'Config file'), (1, 'Instrumented')], cfgStatus, 'lc')
for cfg in cfgFiles:
if self.prepare or not isInstrumented(cfg):
if self.prepare or utils.getUserBool('Do you want to prepare %s for running over the dataset?' % cfg, True):
doInstrument(cfg)
if mustPrepare and not (True in map(isInstrumented, cfgFiles)):
raise ConfigError('A config file must use %s to work properly!' %
str.join(', ', map(lambda x: '@%s@' % x, self.neededVars())))
def reset(self, wms, select):
jobs = self.jobDB.getJobs(JobSelector.create(select, task = self._task))
if jobs:
self._log_user.warning('Resetting the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if utils.getUserBool('Are you sure you want to reset the state of these jobs?', False):
self.cancel(wms, self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING), jobs), False, False)
for jobNum in jobs:
self.jobDB.commit(jobNum, Job())
def __init__(self, config, source):
self._rawSource = source
BasicParameterAdapter.__init__(self, config, source)
self._mapJob2PID = {}
if not os.path.isdir(config.getWorkPath()):
os.makedirs(config.getWorkPath())
self._pathJob2PID = config.getWorkPath('params.map.gz')
self._pathParams = config.getWorkPath('params.dat.gz')
# Find out if init should be performed - overrides userResync!
userInit = config.getState('init', detail='parameters')
needInit = False
if not (os.path.exists(self._pathParams)
and os.path.exists(self._pathJob2PID)):
needInit = True # Init needed if no parameter log exists
if userInit and not needInit and (source.getMaxParameters()
is not None):
utils.eprint(
'Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!'
)
if utils.getUserBool(
'Do you want to perform a syncronization between the current mapping and the new one to avoid this?',
True):
userInit = False
doInit = userInit or needInit
# Find out if resync should be performed
userResync = config.getState('resync', detail='parameters')
config.setState(False, 'resync', detail='parameters')
needResync = False
pHash = self._rawSource.getHash()
self.storedHash = config.get('parameter hash', pHash, persistent=True)
if self.storedHash != pHash:
needResync = True # Resync needed if parameters have changed
self._log.info('Parameter hash has changed')
self._log.debug('\told hash: %s', self.storedHash)
self._log.debug('\tnew hash: %s', pHash)
config.setState(True, 'init', detail='config')
doResync = (userResync or needResync) and not doInit
if not doResync and not doInit: # Reuse old mapping
activity = utils.ActivityLog(
'Loading cached parameter information')
self.readJob2PID()
activity.finish()
return
elif doResync: # Perform sync
activity = utils.ActivityLog('Syncronizing parameter information')
self.storedHash = None
self._resyncState = self.resync()
activity.finish()
elif doInit: # Write current state
self.writeJob2PID(self._pathJob2PID)
ParameterSource.getClass('GCDumpParameterSource').write(
self._pathParams, self)
config.set('parameter hash', self._rawSource.getHash())
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('dataset processor', 'LumiDataProcessor', '+=')
config.set('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor ' +
'LFNPartitionProcessor LumiPartitionProcessor CMSSWPartitionProcessor')
dash_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dashboard'])
dash_config.set('application', 'cmsRun')
SCRAMTask.__init__(self, config, name)
if self._scramProject != 'CMSSW':
raise ConfigError('Project area contains no CMSSW project')
self._oldReleaseTop = None
if self._projectArea:
self._oldReleaseTop = self._parse_scram_file(os.path.join(self._projectArea, '.SCRAM', self._scramArch, 'Environment')).get('RELEASETOP', None)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
self._projectAreaTarballSE = config.getBool(['se runtime', 'se project area'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self._dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0') # this can be a variable like @USER_EVENTS@!
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self._dataSplitter is not None))
# Create project area tarball
if self._projectArea and not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self._projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self._projectArea, self._projectAreaPattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('dataset processor', 'LumiDataProcessor', '+=')
config.set('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor ' +
'LFNPartitionProcessor LumiPartitionProcessor CMSSWPartitionProcessor')
dash_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dashboard'])
dash_config.set('application', 'cmsRun')
SCRAMTask.__init__(self, config, name)
if self._scramProject != 'CMSSW':
raise ConfigError('Project area contains no CMSSW project')
self._oldReleaseTop = None
if self._projectArea:
self._oldReleaseTop = self._parse_scram_file(os.path.join(self._projectArea, '.SCRAM', self._scramArch, 'Environment')).get('RELEASETOP', None)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
self._projectAreaTarballSE = config.getBool(['se runtime', 'se project area'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self._dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0') # this can be a variable like @USER_EVENTS@!
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self._dataSplitter is not None))
# Create project area tarball
if self._projectArea and not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self._projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self._projectArea, self._projectAreaPattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def reset(self, wms, select):
jobs = self.jobDB.getJobs(JobSelector.create(select, task=self._task))
if jobs:
self._log_user.warning('Resetting the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if utils.getUserBool(
'Are you sure you want to reset the state of these jobs?',
False):
self.cancel(
wms,
self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING),
jobs), False, False)
for jobNum in jobs:
self.jobDB.commit(jobNum, Job())
def _findCollision(self, tName, nameDict, varDict, hashKeys, keyFmt, nameFmt = identity):
dupesDict = {}
for (key, name) in nameDict.items():
dupesDict.setdefault(nameFmt(name), []).append(keyFmt(name, key))
ask = True
for name, key_list in sorted(dupesDict.items()):
if len(key_list) > 1:
self._log.warn('Multiple %s keys are mapped to the name %s!', tName, repr(name))
for key in sorted(key_list):
self._log.warn('\t%s hash %s using:', tName, str.join('#', key))
for var, value in self._getFilteredVarDict(varDict, key, hashKeys).items():
self._log.warn('\t\t%s = %s', var, value)
if ask and not utils.getUserBool('Do you want to continue?', False):
sys.exit(os.EX_OK)
ask = False
def __init__(self, config, source):
self._rawSource = source
BasicParameterAdapter.__init__(self, config, source)
self._mapJob2PID = {}
if not os.path.isdir(config.getWorkPath()):
os.makedirs(config.getWorkPath())
self._pathJob2PID = config.getWorkPath('params.map.gz')
self._pathParams = config.getWorkPath('params.dat.gz')
# Find out if init should be performed - overrides userResync!
userInit = config.getState('init', detail = 'parameters')
needInit = False
if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)):
needInit = True # Init needed if no parameter log exists
if userInit and not needInit and (source.getMaxParameters() is not None):
utils.eprint('Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!')
if utils.getUserBool('Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True):
userInit = False
doInit = userInit or needInit
# Find out if resync should be performed
userResync = config.getState('resync', detail = 'parameters')
config.setState(False, 'resync', detail = 'parameters')
needResync = False
pHash = self._rawSource.getHash()
self._storedHash = config.get('parameter hash', pHash, persistent = True)
if self._storedHash != pHash:
needResync = True # Resync needed if parameters have changed
self._log.info('Parameter hash has changed')
self._log.debug('\told hash: %s', self._storedHash)
self._log.debug('\tnew hash: %s', pHash)
config.setState(True, 'init', detail = 'config')
doResync = (userResync or needResync) and not doInit
if not doResync and not doInit: # Reuse old mapping
activity = utils.ActivityLog('Loading cached parameter information')
self._readJob2PID()
activity.finish()
return
elif doResync: # Perform sync
activity = utils.ActivityLog('Syncronizing parameter information')
self._storedHash = None
self._resyncState = self.resync()
activity.finish()
elif doInit: # Write current state
self._writeJob2PID(self._pathJob2PID)
ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams, self)
config.set('parameter hash', self._rawSource.getHash())
def findCollision(tName, nameDict, varDict, hashKeys, keyFmt = lambda x: x):
targetNames = nameDict.values()
for name in list(set(targetNames)):
targetNames.remove(name)
if len(targetNames):
ask = True
for name in targetNames:
utils.eprint("Multiple %s keys are mapped to the same %s name '%s'!" % (tName, tName, keyFmt(name)))
for key in nameDict:
if nameDict[key] == name:
utils.eprint('\t%s hash %s using:' % (tName, keyFmt(key)))
for x in filter(lambda (k, v): k in hashKeys, varDict[keyFmt(key)].items()):
utils.eprint('\t\t%s = %s' % x)
if ask and not utils.getUserBool('Do you want to continue?', False):
sys.exit(0)
ask = False
def doTransfer(self, listDescSourceTarget):
for (desc, source, target) in listDescSourceTarget:
if not self.smPaths:
raise ConfigError("%s can't be transferred because '%s path wasn't set" % (desc, self.smOptPrefix))
for idx, sePath in enumerate(set(self.smPaths)):
utils.vprint('Copy %s to SE %d ' % (desc, idx + 1), -1, newline = False)
sys.stdout.flush()
proc = se_copy(source, os.path.join(sePath, target), self.smForce)
if proc.status(timeout = 5*60, terminate = True) == 0:
utils.vprint('finished', -1)
else:
utils.vprint('failed', -1)
utils.eprint(proc.stderr.read(timeout = 0))
utils.eprint('Unable to copy %s! You can try to copy it manually.' % desc)
if not utils.getUserBool('Is %s (%s) available on SE %s?' % (desc, source, sePath), False):
raise StorageError('%s is missing on SE %s!' % (desc, sePath))
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('partition processor', 'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor')
config.set('dataset processor', 'LumiDataProcessor', '+=')
DataTask.__init__(self, config, name)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
# SCRAM settings
self._configureSCRAMSettings(config)
self.useReqs = config.getBool('software requirements', True, onChange = None)
self._projectAreaTarballSE = config.getBool(['se project area', 'se runtime'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self.dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0')
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self.dataSplitter is not None))
# Create project area tarball
if not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self.projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self.projectArea, self.pattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def doTransfer(self, listDescSourceTarget):
for (desc, source, target) in listDescSourceTarget:
if not self.smPaths:
raise ConfigError("%s can't be transferred because '%s path wasn't set" % (desc, self.smOptPrefix))
for idx, sePath in enumerate(set(self.smPaths)):
activity = Activity('Copy %s to SE %d ' % (desc, idx + 1))
proc = se_copy(source, os.path.join(sePath, target), self.smForce)
proc.status(timeout = 5*60, terminate = True)
activity.finish()
if proc.status(timeout = 0) == 0:
self._log.info('Copy %s to SE %d finished', desc, idx + 1)
else:
self._log.info('Copy %s to SE %d failed', desc, idx + 1)
self._log.critical(proc.stderr.read(timeout = 0))
self._log.critical('Unable to copy %s! You can try to copy it manually.', desc)
if not utils.getUserBool('Is %s (%s) available on SE %s?' % (desc, source, sePath), False):
raise StorageError('%s is missing on SE %s!' % (desc, sePath))
def makeJDL(self, jobNum, module):
cfgPath = os.path.join(self._jobPath, 'job_%d.var' % jobNum)
sbIn = lmap(lambda d_s_t: d_s_t[1], self._getSandboxFilesIn(module))
sbOut = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(module))
wcList = lfilter(lambda x: '*' in x, sbOut)
if len(wcList):
self._writeJobConfig(cfgPath, jobNum, module,
{'GC_WC': str.join(' ', wcList)})
sandboxOutJDL = lfilter(lambda x: x not in wcList,
sbOut) + ['GC_WC.tar.gz']
else:
self._writeJobConfig(cfgPath, jobNum, module, {})
sandboxOutJDL = sbOut
# Warn about too large sandboxes
sbSizes = lmap(os.path.getsize, sbIn)
if sbSizes and (self._warnSBSize >
0) and (sum(sbSizes) > self._warnSBSize * 1024 * 1024):
if not utils.getUserBool(
'Sandbox is very large (%d bytes) and can cause issues with the WMS! Do you want to continue?'
% sum(sbSizes), False):
sys.exit(os.EX_OK)
self._warnSBSize = 0
reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum),
WMS.SITES)
formatStrList = lambda strList: '{ %s }' % str.join(
', ', imap(lambda x: '"%s"' % x, strList))
contents = {
'Executable': '"gc-run.sh"',
'Arguments': '"%d"' % jobNum,
'StdOutput': '"gc.stdout"',
'StdError': '"gc.stderr"',
'InputSandbox': formatStrList(sbIn + [cfgPath]),
'OutputSandbox': formatStrList(sandboxOutJDL),
'Requirements': self._formatRequirements(reqs),
'VirtualOrganisation': '"%s"' % self.vo,
'Rank': '-other.GlueCEStateEstimatedResponseTime',
'RetryCount': 2
}
cpus = dict(reqs).get(WMS.CPUS, 1)
if cpus > 1:
contents['CpuNumber'] = cpus
return utils.DictFormat(' = ').format(contents, format='%s%s%s;\n')
def _processConfigFiles(self, config, cfgFiles, fragment_path, autoPrepare, mustPrepare):
# process list of uninitialized config files
for (cfg, cfg_new, doPrepare) in self._cfgFindUninitialized(config, cfgFiles, autoPrepare, mustPrepare):
if doPrepare and (autoPrepare or utils.getUserBool('Do you want to prepare %s for running over the dataset?' % cfg, True)):
self._cfgStore(cfg, cfg_new, fragment_path)
else:
self._cfgStore(cfg, cfg_new)
result = []
for cfg in cfgFiles:
cfg_new = config.getWorkPath(os.path.basename(cfg))
if not os.path.exists(cfg_new):
raise ConfigError('Config file %r was not copied to the work directory!' % cfg)
isInstrumented = self._cfgIsInstrumented(cfg_new)
if mustPrepare and not isInstrumented:
raise ConfigError('Config file %r must use %s to work properly!' %
(cfg, str.join(', ', imap(lambda x: '@%s@' % x, self.neededVars()))))
if autoPrepare and not isInstrumented:
self._log.warning('Config file %r was not instrumented!', cfg)
result.append(cfg_new)
return result
def resyncMapping(self, newSplitPath, oldBlocks, newBlocks):
activity = Activity('Performing resynchronization of dataset')
(blocksAdded, blocksMissing, blocksMatching) = DataProvider.resyncSources(oldBlocks, newBlocks)
for rmBlock in blocksMissing: # Files in matching blocks are already sorted
sort_inplace(rmBlock[DataProvider.FileList], key = lambda x: x[DataProvider.URL])
activity.finish()
# User overview and setup starts here
resultRedo = []
resultDisable = []
newSplitPathTMP = newSplitPath + '.tmp'
resyncIter = self._resyncIterator(resultRedo, resultDisable, blocksAdded, blocksMissing, blocksMatching)
self.savePartitions(newSplitPathTMP, resyncIter, sourceLenHint = self.getMaxJobs(),
message = 'Performing resynchronization of dataset map (progress is estimated)')
if self._interactive:
# TODO: print info and ask
if not utils.getUserBool('Do you want to use the new dataset partition?', False):
return
os.rename(newSplitPathTMP, newSplitPath)
return (resultRedo, resultDisable)
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set(
'partition processor',
'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor'
)
config.set('dataset processor', 'LumiDataProcessor', '+=')
DataTask.__init__(self, config, name)
self.updateErrorDict(
utils.pathShare('gc-run.cmssw.sh', pkg='grid_control_cms'))
# SCRAM settings
self._configureSCRAMSettings(config)
self.useReqs = config.getBool('software requirements',
True,
onChange=None)
self._projectAreaTarballSE = config.getBool(
['se project area', 'se runtime'], True)
self._projectAreaTarball = config.getWorkPath(
'cmssw-project-area.tar.gz')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError(
'Prefix executable and argument options with either prolog or epilog!'
)
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self.dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0')
fragment = config.getPath(
'instrumentation fragment',
utils.pathShare('fragmentForCMSSW.py', pkg='grid_control_cms'))
self.configFiles = self._processConfigFiles(
config,
list(self._getConfigFiles(config)),
fragment,
autoPrepare=config.getBool('instrumentation', True),
mustPrepare=(self.dataSplitter is not None))
# Create project area tarball
if not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail='sandbox')
if config.getState('init', detail='sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool(
'CMSSW tarball already exists! Do you want to regenerate it?',
True):
return
# Generate CMSSW tarball
if self.projectArea:
utils.genTarball(
self._projectAreaTarball,
utils.matchFiles(self.projectArea, self.pattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail='storage')
def __init__(self, config, name):
config.set('se input timeout', '0:30', override = False)
config.set('dataset provider', 'DBS3Provider', override = False)
config.set('dataset splitter', 'EventBoundarySplitter', override = False)
DataTask.__init__(self, config, name)
self.errorDict.update(dict(self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))))
# SCRAM info
scramProject = config.getList('scram project', [])
if len(scramProject):
self.projectArea = config.getPath('project area', '')
if len(self.projectArea):
raise ConfigError('Cannot specify both SCRAM project and project area')
if len(scramProject) != 2:
raise ConfigError('SCRAM project needs exactly 2 arguments: PROJECT VERSION')
else:
self.projectArea = config.getPath('project area')
# This works in tandem with provider_dbsv2.py !
self.selectedLumis = parseLumiFilter(config.get('lumi filter', ''))
self.useReqs = config.getBool('software requirements', True, onChange = None)
self.seRuntime = config.getBool('se runtime', False)
self.runtimePath = config.getWorkPath('runtime.tar.gz')
if len(self.projectArea):
defaultPattern = '-.* -config bin lib python module */data *.xml *.sql *.cf[if] *.py -*/.git -*/.svn -*/CVS -*/work.*'
self.pattern = config.getList('area files', defaultPattern.split())
if os.path.exists(self.projectArea):
utils.vprint('Project area found in: %s' % self.projectArea, -1)
else:
raise ConfigError('Specified config area %r does not exist!' % self.projectArea)
scramPath = os.path.join(self.projectArea, '.SCRAM')
# try to open it
try:
fp = open(os.path.join(scramPath, 'Environment'), 'r')
self.scramEnv = utils.DictFormat().parse(fp, keyParser = {None: str})
except:
raise ConfigError('Project area file %s/.SCRAM/Environment cannot be parsed!' % self.projectArea)
for key in ['SCRAM_PROJECTNAME', 'SCRAM_PROJECTVERSION']:
if key not in self.scramEnv:
raise ConfigError('Installed program in project area not recognized.')
archs = filter(lambda x: os.path.isdir(os.path.join(scramPath, x)) and not x.startswith('.'), os.listdir(scramPath))
self.scramArch = config.get('scram arch', (archs + [noDefault])[0])
try:
fp = open(os.path.join(scramPath, self.scramArch, 'Environment'), 'r')
self.scramEnv.update(utils.DictFormat().parse(fp, keyParser = {None: str}))
except:
raise ConfigError('Project area file .SCRAM/%s/Environment cannot be parsed!' % self.scramArch)
else:
self.scramEnv = {
'SCRAM_PROJECTNAME': scramProject[0],
'SCRAM_PROJECTVERSION': scramProject[1]
}
self.scramArch = config.get('scram arch')
self.scramVersion = config.get('scram version', 'scramv1')
if self.scramEnv['SCRAM_PROJECTNAME'] != 'CMSSW':
raise ConfigError('Project area not a valid CMSSW project area.')
# Information about search order for software environment
self.searchLoc = []
if config.getState('sandbox'):
userPath = config.get('cmssw dir', '')
if userPath != '':
self.searchLoc.append(('CMSSW_DIR_USER', userPath))
if self.scramEnv.get('RELEASETOP', None):
projPath = os.path.normpath('%s/../../../../' % self.scramEnv['RELEASETOP'])
self.searchLoc.append(('CMSSW_DIR_PRO', projPath))
if len(self.searchLoc):
utils.vprint('Local jobs will try to use the CMSSW software located here:', -1)
for i, loc in enumerate(self.searchLoc):
key, value = loc
utils.vprint(' %i) %s' % (i + 1, value), -1)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
self.configFiles = []
cfgDefault = QM(self.prolog.isActive() or self.epilog.isActive(), [], noDefault)
for cfgFile in config.getPaths('config file', cfgDefault, mustExist = False):
newPath = config.getWorkPath(os.path.basename(cfgFile))
if not os.path.exists(newPath):
if not os.path.exists(cfgFile):
raise ConfigError('Config file %r not found.' % cfgFile)
shutil.copyfile(cfgFile, newPath)
self.configFiles.append(newPath)
# Check that for dataset jobs the necessary placeholders are in the config file
self.prepare = config.getBool('prepare config', False)
fragment = config.getPath('instrumentation fragment',
os.path.join('packages', 'grid_control_cms', 'share', 'fragmentForCMSSW.py'))
if self.dataSplitter != None:
if config.getState('sandbox'):
if len(self.configFiles) > 0:
self.instrumentCfgQueue(self.configFiles, fragment, mustPrepare = True)
else:
self.eventsPerJob = config.get('events per job', '0')
if config.getState(detail = 'sandbox') and self.prepare:
self.instrumentCfgQueue(self.configFiles, fragment)
if not os.path.exists(config.getWorkPath('runtime.tar.gz')):
config.setState(True, detail = 'sandbox')
if config.getState(detail = 'sandbox'):
if os.path.exists(config.getWorkPath('runtime.tar.gz')):
if not utils.getUserBool('Runtime already exists! Do you want to regenerate CMSSW tarball?', True):
return
# Generate runtime tarball (and move to SE)
if self.projectArea:
utils.genTarball(config.getWorkPath('runtime.tar.gz'), utils.matchFiles(self.projectArea, self.pattern))
if self.seRuntime:
config.setState(True, detail = 'storage')
