def deployTask(self, task, monitor):
self.outputFiles = lmap(lambda d_s_t: d_s_t[2],
self._getSandboxFilesOut(task)) # HACK
task.validateVariables()
self.smSEIn.addFiles(
lmap(lambda d_s_t: d_s_t[2],
task.getSEInFiles())) # add task SE files to SM
# Transfer common SE files
if self.config.getState('init', detail='storage'):
self.smSEIn.doTransfer(task.getSEInFiles())
def convert(fnList):
for fn in fnList:
if isinstance(fn, str):
yield (fn, os.path.basename(fn), False)
else:
yield (None, os.path.basename(fn.name), fn)
# Package sandbox tar file
self._log.log(logging.INFO1, 'Packing sandbox')
sandbox = self._getSandboxName(task)
utils.ensureDirExists(os.path.dirname(sandbox), 'sandbox directory')
if not os.path.exists(sandbox) or self.config.getState(
'init', detail='sandbox'):
utils.genTarball(
sandbox,
convert(
self._getSandboxFiles(task, monitor,
[self.smSEIn, self.smSEOut])))
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('dataset processor', 'LumiDataProcessor', '+=')
config.set('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor ' +
'LFNPartitionProcessor LumiPartitionProcessor CMSSWPartitionProcessor')
dash_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dashboard'])
dash_config.set('application', 'cmsRun')
SCRAMTask.__init__(self, config, name)
if self._scramProject != 'CMSSW':
raise ConfigError('Project area contains no CMSSW project')
self._oldReleaseTop = None
if self._projectArea:
self._oldReleaseTop = self._parse_scram_file(os.path.join(self._projectArea, '.SCRAM', self._scramArch, 'Environment')).get('RELEASETOP', None)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
self._projectAreaTarballSE = config.getBool(['se runtime', 'se project area'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self._dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0') # this can be a variable like @USER_EVENTS@!
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self._dataSplitter is not None))
# Create project area tarball
if self._projectArea and not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self._projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self._projectArea, self._projectAreaPattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('dataset processor', 'LumiDataProcessor', '+=')
config.set('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor ' +
'LFNPartitionProcessor LumiPartitionProcessor CMSSWPartitionProcessor')
dash_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dashboard'])
dash_config.set('application', 'cmsRun')
SCRAMTask.__init__(self, config, name)
if self._scramProject != 'CMSSW':
raise ConfigError('Project area contains no CMSSW project')
self._oldReleaseTop = None
if self._projectArea:
self._oldReleaseTop = self._parse_scram_file(os.path.join(self._projectArea, '.SCRAM', self._scramArch, 'Environment')).get('RELEASETOP', None)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
self._projectAreaTarballSE = config.getBool(['se runtime', 'se project area'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self._dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0') # this can be a variable like @USER_EVENTS@!
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self._dataSplitter is not None))
# Create project area tarball
if self._projectArea and not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self._projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self._projectArea, self._projectAreaPattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('partition processor', 'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor')
config.set('dataset processor', 'LumiDataProcessor', '+=')
DataTask.__init__(self, config, name)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
# SCRAM settings
self._configureSCRAMSettings(config)
self.useReqs = config.getBool('software requirements', True, onChange = None)
self._projectAreaTarballSE = config.getBool(['se project area', 'se runtime'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self.dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0')
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self.dataSplitter is not None))
# Create project area tarball
if not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self.projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self.projectArea, self.pattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
def deployTask(self, task, monitor, transferSE, transferSB): self.outputFiles = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(task)) # HACK task.validateVariables() self.smSEIn.addFiles(lmap(lambda d_s_t: d_s_t[2], task.getSEInFiles())) # add task SE files to SM # Transfer common SE files if transferSE: self.smSEIn.doTransfer(task.getSEInFiles()) def convert(fnList): for fn in fnList: if isinstance(fn, str): yield (fn, os.path.basename(fn), False) else: yield (None, os.path.basename(fn.name), fn) # Package sandbox tar file self._log.log(logging.INFO1, 'Packing sandbox') sandbox = self._getSandboxName(task) utils.ensureDirExists(os.path.dirname(sandbox), 'sandbox directory') if not os.path.exists(sandbox) or transferSB: utils.genTarball(sandbox, convert(self._getSandboxFiles(task, monitor, [self.smSEIn, self.smSEOut])))
def deployTask(self, module, monitor):
self.outputFiles = map(lambda (d, s, t): t, self._getSandboxFilesOut(module)) # HACK
module.validateVariables()
self.smSEIn.addFiles(map(lambda (d, s, t): t, module.getSEInFiles())) # add module SE files to SM
# Transfer common SE files
if self.config.getState(detail = 'storage'):
self.smSEIn.doTransfer(module.getSEInFiles())
def convert(fnList):
for fn in fnList:
if isinstance(fn, str):
yield (fn, os.path.basename(fn), False)
else:
yield (None, os.path.basename(fn.name), fn)
# Package sandbox tar file
utils.vprint('Packing sandbox:')
sandbox = self._getSandboxName(module)
utils.ensureDirExists(os.path.dirname(sandbox), 'sandbox directory')
if not os.path.exists(sandbox) or self.config.getState(detail = 'sandbox'):
utils.genTarball(sandbox, convert(self._getSandboxFiles(module, monitor, [self.smSEIn, self.smSEOut])))
def __init__(self, config, name):
config.set('se input timeout', '0:30', override = False)
config.set('dataset provider', 'DBS3Provider', override = False)
config.set('dataset splitter', 'EventBoundarySplitter', override = False)
DataTask.__init__(self, config, name)
self.errorDict.update(dict(self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))))
# SCRAM info
scramProject = config.getList('scram project', [])
if len(scramProject):
self.projectArea = config.getPath('project area', '')
if len(self.projectArea):
raise ConfigError('Cannot specify both SCRAM project and project area')
if len(scramProject) != 2:
raise ConfigError('SCRAM project needs exactly 2 arguments: PROJECT VERSION')
else:
self.projectArea = config.getPath('project area')
# This works in tandem with provider_dbsv2.py !
self.selectedLumis = parseLumiFilter(config.get('lumi filter', ''))
self.useReqs = config.getBool('software requirements', True, onChange = None)
self.seRuntime = config.getBool('se runtime', False)
self.runtimePath = config.getWorkPath('runtime.tar.gz')
if len(self.projectArea):
defaultPattern = '-.* -config bin lib python module */data *.xml *.sql *.cf[if] *.py -*/.git -*/.svn -*/CVS -*/work.*'
self.pattern = config.getList('area files', defaultPattern.split())
if os.path.exists(self.projectArea):
utils.vprint('Project area found in: %s' % self.projectArea, -1)
else:
raise ConfigError('Specified config area %r does not exist!' % self.projectArea)
scramPath = os.path.join(self.projectArea, '.SCRAM')
# try to open it
try:
fp = open(os.path.join(scramPath, 'Environment'), 'r')
self.scramEnv = utils.DictFormat().parse(fp, keyParser = {None: str})
except:
raise ConfigError('Project area file %s/.SCRAM/Environment cannot be parsed!' % self.projectArea)
for key in ['SCRAM_PROJECTNAME', 'SCRAM_PROJECTVERSION']:
if key not in self.scramEnv:
raise ConfigError('Installed program in project area not recognized.')
archs = filter(lambda x: os.path.isdir(os.path.join(scramPath, x)) and not x.startswith('.'), os.listdir(scramPath))
self.scramArch = config.get('scram arch', (archs + [noDefault])[0])
try:
fp = open(os.path.join(scramPath, self.scramArch, 'Environment'), 'r')
self.scramEnv.update(utils.DictFormat().parse(fp, keyParser = {None: str}))
except:
raise ConfigError('Project area file .SCRAM/%s/Environment cannot be parsed!' % self.scramArch)
else:
self.scramEnv = {
'SCRAM_PROJECTNAME': scramProject[0],
'SCRAM_PROJECTVERSION': scramProject[1]
}
self.scramArch = config.get('scram arch')
self.scramVersion = config.get('scram version', 'scramv1')
if self.scramEnv['SCRAM_PROJECTNAME'] != 'CMSSW':
raise ConfigError('Project area not a valid CMSSW project area.')
# Information about search order for software environment
self.searchLoc = []
if config.getState('sandbox'):
userPath = config.get('cmssw dir', '')
if userPath != '':
self.searchLoc.append(('CMSSW_DIR_USER', userPath))
if self.scramEnv.get('RELEASETOP', None):
projPath = os.path.normpath('%s/../../../../' % self.scramEnv['RELEASETOP'])
self.searchLoc.append(('CMSSW_DIR_PRO', projPath))
if len(self.searchLoc):
utils.vprint('Local jobs will try to use the CMSSW software located here:', -1)
for i, loc in enumerate(self.searchLoc):
key, value = loc
utils.vprint(' %i) %s' % (i + 1, value), -1)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
self.configFiles = []
cfgDefault = QM(self.prolog.isActive() or self.epilog.isActive(), [], noDefault)
for cfgFile in config.getPaths('config file', cfgDefault, mustExist = False):
newPath = config.getWorkPath(os.path.basename(cfgFile))
if not os.path.exists(newPath):
if not os.path.exists(cfgFile):
raise ConfigError('Config file %r not found.' % cfgFile)
shutil.copyfile(cfgFile, newPath)
self.configFiles.append(newPath)
# Check that for dataset jobs the necessary placeholders are in the config file
self.prepare = config.getBool('prepare config', False)
fragment = config.getPath('instrumentation fragment',
os.path.join('packages', 'grid_control_cms', 'share', 'fragmentForCMSSW.py'))
if self.dataSplitter != None:
if config.getState('sandbox'):
if len(self.configFiles) > 0:
self.instrumentCfgQueue(self.configFiles, fragment, mustPrepare = True)
else:
self.eventsPerJob = config.get('events per job', '0')
if config.getState(detail = 'sandbox') and self.prepare:
self.instrumentCfgQueue(self.configFiles, fragment)
if not os.path.exists(config.getWorkPath('runtime.tar.gz')):
config.setState(True, detail = 'sandbox')
if config.getState(detail = 'sandbox'):
if os.path.exists(config.getWorkPath('runtime.tar.gz')):
if not utils.getUserBool('Runtime already exists! Do you want to regenerate CMSSW tarball?', True):
return
# Generate runtime tarball (and move to SE)
if self.projectArea:
utils.genTarball(config.getWorkPath('runtime.tar.gz'), utils.matchFiles(self.projectArea, self.pattern))
if self.seRuntime:
config.setState(True, detail = 'storage')
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set(
'partition processor',
'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor'
)
config.set('dataset processor', 'LumiDataProcessor', '+=')
DataTask.__init__(self, config, name)
self.updateErrorDict(
utils.pathShare('gc-run.cmssw.sh', pkg='grid_control_cms'))
# SCRAM settings
self._configureSCRAMSettings(config)
self.useReqs = config.getBool('software requirements',
True,
onChange=None)
self._projectAreaTarballSE = config.getBool(
['se project area', 'se runtime'], True)
self._projectAreaTarball = config.getWorkPath(
'cmssw-project-area.tar.gz')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError(
'Prefix executable and argument options with either prolog or epilog!'
)
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self.dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0')
fragment = config.getPath(
'instrumentation fragment',
utils.pathShare('fragmentForCMSSW.py', pkg='grid_control_cms'))
self.configFiles = self._processConfigFiles(
config,
list(self._getConfigFiles(config)),
fragment,
autoPrepare=config.getBool('instrumentation', True),
mustPrepare=(self.dataSplitter is not None))
# Create project area tarball
if not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail='sandbox')
if config.getState('init', detail='sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool(
'CMSSW tarball already exists! Do you want to regenerate it?',
True):
return
# Generate CMSSW tarball
if self.projectArea:
utils.genTarball(
self._projectAreaTarball,
utils.matchFiles(self.projectArea, self.pattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail='storage')
