def __call__(self, config, old_obj, cur_obj, cur_entry, obj2str): log = logging.getLogger('config.onchange.%s' % self._option.lower()) config = config.changeView(setSections=['interactive']) interaction_def = config.getBool('default', True, onChange=None) interaction_opt = config.getBool(self._option, interaction_def, onChange=None) if interaction_opt: msg = 'The option "%s" was changed from the old value:' % cur_entry.format_opt( ) if utils.getUserBool( msg + ('\n\t> %s\nto the new value:' % obj2str(old_obj).lstrip()) + ('\n\t> %s\nDo you want to abort?' % obj2str(cur_obj).lstrip()), False): raise ConfigError('Abort due to unintentional config change!') if not utils.getUserBool( 'A partial reinitialization (same as --reinit %s) is needed to apply this change! Do you want to continue?' % self._option, True): log.log(logging.INFO1, 'Using stored value %s for option %s', obj2str(old_obj), cur_entry.format_opt()) return old_obj config.setState(True, 'init', detail=self._option) config.setState( True, 'init', detail='config') # This will trigger a write of the new options return cur_obj
def cancel(self, wms, jobs, interactive, showJobs): if len(jobs) == 0: return if showJobs: self._reportClass(self.jobDB, self._task, jobs).display() if interactive and not utils.getUserBool( 'Do you really want to cancel these jobs?', True): return def mark_cancelled(jobNum): jobObj = self.jobDB.get(jobNum) if jobObj is None: return self._update(jobObj, jobNum, Job.CANCELLED) self._eventhandler.onJobUpdate(wms, jobObj, jobNum, {'reason': 'cancelled'}) jobs.reverse() for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)): # Remove deleted job from todo list and mark as cancelled assert (self.jobDB.get(jobNum).wmsId == wmsId) jobs.remove(jobNum) mark_cancelled(jobNum) if len(jobs) > 0: self._log_user.warning( 'There was a problem with cancelling the following jobs:') self._reportClass(self.jobDB, self._task, jobs).display() if (interactive and utils.getUserBool( 'Do you want to mark them as cancelled?', True)) or not interactive: lmap(mark_cancelled, jobs) if interactive: utils.wait(2)
def cancel(self, wms, jobs, interactive = False, showJobs = True): if len(jobs) == 0: return if showJobs: self._reportClass(self.jobDB, self._task, jobs).display() if interactive and not utils.getUserBool('Do you really want to cancel these jobs?', True): return def mark_cancelled(jobNum): jobObj = self.jobDB.get(jobNum) if jobObj is None: return self._update(jobObj, jobNum, Job.CANCELLED) self._eventhandler.onJobUpdate(wms, jobObj, jobNum, {'reason': 'cancelled'}) jobs.reverse() for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)): # Remove deleted job from todo list and mark as cancelled assert(self.jobDB.get(jobNum).wmsId == wmsId) jobs.remove(jobNum) mark_cancelled(jobNum) if len(jobs) > 0: self._log_user.warning('There was a problem with cancelling the following jobs:') self._reportClass(self.jobDB, self._task, jobs).display() if (interactive and utils.getUserBool('Do you want to mark them as cancelled?', True)) or not interactive: lmap(mark_cancelled, jobs) if interactive: utils.wait(2)
def __call__(self, config, old_obj, cur_obj, cur_entry, obj2str): log = logging.getLogger('config.onchange.%s' % self._option.lower()) if config.isInteractive(self._option, default = True): msg = 'The option "%s" was changed from the old value:' % cur_entry.format_opt() if utils.getUserBool(msg + ('\n\t> %s\nto the new value:' % obj2str(old_obj).lstrip()) + ('\n\t> %s\nDo you want to abort?' % obj2str(cur_obj).lstrip()), False): raise ConfigError('Abort due to unintentional config change!') if not utils.getUserBool('A partial reinitialization (same as --reinit %s) is needed to apply this change! Do you want to continue?' % self._option, True): log.log(logging.INFO1, 'Using stored value %s for option %s', obj2str(old_obj), cur_entry.format_opt()) return old_obj config.setState(True, 'init', detail = self._option) config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options return cur_obj
def __call__(self, config, old_obj, cur_obj, cur_entry, obj2str): log = logging.getLogger('config.onChange.%s' % self._option) config = config.changeView(setSections = ['interactive']) interaction_def = config.getBool('default', True, onChange = None) interaction_opt = config.getBool(self._option, interaction_def, onChange = None) if interaction_opt: if utils.getUserBool('The option "%s" was changed from the old value:' % cur_entry.format_opt() + '\n\t%s\nto the new value:\n\t%s\nDo you want to abort?' % (obj2str(old_obj), obj2str(cur_obj)), False): raise ConfigError('Abort due to unintentional config change!') if not utils.getUserBool('A partial reinitialization (same as --reinit %s) is needed to apply this change! Do you want to continue?' % self._option, True): log.log(logging.INFO1, 'Using stored value %s for option %s' % (obj2str(old_obj), cur_entry.format_opt())) return old_obj config.setState(True, 'init', detail = self._option) config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options return cur_obj
def makeJDL(self, jobNum, module): cfgPath = os.path.join(self._jobPath, 'job_%d.var' % jobNum) sbIn = lmap(lambda d_s_t: d_s_t[1], self._getSandboxFilesIn(module)) sbOut = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(module)) wcList = lfilter(lambda x: '*' in x, sbOut) if len(wcList): self._writeJobConfig(cfgPath, jobNum, module, {'GC_WC': str.join(' ', wcList)}) sandboxOutJDL = lfilter(lambda x: x not in wcList, sbOut) + ['GC_WC.tar.gz'] else: self._writeJobConfig(cfgPath, jobNum, module, {}) sandboxOutJDL = sbOut # Warn about too large sandboxes sbSizes = lmap(os.path.getsize, sbIn) if sbSizes and (self._warnSBSize > 0) and (sum(sbSizes) > self._warnSBSize * 1024 * 1024): if not utils.getUserBool('Sandbox is very large (%d bytes) and can cause issues with the WMS! Do you want to continue?' % sum(sbSizes), False): sys.exit(os.EX_OK) self._warnSBSize = 0 reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum), WMS.SITES) formatStrList = lambda strList: '{ %s }' % str.join(', ', imap(lambda x: '"%s"' % x, strList)) contents = { 'Executable': '"gc-run.sh"', 'Arguments': '"%d"' % jobNum, 'StdOutput': '"gc.stdout"', 'StdError': '"gc.stderr"', 'InputSandbox': formatStrList(sbIn + [cfgPath]), 'OutputSandbox': formatStrList(sandboxOutJDL), 'VirtualOrganisation': '"%s"' % self.vo, 'Rank': '-other.GlueCEStateEstimatedResponseTime', 'RetryCount': 2 } return self._jdl_writer.format(reqs, contents)
def _findCollision(self, tName, nameDict, varDict, hashKeys, keyFmt, nameFmt=identity): dupesDict = {} for (key, name) in nameDict.items(): dupesDict.setdefault(nameFmt(name), []).append(keyFmt(name, key)) ask = True for name, key_list in sorted(dupesDict.items()): if len(key_list) > 1: self._log.warn('Multiple %s keys are mapped to the name %s!', tName, repr(name)) for key in sorted(key_list): self._log.warn('\t%s hash %s using:', tName, str.join('#', key)) for var, value in self._getFilteredVarDict( varDict, key, hashKeys).items(): self._log.warn('\t\t%s = %s', var, value) if ask and not utils.getUserBool('Do you want to continue?', False): sys.exit(os.EX_OK) ask = False
def doTransfer(self, listDescSourceTarget): for (desc, source, target) in listDescSourceTarget: if not self.smPaths: raise ConfigError( "%s can't be transferred because '%s path wasn't set" % (desc, self.smOptPrefix)) for idx, sePath in enumerate(set(self.smPaths)): utils.vprint('Copy %s to SE %d ' % (desc, idx + 1), -1, newline=False) sys.stdout.flush() proc = se_copy(source, os.path.join(sePath, target), self.smForce) if proc.status(timeout=5 * 60, terminate=True) == 0: utils.vprint('finished', -1) else: utils.vprint('failed', -1) utils.eprint(proc.stderr.read(timeout=0)) utils.eprint( 'Unable to copy %s! You can try to copy it manually.' % desc) if not utils.getUserBool( 'Is %s (%s) available on SE %s?' % (desc, source, sePath), False): raise StorageError('%s is missing on SE %s!' % (desc, sePath))
def doTransfer(self, listDescSourceTarget): for (desc, source, target) in listDescSourceTarget: if not self.smPaths: raise ConfigError( "%s can't be transferred because '%s path wasn't set" % (desc, self.smOptPrefix)) for idx, sePath in enumerate(set(self.smPaths)): activity = Activity('Copy %s to SE %d ' % (desc, idx + 1)) proc = se_copy(source, os.path.join(sePath, target), self.smForce) proc.status(timeout=5 * 60, terminate=True) activity.finish() if proc.status(timeout=0) == 0: self._log.info('Copy %s to SE %d finished', desc, idx + 1) else: self._log.info('Copy %s to SE %d failed', desc, idx + 1) self._log.critical(proc.stderr.read(timeout=0)) self._log.critical( 'Unable to copy %s! You can try to copy it manually.', desc) if not utils.getUserBool( 'Is %s (%s) available on SE %s?' % (desc, source, sePath), False): raise StorageError('%s is missing on SE %s!' % (desc, sePath))
def _processConfigFiles(self, config, cfgFiles, fragment_path, autoPrepare, mustPrepare): # process list of uninitialized config files for (cfg, cfg_new, doPrepare) in self._cfgFindUninitialized(config, cfgFiles, autoPrepare, mustPrepare): if doPrepare and (autoPrepare or utils.getUserBool( 'Do you want to prepare %s for running over the dataset?' % cfg, True)): self._cfgStore(cfg, cfg_new, fragment_path) else: self._cfgStore(cfg, cfg_new) result = [] for cfg in cfgFiles: cfg_new = config.getWorkPath(os.path.basename(cfg)) if not os.path.exists(cfg_new): raise ConfigError( 'Config file %r was not copied to the work directory!' % cfg) isInstrumented = self._cfgIsInstrumented(cfg_new) if mustPrepare and not isInstrumented: raise ConfigError( 'Config file %r must use %s to work properly!' % (cfg, str.join(', ', imap(lambda x: '@%s@' % x, self.neededVars())))) if autoPrepare and not isInstrumented: self._log.warning('Config file %r was not instrumented!', cfg) result.append(cfg_new) return result
def resyncMapping(self, newSplitPath, oldBlocks, newBlocks): log = utils.ActivityLog('Performing resynchronization of dataset') (blocksAdded, blocksMissing, blocksMatching) = DataProvider.resyncSources(oldBlocks, newBlocks) for rmBlock in blocksMissing: # Files in matching blocks are already sorted sort_inplace(rmBlock[DataProvider.FileList], key=lambda x: x[DataProvider.URL]) log.finish() # User overview and setup starts here resultRedo = [] resultDisable = [] newSplitPathTMP = newSplitPath + '.tmp' resyncIter = self._resyncIterator(resultRedo, resultDisable, blocksAdded, blocksMissing, blocksMatching) self.savePartitions( newSplitPathTMP, resyncIter, sourceLen=self.getMaxJobs(), message= 'Performing resynchronization of dataset map (progress is estimated)' ) if self._interactive: # TODO: print info and ask if not utils.getUserBool( 'Do you want to use the new dataset partition?', False): return None os.rename(newSplitPathTMP, newSplitPath) return (resultRedo, resultDisable)
def instrumentCfgQueue(self, cfgFiles, fragment, mustPrepare = False): def isInstrumented(cfgName): cfg = open(cfgName, 'r').read() for tag in self.neededVars(): if (not '__%s__' % tag in cfg) and (not '@%s@' % tag in cfg): return False return True def doInstrument(cfgName): if not isInstrumented(cfgName) or 'customise_for_gc' not in open(cfgName, 'r').read(): utils.vprint('Instrumenting...', os.path.basename(cfgName), -1) open(cfgName, 'a').write(open(fragment, 'r').read()) else: utils.vprint('%s already contains customise_for_gc and all needed variables' % os.path.basename(cfgName), -1) cfgStatus = [] comPath = os.path.dirname(os.path.commonprefix(cfgFiles)) for cfg in cfgFiles: cfgStatus.append({0: cfg.split(comPath, 1)[1].lstrip('/'), 1: str(isInstrumented(cfg)), 2: cfg}) utils.printTabular([(0, 'Config file'), (1, 'Instrumented')], cfgStatus, 'lc') for cfg in cfgFiles: if self.prepare or not isInstrumented(cfg): if self.prepare or utils.getUserBool('Do you want to prepare %s for running over the dataset?' % cfg, True): doInstrument(cfg) if mustPrepare and not (True in map(isInstrumented, cfgFiles)): raise ConfigError('A config file must use %s to work properly!' % str.join(', ', map(lambda x: '@%s@' % x, self.neededVars())))
def reset(self, wms, select): jobs = self.jobDB.getJobs(JobSelector.create(select, task = self._task)) if jobs: self._log_user.warning('Resetting the following jobs:') self._reportClass(self.jobDB, self._task, jobs).display() if utils.getUserBool('Are you sure you want to reset the state of these jobs?', False): self.cancel(wms, self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING), jobs), False, False) for jobNum in jobs: self.jobDB.commit(jobNum, Job())
def __init__(self, config, source): self._rawSource = source BasicParameterAdapter.__init__(self, config, source) self._mapJob2PID = {} if not os.path.isdir(config.getWorkPath()): os.makedirs(config.getWorkPath()) self._pathJob2PID = config.getWorkPath('params.map.gz') self._pathParams = config.getWorkPath('params.dat.gz') # Find out if init should be performed - overrides userResync! userInit = config.getState('init', detail='parameters') needInit = False if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)): needInit = True # Init needed if no parameter log exists if userInit and not needInit and (source.getMaxParameters() is not None): utils.eprint( 'Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!' ) if utils.getUserBool( 'Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True): userInit = False doInit = userInit or needInit # Find out if resync should be performed userResync = config.getState('resync', detail='parameters') config.setState(False, 'resync', detail='parameters') needResync = False pHash = self._rawSource.getHash() self.storedHash = config.get('parameter hash', pHash, persistent=True) if self.storedHash != pHash: needResync = True # Resync needed if parameters have changed self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self.storedHash) self._log.debug('\tnew hash: %s', pHash) config.setState(True, 'init', detail='config') doResync = (userResync or needResync) and not doInit if not doResync and not doInit: # Reuse old mapping activity = utils.ActivityLog( 'Loading cached parameter information') self.readJob2PID() activity.finish() return elif doResync: # Perform sync activity = utils.ActivityLog('Syncronizing parameter information') self.storedHash = None self._resyncState = self.resync() activity.finish() elif doInit: # Write current state self.writeJob2PID(self._pathJob2PID) ParameterSource.getClass('GCDumpParameterSource').write( self._pathParams, self) config.set('parameter hash', self._rawSource.getHash())
def __init__(self, config, name): config.set('se input timeout', '0:30') config.set('dataset provider', 'DBS3Provider') config.set('dataset splitter', 'EventBoundarySplitter') config.set('dataset processor', 'LumiDataProcessor', '+=') config.set('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor ' + 'LFNPartitionProcessor LumiPartitionProcessor CMSSWPartitionProcessor') dash_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dashboard']) dash_config.set('application', 'cmsRun') SCRAMTask.__init__(self, config, name) if self._scramProject != 'CMSSW': raise ConfigError('Project area contains no CMSSW project') self._oldReleaseTop = None if self._projectArea: self._oldReleaseTop = self._parse_scram_file(os.path.join(self._projectArea, '.SCRAM', self._scramArch, 'Environment')).get('RELEASETOP', None) self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms')) self._projectAreaTarballSE = config.getBool(['se runtime', 'se project area'], True) self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz') # Prolog / Epilog script support - warn about old syntax self.prolog = TaskExecutableWrapper(config, 'prolog', '') self.epilog = TaskExecutableWrapper(config, 'epilog', '') if config.getPaths('executable', []) != []: raise ConfigError('Prefix executable and argument options with either prolog or epilog!') self.arguments = config.get('arguments', '') # Get cmssw config files and check their existance # Check that for dataset jobs the necessary placeholders are in the config file if self._dataSplitter is None: self.eventsPerJob = config.get('events per job', '0') # this can be a variable like @USER_EVENTS@! fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms')) self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment, autoPrepare = config.getBool('instrumentation', True), mustPrepare = (self._dataSplitter is not None)) # Create project area tarball if self._projectArea and not os.path.exists(self._projectAreaTarball): config.setState(True, 'init', detail = 'sandbox') # Information about search order for software environment self.searchLoc = self._getCMSSWPaths(config) if config.getState('init', detail = 'sandbox'): if os.path.exists(self._projectAreaTarball): if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True): return # Generate CMSSW tarball if self._projectArea: utils.genTarball(self._projectAreaTarball, utils.matchFiles(self._projectArea, self._projectAreaPattern)) if self._projectAreaTarballSE: config.setState(True, 'init', detail = 'storage')
def reset(self, wms, select): jobs = self.jobDB.getJobs(JobSelector.create(select, task=self._task)) if jobs: self._log_user.warning('Resetting the following jobs:') self._reportClass(self.jobDB, self._task, jobs).display() if utils.getUserBool( 'Are you sure you want to reset the state of these jobs?', False): self.cancel( wms, self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING), jobs), False, False) for jobNum in jobs: self.jobDB.commit(jobNum, Job())
def _findCollision(self, tName, nameDict, varDict, hashKeys, keyFmt, nameFmt = identity): dupesDict = {} for (key, name) in nameDict.items(): dupesDict.setdefault(nameFmt(name), []).append(keyFmt(name, key)) ask = True for name, key_list in sorted(dupesDict.items()): if len(key_list) > 1: self._log.warn('Multiple %s keys are mapped to the name %s!', tName, repr(name)) for key in sorted(key_list): self._log.warn('\t%s hash %s using:', tName, str.join('#', key)) for var, value in self._getFilteredVarDict(varDict, key, hashKeys).items(): self._log.warn('\t\t%s = %s', var, value) if ask and not utils.getUserBool('Do you want to continue?', False): sys.exit(os.EX_OK) ask = False
def __init__(self, config, source): self._rawSource = source BasicParameterAdapter.__init__(self, config, source) self._mapJob2PID = {} if not os.path.isdir(config.getWorkPath()): os.makedirs(config.getWorkPath()) self._pathJob2PID = config.getWorkPath('params.map.gz') self._pathParams = config.getWorkPath('params.dat.gz') # Find out if init should be performed - overrides userResync! userInit = config.getState('init', detail = 'parameters') needInit = False if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)): needInit = True # Init needed if no parameter log exists if userInit and not needInit and (source.getMaxParameters() is not None): utils.eprint('Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!') if utils.getUserBool('Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True): userInit = False doInit = userInit or needInit # Find out if resync should be performed userResync = config.getState('resync', detail = 'parameters') config.setState(False, 'resync', detail = 'parameters') needResync = False pHash = self._rawSource.getHash() self._storedHash = config.get('parameter hash', pHash, persistent = True) if self._storedHash != pHash: needResync = True # Resync needed if parameters have changed self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self._storedHash) self._log.debug('\tnew hash: %s', pHash) config.setState(True, 'init', detail = 'config') doResync = (userResync or needResync) and not doInit if not doResync and not doInit: # Reuse old mapping activity = utils.ActivityLog('Loading cached parameter information') self._readJob2PID() activity.finish() return elif doResync: # Perform sync activity = utils.ActivityLog('Syncronizing parameter information') self._storedHash = None self._resyncState = self.resync() activity.finish() elif doInit: # Write current state self._writeJob2PID(self._pathJob2PID) ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams, self) config.set('parameter hash', self._rawSource.getHash())
def findCollision(tName, nameDict, varDict, hashKeys, keyFmt = lambda x: x): targetNames = nameDict.values() for name in list(set(targetNames)): targetNames.remove(name) if len(targetNames): ask = True for name in targetNames: utils.eprint("Multiple %s keys are mapped to the same %s name '%s'!" % (tName, tName, keyFmt(name))) for key in nameDict: if nameDict[key] == name: utils.eprint('\t%s hash %s using:' % (tName, keyFmt(key))) for x in filter(lambda (k, v): k in hashKeys, varDict[keyFmt(key)].items()): utils.eprint('\t\t%s = %s' % x) if ask and not utils.getUserBool('Do you want to continue?', False): sys.exit(0) ask = False
def doTransfer(self, listDescSourceTarget): for (desc, source, target) in listDescSourceTarget: if not self.smPaths: raise ConfigError("%s can't be transferred because '%s path wasn't set" % (desc, self.smOptPrefix)) for idx, sePath in enumerate(set(self.smPaths)): utils.vprint('Copy %s to SE %d ' % (desc, idx + 1), -1, newline = False) sys.stdout.flush() proc = se_copy(source, os.path.join(sePath, target), self.smForce) if proc.status(timeout = 5*60, terminate = True) == 0: utils.vprint('finished', -1) else: utils.vprint('failed', -1) utils.eprint(proc.stderr.read(timeout = 0)) utils.eprint('Unable to copy %s! You can try to copy it manually.' % desc) if not utils.getUserBool('Is %s (%s) available on SE %s?' % (desc, source, sePath), False): raise StorageError('%s is missing on SE %s!' % (desc, sePath))
def __init__(self, config, name): config.set('se input timeout', '0:30') config.set('dataset provider', 'DBS3Provider') config.set('dataset splitter', 'EventBoundarySplitter') config.set('partition processor', 'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor') config.set('dataset processor', 'LumiDataProcessor', '+=') DataTask.__init__(self, config, name) self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms')) # SCRAM settings self._configureSCRAMSettings(config) self.useReqs = config.getBool('software requirements', True, onChange = None) self._projectAreaTarballSE = config.getBool(['se project area', 'se runtime'], True) self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz') # Information about search order for software environment self.searchLoc = self._getCMSSWPaths(config) # Prolog / Epilog script support - warn about old syntax self.prolog = TaskExecutableWrapper(config, 'prolog', '') self.epilog = TaskExecutableWrapper(config, 'epilog', '') if config.getPaths('executable', []) != []: raise ConfigError('Prefix executable and argument options with either prolog or epilog!') self.arguments = config.get('arguments', '') # Get cmssw config files and check their existance # Check that for dataset jobs the necessary placeholders are in the config file if self.dataSplitter is None: self.eventsPerJob = config.get('events per job', '0') fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms')) self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment, autoPrepare = config.getBool('instrumentation', True), mustPrepare = (self.dataSplitter is not None)) # Create project area tarball if not os.path.exists(self._projectAreaTarball): config.setState(True, 'init', detail = 'sandbox') if config.getState('init', detail = 'sandbox'): if os.path.exists(self._projectAreaTarball): if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True): return # Generate CMSSW tarball if self.projectArea: utils.genTarball(self._projectAreaTarball, utils.matchFiles(self.projectArea, self.pattern)) if self._projectAreaTarballSE: config.setState(True, 'init', detail = 'storage')
def doTransfer(self, listDescSourceTarget): for (desc, source, target) in listDescSourceTarget: if not self.smPaths: raise ConfigError("%s can't be transferred because '%s path wasn't set" % (desc, self.smOptPrefix)) for idx, sePath in enumerate(set(self.smPaths)): activity = Activity('Copy %s to SE %d ' % (desc, idx + 1)) proc = se_copy(source, os.path.join(sePath, target), self.smForce) proc.status(timeout = 5*60, terminate = True) activity.finish() if proc.status(timeout = 0) == 0: self._log.info('Copy %s to SE %d finished', desc, idx + 1) else: self._log.info('Copy %s to SE %d failed', desc, idx + 1) self._log.critical(proc.stderr.read(timeout = 0)) self._log.critical('Unable to copy %s! You can try to copy it manually.', desc) if not utils.getUserBool('Is %s (%s) available on SE %s?' % (desc, source, sePath), False): raise StorageError('%s is missing on SE %s!' % (desc, sePath))
def makeJDL(self, jobNum, module): cfgPath = os.path.join(self._jobPath, 'job_%d.var' % jobNum) sbIn = lmap(lambda d_s_t: d_s_t[1], self._getSandboxFilesIn(module)) sbOut = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(module)) wcList = lfilter(lambda x: '*' in x, sbOut) if len(wcList): self._writeJobConfig(cfgPath, jobNum, module, {'GC_WC': str.join(' ', wcList)}) sandboxOutJDL = lfilter(lambda x: x not in wcList, sbOut) + ['GC_WC.tar.gz'] else: self._writeJobConfig(cfgPath, jobNum, module, {}) sandboxOutJDL = sbOut # Warn about too large sandboxes sbSizes = lmap(os.path.getsize, sbIn) if sbSizes and (self._warnSBSize > 0) and (sum(sbSizes) > self._warnSBSize * 1024 * 1024): if not utils.getUserBool( 'Sandbox is very large (%d bytes) and can cause issues with the WMS! Do you want to continue?' % sum(sbSizes), False): sys.exit(os.EX_OK) self._warnSBSize = 0 reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum), WMS.SITES) formatStrList = lambda strList: '{ %s }' % str.join( ', ', imap(lambda x: '"%s"' % x, strList)) contents = { 'Executable': '"gc-run.sh"', 'Arguments': '"%d"' % jobNum, 'StdOutput': '"gc.stdout"', 'StdError': '"gc.stderr"', 'InputSandbox': formatStrList(sbIn + [cfgPath]), 'OutputSandbox': formatStrList(sandboxOutJDL), 'Requirements': self._formatRequirements(reqs), 'VirtualOrganisation': '"%s"' % self.vo, 'Rank': '-other.GlueCEStateEstimatedResponseTime', 'RetryCount': 2 } cpus = dict(reqs).get(WMS.CPUS, 1) if cpus > 1: contents['CpuNumber'] = cpus return utils.DictFormat(' = ').format(contents, format='%s%s%s;\n')
def _processConfigFiles(self, config, cfgFiles, fragment_path, autoPrepare, mustPrepare): # process list of uninitialized config files for (cfg, cfg_new, doPrepare) in self._cfgFindUninitialized(config, cfgFiles, autoPrepare, mustPrepare): if doPrepare and (autoPrepare or utils.getUserBool('Do you want to prepare %s for running over the dataset?' % cfg, True)): self._cfgStore(cfg, cfg_new, fragment_path) else: self._cfgStore(cfg, cfg_new) result = [] for cfg in cfgFiles: cfg_new = config.getWorkPath(os.path.basename(cfg)) if not os.path.exists(cfg_new): raise ConfigError('Config file %r was not copied to the work directory!' % cfg) isInstrumented = self._cfgIsInstrumented(cfg_new) if mustPrepare and not isInstrumented: raise ConfigError('Config file %r must use %s to work properly!' % (cfg, str.join(', ', imap(lambda x: '@%s@' % x, self.neededVars())))) if autoPrepare and not isInstrumented: self._log.warning('Config file %r was not instrumented!', cfg) result.append(cfg_new) return result
def resyncMapping(self, newSplitPath, oldBlocks, newBlocks): activity = Activity('Performing resynchronization of dataset') (blocksAdded, blocksMissing, blocksMatching) = DataProvider.resyncSources(oldBlocks, newBlocks) for rmBlock in blocksMissing: # Files in matching blocks are already sorted sort_inplace(rmBlock[DataProvider.FileList], key = lambda x: x[DataProvider.URL]) activity.finish() # User overview and setup starts here resultRedo = [] resultDisable = [] newSplitPathTMP = newSplitPath + '.tmp' resyncIter = self._resyncIterator(resultRedo, resultDisable, blocksAdded, blocksMissing, blocksMatching) self.savePartitions(newSplitPathTMP, resyncIter, sourceLenHint = self.getMaxJobs(), message = 'Performing resynchronization of dataset map (progress is estimated)') if self._interactive: # TODO: print info and ask if not utils.getUserBool('Do you want to use the new dataset partition?', False): return os.rename(newSplitPathTMP, newSplitPath) return (resultRedo, resultDisable)
def __init__(self, config, name): config.set('se input timeout', '0:30') config.set('dataset provider', 'DBS3Provider') config.set('dataset splitter', 'EventBoundarySplitter') config.set( 'partition processor', 'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor' ) config.set('dataset processor', 'LumiDataProcessor', '+=') DataTask.__init__(self, config, name) self.updateErrorDict( utils.pathShare('gc-run.cmssw.sh', pkg='grid_control_cms')) # SCRAM settings self._configureSCRAMSettings(config) self.useReqs = config.getBool('software requirements', True, onChange=None) self._projectAreaTarballSE = config.getBool( ['se project area', 'se runtime'], True) self._projectAreaTarball = config.getWorkPath( 'cmssw-project-area.tar.gz') # Information about search order for software environment self.searchLoc = self._getCMSSWPaths(config) # Prolog / Epilog script support - warn about old syntax self.prolog = TaskExecutableWrapper(config, 'prolog', '') self.epilog = TaskExecutableWrapper(config, 'epilog', '') if config.getPaths('executable', []) != []: raise ConfigError( 'Prefix executable and argument options with either prolog or epilog!' ) self.arguments = config.get('arguments', '') # Get cmssw config files and check their existance # Check that for dataset jobs the necessary placeholders are in the config file if self.dataSplitter is None: self.eventsPerJob = config.get('events per job', '0') fragment = config.getPath( 'instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg='grid_control_cms')) self.configFiles = self._processConfigFiles( config, list(self._getConfigFiles(config)), fragment, autoPrepare=config.getBool('instrumentation', True), mustPrepare=(self.dataSplitter is not None)) # Create project area tarball if not os.path.exists(self._projectAreaTarball): config.setState(True, 'init', detail='sandbox') if config.getState('init', detail='sandbox'): if os.path.exists(self._projectAreaTarball): if not utils.getUserBool( 'CMSSW tarball already exists! Do you want to regenerate it?', True): return # Generate CMSSW tarball if self.projectArea: utils.genTarball( self._projectAreaTarball, utils.matchFiles(self.projectArea, self.pattern)) if self._projectAreaTarballSE: config.setState(True, 'init', detail='storage')
def __init__(self, config, name): config.set('se input timeout', '0:30', override = False) config.set('dataset provider', 'DBS3Provider', override = False) config.set('dataset splitter', 'EventBoundarySplitter', override = False) DataTask.__init__(self, config, name) self.errorDict.update(dict(self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms')))) # SCRAM info scramProject = config.getList('scram project', []) if len(scramProject): self.projectArea = config.getPath('project area', '') if len(self.projectArea): raise ConfigError('Cannot specify both SCRAM project and project area') if len(scramProject) != 2: raise ConfigError('SCRAM project needs exactly 2 arguments: PROJECT VERSION') else: self.projectArea = config.getPath('project area') # This works in tandem with provider_dbsv2.py ! self.selectedLumis = parseLumiFilter(config.get('lumi filter', '')) self.useReqs = config.getBool('software requirements', True, onChange = None) self.seRuntime = config.getBool('se runtime', False) self.runtimePath = config.getWorkPath('runtime.tar.gz') if len(self.projectArea): defaultPattern = '-.* -config bin lib python module */data *.xml *.sql *.cf[if] *.py -*/.git -*/.svn -*/CVS -*/work.*' self.pattern = config.getList('area files', defaultPattern.split()) if os.path.exists(self.projectArea): utils.vprint('Project area found in: %s' % self.projectArea, -1) else: raise ConfigError('Specified config area %r does not exist!' % self.projectArea) scramPath = os.path.join(self.projectArea, '.SCRAM') # try to open it try: fp = open(os.path.join(scramPath, 'Environment'), 'r') self.scramEnv = utils.DictFormat().parse(fp, keyParser = {None: str}) except: raise ConfigError('Project area file %s/.SCRAM/Environment cannot be parsed!' % self.projectArea) for key in ['SCRAM_PROJECTNAME', 'SCRAM_PROJECTVERSION']: if key not in self.scramEnv: raise ConfigError('Installed program in project area not recognized.') archs = filter(lambda x: os.path.isdir(os.path.join(scramPath, x)) and not x.startswith('.'), os.listdir(scramPath)) self.scramArch = config.get('scram arch', (archs + [noDefault])[0]) try: fp = open(os.path.join(scramPath, self.scramArch, 'Environment'), 'r') self.scramEnv.update(utils.DictFormat().parse(fp, keyParser = {None: str})) except: raise ConfigError('Project area file .SCRAM/%s/Environment cannot be parsed!' % self.scramArch) else: self.scramEnv = { 'SCRAM_PROJECTNAME': scramProject[0], 'SCRAM_PROJECTVERSION': scramProject[1] } self.scramArch = config.get('scram arch') self.scramVersion = config.get('scram version', 'scramv1') if self.scramEnv['SCRAM_PROJECTNAME'] != 'CMSSW': raise ConfigError('Project area not a valid CMSSW project area.') # Information about search order for software environment self.searchLoc = [] if config.getState('sandbox'): userPath = config.get('cmssw dir', '') if userPath != '': self.searchLoc.append(('CMSSW_DIR_USER', userPath)) if self.scramEnv.get('RELEASETOP', None): projPath = os.path.normpath('%s/../../../../' % self.scramEnv['RELEASETOP']) self.searchLoc.append(('CMSSW_DIR_PRO', projPath)) if len(self.searchLoc): utils.vprint('Local jobs will try to use the CMSSW software located here:', -1) for i, loc in enumerate(self.searchLoc): key, value = loc utils.vprint(' %i) %s' % (i + 1, value), -1) # Prolog / Epilog script support - warn about old syntax self.prolog = TaskExecutableWrapper(config, 'prolog', '') self.epilog = TaskExecutableWrapper(config, 'epilog', '') if config.getPaths('executable', []) != []: raise ConfigError('Prefix executable and argument options with either prolog or epilog!') self.arguments = config.get('arguments', '') # Get cmssw config files and check their existance self.configFiles = [] cfgDefault = QM(self.prolog.isActive() or self.epilog.isActive(), [], noDefault) for cfgFile in config.getPaths('config file', cfgDefault, mustExist = False): newPath = config.getWorkPath(os.path.basename(cfgFile)) if not os.path.exists(newPath): if not os.path.exists(cfgFile): raise ConfigError('Config file %r not found.' % cfgFile) shutil.copyfile(cfgFile, newPath) self.configFiles.append(newPath) # Check that for dataset jobs the necessary placeholders are in the config file self.prepare = config.getBool('prepare config', False) fragment = config.getPath('instrumentation fragment', os.path.join('packages', 'grid_control_cms', 'share', 'fragmentForCMSSW.py')) if self.dataSplitter != None: if config.getState('sandbox'): if len(self.configFiles) > 0: self.instrumentCfgQueue(self.configFiles, fragment, mustPrepare = True) else: self.eventsPerJob = config.get('events per job', '0') if config.getState(detail = 'sandbox') and self.prepare: self.instrumentCfgQueue(self.configFiles, fragment) if not os.path.exists(config.getWorkPath('runtime.tar.gz')): config.setState(True, detail = 'sandbox') if config.getState(detail = 'sandbox'): if os.path.exists(config.getWorkPath('runtime.tar.gz')): if not utils.getUserBool('Runtime already exists! Do you want to regenerate CMSSW tarball?', True): return # Generate runtime tarball (and move to SE) if self.projectArea: utils.genTarball(config.getWorkPath('runtime.tar.gz'), utils.matchFiles(self.projectArea, self.pattern)) if self.seRuntime: config.setState(True, detail = 'storage')