def __init__(self,what='standard',strict=True): from Configuration.PyReleaseValidation.MatrixReader import MatrixReader self.mrd = MatrixReader(what,noRun=True) self.mrd.prepare('all','',None) self.configBuilders={} self.processes={} self.strict=strict
def showRaw(opt): mrd = MatrixReader(opt) mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList) return 0
def __init__(self, opt): cmd.Cmd.__init__(self) self.opt_ = opt self.matrices_ = {} tmp = MatrixReader(self.opt_) for what in tmp.files: self.opt_.what = what self.matrices_[what] = MatrixReader(self.opt_) self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel, self.opt_.fromScratch) os.system("clear")
def runSelected(opt): mrd = MatrixReader(opt) mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch) # test for wrong input workflows if opt.testList: definedSet = set([dwf.numId for dwf in mrd.workFlows]) testSet = set(opt.testList) undefSet = testSet - definedSet if len(undefSet) > 0: raise ValueError('Undefined workflows: ' + ', '.join(map(str, list(undefSet)))) ret = 0 if opt.show: mrd.show(opt.testList, opt.extended, opt.cafVeto) if opt.testList: print('testListected items:', opt.testList) else: mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads) ret = mRunnerHi.runTests(opt) if opt.wmcontrol: if ret != 0: print('Cannot go on with wmagent injection with failing workflows') else: wfInjector = MatrixInjector(opt, mode=opt.wmcontrol, options=opt.wmoptions) ret = wfInjector.prepare(mrd, mRunnerHi.runDirs) if ret == 0: wfInjector.upload() wfInjector.submit() return ret
def runSelected(opt): mrd = MatrixReader(opt) mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch) ret = 0 if opt.show: mrd.show(opt.testList, opt.extended, opt.cafVeto) if opt.testList: print 'testListected items:', opt.testList else: mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads) ret = mRunnerHi.runTests(opt) if opt.wmcontrol: if ret != 0: print 'Cannot go on with wmagent injection with failing workflows' else: wfInjector = MatrixInjector(opt, mode=opt.wmcontrol, options=opt.wmoptions) ret = wfInjector.prepare(mrd, mRunnerHi.runDirs) if ret == 0: wfInjector.upload() wfInjector.submit() return ret
def runSelected(opt): mrd = MatrixReader(opt) mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch) # test for wrong input workflows if opt.testList: definedWF = [] for dwf in mrd.workFlows: definedWF.append(dwf.numId) for twf in opt.testList: if twf not in definedWF: raise ValueError('Not defined workflow ', twf, ' requested') ret = 0 if opt.show: mrd.show(opt.testList, opt.extended, opt.cafVeto) if opt.testList: print('testListected items:', opt.testList) else: mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads) ret = mRunnerHi.runTests(opt) if opt.wmcontrol: if ret != 0: print('Cannot go on with wmagent injection with failing workflows') else: wfInjector = MatrixInjector(opt, mode=opt.wmcontrol, options=opt.wmoptions) ret = wfInjector.prepare(mrd, mRunnerHi.runDirs) if ret == 0: wfInjector.upload() wfInjector.submit() return ret
def runSelected(opt): mrd = MatrixReader(opt) mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch) ret = 0 if opt.show: mrd.show(opt.testList, opt.extended, opt.cafVeto) if opt.testList : print('testListected items:', opt.testList) else: mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads) ret = mRunnerHi.runTests(opt) if opt.wmcontrol: if ret!=0: print('Cannot go on with wmagent injection with failing workflows') else: wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions) ret= wfInjector.prepare(mrd, mRunnerHi.runDirs) if ret==0: wfInjector.upload() wfInjector.submit() return ret
from Configuration.PyReleaseValidation.MatrixReader import MatrixReader from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner except Exception, e: bdir=os.getenv('CMSSW_RELEASE_BASE', None) if bdir != None: sys.path.insert(0,bdir+'/src/Configuration/PyReleaseValidation/scripts') bdir=os.getenv('CMSSW_BASE', None) if bdir != None: sys.path.insert(0,bdir+'/src/Configuration/PyReleaseValidation/scripts') from runTheMatrix import MatrixReader from runTheMatrix import MatrixRunner newWorkFlows = False opts = MatrixOpts() try: mrd = MatrixReader() except: mrd = MatrixReader(opts) newWorkFlows = True mrd.prepare() for wf in mrd.workFlows: if str(wf.numId) not in testList: continue print "processing ", wf.numId if newWorkFlows: updateNewWF(wf) else: updateOldWF(wf) ret = 0 if show: if not newWorkFlows:
class MatrixToProcess: def __init__(self,what='standard',strict=True): from Configuration.PyReleaseValidation.MatrixReader import MatrixReader self.mrd = MatrixReader(what,noRun=True) self.mrd.prepare('all','',None) self.configBuilders={} self.processes={} self.strict=strict def getKey(self,wfNumber,step): return str(wfNumber)+':'+str(step) def getProcess(self,wfNumber,step): key=self.getKey(wfNumber,step) if not key in self.configBuilders: self.load(wfNumber,step) if not key in self.configBuilders: return None return self.configBuilders[key].process def load(self,wfNumber,step): from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder from Configuration.PyReleaseValidation.cmsDriverOptions import OptionsFromCommand import copy if len(self.configBuilders)!=0 and self.strict: raise Exception('one should never be loading more than one process at a time due to python loading/altering feature') key=self.getKey(wfNumber,step) if key in self.configBuilders: return True for wf in self.mrd.workFlows: if float(wf.numId)!=wfNumber: continue if not hasattr(wf,'cmdStep%d'%(step)): continue if not getattr(wf,'cmdStep%d'%(step)): continue command=getattr(wf,'cmdStep%d'%(step)) opt=OptionsFromCommand(command) if opt: cb = ConfigBuilder(opt,with_input=True,with_output=True) cb.prepare() self.configBuilders[key]=copy.copy(cb) return True print "could not satisfy the request for step",step,"of workflow",wfNumber return False def getConfig(self,wfNumber,step): key=self.getKey(wfNumber,step) if not key in self.configBuilders: self.getProcess(wfNumber,step) if not key in self.configBuilders: return None return self.configBuilders[key].pythonCfgCode def identityTest(self,wfNumber,step): self.getProcess(wfNumber,step) key=self.getKey(wfNumber,step) if not key in self.configBuilders: return None cb=self.configBuilders[key] #need to compare those two for identity cb.process cd.pythonCfgCode def listAll(self): for wf in self.mrd.workFlows: step=1 print '---------------------' print 'process workflow',wf.numId print while self.load(float(wf.numId),step): p=self.getProcess(float(wf.numId),step) print ', '.join(s.label() for s in p.schedule) #print p.outputModules() step+=1
class MatrixToProcess: def __init__(self,what='standard',strict=True): from Configuration.PyReleaseValidation.MatrixReader import MatrixReader self.mrd = MatrixReader(what,noRun=True) self.mrd.prepare('all','',None) self.configBuilders={} self.processes={} self.strict=strict def getKey(self,wfNumber,step): return str(wfNumber)+':'+str(step) def getProcess(self,wfNumber,step): key=self.getKey(wfNumber,step) if not key in self.configBuilders: self.load(wfNumber,step) if not key in self.configBuilders: return None return self.configBuilders[key].process def load(self,wfNumber,step): from Configuration.Applications.ConfigBuilder import ConfigBuilder from Configuration.Applications.cmsDriverOptions import OptionsFromCommand import copy if len(self.configBuilders)!=0 and self.strict: raise Exception('one should never be loading more than one process at a time due to python loading/altering feature') key=self.getKey(wfNumber,step) if key in self.configBuilders: return True for wf in self.mrd.workFlows: if float(wf.numId)!=wfNumber: continue if not hasattr(wf,'cmdStep%d'%(step)): continue if not getattr(wf,'cmdStep%d'%(step)): continue command=getattr(wf,'cmdStep%d'%(step)) opt=OptionsFromCommand(command) if opt: cb = ConfigBuilder(opt,with_input=True,with_output=True) cb.prepare() self.configBuilders[key]=copy.copy(cb) return True print("could not satisfy the request for step",step,"of workflow",wfNumber) return False def getConfig(self,wfNumber,step): key=self.getKey(wfNumber,step) if not key in self.configBuilders: self.getProcess(wfNumber,step) if not key in self.configBuilders: return None return self.configBuilders[key].pythonCfgCode def identityTest(self,wfNumber,step): self.getProcess(wfNumber,step) key=self.getKey(wfNumber,step) if not key in self.configBuilders: return None cb=self.configBuilders[key] #need to compare those two for identity cb.process cd.pythonCfgCode def listAll(self): for wf in self.mrd.workFlows: step=1 print('---------------------') print('process workflow',wf.numId) print() while self.load(float(wf.numId),step): p=self.getProcess(float(wf.numId),step) print(', '.join(s.label() for s in p.schedule)) #print p.outputModules() step+=1
except Exception, e: bdir = os.getenv('CMSSW_RELEASE_BASE', None) if bdir != None: sys.path.insert( 0, bdir + '/src/Configuration/PyReleaseValidation/scripts') bdir = os.getenv('CMSSW_BASE', None) if bdir != None: sys.path.insert( 0, bdir + '/src/Configuration/PyReleaseValidation/scripts') from runTheMatrix import MatrixReader from runTheMatrix import MatrixRunner newWorkFlows = False opts = MatrixOpts() try: mrd = MatrixReader() except: mrd = MatrixReader(opts) newWorkFlows = True mrd.prepare() for wf in mrd.workFlows: if str(wf.numId) not in testList: continue print "processing ", wf.numId if newWorkFlows: updateNewWF(wf) else: updateOldWF(wf) ret = 0 if show: if not newWorkFlows: