def __init__(self,what='standard',strict=True):
from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
self.mrd = MatrixReader(what,noRun=True)
self.mrd.prepare('all','',None)
self.configBuilders={}
self.processes={}
self.strict=strict
def showRaw(opt):
mrd = MatrixReader(opt)
mrd.showRaw(opt.useInput,
opt.refRel,
opt.fromScratch,
opt.raw,
opt.step1Only,
selected=opt.testList)
return 0
def __init__(self, opt):
cmd.Cmd.__init__(self)
self.opt_ = opt
self.matrices_ = {}
tmp = MatrixReader(self.opt_)
for what in tmp.files:
self.opt_.what = what
self.matrices_[what] = MatrixReader(self.opt_)
self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel,
self.opt_.fromScratch)
os.system("clear")
def __init__(self,what='standard',strict=True):
from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
self.mrd = MatrixReader(what,noRun=True)
self.mrd.prepare('all','',None)
self.configBuilders={}
self.processes={}
self.strict=strict
def runSelected(opt):
mrd = MatrixReader(opt)
mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
# test for wrong input workflows
if opt.testList:
definedSet = set([dwf.numId for dwf in mrd.workFlows])
testSet = set(opt.testList)
undefSet = testSet - definedSet
if len(undefSet) > 0:
raise ValueError('Undefined workflows: ' +
', '.join(map(str, list(undefSet))))
ret = 0
if opt.show:
mrd.show(opt.testList, opt.extended, opt.cafVeto)
if opt.testList: print('testListected items:', opt.testList)
else:
mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
ret = mRunnerHi.runTests(opt)
if opt.wmcontrol:
if ret != 0:
print('Cannot go on with wmagent injection with failing workflows')
else:
wfInjector = MatrixInjector(opt,
mode=opt.wmcontrol,
options=opt.wmoptions)
ret = wfInjector.prepare(mrd, mRunnerHi.runDirs)
if ret == 0:
wfInjector.upload()
wfInjector.submit()
return ret
def runSelected(opt):
mrd = MatrixReader(opt)
mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
ret = 0
if opt.show:
mrd.show(opt.testList, opt.extended, opt.cafVeto)
if opt.testList: print 'testListected items:', opt.testList
else:
mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
ret = mRunnerHi.runTests(opt)
if opt.wmcontrol:
if ret != 0:
print 'Cannot go on with wmagent injection with failing workflows'
else:
wfInjector = MatrixInjector(opt,
mode=opt.wmcontrol,
options=opt.wmoptions)
ret = wfInjector.prepare(mrd, mRunnerHi.runDirs)
if ret == 0:
wfInjector.upload()
wfInjector.submit()
return ret
def runSelected(opt):
mrd = MatrixReader(opt)
mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
# test for wrong input workflows
if opt.testList:
definedWF = []
for dwf in mrd.workFlows:
definedWF.append(dwf.numId)
for twf in opt.testList:
if twf not in definedWF:
raise ValueError('Not defined workflow ', twf, ' requested')
ret = 0
if opt.show:
mrd.show(opt.testList, opt.extended, opt.cafVeto)
if opt.testList: print('testListected items:', opt.testList)
else:
mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
ret = mRunnerHi.runTests(opt)
if opt.wmcontrol:
if ret != 0:
print('Cannot go on with wmagent injection with failing workflows')
else:
wfInjector = MatrixInjector(opt,
mode=opt.wmcontrol,
options=opt.wmoptions)
ret = wfInjector.prepare(mrd, mRunnerHi.runDirs)
if ret == 0:
wfInjector.upload()
wfInjector.submit()
return ret
def runSelected(opt):
mrd = MatrixReader(opt)
mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
ret = 0
if opt.show:
mrd.show(opt.testList, opt.extended, opt.cafVeto)
if opt.testList : print('testListected items:', opt.testList)
else:
mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
ret = mRunnerHi.runTests(opt)
if opt.wmcontrol:
if ret!=0:
print('Cannot go on with wmagent injection with failing workflows')
else:
wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions)
ret= wfInjector.prepare(mrd,
mRunnerHi.runDirs)
if ret==0:
wfInjector.upload()
wfInjector.submit()
return ret
def showRaw(opt):
mrd = MatrixReader(opt)
mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList)
return 0
from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner
except Exception, e:
bdir=os.getenv('CMSSW_RELEASE_BASE', None)
if bdir != None:
sys.path.insert(0,bdir+'/src/Configuration/PyReleaseValidation/scripts')
bdir=os.getenv('CMSSW_BASE', None)
if bdir != None:
sys.path.insert(0,bdir+'/src/Configuration/PyReleaseValidation/scripts')
from runTheMatrix import MatrixReader
from runTheMatrix import MatrixRunner
newWorkFlows = False
opts = MatrixOpts()
try:
mrd = MatrixReader()
except:
mrd = MatrixReader(opts)
newWorkFlows = True
mrd.prepare()
for wf in mrd.workFlows:
if str(wf.numId) not in testList: continue
print "processing ", wf.numId
if newWorkFlows: updateNewWF(wf)
else: updateOldWF(wf)
ret = 0
if show:
if not newWorkFlows:
class MatrixToProcess:
def __init__(self,what='standard',strict=True):
from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
self.mrd = MatrixReader(what,noRun=True)
self.mrd.prepare('all','',None)
self.configBuilders={}
self.processes={}
self.strict=strict
def getKey(self,wfNumber,step):
return str(wfNumber)+':'+str(step)
def getProcess(self,wfNumber,step):
key=self.getKey(wfNumber,step)
if not key in self.configBuilders:
self.load(wfNumber,step)
if not key in self.configBuilders:
return None
return self.configBuilders[key].process
def load(self,wfNumber,step):
from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder
from Configuration.PyReleaseValidation.cmsDriverOptions import OptionsFromCommand
import copy
if len(self.configBuilders)!=0 and self.strict:
raise Exception('one should never be loading more than one process at a time due to python loading/altering feature')
key=self.getKey(wfNumber,step)
if key in self.configBuilders:
return True
for wf in self.mrd.workFlows:
if float(wf.numId)!=wfNumber: continue
if not hasattr(wf,'cmdStep%d'%(step)): continue
if not getattr(wf,'cmdStep%d'%(step)): continue
command=getattr(wf,'cmdStep%d'%(step))
opt=OptionsFromCommand(command)
if opt:
cb = ConfigBuilder(opt,with_input=True,with_output=True)
cb.prepare()
self.configBuilders[key]=copy.copy(cb)
return True
print "could not satisfy the request for step",step,"of workflow",wfNumber
return False
def getConfig(self,wfNumber,step):
key=self.getKey(wfNumber,step)
if not key in self.configBuilders: self.getProcess(wfNumber,step)
if not key in self.configBuilders: return None
return self.configBuilders[key].pythonCfgCode
def identityTest(self,wfNumber,step):
self.getProcess(wfNumber,step)
key=self.getKey(wfNumber,step)
if not key in self.configBuilders: return None
cb=self.configBuilders[key]
#need to compare those two for identity
cb.process
cd.pythonCfgCode
def listAll(self):
for wf in self.mrd.workFlows:
step=1
print '---------------------'
print 'process workflow',wf.numId
print
while self.load(float(wf.numId),step):
p=self.getProcess(float(wf.numId),step)
print ', '.join(s.label() for s in p.schedule)
#print p.outputModules()
step+=1
class MatrixToProcess:
def __init__(self,what='standard',strict=True):
from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
self.mrd = MatrixReader(what,noRun=True)
self.mrd.prepare('all','',None)
self.configBuilders={}
self.processes={}
self.strict=strict
def getKey(self,wfNumber,step):
return str(wfNumber)+':'+str(step)
def getProcess(self,wfNumber,step):
key=self.getKey(wfNumber,step)
if not key in self.configBuilders:
self.load(wfNumber,step)
if not key in self.configBuilders:
return None
return self.configBuilders[key].process
def load(self,wfNumber,step):
from Configuration.Applications.ConfigBuilder import ConfigBuilder
from Configuration.Applications.cmsDriverOptions import OptionsFromCommand
import copy
if len(self.configBuilders)!=0 and self.strict:
raise Exception('one should never be loading more than one process at a time due to python loading/altering feature')
key=self.getKey(wfNumber,step)
if key in self.configBuilders:
return True
for wf in self.mrd.workFlows:
if float(wf.numId)!=wfNumber: continue
if not hasattr(wf,'cmdStep%d'%(step)): continue
if not getattr(wf,'cmdStep%d'%(step)): continue
command=getattr(wf,'cmdStep%d'%(step))
opt=OptionsFromCommand(command)
if opt:
cb = ConfigBuilder(opt,with_input=True,with_output=True)
cb.prepare()
self.configBuilders[key]=copy.copy(cb)
return True
print("could not satisfy the request for step",step,"of workflow",wfNumber)
return False
def getConfig(self,wfNumber,step):
key=self.getKey(wfNumber,step)
if not key in self.configBuilders: self.getProcess(wfNumber,step)
if not key in self.configBuilders: return None
return self.configBuilders[key].pythonCfgCode
def identityTest(self,wfNumber,step):
self.getProcess(wfNumber,step)
key=self.getKey(wfNumber,step)
if not key in self.configBuilders: return None
cb=self.configBuilders[key]
#need to compare those two for identity
cb.process
cd.pythonCfgCode
def listAll(self):
for wf in self.mrd.workFlows:
step=1
print('---------------------')
print('process workflow',wf.numId)
print()
while self.load(float(wf.numId),step):
p=self.getProcess(float(wf.numId),step)
print(', '.join(s.label() for s in p.schedule))
#print p.outputModules()
step+=1
except Exception, e:
bdir = os.getenv('CMSSW_RELEASE_BASE', None)
if bdir != None:
sys.path.insert(
0, bdir + '/src/Configuration/PyReleaseValidation/scripts')
bdir = os.getenv('CMSSW_BASE', None)
if bdir != None:
sys.path.insert(
0, bdir + '/src/Configuration/PyReleaseValidation/scripts')
from runTheMatrix import MatrixReader
from runTheMatrix import MatrixRunner
newWorkFlows = False
opts = MatrixOpts()
try:
mrd = MatrixReader()
except:
mrd = MatrixReader(opts)
newWorkFlows = True
mrd.prepare()
for wf in mrd.workFlows:
if str(wf.numId) not in testList: continue
print "processing ", wf.numId
if newWorkFlows: updateNewWF(wf)
else: updateOldWF(wf)
ret = 0
if show:
if not newWorkFlows:
