def manageSecondaries(self,updates,reportAll,reports) :
def doUpdate(name) : return updates==True or type(updates)==str and name in updates.split(',')
def doReport(name) : return reportAll==True or type(reports)==str and name in reports.split(',')
def manage(conf,secondary) :
if self.__nocheck and not doUpdate(secondary.name) : return
org = organizer(conf['tag'], [ss for ss in self.sampleSpecs(conf['tag'])
if ss['name'] in secondary.baseSamples() or not secondary.baseSamples()])
index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None))
if index==None :
print " !! Not found: %s %s"%(secondary.name,secondary.moreNames)
print org
return
org.dropSteps( allButIndices = [index])
if doUpdate(secondary.name) : secondary.doCache(org)
else : secondary.checkCache(org)
def report(conf,secondary) :
secondary.allSamples = [ss.weightedName for ss in self.filteredSamples(conf)]
if doReport(secondary.name) : secondary.reportCache()
confLoopers = []
for conf in self.readyConfs:
looper_list = self.listsOfLoopers[conf['tag']]
if looper_list:
confLoopers.append((conf, looper_list[0]))
for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False)
args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps[:self.indexOfInvertedLabel(looper.steps)])] for conf,looper in confLoopers],[])
if reports==True :
print '\n'.join(sorted(set(s.name for c,s in args)))
sys.exit(0)
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(report), args)
def loop(self) :
wl = self.workList()
if self.__jobId!=None : self.func(*(wl[0]))
elif not self.__profile : utils.operateOnListUsingQueue(self.__loop, utils.qWorker(self.func), wl)
else :
import cProfile
self.listOfLoopersForProf = wl
cProfile.run("someInstance.profileLoop()","resultProfile.out")
def makeInputFileLists(self) :
def makeFileList(name) :
if os.path.exists(self.inputFilesListFile(name)) and self.useCachedFileLists() : return
fileNames = eval(self.sampleDict[name].filesCommand)
assert fileNames, "The command '%s' produced an empty list of files"%self.sampleDict[name].filesCommand
tmpDir,localFileName,globalFileName = self.globalToLocal(self.inputFilesListFile(name))
utils.writePickle(localFileName, fileNames)
self.localToGlobal(tmpDir, localFileName, globalFileName)
sampleNames = set(sum([[sampleSpec.name for sampleSpec in self.filteredSamples(conf)] for conf in self.configurations],[]))
utils.operateOnListUsingQueue(self.__loop, utils.qWorker(makeFileList), [(name,) for name in sampleNames] )
def makeInputFileLists(self) :
def nEventsFile(fileName):
if not configuration.computeEntriesAtMakeFileList(): return None
return utils.nEventsFile(fileName,'/'.join(self.mainTree()))
def makeFileList(name) :
fName = self.inputFilesListFile(name)
if os.path.exists(fName) and self.useCachedFileLists() : return
fileNames = eval(self.sampleDict[name].filesCommand)
assert fileNames, "The command '%s' produced an empty list of files"%self.sampleDict[name].filesCommand
utils.mkdir(os.path.dirname(fName))
utils.writePickle(fName, zip(fileNames,map(nEventsFile, fileNames)))
sampleNames = set(sum([[sampleSpec.name for sampleSpec in self.filteredSamples(conf)] for conf in self.configurations],[]))
utils.operateOnListUsingQueue(self.__loop, utils.qWorker(makeFileList), [(name,) for name in sampleNames] )
def manageSecondaries(self,update) :
def manage(conf,secondary) :
org = self.organizer(conf)
index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None))
if index==None :
print " !! Not found: %s %s"%(secondary.name,secondary.moreNames)
return
org.dropSteps( allButIndices = [index])
if update==True or (type(update)==str and secondary.name in update.split(',')) :
secondary.doCache(org)
else : secondary.checkCache(org)
if self.__nocheck : return
confLoopers = [(conf,self.listsOfLoopers[conf['tag']][0]) for conf in self.readyConfs]
for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False)
args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps)] for conf,looper in confLoopers],[])
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
def __init__(self,cmds, nCores=4, site=None):
self.nCores = nCores
self.cmds = cmds
if site == 'ic':
localdir = '/'.join(__file__.split('/')[:-1])
prelude = '''
export SCRAM_ARCH=slc5_amd64_gcc472
source /vols/cms/grid/setup.sh
cd /home/hep/bbetchar/work/CMSSW_6_2_0_patch1/src
eval `scram runtime -sh`
cd %s
'''%localdir
def sub(iJob,cmd):
with tempfile.NamedTemporaryFile('w',prefix='job%04d_'%iJob,suffix='.sh') as tmp:
print>>tmp, prelude
print>>tmp, cmd
tmp.flush()
os.system('./ic_cmsSub.sh %s'%tmp.name)
utils.operateOnListUsingQueue(self.nCores, utils.qWorker(sub), enumerate(self.cmds))
else:
self.default()
def manageSecondaries(self,updates,reports) :
def doUpdate(name) : return updates==True or type(updates)==str and name in updates.split(',')
def doReport(name) : return reports==True or type(reports)==str and name in reports.split(',')
def manage(conf,secondary) :
if self.__nocheck and not doUpdate(secondary.name) : return
org = organizer(conf['tag'], [ss for ss in self.sampleSpecs(conf['tag'])
if ss['name'] in secondary.baseSamples() or not secondary.baseSamples()])
index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None))
if index==None :
print " !! Not found: %s %s"%(secondary.name,secondary.moreNames)
return
org.dropSteps( allButIndices = [index])
if doUpdate(secondary.name) : secondary.doCache(org)
else : secondary.checkCache(org)
def report(conf,secondary) :
if doReport(secondary.name) : secondary.reportCache()
confLoopers = [(conf,self.listsOfLoopers[conf['tag']][0]) for conf in self.readyConfs]
for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False)
args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps)] for conf,looper in confLoopers],[])
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(report), args)
def mergeAllOutput(self) :
args = sum([[(conf['tag'],looper) for looper in self.listsOfLoopers[conf['tag']]] for conf in self.configurations],[])
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(self.mergeOutput), args)
def concludeAll(self) : utils.operateOnListUsingQueue( configuration.nCoresDefault(), utils.qWorker(self.conclude), zip(self.readyConfs) ) def organizer(self, config) : return organizer.organizer(config['tag'], self.sampleSpecs(config['tag']))
def concludeAll(self) : utils.operateOnListUsingQueue( configuration.nCoresDefault(), utils.qWorker(self.conclude), zip(self.readyConfs) ) def organizer(self, config, verbose = True, prefixesNoScale=[]) :
def default(self):
utils.operateOnListUsingQueue(self.nCores, utils.qWorker(os.system), zip(self.cmds))
def recordedInvMicrobarnsShotgun(jsons, cores = 2, cacheDir = './' ) :
pickles = ["%s/%d.pickle"%(cacheDir,hash(str(sorted([(key,val) for key,val in json.iteritems()])))) for json in jsons]
def worker(pickle, json) :
if not os.path.exists(pickle) : utils.writePickle(pickle, recordedInvMicrobarns(json))
utils.operateOnListUsingQueue(cores, utils.qWorker(worker), zip(pickles,jsons))
return [utils.readPickle(pickle) for pickle in pickles ]
def recordedInvMicrobarns(json) :
pars = lumiQueryAPI.ParametersObject()
pars.noWarnings = True
pars.norm = 1.0
pars.lumiversion = '0001'
pars.beammode = 'STABLE BEAMS'
session,svc = lumiQueryAPI.setupSession( connectString = 'frontier://LumiCalc/CMS_LUMI_PROD',
parameters = pars, siteconfpath = None, debug = False)
lumidata = lumiQueryAPI.recordedLumiForRange (session, pars, jsonToIFP(json))
return sum(lumiQueryAPI.calculateTotalRecorded(dataperRun[2]) for dataperRun in lumidata if dataperRun[1])
def recordedInvMicrobarnsShotgun(jsons, cores = 2, cacheDir = './' ) :
pickles = ["%s/%d.pickle"%(cacheDir,hash(str(sorted([(key,val) for key,val in json.iteritems()])))) for json in jsons]
def worker(pickle, json) :
if not os.path.exists(pickle) : utils.writePickle(pickle, recordedInvMicrobarns(json))
utils.operateOnListUsingQueue(cores, utils.qWorker(worker), zip(pickles,jsons))
return [utils.readPickle(pickle) for pickle in pickles ]
if __name__=='__main__' :
print
import utils
if len(sys.argv)<2 : print 'Pass list of "{json}" and/or filenames as argument'; sys.exit(0)
def output(arg) :
json = eval(arg if '{' in arg else open(arg).readline())
lumi = recordedInvMicrobarns(json)
print "%.4f/pb in %s"%(lumi/1e6,arg)
print
utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(output), [(a,) for a in sys.argv[1:]])