def getAnalysisBeta(resDir, p = 0.875, scoreTres = 0.75, betaMin = 0.01, betaMax = 0.99, betaStep = 0.05):
"""
Gives the analysis for for a range of beta with other parameters fixed
"""
result = []
if not os.path.isdir(resDir):
print 'The directory mentioned does not exist:' + resDir
sys.exit()
pDir = os.path.join(resDir, 'p_'+ str(p))
if not os.path.isdir(pDir):
print 'The directory mentioned does not exist:' + pDir
sys.exit()
beta = betaMin
while beta <= betaMax:
bDir = os.path.join(pDir, 'beta_' + str(beta))
if not os.path.isdir(bDir):
print 'The directory mentioned does not exist:' + bDir
sys.exit()
data = ut.readPickle(os.path.join(bDir, 'result.pkl'))
found = False
for tup in data:
if abs(tup[0] - scoreTres) < eps:
found = True
print 'The value of scoreTres:' + str(tup[0])
result.append((beta,tup[1]))
break
if found is not True:
print 'Could not find the value for iteration with beta=' + str(beta)
sys.exit()
beta += betaStep
return result
def looper(spec) :
assert spec.weightedName not in sampleNames,"Duplicate sample name %s is not allowed."%spec.weightedName ; sampleNames.add(spec.weightedName)
pars = dict( list(conf.iteritems()) + [("baseSample",spec.name), ("sample",spec.weightedName) ] )
tup = self.sampleDict[spec.name]
inputFiles = utils.readPickle(self.inputFilesListFile(spec.name), "Have you looped?")[:spec.nFilesMax]
nEventsMax,nFilesMax = parseForNumberEvents(spec, tup, len(inputFiles), self.__nSlices)
inputFiles = inputFiles[:nFilesMax]
adjustedSteps = ([steps.master(xs = tup.xs, xsPostWeights = spec.xsPostWeights,
lumi = spec.overrideLumi if spec.overrideLumi!=None else tup.lumi,
lumiWarn = lumiWarn(tup.lumi, nEventsMax, spec.nFilesMax) )] +
self.listOfSteps(pars) )
nonSteps = [s for s in adjustedSteps if not issubclass(type(s), analysisStep)]
assert not nonSteps, "\n\nWarning, the following items from listOfSteps() are not analysisSteps:\n"+('\n'.join(' '+str(s) for s in nonSteps))
for step in filter(lambda s: s.only not in ['','data' if tup.lumi else 'sim'], adjustedSteps) : step.disabled = True
return analysisLooper(mainTree=self.mainTree(),
otherTreesToKeepWhenSkimming=self.otherTreesToKeepWhenSkimming(),
nEventsMax=nEventsMax,
leavesToBlackList=self.leavesToBlackList(),
steps=adjustedSteps,
calculables=allCalculables(self.listOfCalculables(pars),
spec.weights,
adjustedSteps),
inputFiles=inputFiles,
name=pars["sample"],
moveOutputFiles=self.moveOutputFiles,
localStem=self.localStem,
subDir="%(tag)s" % conf,
globalStem=self.globalStem,
quietMode=(self.__loop > 1) or self.__quiet,
skip=self.__skip
)
def mergeFunc(self, nSlices) :
cleanUpList = []
self.setupSteps(minimal = True)
self.calculablesUsed = set()
self.leavesUsed = set()
products = [collections.defaultdict(list) for step in self.steps]
for iSlice in range(nSlices) :
pickleFileBlocks = self.pickleFileName.split('/')
pickleFileBlocks.insert(-1,self.name)
pickleFileBlocks[-1] = pickleFileBlocks[-1].replace(self.name,self.childName(nSlices,iSlice))
cleanUpList.append( '/'.join(pickleFileBlocks[:-1]) )
dataByStep,calcsUsed,leavesUsed = utils.readPickle( '/'.join(pickleFileBlocks) )
self.calculablesUsed |= calcsUsed
self.leavesUsed |= leavesUsed
for stepDict,data in zip(products, dataByStep) :
for key,val in data.iteritems() : stepDict[key].append(val)
for step,stepDict in filter(lambda s: s[0].isSelector, zip(self.steps, products)) :
step.increment(True, sum(stepDict["nPass"]))
step.increment(False,sum(stepDict["nFail"]))
self.printStats()
print utils.hyphens
for iStep,step,stepDict in zip(range(len(self.steps)),self.steps,products) :
if iStep : rootFile.cd('/'.join(step.name for step in self.steps[:iStep+1] ))
step.mergeFunc(stepDict)
if not iStep: rootFile = r.TFile.Open(self.steps[0].outputFileName, "UPDATE")
rootFile.Close()
for dirName in cleanUpList : os.system("rm -fr %s"%dirName)
def mergeOutput(self,tag,looper) :
if not os.path.exists(self.jobsFile(tag,looper.name)) : return
nSlices = utils.readPickle(self.jobsFile(tag,looper.name)) # number of slices used when --loop was passed
incompleteSlices = looper.incompleteSlices(nSlices)
if not incompleteSlices:
looper.mergeFunc(nSlices)
for item in incompleteSlices:
if self.__resubmit:
batch.submitJob(item)
else:
print item
def looper(spec) :
assert spec.weightedName not in sampleNames,"Duplicate sample name %s is not allowed."%spec.weightedName ; sampleNames.add(spec.weightedName)
pars = dict( list(conf.iteritems()) + [("baseSample",spec.name), ("sample",spec.weightedName) ] )
tup = self.sampleDict[spec.name]
inputFiles = utils.readPickle(self.inputFilesListFile(spec.name))[:spec.nFilesMax]
nEventsMax,nFilesMax = parseForNumberEvents(spec, tup, len(inputFiles), self.__nSlices)
inputFiles = inputFiles[:nFilesMax]
adjustedSteps = [ steps.Master.Master(xs = tup.xs, xsPostWeights = spec.xsPostWeights,
lumi = spec.overrideLumi if spec.overrideLumi!=None else tup.lumi,
lumiWarn = lumiWarn(tup.lumi, nEventsMax, spec.nFilesMax) )
] + steps.adjustSteps(self.listOfSteps(pars), "Data" if tup.lumi else "Mc")
return analysisLooper( mainTree = self.mainTree(), otherTreesToKeepWhenSkimming = self.otherTreesToKeepWhenSkimming(),
nEventsMax = nEventsMax, leavesToBlackList = self.leavesToBlackList(),
steps = adjustedSteps, calculables = allCalculables( self.listOfCalculables(pars), spec.weights, adjustedSteps ),
inputFiles = inputFiles, name = pars["sample"],
localStem = self.localStem, subDir = "%(tag)s"%conf,
globalStem = self.globalStem, quietMode = self.__loop>1 )
def mergeOutput(self,tag,looper) :
if not os.path.exists(self.jobsFile(tag,looper.name)) : return
nSlices = utils.readPickle(self.jobsFile(tag,looper.name))
if looper.readyMerge(nSlices) : looper.mergeFunc(nSlices)
def recordedInvMicrobarnsShotgun(jsons, cores = 2, cacheDir = './' ) :
pickles = ["%s/%d.pickle"%(cacheDir,hash(str(sorted([(key,val) for key,val in json.iteritems()])))) for json in jsons]
def worker(pickle, json) :
if not os.path.exists(pickle) : utils.writePickle(pickle, recordedInvMicrobarns(json))
utils.operateOnListUsingQueue(cores, utils.qWorker(worker), zip(pickles,jsons))
return [utils.readPickle(pickle) for pickle in pickles ]