def makeXsecData(options, data):
dummy,description = treedescription()
allpar = description.split(':')
xsecdata = {}
if data == {}:
print "Your data array is empty! Exiting"
sys.exit()
for key in data:
CLsexp = float(data[key][allpar.index("CLsexp")])
upperLimit = float(data[key][allpar.index("upperLimit")])
expectedUpperLimit = float(data[key][allpar.index("expectedUpperLimit")])
expectedUpperLimitMinus1Sig = float(data[key][allpar.index("expectedUpperLimitMinus1Sig")])
expectedUpperLimitPlus1Sig = float(data[key][allpar.index("expectedUpperLimitPlus1Sig")])
expectedUpperLimitMinus2Sig = float(data[key][allpar.index("expectedUpperLimitMinus2Sig")])
expectedUpperLimitPlus2Sig = float(data[key][allpar.index("expectedUpperLimitPlus2Sig")])
fID = int(float(data[key][allpar.index("fID")]))
#print "-1: %.3f, -2: %.3f" % (expectedUpperLimitMinus1Sig, expectedUpperLimitMinus2Sig)
# TODO need to extract this plus uncertainty from SignalDB
# NOTE: hardcoded for 1 proc, hence the extra [0]
xsec = xsecs[key][0] * 1000
xsecUp = (1+xsecs[key][1]) * xsec
xsecDown = (1-xsecs[key][1]) * xsec
print "%s: xsec=%f relunc=%f up=%f down=%f" % (key, xsec, xsecs[key][1], xsecUp, xsecDown)
xsec2 = xsec/2
xsec4 = xsec/4
xsec8 = xsec/8
xsec2Up = xsecUp/2
xsec4Up = xsecUp/4
xsec8Up = xsecUp/8
xsec2Down = xsecDown/2
xsec4Down = xsecDown/4
xsec8Down = xsecDown/8
xsecdata[key] = {"upperLimit": upperLimit, "expectedUpperLimit" : expectedUpperLimit, "expectedUpperLimitMinus1Sig" : expectedUpperLimitMinus1Sig,
"expectedUpperLimitPlus1Sig" : expectedUpperLimitPlus1Sig, "expectedUpperLimitMinus2Sig" : expectedUpperLimitMinus2Sig,
"expectedUpperLimitPlus2Sig" : expectedUpperLimitPlus2Sig, "xsec" : xsec, "xsecUp" : xsecUp, "xsecDown" : xsecDown, "xsec2": xsec2,
"xsec2Up" : xsec2Up, "xsec2Down" : xsec2Down,
"xsec4Up" : xsec4Up, "xsec4Down" : xsec4Down,
"xsec8Up" : xsec8Up, "xsec8Down" : xsec8Down,
"xsec4": xsec4, "xsec8": xsec8, "fID" : fID, "CLsexp": CLsexp}
return xsecdata
def harvestToDict(harvestInputFileName=""):
print ">>> entering harvestToDict()"
massPlaneDict = {}
harvestInput = open(harvestInputFileName, "r")
print harvestInputFileName
descriptionFileName = harvestInputFileName.rsplit(
'/', 1)[0] + "/summary_harvest_tree_description.*"
os.system("cp %s ." % descriptionFileName)
descriptionFileName = harvestInputFileName + "_infoFile"
os.system("cp %s output_infoFile" % descriptionFileName)
os.system("cp %s output" % harvestInputFileName)
from summary_harvest_tree_description import treedescription
dummy, fieldNames = treedescription()
fieldNames = fieldNames.split(':')
if ".json" in harvestInputFileName and 0:
print ">>> Interpreting json file"
with open(harvestInput) as inputJSONFile:
inputJSONFile = json.load(inputJSONFile)
else:
print ">>> Interpreting text file"
for massLine in harvestInput.readlines():
values = massLine.split()
values = dict(zip(fieldNames, values))
massPoint = (float(values["m0/F"]), float(values["m12/F"]))
massPlaneDict[massPoint] = {
"CLs":
ROOT.RooStats.PValueToSignificance(float(values["CLs/F"])),
"CLsexp":
ROOT.RooStats.PValueToSignificance(float(values["CLsexp/F"])),
"clsu1s":
ROOT.RooStats.PValueToSignificance(float(values["clsu1s/F"])),
# "clsu2s": ROOT.RooStats.PValueToSignificance( float(values["clsu2s/F"]) ) ,
"clsd1s":
ROOT.RooStats.PValueToSignificance(float(values["clsd1s/F"])),
# "clsd2s": ROOT.RooStats.PValueToSignificance( float(values["clsd2s/F"]) ) ,
}
print massPlaneDict
return massPlaneDict
def main():
options = parseCmdLine(sys.argv)
dummy, description = treedescription()
allpar = description.split(':')
xsecIdx = allpar.index('xsec')
ULIdx = allpar.index('upperLimit')
if options.expected:
ULIdx = allpar.index('expectedUpperLimit')
m0Idx = allpar.index('m0')
m12Idx = allpar.index('m12')
fIdIdx = allpar.index('fID')
p0Idx = allpar.index('p0')
p1Idx = allpar.index('p1')
CLsIdx = allpar.index('CLs')
CLsexpIdx = allpar.index('CLsexp')
covqualIdx = allpar.index('covqual')
dodgycovIdx = allpar.index('dodgycov')
failedcovIdx = allpar.index('failedcov')
failedfitIdx = allpar.index('failedfit')
failedp0Idx = allpar.index('failedp0')
failedstatusIdx = allpar.index('failedstatus')
fitstatusIdx = allpar.index('fitstatus')
#f = open(options.filename, "r")
# get files to read
f = open(
"Outputs/%s_combined_fixSigXSecNominal__1_harvest_list_infoFile" %
options.grid, "r")
filenames = {}
for l in f.readlines():
(N, fname) = l.strip().split(":")
filenames[int(
N.strip())] = "Outputs/%s_%s_fixSigXSecNominal__1_harvest_list" % (
options.grid, fname.strip())
f.close()
data = {}
CLsdata = {}
for N in filenames:
f = open(filenames[N])
for l in f.readlines():
vals = l.strip().split(' ')
key = "%d_%d" % (int(float(vals[m0Idx])), int(float(vals[m12Idx])))
if key != options.point:
continue
data[N] = [float(vals[ULIdx]), float(vals[xsecIdx]), N]
CLsdata[N] = [
float(vals[p0Idx]),
float(vals[p1Idx]),
float(vals[CLsIdx]),
float(vals[CLsexpIdx])
]
f.close()
if options.seperate:
print "# x\ty\tUL\txsec\tSR\tCLs"
else:
print "# x\ty\tUL*xsec\tSR"
for N in sorted(data):
(x, y) = options.point.split("_")
x = int(x)
y = int(y)
if options.seperate:
print "%d\t%d\t%f\t%f\t%d\t%.2f\t%.2f (exp)" % (
x, y, data[N][0], data[N][1], data[N][2], CLsdata[N][2],
CLsdata[N][3])
else:
print "%d\t%d\t%f\t%d" % (x, y, data[N][0] * data[N][1],
data[N][2])
covqual = int(float(vals[covqualIdx]))
dodgycov = int(float(vals[dodgycovIdx]))
failedcov = int(float(vals[failedcovIdx]))
failedfit = int(float(vals[failedfitIdx]))
failedp0 = int(float(vals[failedp0Idx]))
failedstatus = int(float(vals[failedstatusIdx]))
fitstatus = int(float(vals[fitstatusIdx]))
# verbose print line to see what's going on with a point
print "\t ==> covqual = %d, dodgycov = %d, failedcov = %d, failedfit = %d, failedp0 = %d, failedstatus = %d, fitstatus = %d" % (
covqual, dodgycov, failedcov, failedfit, failedp0, failedstatus,
fitstatus)
return
#g.SetPoint(counter,i[0],StatTools.GetSigma(i[1]))
counter+=1
g.SetName(self.name)
g.SetLineColor(color)
g.SetMarkerColor(color)
g.SetLineWidth(2)
self.tgraph=g
return g
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
# Some global variables
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
from summary_harvest_tree_description import treedescription
dummy,description = treedescription()
allpar = description.split(':')
print dummy,description
# INPUTDIR has the workspaces from HistFitter
INPUTDIR="../../results/"
INPUTDIR_SIGNALDB="/afs/cern.ch/atlas/groups/susy/0lepton/ZeroLeptonFitter/ZeroLepton-00-00-36_Light/"
# OUTPUTDIR is where the combination of these using hadd will go, as well as
# all the list files for the contours, the histograms and the plots
OUTPUTDIR="Outputs/"
PLOTSDIR="plots/"
try:
os.mkdir(OUTPUTDIR)
except:
def main():
options = parseCmdLine(sys.argv)
dummy, description = treedescription()
allpar = description.split(':')
if options.filename.find("Moriond13") != -1:
# from HistFitter v24 source code
description = "p0:p1:CLs:"
description += "mode:nexp:"
description += "seed:"
description += "CLsexp:"
description += "fID:sigma0:sigma1:"
description += "clsu1s:clsd1s:clsu2s:clsd2s:"
description += "upperLimit:upperLimitEstimatedError:expectedUpperLimit:expectedUpperLimitPlus1Sig:expectedUpperLimitPlus2Sig:expectedUpperLimitMinus1Sig:expectedUpperLimitMinus2Sig:xsec:excludedXsec:"
description += "m0:m12"
allpar = description.split(':')
xsecIdx = allpar.index('xsec')
ULIdx = allpar.index('upperLimit')
if options.expected:
ULIdx = allpar.index('expectedUpperLimit')
if not "pMSSM" in options.filename:
m0Idx = allpar.index('m0')
m12Idx = allpar.index('m12')
else:
m0Idx = allpar.index('M1')
m12Idx = allpar.index('M2')
fIdIdx = allpar.index('fID')
p0Idx = allpar.index('p0')
p1Idx = allpar.index('p1')
CLsIdx = allpar.index('CLs')
CLsexpIdx = allpar.index('CLsexp')
covqualIdx = allpar.index('covqual')
dodgycovIdx = allpar.index('dodgycov')
failedcovIdx = allpar.index('failedcov')
failedfitIdx = allpar.index('failedfit')
failedp0Idx = allpar.index('failedp0')
failedstatusIdx = allpar.index('failedstatus')
fitstatusIdx = allpar.index('fitstatus')
f = open(options.filename, "r")
data = {}
CLsdata = {}
for l in f.readlines():
vals = l.strip().split(' ')
key = "%d_%d" % (int(float(vals[m0Idx])), int(float(vals[m12Idx])))
data[key] = [
float(vals[ULIdx]),
float(vals[xsecIdx]),
int(vals[fIdIdx])
]
CLsdata[key] = [
float(vals[p0Idx]),
float(vals[p1Idx]),
float(vals[CLsIdx]),
float(vals[CLsexpIdx])
]
if options.seperate:
print "# x\ty\tUL\txsec\tSR\tCLs"
elif options.hepdata:
print makeHepDataHeader(options.filename)
else:
print "# x\ty\tUL*xsec\tSR"
for key in sorted(data.iterkeys(), key=lambda a: map(int, a.split('_'))):
(x, y) = key.split("_")
x = int(x)
y = int(y)
## HACK to filter e.g. rectangle (easier to isolate crashed points)
#if not (x > 200 and x < 300): continue
#if not (y > 100 and y < 200): continue
#if not y == 0: continue
if options.seperate:
print "%d\t%d\t%f\t%f\t%d\t%e\t%e (exp)" % (
x, y, data[key][0], data[key][1], data[key][2],
CLsdata[key][2], CLsdata[key][3])
elif options.hepdata:
UL = data[key][0] * data[key][1]
SR = GetSRName(data[key][2])
print "%d; %d; %e; %s" % (x, y, UL, SR)
else:
print "%d\t%d\t%f\t%d" % (x, y, data[key][0] * data[key][1],
data[key][2])
covqual = int(float(vals[covqualIdx]))
dodgycov = int(float(vals[dodgycovIdx]))
failedcov = int(float(vals[failedcovIdx]))
failedfit = int(float(vals[failedfitIdx]))
failedp0 = int(float(vals[failedp0Idx]))
failedstatus = int(float(vals[failedstatusIdx]))
fitstatus = int(float(vals[fitstatusIdx]))
# verbose print line to see what's going on with a point
#print "\t ==> covqual = %d, dodgycov = %d, failedcov = %d, failedfit = %d, failedp0 = %d, failedstatus = %d, fitstatus = %d" % (covqual, dodgycov, failedcov, failedfit, failedp0, failedstatus, fitstatus)
return
def readData(options, filename):
try:
f = open(filename)
except:
print "Cannot open file %s" % filename
return
print "Reading data from %s" % filename
dummy,description = treedescription()
allpar = description.split(':')
#extend this to more pars on gridname if wanted
par1_s = "mwino"
data = {}
for l in f.readlines():
d = l.strip().split()
selectpar = "expectedUpperLimit" # makes NO sense to use something different here
pval = float( d[allpar.index(selectpar)])
par1 = int(float( d[allpar.index(par1_s)]))
key = "%d" % (par1)
# float strings -> so make them float, then an int to throw away .0000 and then to bool
failedcov = bool(int(float(d[allpar.index("failedcov")]))) # Mediocre cov matrix quality
covqual = int(float(d[allpar.index("covqual")])) # covqual
failedfit = bool(int(float(d[allpar.index("failedfit")]))) # Fit failure
failedp0 = bool(int(float(d[allpar.index("failedp0")]))) # Base p0 ~ 0.5 (this can reject good fits)!
fitstatus = bool(int(float(d[allpar.index("fitstatus")]))) # Fit status from Minuit
nofit = bool(int(float(d[allpar.index("nofit")]))) # Whether there's a fit present
if failedfit or failedcov:
print "WARNING: fit failed for %s" % key
continue
if covqual < 3 and covqual != -1:
print "WARNING: covqual=%d for %s" % (covqual, key)
continue
# this whole block is stolen from makeContoursNew.py. Do we need a module for this? Yes, we do.... --GJ, 4 feb 15
if selectpar == "expectedUpperLimit" and pval < 0.00001:
print "INFO: %s removing %s < 0.00001 for %s" % (filename, selectpar, key)
continue
# 110 is 20 times our default step -> this is almost certainly a bug
if selectpar == "expectedUpperLimit" and pval == 110.0:
print "INFO: %s removing expUL==110.0 for %s" % (filename, key)
continue
# if -1sig, -2sig == 0 and +1sig, 2sig == 100 -> almost certainly a bug too
if selectpar == "expectedUpperLimit" and float(d[allpar.index("expectedUpperLimitMinus1Sig")]) == 0.0 and float(d[allpar.index("expectedUpperLimitMinus2Sig")]) == 0.0 and float(d[allpar.index("expectedUpperLimitPlus1Sig")]) == 100.0 and float(d[allpar.index("expectedUpperLimitPlus2Sig")]) == 100.0:
print "INFO: %s removing point %s with expULMinus1Sig == expULMinus2Sig == 0 and expULPlus1Sig == expULPlus2Sig == 100" % (filename, key)
continue
if selectpar == "expectedUpperLimit" and float(d[allpar.index("upperLimit")]) == 0.0:
print "INFO: %s removing obsUL=0.0 for %s" % (filename, key)
continue
# throw away points with CLsexp > 0.99 and UL < 1.0 and CLs=-1 and UL<1 when merging on UL
CLsExp = float( d[allpar.index("CLsexp")])
if selectpar == "expectedUpperLimit" and pval < 1.0 and (CLsExp>0.99 or CLsExp<0) and float( d[allpar.index("upperLimit")])<1:
if CLsExp>0.99: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=1 for %s" % (filename, key)
elif CLsExp<0: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=-1 for %s" % (filename, key)
d[allpar.index("CLsexp")] = str(0.01)
d[allpar.index("CLs")] = str(0.01)
d[allpar.index("clsu1s")] = str(0.01)
d[allpar.index("clsd1s")] = str(0.01)
d[allpar.index("p1")] = str(0.01)
data[key] = d
def makeXsecData(options, data):
dummy,description = treedescription()
allpar = description.split(':')
xsecdata = {}
xsecs = readCrossSections(options)
if data == {}:
print "Your data array is empty! Exiting"
sys.exit()
for key in data:
# extend here to par3 if needed
(par1, par2) = (int(x) for x in key.split("_"))
if not useThisPoint(options, par1, par2):
print "Skipping %s" % key
continue
CLsexp = float(data[key][allpar.index("CLsexp")])
upperLimit = float(data[key][allpar.index("upperLimit")])
expectedUpperLimit = float(data[key][allpar.index("expectedUpperLimit")])
expectedUpperLimitMinus1Sig = float(data[key][allpar.index("expectedUpperLimitMinus1Sig")])
expectedUpperLimitPlus1Sig = float(data[key][allpar.index("expectedUpperLimitPlus1Sig")])
expectedUpperLimitMinus2Sig = float(data[key][allpar.index("expectedUpperLimitMinus2Sig")])
expectedUpperLimitPlus2Sig = float(data[key][allpar.index("expectedUpperLimitPlus2Sig")])
fID = int(float(data[key][allpar.index("fID")]))
#print "-1: %.3f, -2: %.3f" % (expectedUpperLimitMinus1Sig, expectedUpperLimitMinus2Sig)
# TODO need to extract this plus uncertainty from SignalDB
# NOTE: hardcoded for 1 proc, hence the extra [0]
xsec = xsecs[key][0][0]*1000
xsecUp = (1+xsecs[key][0][1]) * xsec
xsecDown = (1-xsecs[key][0][1]) * xsec
print "%s: xsec=%.2e relunc=%.2e up=%.2e down=%.2e" % (key, xsec, xsecs[key][0][1], xsecUp, xsecDown)
xsec2 = xsec/2
xsec4 = xsec/4
xsec8 = xsec/8
xsec2Up = xsecUp/2
xsec4Up = xsecUp/4
xsec8Up = xsecUp/8
xsec2Down = xsecDown/2
xsec4Down = xsecDown/4
xsec8Down = xsecDown/8
xsecdata[key] = {"upperLimit": upperLimit, "expectedUpperLimit" : expectedUpperLimit, "expectedUpperLimitMinus1Sig" : expectedUpperLimitMinus1Sig,
"expectedUpperLimitPlus1Sig" : expectedUpperLimitPlus1Sig, "expectedUpperLimitMinus2Sig" : expectedUpperLimitMinus2Sig,
"expectedUpperLimitPlus2Sig" : expectedUpperLimitPlus2Sig, "xsec" : xsec, "xsecUp" : xsecUp, "xsecDown" : xsecDown, "xsec2": xsec2,
"xsec2Up" : xsec2Up, "xsec2Down" : xsec2Down,
"xsec4Up" : xsec4Up, "xsec4Down" : xsec4Down,
"xsec8Up" : xsec8Up, "xsec8Down" : xsec8Down,
"xsec4": xsec4, "xsec8": xsec8, "fID" : fID, "CLsexp": CLsexp}
return xsecdata
def readData(options, filename):
try:
f = open(filename)
except:
print "Cannot open file %s" % filename
return
print "Reading data from %s" % filename
dummy,description = treedescription()
allpar = description.split(':')
#extend this to more pars on gridname if wanted
par1_s = "m0"
par2_s = "m12"
par3_s = ""
data = {}
for l in f.readlines():
d = l.strip().split()
selectpar = "expectedUpperLimit" # makes NO sense to use something different here
pval = float( d[allpar.index(selectpar)])
par1 = int(float( d[allpar.index(par1_s)]))
par2 = int(float( d[allpar.index(par2_s)]))
key = "%d_%d" % (par1, par2)
# float strings -> so make them float, then an int to throw away .0000 and then to bool
failedcov = bool(int(float(d[allpar.index("failedcov")]))) # Mediocre cov matrix quality
covqual = int(float(d[allpar.index("covqual")])) # covqual
failedfit = bool(int(float(d[allpar.index("failedfit")]))) # Fit failure
failedp0 = bool(int(float(d[allpar.index("failedp0")]))) # Base p0 ~ 0.5 (this can reject good fits)!
fitstatus = bool(int(float(d[allpar.index("fitstatus")]))) # Fit status from Minuit
nofit = bool(int(float(d[allpar.index("nofit")]))) # Whether there's a fit present
if failedfit or failedcov:
print "WARNING: fit failed for %s" % key
continue
if covqual < 3 and covqual != -1:
print "WARNING: covqual=%d for %s" % (covqual, key)
continue
if par3_s != "":
par3 = int(float( d[allpar.index(par3_s)]))
key = "%d_%d_%d" % (par1, par2, par3)
# this whole block is stolen from makeContoursNew.py. Do we need a module for this? Yes, we do.... --GJ, 4 feb 15
if selectpar == "expectedUpperLimit" and pval < 0.00001:
print "INFO: %s removing %s < 0.00001 for %s" % (filename, selectpar, key)
continue
# 110 is 20 times our default step -> this is almost certainly a bug
if selectpar == "expectedUpperLimit" and pval == 110.0:
print "INFO: %s removing expUL==110.0 for %s" % (filename, key)
continue
# if -1sig, -2sig == 0 and +1sig, 2sig == 100 -> almost certainly a bug too
if selectpar == "expectedUpperLimit" and float(d[allpar.index("expectedUpperLimitMinus1Sig")]) == 0.0 and float(d[allpar.index("expectedUpperLimitMinus2Sig")]) == 0.0 and float(d[allpar.index("expectedUpperLimitPlus1Sig")]) == 100.0 and float(d[allpar.index("expectedUpperLimitPlus2Sig")]) == 100.0:
print "INFO: %s removing point %s with expULMinus1Sig == expULMinus2Sig == 0 and expULPlus1Sig == expULPlus2Sig == 100" % (filename, key)
continue
if selectpar == "expectedUpperLimit" and float(d[allpar.index("upperLimit")]) == 0.0:
print "INFO: %s removing obsUL=0.0 for %s" % (filename, key)
continue
# throw away points with CLsexp > 0.99 and UL < 1.0 and CLs=-1 and UL<1 when merging on UL
CLsExp = float( d[allpar.index("CLsexp")])
if selectpar == "expectedUpperLimit" and pval < 1.0 and (CLsExp>0.99 or CLsExp<0) and float( d[allpar.index("upperLimit")])<1:
if CLsExp>0.99: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=1 for %s" % (filename, key)
elif CLsExp<0: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=-1 for %s" % (filename, key)
d[allpar.index("CLsexp")] = str(0.01)
d[allpar.index("CLs")] = str(0.01)
d[allpar.index("clsu1s")] = str(0.01)
d[allpar.index("clsd1s")] = str(0.01)
d[allpar.index("p1")] = str(0.01)
data[key] = d
def main():
options = parseCmdLine(sys.argv)
dummy,description = treedescription()
allpar = description.split(':')
xsecIdx = allpar.index('xsec')
ULIdx = allpar.index('upperLimit')
if options.expected:
ULIdx = allpar.index('expectedUpperLimit')
m0Idx = allpar.index('m0')
m12Idx = allpar.index('m12')
fIdIdx = allpar.index('fID')
p0Idx = allpar.index('p0')
p1Idx = allpar.index('p1')
CLsIdx = allpar.index('CLs')
CLsexpIdx = allpar.index('CLsexp')
covqualIdx = allpar.index('covqual')
dodgycovIdx = allpar.index('dodgycov')
failedcovIdx = allpar.index('failedcov')
failedfitIdx = allpar.index('failedfit')
failedp0Idx = allpar.index('failedp0')
failedstatusIdx = allpar.index('failedstatus')
fitstatusIdx = allpar.index('fitstatus')
#f = open(options.filename, "r")
# get files to read
f = open("Outputs/%s_combined_fixSigXSecNominal__1_harvest_list_infoFile" % options.grid, "r")
filenames = {}
for l in f.readlines():
(N, fname) = l.strip().split(":")
filenames[int(N.strip())] = "Outputs/%s_%s_fixSigXSecNominal__1_harvest_list" % (options.grid, fname.strip())
f.close()
data = {}
CLsdata = {}
for N in filenames:
f = open(filenames[N])
for l in f.readlines():
vals = l.strip().split(' ')
key = "%d_%d" % (int(float(vals[m0Idx])), int(float(vals[m12Idx])))
if key != options.point:
continue
data[N] = [float(vals[ULIdx]), float(vals[xsecIdx]), N]
CLsdata[N] = [float(vals[p0Idx]), float(vals[p1Idx]), float(vals[CLsIdx]), float(vals[CLsexpIdx])]
f.close()
if options.seperate:
print "# x\ty\tUL\txsec\tSR\tCLs"
else:
print "# x\ty\tUL*xsec\tSR"
for N in sorted(data):
(x, y) = options.point.split("_")
x = int(x)
y = int(y)
if options.seperate:
print "%d\t%d\t%f\t%f\t%d\t%.2f\t%.2f (exp)" % (x, y, data[N][0], data[N][1], data[N][2], CLsdata[N][2], CLsdata[N][3])
else:
print "%d\t%d\t%f\t%d" % (x, y, data[N][0] * data[N][1], data[N][2])
covqual = int(float(vals[covqualIdx]))
dodgycov = int(float(vals[dodgycovIdx]))
failedcov = int(float(vals[failedcovIdx]))
failedfit = int(float(vals[failedfitIdx]))
failedp0 = int(float(vals[failedp0Idx]))
failedstatus = int(float(vals[failedstatusIdx]))
fitstatus = int(float(vals[fitstatusIdx]))
# verbose print line to see what's going on with a point
print "\t ==> covqual = %d, dodgycov = %d, failedcov = %d, failedfit = %d, failedp0 = %d, failedstatus = %d, fitstatus = %d" % (covqual, dodgycov, failedcov, failedfit, failedp0, failedstatus, fitstatus)
return
def makeXsecData(options, data):
dummy, description = treedescription()
allpar = description.split(':')
xsecdata = {}
xsecs = readCrossSections(options)
if data == {}:
print "Your data array is empty! Exiting"
sys.exit()
for key in data:
# extend here to par3 if needed
(par1, par2) = (int(x) for x in key.split("_"))
if not useThisPoint(options, par1, par2):
print "Skipping %s" % key
continue
CLsexp = float(data[key][allpar.index("CLsexp")])
upperLimit = float(data[key][allpar.index("upperLimit")])
expectedUpperLimit = float(
data[key][allpar.index("expectedUpperLimit")])
expectedUpperLimitMinus1Sig = float(
data[key][allpar.index("expectedUpperLimitMinus1Sig")])
expectedUpperLimitPlus1Sig = float(
data[key][allpar.index("expectedUpperLimitPlus1Sig")])
expectedUpperLimitMinus2Sig = float(
data[key][allpar.index("expectedUpperLimitMinus2Sig")])
expectedUpperLimitPlus2Sig = float(
data[key][allpar.index("expectedUpperLimitPlus2Sig")])
fID = int(float(data[key][allpar.index("fID")]))
#print "-1: %.3f, -2: %.3f" % (expectedUpperLimitMinus1Sig, expectedUpperLimitMinus2Sig)
# TODO need to extract this plus uncertainty from SignalDB
# NOTE: hardcoded for 1 proc, hence the extra [0]
xsec = xsecs[key][0][0] * 1000
xsecUp = (1 + xsecs[key][0][1]) * xsec
xsecDown = (1 - xsecs[key][0][1]) * xsec
print "%s: xsec=%.2e relunc=%.2e up=%.2e down=%.2e" % (
key, xsec, xsecs[key][0][1], xsecUp, xsecDown)
xsec2 = xsec / 2
xsec4 = xsec / 4
xsec8 = xsec / 8
xsec2Up = xsecUp / 2
xsec4Up = xsecUp / 4
xsec8Up = xsecUp / 8
xsec2Down = xsecDown / 2
xsec4Down = xsecDown / 4
xsec8Down = xsecDown / 8
xsecdata[key] = {
"upperLimit": upperLimit,
"expectedUpperLimit": expectedUpperLimit,
"expectedUpperLimitMinus1Sig": expectedUpperLimitMinus1Sig,
"expectedUpperLimitPlus1Sig": expectedUpperLimitPlus1Sig,
"expectedUpperLimitMinus2Sig": expectedUpperLimitMinus2Sig,
"expectedUpperLimitPlus2Sig": expectedUpperLimitPlus2Sig,
"xsec": xsec,
"xsecUp": xsecUp,
"xsecDown": xsecDown,
"xsec2": xsec2,
"xsec2Up": xsec2Up,
"xsec2Down": xsec2Down,
"xsec4Up": xsec4Up,
"xsec4Down": xsec4Down,
"xsec8Up": xsec8Up,
"xsec8Down": xsec8Down,
"xsec4": xsec4,
"xsec8": xsec8,
"fID": fID,
"CLsexp": CLsexp
}
return xsecdata
def main():
options = parseCmdLine(sys.argv)
dummy,description = treedescription()
allpar = description.split(':')
if options.filename.find("Moriond13") != -1:
# from HistFitter v24 source code
description = "p0:p1:CLs:"
description += "mode:nexp:"
description += "seed:"
description += "CLsexp:"
description += "fID:sigma0:sigma1:"
description += "clsu1s:clsd1s:clsu2s:clsd2s:"
description += "upperLimit:upperLimitEstimatedError:expectedUpperLimit:expectedUpperLimitPlus1Sig:expectedUpperLimitPlus2Sig:expectedUpperLimitMinus1Sig:expectedUpperLimitMinus2Sig:xsec:excludedXsec:"
description += "m0:m12"
allpar = description.split(':')
xsecIdx = allpar.index('xsec')
ULIdx = allpar.index('upperLimit')
if options.expected:
ULIdx = allpar.index('expectedUpperLimit')
if not "pMSSM" in options.filename:
m0Idx = allpar.index('m0')
m12Idx = allpar.index('m12')
else:
m0Idx = allpar.index('M1')
m12Idx = allpar.index('M2')
fIdIdx = allpar.index('fID')
p0Idx = allpar.index('p0')
p1Idx = allpar.index('p1')
CLsIdx = allpar.index('CLs')
CLsexpIdx = allpar.index('CLsexp')
covqualIdx = allpar.index('covqual')
dodgycovIdx = allpar.index('dodgycov')
failedcovIdx = allpar.index('failedcov')
failedfitIdx = allpar.index('failedfit')
failedp0Idx = allpar.index('failedp0')
failedstatusIdx = allpar.index('failedstatus')
fitstatusIdx = allpar.index('fitstatus')
f = open(options.filename, "r")
data = {}
CLsdata = {}
for l in f.readlines():
vals = l.strip().split(' ')
key = "%d_%d" % (int(float(vals[m0Idx])), int(float(vals[m12Idx])))
data[key] = [float(vals[ULIdx]), float(vals[xsecIdx]), int(vals[fIdIdx])]
CLsdata[key] = [float(vals[p0Idx]), float(vals[p1Idx]), float(vals[CLsIdx]), float(vals[CLsexpIdx])]
if options.seperate:
print "# x\ty\tUL\txsec\tSR\tCLs"
elif options.hepdata:
print makeHepDataHeader(options.filename)
else:
print "# x\ty\tUL*xsec\tSR"
for key in sorted(data.iterkeys(), key=lambda a:map(int,a.split('_'))):
(x, y) = key.split("_")
x = int(x)
y = int(y)
## HACK to filter e.g. rectangle (easier to isolate crashed points)
#if not (x > 200 and x < 300): continue
#if not (y > 100 and y < 200): continue
#if not y == 0: continue
if options.seperate:
print "%d\t%d\t%f\t%f\t%d\t%e\t%e (exp)" % (x, y, data[key][0], data[key][1], data[key][2], CLsdata[key][2], CLsdata[key][3])
elif options.hepdata:
UL = data[key][0] * data[key][1]
SR = GetSRName(data[key][2])
print "%d; %d; %e; %s" % (x, y, UL, SR)
else:
print "%d\t%d\t%f\t%d" % (x, y, data[key][0] * data[key][1], data[key][2])
covqual = int(float(vals[covqualIdx]))
dodgycov = int(float(vals[dodgycovIdx]))
failedcov = int(float(vals[failedcovIdx]))
failedfit = int(float(vals[failedfitIdx]))
failedp0 = int(float(vals[failedp0Idx]))
failedstatus = int(float(vals[failedstatusIdx]))
fitstatus = int(float(vals[fitstatusIdx]))
# verbose print line to see what's going on with a point
#print "\t ==> covqual = %d, dodgycov = %d, failedcov = %d, failedfit = %d, failedp0 = %d, failedstatus = %d, fitstatus = %d" % (covqual, dodgycov, failedcov, failedfit, failedp0, failedstatus, fitstatus)
return
#!/usr/bin/env python
# usage :
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
# Import Modules #
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
import sys, os, string, shutil, pickle, subprocess
import ROOT
ROOT.gROOT.Reset()
ROOT.gROOT.SetBatch(True)
#=========================================================================
from summary_harvest_tree_description import treedescription
dummy, description = treedescription()
allpar = description.split(':')
print dummy, description
myMap = {}
selectpar = "CLsexp"
par1 = "m0"
par2 = "m12"
xsecs = ["Nominal", "Up", "Down"]
regions = [
"SRAmedium", "SRBmedium", "SRCloose", "SRCmedium", "SREloose", "SREmedium"
]
#regions = ["SRCmedium"]
for xsec in xsecs:
myMap = {}
def Oring(config):
# print which SRs you have to put in GetSRName.C
for indx,ana in enumerate(config.anaList):
print indx, ana
pass;
import sys, os, string, shutil, pickle, subprocess
import ROOT
from summary_harvest_tree_description import treedescription
dummy,description = treedescription()
allpar = description.split(':')
print allpar;
ROOT.gROOT.Reset()
ROOT.gROOT.SetBatch(True)
# For all xsecs, merge on best selectpar (normally expected CLs)
for xsecStr in allXS:
myMap = {}
myNomMap = {}
myChosenAna = {}
# skip when xsec != Nominal in discovery mode
if config.discovery and xsecStr != "Nominal":
continue;
# if no UL's available, merge on CLsexp. Otherwise, use expectedUpperLimit
# you should use CLsexp even if it's for observed limit because it can be bias?
selectpar = "CLsexp"
#selectpar = "CLs"
if config.makeUL and xsecStr == "Nominal": # ULs only available for Nominal
selectpar = "expectedUpperLimit"
pass;
# select best SR based on p0exp
if config.discovery:
selectpar = "p0exp"
pass;
print "Using selectpar = %s" % selectpar
# default
par1_s = "m0"
par2_s = "m12"
par3_s = ""
listSuffix = "__1_harvest_list"
# modifification depending on signal grid
if config.grid.find("GG_onestepCC")!=-1:
print "Ordering GG_onestepCC"
par1_s = "mgluino"
par2_s = "mchargino"
par3_s = "mlsp"
listSuffix = "__mlspNE60_harvest_list"
pass;
if config.grid.find("SS_onestepCC")!=-1:
print "Ordering SS_onestepCC"
par1_s = "msquark"
par2_s = "mchargino"
par3_s = "mlsp"
listSuffix = "__mlspNE60_harvest_list"
pass;
if config.grid.find("SM_GG_N2")!=-1 :
print "Ordering SM_GG_N2"
par1_s = "mgluino"
par2_s = "mlsp2"
par3_s = "mlsp"
pass;
if config.grid.find("GG_onestep_mlsp60")!=-1:
print "Ordering GG_onestep_mlsp60"
par1_s = "mgluino"
par2_s = "mchargino"
par3_s = "mlsp"
listSuffix = "__mlspEE60_harvest_list"
pass;
if config.discovery:
listSuffix = "_discovery_1_harvest_list"
pass;
infoFilename = config.outputDir+config.outputName+"_combined_fixSigXSec"+xsecStr+listSuffix+"_infoFile"
file_info = open(infoFilename,"w")
# loop over ana (SRs)
print " Oring :",config.anaList;
for indx,ana in enumerate(config.anaList):
filename = config.outputDir+config.outputName+"_"+ana+"_fixSigXSec"+xsecStr+listSuffix
print filename
nomfilename = config.outputDir+config.outputName+"_"+ana+"_fixSigXSecNominal"+listSuffix
print nomfilename
infoline="%i : %s\n"%(indx+1,ana)
file_info.write(infoline)
if not os.path.exists(filename):
print "file does not exist -> skip"
continue;
infile = open(filename,'r')
infilelines = infile.readlines()
nomfile = open(nomfilename,'r')
nomfilelines = nomfile.readlines()
for i, line in enumerate(nomfilelines):
vals = line.strip().split(' ')
if len(allpar) != len(vals):
print 'PRB!!!!!!!!!!!!!!!!!!!!'
print "summary file says %d components; file has %d per line" % (len(allpar),len(vals))
continue;
vals[allpar.index("fID/C")]=(indx+1)
pval = float( vals[allpar.index(selectpar+"/F")])
par1 = float( vals[allpar.index(par1_s+"/F")])
par2 = float( vals[allpar.index(par2_s+"/F")])
key = "%d_%d" % (par1, par2)
if config.grid.find("GG_onestepCC")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
if config.grid.find("SS_onestepCC")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
if config.grid.find("SM_GG_N2")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
if config.grid.find("GG_onestep_mlsp60")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
#print "DEBUG: %s selectpar=%.2e for %s" % (ana, pval, key)
# ignore negative pvalue
if pval < 0:
print "INFO: %s removing negative selectpar (%s = %.2e) for %s" % (ana, selectpar, pval, key)
continue;
# ignore expectedUpperLimit < 0.00001
if selectpar == "expectedUpperLimit" and pval < 0.00001:
print "INFO: %s removing %s < 0.00001 for %s" % (ana, selectpar, key)
continue;
# 110 is 20 times our default step -> this is almost certainly a bug
if selectpar == "expectedUpperLimit" and pval == 110.0:
print "INFO: %s removing expUL==110.0 for %s" % (ana, key)
continue;
# if -1sig, -2sig == 0 and +1sig, 2sig == 100 -> almost certainly a bug too
if selectpar == "expectedUpperLimit" and float(vals[allpar.index("expectedUpperLimitMinus1Sig")]) == 0.0 and float(vals[allpar.index("expectedUpperLimitMinus2Sig")]) == 0.0 and float(vals[allpar.index("expectedUpperLimitPlus1Sig")]) == 100.0 and float(vals[allpar.index("expectedUpperLimitPlus2Sig")]) == 100.0:
print "INFO: %s removing point %s with expULMinus1Sig == expULMinus2Sig == 0 and expULPlus1Sig == expULPlus2Sig == 100" % (ana, key)
continue;
# ignore observed upperLimit=0 when merging on UL
if selectpar == "expectedUpperLimit" and float(vals[allpar.index("upperLimit")]) == 0.0:
print "INFO: %s removing obsUL=0.0 for %s" % (ana, key)
continue;
# throw away points with CLsexp > 0.99 and UL < 1.0 and CLs=-1 and UL<1 when merging on UL
CLsExp = float( vals[allpar.index("CLsexp/F")])
if selectpar == "expectedUpperLimit" and pval < 1.0 and (CLsExp>0.99 or CLsExp<0) and float( vals[allpar.index("upperLimit")])<1:
if CLsExp>0.99: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=1 for %s" % (ana, key)
elif CLsExp<0: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=-1 for %s" % (ana, key)
vals[allpar.index("CLsexp/F")] = str(0.01)
vals[allpar.index("CLs/F")] = str(0.01)
vals[allpar.index("clsu1s/F")] = str(0.01)
vals[allpar.index("clsd1s/F")] = str(0.01)
vals[allpar.index("p1/F")] = str(0.01)
pass;
newline=""
#print vals;
for val in vals:
newline += (str)(val)+" "
pass;
newline = newline.rstrip(" ")
newline += "\n"
# float strings -> so make them float, then an int to throw away .0000 and then to bool
failedcov = bool(int(float(vals[allpar.index("failedcov/F")]))) # Mediocre cov matrix quality
covqual = int(float(vals[allpar.index("covqual/F")])) # covqual
failedfit = bool(int(float(vals[allpar.index("failedfit/F")]))) # Fit failure
failedp0 = bool(int(float(vals[allpar.index("failedp0/F")]))) # Base p0 ~ 0.5 (this can reject good fits)!
fitstatus = bool(int(float(vals[allpar.index("fitstatus/F")]))) # Fit status from Minuit
nofit = bool(int(float(vals[allpar.index("nofit/F")]))) # Whether there's a fit present
# ignore some checker
"""
# ignore failed fit
if failedfit:
print "INFO: %s removing failedfit=true for %s" % (ana, key)
continue
# ignore bad mediocre cov matrix quality
if failedcov:
print "INFO: %s removing failedcov=true for %s" % (ana, key)
# ignore if covqual<3 & covqual!=-1
if covqual < 3 and covqual != -1:
print "INFO: %s removing check if (covqual<3 and covqual!=-1) for %s (found covqual=%d)" % (ana, key, covqual)
continue
"""
key = (par1,par2)
if config.grid.find("GG_onestepCC")!=-1:
key = (par1,par2,par3)
pass;
if config.grid.find("SS_onestepCC")!=-1:
key = (par1,par2,par3)
pass;
if config.grid.find("SM_GG_N2")!=-1:
key = (par1,par2,par3)
pass;
if config.grid.find("GG_onestep_mlsp60")!=-1:
key = (par1,par2,par3)
pass;
if key not in myNomMap.keys():
myNomMap[key] = [pval,newline]
myChosenAna[key] = ana
else:
if pval < myNomMap[key][0] and pval>=0 and (key != (1000.0, 500.0) or ana != "SRJigsawSRS2a"):
print "DEBUG: %s found new best value - selectpar=%.2e < previous=%e for %s" % (ana, pval, myNomMap[key][0], key)
myNomMap[key][0] = pval
myNomMap[key][1] = newline
myChosenAna[key] = ana
pass;
pass;
pass;
nomfile.close()
for i, line in enumerate(infilelines):
vals = line.strip().split(' ')
if len(allpar) != len(vals):
print 'PRB!!!!!!!!!!!!!!!!!!!!'
print "summary file says %d components; file has %d per line" % (len(allpar),len(vals))
continue;
vals[allpar.index("fID/C")]=(indx+1)
pval = float( vals[allpar.index(selectpar+"/F")])
par1 = float( vals[allpar.index(par1_s+"/F")])
par2 = float( vals[allpar.index(par2_s+"/F")])
key = "%d_%d" % (par1, par2)
if config.grid.find("GG_onestepCC")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
if config.grid.find("SS_onestepCC")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
if config.grid.find("SM_GG_N2")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
if config.grid.find("GG_onestep_mlsp60")!=-1:
par3 = float( vals[allpar.index(par3_s+"/F")])
key += "_%d" % par3
pass;
#print "DEBUG: %s selectpar=%.2e for %s" % (ana, pval, key)
# ignore negative pvalue
if pval < 0:
print "INFO: %s removing negative selectpar (%s = %.2e) for %s" % (ana, selectpar, pval, key)
continue;
# ignore expectedUpperLimit < 0.00001
if selectpar == "expectedUpperLimit" and pval < 0.00001:
print "INFO: %s removing %s < 0.00001 for %s" % (ana, selectpar, key)
continue;
# 110 is 20 times our default step -> this is almost certainly a bug
if selectpar == "expectedUpperLimit" and pval == 110.0:
print "INFO: %s removing expUL==110.0 for %s" % (ana, key)
continue;
# if -1sig, -2sig == 0 and +1sig, 2sig == 100 -> almost certainly a bug too
if selectpar == "expectedUpperLimit" and float(vals[allpar.index("expectedUpperLimitMinus1Sig")]) == 0.0 and float(vals[allpar.index("expectedUpperLimitMinus2Sig")]) == 0.0 and float(vals[allpar.index("expectedUpperLimitPlus1Sig")]) == 100.0 and float(vals[allpar.index("expectedUpperLimitPlus2Sig")]) == 100.0:
print "INFO: %s removing point %s with expULMinus1Sig == expULMinus2Sig == 0 and expULPlus1Sig == expULPlus2Sig == 100" % (ana, key)
continue;
# ignore observed upperLimit=0 when merging on UL
if selectpar == "expectedUpperLimit" and float(vals[allpar.index("upperLimit")]) == 0.0:
print "INFO: %s removing obsUL=0.0 for %s" % (ana, key)
continue;
# throw away points with CLsexp > 0.99 and UL < 1.0 and CLs=-1 and UL<1 when merging on UL
CLsExp = float( vals[allpar.index("CLsexp/F")])
if selectpar == "expectedUpperLimit" and pval < 1.0 and (CLsExp>0.99 or CLsExp<0) and float( vals[allpar.index("upperLimit")])<1:
if CLsExp>0.99: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=1 for %s" % (ana, key)
elif CLsExp<0: print "INFO: %s replacing CLsexp with 0.01 since UL < 1.0 and CLsexp=-1 for %s" % (ana, key)
vals[allpar.index("CLsexp/F")] = str(0.01)
vals[allpar.index("CLs/F")] = str(0.01)
vals[allpar.index("clsu1s/F")] = str(0.01)
vals[allpar.index("clsd1s/F")] = str(0.01)
vals[allpar.index("p1/F")] = str(0.01)
pass;
newline=""
#print vals;
for val in vals:
newline += (str)(val)+" "
pass;
newline = newline.rstrip(" ")
newline += "\n"
# float strings -> so make them float, then an int to throw away .0000 and then to bool
failedcov = bool(int(float(vals[allpar.index("failedcov/F")]))) # Mediocre cov matrix quality
covqual = int(float(vals[allpar.index("covqual/F")])) # covqual
failedfit = bool(int(float(vals[allpar.index("failedfit/F")]))) # Fit failure
failedp0 = bool(int(float(vals[allpar.index("failedp0/F")]))) # Base p0 ~ 0.5 (this can reject good fits)!
fitstatus = bool(int(float(vals[allpar.index("fitstatus/F")]))) # Fit status from Minuit
nofit = bool(int(float(vals[allpar.index("nofit/F")]))) # Whether there's a fit present
# ignore some checker
"""
# ignore failed fit
if failedfit:
print "INFO: %s removing failedfit=true for %s" % (ana, key)
continue
# ignore bad mediocre cov matrix quality
if failedcov:
print "INFO: %s removing failedcov=true for %s" % (ana, key)
# ignore if covqual<3 & covqual!=-1
if covqual < 3 and covqual != -1:
print "INFO: %s removing check if (covqual<3 and covqual!=-1) for %s (found covqual=%d)" % (ana, key, covqual)
continue
"""
key = (par1,par2)
if config.grid.find("GG_onestepCC")!=-1:
key = (par1,par2,par3)
pass;
if config.grid.find("SS_onestepCC")!=-1:
key = (par1,par2,par3)
pass;
if config.grid.find("SM_GG_N2")!=-1:
key = (par1,par2,par3)
pass;
if config.grid.find("GG_onestep_mlsp60")!=-1:
key = (par1,par2,par3)
pass;
if key in myChosenAna.keys():
if ana == myChosenAna[key]:
# print "DEBUG: %s found new best value - selectpar=%.2e < previous=%e for %s" % (ana, pval, myMap[key][0], key)
myMap[key]= [pval,newline]
pass;
pass;
pass;
# if key not in myMap.keys():
# myMap[key] = [pval,newline]
# else:
# if pval < myMap[key][0] and pval>=0 and (key != (1000.0, 500.0) or ana != "SRJigsawSRS2a"):
# print "DEBUG: %s found new best value - selectpar=%.2e < previous=%e for %s" % (ana, pval, myMap[key][0], key)
# myMap[key][0] = pval
# myMap[key][1] = newline
# pass;
# pass;
# pass;
infile.close()