def lLHDFromLimits(self):
""" to do some statistics on the chi2 """
nsig = 1.0
nobs, nbg = 100, 100.
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
ulcomp = UpperLimitComputer()
ulobs = ulcomp.ulSigma(m)
ulexp = ulcomp.ulSigma(m, expected=True)
print("ulobs", ulobs)
print("ulexp", ulexp)
f = open("llhds.csv", "wt")
dx = .5
totdir, totlim, totmarg = 0., 0., 0.
for nsig in np.arange(0.1, 100., dx):
print()
print("nsig=", nsig)
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
llhdcomp = LikelihoodComputer(m)
llhddir = llhdcomp.likelihood(nsig)
chi2dir = llhdcomp.chi2(nsig)
llhdmarg = llhdcomp.likelihood(nsig, marginalize=True)
chi2marg = llhdcomp.chi2(nsig, marginalize=True)
print("llhd direct", llhddir, chi2dir)
print("llhd marg", llhdmarg, chi2marg)
llhdlim = likelihoodFromLimits(ulobs, ulexp, nsig)
chi2lim = chi2FromLimits(llhdlim, ulexp)
print("llhd from limits", llhdlim, chi2lim)
totdir += llhddir * dx
totlim += llhdlim * dx
totmarg += llhdmarg * dx
f.write("%s,%s,%s,%s\n" % (nsig, llhddir, llhdlim, llhdmarg))
print("total direct", totdir)
print("total limit", totlim)
print("total marg", totmarg)
f.close()
def totalChi2(self, nsig, marginalize=False, deltas_rel=0.2):
"""
Computes the total chi2 for a given number of observed events, given a
predicted signal "nsig", with nsig being a vector with one entry per
dataset. nsig has to obey the datasetOrder. Deltas is the error on
the signal efficiency.
:param nsig: predicted signal (list)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:returns: chi2 (float)
"""
if len(self._datasets) == 1:
if isinstance(nsig, list):
nsig = nsig[0]
return self._datasets[0].chi2(nsig, marginalize=marginalize)
if not hasattr(self.globalInfo, "covariance"):
logger.error(
"Asked for combined likelihood, but no covariance error given."
)
return None
nobs = [x.dataInfo.observedN for x in self._datasets]
bg = [x.dataInfo.expectedBG for x in self._datasets]
cov = self.globalInfo.covariance
computer = LikelihoodComputer(
Data(nobs, bg, cov, deltas_rel=deltas_rel))
return computer.chi2(nsig, marginalize=marginalize)
def totalChi2(self, nsig, marginalize=False, deltas_rel=0.2):
"""
Computes the total chi2 for a given number of observed events, given a
predicted signal "nsig", with nsig being a vector with one entry per
dataset. nsig has to obey the datasetOrder. Deltas is the error on
the signal efficiency.
:param nsig: predicted signal (list)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:returns: chi2 (float)
"""
if hasattr(self.globalInfo, "covariance" ):
if len(self._datasets) == 1:
if isinstance(nsig,list):
nsig = nsig[0]
return self._datasets[0].chi2(nsig, marginalize=marginalize)
nobs = [x.dataInfo.observedN for x in self._datasets ]
bg = [x.dataInfo.expectedBG for x in self._datasets ]
cov = self.globalInfo.covariance
computer = LikelihoodComputer(Data(nobs, bg, cov, deltas_rel=deltas_rel))
return computer.chi2(nsig, marginalize=marginalize)
elif hasattr(self.globalInfo, "jsonFiles"):
# Getting the path to the json files
# Loading the jsonFiles
ulcomputer, combinations = self.getPyhfComputer( nsig )
if ulcomputer.nWS == 1:
return ulcomputer.chi2()
else:
# Looking for the best combination
if self.bestCB == None:
ulMin = float('+inf')
for i_ws in range(ulcomputer.nWS):
logger.debug("Performing best expected combination")
ul = ulcomputer.ulSigma(expected=True, workspace_index=i_ws)
if ul < ulMin:
ulMin = ul
i_best = i_ws
self.bestCB = combinations[i_best] # Keeping the index of the best combination for later
logger.debug('Best combination : %d' % self.bestCB)
return ulcomputer.chi2(workspace_index=combinations.index(self.bestCB))
else:
logger.error("Asked for combined likelihood, but no covariance error given." )
return None
def chi2(self, nsig, deltas_rel=0.2, marginalize=False):
"""
Computes the chi2 for a given number of observed events "nobs",
given number of signal events "nsig", and error on signal "deltas".
nobs, expectedBG and bgError are part of dataInfo.
:param nsig: predicted signal (float)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:param marginalize: if true, marginalize nuisances. Else, profile them.
:return: chi2 (float)
"""
m = Data(self.dataInfo.observedN, self.dataInfo.expectedBG,
self.dataInfo.bgError**2,deltas_rel=deltas_rel)
computer = LikelihoodComputer(m)
ret = computer.chi2(nsig, marginalize=marginalize)
return ret
def likelihood(self, nsig, deltas_rel=0.2, marginalize=False):
"""
Computes the likelihood to observe nobs events,
given a predicted signal "nsig", assuming "deltas"
error on the signal efficiency.
The values observedN, expectedBG, and bgError are part of dataInfo.
:param nsig: predicted signal (float)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:param marginalize: if true, marginalize nuisances. Else, profile them.
:returns: likelihood to observe nobs events (float)
"""
m = Data(self.dataInfo.observedN,
self.dataInfo.expectedBG,
self.dataInfo.bgError**2,
deltas_rel=deltas_rel)
computer = LikelihoodComputer(m)
return computer.likelihood(nsig, marginalize=marginalize)
def testPredictionInterface(self):
""" A simple test to see that the interface in datasetObj
and TheoryPrediction to the statistics tools is working correctly
"""
expRes = database.getExpResults(analysisIDs=['CMS-SUS-13-012'])[0]
slhafile = "./testFiles/slha/simplyGluino.slha"
smstoplist = slhaDecomposer.decompose(slhafile)
prediction = theoryPredictionsFor(expRes, smstoplist, deltas_rel=0.)[0]
pred_signal_strength = prediction.xsection.value
prediction.computeStatistics()
ill = math.log(prediction.likelihood)
ichi2 = prediction.chi2
nsig = (pred_signal_strength * expRes.globalInfo.lumi).asNumber()
m = Data(4, 2.2, 1.1**2, None, nsignal=nsig, deltas_rel=0.2)
computer = LikelihoodComputer(m)
dll = math.log(computer.likelihood(nsig, marginalize=False))
self.assertAlmostEqual(ill, dll, places=2)
dchi2 = computer.chi2(nsig, marginalize=False)
# print ( "dchi2,ichi2",dchi2,ichi2)
self.assertAlmostEqual(ichi2, dchi2, places=2)
def testChi2FromLimits(self):
""" test the chi2 value that we obtain from limits """
nsig = 35.0
nobs, nbg = 110, 100.
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
ulcomp = UpperLimitComputer()
ulobs = ulcomp.ulSigma(m)
ulexp = ulcomp.ulSigma(m, expected=True)
dx = .5
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
llhdcomp = LikelihoodComputer(m)
llhddir = llhdcomp.likelihood(nsig)
chi2dir = llhdcomp.chi2(nsig)
llhdmarg = llhdcomp.likelihood(nsig, marginalize=True)
chi2marg = llhdcomp.chi2(nsig, marginalize=True)
llhdlim = likelihoodFromLimits(ulobs, ulexp, nsig)
chi2lim = chi2FromLimits(llhdlim, ulexp)
## relative error on chi2, for this example is about 4%
rel = abs(chi2lim - chi2marg) / chi2marg
self.assertAlmostEqual(rel, 0.04, 1)
def testLikelihood(self):
"""
Compare the computed chi2 from a given observed
and expected upper limit and a theory prediction
with the previously known result for the value of
the chi2.
All values of nobs, nsig, nb, deltab come from the
SModelS database and are for the T1 simplified model
from efficiency results of one of
ATLAS-SUSY-2013-02
ATLAS-CONF-2013-047
CMS-SUS-13-012
ATLAS-CONF-2013-054
"""
expected_values = [
# mgluino mlsp nsig nobs nb deltab llhd chi2
# ---------- ---------- --------------- ---------------- ---------- ---------- -------------------- ----------------
{
'mgluino': 500,
'mlsp': 200,
'nsig': 384.898,
'nobs': 298.413,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00024197,
'chi2': 7.32614
}
]
{
'mgluino': 500,
'mlsp': 300,
'nsig': 185.166,
'nobs': 223.619,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00215989,
'chi2': 3.67088900
},
{
'mgluino': 500,
'mlsp': 400,
'nsig': 450.820,
'nobs': 2331.38,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00075499,
'chi2': 2.85026
},
{
'mgluino': 600,
'mlsp': 100,
'nsig': 476.150,
'nobs': 437.874,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00100575,
'chi2': 3.52406595
},
{
'mgluino': 600,
'mlsp': 200,
'nsig': 264.387,
'nobs': 232.912,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00028921,
'chi2': 7.57051432
},
{
'mgluino': 600,
'mlsp': 300,
'nsig': 171.766,
'nobs': 211.038,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00198061,
'chi2': 4.02903
},
{
'mgluino': 600,
'mlsp': 400,
'nsig': 66.9991,
'nobs': 150.393,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00845030,
'chi2': 1.89194013
},
{
'mgluino': 600,
'mlsp': 500,
'nsig': 157.571,
'nobs': 2167.25,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00217371,
'chi2': 0.845615
},
{
'mgluino': 700,
'mlsp': 100,
'nsig': 307.492,
'nobs': 325.060,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00159632,
'chi2': 3.41955
},
{
'mgluino': 700,
'mlsp': 200,
'nsig': 211.534,
'nobs': 228.763,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00061670,
'chi2': 6.26852955
},
{
'mgluino': 700,
'mlsp': 300,
'nsig': 147.084,
'nobs': 167.631,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00012707,
'chi2': 10.1408114
},
{
'mgluino': 700,
'mlsp': 400,
'nsig': 420.524,
'nobs': 2332.28,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00100854,
'chi2': 2.28195399
},
{
'mgluino': 700,
'mlsp': 500,
'nsig': 186.726,
'nobs': 2162.70,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00165411,
'chi2': 1.39601
},
{
'mgluino': 700,
'mlsp': 600,
'nsig': 5.18888,
'nobs': 24.3271,
'nb': 37.0,
'deltab': 6.0,
'llhd': 0.00545866,
'chi2': 4.3836
},
{
'mgluino': 800,
'mlsp': 100,
'nsig': 169.670,
'nobs': 213.312,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00116298,
'chi2': 5.09803029
},
{
'mgluino': 800,
'mlsp': 200,
'nsig': 152.221,
'nobs': 212.732,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00223053,
'chi2': 3.81519907
},
{
'mgluino': 800,
'mlsp': 300,
'nsig': 98.6749,
'nobs': 175.141,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00298021,
'chi2': 3.72566221
},
{
'mgluino': 800,
'mlsp': 400,
'nsig': 59.3935,
'nobs': 141.966,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00262008,
'chi2': 4.30322736
},
{
'mgluino': 800,
'mlsp': 500,
'nsig': 27.7738,
'nobs': 123.172,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.01605069,
'chi2': 0.903864
},
{
'mgluino': 800,
'mlsp': 600,
'nsig': 6.40339,
'nobs': 39.2979,
'nb': 33.0,
'deltab': 6.0,
'llhd': 0.04536718,
'chi2': -0.000501411
},
{
'mgluino': 800,
'mlsp': 700,
'nsig': 4.38635,
'nobs': 132.824,
'nb': 125.0,
'deltab': 10.0,
'llhd': 0.02525385,
'chi2': 0.0565348
},
{
'mgluino': 900,
'mlsp': 100,
'nsig': 18.8255,
'nobs': 14.1228,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.02122262,
'chi2': 2.85426343
},
{
'mgluino': 900,
'mlsp': 200,
'nsig': 16.0543,
'nobs': 6.77062,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.00187567,
'chi2': 8.43890579
},
{
'mgluino': 900,
'mlsp': 300,
'nsig': 64.4188,
'nobs': 142.220,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00181163,
'chi2': 5.00642964
},
{
'mgluino': 900,
'mlsp': 400,
'nsig': 44.8312,
'nobs': 140.979,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00692173,
'chi2': 2.34800741
},
{
'mgluino': 900,
'mlsp': 500,
'nsig': 24.4723,
'nobs': 120.688,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00601021,
'chi2': 2.72454478
},
{
'mgluino': 900,
'mlsp': 600,
'nsig': 67.0446,
'nobs': 2165.25,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00328372,
'chi2': 0.0371125
},
{
'mgluino': 900,
'mlsp': 700,
'nsig': 1.00167,
'nobs': 5.0,
'nb': 5.2,
'deltab': 1.4,
'llhd': 0.13964962,
'chi2': 0.107139
},
{
'mgluino': 900,
'mlsp': 800,
'nsig': 0.86634,
'nobs': 24.0,
'nb': 37.0,
'deltab': 6.0,
'llhd': 0.01303119,
'chi2': 2.638323
},
{
'mgluino': 1000,
'mlsp': 100,
'nsig': 11.7426,
'nobs': 11.8786,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.05712388,
'chi2': 1.06934
},
{
'mgluino': 1000,
'mlsp': 200,
'nsig': 9.85815,
'nobs': 7.98535,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.03180710,
'chi2': 2.63593288
},
{
'mgluino': 1000,
'mlsp': 300,
'nsig': 6.80275,
'nobs': 6.14772,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.04255251,
'chi2': 2.25703866
},
{
'mgluino': 1000,
'mlsp': 400,
'nsig': 25.8451,
'nobs': 120.523,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00525390,
'chi2': 2.99137140
},
{
'mgluino': 1000,
'mlsp': 500,
'nsig': 18.6299,
'nobs': 122.095,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01056408,
'chi2': 1.59516468
},
{
'mgluino': 1000,
'mlsp': 600,
'nsig': 10.2636,
'nobs': 119.968,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01536934,
'chi2': 0.84011292
},
{
'mgluino': 1000,
'mlsp': 700,
'nsig': 4.59470,
'nobs': 121.728,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.02078487,
'chi2': 0.23333618
},
{
'mgluino': 1000,
'mlsp': 800,
'nsig': 1.91162,
'nobs': 121.196,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.02193946,
'chi2': 0.12552973
},
{
'mgluino': 1000,
'mlsp': 900,
'nsig': 0.62255,
'nobs': 133.0,
'nb': 125.0,
'deltab': 10.0,
'llhd': 0.02290147,
'chi2': 0.253293
},
{
'mgluino': 1100,
'mlsp': 100,
'nsig': 5.95636,
'nobs': 5.94515,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.05236744,
'chi2': 1.87080349
},
{
'mgluino': 1100,
'mlsp': 200,
'nsig': 5.51938,
'nobs': 5.92472,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.06002489,
'chi2': 1.59429
},
{
'mgluino': 1100,
'mlsp': 300,
'nsig': 3.93082,
'nobs': 6.07873,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.09732480,
'chi2': 0.61881103
},
{
'mgluino': 1100,
'mlsp': 400,
'nsig': 2.80428,
'nobs': 6.54033,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.12350249,
'chi2': 0.0882501
},
{
'mgluino': 1100,
'mlsp': 500,
'nsig': 12.6778,
'nobs': 125.271,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01749246,
'chi2': 0.560614
},
{
'mgluino': 1100,
'mlsp': 600,
'nsig': 8.02475,
'nobs': 119.742,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01700322,
'chi2': 0.64005829
},
{
'mgluino': 1100,
'mlsp': 700,
'nsig': 4.74108,
'nobs': 120.211,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01979279,
'chi2': 0.334838
},
{
'mgluino': 1100,
'mlsp': 800,
'nsig': 1.79622,
'nobs': 120.858,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.02187799,
'chi2': 0.134012
},
{
'mgluino': 1100,
'mlsp': 900,
'nsig': 4.82397,
'nobs': 2166.20,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00313424,
'chi2': 0.119045
},
{
'mgluino': 1100,
'mlsp': 1000,
'nsig': 0.1606,
'nobs': 25.0,
'nb': 37.0,
'deltab': 6.0,
'llhd': 0.01796058,
'chi2': 1.9806
},
{
'mgluino': 2100,
'mlsp': 1000,
'nsig': 0.1606,
'nobs': 2.0,
'nb': 0.7,
'deltab': 6.0,
'llhd': 0.108669,
'chi2': -0.161304
}
for d in expected_values:
nobs = d['nobs']
nsig = d['nsig']
nb = d['nb']
deltab = d['deltab']
# print ("ns="+str(nsig)+"; nobs = "+str(nobs)+"; nb="+str(nb)+"; db="+str(deltab))
# Chi2 as computed by statistics module:
m = Data(nobs, nb, deltab**2, deltas_rel=0.2)
computer = LikelihoodComputer(m)
chi2_actual = computer.chi2(nsig, marginalize=True) ## , .2*nsig )
chi2_expected = d['chi2']
if not chi2_expected == None and not np.isnan(chi2_expected):
# chi2_expected = self.round_to_sign(chi2_expected, 2)
# Check that chi2 values agree:
self.assertAlmostEqual(abs(chi2_actual - chi2_expected) /
chi2_expected,
0.,
places=2)
else:
self.assertTrue(chi2_actual == None or np.isnan(chi2_actual))
# likelihood as computed by statistics module:
# computer = LikelihoodComputer( nobs, nb, deltab**2 )
#likelihood_actual = statistics.likelihood( nsig,
# nobs, nb, deltab, deltas)
likelihood_actual = computer.likelihood(nsig, marginalize=False)
# likelihood_actual = statistics.likelihood()
# logger.error("llk= "+str(likelihood_actual)+" nsig="+str(nsig)+" nobs = "+str(nobs)+" nb="+str(nb)+"+-"+str(deltab))
#print('llhdactual', likelihood_actual)
if not likelihood_actual == None and not np.isnan(
likelihood_actual):
likelihood_actual = self.round_to_sign(likelihood_actual, 4)
# The previously computed likelihood:
# (using: ntoys=100000)
likelihood_expected = d['llhd']
#print('llhdexp', likelihood_expected)
if not likelihood_expected == None and not np.isnan(
likelihood_expected):
likelihood_expected = self.round_to_sign(
likelihood_expected, 4)
# Check that likelihood values agree:
self.assertAlmostEqual(likelihood_actual,
likelihood_expected,
delta=2 * 1e-1)
else:
self.assertTrue(likelihood_actual == None
or np.isnan(likelihood_actual))