def lLHDFromLimits(self):
""" to do some statistics on the chi2 """
nsig = 1.0
nobs, nbg = 100, 100.
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
ulcomp = UpperLimitComputer()
ulobs = ulcomp.ulSigma(m)
ulexp = ulcomp.ulSigma(m, expected=True)
print("ulobs", ulobs)
print("ulexp", ulexp)
f = open("llhds.csv", "wt")
dx = .5
totdir, totlim, totmarg = 0., 0., 0.
for nsig in np.arange(0.1, 100., dx):
print()
print("nsig=", nsig)
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
llhdcomp = LikelihoodComputer(m)
llhddir = llhdcomp.likelihood(nsig)
chi2dir = llhdcomp.chi2(nsig)
llhdmarg = llhdcomp.likelihood(nsig, marginalize=True)
chi2marg = llhdcomp.chi2(nsig, marginalize=True)
print("llhd direct", llhddir, chi2dir)
print("llhd marg", llhdmarg, chi2marg)
llhdlim = likelihoodFromLimits(ulobs, ulexp, nsig)
chi2lim = chi2FromLimits(llhdlim, ulexp)
print("llhd from limits", llhdlim, chi2lim)
totdir += llhddir * dx
totlim += llhdlim * dx
totmarg += llhdmarg * dx
f.write("%s,%s,%s,%s\n" % (nsig, llhddir, llhdlim, llhdmarg))
print("total direct", totdir)
print("total limit", totlim)
print("total marg", totmarg)
f.close()
def testPathologicalModel2(self):
C=[ 1. ]
m=Data(observed=[0],
backgrounds=[.0],
covariance= C,
third_moment = [ 0. ] * 8,
nsignal=[x/100. for x in [0.1] ],
name="pathological model 2",deltas_rel=0. )
m.zeroSignal()
ulComp = UpperLimitComputer(ntoys=10000, cl=.95 )
ul = ulComp.ulSigma(m, marginalize=True )
ulProf = ulComp.ulSigma(m, marginalize=False )
self.assertAlmostEqual(ul/(3049.*sum(m.nsignal)), 1., 1 )
self.assertAlmostEqual(ulProf/(1920.*sum(m.nsignal)), 1., 1 )
def testPathologicalModel2(self):
C = [1.]
m = Data(observed=[0],
backgrounds=[.0],
covariance=C,
third_moment=[0.] * 8,
nsignal=[x / 100. for x in [0.1]],
name="pathological model 2",
deltas_rel=0.)
m.zeroSignal()
ulComp = UpperLimitComputer(ntoys=10000, cl=.95)
ul = ulComp.ulSigma(m, marginalize=True)
ulProf = ulComp.ulSigma(m, marginalize=False)
self.assertAlmostEqual(ul / (3049. * sum(m.nsignal)), 1., 1)
self.assertAlmostEqual(ulProf / (1920. * sum(m.nsignal)), 1., 1)
def testModel8(self):
C = [
18774.2, -2866.97, -5807.3, -4460.52, -2777.25, -1572.97, -846.653,
-442.531, -2866.97, 496.273, 900.195, 667.591, 403.92, 222.614,
116.779, 59.5958, -5807.3, 900.195, 1799.56, 1376.77, 854.448,
482.435, 258.92, 134.975, -4460.52, 667.591, 1376.77, 1063.03,
664.527, 377.714, 203.967, 106.926, -2777.25, 403.92, 854.448,
664.527, 417.837, 238.76, 129.55, 68.2075, -1572.97, 222.614,
482.435, 377.714, 238.76, 137.151, 74.7665, 39.5247, -846.653,
116.779, 258.92, 203.967, 129.55, 74.7665, 40.9423, 21.7285,
-442.531, 59.5958, 134.975, 106.926, 68.2075, 39.5247, 21.7285,
11.5732
]
nsignal = [
x / 100. for x in [47, 29.4, 21.1, 14.3, 9.4, 7.1, 4.7, 4.3]
]
m = Data(
observed=[1964, 877, 354, 182, 82, 36, 15, 11],
backgrounds=[2006.4, 836.4, 350., 147.1, 62.0, 26.2, 11.1, 4.7],
covariance=C,
third_moment=[0.] * 8,
nsignal=nsignal,
name="CMS-NOTE-2017-001 model",
deltas_rel=0.)
ulComp = UpperLimitComputer(ntoys=2000, cl=.95)
ul = ulComp.ulSigma(m)
self.assertAlmostEqual(ul / (131.828 * sum(nsignal)), 1.0, 1)
ulProf = ulComp.ulSigma(m, marginalize=False)
#print ( "ul,ulprof=", ul,ulProf )
self.assertAlmostEqual(ulProf / (131.828 * sum(nsignal)), 1.0, 1)
def totalChi2(self, nsig, marginalize=False, deltas_rel=0.2):
"""
Computes the total chi2 for a given number of observed events, given a
predicted signal "nsig", with nsig being a vector with one entry per
dataset. nsig has to obey the datasetOrder. Deltas is the error on
the signal efficiency.
:param nsig: predicted signal (list)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:returns: chi2 (float)
"""
if len(self._datasets) == 1:
if isinstance(nsig, list):
nsig = nsig[0]
return self._datasets[0].chi2(nsig, marginalize=marginalize)
if not hasattr(self.globalInfo, "covariance"):
logger.error(
"Asked for combined likelihood, but no covariance error given."
)
return None
nobs = [x.dataInfo.observedN for x in self._datasets]
bg = [x.dataInfo.expectedBG for x in self._datasets]
cov = self.globalInfo.covariance
computer = LikelihoodComputer(
Data(nobs, bg, cov, deltas_rel=deltas_rel))
return computer.chi2(nsig, marginalize=marginalize)
def testChi2FromLimits(self):
""" test the chi2 value that we obtain from limits """
nsig = 35.0
nobs, nbg = 110, 100.
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
ulcomp = UpperLimitComputer()
ulobs = ulcomp.ulSigma(m)
ulexp = ulcomp.ulSigma(m, expected=True)
dx = .5
m = Data(nobs, nbg, 0.001, None, nsig, deltas_rel=0.)
llhdcomp = LikelihoodComputer(m)
llhddir = llhdcomp.likelihood(nsig)
chi2dir = llhdcomp.chi2(nsig)
llhdmarg = llhdcomp.likelihood(nsig, marginalize=True)
chi2marg = llhdcomp.chi2(nsig, marginalize=True)
llhdlim = likelihoodFromLimits(ulobs, ulexp, nsig)
chi2lim = chi2FromLimits(llhdlim, ulexp)
## relative error on chi2, for this example is about 4%
rel = abs(chi2lim - chi2marg) / chi2marg
self.assertAlmostEqual(rel, 0.04, 1)
def getSRUpperLimit(self,
alpha=0.05,
expected=False,
compute=False,
deltas_rel=0.2):
"""
Computes the 95% upper limit on the signal*efficiency for a given dataset (signal region).
Only to be used for efficiency map type results.
:param alpha: Can be used to change the C.L. value. The default value is 0.05 (= 95% C.L.)
:param expected: Compute expected limit ( i.e. Nobserved = NexpectedBG )
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:param compute: If True, the upper limit will be computed
from expected and observed number of events. If False, the value listed
in the database will be used instead.
:return: upper limit value
"""
if not self.getType() == 'efficiencyMap':
logger.error(
"getSRUpperLimit can only be used for efficiency map results!")
raise SModelSError()
if not compute:
if expected:
try:
return self.dataInfo.expectedUpperLimit
except AttributeError:
logger.info(
"expectedUpperLimit field not found. Using observed UL instead."
)
return self.dataInfo.upperLimit
else:
return self.dataInfo.upperLimit
Nobs = self.dataInfo.observedN #Number of observed events
if expected:
Nobs = self.dataInfo.expectedBG
Nexp = self.dataInfo.expectedBG #Number of expected BG events
bgError = self.dataInfo.bgError # error on BG
m = Data(Nobs, Nexp, bgError, detlas_rel=deltas_rel)
computer = UpperLimitComputer(cl=1. - alpha)
maxSignalXsec = computer.ulSigma(m)
maxSignalXsec = maxSignalXsec / self.globalInfo.lumi
return maxSignalXsec
def totalChi2(self, nsig, marginalize=False, deltas_rel=0.2):
"""
Computes the total chi2 for a given number of observed events, given a
predicted signal "nsig", with nsig being a vector with one entry per
dataset. nsig has to obey the datasetOrder. Deltas is the error on
the signal efficiency.
:param nsig: predicted signal (list)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:returns: chi2 (float)
"""
if hasattr(self.globalInfo, "covariance" ):
if len(self._datasets) == 1:
if isinstance(nsig,list):
nsig = nsig[0]
return self._datasets[0].chi2(nsig, marginalize=marginalize)
nobs = [x.dataInfo.observedN for x in self._datasets ]
bg = [x.dataInfo.expectedBG for x in self._datasets ]
cov = self.globalInfo.covariance
computer = LikelihoodComputer(Data(nobs, bg, cov, deltas_rel=deltas_rel))
return computer.chi2(nsig, marginalize=marginalize)
elif hasattr(self.globalInfo, "jsonFiles"):
# Getting the path to the json files
# Loading the jsonFiles
ulcomputer, combinations = self.getPyhfComputer( nsig )
if ulcomputer.nWS == 1:
return ulcomputer.chi2()
else:
# Looking for the best combination
if self.bestCB == None:
ulMin = float('+inf')
for i_ws in range(ulcomputer.nWS):
logger.debug("Performing best expected combination")
ul = ulcomputer.ulSigma(expected=True, workspace_index=i_ws)
if ul < ulMin:
ulMin = ul
i_best = i_ws
self.bestCB = combinations[i_best] # Keeping the index of the best combination for later
logger.debug('Best combination : %d' % self.bestCB)
return ulcomputer.chi2(workspace_index=combinations.index(self.bestCB))
else:
logger.error("Asked for combined likelihood, but no covariance error given." )
return None
def chi2(self, nsig, deltas_rel=0.2, marginalize=False):
"""
Computes the chi2 for a given number of observed events "nobs",
given number of signal events "nsig", and error on signal "deltas".
nobs, expectedBG and bgError are part of dataInfo.
:param nsig: predicted signal (float)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:param marginalize: if true, marginalize nuisances. Else, profile them.
:return: chi2 (float)
"""
m = Data(self.dataInfo.observedN, self.dataInfo.expectedBG,
self.dataInfo.bgError**2,deltas_rel=deltas_rel)
computer = LikelihoodComputer(m)
ret = computer.chi2(nsig, marginalize=marginalize)
return ret
def getCombinedUpperLimitFor(self, nsig, expected=False, deltas_rel=0.2):
"""
Get combined upper limit.
:param nsig: list of signal events in each signal region/dataset. The list
should obey the ordering in globalInfo.datasetOrder.
:param expected: return expected, not observed value
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:returns: upper limit on sigma*eff
"""
if not hasattr(self.globalInfo, "covariance"):
logger.error(
"no covariance matrix given in globalInfo.txt for %s" %
self.globalInfo.id)
raise SModelSError(
"no covariance matrix given in globalInfo.txt for %s" %
self.globalInfo.id)
cov = self.globalInfo.covariance
if type(cov) != list:
raise SModelSError("covariance field has wrong type: %s" %
type(cov))
if len(cov) < 1:
raise SModelSError("covariance matrix has length %d." % len(cov))
computer = UpperLimitComputer(ntoys=10000)
nobs = [x.dataInfo.observedN for x in self._datasets]
bg = [x.dataInfo.expectedBG for x in self._datasets]
no = nobs
ret = computer.ulSigma(Data(observed=no,
backgrounds=bg,
covariance=cov,
third_moment=None,
nsignal=nsig,
deltas_rel=deltas_rel),
marginalize=self._marginalize,
expected=expected)
#Convert limit on total number of signal events to a limit on sigma*eff
ret = ret / self.globalInfo.lumi
return ret
def likelihood(self, nsig, deltas_rel=0.2, marginalize=False):
"""
Computes the likelihood to observe nobs events,
given a predicted signal "nsig", assuming "deltas"
error on the signal efficiency.
The values observedN, expectedBG, and bgError are part of dataInfo.
:param nsig: predicted signal (float)
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:param marginalize: if true, marginalize nuisances. Else, profile them.
:returns: likelihood to observe nobs events (float)
"""
m = Data(self.dataInfo.observedN,
self.dataInfo.expectedBG,
self.dataInfo.bgError**2,
deltas_rel=deltas_rel)
computer = LikelihoodComputer(m)
return computer.likelihood(nsig, marginalize=marginalize)
def createModel(self, n=3):
import model_90 as m9
S = m9.third_moment.tolist()[:n]
D = m9.observed.tolist()[:n]
B = m9.background.tolist()[:n]
sig = [x / 100. for x in m9.signal.tolist()[:n]]
C_ = m9.covariance.tolist()
ncov = int(sqrt(len(C_)))
C = []
for i in range(n):
C.append(C_[ncov * i:ncov * i + n])
m = Data(observed=D,
backgrounds=B,
covariance=C,
third_moment=S,
nsignal=sig,
name="model%d" % n,
deltas_rel=0.)
return m
def testPredictionInterface(self):
""" A simple test to see that the interface in datasetObj
and TheoryPrediction to the statistics tools is working correctly
"""
expRes = database.getExpResults(analysisIDs=['CMS-SUS-13-012'])[0]
slhafile = "./testFiles/slha/simplyGluino.slha"
smstoplist = slhaDecomposer.decompose(slhafile)
prediction = theoryPredictionsFor(expRes, smstoplist, deltas_rel=0.)[0]
pred_signal_strength = prediction.xsection.value
prediction.computeStatistics()
ill = math.log(prediction.likelihood)
ichi2 = prediction.chi2
nsig = (pred_signal_strength * expRes.globalInfo.lumi).asNumber()
m = Data(4, 2.2, 1.1**2, None, nsignal=nsig, deltas_rel=0.2)
computer = LikelihoodComputer(m)
dll = math.log(computer.likelihood(nsig, marginalize=False))
self.assertAlmostEqual(ill, dll, places=2)
dchi2 = computer.chi2(nsig, marginalize=False)
# print ( "dchi2,ichi2",dchi2,ichi2)
self.assertAlmostEqual(ichi2, dchi2, places=2)
def testLikelihood(self):
"""
Compare the computed chi2 from a given observed
and expected upper limit and a theory prediction
with the previously known result for the value of
the chi2.
All values of nobs, nsig, nb, deltab come from the
SModelS database and are for the T1 simplified model
from efficiency results of one of
ATLAS-SUSY-2013-02
ATLAS-CONF-2013-047
CMS-SUS-13-012
ATLAS-CONF-2013-054
"""
expected_values = [
# mgluino mlsp nsig nobs nb deltab llhd chi2
# ---------- ---------- --------------- ---------------- ---------- ---------- -------------------- ----------------
{
'mgluino': 500,
'mlsp': 200,
'nsig': 384.898,
'nobs': 298.413,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00024197,
'chi2': 7.32614
}
]
{
'mgluino': 500,
'mlsp': 300,
'nsig': 185.166,
'nobs': 223.619,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00215989,
'chi2': 3.67088900
},
{
'mgluino': 500,
'mlsp': 400,
'nsig': 450.820,
'nobs': 2331.38,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00075499,
'chi2': 2.85026
},
{
'mgluino': 600,
'mlsp': 100,
'nsig': 476.150,
'nobs': 437.874,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00100575,
'chi2': 3.52406595
},
{
'mgluino': 600,
'mlsp': 200,
'nsig': 264.387,
'nobs': 232.912,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00028921,
'chi2': 7.57051432
},
{
'mgluino': 600,
'mlsp': 300,
'nsig': 171.766,
'nobs': 211.038,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00198061,
'chi2': 4.02903
},
{
'mgluino': 600,
'mlsp': 400,
'nsig': 66.9991,
'nobs': 150.393,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.00845030,
'chi2': 1.89194013
},
{
'mgluino': 600,
'mlsp': 500,
'nsig': 157.571,
'nobs': 2167.25,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00217371,
'chi2': 0.845615
},
{
'mgluino': 700,
'mlsp': 100,
'nsig': 307.492,
'nobs': 325.060,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00159632,
'chi2': 3.41955
},
{
'mgluino': 700,
'mlsp': 200,
'nsig': 211.534,
'nobs': 228.763,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00061670,
'chi2': 6.26852955
},
{
'mgluino': 700,
'mlsp': 300,
'nsig': 147.084,
'nobs': 167.631,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00012707,
'chi2': 10.1408114
},
{
'mgluino': 700,
'mlsp': 400,
'nsig': 420.524,
'nobs': 2332.28,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00100854,
'chi2': 2.28195399
},
{
'mgluino': 700,
'mlsp': 500,
'nsig': 186.726,
'nobs': 2162.70,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00165411,
'chi2': 1.39601
},
{
'mgluino': 700,
'mlsp': 600,
'nsig': 5.18888,
'nobs': 24.3271,
'nb': 37.0,
'deltab': 6.0,
'llhd': 0.00545866,
'chi2': 4.3836
},
{
'mgluino': 800,
'mlsp': 100,
'nsig': 169.670,
'nobs': 213.312,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00116298,
'chi2': 5.09803029
},
{
'mgluino': 800,
'mlsp': 200,
'nsig': 152.221,
'nobs': 212.732,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00223053,
'chi2': 3.81519907
},
{
'mgluino': 800,
'mlsp': 300,
'nsig': 98.6749,
'nobs': 175.141,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00298021,
'chi2': 3.72566221
},
{
'mgluino': 800,
'mlsp': 400,
'nsig': 59.3935,
'nobs': 141.966,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00262008,
'chi2': 4.30322736
},
{
'mgluino': 800,
'mlsp': 500,
'nsig': 27.7738,
'nobs': 123.172,
'nb': 111.0,
'deltab': 11.0,
'llhd': 0.01605069,
'chi2': 0.903864
},
{
'mgluino': 800,
'mlsp': 600,
'nsig': 6.40339,
'nobs': 39.2979,
'nb': 33.0,
'deltab': 6.0,
'llhd': 0.04536718,
'chi2': -0.000501411
},
{
'mgluino': 800,
'mlsp': 700,
'nsig': 4.38635,
'nobs': 132.824,
'nb': 125.0,
'deltab': 10.0,
'llhd': 0.02525385,
'chi2': 0.0565348
},
{
'mgluino': 900,
'mlsp': 100,
'nsig': 18.8255,
'nobs': 14.1228,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.02122262,
'chi2': 2.85426343
},
{
'mgluino': 900,
'mlsp': 200,
'nsig': 16.0543,
'nobs': 6.77062,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.00187567,
'chi2': 8.43890579
},
{
'mgluino': 900,
'mlsp': 300,
'nsig': 64.4188,
'nobs': 142.220,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00181163,
'chi2': 5.00642964
},
{
'mgluino': 900,
'mlsp': 400,
'nsig': 44.8312,
'nobs': 140.979,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00692173,
'chi2': 2.34800741
},
{
'mgluino': 900,
'mlsp': 500,
'nsig': 24.4723,
'nobs': 120.688,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00601021,
'chi2': 2.72454478
},
{
'mgluino': 900,
'mlsp': 600,
'nsig': 67.0446,
'nobs': 2165.25,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00328372,
'chi2': 0.0371125
},
{
'mgluino': 900,
'mlsp': 700,
'nsig': 1.00167,
'nobs': 5.0,
'nb': 5.2,
'deltab': 1.4,
'llhd': 0.13964962,
'chi2': 0.107139
},
{
'mgluino': 900,
'mlsp': 800,
'nsig': 0.86634,
'nobs': 24.0,
'nb': 37.0,
'deltab': 6.0,
'llhd': 0.01303119,
'chi2': 2.638323
},
{
'mgluino': 1000,
'mlsp': 100,
'nsig': 11.7426,
'nobs': 11.8786,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.05712388,
'chi2': 1.06934
},
{
'mgluino': 1000,
'mlsp': 200,
'nsig': 9.85815,
'nobs': 7.98535,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.03180710,
'chi2': 2.63593288
},
{
'mgluino': 1000,
'mlsp': 300,
'nsig': 6.80275,
'nobs': 6.14772,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.04255251,
'chi2': 2.25703866
},
{
'mgluino': 1000,
'mlsp': 400,
'nsig': 25.8451,
'nobs': 120.523,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.00525390,
'chi2': 2.99137140
},
{
'mgluino': 1000,
'mlsp': 500,
'nsig': 18.6299,
'nobs': 122.095,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01056408,
'chi2': 1.59516468
},
{
'mgluino': 1000,
'mlsp': 600,
'nsig': 10.2636,
'nobs': 119.968,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01536934,
'chi2': 0.84011292
},
{
'mgluino': 1000,
'mlsp': 700,
'nsig': 4.59470,
'nobs': 121.728,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.02078487,
'chi2': 0.23333618
},
{
'mgluino': 1000,
'mlsp': 800,
'nsig': 1.91162,
'nobs': 121.196,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.02193946,
'chi2': 0.12552973
},
{
'mgluino': 1000,
'mlsp': 900,
'nsig': 0.62255,
'nobs': 133.0,
'nb': 125.0,
'deltab': 10.0,
'llhd': 0.02290147,
'chi2': 0.253293
},
{
'mgluino': 1100,
'mlsp': 100,
'nsig': 5.95636,
'nobs': 5.94515,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.05236744,
'chi2': 1.87080349
},
{
'mgluino': 1100,
'mlsp': 200,
'nsig': 5.51938,
'nobs': 5.92472,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.06002489,
'chi2': 1.59429
},
{
'mgluino': 1100,
'mlsp': 300,
'nsig': 3.93082,
'nobs': 6.07873,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.09732480,
'chi2': 0.61881103
},
{
'mgluino': 1100,
'mlsp': 400,
'nsig': 2.80428,
'nobs': 6.54033,
'nb': 4.9,
'deltab': 1.6,
'llhd': 0.12350249,
'chi2': 0.0882501
},
{
'mgluino': 1100,
'mlsp': 500,
'nsig': 12.6778,
'nobs': 125.271,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01749246,
'chi2': 0.560614
},
{
'mgluino': 1100,
'mlsp': 600,
'nsig': 8.02475,
'nobs': 119.742,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01700322,
'chi2': 0.64005829
},
{
'mgluino': 1100,
'mlsp': 700,
'nsig': 4.74108,
'nobs': 120.211,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.01979279,
'chi2': 0.334838
},
{
'mgluino': 1100,
'mlsp': 800,
'nsig': 1.79622,
'nobs': 120.858,
'nb': 126.0,
'deltab': 13.0,
'llhd': 0.02187799,
'chi2': 0.134012
},
{
'mgluino': 1100,
'mlsp': 900,
'nsig': 4.82397,
'nobs': 2166.20,
'nb': 2120.0,
'deltab': 110.0,
'llhd': 0.00313424,
'chi2': 0.119045
},
{
'mgluino': 1100,
'mlsp': 1000,
'nsig': 0.1606,
'nobs': 25.0,
'nb': 37.0,
'deltab': 6.0,
'llhd': 0.01796058,
'chi2': 1.9806
},
{
'mgluino': 2100,
'mlsp': 1000,
'nsig': 0.1606,
'nobs': 2.0,
'nb': 0.7,
'deltab': 6.0,
'llhd': 0.108669,
'chi2': -0.161304
}
for d in expected_values:
nobs = d['nobs']
nsig = d['nsig']
nb = d['nb']
deltab = d['deltab']
# print ("ns="+str(nsig)+"; nobs = "+str(nobs)+"; nb="+str(nb)+"; db="+str(deltab))
# Chi2 as computed by statistics module:
m = Data(nobs, nb, deltab**2, deltas_rel=0.2)
computer = LikelihoodComputer(m)
chi2_actual = computer.chi2(nsig, marginalize=True) ## , .2*nsig )
chi2_expected = d['chi2']
if not chi2_expected == None and not np.isnan(chi2_expected):
# chi2_expected = self.round_to_sign(chi2_expected, 2)
# Check that chi2 values agree:
self.assertAlmostEqual(abs(chi2_actual - chi2_expected) /
chi2_expected,
0.,
places=2)
else:
self.assertTrue(chi2_actual == None or np.isnan(chi2_actual))
# likelihood as computed by statistics module:
# computer = LikelihoodComputer( nobs, nb, deltab**2 )
#likelihood_actual = statistics.likelihood( nsig,
# nobs, nb, deltab, deltas)
likelihood_actual = computer.likelihood(nsig, marginalize=False)
# likelihood_actual = statistics.likelihood()
# logger.error("llk= "+str(likelihood_actual)+" nsig="+str(nsig)+" nobs = "+str(nobs)+" nb="+str(nb)+"+-"+str(deltab))
#print('llhdactual', likelihood_actual)
if not likelihood_actual == None and not np.isnan(
likelihood_actual):
likelihood_actual = self.round_to_sign(likelihood_actual, 4)
# The previously computed likelihood:
# (using: ntoys=100000)
likelihood_expected = d['llhd']
#print('llhdexp', likelihood_expected)
if not likelihood_expected == None and not np.isnan(
likelihood_expected):
likelihood_expected = self.round_to_sign(
likelihood_expected, 4)
# Check that likelihood values agree:
self.assertAlmostEqual(likelihood_actual,
likelihood_expected,
delta=2 * 1e-1)
else:
self.assertTrue(likelihood_actual == None
or np.isnan(likelihood_actual))
def testUpperLimit(self):
m = Data(100., 100., 0.001, None, 1.0, deltas_rel=0.)
comp = UpperLimitComputer()
re = comp.ulSigma(m)
self.assertAlmostEqual(re / (1.06 * 20.), 1., 1)
def getCombinedUpperLimitFor(self, nsig, expected=False, deltas_rel=0.2):
"""
Get combined upper limit. If covariances are given in globalInfo then simplified likelihood is used, else if json files are given pyhf cimbination is performed.
:param nsig: list of signal events in each signal region/dataset. The list
should obey the ordering in globalInfo.datasetOrder.
:param expected: return expected, not observed value
:param deltas_rel: relative uncertainty in signal (float). Default value is 20%.
:returns: upper limit on sigma*eff
"""
if hasattr(self.globalInfo, "covariance" ):
cov = self.globalInfo.covariance
if type(cov) != list:
raise SModelSError( "covariance field has wrong type: %s" % type(cov))
if len(cov) < 1:
raise SModelSError( "covariance matrix has length %d." % len(cov))
computer = UpperLimitComputer(ntoys=10000)
nobs = [x.dataInfo.observedN for x in self._datasets]
bg = [x.dataInfo.expectedBG for x in self._datasets]
no = nobs
ret = computer.ulSigma(Data(observed=no, backgrounds=bg, covariance=cov,
third_moment=None, nsignal=nsig, deltas_rel=deltas_rel),
marginalize=self._marginalize,
expected=expected)
if ret != None:
#Convert limit on total number of signal events to a limit on sigma*eff
ret = ret/self.globalInfo.lumi
logger.debug("SL upper limit : {}".format(ret))
return ret
elif hasattr(self.globalInfo, "jsonFiles" ):
logger.debug("Using pyhf")
if all([s == 0 for s in nsig]):
logger.warning("All signals are empty")
return None
ulcomputer, combinations = self.getPyhfComputer( nsig )
if ulcomputer.nWS == 1:
ret = ulcomputer.ulSigma(expected=expected)
ret = ret/self.globalInfo.lumi
logger.debug("pyhf upper limit : {}".format(ret))
return ret
else:
# Looking for the best combination
logger.debug('self.bestCB : {}'.format(self.bestCB))
if self.bestCB == None:
logger.debug("Performing best expected combination")
ulMin = float('+inf')
for i_ws in range(ulcomputer.nWS):
ul = ulcomputer.ulSigma(expected=True, workspace_index=i_ws)
if ul == None:
continue
if ul < ulMin:
ulMin = ul
i_best = i_ws
self.bestCB = combinations[i_best] # Keeping the index of the best combination for later
logger.debug('Best combination : %s' % self.bestCB)
# Computing upper limit using best combination
if expected:
try:
ret = ulMin/self.globalInfo.lumi
except NameError:
ret = ulcomputer.ulSigma(expected=True, workspace_index=combinations.index(self.bestCB))
ret = ret/self.globalInfo.lumi
else:
ret = ulcomputer.ulSigma(expected=False, workspace_index=combinations.index(self.bestCB))
ret = ret/self.globalInfo.lumi
logger.debug("pyhf upper limit : {}".format(ret))
return ret
else:
logger.error ( "no covariance matrix or json file given in globalInfo.txt for %s" % self.globalInfo.id )
raise SModelSError( "no covariance matrix or json file given in globalInfo.txt for %s" % self.globalInfo.id )
def run ( nobs, nExp, nExpErr, nsig ):
""" run the procedure, with:
:param nobs: number of observed
:param nExp: number expected
:param nExpErr: error on number expected
"""
print ( "Starting run with nobs", nobs, "nExp", nExp )
data = Data ( nobs, nExp, nExpErr**2, nsignal = nsig )
computer = UpperLimitComputer ( 10000 )
marginalize = False # True
ULobs = computer.ulSigma ( data, marginalize=marginalize )
ULexp = computer.ulSigma ( data, expected=True, marginalize=marginalize )
print(r'Nobs = %1.2f, Nbg = %1.2f +- %1.2f, Nsig < %1.2f, Nsig (expected) < %1.2f'
%(nobs,nExp,nExpErr,ULobs,ULexp))
sigma = ULexp/1.96
mu0 = 0
def erfroot ( x ):
return 0.95- (special.erf((ULobs-x)/(np.sqrt(2)*sigma)) + special.erf(x/(np.sqrt(2)*sigma)))/(1+special.erf(x/(np.sqrt(2)*sigma)) )
if nobs > nExp:
ua,ub = 0,2*(3*nExpErr)
xa,xb=1,1
while xa*xb > .0:
xa,xb= erfroot ( ua ), erfroot ( ub )
ub = 2*ub
mu0 = optimize.brentq(lambda x: erfroot(x), ua,ub )
print ( "mu0", mu0 )
# mu0 = nobs - nExp
# print ( "mu0", mu0, "nobs", nobs, "nExp", nExp )
def llhdFromLimits(mu,mu0,sigma):
return stats.norm.pdf(x=mu,loc=mu0,scale=sigma)
normLim = 1 - stats.norm.cdf(0,loc=mu0,scale=sigma)
nsteps = 100 # 100
mulim = nobs-nExp+4*nExpErr
norm = p((nobs-nExp)+5*nExpErr,nsig,nExp,nExpErr,nobs)
while True:
Lmax1 = likelihood ( mulim, nsig,nExp,nExpErr,nobs)/norm
Lmax2 = llhdFromLimits ( mulim, mu0, sigma )/normLim
if max(Lmax1,Lmax2)<.01:
break
mulim = 1.2 * mulim
muvals = np.linspace(0, mulim, nsteps )
llhds = np.array([[mu,likelihood(mu,nsig,nExp,nExpErr,nobs)/norm] for mu in muvals])
llhdsApp = np.array([[mu,llhdFromLimits(mu,mu0,sigma)/normLim] for mu in muvals])
print ( "llhds", llhds[::20] )
print ( "llhdsApp", llhdsApp[::20] )
mumax = llhds[np.argmax(llhds[:,1])][0]
mumaxApp = llhdsApp[np.argmax(llhdsApp[:,1])][0]
f = plt.figure(figsize=(8,4))
plt.plot(llhds[:,0],llhds[:,1],label='Full',linewidth=3,color="black")
plt.plot(llhdsApp[:,0],llhdsApp[:,1],label='From Limits',linestyle='--',linewidth=3)
plt.legend()
plt.xlabel(r'$\mu$')
plt.ylabel(r'$L$')
plt.text(0,llhdsApp[:,1].min(),'$\mu_{max} = %1.2f$\n$\mu_{max}^{app} = %1.2f$' %(mumax,mumaxApp),fontsize=14)
plt.title(r'$N_{obs} = %1.2f, N_{bg} = %1.2f \pm %1.2f, \sigma_{UL}^{obs} = %1.2f, \sigma_{UL}^{exp} = %1.2f, \Delta = %1.2f$'
%(nobs,nExp,nExpErr,ULobs,ULexp,abs(ULobs-ULexp)/(ULobs+ULexp)),fontsize=14)
plt.savefig("llhd.png")