def __init__(self, protomodel, pid1, pid2, nproc, rundir):
self.rundirarg = rundir
self.rundir = setup(rundir)
self.M = protomodel
self.pid1 = pid1
self.pid2 = pid2
self.nproc = nproc
expected = False
select = "all"
dbpath = rundir + "/default.pcl"
self.predictor = Predictor(0,
dbpath=dbpath,
expected=expected,
select=select)
def __init__(self,
walkerid=0,
nsteps=10000,
strategy="aggressive",
dump_training=False,
cheatcode=0,
dbpath="./database.pcl",
expected=False,
select="all",
catch_exceptions=True,
rundir=None,
nevents=100000,
do_combine=False,
record_history=False,
seed=None,
stopTeleportationAfter=-1):
""" initialise the walker
:param nsteps: maximum number of steps to perform, negative is infinity
:param cheatcode: cheat mode. 0 is no cheating, 1 is with ranges, 2
is the Z323 model.
:param expected: remove possible signals from database
:param select: select only subset of results (all for all, em for efficiency maps only, ul for upper limits only, alternatively select for txnames via e.g. "txnames:T1,T2"
:param catch_exceptions: should we catch exceptions
:param nevents: number of MC events when computing cross-sections
:param do_combine: if true, then also perform combinations, either via
simplified likelihoods or via pyhf
:param record_history: if true, attach a history recorder class
:param seed: random seed, int or None
:param stopTeleportationAfter: int or None. we stop teleportation after this step nr.
If negative or None, we dont teleport at all
"""
if type(walkerid) != int or type(nsteps) != int or type(
strategy) != str:
self.pprint("Wrong call of constructor: %s, %s, %s" %
(walkerid, nsteps, strategy))
sys.exit(-2)
self.walkerid = walkerid ## walker id, for parallel runs
self.rundir = rundir
if rundir == None:
self.rundir = "./"
if seed is not None:
from ptools import helpers
helpers.seedRandomNumbers(seed + walkerid)
self.pprint(f"setting random seed to {seed}")
#Initialize Predictor
self.predictor = Predictor(self.walkerid,
dbpath=dbpath,
expected=expected,
select=select,
do_combine=do_combine)
#Initialize Hiscore (with access to the predictor)
self.hiscoreList = Hiscore(walkerid,
True,
"%s/H%d.hi" % (self.rundir, walkerid),
backup=False,
predictor=self.predictor)
self.hiscoreList.nkeep = 1
#Initialize ProtoModel and Manipulator:
protomodel = ProtoModel(
self.walkerid,
keep_meta=True,
nevents=nevents,
dbversion=self.predictor.database.databaseVersion)
self.manipulator = Manipulator(protomodel,
strategy,
do_record=record_history,
seed=seed)
self.catch_exceptions = catch_exceptions
self.maxsteps = nsteps
if stopTeleportationAfter == None:
stopTeleportationAfter = -1
# stopTeleportationAfter = self.maxsteps/3.
self.stopTeleportationAfter = stopTeleportationAfter
self.accelerator = None
if record_history:
from ptools.history import History
self.recorder = History(f"{self.rundir}/history{walkerid}.list")
self.manipulator.do_record = True
jobid = "unknown"
if "SLURM_JOBID" in os.environ:
jobid = os.environ["SLURM_JOBID"]
self.pprint("Ramping up with slurm jobid %s" % jobid)
if cheatcode <= 0:
self.takeStep() # the first step should be considered as "taken"
#Set current Z and K values to threshold values
self.currentZ = -0.1
self.currentK = -20.0
else:
self.manipulator.cheat(cheatcode)
self.predictor.predict(self.protomodel)
self.pprint ( "Cheat model gets Z=%.2f, K=%.2f" % \
( self.manipulator.M.Z, self.manipulator.M.K ) )
# self.printStats ( substep=4 )
self.manipulator.backupModel()
self.hiscoreList.newResult(self.manipulator)
self.printStats(substep=5)
self.currentK = self.manipulator.M.K
self.currentZ = self.manipulator.M.Z
if dump_training:
from accelerator import Accelerator
## we use the accelerator only to dump the training data
self.accelerator = Accelerator(walkerid=walkerid,
dump_training=True,
is_trained=False)
class RandomWalker:
def __init__(self,
walkerid=0,
nsteps=10000,
strategy="aggressive",
dump_training=False,
cheatcode=0,
dbpath="./database.pcl",
expected=False,
select="all",
catch_exceptions=True,
rundir=None,
nevents=100000,
do_combine=False,
record_history=False,
seed=None,
stopTeleportationAfter=-1):
""" initialise the walker
:param nsteps: maximum number of steps to perform, negative is infinity
:param cheatcode: cheat mode. 0 is no cheating, 1 is with ranges, 2
is the Z323 model.
:param expected: remove possible signals from database
:param select: select only subset of results (all for all, em for efficiency maps only, ul for upper limits only, alternatively select for txnames via e.g. "txnames:T1,T2"
:param catch_exceptions: should we catch exceptions
:param nevents: number of MC events when computing cross-sections
:param do_combine: if true, then also perform combinations, either via
simplified likelihoods or via pyhf
:param record_history: if true, attach a history recorder class
:param seed: random seed, int or None
:param stopTeleportationAfter: int or None. we stop teleportation after this step nr.
If negative or None, we dont teleport at all
"""
if type(walkerid) != int or type(nsteps) != int or type(
strategy) != str:
self.pprint("Wrong call of constructor: %s, %s, %s" %
(walkerid, nsteps, strategy))
sys.exit(-2)
self.walkerid = walkerid ## walker id, for parallel runs
self.rundir = rundir
if rundir == None:
self.rundir = "./"
if seed is not None:
from ptools import helpers
helpers.seedRandomNumbers(seed + walkerid)
self.pprint(f"setting random seed to {seed}")
#Initialize Predictor
self.predictor = Predictor(self.walkerid,
dbpath=dbpath,
expected=expected,
select=select,
do_combine=do_combine)
#Initialize Hiscore (with access to the predictor)
self.hiscoreList = Hiscore(walkerid,
True,
"%s/H%d.hi" % (self.rundir, walkerid),
backup=False,
predictor=self.predictor)
self.hiscoreList.nkeep = 1
#Initialize ProtoModel and Manipulator:
protomodel = ProtoModel(
self.walkerid,
keep_meta=True,
nevents=nevents,
dbversion=self.predictor.database.databaseVersion)
self.manipulator = Manipulator(protomodel,
strategy,
do_record=record_history,
seed=seed)
self.catch_exceptions = catch_exceptions
self.maxsteps = nsteps
if stopTeleportationAfter == None:
stopTeleportationAfter = -1
# stopTeleportationAfter = self.maxsteps/3.
self.stopTeleportationAfter = stopTeleportationAfter
self.accelerator = None
if record_history:
from ptools.history import History
self.recorder = History(f"{self.rundir}/history{walkerid}.list")
self.manipulator.do_record = True
jobid = "unknown"
if "SLURM_JOBID" in os.environ:
jobid = os.environ["SLURM_JOBID"]
self.pprint("Ramping up with slurm jobid %s" % jobid)
if cheatcode <= 0:
self.takeStep() # the first step should be considered as "taken"
#Set current Z and K values to threshold values
self.currentZ = -0.1
self.currentK = -20.0
else:
self.manipulator.cheat(cheatcode)
self.predictor.predict(self.protomodel)
self.pprint ( "Cheat model gets Z=%.2f, K=%.2f" % \
( self.manipulator.M.Z, self.manipulator.M.K ) )
# self.printStats ( substep=4 )
self.manipulator.backupModel()
self.hiscoreList.newResult(self.manipulator)
self.printStats(substep=5)
self.currentK = self.manipulator.M.K
self.currentZ = self.manipulator.M.Z
if dump_training:
from accelerator import Accelerator
## we use the accelerator only to dump the training data
self.accelerator = Accelerator(walkerid=walkerid,
dump_training=True,
is_trained=False)
def hostname(self):
return socket.gethostname()
def setWalkerId(self, Id):
self.walkerid = Id
self.manipulator.setWalkerId(Id)
if self.accelerator != None:
self.accelerator.walkerid = Id
@classmethod
def fromProtoModel(cls,
protomodel,
nsteps=10000,
strategy="aggressive",
walkerid=0,
dump_training=False,
dbpath="<rundir>/database.pcl",
expected=False,
select="all",
catch_exceptions=True,
keep_meta=True,
rundir=None,
do_combine=False,
seed=None,
stopTeleportationAfter=-1):
ret = cls( walkerid, nsteps=nsteps, dbpath = dbpath, expected=expected,
select=select, catch_exceptions = catch_exceptions, rundir = rundir,
do_combine = do_combine, seed = seed, stopTeleportationAfter = \
stopTeleportationAfter )
ret.manipulator.M = protomodel
ret.manipulator.setWalkerId(walkerid)
ret.manipulator.backupModel()
if dump_training:
## we use the accelerator only to dump the training data
from accelerator import Accelerator
ret.accelerator = Accelerator(walkerid=walkerid,
dump_training=True,
is_trained=False)
return ret
@classmethod
def fromDictionary(cls,
dictionary,
nsteps=10000,
strategy="aggressive",
walkerid=0,
dump_training=False,
dbpath="<rundir>/database.pcl",
expected=False,
select="all",
catch_exceptions=True,
keep_meta=True,
rundir=None,
nevents=100000,
do_combine=False,
seed=None,
stopTeleportationAfter=-1):
ret = cls(walkerid,
nsteps=nsteps,
dbpath=dbpath,
expected=expected,
select=select,
catch_exceptions=catch_exceptions,
rundir=rundir,
nevents=nevents,
do_combine=do_combine,
seed=seed,
stopTeleportationAfter=stopTeleportationAfter)
ret.manipulator.M = ProtoModel( walkerid, keep_meta, \
dbversion = ret.predictor.database.databaseVersion )
ret.manipulator.initFromDict(dictionary)
ret.manipulator.setWalkerId(walkerid)
ret.manipulator.M.createNewSLHAFileName()
# ret.printStats ( substep=3 )
ret.manipulator.backupModel()
return ret
def pprint(self, *args):
""" logging """
if not hasattr(self, "walkerid"):
self.walkerid = -1
print("[walk:%d:%s-%s] %s" %
(self.walkerid, self.hostname(), time.strftime("%H:%M:%S"),
" ".join(map(str, args))))
self.log(*args)
@property
def protomodel(self):
return self.manipulator.M
@protomodel.setter
def protomodel(self, protomodel):
self.manipulator.M = protomodel
def printStats(self, substep=None):
""" print the stats, i.e. number of unfrozen particles.
for debugging. """
nUnfrozen = len(self.protomodel.unFrozenParticles())
nTotal = len(self.protomodel.particles)
pidsp = self.protomodel.unFrozenParticles()
pidsp.sort()
namer = SParticleNames(False)
prtcles = ", ".join(map(namer.asciiName, pidsp))
pidsbc = list(self.manipulator.getAllPidsOfBestCombo())
pidsbc.sort()
prtclesbc = ", ".join(map(namer.asciiName, pidsbc))
self.pprint ( "Step %d has %d/%d unfrozen particles: %s [in best combo: %s]" % \
( self.protomodel.step, nUnfrozen, nTotal, \
prtcles, prtclesbc ) )
if len(pidsbc) > 0 and not set(pidsbc).issubset(set(pidsp)):
self.pprint(
" `-- error! best combo pids arent subset of masses pids!!!")
self.manipulator.M.bestCombo = None
def onestep(self):
#Add one step
self.protomodel.step += 1
self.pprint("Step %d begins." % (self.protomodel.step))
self.printStats()
#Remove data about best combo
self.protomodel.cleanBestCombo()
# self.printStats( substep=11 )
printMemUsage = False
if printMemUsage:
self.pprint ( "memory footprint (kb): walker %d, model %d, accelerator %d" %\
( asizeof(self)/1024,asizeof(self.protomodel)/1024,asizeof(self.accelerator)/1024 ))
#Trim the model, so we start only with the relevant particles for the
#best combination in the previous step:
self.log("freeze pids that arent in best combo, we dont need them:")
nfrozen = self.manipulator.freezePidsNotInBestCombo()
self.log(" `- froze %d particles not in best combo" % nfrozen)
# self.printStats( substep=12 )
#Take a step in the model space:
self.manipulator.randomlyChangeModel()
# self.printStats( substep=13 )
nUnfrozen = len(self.protomodel.unFrozenParticles())
## number of pids in best combo, as a check
#Try to create a simpler model
#(merge pre-defined particles of their mass difference is below dm)
protomodelSimp = self.manipulator.simplifyModel(dm=200.0)
# self.printStats( substep=14 )
if self.catch_exceptions:
try:
self.predictor.predict(self.manipulator.M)
if protomodelSimp:
self.predictor.predict(protomodelSimp)
except Exception as e:
self.pprint(
"error ``%s'' (%s) encountered when trying to predict. lets revert and not count it as a step."
% (str(e), type(e)))
self.manipulator.restoreModel()
self.manipulator.M.step -= 1 # we dont count that step.
import traceback
traceback.print_exc()
return
else:
self.predictor.predict(self.manipulator.M)
if protomodelSimp:
self.predictor.predict(protomodelSimp)
#Now keep the model with highest score:
if protomodelSimp:
if self.manipulator.M.K is None or (
protomodelSimp.K is not None and
(protomodelSimp.K > self.manipulator.M.K)):
self.manipulator.M = protomodelSimp
#If no combination could be found, return
if self.manipulator.M.Z is None:
return
#the muhat multiplier gets multiplied into the signal strengths
self.manipulator.rescaleSignalBy(self.protomodel.muhat)
self.log ( "Top r values after rescaling are: %.2f, %.2f" % \
( self.manipulator.M.rvalues[0], self.manipulator.M.rvalues[1] ) )
self.log ( "Step %d: found highest Z: %.2f" % \
( self.protomodel.step, self.protomodel.Z ) )
nUnfrozen = len(self.protomodel.unFrozenParticles())
self.pprint ( "best combo for strategy ``%s'' is %s: %s: [K=%.2f, Z=%.2f, %d unfrozen]" % \
( self.manipulator.strategy, self.protomodel.letters, self.protomodel.description, self.protomodel.K, self.protomodel.Z, nUnfrozen ) )
smaxstp = "%s" % self.maxsteps
if self.maxsteps < 0:
smaxstp = "inf"
#For low scoring models, teleport to a high score model:
if self.checkIfToTeleport(pmax=0.5, norm=10.0):
# if we teleport the rest becomes irrelevant
return
self.printStats()
self.log ( "Step %d check if result goes into hiscore list" % \
( self.protomodel.step ) )
srs = "%s" % ", ".join(
["%.2f" % x for x in self.protomodel.rvalues[:3]])
self.log("r values before calling .newResult are at %s" % srs)
self.hiscoreList.newResult(self.manipulator) ## add to high score list
srs = "%s" % ", ".join(
["%.2f" % x for x in self.protomodel.rvalues[:3]])
self.log("r values after calling .newResult are at %s" % srs)
self.log("done check for result to go into hiscore list")
self.log("Step %d/%s finished." % (self.protomodel.step, smaxstp))
def checkIfToTeleport(self, pmax=0.1, norm=10.0):
""" check if we should teleport to a high score model. If yes, then we
should then also perform the teleportation. The teleportation is
done only if the model has a score smaller then the best score in
hiscoreList. The teleportation probability is given by
pmax*(1-exp^(K-bestK)/norm), so pmax is the maximum probability
(when K -> -infinity).
:param pmax: Maximum probability for teleportation.
:param norm: Normalization for K distance.
"""
if self.protomodel.step > self.stopTeleportationAfter:
self.log ( "teleportation is turned off after step #%d" % \
self.stopTeleportationAfter )
return False
#self.log ( "teleportation turned off" )
#return False
import random
bestK = self.hiscoreList.globalMaxK()
if bestK < 1.:
self.log("bestK is smaller than one. no teleporting.")
return False
ourK = -2.
if hasattr(self.manipulator.M, "K") and self.manipulator.M.K > -2:
ourK = self.manipulator.M.K
#The current model already is the best, do nothing.
if ourK >= bestK:
return False
#Otherwise compute the teleportation probability:
dK = (ourK - bestK) / norm
prob = pmax * (1. - math.exp(dK))
a = random.uniform(0., 1.)
doTP = (a < prob) ## do teleport, yes or no
sDoTP = "a>p: dont teleport."
if doTP:
sDoTP = "a<p: do teleport."
self.log ( "check if to teleport, Kmax=%.2f, ours is=%.2f, p=%.2f, a=%.2f, %s" % \
( bestK, ourK, prob, a, sDoTP ) )
if doTP:
self.manipulator.teleportToHiscore()
return doTP
def takeStep(self):
""" take the step, save it as last step """
## Backup model
self.manipulator.backupModel()
# Update current K and Z values
self.currentK = self.protomodel.K
self.currentZ = self.protomodel.Z
self.manipulator.record("take step")
def highlight(self, msgType="info", *args):
""" logging, hilit """
col = colorama.Fore.GREEN
print("%s[walk:%d] %s%s" % (col, self.walkerid, " ".join(map(
str, args)), colorama.Fore.RESET))
def decideOnTakingStep(self):
""" depending on the ratio of K values, decide on whether to take the step or not.
If ratio > 1., take the step, if < 1, let chance decide. """
import random
ratio = 1.
K = self.currentK
newK = self.protomodel.K
if K > -20. and newK < K:
ratio = numpy.exp(.5 * (newK - K))
if ratio >= 1.:
self.highlight ( "info", "K: %.3f -> %.3f: r=%.4f, take the step" % \
( self.currentK, self.protomodel.K, ratio ) )
if self.protomodel.K > 0. and self.protomodel.K < 0.7 * self.currentK:
self.pprint(" `- weirdly, though, K decreases. Please check.")
sys.exit(-2)
self.takeStep()
else:
u = random.uniform(0., 1.)
if u > ratio:
self.pprint("u=%.2f > %.2f; K: %.2f -> %.2f: revert." %
(u, ratio, self.currentK, self.protomodel.K))
self.manipulator.restoreModel(reportReversion=True)
if hasattr(self, "oldgrad") and self.accelerator != None:
self.accelerator.grad = self.oldgrad
else:
self.pprint(
"u=%.2f <= %.2f ; %.2f -> %.2f: take the step, even though old is better."
% (u, ratio, self.currentK, self.protomodel.Z))
self.takeStep()
def log(self, *args):
""" logging to file """
with open("%s/walker%d.log" % (self.rundir, self.walkerid), "a") as f:
f.write("[randomWalker-%s] %s\n" %
(time.strftime("%H:%M:%S"), " ".join(map(str, args))))
def record(self):
""" if recorder is defined, then record. """
## do we have a history recorder?
if not hasattr(self, "recorder"):
return
self.recorder.add(self.manipulator)
def walk(self, catchem=False):
""" Now perform the random walk """
# self.printStats ( substep = 2 )
self.manipulator.backupModel()
if len(self.manipulator.M.unFrozenParticles(withLSP=False)) < 1:
## start with unfreezing a random particle
self.manipulator.randomlyUnfreezeParticle(force=True)
self.manipulator.backupModel()
while self.maxsteps < 0 or self.protomodel.step < self.maxsteps:
if not catchem:
self.onestep()
else:
try:
self.onestep()
except Exception as e:
# https://bioinfoexpert.com/2016/01/18/tracing-exceptions-in-multiprocessing-in-python/
self.pprint ( "taking a step resulted in exception: %s, %s" % \
(type(e), e ) )
import traceback
traceback.print_stack(limit=None)
except_type, except_class, tb = sys.exc_info()
extracted = traceback.extract_tb(tb)
for point in extracted:
self.pprint("extracted: %s" % point)
with open("%s/exceptions.log" % self.rundir, "a") as f:
f.write ( "%s: taking a step resulted in exception: %s, %s\n" % \
(time.asctime(), type(e), e ) )
f.write ( " `- exception occured in walker #%s\n" % \
self.protomodel.walkerid )
sys.exit(-1)
#If no combination was found, go back
if self.protomodel.K is None:
self.manipulator.restoreModel()
continue
# obtain the ratio of posteriors
self.decideOnTakingStep()
self.record()
self.manipulator.M.delCurrentSLHA()
self.pprint("Was asked to stop after %d steps" % self.maxsteps)
#!/usr/bin/env python3
""" restore hiscore.hi from the dictionary file """
# from ptools import hiscoreTools
from walker.hiscore import Hiscore
from builder.manipulator import Manipulator
from builder.protomodel import ProtoModel
from tester.predictor import Predictor
from smodels.tools.smodelsLogging import logger
logger.setLevel("ERROR")
import subprocess
## maybe add copying of real*.dict to hiscores.dict
## maybe add check for database pickle file
subprocess.getoutput("rm H*hi")
subprocess.getoutput("rm Kold.conf")
pmodel = ProtoModel(0)
pr = Predictor(0)
ma = Manipulator(pmodel)
ma.initFromDictFile("hiscores.dict", initTestStats=True)
print("The previous K value was", ma.M.K)
pr.predict(ma.M)
print("We end up with K=", ma.M.K)
hi = Hiscore(0, True, picklefile="H1.hi", hiscores=[ma.M])
hi.save()
a = subprocess.getoutput("./upHi.py")
print(a)
def testPredictions(self):
with open('randomModels_default.pcl', 'rb') as f:
protomodel = pickle.load(f)[0]
protomodel.dbversion = None
protomodel.codeversion = None
pNew = protomodel.copy()
pNew.templateSLHA = os.path.abspath(
'../builder/templates/template1g.slha')
pNew.Z = None
pNew.llhd = None
pNew.bestCombo = None
pNew.muhat = None
pNew.mumax = None
pNew.tpList = []
#Test against old version:
predictor = Predictor(0,
dbpath='./database.pcl',
expected=False,
select='all')
predictor.rthreshold = 1.7
predictor.predict(pNew)
#OBS: Since the original protomodel already has all of its cross-sections rescaled, we do not
#need to rescale pNew again (since muhat should be 1)
#Hard coded combarison against (old) version (minor xsec differences and the change in prior):
oldPrior = predictor.combiner.computePrior(pNew,
name="expo2",
nll=True)
newPrior = predictor.combiner.computePrior(pNew,
name="expo1",
nll=True)
newK = pNew.K - 2 * oldPrior + 2 * newPrior
self.assertTrue(abs(3.9 - newK) / 3.9 < 0.1)
self.assertTrue(abs(2.3 - pNew.Z) / 2.3 < 0.1)
self.assertEqual(15, len(pNew.rvalues))
self.assertEqual('BDGK', pNew.letters)
#From previous (pre-refac) version:
rvalues = [
1.725, 1.216, 1.133, 0.824, 0.536, 0.285, 0.27, 0.198, 0.147,
0.112, 0.074, 0.073, 0.052, 0.049, 0.00242
]
for i, r in enumerate(rvalues):
self.assertTrue(
abs(pNew.rvalues[i] - r) / r <
0.05) #Allow no more than 5% differences (due to xsec)
#Compare against new default:
predictor.rthreshold = 1.3
predictor.predict(pNew)
self.assertAlmostEqual(protomodel.Z, pNew.Z, 3)
self.assertAlmostEqual(protomodel.K, pNew.K, 3)
self.assertAlmostEqual(protomodel.muhat, pNew.muhat, 3)
self.assertAlmostEqual(protomodel.mumax, pNew.mumax, 3)
self.assertAlmostEqual(protomodel.llhd, pNew.llhd, 3)
np.testing.assert_almost_equal(protomodel.rvalues, pNew.rvalues, 3)
self.assertEqual(len(protomodel.bestCombo), len(pNew.bestCombo))
for i, pred in enumerate(protomodel.bestCombo):
self.assertEqual(str(pred.expResult),
str(pNew.bestCombo[i].expResult))
self.assertEqual(str(pred.dataset), str(pNew.bestCombo[i].dataset))
self.assertAlmostEqual(pred.xsection.value.asNumber(),
pNew.bestCombo[i].xsection.value.asNumber(),
3)
self.assertAlmostEqual(pred.upperLimit.asNumber(),
pNew.bestCombo[i].upperLimit.asNumber(), 3)
for i, pred in enumerate(protomodel.tpList):
self.assertAlmostEqual(pred[0], pNew.tpList[i][0])
self.assertEqual(str(pred[2].expResult),
str(pNew.tpList[i][2].expResult))
self.assertAlmostEqual(pred[2].xsection.value.asNumber(),
pNew.tpList[i][2].xsection.value.asNumber(),
3)
#Remove files generated during run
for f in glob.glob('.cur*slha'):
os.remove(f)
class LlhdScanner:
""" class that encapsulates a likelihood sweep """
def __init__(self, protomodel, pid1, pid2, nproc, rundir):
self.rundirarg = rundir
self.rundir = setup(rundir)
self.M = protomodel
self.pid1 = pid1
self.pid2 = pid2
self.nproc = nproc
expected = False
select = "all"
dbpath = rundir + "/default.pcl"
self.predictor = Predictor(0,
dbpath=dbpath,
expected=expected,
select=select)
def pprint(self, *args):
""" pretty print """
t = time.strftime("%H:%M:%S")
line = "[llhdscanner:%s] %s" % (t, " ".join(map(str, args)))
print(line)
with open("llhdscan%d.log" % self.pid1, "at") as f:
f.write(line + "\n")
def describeRange(self, r):
""" describe range r in a string """
if len(r) == 0:
return ""
if len(r) == 1:
return "%d" % r[0]
if len(r) == 2:
return "%d,%d" % (r[0], r[1])
return "%d,%d ... %d" % (r[0], r[1], r[-1])
def getMassPoints(self, rpid1, rpid2):
""" run for the given mass points
:param rpid1: list of masses for pid1
:param rpid2: list of masses for pid2
:returns: masspoints
"""
if self.nproc == 1:
return runThread ( 0, self.rundir, self.M, self.pid1, self.pid2, \
self.mpid1, self.mpid2, self.nevents, rpid1, rpid2,
self.predictor, None )
chunkedRPid1 = [list(rpid1[i::self.nproc]) for i in range(self.nproc)]
processes = []
manager = multiprocessing.Manager()
return_dict = manager.dict()
# print ( "chunked", chunkedRPid1 )
for ctr, chunk in enumerate(chunkedRPid1):
self.M.walkerid = ctr
p = multiprocessing.Process(
target=runThread,
args=(ctr, self.rundir, self.M, self.pid1, self.pid2,
self.mpid1, self.mpid2, self.nevents, chunk, rpid2,
self.predictor, return_dict))
p.start()
processes.append(p)
for p in processes:
p.join()
masspoints = []
hasStored = set()
for k, v in return_dict.items():
# print ( "collecting from thread %s" % str(k) )
for mp in v:
key = (mp[0], mp[1])
# print ( "key", key )
if key in hasStored:
continue
hasStored.add(key)
masspoints.append(mp)
# for m in masspoints:
# print ( "mass point", m[0], ",", m[1], ": nres", len(m[2]) )
return masspoints
def scanLikelihoodFor(self, min1, max1, dm1, min2, max2, dm2, nevents,
topo, output):
""" plot the likelihoods as a function of pid1 and pid2
:param output: prefix for output file [mp]
"""
self.nevents = nevents
pid1 = self.pid1
pid2 = self.pid2
if pid2 != self.M.LSP:
print(
"[llhdscanner] we currently assume pid2 to be the LSP, but it is %d"
% pid2)
import numpy
c = Combiner()
anaIds = c.getAnaIdsWithPids(self.M.bestCombo, [pid1, pid2])
## mass range for pid1
self.mpid1 = self.M.masses[pid1]
self.mpid2 = self.M.masses[pid2]
rpid1 = numpy.arange(min1, max1 + 1e-8, dm1)
rpid2 = numpy.arange(min2, max2 + 1e-8, dm2)
print("[llhdscanner] range for %d: %s" %
(pid1, self.describeRange(rpid1)))
print("[llhdscanner] range for %d: %s" %
(pid2, self.describeRange(rpid2)))
print("[llhdscanner] total %d points, %d events for %s" %
(len(rpid1) * len(rpid2), nevents, topo))
self.M.createNewSLHAFileName(prefix="llhd%d" % pid1)
#self.M.initializePredictor()
self.predictor.filterForTopos(topo)
self.M.walkerid = 0
thread0 = LlhdThread ( 0, self.rundir, self.M, self.pid1, self.pid2, \
self.mpid1, self.mpid2, self.nevents, self.predictor )
llhds, robs = thread0.getPredictions(False)
thread0.clean()
self.pprint ( "protomodel point: m1 %d, m2 %d, %d llhds" % \
( self.mpid1, self.mpid2, len(llhds) ) )
masspoints = [(self.mpid1, self.mpid2, llhds, robs)]
if True:
## freeze out all other particles
for pid_, m_ in self.M.masses.items():
if pid_ not in [self.pid1, self.pid2]:
self.M.masses[pid_] = 1e6
newpoints = self.getMassPoints(rpid1, rpid2)
masspoints += newpoints
import pickle
picklefile = "%s%d%d.pcl" % (output, pid1, pid2)
if os.path.exists(picklefile):
subprocess.getoutput("cp %s %s.old" % (picklefile, picklefile))
self.pprint("now saving to %s" % picklefile)
f = open(picklefile, "wb")
pickle.dump(masspoints, f)
pickle.dump(self.mpid1, f)
pickle.dump(self.mpid2, f)
pickle.dump(nevents, f)
pickle.dump(topo, f)
pickle.dump(time.asctime(), f)
f.close()
self.M.delCurrentSLHA()
def overrideWithDefaults(self, args):
mins = { 1000005: 100., 1000006: 100., 2000006: 100., 1000021: 300., \
1000001: 250., 1000002: 250., 1000003: 250., 1000004: 250. }
maxs = { 1000005: 1500., 1000006: 1460., 2000006: 1260., 1000021: 2351., \
1000001: 2051., 1000002: 2051., 1000003: 2051., 1000004: 2051. }
#dm = { 1000005: 16., 1000006: 16., 2000006: 16., 1000021: 20., \
# 1000001: 20., 1000002: 20., 1000003: 20., 1000004: 20. }
dm = { 1000005: 10., 1000006: 10., 2000006: 10., 1000021: 15., \
1000001: 12., 1000002: 12., 1000003: 12., 1000004: 12. }
topo = { 1000005: "T2bb",1000006: "T2tt", 2000006: "T2tt", 1000021: "T1", \
1000001: "T2", 1000002: "T2", 1000003: "T2", 1000004: "T2" }
### make the LSP scan depend on the mother
LSPmins = { 1000005: 5., 1000006: 5., 2000006: 5., 1000021: 5., \
1000001: 5., 1000002: 5., 1000003: 5., 1000004: 5. }
LSPmaxs = { 1000005: 800., 1000006: 900., 2000006: 800., 1000021: 1800., \
1000001: 1700., 1000002: 1700., 1000003: 1700., 1000004: 1700. }
#LSPdm = { 1000005: 12., 1000006: 14., 2000006: 14., 1000021: 14., \
# 1000001: 12., 1000002: 12., 1000003: 12., 1000004: 12. }
LSPdm = { 1000005: 10., 1000006: 10., 2000006: 10., 1000021: 15., \
1000001: 10., 1000002: 10., 1000003: 10., 1000004: 10. }
if not args.pid1 in mins:
print(
"[llhdscanner] asked for defaults for %d, but none defined." %
args.pid1)
return args
if args.min1 == None:
args.min1 = mins[args.pid1]
if args.max1 == None:
args.max1 = maxs[args.pid1]
if args.deltam1 == None:
args.deltam1 = dm[args.pid1]
if args.min2 == None:
args.min2 = LSPmins[args.pid1]
if args.max2 == None:
args.max2 = LSPmaxs[args.pid1]
if args.deltam2 == None:
args.deltam2 = LSPdm[args.pid1]
if args.topo == None:
args.topo = topo[args.pid1]
return args
def produceSSMs( hi, pid1, pid2, nevents = 100000, dry_run=False,
nproc=5, fac = 1.008, rundir= "" ):
""" produce pickle files for ssm scan, for (pid1,pid2), with nevents
:param hi: hiscore list object
:param nproc: number of processes
:param fac: factor with which to multiply interval
"""
if fac == None:
fac = 1.008
print ( "produceSSMs", pid1, pid2 )
model = hi.hiscores[0]
pids = ( pid1, pid2 )
if pid2 < pid1:
pids = ( pid2, pid1 )
if not pids in model.ssmultipliers:
print ( "[scanner] could not find pids %s in multipliers" % ( str(pids) ) )
print ( "only", model.ssmultipliers )
return
ssm = model.ssmultipliers[pids]
# print ( "[scanner] starting with %s: %.2f" % ( pids, ssm ) )
Zs = {}
fm = .6 ## lower bound (relative) on mass
# mrange = numpy.arange ( ssm * fm, ssm / fm, .01*ssm )
ssmrangetot = [ ssm ]
ssm1,ssm2 = ssm, ssm
dssm = fac
while ssm1 > fm * ssm:
ssm1 = ssm/dssm
ssm2 = ssm*dssm
ssmrangetot.append( ssm1 )
ssmrangetot.append( ssm2 )
dssm = dssm * fac
ssmrangetot.sort()
if nproc > len(ssmrangetot):
nproc = len(ssmrangetot)
ssmranges = [ ssmrangetot[i::nproc] for i in range(nproc) ]
print ( "[scanner] start scanning with ssm(%d,%d)=%.2f with %d procs, %d ssm points, %d events" % \
( pid1, pid2, ssm, nproc, len(ssmrangetot), nevents ) )
import multiprocessing
pool = multiprocessing.Pool ( processes = len(ssmranges) )
expected = False
select = "all"
dbpath = rundir + "/default.pcl"
predictor = Predictor( 0, dbpath=dbpath,
expected=expected, select=select )
args = [ { "model": model, "pids": pids, "nevents": nevents, "ssm": ssm,
"predictor": predictor, "rundir": rundir, "dry_run": dry_run,
"i": i, "ssmrange": x } for i,x in enumerate(ssmranges) ]
Zs={}
tmp = pool.map ( ssmProcess, args )
for r in tmp:
Zs.update(r)
if dry_run:
return
import pickle
filename = "ssm%d%d.pcl" % (pids[0],pids[1])
if os.path.exists ( filename ):
subprocess.getoutput ( "cp %s %sold" % ( filename, filename ) )
with open ( filename, "wb" ) as f:
pickle.dump ( Zs, f )
pickle.dump ( ssm, f )
pickle.dump ( nevents, f )
pickle.dump ( time.asctime(), f )
f.close()
def produce( hi, pid=1000022, nevents = 100000, dry_run=False,
nproc=5, fac = 1.008, rundir = "", preserve_xsecs = False ):
""" produce pickle files of mass scans for pid, with nevents
:param hi: hiscore list object
:param nproc: number of processes
:param fac: factor with which to multiply interval
:param preserve_xsecs: adjust the SSMs to that xsecs are preserved
"""
if type(pid) in [ list, tuple, set ]:
pid = set(map(abs,pid))
for p in pid:
produce ( hi, p, nevents, dry_run, nproc, fac, rundir = rundir,
preserve_xsecs = preserve_xsecs )
return
pid = abs(pid)
model = hi.hiscores[0]
if fac == None:
fac = 1.008
if model.masses[pid]<150.:
fac = 1.007
if model.masses[pid]<80.:
fac = 1.006
if preserve_xsecs and not hasattr ( model, "stored_xsecs" ):
print ( "[scanner] preserve_xsec mode, so computing the xsecs now" )
model.computeXSecs( keep_slha = True )
if model == None:
print ( "[scanner] cannot find a model in %s" % hi.pickleFile )
apid = abs(pid)
mass = model.masses[pid]
if mass > 9e5:
print ( "mass %d too high. Wont produce." % mass )
return
# model.createNewSLHAFileName ( prefix = "scan%s" % pid )
Zs = {}
fm = .6 ## lower bound (relative) on mass
# mrange = numpy.arange ( mass * fm, mass / fm, .008*mass )
mrangetot = [ mass ]
m1,m2 = mass, mass
dm = fac
while m1 > fm * mass:
m1 = mass/dm
m2 = mass*dm
dm = dm * fac
mrangetot.append( m1 )
if pid in [ 1000006, 2000006 ] and m2 > 1550.:
continue ## there is nothing to see beyond
mrangetot.append( m2 )
mrangetot.sort()
mranges = [ mrangetot[i::nproc] for i in range(nproc) ]
print ( "[scanner] start scanning with m(%d)=%.1f with %d procs, %d mass points, %d events" % \
( pid, mass, nproc, len(mrangetot), nevents ) )
expected = False
select = "all"
dbpath = rundir + "/default.pcl"
predictor = Predictor( 0, dbpath=dbpath,
expected=expected, select=select )
import multiprocessing
pool = multiprocessing.Pool ( processes = len(mranges) )
args = [ { "model": model, "rundir": rundir, "pid": pid, "nevents": nevents, "predictor": predictor, "preserve_xsecs": preserve_xsecs, "dry_run": dry_run,
"i": i, "mrange": x } for i,x in enumerate(mranges) ]
Zs={}
tmp = pool.map ( predProcess, args )
# model.delCurrentSLHA()
for r in tmp:
Zs.update(r)
if dry_run:
return
import pickle
with open ( "scanM%s.pcl" % pid, "wb" ) as f:
pickle.dump ( Zs, f )
pickle.dump ( mass, f )
pickle.dump ( nevents, f )
pickle.dump ( time.asctime(), f )
pickle.dump ( preserve_xsecs, f )
f.close()
args = argparser.parse_args()
drawtimestamp = not args.notimestamp
rundir = setup( args.rundir )
nproc = args.nproc
if nproc < 1:
nproc = nCPUs() + nproc
allpids = findPids( rundir )
pids = args.pid
if pids == 0:
pids = allpids
if args.produce:
hi = getHiscore( args.force_copy, rundir )
if args.pid2 > 0:
produceSSMs( hi, args.pid, args.pid2, args.nevents, args.dry_run, nproc, args.factor, rundir = rundir )
else:
produce( hi, pids, args.nevents, args.dry_run, nproc, args.factor, rundir = rundir, preserve_xsecs = args.preserve_xsecs )
pred = Predictor( 0 )
rthreshold = pred.rthreshold
if args.draw:
if args.pid != 0:
draw( pids, args.interactive, args.pid2, args.copy, drawtimestamp, rundir, \
plotrmax = False, rthreshold = rthreshold, upload = args.upload )
else:
for pid in allpids:
try:
draw( pid, args.interactive, args.pid2, args.copy, drawtimestamp,
rundir, plotrmax = False, rthreshold = rthreshold,
upload = args.upload )
except Exception as e:
print ( "[scanner] skipping %d: %s" % ( pid, e ) )
def main ( args ):
""" the function that updates the hiscore.hi file
:param args: detailed, outfile, infile, print, fetch, nmax,
check, interactive, nevents.
see "if __main__" part below.
:returns: { "Z": highest significance,
"step": step, "model": model, "K": bayesian_K }
"""
ret = { "Z": 0., "step": 0, "model": None, "K": -100. }
if args.detailed:
args.print = True
if args.outfile.lower() in [ "none", "", "false" ]:
args.outfile = None
infile = args.infile
if type(infile) is str and infile.lower() in [ "none", "" ]:
infile = None
trundir = None
if hasattr ( args, "rundir" ):
trundir = args.rundir
rundir = setup( trundir )
if infile == "default":
infile = "%s/hiscore.hi" % rundir
if args.outfile == infile:
print ( "[hiscore] outputfile is same as input file. will assume that you do not want me to write out at all." )
args.outfile = None
if args.fetch:
import subprocess
cmd = "scp gpu:/local/wwaltenberger/git/smodels-utils/prototools/H*.hi ."
print ( "[hiscore] %s" % cmd )
out = subprocess.getoutput ( cmd )
print ( out )
if infile is None:
print ( "[hiscore] compiling a hiscore list with %d protomodels" % args.nmax )
protomodels = compileList( args.nmax ) ## compile list from H<n>.hi files
else:
with open(infile,"rb") as f:
try:
protomodels = pickle.load ( f )
try:
pickle.load ( f )
except EOFError:
pass
f.close()
except (BlockingIOError,OSError) as e:
print ( "file handling error on %s: %s" % ( infile, e ) )
## make sure we dont block!
raise e
if protomodels[0] == None:
print ( "[hiscore] error, we have an empty hiscore list" )
return ret
sin = infile
if sin == None:
sin = "H*.hi"
pevs = pprintEvs ( protomodels[0] )
print ( "[hiscore] hiscore from %s[%d] is at K=%.3f, Z=%.3f (%s)" % \
( sin, protomodels[0].walkerid, protomodels[0].K, protomodels[0].Z, pevs ) )
# nevents = args.nevents
if args.nmax > 0:
protomodels = protomodels[:args.nmax]
# print ( "we are here", args.outfile, hasattr ( protomodels[0], "analysisContributions" ) )
if type(args.outfile)==str and (".pcl" in args.outfile or ".hi" in args.outfile ):
if not hasattr ( protomodels[0], "analysisContributions" ):
print ( "[hiscore] why does the winner not have analysis contributions?" )
ma = Manipulator ( protomodels[0] )
from walker.hiscore import Hiscore
hi = Hiscore( 0, True, f"{rundir}/hiscore.hi" )
hi.computeAnalysisContributions(ma)
protomodels[0]=ma.M
hi.save()
if not hasattr ( protomodels[0], "particleContributions" ):
print ( "[hiscore] why does the winner not have particle contributions?" )
ma = Manipulator ( protomodels[0] )
from walker.hiscore import Hiscore
from tester.predictor import Predictor
predictor = None
dbpath = args.dbpath
if not "/" in dbpath:
dbpath = f"{rundir}/{args.dbpath}"
if hasattr ( args, "dbpath" ):
dbpath = args.dbpath
if os.path.exists ( dbpath ):
predictor = Predictor ( 0, dbpath = dbpath,
expected = False, select= "all" )
hi = Hiscore( 0, True, f"{rundir}/hiscore.hi", predictor = predictor )
hi.computeParticleContributions(ma)
protomodels[0]=ma.M
hi.save()
if args.outfile is not None:
storeList ( protomodels, args.outfile )
if args.check:
protomodel = protomodels[0]
protomodel.predict()
print ( "[hiscore] args.check, implement" )
if args.print:
printProtoModels ( protomodels, args.detailed, min ( 10, args.nmax ) )
if len(protomodels)>0 and protomodels[0] != None:
ret["Z"]=protomodels[0].Z
ret["K"]=protomodels[0].K
ret["step"]=protomodels[0].step
ret["model"]=protomodels[0]
return ret
return ret
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] Modules: %smanipulator, hiscore, combiner, predictor, helpers%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] Classes: %sProtoModel, Combiner, Predictor, Hiscore, Database%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] Instantiations: %sma, co, hi, pr%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
from tester import combiner
from walker import hiscore
from tester import predictor
from tester.combiner import Combiner
from tester.predictor import Predictor
from builder.protomodel import ProtoModel
from smodels.tools.physicsUnits import pb, fb, GeV, TeV
from smodels.experiment.databaseObj import Database
import copy, numpy, scipy, scipy.stats, math
co = Combiner() # instantiate for convenience
pr = Predictor( 0 ) # instantiate for convenience
from ptools import helpers
# import hiscore #Keep it for convenience
if args.execute not in [ "", None ]:
if os.path.exists ( args.execute ):
with open ( args.execute, "rt" ) as f:
exec ( f.read() )
if not args.nointeractive:
import IPython
IPython.embed( using=False )
ma.M.delCurrentSLHA()