def plotDecays(protomodel, verbosity, outfile="decays.png"):
print("[plotHiscore] now draw %s" % outfile)
options = {
"tex": True,
"color": True,
"dot": True,
"squarks": True,
"weakinos": True,
"sleptons": True,
"neato": True,
"separatecharm": True,
"integratesquarks": False,
"leptons": True
}
options["rmin"] = 0.005
## FIXME add cross sections.
if verbosity == "debug":
soptions = ""
for k, v in options.items():
if v == True:
soptions += "--%s " % k
ma = Manipulator(protomodel)
ssms = ma.simplifySSMs()
# soptions+=' --ssmultipliers "%s"' % ssms
print ( "%s../smodels_utils/plotting/decayPlotter.py -f %s -o %s %s%s" % \
( colorama.Fore.GREEN, protomodel.currentSLHA, outfile, soptions,
colorama.Fore.RESET ) )
decayPlotter.draw(protomodel.currentSLHA,
outfile,
options,
verbosity=verbosity,
ssmultipliers=protomodel.ssmultipliers)
def getUnfrozenSSMs(protomodel, frozen, includeOnes=False, dropLSPLSP=True):
""" of all SSMs, leave out the ones with frozen particles
:param protomodel: the, well, protomodel
:param frozen: list of pids of frozen particles
:param includeOnes: if False, then also filter out values close to unity
:param dropLSPLSP: if True, dont include LSP,LSP production
:returns: dictionary of SSMs without frozen particles
"""
# ssms = protomodel.ssmultipliers
ma = Manipulator(protomodel)
ssms = ma.simplifySSMs(threshold=.01 * fb)
D = {}
for pids, v in ssms.items():
hasFrozenParticle = False
if dropLSPLSP:
if pids == (ProtoModel.LSP, ProtoModel.LSP):
continue
for pid in pids:
if pid in frozen or -pid in frozen:
hasFrozenParticle = True
if hasFrozenParticle:
continue
if not includeOnes and abs(v - 1.) < 1e-2:
continue
D[pids] = v
return D
def main( rundir = None, maxruns=3, doPlots=True, uploadTo="latest" ):
""" eternal loop that updates hiscore.hi and states.dict
:param maxruns: maximally iterate that many times
:param doPlots: if False, suppress plotting
:param uploadTo: upload plots to directory "~/git/smodels.github.io/<uploadTo>"
"""
rundir = setup( rundir )
i = 0
Z, Zold, step, K, Kold = 0., 0., 0, -90., -90.
Zfile = "%s/Zold.conf" % rundir
if os.path.exists ( Zfile ):
with open ( Zfile, "rt" ) as f:
Zold = float ( f.read().strip() )
Kfile = "%s/Kold.conf" % rundir
if os.path.exists ( Kfile ):
with open ( Kfile, "rt" ) as f:
Kold = float ( f.read().strip() )
while True:
i+=1
if maxruns != None and i > maxruns:
break
D = updateHiscores( rundir )
Z,step,model,K = D["Z"],D["step"],D["model"],D["K"]
if K > Kold + .001:
from builder.manipulator import Manipulator
m = Manipulator ( model )
T=str(int(time.time()))
m.writeDictFile ( "pmodel-%s.py" % T, comment="history keeper" )
with open ( "%shistory.txt" % rundir, "at" ) as f:
f.write ( "%s, step=%d, Z=%.4f, K=%.4f, t=%s\n" % ( time.asctime(),step,Z,K,T) )
f.close()
with open ( Zfile, "wt" ) as f:
f.write ( "%s\n" % str(Z) )
f.close()
with open ( Kfile, "wt" ) as f:
f.write ( "%s\n" % str(K) )
f.close()
Zold = Z
Kold = K
if doPlots:
plot ( Z, K, rundir, uploadTo )
updateStates( rundir )
time.sleep(60.)
if os.path.exists ( Kfile ): ## so we can meddle from outside
with open ( Kfile, "rt" ) as f:
Kold = float ( f.read().strip() )
def writeListToDictFile ( self, dictFile=None ):
""" write the models in append mode in a single dictFile.
:param dictFile: write to dictFile. If None, then self.pickleFile
is used, but with ".dict" as extension.
"""
if dictFile==None:
dictFile = self.pickleFile
if dictFile.endswith(".pcl"):
dictFile = dictFile[:-4]+".py"
f=open(dictFile,"wt")
f.write("[")
f.close()
for protomodel in self.hiscores:
ma = Manipulator ( protomodel )
ma.writeDictFile ( outfile = dictFile, appendMode=True )
f=open(dictFile,"at")
f.write("]\n")
f.close()
def testRandomModel(self):
helpers.seedRandomNumbers(456)
with open('randomModels_default.pcl', 'rb') as f:
pList = pickle.load(
f) #List with original models and modified ones
pNew = copy.deepcopy(
pList[0]
) #Obs: Can not use ProtoModel.copy(), since it calls random
pNew.templateSLHA = os.path.abspath(
'../builder/templates/template1g.slha')
m = Manipulator(pNew)
for p in pList:
self.assertEqual(pNew.masses, p.masses)
self.assertEqual(pNew.decays, p.decays)
self.assertEqual(pNew.ssmultipliers, p.ssmultipliers)
self.assertEqual(pNew._stored_xsecs, p._stored_xsecs)
m.randomlyChangeModel()
def ssmProcess ( args ):
""" one thread that computes predictions for ssms given in ssmrange
"""
i = args["i"]
import time
# time.sleep(5*i) ## would that help??
print ( "[scanner:%d] starting thread" % ( i ) )
model = args["model"]
pids = args["pids"]
predictor = args["predictor"]
nevents = args["nevents"]
ssmrange = args["ssmrange"]
ssm = args["ssm"]
model.walkerid = 200000+10000*i + model.walkerid
model.createNewSLHAFileName ( prefix = "ssm%dp%d%d%.2f" % ( i, pids[0], pids[1], ssm ) )
if not pids in model.ssmultipliers:
print ( "[scanner:%d] error cannot find pids %s" % (i, str(pids) ) )
return
ret = {}
ts = time.strftime("%H:%M:%S" )
model.delXSecs()
# model.predict ( nevents = nevents, recycle_xsecs = True )
predictor.predict ( model )
print ( "[scanner:%d-%s] before we begin, Z is %.3f" % ( i, ts, model.Z ) )
for ctr,ssm in enumerate(ssmrange):
ssmold = model.ssmultipliers[pids]
print ( "[scanner:%d] we change the ssm from %.3f to %.3f" % \
( i, ssmold, ssm ) )
ma = Manipulator ( model )
ma.changeSSM ( pids, ssm )
model = ma.M
ts = time.strftime("%H:%M:%S" )
print ( "[scanner:%d-%s] start with %d/%d, ssm=%.2f (%d events)" % \
( i, ts, ctr, len(ssmrange), ssm, nevents ) )
# model.predict ( nevents = nevents, recycle_xsecs = True )
predictor.predict ( model ) # #nevents = nevents, recycle_xsecs = True )
print ( "[scanner:%d-%s] `- Z=%.3f" % ( i, ts, model.Z ) )
ret[model.muhat*ssm]=(model.Z,model.rvalues[0],model.K,model.muhat)
return ret
def testRandomUnfreeze(self):
helpers.seedRandomNumbers(456)
with open('randomModels_default.pcl', 'rb') as f:
model = pickle.load(f)[0]
m = Manipulator(model)
m.randomlyUnfreezeParticle(sigma=0.5)
self.assertEqual(sorted(model.unFrozenParticles()),
[1000001, 1000021, 1000022])
masses = {1000022: 215.71, 1000001: 941.23, 1000021: 1068.91}
for pid, mass in masses.items():
self.assertAlmostEqual(model.masses[pid], mass, places=1)
decays = {
1000022: {},
1000001: {
1000022: 1.0
},
1000021: {
(1000022, 6): 0.0,
(1000022, 5): 0.5035,
(1000022, 21): 0.248,
(1000022, 1): 0.202,
(1000001, 1): 0.0,
(1000022, 4): 0.0456
}
}
for pid, decay in decays.items():
for dpid, br in decay.items():
self.assertAlmostEqual(model.decays[pid][dpid], br, places=1)
m.randomlyUnfreezeParticle(force=True)
self.assertEqual(sorted(model.unFrozenParticles()),
[1000001, 1000021, 1000022, 1000024])
def plot(number, verbosity, picklefile, options, dbpath):
## plot hiscore number "number"
protomodel = obtain(number, picklefile)
protoslha = protomodel.createSLHAFile()
subprocess.getoutput("cp %s hiscore.slha" % protoslha)
m = Manipulator(protomodel)
print("[plotHiscore] now write pmodel.py")
m.writeDictFile()
opts = ["ruler", "decays", "predictions", "copy", "html"]
for i in opts:
if not i in options:
options[i] = True
plotruler = options["ruler"]
horizontal = False
if "horizontal" in options and options["horizontal"]:
horizontal = True
# print ( "[plotHiscore] slha file exists?", os.path.exists ( protomodel.currentSLHA ), protomodel.currentSLHA )
if plotruler:
plotRuler(protomodel, verbosity, horizontal)
plotdecays = options["decays"]
if plotdecays:
plotDecays(protomodel, verbosity)
if options["predictions"]:
discussPredictions(protomodel)
if options["html"] or options["tex"]:
texdoc = writeTex(protomodel, options["keep_tex"])
if options["html"]:
writeIndexHtml(protomodel)
if options["tex"]:
writeIndexTex(protomodel, texdoc)
writeRawNumbersLatex(protomodel)
writeRawNumbersHtml(protomodel)
protomodel.delCurrentSLHA()
def clean ( self ):
""" clean hiscore list, i.e. remove cruft from protomodels.
leave first one as it is """
for ctr,h in enumerate(self.hiscores[1:]):
if h != None:
m=Manipulator ( h )
m.rescaleSignalBy(m.M.muhat)
m.delBackup ( )
m.M.cleanBestCombo ()
self.hiscores[ctr+1]=m.M
def testUnfreeze(self):
helpers.seedRandomNumbers(123)
with open('randomModels_default.pcl', 'rb') as f:
modelList = pickle.load(f)
m = Manipulator(modelList[4])
r = m.unFreezeParticle(
pid=2000006) #Should not work since 1000006 is frozen!
self.assertEqual(r, 0)
self.assertEqual(sorted(m.M.unFrozenParticles()), [1000022, 1000024])
r = m.unFreezeParticle(
pid=1000006) #Should not work since 1000006 is frozen!
r = m.unFreezeParticle(
pid=2000006) #Now it should work since 1000006 is unfrozen!
self.assertEqual(r, 1)
self.assertEqual(sorted(m.M.unFrozenParticles()),
[1000006, 1000022, 1000024, 2000006])
self.assertTrue(
m.M.masses[2000006] >
m.M.masses[1000006]) #stop2 mass should be higher than stop1
minMass = min([mass for mass in m.M.masses.values()])
self.assertTrue(minMass == m.M.masses[
m.M.LSP]) #all masses should smaller than the LSP mass
#Check if the only open decays are created:
m = Manipulator(modelList[1])
r = m.unFreezeParticle(pid=1000021, force=True)
self.assertEqual(r, 1)
self.assertEqual(sorted(m.M.unFrozenParticles()),
[1000001, 1000021, 1000022])
self.assertEqual(sorted(m.M.decays.keys()),
[1000001, 1000021, 1000022])
self.assertEqual(sorted(m.M.decays[1000021].keys()), [(1000001, 1),
(1000022, 1),
(1000022, 4),
(1000022, 5),
(1000022, 6),
(1000022, 21)])
for pid in m.M.decays[1000021]:
self.assertTrue(m.M.masses[1000021] > m.M.masses[max(pid)])
def __init__(self,
walkerid=0,
nsteps=10000,
strategy="aggressive",
dump_training=False,
cheatcode=0,
dbpath="./database.pcl",
expected=False,
select="all",
catch_exceptions=True,
rundir=None,
nevents=100000,
do_combine=False,
record_history=False,
seed=None,
stopTeleportationAfter=-1):
""" initialise the walker
:param nsteps: maximum number of steps to perform, negative is infinity
:param cheatcode: cheat mode. 0 is no cheating, 1 is with ranges, 2
is the Z323 model.
:param expected: remove possible signals from database
:param select: select only subset of results (all for all, em for efficiency maps only, ul for upper limits only, alternatively select for txnames via e.g. "txnames:T1,T2"
:param catch_exceptions: should we catch exceptions
:param nevents: number of MC events when computing cross-sections
:param do_combine: if true, then also perform combinations, either via
simplified likelihoods or via pyhf
:param record_history: if true, attach a history recorder class
:param seed: random seed, int or None
:param stopTeleportationAfter: int or None. we stop teleportation after this step nr.
If negative or None, we dont teleport at all
"""
if type(walkerid) != int or type(nsteps) != int or type(
strategy) != str:
self.pprint("Wrong call of constructor: %s, %s, %s" %
(walkerid, nsteps, strategy))
sys.exit(-2)
self.walkerid = walkerid ## walker id, for parallel runs
self.rundir = rundir
if rundir == None:
self.rundir = "./"
if seed is not None:
from ptools import helpers
helpers.seedRandomNumbers(seed + walkerid)
self.pprint(f"setting random seed to {seed}")
#Initialize Predictor
self.predictor = Predictor(self.walkerid,
dbpath=dbpath,
expected=expected,
select=select,
do_combine=do_combine)
#Initialize Hiscore (with access to the predictor)
self.hiscoreList = Hiscore(walkerid,
True,
"%s/H%d.hi" % (self.rundir, walkerid),
backup=False,
predictor=self.predictor)
self.hiscoreList.nkeep = 1
#Initialize ProtoModel and Manipulator:
protomodel = ProtoModel(
self.walkerid,
keep_meta=True,
nevents=nevents,
dbversion=self.predictor.database.databaseVersion)
self.manipulator = Manipulator(protomodel,
strategy,
do_record=record_history,
seed=seed)
self.catch_exceptions = catch_exceptions
self.maxsteps = nsteps
if stopTeleportationAfter == None:
stopTeleportationAfter = -1
# stopTeleportationAfter = self.maxsteps/3.
self.stopTeleportationAfter = stopTeleportationAfter
self.accelerator = None
if record_history:
from ptools.history import History
self.recorder = History(f"{self.rundir}/history{walkerid}.list")
self.manipulator.do_record = True
jobid = "unknown"
if "SLURM_JOBID" in os.environ:
jobid = os.environ["SLURM_JOBID"]
self.pprint("Ramping up with slurm jobid %s" % jobid)
if cheatcode <= 0:
self.takeStep() # the first step should be considered as "taken"
#Set current Z and K values to threshold values
self.currentZ = -0.1
self.currentK = -20.0
else:
self.manipulator.cheat(cheatcode)
self.predictor.predict(self.protomodel)
self.pprint ( "Cheat model gets Z=%.2f, K=%.2f" % \
( self.manipulator.M.Z, self.manipulator.M.K ) )
# self.printStats ( substep=4 )
self.manipulator.backupModel()
self.hiscoreList.newResult(self.manipulator)
self.printStats(substep=5)
self.currentK = self.manipulator.M.K
self.currentZ = self.manipulator.M.Z
if dump_training:
from accelerator import Accelerator
## we use the accelerator only to dump the training data
self.accelerator = Accelerator(walkerid=walkerid,
dump_training=True,
is_trained=False)
class RandomWalker:
def __init__(self,
walkerid=0,
nsteps=10000,
strategy="aggressive",
dump_training=False,
cheatcode=0,
dbpath="./database.pcl",
expected=False,
select="all",
catch_exceptions=True,
rundir=None,
nevents=100000,
do_combine=False,
record_history=False,
seed=None,
stopTeleportationAfter=-1):
""" initialise the walker
:param nsteps: maximum number of steps to perform, negative is infinity
:param cheatcode: cheat mode. 0 is no cheating, 1 is with ranges, 2
is the Z323 model.
:param expected: remove possible signals from database
:param select: select only subset of results (all for all, em for efficiency maps only, ul for upper limits only, alternatively select for txnames via e.g. "txnames:T1,T2"
:param catch_exceptions: should we catch exceptions
:param nevents: number of MC events when computing cross-sections
:param do_combine: if true, then also perform combinations, either via
simplified likelihoods or via pyhf
:param record_history: if true, attach a history recorder class
:param seed: random seed, int or None
:param stopTeleportationAfter: int or None. we stop teleportation after this step nr.
If negative or None, we dont teleport at all
"""
if type(walkerid) != int or type(nsteps) != int or type(
strategy) != str:
self.pprint("Wrong call of constructor: %s, %s, %s" %
(walkerid, nsteps, strategy))
sys.exit(-2)
self.walkerid = walkerid ## walker id, for parallel runs
self.rundir = rundir
if rundir == None:
self.rundir = "./"
if seed is not None:
from ptools import helpers
helpers.seedRandomNumbers(seed + walkerid)
self.pprint(f"setting random seed to {seed}")
#Initialize Predictor
self.predictor = Predictor(self.walkerid,
dbpath=dbpath,
expected=expected,
select=select,
do_combine=do_combine)
#Initialize Hiscore (with access to the predictor)
self.hiscoreList = Hiscore(walkerid,
True,
"%s/H%d.hi" % (self.rundir, walkerid),
backup=False,
predictor=self.predictor)
self.hiscoreList.nkeep = 1
#Initialize ProtoModel and Manipulator:
protomodel = ProtoModel(
self.walkerid,
keep_meta=True,
nevents=nevents,
dbversion=self.predictor.database.databaseVersion)
self.manipulator = Manipulator(protomodel,
strategy,
do_record=record_history,
seed=seed)
self.catch_exceptions = catch_exceptions
self.maxsteps = nsteps
if stopTeleportationAfter == None:
stopTeleportationAfter = -1
# stopTeleportationAfter = self.maxsteps/3.
self.stopTeleportationAfter = stopTeleportationAfter
self.accelerator = None
if record_history:
from ptools.history import History
self.recorder = History(f"{self.rundir}/history{walkerid}.list")
self.manipulator.do_record = True
jobid = "unknown"
if "SLURM_JOBID" in os.environ:
jobid = os.environ["SLURM_JOBID"]
self.pprint("Ramping up with slurm jobid %s" % jobid)
if cheatcode <= 0:
self.takeStep() # the first step should be considered as "taken"
#Set current Z and K values to threshold values
self.currentZ = -0.1
self.currentK = -20.0
else:
self.manipulator.cheat(cheatcode)
self.predictor.predict(self.protomodel)
self.pprint ( "Cheat model gets Z=%.2f, K=%.2f" % \
( self.manipulator.M.Z, self.manipulator.M.K ) )
# self.printStats ( substep=4 )
self.manipulator.backupModel()
self.hiscoreList.newResult(self.manipulator)
self.printStats(substep=5)
self.currentK = self.manipulator.M.K
self.currentZ = self.manipulator.M.Z
if dump_training:
from accelerator import Accelerator
## we use the accelerator only to dump the training data
self.accelerator = Accelerator(walkerid=walkerid,
dump_training=True,
is_trained=False)
def hostname(self):
return socket.gethostname()
def setWalkerId(self, Id):
self.walkerid = Id
self.manipulator.setWalkerId(Id)
if self.accelerator != None:
self.accelerator.walkerid = Id
@classmethod
def fromProtoModel(cls,
protomodel,
nsteps=10000,
strategy="aggressive",
walkerid=0,
dump_training=False,
dbpath="<rundir>/database.pcl",
expected=False,
select="all",
catch_exceptions=True,
keep_meta=True,
rundir=None,
do_combine=False,
seed=None,
stopTeleportationAfter=-1):
ret = cls( walkerid, nsteps=nsteps, dbpath = dbpath, expected=expected,
select=select, catch_exceptions = catch_exceptions, rundir = rundir,
do_combine = do_combine, seed = seed, stopTeleportationAfter = \
stopTeleportationAfter )
ret.manipulator.M = protomodel
ret.manipulator.setWalkerId(walkerid)
ret.manipulator.backupModel()
if dump_training:
## we use the accelerator only to dump the training data
from accelerator import Accelerator
ret.accelerator = Accelerator(walkerid=walkerid,
dump_training=True,
is_trained=False)
return ret
@classmethod
def fromDictionary(cls,
dictionary,
nsteps=10000,
strategy="aggressive",
walkerid=0,
dump_training=False,
dbpath="<rundir>/database.pcl",
expected=False,
select="all",
catch_exceptions=True,
keep_meta=True,
rundir=None,
nevents=100000,
do_combine=False,
seed=None,
stopTeleportationAfter=-1):
ret = cls(walkerid,
nsteps=nsteps,
dbpath=dbpath,
expected=expected,
select=select,
catch_exceptions=catch_exceptions,
rundir=rundir,
nevents=nevents,
do_combine=do_combine,
seed=seed,
stopTeleportationAfter=stopTeleportationAfter)
ret.manipulator.M = ProtoModel( walkerid, keep_meta, \
dbversion = ret.predictor.database.databaseVersion )
ret.manipulator.initFromDict(dictionary)
ret.manipulator.setWalkerId(walkerid)
ret.manipulator.M.createNewSLHAFileName()
# ret.printStats ( substep=3 )
ret.manipulator.backupModel()
return ret
def pprint(self, *args):
""" logging """
if not hasattr(self, "walkerid"):
self.walkerid = -1
print("[walk:%d:%s-%s] %s" %
(self.walkerid, self.hostname(), time.strftime("%H:%M:%S"),
" ".join(map(str, args))))
self.log(*args)
@property
def protomodel(self):
return self.manipulator.M
@protomodel.setter
def protomodel(self, protomodel):
self.manipulator.M = protomodel
def printStats(self, substep=None):
""" print the stats, i.e. number of unfrozen particles.
for debugging. """
nUnfrozen = len(self.protomodel.unFrozenParticles())
nTotal = len(self.protomodel.particles)
pidsp = self.protomodel.unFrozenParticles()
pidsp.sort()
namer = SParticleNames(False)
prtcles = ", ".join(map(namer.asciiName, pidsp))
pidsbc = list(self.manipulator.getAllPidsOfBestCombo())
pidsbc.sort()
prtclesbc = ", ".join(map(namer.asciiName, pidsbc))
self.pprint ( "Step %d has %d/%d unfrozen particles: %s [in best combo: %s]" % \
( self.protomodel.step, nUnfrozen, nTotal, \
prtcles, prtclesbc ) )
if len(pidsbc) > 0 and not set(pidsbc).issubset(set(pidsp)):
self.pprint(
" `-- error! best combo pids arent subset of masses pids!!!")
self.manipulator.M.bestCombo = None
def onestep(self):
#Add one step
self.protomodel.step += 1
self.pprint("Step %d begins." % (self.protomodel.step))
self.printStats()
#Remove data about best combo
self.protomodel.cleanBestCombo()
# self.printStats( substep=11 )
printMemUsage = False
if printMemUsage:
self.pprint ( "memory footprint (kb): walker %d, model %d, accelerator %d" %\
( asizeof(self)/1024,asizeof(self.protomodel)/1024,asizeof(self.accelerator)/1024 ))
#Trim the model, so we start only with the relevant particles for the
#best combination in the previous step:
self.log("freeze pids that arent in best combo, we dont need them:")
nfrozen = self.manipulator.freezePidsNotInBestCombo()
self.log(" `- froze %d particles not in best combo" % nfrozen)
# self.printStats( substep=12 )
#Take a step in the model space:
self.manipulator.randomlyChangeModel()
# self.printStats( substep=13 )
nUnfrozen = len(self.protomodel.unFrozenParticles())
## number of pids in best combo, as a check
#Try to create a simpler model
#(merge pre-defined particles of their mass difference is below dm)
protomodelSimp = self.manipulator.simplifyModel(dm=200.0)
# self.printStats( substep=14 )
if self.catch_exceptions:
try:
self.predictor.predict(self.manipulator.M)
if protomodelSimp:
self.predictor.predict(protomodelSimp)
except Exception as e:
self.pprint(
"error ``%s'' (%s) encountered when trying to predict. lets revert and not count it as a step."
% (str(e), type(e)))
self.manipulator.restoreModel()
self.manipulator.M.step -= 1 # we dont count that step.
import traceback
traceback.print_exc()
return
else:
self.predictor.predict(self.manipulator.M)
if protomodelSimp:
self.predictor.predict(protomodelSimp)
#Now keep the model with highest score:
if protomodelSimp:
if self.manipulator.M.K is None or (
protomodelSimp.K is not None and
(protomodelSimp.K > self.manipulator.M.K)):
self.manipulator.M = protomodelSimp
#If no combination could be found, return
if self.manipulator.M.Z is None:
return
#the muhat multiplier gets multiplied into the signal strengths
self.manipulator.rescaleSignalBy(self.protomodel.muhat)
self.log ( "Top r values after rescaling are: %.2f, %.2f" % \
( self.manipulator.M.rvalues[0], self.manipulator.M.rvalues[1] ) )
self.log ( "Step %d: found highest Z: %.2f" % \
( self.protomodel.step, self.protomodel.Z ) )
nUnfrozen = len(self.protomodel.unFrozenParticles())
self.pprint ( "best combo for strategy ``%s'' is %s: %s: [K=%.2f, Z=%.2f, %d unfrozen]" % \
( self.manipulator.strategy, self.protomodel.letters, self.protomodel.description, self.protomodel.K, self.protomodel.Z, nUnfrozen ) )
smaxstp = "%s" % self.maxsteps
if self.maxsteps < 0:
smaxstp = "inf"
#For low scoring models, teleport to a high score model:
if self.checkIfToTeleport(pmax=0.5, norm=10.0):
# if we teleport the rest becomes irrelevant
return
self.printStats()
self.log ( "Step %d check if result goes into hiscore list" % \
( self.protomodel.step ) )
srs = "%s" % ", ".join(
["%.2f" % x for x in self.protomodel.rvalues[:3]])
self.log("r values before calling .newResult are at %s" % srs)
self.hiscoreList.newResult(self.manipulator) ## add to high score list
srs = "%s" % ", ".join(
["%.2f" % x for x in self.protomodel.rvalues[:3]])
self.log("r values after calling .newResult are at %s" % srs)
self.log("done check for result to go into hiscore list")
self.log("Step %d/%s finished." % (self.protomodel.step, smaxstp))
def checkIfToTeleport(self, pmax=0.1, norm=10.0):
""" check if we should teleport to a high score model. If yes, then we
should then also perform the teleportation. The teleportation is
done only if the model has a score smaller then the best score in
hiscoreList. The teleportation probability is given by
pmax*(1-exp^(K-bestK)/norm), so pmax is the maximum probability
(when K -> -infinity).
:param pmax: Maximum probability for teleportation.
:param norm: Normalization for K distance.
"""
if self.protomodel.step > self.stopTeleportationAfter:
self.log ( "teleportation is turned off after step #%d" % \
self.stopTeleportationAfter )
return False
#self.log ( "teleportation turned off" )
#return False
import random
bestK = self.hiscoreList.globalMaxK()
if bestK < 1.:
self.log("bestK is smaller than one. no teleporting.")
return False
ourK = -2.
if hasattr(self.manipulator.M, "K") and self.manipulator.M.K > -2:
ourK = self.manipulator.M.K
#The current model already is the best, do nothing.
if ourK >= bestK:
return False
#Otherwise compute the teleportation probability:
dK = (ourK - bestK) / norm
prob = pmax * (1. - math.exp(dK))
a = random.uniform(0., 1.)
doTP = (a < prob) ## do teleport, yes or no
sDoTP = "a>p: dont teleport."
if doTP:
sDoTP = "a<p: do teleport."
self.log ( "check if to teleport, Kmax=%.2f, ours is=%.2f, p=%.2f, a=%.2f, %s" % \
( bestK, ourK, prob, a, sDoTP ) )
if doTP:
self.manipulator.teleportToHiscore()
return doTP
def takeStep(self):
""" take the step, save it as last step """
## Backup model
self.manipulator.backupModel()
# Update current K and Z values
self.currentK = self.protomodel.K
self.currentZ = self.protomodel.Z
self.manipulator.record("take step")
def highlight(self, msgType="info", *args):
""" logging, hilit """
col = colorama.Fore.GREEN
print("%s[walk:%d] %s%s" % (col, self.walkerid, " ".join(map(
str, args)), colorama.Fore.RESET))
def decideOnTakingStep(self):
""" depending on the ratio of K values, decide on whether to take the step or not.
If ratio > 1., take the step, if < 1, let chance decide. """
import random
ratio = 1.
K = self.currentK
newK = self.protomodel.K
if K > -20. and newK < K:
ratio = numpy.exp(.5 * (newK - K))
if ratio >= 1.:
self.highlight ( "info", "K: %.3f -> %.3f: r=%.4f, take the step" % \
( self.currentK, self.protomodel.K, ratio ) )
if self.protomodel.K > 0. and self.protomodel.K < 0.7 * self.currentK:
self.pprint(" `- weirdly, though, K decreases. Please check.")
sys.exit(-2)
self.takeStep()
else:
u = random.uniform(0., 1.)
if u > ratio:
self.pprint("u=%.2f > %.2f; K: %.2f -> %.2f: revert." %
(u, ratio, self.currentK, self.protomodel.K))
self.manipulator.restoreModel(reportReversion=True)
if hasattr(self, "oldgrad") and self.accelerator != None:
self.accelerator.grad = self.oldgrad
else:
self.pprint(
"u=%.2f <= %.2f ; %.2f -> %.2f: take the step, even though old is better."
% (u, ratio, self.currentK, self.protomodel.Z))
self.takeStep()
def log(self, *args):
""" logging to file """
with open("%s/walker%d.log" % (self.rundir, self.walkerid), "a") as f:
f.write("[randomWalker-%s] %s\n" %
(time.strftime("%H:%M:%S"), " ".join(map(str, args))))
def record(self):
""" if recorder is defined, then record. """
## do we have a history recorder?
if not hasattr(self, "recorder"):
return
self.recorder.add(self.manipulator)
def walk(self, catchem=False):
""" Now perform the random walk """
# self.printStats ( substep = 2 )
self.manipulator.backupModel()
if len(self.manipulator.M.unFrozenParticles(withLSP=False)) < 1:
## start with unfreezing a random particle
self.manipulator.randomlyUnfreezeParticle(force=True)
self.manipulator.backupModel()
while self.maxsteps < 0 or self.protomodel.step < self.maxsteps:
if not catchem:
self.onestep()
else:
try:
self.onestep()
except Exception as e:
# https://bioinfoexpert.com/2016/01/18/tracing-exceptions-in-multiprocessing-in-python/
self.pprint ( "taking a step resulted in exception: %s, %s" % \
(type(e), e ) )
import traceback
traceback.print_stack(limit=None)
except_type, except_class, tb = sys.exc_info()
extracted = traceback.extract_tb(tb)
for point in extracted:
self.pprint("extracted: %s" % point)
with open("%s/exceptions.log" % self.rundir, "a") as f:
f.write ( "%s: taking a step resulted in exception: %s, %s\n" % \
(time.asctime(), type(e), e ) )
f.write ( " `- exception occured in walker #%s\n" % \
self.protomodel.walkerid )
sys.exit(-1)
#If no combination was found, go back
if self.protomodel.K is None:
self.manipulator.restoreModel()
continue
# obtain the ratio of posteriors
self.decideOnTakingStep()
self.record()
self.manipulator.M.delCurrentSLHA()
self.pprint("Was asked to stop after %d steps" % self.maxsteps)
#!/usr/bin/env python3
""" restore hiscore.hi from the dictionary file """
# from ptools import hiscoreTools
from walker.hiscore import Hiscore
from builder.manipulator import Manipulator
from builder.protomodel import ProtoModel
from tester.predictor import Predictor
from smodels.tools.smodelsLogging import logger
logger.setLevel("ERROR")
import subprocess
## maybe add copying of real*.dict to hiscores.dict
## maybe add check for database pickle file
subprocess.getoutput("rm H*hi")
subprocess.getoutput("rm Kold.conf")
pmodel = ProtoModel(0)
pr = Predictor(0)
ma = Manipulator(pmodel)
ma.initFromDictFile("hiscores.dict", initTestStats=True)
print("The previous K value was", ma.M.K)
pr.predict(ma.M)
print("We end up with K=", ma.M.K)
hi = Hiscore(0, True, picklefile="H1.hi", hiscores=[ma.M])
hi.save()
a = subprocess.getoutput("./upHi.py")
print(a)
def main ( args ):
""" the function that updates the hiscore.hi file
:param args: detailed, outfile, infile, print, fetch, nmax,
check, interactive, nevents.
see "if __main__" part below.
:returns: { "Z": highest significance,
"step": step, "model": model, "K": bayesian_K }
"""
ret = { "Z": 0., "step": 0, "model": None, "K": -100. }
if args.detailed:
args.print = True
if args.outfile.lower() in [ "none", "", "false" ]:
args.outfile = None
infile = args.infile
if type(infile) is str and infile.lower() in [ "none", "" ]:
infile = None
trundir = None
if hasattr ( args, "rundir" ):
trundir = args.rundir
rundir = setup( trundir )
if infile == "default":
infile = "%s/hiscore.hi" % rundir
if args.outfile == infile:
print ( "[hiscore] outputfile is same as input file. will assume that you do not want me to write out at all." )
args.outfile = None
if args.fetch:
import subprocess
cmd = "scp gpu:/local/wwaltenberger/git/smodels-utils/prototools/H*.hi ."
print ( "[hiscore] %s" % cmd )
out = subprocess.getoutput ( cmd )
print ( out )
if infile is None:
print ( "[hiscore] compiling a hiscore list with %d protomodels" % args.nmax )
protomodels = compileList( args.nmax ) ## compile list from H<n>.hi files
else:
with open(infile,"rb") as f:
try:
protomodels = pickle.load ( f )
try:
pickle.load ( f )
except EOFError:
pass
f.close()
except (BlockingIOError,OSError) as e:
print ( "file handling error on %s: %s" % ( infile, e ) )
## make sure we dont block!
raise e
if protomodels[0] == None:
print ( "[hiscore] error, we have an empty hiscore list" )
return ret
sin = infile
if sin == None:
sin = "H*.hi"
pevs = pprintEvs ( protomodels[0] )
print ( "[hiscore] hiscore from %s[%d] is at K=%.3f, Z=%.3f (%s)" % \
( sin, protomodels[0].walkerid, protomodels[0].K, protomodels[0].Z, pevs ) )
# nevents = args.nevents
if args.nmax > 0:
protomodels = protomodels[:args.nmax]
# print ( "we are here", args.outfile, hasattr ( protomodels[0], "analysisContributions" ) )
if type(args.outfile)==str and (".pcl" in args.outfile or ".hi" in args.outfile ):
if not hasattr ( protomodels[0], "analysisContributions" ):
print ( "[hiscore] why does the winner not have analysis contributions?" )
ma = Manipulator ( protomodels[0] )
from walker.hiscore import Hiscore
hi = Hiscore( 0, True, f"{rundir}/hiscore.hi" )
hi.computeAnalysisContributions(ma)
protomodels[0]=ma.M
hi.save()
if not hasattr ( protomodels[0], "particleContributions" ):
print ( "[hiscore] why does the winner not have particle contributions?" )
ma = Manipulator ( protomodels[0] )
from walker.hiscore import Hiscore
from tester.predictor import Predictor
predictor = None
dbpath = args.dbpath
if not "/" in dbpath:
dbpath = f"{rundir}/{args.dbpath}"
if hasattr ( args, "dbpath" ):
dbpath = args.dbpath
if os.path.exists ( dbpath ):
predictor = Predictor ( 0, dbpath = dbpath,
expected = False, select= "all" )
hi = Hiscore( 0, True, f"{rundir}/hiscore.hi", predictor = predictor )
hi.computeParticleContributions(ma)
protomodels[0]=ma.M
hi.save()
if args.outfile is not None:
storeList ( protomodels, args.outfile )
if args.check:
protomodel = protomodels[0]
protomodel.predict()
print ( "[hiscore] args.check, implement" )
if args.print:
printProtoModels ( protomodels, args.detailed, min ( 10, args.nmax ) )
if len(protomodels)>0 and protomodels[0] != None:
ret["Z"]=protomodels[0].Z
ret["K"]=protomodels[0].K
ret["step"]=protomodels[0].step
ret["model"]=protomodels[0]
return ret
return ret
args = argparser.parse_args()
from walker.hiscore import Hiscore
if not os.path.exists ( args.infile ):
print ( f"[hiscoreTools] error: input file {args.infile} does not exist." )
sys.exit()
hi = Hiscore ( 0, False, args.infile )
protomodels = hi.hiscores
import builder
protomodel = protomodels
from builder.protomodel import ProtoModel
# so we can also use Andre's pcl files
if type(protomodels)==ProtoModel:
protomodel = protomodels
else:
protomodel = protomodels[0]
ma = Manipulator ( protomodel )
ma.M.createNewSLHAFileName()
print ( "[hiscoreTools] starting interactive session." )
print ( "[hiscoreTools] Variables: %sprotomodels, pb, fb%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] python: %scopy, numpy, scipy, scipy.stats, math%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] Modules: %smanipulator, hiscore, combiner, predictor, helpers%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] Classes: %sProtoModel, Combiner, Predictor, Hiscore, Database%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
print ( "[hiscoreTools] Instantiations: %sma, co, hi, pr%s" % \
( colorama.Fore.RED, colorama.Fore.RESET ) )
from tester import combiner
from walker import hiscore
from tester import predictor