def checkForRedundancy(self,databaseParticles):
""" In case of efficiency maps, check if any txnames have overlapping
constraints. This would result in double counting, so we dont
allow it. """
if self.getType() == "upperLimit":
return False
logger.debug ( "checking for redundancy" )
datasetElements = []
for tx in self.txnameList:
if hasattr(tx, 'finalState'):
finalState = tx.finalState
else:
finalState = ['MET','MET']
if hasattr(tx, 'intermediateState'):
intermediateState = tx.intermediateState
else:
intermediateState = None
for el in elementsInStr(str(tx.constraint)):
newEl = Element(el,finalState,intermediateState,
model=databaseParticles)
datasetElements.append(newEl)
combos = itertools.combinations(datasetElements, 2)
for x,y in combos:
if x == y and _complainAboutOverlappingConstraints:
errmsg ="Constraints (%s) and (%s) appearing in dataset %s:%s overlap "\
"(may result in double counting)." % \
(x,y,self.getID(),self.globalInfo.id )
logger.error( errmsg )
raise SModelSError ( errmsg )
def __init__(self, path=None):
self.path = path
if path:
logger.debug('Creating object based on %s' %self.path)
#Open the info file and get the information:
if not os.path.isfile(path):
logger.error("Info file %s not found" % path)
raise SModelSError()
from smodels.tools.stringTools import concatenateLines
infoFile = open(self.path)
content = concatenateLines ( infoFile.readlines() )
infoFile.close()
#Get tags in info file:
tags = [line.split(':', 1)[0].strip() for line in content]
for i,tag in enumerate(tags):
if not tag: continue
line = content[i]
value = line.split(':',1)[1].strip()
if tags.count(tag) == 1:
self.addInfo(tag,value)
else:
logger.info("Ignoring unknown field %s found in file %s"
% (tag, self.path))
continue
def __init__(self, path=None):
self.path = path
if path:
logger.debug('Creating object based on %s' % self.path)
#Open the info file and get the information:
if not os.path.isfile(path):
logger.error("Info file %s not found" % path)
raise SModelSError()
from smodels.tools.stringTools import concatenateLines
infoFile = open(self.path)
content = concatenateLines(infoFile.readlines())
infoFile.close()
#Get tags in info file:
tags = [line.split(':', 1)[0].strip() for line in content]
for i, tag in enumerate(tags):
if not tag: continue
line = content[i]
value = line.split(':', 1)[1].strip()
if tags.count(tag) == 1:
self.addInfo(tag, value)
else:
logger.info("Ignoring unknown field %s found in file %s" %
(tag, self.path))
continue
def _validateBase(self, path):
"""
Validates the base directory to locate the database.
Raises an exception if something is wrong with the path.
"""
logger.debug('Try to set the path for the database to: %s', path)
tmp = os.path.realpath(path)
if os.path.isfile(tmp):
self._base = os.path.dirname(tmp)
self.force_load = "pcl"
self.pclfilename = os.path.basename(tmp)
return
if tmp[-4:] == ".pcl":
if not os.path.exists(tmp):
logger.error("File not found: %s" % tmp)
sys.exit()
logger.error("Supplied a pcl filename, but %s is not a file." %
tmp)
sys.exit()
path = tmp + '/'
if not os.path.exists(path):
logger.error('%s is no valid path!' % path)
raise DatabaseNotFoundException("Database not found")
self._base = path
def getValueFor(self, massarray):
"""
Interpolates the value and returns the UL or efficiency for the
respective massarray
:param massarray: mass array values (with units), i.e.
[[100*GeV,10*GeV],[100*GeV,10*GeV]]
"""
porig = self.flattenMassArray(massarray) ## flatten
self.massarray = massarray
if len(porig) != self.full_dimensionality:
logger.error ( "dimensional error. I have been asked to compare a "\
"%d-dimensional mass vector with %d-dimensional data!" % \
( len(porig), self.full_dimensionality ) )
return None
p = ((np.matrix(porig)[0] - self.delta_x)).tolist()[0]
P = np.dot(p, self._V) ## rotate
dp = self.countNonZeros(P)
self.projected_value = self.interpolate([P[:self.dimensionality]])
# self.projected_value = griddata( self.Mp, self.xsec, [ P[:self.dimensionality] ], method="linear")[0]
# self.projected_value = float(self.projected_value)
if dp > self.dimensionality: ## we have data in different dimensions
if self._accept_errors_upto == None:
return None
logger.debug( "attempting to interpolate outside of convex hull "\
"(d=%d,dp=%d,masses=%s)" %
( self.dimensionality, dp, str(massarray) ) )
return self._interpolateOutsideConvexHull(massarray)
return self._returnProjectedValue()
def __init__(self, path=None, info=None, createInfo=True, discard_zeroes=True):
""" :param discard_zeroes: discard txnames with zero-only results """
self.path = path
self.globalInfo = info
self.txnameList = []
if path and createInfo:
logger.debug('Creating object based on data folder : %s' %self.path)
#Get data folder info:
if not os.path.isfile(os.path.join(path,"dataInfo.txt")):
logger.error("dataInfo.txt file not found in " + path)
raise TypeError
self.dataInfo = infoObj.Info(os.path.join(path,"dataInfo.txt"))
#Get list of TxName objects:
for txtfile in glob.iglob(os.path.join(path,"*.txt")):
try:
txname = txnameObj.TxName(txtfile,self.globalInfo,self.dataInfo)
if discard_zeroes and txname.hasOnlyZeroes():
logger.debug ( "%s, %s has only zeroes. discard it." % \
( self.path, txname.txName ) )
continue
self.txnameList.append(txname)
except TypeError: continue
self.txnameList.sort()
self.checkForRedundancy()
def getValueForPoint(self, point):
"""
Returns the UL or efficiency for the point (in coordinates) using interpolation
:param point: Point in coordinate space (length = self.full_dimensionality)
:return: Value of UL or efficiency (float) without units
"""
#Make sure the point is a numpy array
point = np.array(point)
self.projected_value = self.interpolate(point[:self.dimensionality])
#Check if input point has larger dimensionality:
dp = self.countNonZeros(point)
if dp > self.dimensionality: ## we have data in different dimensions
if self._accept_errors_upto == None:
return None
logger.debug( "attempting to interpolate outside of convex hull "\
"(d=%d,dp=%d,point=%s)" %
( self.dimensionality, dp, str(point) ) )
val = self._interpolateOutsideConvexHull(point)
else:
val = self._returnProjectedValue()
return val
def interpolate(self, point, fill_value=np.nan):
tol = 1e-6
# tol = sys.float_info.epsilon * 1e10
simplex = self.tri.find_simplex(point, tol=tol)
if simplex==-1: ## not inside any simplex?
return fill_value
#Transformation matrix for the simplex:
simplexTrans = np.take(self.tri.transform, simplex, axis=0)
#Space dimension:
d = simplexTrans.shape[-1]
#Rotation and translation to baryocentric coordinates:
delta_x = simplexTrans[d,:]
rot = simplexTrans[:d,:]
bary = np.dot(rot,point-delta_x) #Point coordinates in the baryocentric system
#Weights for the vertices:
wts = np.append(bary, 1. - bary.sum())
#Vertex indices:
vertices = np.take(self.tri.simplices, simplex, axis=0)
#Compute the value:
values = np.array(self.y_values)
ret = np.dot(np.take(values, vertices),wts)
minXsec = min(np.take(values, vertices))
if ret < minXsec:
logger.debug('Interpolation below simplex values. Will take the smallest simplex value.')
ret = minXsec
return float(ret)
def likelihood(self, workspace_index=None):
"""
Returns the value of the likelihood.
Inspired by the `pyhf.infer.mle` module but for non-log likelihood
"""
logger.debug("Calling likelihood")
self.__init__(self.data)
if self.nWS == 1:
workspace = self.workspaces[0]
elif workspace_index != None:
if self.zeroSignalsFlag[workspace_index] == True:
logger.warning("Workspace number %d has zero signals" % workspace_index)
return None
else:
workspace = self.workspaces[workspace_index]
# Same modifiers_settings as those used when running the 'pyhf cls' command line
msettings = {'normsys': {'interpcode': 'code4'}, 'histosys': {'interpcode': 'code4p'}}
model = workspace.model(modifier_settings=msettings)
test_poi = 1.
_, nllh = pyhf.infer.mle.fixed_poi_fit(test_poi, workspace.data(model), model, return_fitted_val=True)
ret = nllh.tolist()
try:
ret = float(ret)
except:
ret = float(ret[0])
return np.exp(-ret/2.)
def computeV(self, values):
""" compute rotation matrix _V, and triangulation self.tri """
if self._V != None:
return
Morig = []
self.xsec = []
for x, y in values:
self.xsec.append(y / self.unit)
xp = self.flattenMassArray(x)
Morig.append(xp)
aM = np.matrix(Morig)
MT = aM.T.tolist()
self.delta_x = np.matrix([sum(x) / len(Morig) for x in MT])[0]
M = []
# print "here"
for Mx in Morig:
m = (np.matrix(Mx) - self.delta_x).tolist()[0]
M.append(m)
# M.append ( [ self.round_to_n ( x, 7 ) for x in m ] )
U, s, Vt = svd(M)
V = Vt.T
self._V = V ## self.round ( V )
Mp = []
## the dimensionality of the whole mass space, disrespecting equal branches
## assumption
self.full_dimensionality = len(xp)
self.dimensionality = 0
for m in M:
mp = np.dot(m, V)
Mp.append(mp)
nz = self.countNonZeros(mp)
if nz > self.dimensionality:
self.dimensionality = nz
## print "dim=",self.dimensionality
# self.MpCut=[]
MpCut = []
for i in Mp:
if self.dimensionality > 1:
MpCut.append(i[:self.dimensionality].tolist())
else:
MpCut.append([i[0].tolist(), 0.])
if self.dimensionality == 1:
logger.debug(
"1-D data found. Extending to a small 2-D band around the line."
)
MpCut += [[pt[0], pt[1] + 0.0001]
for pt in MpCut] + [[pt[0], pt[1] - 0.0001]
for pt in MpCut]
self._1dim = True
self.xsec += self.xsec + self.xsec
self.dimensionality = 2
else:
self._1dim = False
# self.Mp=MpCut ## also keep the rotated points, with truncated zeros
self.tri = qhull.Delaunay(MpCut)
def createBinaryFile(self, filename=None):
""" create a pcl file from the text database,
potentially overwriting an old pcl file. """
print("Creating binary database")
print("(this may take a few minutes, but it's done only once!)")
t0 = time.time()
logger.info("Creating binary database ")
logger.info("(this may take a few minutes, but it's done only once!)")
logger.debug(" * compute last modified timestamp.")
self.lastModifiedAndFileCount()
logger.debug ( " * compute timestamp: %s filecount: %d" % \
( time.ctime ( self.txt_mtime[0] ), self.txt_mtime[1] ) )
binfile = filename
if binfile == None:
binfile = self.binfile
logger.debug(" * create %s" % self.binfile)
with open(binfile, "wb") as f:
logger.debug(" * load text database")
self.loadTextDatabase()
logger.debug(" * write %s version %s" %
(self.binfile, self.sw_format_version))
ptcl = serializer.HIGHEST_PROTOCOL
self.pcl_python = sys.version
serializer.dump(self.pcl_python, f, protocol=ptcl)
serializer.dump(self.sw_format_version, f, protocol=ptcl)
serializer.dump(self.txt_mtime, f, protocol=ptcl)
serializer.dump(self._databaseVersion, f, protocol=ptcl)
serializer.dump(self.hasFastLim, f, protocol=ptcl)
serializer.dump(self.expResultList, f, protocol=ptcl)
logger.info ( " * done writing %s in %.1f secs." % \
( binfile, time.time()-t0 ) )
def __init__ ( self, data, cl=0.95):
"""
:param data: instance of `PyhfData` holding the signals information
:param cl: confdence level at which the upper limit is desired to be computed
:ivar data: created from :param data:
:ivar nsignals: signal predictions list divided into sublists, one for each json file
:ivar inputJsons: list of input json files as python json instances
:ivar channelsInfo: list of channels information for the json files
:ivar zeroSignalsFlag: list boolean flags in case all signals are zero for a specific json
:ivar nWS: number of workspaces = number of json files
:ivar patches: list of patches to be applied to the inputJsons as python dictionary instances
:ivar workspaces: list of workspaces resulting from the patched inputJsons
:ivar cl: created from :param cl:
:ivar scale: scale that is applied to the signal predictions, dynamically changes throughout the upper limit calculation
:ivar alreadyBeenThere: boolean flag that identifies when the :ivar nsignals: accidentally passes twice at two identical values
"""
self.data = data
self.nsignals = self.data.nsignals
logger.debug("Signals : {}".format(self.nsignals))
self.inputJsons = self.data.inputJsons
self.channelsInfo = self.data.channelsInfo
self.zeroSignalsFlag = self.data.zeroSignalsFlag
self.nWS = self.data.nWS
self.patches = self.patchMaker()
self.workspaces = self.wsMaker()
self.cl = cl
self.scale = 1.
self.alreadyBeenThere = False # boolean to detect wether self.signals has returned to an older value
self.checkPyhfVersion()
self.welcome()
def compute(self,
sqrts,
slhafile,
lhefile=None,
unlink=True,
loFromSlha=None,
pythiacard=None):
"""
Run pythia and compute SUSY cross sections for the input SLHA file.
:param sqrts: sqrt{s} to run Pythia, given as a unum (e.g. 7.*TeV)
:param slhafile: SLHA file
:param lhefile: LHE file. If None, do not write pythia output to file. If
file does not exist, write pythia output to this file name. If
file exists, read LO xsecs from this file (does not run pythia).
:param unlink: Clean up temp directory after running pythia
:param loFromSlha: If True, uses the LO xsecs from the SLHA file to compute the
higher order xsecs
:param pythiaCard: Optional path to pythia.card. If None, uses /etc/pythia.card
:returns: XSectionList object
"""
sqrts = self._checkSqrts(sqrts)
self._checkSLHA(slhafile)
if lhefile:
if os.path.isfile(lhefile):
logger.warning("Using LO cross sections from " + lhefile)
logger.error(
"Cross section retrieval from lhefile currently not implemented"
)
sys.exit()
else:
logger.info("Writing pythia LHE output to " + lhefile)
if loFromSlha:
logger.info("Using LO cross sections from " + slhafile)
xsecsInfile = crossSection.getXsecFromSLHAFile(slhafile)
loXsecs = crossSection.XSectionList()
for xsec in xsecsInfile:
if xsec.info.order == 0 and xsec.info.sqrts == sqrts:
loXsecs.add(xsec)
else:
logger.info("get LO cross sections from pythia%d" %
self.pythiaVersion)
tool = toolBox.ToolBox().get("pythia%d" % self.pythiaVersion)
tool.nevents = self.nevents
tool.sqrts = sqrts / TeV
tool.pythiacard = pythiacard
loXsecs = tool.run(slhafile, lhefile, unlink=unlink)
self.loXsecs = loXsecs
self.loXsecs.sort()
self.xsecs = self.addHigherOrders(sqrts, slhafile)
self.xsecs.sort()
#for xsec in self.loXsecs:
# logger.debug ( "now writing out xsecs: %s" % xsec.value )
logger.debug("how many NLL xsecs? %d" % len(self.xsecs))
return self.xsecs
def loadTextDatabase ( self ):
""" simply loads the textdabase """
if self.txt_meta.databaseVersion and len(self.expResultList)>0:
logger.debug ( "Asked to load database, but has already been loaded. Ignore." )
return
logger.info ( "Parsing text database at %s" % self.txt_meta.pathname )
self.expResultList = self._loadExpResults()
def __init__(self, path=None, info=None, createInfo=True):
self.path = path
self.globalInfo = info
self.txnameList = []
if path and createInfo:
logger.debug('Creating object based on data folder : %s' %
self.path)
#Get data folder info:
if not os.path.isfile(os.path.join(path, "dataInfo.txt")):
logger.error("dataInfo.txt file not found in " + path)
raise TypeError
self.dataInfo = infoObj.Info(os.path.join(path, "dataInfo.txt"))
#Get list of TxName objects:
for txtfile in glob.iglob(os.path.join(path, "*.txt")):
try:
txname = txnameObj.TxName(txtfile, self.globalInfo,
self.dataInfo)
self.txnameList.append(txname)
except TypeError:
continue
self.txnameList.sort()
def _setParticles(self, databaseParticles=None):
"""
Set the databaseParticles attribute.
If databaseParticles is None and the self.databaseParticles is None,
try to use the particles stored in the first ExpResult
in the database (ExptResult.globalInfo._databaseParticles).
If not found, fallback to the final states defined in defaultFinalStates.py.
:param databaseParticles: Model object containing the final state particles
used in the database.
"""
#If not yet defined, set the attribute to None:
if not hasattr(self, 'databaseParticles'):
self.databaseParticles = None
#If input is given, use it to set the databaseParticles attribute:
if databaseParticles:
logger.debug("Setting database particles from %s" %
str(databaseParticles))
self.databaseParticles = databaseParticles
#If still None, fallback to default:
if self.databaseParticles is None:
logging.debug("databaseParticles not found. Using default state.")
from smodels.experiment.defaultFinalStates import finalStates
self.databaseParticles = finalStates
def __init__(self, path=None, info=None, createInfo=True,
discard_zeroes=True, databaseParticles = None):
"""
:param discard_zeroes: discard txnames with zero-only results
"""
self.path = path
self.globalInfo = info
self.txnameList = []
if path and createInfo:
logger.debug('Creating object based on data folder : %s' %self.path)
#Get data folder info:
if not os.path.isfile(os.path.join(path,"dataInfo.txt")):
logger.error("dataInfo.txt file not found in " + path)
raise TypeError
self.dataInfo = infoObj.Info(os.path.join(path,"dataInfo.txt"))
#Get list of TxName objects:
for txtfile in glob.iglob(os.path.join(path,"*.txt")):
try:
txname = txnameObj.TxName(txtfile,self.globalInfo,
self.dataInfo, databaseParticles)
if discard_zeroes and txname.hasOnlyZeroes():
logger.debug ( "%s, %s has only zeroes. discard it." % \
( self.path, txname.txName ) )
continue
self.txnameList.append(txname)
except TypeError: continue
self.txnameList.sort()
self.checkForRedundancy(databaseParticles)
def updateBinaryFile ( self ):
""" write a binar db file, but only if
necessary. """
if self.needsUpdate():
logger.debug ( "Binary db file needs an update." )
self.createBinaryFile()
else:
logger.debug ( "Binary db file does not need an update." )
def updateBinaryFile(self):
""" write a binar db file, but only if
necessary. """
if self.needsUpdate():
logger.debug("Binary db file needs an update.")
self.createBinaryFile()
else:
logger.debug("Binary db file does not need an update.")
def loadTextDatabase(self):
""" simply loads the textdabase """
if self.txt_meta.databaseVersion and len(self.expResultList) > 0:
logger.debug(
"Asked to load database, but has already been loaded. Ignore.")
return
logger.info("Parsing text database at %s" % self.txt_meta.pathname)
self.expResultList = self._loadExpResults()
def fetchFromScratch(self, path, store, discard_zeroes):
""" fetch database from scratch, together with
description.
:param store: filename to store json file.
"""
def sizeof_fmt(num, suffix='B'):
for unit in ['', 'K', 'M', 'G', 'T', 'P']:
if abs(num) < 1024.:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
import requests
try:
r = requests.get(path)
except Exception as e:
logger.error("Exception when trying to fetch database: %s" % e)
logger.error(
"Consider supplying a different database path in the ini file (possibly a local one)"
)
sys.exit()
if r.status_code != 200:
logger.error ( "Error %d: could not fetch %s from server." % \
( r.status_code, path ) )
sys.exit()
## its new so store the description
with open(store, "w") as f:
f.write(r.text)
if not "url" in r.json().keys():
logger.error("cannot parse json file %s." % path)
sys.exit()
size = r.json()["size"]
logger.info ( "need to fetch %s. size is %s." % \
( r.json()["url"], sizeof_fmt ( size ) ) )
t0 = time.time()
r2 = requests.get(r.json()["url"], stream=True)
filename = "./" + r2.url.split("/")[-1]
with open(filename, "wb") as dump:
if not self.inNotebook(): ## \r doesnt work in notebook
print(" " + " " * 51 + "<", end="\r")
print("loading >", end="")
for x in r2.iter_content(chunk_size=int(size / 50)):
dump.write(x)
dump.flush()
print(".", end="")
sys.stdout.flush()
if self.inNotebook():
print("done.")
else:
print("")
dump.close()
logger.info("fetched %s in %d secs." % (r2.url, time.time() - t0))
logger.debug("store as %s" % filename)
#with open( filename, "wb" ) as f:
# f.write ( r2.content )
# f.close()
self.force_load = "pcl"
return ("./", "%s" % filename)
def fetchFromScratch ( self, path, store, discard_zeroes ):
""" fetch database from scratch, together with
description.
:param store: filename to store json file.
"""
def sizeof_fmt(num, suffix='B'):
for unit in [ '','K','M','G','T','P' ]:
if abs(num) < 1024.:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
import requests
try:
r = requests.get( path )
except Exception as e:
logger.error ( "Exception when trying to fetch database: %s" % e )
logger.error ( "Consider supplying a different database path in the ini file (possibly a local one)" )
sys.exit()
if r.status_code != 200:
logger.error ( "Error %d: could not fetch %s from server." % \
( r.status_code, path ) )
sys.exit()
## its new so store the description
with open( store, "w" ) as f:
f.write ( r.text )
if not "url" in r.json().keys():
logger.error ( "cannot parse json file %s." % path )
sys.exit()
size = r.json()["size"]
logger.info ( "need to fetch %s. size is %s." % \
( r.json()["url"], sizeof_fmt ( size ) ) )
t0=time.time()
r2=requests.get ( r.json()["url"], stream=True )
filename= "./" + r2.url.split("/")[-1]
with open ( filename, "wb" ) as dump:
if not self.inNotebook(): ## \r doesnt work in notebook
print ( " " + " "*51 + "<", end="\r" )
print ( "loading >", end="" )
for x in r2.iter_content(chunk_size=int ( size / 50 ) ):
dump.write ( x )
dump.flush ()
print ( ".", end="" )
sys.stdout.flush()
if self.inNotebook():
print ( "done." )
else:
print( "" )
dump.close()
logger.info ( "fetched %s in %d secs." % ( r2.url, time.time()-t0 ) )
logger.debug ( "store as %s" % filename )
#with open( filename, "wb" ) as f:
# f.write ( r2.content )
# f.close()
self.force_load = "pcl"
return ( "./", "%s" % filename )
def fetchFromServer(self, path, discard_zeroes):
import requests, time, json
self.source = "http"
if "ftp://" in path:
self.source = "ftp"
cDir = cacheDirectory(create=True)
store = os.path.join(
cDir,
path.replace(":", "_").replace("/", "_").replace(".", "_"))
logger.debug("need to fetch from server: %s and store to %s" %
(path, store))
if not os.path.isfile(store):
## completely new! fetch the description and the db!
return self.fetchFromScratch(path, store, discard_zeroes)
with open(store, "r") as f:
jsn = json.load(f)
filename = os.path.join(cDir, jsn["url"].split("/")[-1])
class _: ## pseudo class for pseudo requests
def __init__(self):
self.status_code = -1
r = _()
try:
r = requests.get(path, timeout=2)
except requests.exceptions.RequestException as e:
pass
if r.status_code != 200:
logger.warning( "Error %d: could not fetch %s from server." % \
( r.status_code, path ) )
if not os.path.isfile(filename):
logger.error(
"Cant find a local copy of the pickle file. Exit.")
sys.exit()
logger.warning(
"I do however have a local copy of the file at %s. I work with that."
% filename)
self.force_load = "pcl"
return (cDir, filename)
#return ( cDir, os.path.basename ( filename ) )
if not os.path.exists(filename):
return self.fetchFromScratch(path, store, discard_zeroes)
stats = os.stat(filename)
if stats.st_size < jsn["size"] - 2048:
## size doesnt match (2048 is to allow for slightly different file
## sizes reported by the OS). redownload!
return self.fetchFromScratch(path, store, discard_zeroes)
if r.json()["lastchanged"] > jsn["lastchanged"]:
## has changed! redownload everything!
return self.fetchFromScratch(path, store, discard_zeroes)
if not os.path.isfile(filename):
return self.fetchFromScratch(path, store, discard_zeroes)
self.force_load = "pcl"
return ("./", filename)
def _returnProjectedValue(self):
## None is returned without units'
if self.projected_value is None or math.isnan(self.projected_value):
logger.debug ( "projected value is None. Projected point not in " \
"convex hull? original point=%s" % self.massarray )
return None
#Set value to zero if it is lower than machine precision (avoids fake negative values)
if abs(self.projected_value) < 100. * sys.float_info.epsilon:
self.projected_value = 0.
return self.projected_value * self.unit
def writeIgnoreMessage(keys, rEven, rOdd):
msg = ""
for pid in keys:
if not pid in list(rEven) + list(rOdd):
logger.warning("Particle %i not defined in particles.py, its decays will be ignored" %(pid))
continue
if pid in rEven:
msg += "%s, " % rEven[pid]
continue
if len(msg)>0:
logger.debug ( "Ignoring %s decays" % msg[:-2] )
def writePickle(self, dbVersion):
""" write the pickle file """
meta = metaObj.Meta ( self.path, self.discard_zeroes, databaseVersion=dbVersion )
pclfile = "%s/.%s" % ( self.path, meta.getPickleFileName() )
logger.debug ( "writing expRes pickle file %s, mtime=%s" % (pclfile, meta.cTime() ) )
f=open( pclfile, "wb" )
ptcl = serializer.HIGHEST_PROTOCOL
# ptcl = 2
serializer.dump(meta, f, protocol=ptcl)
serializer.dump(self, f, protocol=ptcl)
f.close()
def _interpolateOutsideConvexHull(self, massarray):
""" experimental routine, meant to check if we can interpolate outside
convex hull """
de = self._estimateExtrapolationError(massarray)
if de < self._accept_errors_upto:
return self._returnProjectedValue()
if not math.isnan(de):
logger.debug ( "Expected propagation error of %f too large to " \
"propagate." % de )
return None
def writeIgnoreMessage(keys, rEven, rOdd):
msg = ""
for pid in keys:
if not pid in list(rEven) + list(rOdd):
logger.warning(
"Particle %i not defined in particles.py, its decays will be ignored"
% (pid))
continue
if pid in rEven:
msg += "%s, " % rEven[pid]
continue
if len(msg) > 0:
logger.debug("Ignoring %s decays" % msg[:-2])
def _interpolateOutsideConvexHull(self, massarray):
""" experimental routine, meant to check if we can interpolate outside
convex hull """
porig = self.flattenMassArray(massarray) ## flatten
p = ((np.matrix(porig)[0] - self.delta_x)).tolist()[0]
P = np.dot(p, self._V)
de = self._estimateExtrapolationError(massarray)
if de < self._accept_errors_upto:
return self._returnProjectedValue()
if not math.isnan(de):
logger.debug ( "Expected propagation error of %f too large to " \
"propagate." % de )
return None
def createBinaryFile(self, filename=None):
""" create a pcl file from the text database,
potentially overwriting an old pcl file. """
## make sure we have a model to pickle with the database!
if self.txt_meta == None:
logger.error(
"Trying to create database pickle, but no txt_meta defined.")
raise SModelSError()
logger.debug( "database timestamp: %s, filecount: %s" % \
( time.ctime( self.txt_meta.mtime ), self.txt_meta.filecount ) )
binfile = filename
if binfile == None:
binfile = self.pcl_meta.pathname
if not hasattr(self,'databaseParticles') or \
type(self.databaseParticles) == type(None):
self._setParticles(self._getParticles())
logger.debug(" * create %s" % binfile)
with open(binfile, "wb") as f:
logger.debug(" * load text database")
self.loadTextDatabase()
logger.debug( " * write %s db version %s, format version %s, %s" % \
( binfile, self.txt_meta.databaseVersion,
self.txt_meta.format_version, self.txt_meta.cTime() ) )
# ptcl = serializer.HIGHEST_PROTOCOL
ptcl = min(
4, serializer.HIGHEST_PROTOCOL
) ## 4 is default protocol in python3.8, and highest protocol in 3.7
serializer.dump(self.txt_meta, f, protocol=ptcl)
serializer.dump(self.expResultList, f, protocol=ptcl)
serializer.dump(self.databaseParticles, f, protocol=ptcl)
logger.info("%s created." % (binfile))
def writePickle(self, dbVersion):
""" write the pickle file """
meta = metaObj.Meta(self.path,
self.discard_zeroes,
databaseVersion=dbVersion)
pclfile = "%s/.%s" % (self.path, meta.getPickleFileName())
logger.debug("writing expRes pickle file %s, mtime=%s" %
(pclfile, meta.cTime()))
f = open(pclfile, "wb")
ptcl = min(4, serializer.HIGHEST_PROTOCOL)
serializer.dump(meta, f, protocol=ptcl)
serializer.dump(self, f, protocol=ptcl)
f.close()
def compile(self):
"""
Try to compile the tool.
"""
logger.debug("Trying to compile %s", self.name)
cmd = "cd %s; make" % self.srcPath
out = executor.getoutput(cmd)
# out = subprocess.check_output ( cmd, shell=True, universal_newlines=True )
logger.debug(out)
if not os.path.exists ( self.executablePath ):
logger.error ( "Compilation of %s failed. Is the %s compiler installed?" % ( self.name, self.compiler ) )
sys.exit()
logger.info ( "Compilation of %s succeeded!" % ( self.name ) )
return True
def main(args):
canonizer = ArgsStandardizer()
setLogLevel(args.verbosity)
if not hasattr(args, "noautocompile"):
args.noautocompile = False
if args.query:
return canonizer.queryCrossSections(args.filename)
if args.colors:
from smodels.tools.colors import colors
colors.on = True
sqrtses = canonizer.getSqrtses(args)
order = canonizer.getOrder(args)
canonizer.checkAllowedSqrtses(order, sqrtses)
inputFiles = canonizer.getInputFiles(args)
ncpus = canonizer.checkNCPUs(args.ncpus, inputFiles)
pythiaVersion = canonizer.getPythiaVersion(args)
ssmultipliers = None
if hasattr(args, "ssmultipliers"):
ssmultipliers = canonizer.getSSMultipliers(args.ssmultipliers)
if ssmultipliers != None:
for pids, multiplier in ssmultipliers.items():
if type(pids) != tuple:
logger.error(
"keys of ssmultipliers need to be supplied as tuples")
sys.exit()
if type(multiplier) not in [int, float]:
logger.error(
"values of ssmultipliers need to be supplied as ints or floats"
)
sys.exit()
pythiacard = None
if hasattr(args, 'pythiacard'):
pythiacard = args.pythiacard
children = []
for i in range(ncpus):
pid = os.fork()
chunk = inputFiles[i::ncpus]
if pid < 0:
logger.error("fork did not succeed! Pid=%d" % pid)
sys.exit()
if pid == 0:
logger.debug("chunk #%d: pid %d (parent %d)." %
(i, os.getpid(), os.getppid()))
logger.debug(" `-> %s" % " ".join(chunk))
computer = XSecComputer( order, args.nevents, pythiaVersion, \
not args.noautocompile )
toFile = canonizer.writeToFile(args)
computer.computeForBunch ( sqrtses, chunk, not args.keep,
args.LOfromSLHA, toFile, pythiacard=pythiacard, \
ssmultipliers = ssmultipliers )
os._exit(0)
if pid > 0:
children.append(pid)
for child in children:
r = os.waitpid(child, 0)
logger.debug("child %d terminated: %s" % (child, r))
logger.debug("all children terminated.")
def run(self,
slhaFile,
cfgfile=None,
do_unlink=True,
do_compile=False,
do_check=True):
"""
Run Pythia.
:param slhaFile: SLHA file
:param cfgfile: optionally supply a new config file; if not supplied,
use the one supplied at construction time;
this config file will not be touched or copied;
it will be taken as is
:param do_unlink: clean up temporary files after run?
:param do_compile: if true, we try to compile binary if it isnt installed.
:param do_check: check installation, before running
:returns: stdout and stderr, or error message
"""
if do_check:
ci = self.checkInstallation()
if not ci:
if not do_compile:
logger.error("couldnt find pythia6 binary.")
self.complain()
logger.warning(
"couldnt find pythia6 binary. I have been asked to try to compile it, though. Lets see."
)
# self.compile()
self.complain()
slha = self.checkFileExists(slhaFile)
cfg = self.absPath(cfgfile)
logger.debug("running with " + str(cfg))
import shutil
shutil.copy(slha, self.tempDirectory() + "/fort.61")
cmd = "cd %s ; %s < %s" % \
(self.tempDirectory(), self.executablePath, cfg)
logger.debug("Now running " + str(cmd))
out = executor.getoutput(cmd)
# out = subprocess.check_output ( cmd, shell=True, universal_newlines=True )
if do_unlink:
self.unlink(unlinkdir=True)
else:
f = open(self.tempDirectory() + "/log", "w")
f.write(cmd + "\n\n\n")
f.write(out + "\n")
f.close()
return out
def chi2(self, workspace_index=None):
"""
Returns the chi square
"""
self.__init__(self.data)
logger.debug("Calling chi2")
if self.nWS == 1:
workspace = self.workspaces[0]
elif workspace_index != None:
if self.zeroSignalsFlag[workspace_index] == True:
logger.warning("Workspace number %d has zero signals" % workspace_index)
return None
else:
workspace = self.workspaces[workspace_index]
# Same modifiers_settings as those used when running the 'pyhf cls' command line
msettings = {'normsys': {'interpcode': 'code4'}, 'histosys': {'interpcode': 'code4p'}}
model = workspace.model(modifier_settings=msettings)
_, nllh = pyhf.infer.mle.fit(workspace.data(model), model, return_fitted_val=True)
logger.debug(workspace['channels'][0]['samples'][0])
logger.debug('nllh : {}'.format(nllh))
# Computing the background numbers and fetching the observations
for ch in workspace['channels']:
chName = ch['name']
# Backgrounds
for sp in ch['samples']:
if sp['name'] != 'bsm':
try:
bkg = [b + d for b, d in zip(bkg, sp['data'])]
except NameError: # If bkg doesn't exit, intialize it
bkg = sp['data']
# Observations
for observation in workspace['observations']:
if observation['name'] == chName:
obs = observation['data']
dn = [ob - bk for ob, bk in zip(obs, bkg)]
# Feeding dn as signal input
for sp in ch['samples']:
if sp['name'] == 'bsm':
sp['data'] = dn
logger.debug(workspace['channels'][0]['samples'][0])
_, maxNllh = pyhf.infer.mle.fixed_poi_fit(1., workspace.data(model), model, return_fitted_val=True)
logger.debug('maxNllh : {}'.format(maxNllh))
ret = (maxNllh - nllh).tolist()
try:
ret = float(ret)
except:
ret = float(ret[0])
return ret
def compile(self):
"""
Try to compile the tool.
"""
logger.debug("Trying to compile %s", self.name)
cmd = "cd %s; make" % self.srcPath
out = executor.getoutput(cmd)
# out = subprocess.check_output ( cmd, shell=True, universal_newlines=True )
logger.debug(out)
if not os.path.exists(self.executablePath):
logger.error(
"Compilation of %s failed. Is the %s compiler installed?" %
(self.name, self.compiler))
sys.exit()
logger.info("Compilation of %s succeeded!" % (self.name))
return True
def _formatObj(self,obj):
"""
Method for formatting the output depending on the type of object
and output.
:param obj: A object to be printed. Must match one of the types defined in formatObj
"""
typeStr = type(obj).__name__
try:
formatFunction = getattr(self,'_format'+typeStr)
return formatFunction(obj)
except AttributeError as e:
logger.debug('Error formating object %s: \n %s' %(typeStr,e))
return False
def fixpermissions():
""" make sure that all filepermissions are such that
we can compile the wrappers for pythia and nllfast. """
from smodels.tools.smodelsLogging import logger
import os, glob
Dir = "%ssmodels/lib/" % installDirectory()
try:
Dirs = [ "%spythia6" % Dir, "%spythia8" % Dir ]
Dirs += glob.glob("%snllfast/nllfast-*" % Dir )
Dirs += glob.glob("%spythia8/xml.doc" % Dir )
for p in Dirs:
logger.debug ( "chmod 777 %s" % (p) )
os.chmod ( p, 0o777 )
except Exception as e:
print ( "chmod failed (permission error). Please try as root, i.e.:" )
print ( "sudo smodelsTools.py fixpermissions" )
def __init__(self, importname, optional=False ):
"""
Initializes the ExternalPythonTool object. Useful for installation.
:params optional: optional package, not needed for core SModelS.
"""
self.name = importname
self.optional = optional
self.pythonPath = ""
try:
i = __import__(importname)
self.pythonPath = i.__file__.replace("/__init__.pyc", "")
except ImportError as e:
if optional:
logger.debug("could not find %s: %s (but its not necessary for smodels, so dont worry)" % (importname, e))
else:
logger.error("could not find %s: %s" % (importname, e))
def _formatObj(self, obj):
"""
Method for formatting the output depending on the type of object
and output.
:param obj: A object to be printed. Must match one of the types defined in formatObj
"""
typeStr = type(obj).__name__
try:
formatFunction = getattr(self, '_format' + typeStr)
return formatFunction(obj)
except AttributeError as e:
logger.debug('Error formating object %s: \n %s' % (typeStr, e))
return False
def _returnProjectedValue(self):
"""
Return interpolation result with the appropriate units.
"""
## None is returned without units'
if self.projected_value is None or math.isnan(self.projected_value):
logger.debug ( "projected value is None. Projected point not in " \
"convex hull? original point=%s" % self.massarray )
return None
#Set value to zero if it is lower than machine precision (avoids fake negative values)
if abs(self.projected_value) < 100.*sys.float_info.epsilon:
self.projected_value = 0.
return self.projected_value*self.units[-1]
def fetchFromServer ( self, path, discard_zeroes ):
import requests, time, json
logger.debug ( "need to fetch from server: %s" % path )
self.source = "http"
if "ftp://" in path:
self.source = "ftp"
store = "." + path.replace ( ":","_" ).replace( "/", "_" ).replace(".","_" )
if not os.path.isfile ( store ):
## completely new! fetch the description and the db!
return self.fetchFromScratch ( path, store, discard_zeroes )
with open(store,"r") as f:
jsn = json.load(f)
filename= "./" + jsn["url"].split("/")[-1]
class _: ## pseudo class for pseudo requests
def __init__ ( self ): self.status_code = -1
r=_()
try:
r = requests.get( path )
except Exception:
pass
if r.status_code != 200:
logger.warning ( "Error %d: could not fetch %s from server." % \
( r.status_code, path ) )
if not os.path.isfile ( filename ):
logger.error ( "Cant find a local copy of the pickle file. Exit." )
sys.exit()
logger.warning ( "I do however have a local copy of the file. I work with that." )
self.force_load = "pcl"
# next step: check the timestamps
return ( "./", filename )
if r.json()["lastchanged"] > jsn["lastchanged"]:
## has changed! redownload everything!
return self.fetchFromScratch ( path, store, discard_zeroes )
if not os.path.isfile ( filename ):
return self.fetchFromScratch ( path, store, discard_zeroes )
self.force_load = "pcl"
# next step: check the timestamps
return ( "./", filename )
def unlink(self, unlinkdir=True):
"""
Remove temporary files.
:param unlinkdir: remove temp directory completely
"""
if self.tempdir == None:
return
if self.keepTempDir:
logger.warn("Keeping everything in " + self.tempdir)
return
logger.debug( "Unlinking " + self.tempdir )
for inputFile in ["fort.61", "fort.68", "log"]:
if os.path.exists(self.tempdir + "/" + inputFile):
os.unlink(self.tempdir + "/" + inputFile)
if unlinkdir:
for inputFile in ["temp.cfg"]:
os.unlink(self.tempdir + "/" + inputFile)
if os.path.exists(self.tempdir):
os.rmdir(self.tempdir)
self.tempdir = None
def checkPathName( self, path, discard_zeroes ):
"""
checks the path name,
returns the base directory and the pickle file name.
If path starts with http or ftp, fetch the description file
and the database.
returns the base directory and the pickle file name
"""
logger.debug('Try to set the path for the database to: %s', path)
if path.startswith( ( "http://", "https://", "ftp://" ) ):
return self.fetchFromServer ( path, discard_zeroes )
if path.startswith( ( "file://" ) ):
path=path[7:]
tmp = os.path.realpath(path)
if os.path.isfile ( tmp ):
base = os.path.dirname ( tmp )
return ( base, tmp )
if tmp[-4:]==".pcl":
self.source="pcl"
if not os.path.exists ( tmp ):
if self.force_load == "pcl":
logger.error ( "File not found: %s" % tmp )
sys.exit()
logger.info ( "File not found: %s. Will generate." % tmp )
base = os.path.dirname ( tmp )
return ( base, tmp )
logger.error ( "Supplied a pcl filename, but %s is not a file." % tmp )
sys.exit()
path = tmp + '/'
if not os.path.exists(path):
logger.error('%s is no valid path!' % path)
raise DatabaseNotFoundException("Database not found")
m=Meta ( path, discard_zeroes = discard_zeroes )
self.source="txt"
return ( path, path + m.getPickleFileName() )
def removeLowerOrder(self):
"""
Keep only the highest order cross section for each process in the list.
Remove order information and set default labels.
"""
newList = XSectionList()
for pids in self.getPIDpairs():
xsecs = self.getXsecsFor(pids)
for i, ixsec in enumerate(xsecs):
newxsec = ixsec.copy()
removeXsec = False
isqrts = ixsec.info.sqrts
iorder = ixsec.info.order
# Check if the xsec appear with the same sqrts but at a higher
# order
for j, jxsec in enumerate(xsecs):
if i == j:
continue
jsqrts = jxsec.info.sqrts
jorder = jxsec.info.order
if jsqrts == isqrts and jorder > iorder:
removeXsec = True
break
if not removeXsec:
# Erase cross section labels and information
newxsec.info.label = str(newxsec.info.sqrts)
newxsec.info.order = None
newList.add(newxsec)
if len(self) != len(newList):
logger.debug("Ignoring %i lower order cross sections",
(len(self) - len(newList)))
self.xSections = newList.xSections
def getValueFor(self,massarray):
"""
Interpolates the value and returns the UL or efficiency for the
respective massarray
:param massarray: mass array values (with units), i.e.
[[100*GeV,10*GeV],[100*GeV,10*GeV]]
"""
porig = self.removeUnits(massarray)
porig = self.formatInput(porig,self.dataShape) #Remove entries which match wildcards
porig = self.flattenArray(porig) ## flatten
self.massarray = massarray ## only for bookkeeping and better error msgs
if len(porig) != self.full_dimensionality:
logger.error ( "dimensional error. I have been asked to compare a "\
"%d-dimensional mass vector with %d-dimensional data!" % \
( len(porig), self.full_dimensionality ) )
return None
p = ((np.array([porig]) - self.delta_x )).tolist()[0]
P = np.dot(p,self._V) ## rotate
#Get value for the truncated point:
self.projected_value = self.interpolate(P[:self.dimensionality])
#Check if input point has larger dimensionality:
dp = self.countNonZeros(P)
if dp > self.dimensionality: ## we have data in different dimensions
if self._accept_errors_upto == None:
return None
logger.debug( "attempting to interpolate outside of convex hull "\
"(d=%d,dp=%d,masses=%s)" %
( self.dimensionality, dp, str(massarray) ) )
return self._interpolateOutsideConvexHull(massarray)
return self._returnProjectedValue()
def _run(self, slhaFile, cfgfile=None, unlink=True, do_compile=False,
do_check=True ):
"""
Really Run Pythia.
:param slhaFile: SLHA file
:param cfgfile: optionally supply a new config file; if not supplied,
use the one supplied at construction time;
this config file will not be touched or copied;
it will be taken as is
:param unlink: clean up temporary files after run?
:param do_compile: if true, we try to compile binary if it isnt installed.
:param do_check: check installation, before running
:returns: stdout and stderr, or error message
"""
if do_check:
self.checkInstallation( do_compile )
slha = self.checkFileExists(slhaFile)
cfg = self.absPath(cfgfile)
ck_cfg = self.checkFileExists(cfg)
logger.debug("running with " + str(cfg))
import shutil
shutil.copy(slha, self.tempDirectory() + "/fort.61")
cmd = "cd %s ; %s < %s" % \
(self.tempDirectory(), self.executablePath, cfg)
logger.debug("Now running " + str(cmd))
out = executor.getoutput(cmd)
if unlink:
self.unlink( unlinkdir=True )
else:
f = open(self.tempDirectory() + "/log", "w")
f.write (cmd + "\n\n\n")
f.write (out + "\n")
f.close()
return out
def checkForRedundancy ( self ):
""" In case of efficiency maps, check if any txnames have overlapping
constraints. This would result in double counting, so we dont
allow it. """
if self.getType() == "upperLimit":
return False
logger.debug ( "checking for redundancy" )
datasetElements = []
for tx in self.txnameList:
if hasattr(tx, 'finalState'):
finalState = tx.finalState
else:
finalState = ['MET','MET']
for el in elementsInStr(str(tx.constraint)):
newEl = Element(el,finalState)
datasetElements.append(newEl)
combos = itertools.combinations ( datasetElements, 2 )
for x,y in combos:
if x.particlesMatch ( y ):
errmsg ="Constraints (%s) appearing in dataset %s, %s overlap "\
"(may result in double counting)." % \
(x,self.getID(),self.globalInfo.id )
logger.error( errmsg )
raise SModelSError ( errmsg )
def checkBinaryFile ( self ):
nu=self.needsUpdate()
logger.debug ( "Checking binary db file." )
logger.debug ( "Binary file dates to %s(%d)" % \
( time.ctime(self.pcl_meta.mtime),self.pcl_meta.filecount ) )
logger.debug ( "Database dates to %s(%d)" % \
( time.ctime(self.txt_meta.mtime),self.txt_meta.filecount ) )
if nu:
logger.info ( "Binary db file needs an update." )
else:
logger.info ( "Binary db file does not need an update." )
return nu
def createBinaryFile(self, filename=None):
""" create a pcl file from the text database,
potentially overwriting an old pcl file. """
if self.txt_meta == None:
logger.error ( "Trying to create database pickle, but no txt_meta defined." )
sys.exit()
logger.debug ( "database timestamp: %s, filecount: %d" % \
( time.ctime ( self.txt_meta.mtime ), self.txt_meta.filecount ) )
binfile = filename
if binfile == None:
binfile = self.pcl_meta.pathname
logger.debug ( " * create %s" % binfile )
with open ( binfile, "wb" ) as f:
logger.debug ( " * load text database" )
self.loadTextDatabase()
logger.debug ( " * write %s db version %s, format version %s, %s" % \
( binfile, self.txt_meta.databaseVersion,
self.txt_meta.format_version, self.txt_meta.cTime() ) )
ptcl = serializer.HIGHEST_PROTOCOL
# ptcl = 2
serializer.dump(self.txt_meta, f, protocol=ptcl)
serializer.dump(self.expResultList, f, protocol=ptcl)
logger.info ( "%s created." % ( binfile ) )
def run( self, slhaFile, lhefile=None, unlink=True ):
"""
Run pythia8.
:param slhaFile: SLHA file
:param lhefile: option to write LHE output to file; if None, do not write
output to disk. If lhe file exists, use its events for
xsecs calculation.
:param unlink: clean up temporary files after run?
:returns: List of cross sections
"""
#Change pythia configuration file, if defined:
if self.pythiacard:
pythiacard_default = self.cfgfile
self.cfgfile = self.pythiacard
self.xsecs = {}
logger.debug ( "wrapper.run()" )
slha = self.checkFileExists(slhaFile)
logger.debug ( "file check: " + slha )
cfg = self.absPath(self.cfgfile)
logger.debug("running with cfgfile " + str(cfg))
lhefile = self.tempDirectory() + "/events.lhe"
cmd = "%s -n %d -f %s -s %d -c %s -l %s" % \
( self.executablePath, self.nevents, slha, self.sqrts, cfg, lhefile )
xmldoc = self.executablePath.replace ( "pythia8.exe", "xml.doc" )
logger.debug ( "exe path=%s" % self.executablePath )
self.checkInstallation ( compile=True )
if os.path.exists (xmldoc ):
logger.debug ( "xml.doc found at %s." % xmldoc )
with open ( xmldoc ) as f:
xmlDir = f.read()
toadd = os.path.join ( os.path.dirname ( xmldoc ) , xmlDir.strip() )
logger.debug ( "adding -x %s" % toadd )
cmd += " -x %s" % toadd
logger.debug("Now running ''%s''" % str(cmd) )
out = executor.getoutput(cmd)
logger.debug ( "out=%s" % out )
if not os.path.isfile(lhefile):
raise SModelSError( "LHE file %s not found" % lhefile )
lheF = open(lhefile,'r')
lhedata = lheF.read()
lheF.close()
os.remove(lhefile)
if not "<LesHouchesEvents" in lhedata:
raise SModelSError("No LHE events found in pythia output")
if not unlink:
tempfile = self.tempDirectory() + "/log"
f = open( tempfile, "w")
f.write (cmd + "\n\n\n")
f.write (out + "\n")
f.write (lhedata + "\n")
f.close()
logger.info ( "stored everything in %s" % tempfile )
# Create memory only file object
if sys.version[0]=="2":
lhedata = unicode( lhedata )
lheFile = io.StringIO(lhedata)
ret = getXsecFromLHEFile(lheFile)
#Reset pythia card to its default value
if self.pythiacard:
self.cfgfile = pythiacard_default
return ret
def decompose(slhafile, sigcut=.1 * fb, doCompress=False, doInvisible=False,
minmassgap=-1.*GeV, useXSecs=None):
"""
Perform SLHA-based decomposition.
:param slhafile: the slha input file. May be an URL (though http, ftp only).
:param sigcut: minimum sigma*BR to be generated, by default sigcut = 0.1 fb
:param doCompress: turn mass compression on/off
:param doInvisible: turn invisible compression on/off
:param minmassgap: maximum value (in GeV) for considering two R-odd particles
degenerate (only revelant for doCompress=True )
:param useXSecs: optionally a dictionary with cross sections for pair
production, by default reading the cross sections
from the SLHA file.
:returns: list of topologies (TopologyList object)
"""
if slhafile.startswith("http") or slhafile.startswith("ftp"):
logger.info ( "asked for remote slhafile %s. will fetch it." % slhafile )
import requests
import os.path
r=requests.get(slhafile)
if r.status_code != 200:
logger.error ( "could not retrieve remote file %d: %s" % ( r.status_code, r.reason ) )
raise SModelSError()
basename = os.path.basename ( slhafile )
f=open ( basename, "w" )
f.write ( r.text )
f.close()
slhafile = basename
t1 = time.time()
if doCompress and minmassgap / GeV < 0.:
logger.error("Asked for compression without specifying minmassgap. Please set minmassgap.")
raise SModelSError()
if type(sigcut) == type(1.):
sigcut = sigcut * fb
try:
f=pyslha.readSLHAFile ( slhafile )
except pyslha.ParseError as e:
logger.error ( "The file %s cannot be parsed as an SLHA file: %s" % (slhafile, e) )
raise SModelSError()
# Get cross section from file
xSectionList = crossSection.getXsecFromSLHAFile(slhafile, useXSecs)
# Get BRs and masses from file
brDic, massDic = _getDictionariesFromSLHA(slhafile)
# Only use the highest order cross sections for each process
xSectionList.removeLowerOrder()
# Order xsections by PDGs to improve performance
xSectionList.order()
#Reweight decays by fraction of prompt decays and add fraction of long-lived
brDic = _getPromptDecays(slhafile,brDic)
# Get maximum cross sections (weights) for single particles (irrespective
# of sqrtS)
maxWeight = {}
for pid in xSectionList.getPIDs():
maxWeight[pid] = xSectionList.getXsecsFor(pid).getMaxXsec()
# Generate dictionary, where keys are the PIDs and values
# are the list of cross sections for the PID pair (for performance)
xSectionListDict = {}
for pids in xSectionList.getPIDpairs():
xSectionListDict[pids] = xSectionList.getXsecsFor(pids)
# Create 1-particle branches with all possible mothers
branchList = []
for pid in maxWeight:
branchList.append(Branch())
branchList[-1].PIDs = [[pid]]
if not pid in massDic:
logger.error ( "pid %d does not appear in masses dictionary %s in slhafile %s" %
( pid, massDic, slhafile ) )
branchList[-1].masses = [massDic[pid]]
branchList[-1].maxWeight = maxWeight[pid]
# Generate final branches (after all R-odd particles have decayed)
finalBranchList = decayBranches(branchList, brDic, massDic, sigcut)
# Generate dictionary, where keys are the PIDs and values are the list of branches for the PID (for performance)
branchListDict = {}
for branch in finalBranchList:
if len(branch.PIDs) != 1:
logger.error("During decomposition the branches should \
not have multiple PID lists!")
return False
if branch.PIDs[0][0] in branchListDict:
branchListDict[branch.PIDs[0][0]].append(branch)
else:
branchListDict[branch.PIDs[0][0]] = [branch]
for pid in xSectionList.getPIDs():
if not pid in branchListDict: branchListDict[pid] = []
#Sort the branch lists by max weight to improve performance:
for pid in branchListDict:
branchListDict[pid] = sorted(branchListDict[pid],
key=lambda br: br.maxWeight, reverse=True)
smsTopList = topology.TopologyList()
# Combine pairs of branches into elements according to production
# cross section list
for pids in xSectionList.getPIDpairs():
weightList = xSectionListDict[pids]
minBR = (sigcut/weightList.getMaxXsec()).asNumber()
if minBR > 1.: continue
for branch1 in branchListDict[pids[0]]:
BR1 = branch1.maxWeight/maxWeight[pids[0]] #Branching ratio for first branch
if BR1 < minBR: break #Stop loop if BR1 is already too low
for branch2 in branchListDict[pids[1]]:
BR2 = branch2.maxWeight/maxWeight[pids[1]] #Branching ratio for second branch
if BR2 < minBR: break #Stop loop if BR2 is already too low
finalBR = BR1*BR2
if type(finalBR) == type(1.*fb):
finalBR = finalBR.asNumber()
if finalBR < minBR: continue # Skip elements with xsec below sigcut
if len(branch1.PIDs) != 1 or len(branch2.PIDs) != 1:
logger.error("During decomposition the branches should \
not have multiple PID lists!")
return False
newElement = element.Element([branch1, branch2])
newElement.weight = weightList*finalBR
newElement.sortBranches() #Make sure elements are sorted BEFORE adding them
smsTopList.addElement(newElement)
smsTopList.compressElements(doCompress, doInvisible, minmassgap)
smsTopList._setElementIds()
logger.debug("slhaDecomposer done in %.2f s." % (time.time() -t1 ) )
return smsTopList
def loadParameters(self):
"""
Reads the parameters from the plotting parameter file.
"""
logger.info("Reading parameters from %s ..." %(self.parameterFile))
parFile = self.parameterFile
import imp
try:
with open(self.parameterFile, 'rb') as fParameters: ## imports parameter file
parameters = imp.load_module("parameters",fParameters,self.parameterFile,('.py', 'rb', imp.PY_SOURCE))
except:
logger.error("Error loading parameters file %s" %self.parameterFile)
return False
if not hasattr(parameters, 'slha_hover_information'):
logger.debug("slha_hover_information dictionary was not found in %s. SLHA data will not be included in info box." %parFile)
self.slha_hover_information = {}
else:
self.slha_hover_information = parameters.slha_hover_information
if not hasattr(parameters, 'ctau_hover_information'):
logger.debug("ctau_hover_information dictionary was not found in %s. Lifetime data will not be included in info box." %parFile)
self.ctau_hover_information = {}
else:
self.ctau_hover_information = parameters.ctau_hover_information
if not hasattr(parameters, 'BR_hover_information'):
logger.debug("BR_hover_information dictionary was not found in %s. Branching ratio data will not be included in info box." %parFile)
self.BR_hover_information = {}
else:
self.BR_hover_information = parameters.BR_hover_information
if not hasattr(parameters, 'SModelS_hover_information'):
logger.debug("SModelS_hover_information dictionary was not found in %s. SModelS data will not be included in info box." %parFile)
self.SModelS_hover_information = {}
else:
self.SModelS_hover_information = list(set(parameters.SModelS_hover_information))
if not hasattr(parameters, 'plot_data'):
logger.debug("plot_data list was not found in %s. All points will be plotted" %parFile)
self.plot_data = ['all']
else:
self.plot_data = list(set(parameters.plot_data))
if not hasattr(parameters, 'variable_x'):
raise SModelSError("variable_x was not found in %s. Please define the variable to be plotted in the x-axis." %parFile)
else:
self.variable_x = parameters.variable_x
if not hasattr(parameters, 'variable_y'):
raise SModelSError("variable_y was not found in %s. Please define the variable to be plotted in the y-axis." %parFile)
else:
self.variable_y = parameters.variable_y
if not hasattr(parameters, 'plot_list'):
raise SModelSError("plot_list was not found in %s. Please define the list of plots to be plotted." %parFile)
else:
self.plot_list = list(set(parameters.plot_list))
if not hasattr(parameters,'BR_get_top'):
logger.debug("BR_get_top not found in %s. Will include all decay channels")
self.BR_get_top = 'all'
else:
self.BR_get_top = parameters.BR_get_top
if not hasattr(parameters,'plot_title'):
logger.warning("plot_title not defined in %s. Using default title" %parFile)
self.plot_title = 'interactive-plots'
else:
self.plot_title = parameters.plot_title
def createExpResult ( self, root ):
""" create, from pickle file or text files """
txtmeta = Meta ( root, discard_zeroes = self.txt_meta.discard_zeroes,
hasFastLim=None, databaseVersion = self.databaseVersion )
pclfile = "%s/.%s" % ( root, txtmeta.getPickleFileName() )
logger.debug ( "Creating %s, pcl=%s" % (root,pclfile ) )
expres = None
try:
# logger.info ( "%s exists? %d" % ( pclfile,os.path.exists ( pclfile ) ) )
if not self.force_load=="txt" and os.path.exists ( pclfile ):
# logger.info ( "%s exists" % ( pclfile ) )
with open(pclfile,"rb" ) as f:
logger.debug ( "Loading: %s" % pclfile )
## read meta from pickle
pclmeta = serializer.load ( f )
if not pclmeta.needsUpdate ( txtmeta ):
logger.debug ( "we can use expres from pickle file %s" % pclfile )
expres = serializer.load ( f )
else:
logger.debug ( "we cannot use expres from pickle file %s" % pclfile )
logger.debug ( "txt meta %s" % txtmeta )
logger.debug ( "pcl meta %s" % pclmeta )
logger.debug ( "pcl meta needs update %s" % pclmeta.needsUpdate ( txtmeta ) )
except IOError as e:
logger.error ( "exception %s" % e )
if not expres: ## create from text file
expres = ExpResult(root, discard_zeroes = self.txt_meta.discard_zeroes )
if self.subpickle and expres: expres.writePickle( self.databaseVersion )
if expres:
contact = expres.globalInfo.getInfo("contact")
if contact and "fastlim" in contact.lower():
self.txt_meta.hasFastLim = True
return expres
def __init__(self, path, globalObj, infoObj):
self.path = path
self.globalInfo = globalObj
self._infoObj = infoObj
self.txnameData = None
self.txnameDataExp = None ## expected Data
self._topologyList = TopologyList()
logger.debug('Creating object based on txname file: %s' %self.path)
#Open the info file and get the information:
if not os.path.isfile(path):
logger.error("Txname file %s not found" % path)
raise SModelSError()
txtFile = open(path,'r')
txdata = txtFile.read()
txtFile.close()
if not "txName" in txdata: raise TypeError
if not 'upperLimits' in txdata and not 'efficiencyMap' in txdata:
raise TypeError
content = concatenateLines(txdata.split("\n"))
#Get tags in info file:
tags = [line.split(':', 1)[0].strip() for line in content]
data = None
expectedData = None
dataType = None
for i,tag in enumerate(tags):
if not tag: continue
line = content[i]
value = line.split(':',1)[1].strip()
if tags.count(tag) != 1:
logger.info("Duplicated field %s found in file %s" \
% (tag, self.path))
if ';' in value: value = value.split(';')
if tag == 'upperLimits' or tag == 'efficiencyMap':
data = value
dataType = tag
elif tag == 'expectedUpperLimits':
expectedData = value
dataType = 'upperLimits'
else:
self.addInfo(tag,value)
ident = self.globalInfo.id+":"+dataType[0]+":"+ str(self._infoObj.dataId)
ident += ":" + self.txName
self.txnameData = TxNameData(data, dataType, ident )
if expectedData:
self.txnameDataExp = TxNameData( expectedData, dataType, ident )
#Builds up a list of elements appearing in constraints:
if hasattr(self,'finalState'):
finalState = self.finalState
else:
finalState = ["MET","MET"]
elements = []
if hasattr(self,'constraint'):
elements += [Element(el,finalState) for el in elementsInStr(str(self.constraint))]
if hasattr(self,'condition') and self.condition:
conds = self.condition
if not isinstance(conds,list): conds = [conds]
for cond in conds:
for el in elementsInStr(str(cond)):
newEl = Element(el,finalState)
if not newEl in elements: elements.append(newEl)
# Builds up TopologyList with all the elements appearing in constraints
# and conditions:
for el in elements:
self._topologyList.addElement(el)
def getExpResults(self, analysisIDs=['all'], datasetIDs=['all'], txnames=['all'],
dataTypes = ['all'], useSuperseded=False, useNonValidated=False,
onlyWithExpected = False ):
"""
Returns a list of ExpResult objects.
Each object refers to an analysisID containing one (for UL) or more
(for Efficiency maps) dataset (signal region) and each dataset
containing one or more TxNames. If analysisIDs is defined, returns
only the results matching one of the IDs in the list. If dataTypes is
defined, returns only the results matching a dataType in the list. If
datasetIDs is defined, returns only the results matching one of the IDs
in the list. If txname is defined, returns only the results matching
one of the Tx names in the list.
:param analysisID: list of analysis ids ([CMS-SUS-13-006,...]). Can
be wildcarded with usual shell wildcards: * ? [<letters>]
Furthermore, the centre-of-mass energy can be chosen
as suffix, e.g. ":13*TeV". Note that the asterisk
in the suffix is not a wildcard.
:param datasetIDs: list of dataset ids ([ANA-CUT0,...]). Can be wildcarded
with usual shell wildcards: * ? [<letters>]
:param txnames: list of txnames ([TChiWZ,...]). Can be wildcarded with
usual shell wildcards: * ? [<letters>]
:param dataTypes: dataType of the analysis (all, efficiencyMap or upperLimit)
Can be wildcarded with usual shell wildcards: * ? [<letters>]
:param useSuperseded: If False, the supersededBy results will not be included
:param useNonValidated: If False, the results with validated = False
will not be included
:param onlyWithExpected: Return only those results that have expected values
also. Note that this is trivially fulfilled for all efficiency maps.
:returns: list of ExpResult objects or the ExpResult object if the list
contains only one result
"""
if type(analysisIDs)==str: analysisIDs=[analysisIDs]
if type(datasetIDs)==str: datasetIDs=[datasetIDs]
if type(txnames)==str: txnames=[txnames]
if type(dataTypes)==str: dataTypes=[dataTypes]
import fnmatch
expResultList = []
for expResult in self.expResultList:
superseded = None
if hasattr(expResult.globalInfo,'supersededBy'):
superseded = expResult.globalInfo.supersededBy.replace(" ","")
if superseded and (not useSuperseded):
continue
analysisID = expResult.globalInfo.getInfo('id')
sqrts = expResult.globalInfo.getInfo('sqrts')
# Skip analysis not containing any of the required ids:
if analysisIDs != ['all']:
hits=False
for patternString in analysisIDs:
# Extract centre-of-mass energy
# Assuming 0 or 1 colons.
pattern = patternString.split(':')
hits = fnmatch.filter ( [ analysisID ], pattern[0] )
if len ( pattern ) > 1:
# Parse suffix
# Accepted Strings: ":13", ":13*TeV", ":13TeV", ":13 TeV"
# Everything else will yield an error at the unum-conversion (eval())
if pattern[1].endswith('TeV'):
pattern[1] = pattern[1][:-3]
if pattern[1][-1] in [" ", "*"]:
pattern[1] = pattern[1][:-1]
pattern[1] += "*TeV"
if sqrts != eval(pattern[1]):
hits = False
if hits:
break
# continue
if not hits:
continue
newExpResult = ExpResult()
newExpResult.path = expResult.path
newExpResult.globalInfo = expResult.globalInfo
newExpResult.datasets = []
for dataset in expResult.datasets:
if dataTypes != ['all']:
hits=False
for pattern in dataTypes:
hits = fnmatch.filter ( [ dataset.dataInfo.dataType ], pattern )
if hits:
break
#continue
if not hits:
continue
if hasattr(dataset.dataInfo, 'dataID') and datasetIDs != ['all']:
hits=False
for pattern in datasetIDs:
hits = fnmatch.filter ( [ dataset.dataInfo.dataID ], pattern )
if hits:
break
# continue
if not hits:
continue
newDataSet = datasetObj.DataSet( dataset.path, dataset.globalInfo,
False, discard_zeroes=self.txt_meta.discard_zeroes )
newDataSet.dataInfo = dataset.dataInfo
newDataSet.txnameList = []
for txname in dataset.txnameList:
if type(txname.validated) == str:
txname.validated = txname.validated.lower()
# print ( "txname",txname.validated,type(txname.validated) )
if (txname.validated not in [True, False, "true", "false", "n/a", "tbd", None, "none"]):
logger.error("value of validated field '%s' in %s unknown." % (txname.validated, expResult))
if txname.validated in [None, "none"]: ## FIXME after 1.1.1 this becomes a warning msg?
logger.debug("validated is None in %s/%s/%s. Please set to True, False, N/A, or tbd." % \
( expResult.globalInfo.id, dataset.dataInfo.dataId, txname ) )
if txname.validated not in [ None, True, "true", "n/a", "tbd" ] and (not useNonValidated ):
# if txname.validated is False and (not useNonValidated):
continue
if txnames != ['all']:
#Replaced by wildcard-evaluation below (2018-04-06 mat)
hits=False
for pattern in txnames:
hits = fnmatch.filter ( [ txname.txName ], pattern )
if hits:
continue
if not hits:
continue
if onlyWithExpected and dataset.dataInfo.dataType == \
"upperLimit" and not txname.txnameDataExp:
continue
newDataSet.txnameList.append(txname)
# Skip data set not containing any of the required txnames:
if not newDataSet.txnameList or newDataSet.txnameList == []:
continue
newExpResult.datasets.append(newDataSet)
# Skip analysis not containing any of the required txnames:
if not newExpResult.getTxNames():
continue
expResultList.append(newExpResult)
return expResultList
