def test3ThreadingSupport(self):
"""Test whether the GIL can be properly released"""
try:
gROOT.GetVersion._threaded = 1
except AttributeError:
# Attribute name change in new Cppyy
gROOT.GetVersion.__release_gil__ = 1
gROOT.GetVersion()
def checkPyROOT(self):
# Loading ROOT library
self.PrintLibrary("PyRoot libraries")
try:
from ROOT import gROOT
except:
self.PrintFAIL(warning=False)
logging.error("'root-config --libdir' indicates a wrong path for ROOT"\
+" libraries. Please specify the ROOT library path"\
+" into the environnement variable $PYTHONPATH")
return False
# Setting ROOT batch mode
if not self.script:
from ROOT import TApplication
from ROOT import gApplication
TApplication.NeedGraphicsLibs()
gApplication.InitializeGraphics()
gROOT.SetBatch(True)
# Checking ROOT release
RootVersion = str(gROOT.GetVersionInt())
if len(RootVersion) < 3:
self.PrintFAIL(warning=False)
logging.error('Bad release of ROOT : '+gROOT.GetVersion()+\
'. MadAnalysis5 needs ROOT 5.27 or higher.\n Please upgrade your version of ROOT.')
return False
RootVersionA = int(RootVersion[0])
RootVersionB = int(RootVersion[1] + RootVersion[2])
if RootVersionA != 5 or RootVersionB < 27:
self.PrintFAIL(warning=False)
logging.error('Bad release of ROOT : '+gROOT.GetVersion()+\
'. MadAnalysis5 needs ROOT 5.27 or higher.\n Please upgrade your version of ROOT.')
return False
self.configLinux.root_version = RootVersion
self.PrintOK()
return True
from __future__ import print_function
from pprint import pprint
from sample_builder.sbyields import SBYields
from ROOT import gROOT
print("My ROOT version is {}".format(gROOT.GetVersion()))
def utest_yields(debug=False):
root_file_name = "../root_files/submitDir_v10_mc16ade.root"
region_prefix = "2tag2pjet_0ptv_SRLRJwindow"
masses = [1000, 1100, 1200, 1400, 1600, 1800, 2000, 2500, 3000]
binning = [0., 4000.] # One bin
pickle_file_name = '../pickle_files/yields.data'
sby = SBYields(root_file_name, region_prefix, masses, binning)
# sby.shape_systs = ['SysFATJET_Medium_JET_Comb_Baseline_Kin',
# 'SysFATJET_Medium_JET_Comb_TotalStat_Kin',
# 'SysFATJET_Medium_JET_Comb_Modelling_Kin',
# 'SysFATJET_Medium_JET_Comb_Tracking_Kin',
# 'SysFATJET_JER', 'SysFATJET_JMR',
# 'SysTAUS_TRUEHADDITAU_EFF_JETID_TOTAL',
# 'SysTAUS_TRUEHADDITAU_SME_TES_TOTAL',
# 'SysMET_SoftTrk_ResoPerp', 'SysMET_SoftTrk_ResoPara',
# 'SysMET_JetTrk_Scale', 'SysMET_SoftTrk_Scale',
# 'SysPRW_DATASF', ]
sby.diboson = ['WWPw', 'WZPw', 'ZZPw']
sby.Wjets = ['Wbb', 'Wbc', 'Wbl', 'Wcc', 'Wcl', 'Wl']
def get_geant_version():
'''
Extracting GEANT4 version from $G4LIB, e.g.
/cvmfs/lz.opensciencegrid.org/geant4/geant4.9.5.p02/lib64/Geant4-9.5.2
TODO: change to use `geant4-config --version`
'''
# geant4.10.02
g4lib = os.environ.get('G4LIB')
tmp = g4lib.split('/')[-1].split('-')[-1]
tmp2 = '.'.join(tmp.split('.')[:-1])
version = '4.{0}'.format(tmp2)
return version
GEANT_VERSION = get_geant_version()
ROOT_VERSION = gROOT.GetVersion().replace('/', '.')
class Benchmark(object):
def __enter__(self):
self.usage_start = resource.getrusage(resource.RUSAGE_CHILDREN)
self.meta_data = {}
return self
def __exit__(self, *args):
self.usage_end = resource.getrusage(resource.RUSAGE_CHILDREN)
self.duration = self.usage_end.ru_utime - self.usage_start.ru_utime
# in Kb
self.rss_usage = self.usage_end.ru_maxrss - self.usage_start.ru_maxrss
self.meta_data['Duration'] = self.duration
implementation = platform.python_implementation()
compiler = platform.python_compiler()
cur_system = platform.uname()
nversion = 'Not Installed'
cpversion = 'Not Installed'
rootversion = 'Not Installed'
if importlib.util.find_spec('numpy'):
import numpy
nversion = numpy.__version__
if importlib.util.find_spec('cupy'):
import cupy
cpversion = cupy.__version__
if importlib.util.find_spec('ROOT'):
from ROOT import gROOT
rootversion = gROOT.GetVersion()
assert importlib.util.find_spec(
'numpy'), 'Numpy is needed, please install using: pip install numpy'
assert importlib.util.find_spec(
'matplotlib'
), 'Matplotlib is needed, please install using: pip install matplotlib'
assert importlib.util.find_spec(
'h5py'), 'H5py is needed, please install using: pip install h5py'
assert importlib.util.find_spec(
'uproot'), 'Uproot is needed, please install using: pip install uproot'
s1 = f'''.__________________________________________________.
| ._____ _ .____ .__ __. |
| .____ _. _. / (_)| _ \ | \/ | |
| | _ | | | | | (----'_ | |_) || \ / | |
def checkROOT(self):
# Checking if ROOT is present
logging.info("Checking ROOT libraries ...")
# Trying to call root-config
rootdirs = commands.getstatusoutput('root-config --libdir --incdir')
if rootdirs[0] > 0:
logging.error('ROOT module called "root-config" is not detected.\n'\
+'Two explanations :n'\
+' - ROOT is not installed. You can download it '\
+'from http://root.cern.ch\n'\
+' - ROOT binary folder must be placed in the '\
+'global environment variable $PATH')
return False
# Extracting ROOT library and header path
root_tmp = rootdirs[1].split()
self.includes.append(root_tmp[1])
self.libs.append(root_tmp[0])
os.environ['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH'] + \
":" + root_tmp[0]
os.environ['DYLD_LIBRARY_PATH'] = os.environ['DYLD_LIBRARY_PATH'] + \
":" + root_tmp[0]
os.environ['LIBRARY_PATH'] = os.environ['LIBRARY_PATH'] + \
":" + root_tmp[0]
os.environ['CPLUS_INCLUDE_PATH'] = os.environ['CPLUS_INCLUDE_PATH'] + \
":" + root_tmp[1]
# Adding ROOT library path to Python path
sys.path.append(root_tmp[0])
# Looking for libPyROOT.so
find = False
for item in self.libs:
files = glob.glob(item + "/libPyROOT.so")
if len(files) != 0:
self.configLinux.libraries['PyROOT'] = files[0] + ":" + str(
os.stat(files[0]).st_mtime)
find = True
break
if not find:
logging.error(
"ROOT library called 'libPyROOT.so' is not found. Please check that ROOT is properly installed."
)
return False
# Looking for ROOT.py
find = False
for item in self.libs:
files = glob.glob(item + "/ROOT.py")
if len(files) != 0:
self.configLinux.libraries['ROOT'] = files[0] + ":" + str(
os.stat(files[0]).st_mtime)
find = True
break
if not find:
logging.error(
"ROOT file called 'ROOT.py' is not found. Please check that ROOT is properly installed."
)
return False
# Looking for TH1F.h
find = False
for item in self.includes:
files = glob.glob(item + "/TH1F.h")
if len(files) != 0:
self.configLinux.headers['ROOT'] = files[0] + ":" + str(
os.stat(files[0]).st_mtime)
find = True
break
if not find:
logging.error("ROOT headers are not found. " +\
"Please check that ROOT is properly installed.")
return False
# Loading ROOT library
logging.info("Loading ROOT libraries ...")
try:
from ROOT import gROOT
except:
logging.error("'root-config --libdir' indicates a wrong path for ROOT"\
+" libraries. Please specify the ROOT library path"\
+" into the environnement variable $PYTHONPATH")
return False
# Setting ROOT batch mode
if not self.script:
from ROOT import TApplication
from ROOT import gApplication
TApplication.NeedGraphicsLibs()
gApplication.InitializeGraphics()
gROOT.SetBatch(True)
# Checking ROOT release
RootVersion = str(gROOT.GetVersionInt())
if len(RootVersion) < 3:
logging.error('Bad release of ROOT : '+gROOT.GetVersion()+\
'. MadAnalysis5 needs ROOT 5.27 or higher.\n Please upgrade your version of ROOT.')
return False
RootVersionA = int(RootVersion[0])
RootVersionB = int(RootVersion[1] + RootVersion[2])
if RootVersionA != 5 or RootVersionB < 27:
logging.error('Bad release of ROOT : '+gROOT.GetVersion()+\
'. MadAnalysis5 needs ROOT 5.27 or higher.\n Please upgrade your version of ROOT.')
return False
self.configLinux.root_version = RootVersion
return True
def main():
print "\n\tmkanalyzer.py"
# Decode command line
argv = sys.argv[1:]
argc = len(argv)
if argc < 1: usage()
filename = nameonly(argv[0])
if argc > 1:
varfilename = argv[1]
else:
varfilename = "variables.txt"
if not os.path.exists(varfilename):
print "mkanalyzer.py - can't find variable file: %s" % varfilename
sys.exit(0)
# Check for macro mode
macroMode = argc > 2
# Read variable names
records = map(strip, open(varfilename, "r").readlines())
# Get tree name(s)
t = split(records[0])
if lower(t[0]) == "tree:":
treename = t[1]
else:
treename = "Events"
start = 1
for record in records[1:]:
record = strip(record)
if record == "": break
t = split(record)
if lower(t[0]) == "tree:":
treename += " %s" % t[1]
start += 1
# --------------------------------------------------------------------
# Done with header, so loop over branch names
# and get list of potential struct names (first field of
# varname.
# --------------------------------------------------------------------
# get ROOT version
rversion = gROOT.GetVersion()[0]
records = records[start:]
tokens = []
tmpmap = {}
for index in xrange(len(records)):
record = records[index]
if record == "": continue
# split record into its fields
# varname = variable name as determined by mkvariables.py
tokens.append(split(record, '/'))
rtype, branchname, varname, count = tokens[-1]
# varname should have the format
# <objname>_<field-name>
t = split(varname,'_')
if len(t) > 1: # Need at least two fields for a struct
key = t[0]
if not tmpmap.has_key(key): tmpmap[key] = 0
tmpmap[key] += 1
# If we have at least two fields, we'll create a struct
structname = {}
for key in tmpmap.keys():
if tmpmap[key] > 1:
structname[key] = tmpmap[key]
# --------------------------------------------------------------------
# Loop over tokens
# --------------------------------------------------------------------
usednames = {}
varmap = {}
vectormap = {}
for index, (rtype, branchname, varname, count) in enumerate(tokens):
# if rtype is indexed with an asterisk, this indicates that this
# type is intrinsically a vector type to be handled directly by
# Root.
if rtype[-1] == '*':
intrinsicVtype = True
rtype = rtype[:-1]
else:
intrinsicVtype = False
# Check if current variable is a leaf counter (flagged with an "*")
t = split(count)
count = atoi(t[0]) # Change type to an integer
iscounter = t[-1] == "*"
# make sure names are unique. If they aren't bail out!
if varmap.has_key(varname):
print "** error ** duplicate variable name %s; "\
"please fix variables.txt by hand"
sys.exit(0)
# do something about annoying types
if find(varname, '[') > -1 and count > 1:
print "** warning: access to %s[%d] "\
"must be hand-coded" % (branchname, count)
continue
if find(lower(rtype), 'ref') > -1:
print "** warning: access to %s of type %s "\
"must be hand-coded" % (branchname, rtype)
continue
# special handling for Delphes
if find(varname, '_size') > -1:
varname = 'n%s' % replace(varname, '_size', '')
if rtype == "bool":
rtype = "int"
elif rtype == "long64":
rtype = "long"
elif rtype == "int32":
rtype = "int"
elif rtype == "uchar":
rtype = "int"
elif rtype == "uint":
rtype = "int"
objname = ''
fldname = ''
# If this is a leaf counter, which happens to have the same name
# as a struct, then just ignore for now
if iscounter:
if structname.has_key(varname):
varname = '%s_' % varname
else:
# Get object and field names
t = split(varname,'_')
if len(t) > 0:
# we have at least two fields in varname
key = t[0]
if structname.has_key(key):
# This branch potentially belongs to a struct.
objname = key
# Make sure the count for this branch matches that
# of existing struct
if not usednames.has_key(objname):
usednames[objname] = count;
if usednames[objname] == count:
fldname = replace(varname, '%s_' % objname, '')
else:
objname = ''
fldname = ''
# update map for all variables
varmap[varname] = [rtype, branchname, count, iscounter, intrinsicVtype]
# vector types must have the same object name and a max count > 1
if count > 1:
if fldname != "":
# Make sure fldname is a valid c++ name
if fldname[0] in ['0','1','2','3','4','5','6','7','8','9']:
fldname = 'f%s' % fldname
if not vectormap.has_key(objname): vectormap[objname] = []
vectormap[objname].append((rtype, fldname, varname, count))
#print "%s.%s (%s)" % (objname, fldname, count)
# Declare all variables
keys = varmap.keys()
keys.sort()
declarevec= []
declare = []
init = []
setb = []
addb = []
impl = []
vinit = []
ptrtoref = []
# first create counters
counters = {}
for index, varname in enumerate(keys):
rtype, branchname, count, iscounter, intrinsicVtype = varmap[varname]
if iscounter:
#objname = varname[1:] # n<object-name>
counters[varname] = branchname
addb.append(' output->add("%s", %s);' % (branchname, varname))
addb.append('')
for index, varname in enumerate(keys):
rtype, branchname, count, iscounter, intrinsicVtype = varmap[varname]
if macroMode:
if iscounter:
impl.append(' countvalue& v%d = (*varmap)["%s"];'%\
(index, branchname))
impl.append(' if ( v%d.count )' % index)
impl.append(' %s = *v%d.count;' % (varname, index))
impl.append(' else')
impl.append(' %s = 0;' % varname)
impl.append('')
elif count == 1:
impl.append(' countvalue& v%d = (*varmap)["%s"];' %\
(index, branchname))
impl.append(' if ( v%d.value )' % index)
impl.append(' %s = *v%d.value;' % (varname, index))
impl.append(' else')
impl.append(' %s = 0;' % varname)
impl.append('')
else:
# this is a vector
impl.append(' countvalue& v%d = (*varmap)["%s"];' % \
(index, branchname))
impl.append(' if ( v%d.value )' % index)
impl.append(' {')
impl.append(' %s.resize(*v%d.count);' % (varname, index))
impl.append(' copy(v%d.value, v%d.value+*v%d.count, '\
'%s.begin());'% (index, index, index, varname))
impl.append(' }')
impl.append(' else')
impl.append(' %s.clear();' % varname)
impl.append('')
setb.append(' input->select("%s",' % branchname)
if intrinsicVtype:
setb.append(' %s, true);' % varname)
else:
setb.append(' %s);' % varname)
add_saveObjects = True
if count == 1:
declare.append(" %s\t%s;" % (rtype, varname))
else:
# --------------------------------------
# this is a vector
# --------------------------------------
if intrinsicVtype:
declarevec.append(" // intrinsic vector type")
declarevec.append(" std::vector<%s> \t%s;" %(rtype, varname))
declarevec.append("")
add_saveObjects = False # SAVE NOT YET IMPLEMENTED
else:
declarevec.append(" std::vector<%s>\t%s;" % (rtype, varname))
init.append(" %s\t= std::vector<%s>(%d,0);" % \
(varname, rtype, count))
# first make a concerted effort to identify the counter
# variable name for this object
objname = split(varname, '_')[0]
if not counters.has_key(objname):
key = 'n' + objname
if not counters.has_key(key):
key = objname + "_"
if not counters.has_key(key):
print '** mkanalyzer.py - '\
'unable to find counter for '\
'object %s' % objname
add_saveObjects = False
else:
objname = key
branchname += "[%s]" % counters[objname]
if not iscounter:
addb.append(' output->add("%s",' % branchname)
addb.append(' %s);' % varname)
# Create structs for vector variables
pragma = []
if rversion == '5':
pragma.append('#pragma link C++ class vector<vector<int> >;')
pragma.append('#pragma link C++ class vector<vector<float> >;')
pragma.append('#pragma link C++ class vector<vector<double> >;')
keys = vectormap.keys()
keys.sort()
structvec = []
structdecl = []
structimpl = []
structimplall = []
selectdecl = []
selectimpl = []
selectimpl.append(' // Select objects for which the select'\
' function was called')
selectimpl.append(' void saveObjects()')
selectimpl.append(' {')
if add_saveObjects:
selectimpl.append(' int n = 0;')
structimplall.append(' void fillObjects()')
structimplall.append(' {')
for index, objname in enumerate(keys):
values = vectormap[objname]
varname= values[0][-2]
count = values[0][-1]
structimplall.append(' fill%ss();' % objname)
structimpl.append(' void fill%ss()' % objname)
structimpl.append(' {')
structimpl.append(' %s.resize(%s.size());' % (objname, varname))
structimpl.append(' for(unsigned int i=0; i < %s.size(); ++i)' % \
objname)
structimpl.append(' {')
if add_saveObjects:
selectimpl.append('')
selectimpl.append(' n = 0;')
selectimpl.append(' try')
selectimpl.append(' {')
selectimpl.append(' n = indexmap["%s"].size();' % objname)
selectimpl.append(' }')
selectimpl.append(' catch (...)')
selectimpl.append(' {}')
selectimpl.append(' if ( n > 0 )')
selectimpl.append(' {')
selectimpl.append(' std::vector<int>& '\
'index = indexmap["%s"];' % objname)
selectimpl.append(' for(int i=0; i < n; ++i)')
selectimpl.append(' {')
selectimpl.append(' int j = index[i];')
structdecl.append(' struct %s_s' % objname)
structdecl.append(' {')
for rtype, fldname, varname, count in values:
# treat bools as ints
if rtype == "bool":
cast = '(bool)'
else:
cast = ''
# this is a vector
structdecl.append(' %s\t%s;' % (rtype, fldname))
structimpl.append(' %s[i].%s\t= %s%s[i];' % (objname,
fldname,
cast,
varname))
if add_saveObjects:
selectimpl.append(' %s[i]\t= %s[j];' % (varname,
varname))
structvec.append(' std::vector<eventBuffer::%s_s> %s;' % \
(objname, objname))
init.append(' %s\t= std::vector<eventBuffer::%s_s>(%d);' % (objname,
objname,
count))
pragma.append('')
pragma.append('#pragma link C++ class vector<eventBuffer::%s_s>;' % \
objname)
pragma.append('')
structdecl.append('')
structdecl.append(' std::ostream& operator<<(std::ostream& os)')
structdecl.append(' {')
structdecl.append(' char r[1024];')
structdecl.append(' os << "%s" << std::endl;' % objname)
for rtype, fldname, varname, count in values:
structdecl.append(' sprintf(r, " %s: %s\\n", "%s",'\
' ( double)%s); '\
'os << r;' % ("%-32s", "%f", fldname, fldname))
structdecl.append(' return os;')
structdecl.append(' }')
structdecl.append(' };\n')
structimpl.append(' }')
structimpl.append(' }\n') # end of fill<object>s()
if add_saveObjects:
selectimpl.append(' }')
selectimpl.append(' n%s = n;' % objname)
selectimpl.append(' }')
structimplall.append(' }') # end of fillObjects()
selectimpl.append(' }') # end of saveObjects()
# Write out files
if macroMode:
names = {'NAME': upper(filename),
'name': filename,
'time': ctime(),
'author': AUTHOR,
'vardecl': join("", declare, "\n"),
'selection': join(" ", select, "\n"),
'structdecl': join("", structdecl, "\n"),
'structimpl': join("", structimpl, "\n"),
'structimplall': join("", structimplall, "\n"),
'selectimpl': join("", selectimpl, "\n"),
'impl': join("", impl, "\n"),
'treename': treename,
'percent': '%' }
record = MACRO_DECL_H % names
outfilename = "%s_decl.h" % filename
open(outfilename, "w").write(record)
record = MACRO_IMPL_H % names
outfilename = "%s_impl.h" % filename
open(outfilename, "w").write(record)
sys.exit(0)
# Put everything into a directory
cmd = '''
mkdir -p %(dir)s/tmp
mkdir -p %(dir)s/lib
mkdir -p %(dir)s/src
mkdir -p %(dir)s/python
mkdir -p %(dir)s/include
cp %(hpp)s %(dir)s/include
cp %(cpp)s %(dir)s/src
''' % {'dir': filename,
'hpp': TREESTREAM_HPP,
'cpp': TREESTREAM_CPP}
os.system(cmd)
cmd = '''
cp %(hpp)s %(dir)s/include
cp %(cpp)s %(dir)s/src
cp %(py)s %(dir)s/python
''' % {'dir': filename,
'hpp': TNM_HPP,
'cpp': TNM_CPP,
'py' : TNM_PY
}
os.system(cmd)
# Create Makefile
names = {'name': filename,
'filename': filename,
'time': ctime(),
'author': AUTHOR,
'percent': '%'
}
record = MAKEFILE % names
open("%s/Makefile" % filename, "w").write(record)
# Create C++ code
declarevec += [""] + declare
names = {'NAME': upper(filename),
'name': filename,
'time': ctime(),
'author': AUTHOR,
'vinit': join("", vinit, "\n"),
'ptrtoref': join("", ptrtoref, "\n"),
'vardecl': join("", declarevec, "\n"),
'init': join("", init, "\n"),
'setb': join(" ", setb, "\n"),
'addb': join(" ", addb, "\n"),
'structdecl': join("", structdecl, "\n"),
'structimpl': join("", structimpl, "\n"),
'structimplall': join("", structimplall, "\n"),
'selectimpl': join("", selectimpl, "\n"),
'structvec': join("", structvec, "\n"),
'treename': treename,
'percent': '%' }
record = TEMPLATE_H % names
outfilename = "%s/include/eventBuffer.h" % filename
open(outfilename,"w").write(record)
# Create cc file if one does not yet exist
outfilename = "%s/%s.cc" % (filename, filename)
if not os.path.exists(outfilename):
record = TEMPLATE_CC % names
open(outfilename,"w").write(record)
# Create README
outfilename = "%s/README" % filename
record = README % names
open(outfilename,"w").write(record)
# write out linkdef
outfilename = "%s/include/linkdef.h" % filename
names['pragma'] = join("", pragma, "\n")
record = LINKDEF % names
open(outfilename,"w").write(record)
# write setup.sh
record = '''
if [ "$PYTHONPATH" == "" ]; then
export PYTHONPATH=$PWD/python
else
export PYTHONPATH=$PWD/python:$PYTHONPATH
fi
if [ "$LD_LIBRARY_PATH" == "" ]; then
export LD_LIBRARY_PATH=$PWD/lib
else
export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH
fi
'''
outfilename = "%s/setup.sh" % filename
open(outfilename,"w").write(record)
# Create python program if one does not yet exist
outfilename = "%s/%s.py" % (filename, filename)
if not os.path.exists(outfilename):
record = PYTEMPLATE % names
open(outfilename,"w").write(record)
os.system("chmod +x %s" % outfilename)
print "\tdone!"
if p_status != 0:
print " - ROOT " + "NOT FOUND"
else:
words = output.split()
if len(words) == 0:
print " - ROOT FOUND " + "version UNKOWN"
else:
print " - ROOT FOUND " + "version " + words[0]
# root for python
try:
from ROOT import TCanvas, TF1, gROOT
except:
print " - pyROOT NOT FOUND"
else:
myversion = gROOT.GetVersion()
print " - pyROOT FOUND version " + myversion
# geant4
p = subprocess.Popen("geant4-config --version",
stdout=subprocess.PIPE,
shell=True,
stderr=subprocess.PIPE)
(output, err) = p.communicate()
p_status = p.wait()
if p_status != 0:
print " - Geant4 NOT FOUND"
else:
words = output.split()
if len(words) == 0:
print " - Geant4 FOUND version UNKOWN"
def test3ThreadingSupport(self):
"""Test whether the GIL can be properly released"""
gROOT.GetVersion._threaded = 1
gROOT.GetVersion()
