def __setattr__(self, name, value):
self.setlog()
# Find the absolut path to the file
if name == 'Source' and value.lower().endswith('.xml'):
filepath = value
# find the absolute file path
import os
from AthenaCommon.Utils.unixtools import FindFile
# look for it in the local and then all XML directories
filepath = FindFile(filepath, ['.'] +
os.environ['XMLPATH'].split(os.pathsep),
os.R_OK)
# Not found in user space - look in the AtlasAuthentication package
if not filepath:
filepath = 'TrigConfOffline/' + value
filepath = FindFile(filepath, ['.'] +
os.environ['XMLPATH'].split(os.pathsep),
os.R_OK)
if not filepath:
self._log.warning("Rules source file %s not found" % (value))
else:
value = filepath
self._log.debug("Rules source file found in %s" % (value))
if name in self._properties:
self._log.debug('Set property: %s = %s' % (name, value))
self.__dict__[name] = value
if name == 'OutputLevel':
self._log.setLevel(value)
else:
self._log.warning('Unknown property: %s = %s - property not set' %
(name, value))
def _trace_include( self, frame, event, arg ):
fn = frame.f_code.co_filename
if fn.find ('importlib._bootstrap') >= 0:
return self._trace_include
if not os.path.exists( fn ):
fn = FindFile( basename2( fn ), sys.path, os.R_OK )
if not ( fn and self._doTrace( fn ) ):
return self._trace_include
if fn not in _filecache:
# wait until importing of the module is done to minimize pollution
f = frame.f_back
while f is not None:
try:
if 'import' in _filecache[ f.f_code.co_filename ][ f.f_lineno ]:
return self._trace_include
except Exception:
pass
f = f.f_back
del f
# import is done, and we're back, accept this file from this point on
_filecache[ fn ] = open( fn, 'r' ).readlines() or '\n'
_linecache[ fn ] = sys.maxsize, self.fid
self.fid += 1
lno = frame.f_lineno
aln = lno - 1 > 0 and lno - 1 or 0
ncur, fid = _linecache[ fn ]
buf = _filecache[ fn ]
if self._fcurrent != fn:
self.msg.info( 'continued trace of "%s"', basename2( fn ) )
self._fcurrent = fn
if event == 'line':
for i in range( ncur, aln ):
self._oneline( fid, i, silentMarker, buf )
if ncur <= aln:
self._oneline( fid, aln, activeMarker, buf )
elif 0 <= aln:
self._oneline( fid, aln, tracedMarker, buf )
if ncur < lno:
_linecache[ fn ] = lno, fid
elif event == 'call':
if lno < ncur:
self._oneline( fid, aln, callMarker, buf )
elif event == 'return':
if lno < ncur:
fln = frame.f_code.co_firstlineno - 1
self._oneline( fid, fln, returnMarker, None )
return self._trace_include
def check_job_options(jo_path):
'''
Check if the job options file exists locally or in JOBOPTSEARCHPATH.
Returns True if the file is found and False otherwise.
'''
# Check if job options exists locally
if os.path.isfile(jo_path):
return True
# Try to find the file in JOBOPTSEARCHPATH
found = FindFile(jo_path, os.environ['JOBOPTSEARCHPATH'].split(os.pathsep),
os.R_OK)
return found is not None
def load_input_json():
'''Reads the json file with input definitions and returns the data as dictionary'''
log = get_logger()
input_json_fullpath = FindFile(input_json, os.environ['DATAPATH'].split(os.pathsep), os.R_OK)
if not input_json_fullpath:
log.error('Failed to determine full path for input JSON %s', input_json)
return None
log.debug('Reading %s', input_json_fullpath)
with open(input_json_fullpath) as data_file:
return json.load(data_file)
def _getFileLocalOrPath(filename, pathenv):
"""looks for filename in local directory and then in all paths specified in environment variable 'pathenv'
returns path/filename if existing, otherwise None
"""
if os.path.exists(filename):
log.info( "Using local file %s", filename)
return filename
pathlist = os.getenv(pathenv,'').split(os.pathsep)
resolvedfilename = FindFile(filename, pathlist, os.R_OK)
if resolvedfilename:
return resolvedfilename
log.fatal("No file %s found locally nor in %s" % (filename, os.getenv('CORAL_DBLOOKUP_PATH')) )
return None
def getJetBadChanCorrTool(finder, mainParam, input, **options):
options = checkAndUpdateOptions(finder=finder,
mainParam=mainParam,
input=input,
**options)
from JetMomentTools.JetMomentToolsConf import JetBadChanCorrTool
from CaloClusterCorrection.StandardCellWeightCalib import getCellWeightTool
cellcalibtool = getCellWeightTool(options['finder'],
options['mainParam'],
input,
onlyCellWeight=True)
coneSize = options['mainParam']
BadChanCorrT = JetBadChanCorrTool("JetBadChanCorrTool")
# Explicitly list flags
BadChanCorrT.ConeDr = coneSize
BadChanCorrT.CellCalibrator = cellcalibtool
BadChanCorrT.UseCalibScale = False
BadChanCorrT.AttachCorrCell = True
BadChanCorrT.AttachCorrDOTX = True
BadChanCorrT.AttachCorrJet = True
BadChanCorrT.MissingCellTool = getDefaultMissingCellTool(coneSize)
# Optional
BadChanCorrT.AttachCorrJetForCell = False
if options['addJetQualityMoments'] == True:
BadChanCorrT.AttachCorrJetForCell = True
# new setup for THistService
import os
dataPathList = os.environ['DATAPATH'].split(os.pathsep)
dataPathList.insert(0, os.curdir)
from AthenaCommon.Utils.unixtools import FindFile
RefFileName = FindFile("JetBadChanCorrTool.root", dataPathList, os.R_OK)
from AthenaCommon.AppMgr import ServiceMgr
if not hasattr(ServiceMgr, 'THistSvc'):
from GaudiSvc.GaudiSvcConf import THistSvc
ServiceMgr += THistSvc()
ServiceMgr.THistSvc.Input += [
"JetBadChanCorrTool DATAFILE=\'%s\' OPT=\'READ\'" % RefFileName
]
return BadChanCorrT
def configure(self, test):
json_file = 'TrigInDetValidation/comparitor.json'
json_fullpath = FindFile(json_file,
os.environ['DATAPATH'].split(os.pathsep),
os.R_OK)
if not json_fullpath:
print('Failed to determine full path for input JSON %s', json_file)
return None
with open(json_fullpath) as f:
data = json.load(f)
self.output_dir = 'HLT' + self.level + '-plots'
flag = self.level + self.slice
if (self.lowpt):
self.output_dir = self.output_dir + '-lowpt'
flag = flag + 'Lowpt'
data_object = data[flag]
self.chains = data_object['chains']
if (self.test == 'ttbar'):
self.output_dir = self.output_dir + "-" + self.slice
if (self.type == 'offl'):
self.output_dir = self.output_dir + '-offl'
self.input_file = 'data-hists-offline.root'
self.args += self.input_file + ' '
if (self.reference == None):
# if no referenc found, use input file as reference - athout it doesn't matter - could use --noref
self.args += self.input_file + ' '
else:
self.args += self.ref_file + ' '
self.args += self.chains + ' '
self.args += ' -d ' + self.output_dir
print("TIDAComparitor " + self.args)
super(TrigInDetCompStep, self).configure(test)
def badChanInputHistoHelper():
# keep it as a function for now : might be easier to deal with when
# root-only config will be needed.
# new setup for THistService
dataPathList = os.environ['DATAPATH'].split(os.pathsep)
dataPathList.insert(0, os.curdir)
from AthenaCommon.Utils.unixtools import FindFile
RefFileName = FindFile("JetBadChanCorrTool.root", dataPathList, os.R_OK)
from AthenaCommon.AppMgr import ServiceMgr
if not hasattr(ServiceMgr, 'THistSvc'):
from GaudiSvc.GaudiSvcConf import THistSvc
ServiceMgr += THistSvc()
ServiceMgr.THistSvc.Input += [
"JetBadChanCorrTool DATAFILE=\'%s\' OPT=\'READ\'" % RefFileName
]
badChanInputHistoHelper.called = True
badChanInputHistoHelper()
from JetMomentTools.JetMomentToolsConf import JetBadChanCorrTool
from JetRec.JetRecStandard import jtm
jtm += JetBadChanCorrTool("JetBadChanCorrTool",
MissingCellMap="MissingCaloCellsMap")
#
# ========== Deal with calibration histogram files
#
# -- for analysis (should be in COOL now):
if BTaggingFlags.Runmodus == 'analysis':
if (BTaggingFlags.lifetime1D | BTaggingFlags.lifetime2D
| BTaggingFlags.lifetime3D | BTaggingFlags.secVtxFitBU
| BTaggingFlags.secVtxFitTD):
dataPathList = os.environ['DATAPATH'].split(os.pathsep)
dataPathList.insert(0, os.curdir)
from AthenaCommon.Utils.unixtools import FindFile
JetTagsigname = FindFile(
'all_Cone4TowerParticleJets_matchQuark_FastVertexFitter_sig.root',
dataPathList, os.R_OK)
JetTagbkgname = FindFile(
'all_Cone4TowerParticleJets_matchQuark_FastVertexFitter_bkg.root',
dataPathList, os.R_OK)
svcMgr.THistSvc.Input += [
"SigCali DATAFILE='" + JetTagsigname + "' OPT='OLD'"
]
svcMgr.THistSvc.Input += [
"BkgCali DATAFILE='" + JetTagbkgname + "' OPT='OLD'"
]
writeSmoothedHistos = False
if writeSmoothedHistos:
svcMgr.THistSvc.Output += [
"ControlFile DATAFILE='smoothcontrol.root' OPT='RECREATE'"
# Set the FilePrefix variable
# find the files
import string
dataPathList = os.environ['DATAPATH'].split(os.pathsep)
dataPathList.insert(0, os.curdir)
from AthenaCommon.Utils.unixtools import FindFile
filename = FindFile('prefix.txt', dataPathList, os.R_OK)
if filename is None:
print 'ERROR: SetFlags.py could not find list of file prefixes, prefix.txt'
text = open(filename, 'r')
for line in text.readlines():
words = string.split(line)
if len(words) >= 2:
Run = words[0]
if string.atoi(Run) == RunNumber:
FilePrefix = words[1]
text.close()
print 'Prefix ', FilePrefix
from JetTagTools.JetTagToolsConf import Analysis__EmulTag
Emul2D = Analysis__EmulTag(name="Emul2D", Tagger=["IP2D", "2d"])
Emul3D = Analysis__EmulTag(name="Emul3D", Tagger=["IP3D", "3d"])
EmulSV = Analysis__EmulTag(name="EmulSV", Tagger=["SV1IP3D", "6d"])
EmulOF = Analysis__EmulTag(name="EmulOF", Mode=1)
from AthenaCommon.AppMgr import ToolSvc
ToolSvc += Emul2D
ToolSvc += Emul3D
ToolSvc += EmulSV
ToolSvc += EmulOF
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
if not hasattr(svcMgr, 'THistSvc'):
from GaudiSvc.GaudiSvcConf import THistSvc
svcMgr += THistSvc()
import os
dataPathList = os.environ['DATAPATH'].split(os.pathsep)
dataPathList.insert(0, os.curdir)
from AthenaCommon.Utils.unixtools import FindFile
weightFile = FindFile("Weight.root", dataPathList, os.R_OK)
paramFile = FindFile("Param.root", dataPathList, os.R_OK)
svcMgr.THistSvc.Input += ["WeightFile DATAFILE='" + weightFile + "' OPT='OLD'"]
svcMgr.THistSvc.Input += ["ParamFile DATAFILE='" + paramFile + "' OPT='OLD'"]
from BTagging.BTaggingConf import Analysis__BJetBuilderFast
AtlfReTagging = Analysis__BJetBuilderFast(
name="AtlfReTagging", TagToolList=[Emul2D, Emul3D, EmulSV, EmulOF])
topSequence += AtlfReTagging
print AtlfReTagging
if not os.access('myInvWeight05BERT200.db',os.R_OK):
os.system('cp -f /afs/cern.ch/user/m/menke/public/AtlasOffline-14.2.21/myInvWeight05BERT200.db .')
if not os.access('myOOC05BERT200.db',os.R_OK):
os.system('cp -f /afs/cern.ch/user/m/menke/public/AtlasOffline-14.2.21/myOOC05BERT200.db .')
if not os.access('myOOCPi005BERT200.db',os.R_OK):
os.system('cp -f /afs/cern.ch/user/m/menke/public/AtlasOffline-14.2.21/myOOCPi005BERT200.db .')
if not os.access('myDBHadDMCoeff200.db',os.R_OK):
os.system('cp -f /afs/ipp-garching.mpg.de/home/g/gdp/public/HadDMCoeff2/myDBHadDMCoeff200.db .')
from AthenaCommon.Utils.unixtools import FindFile
localCondFile='PoolCat_oflcond.xml'
import os
dataPathList = os.environ['DATAPATH'].split(':')
for j in range(0,len(dataPathList)):
dataPathList[j] += '/poolcond/'
condFile=FindFile(localCondFile,dataPathList,os.R_OK)
if condFile:
os.system('mkdir poolcond; cp '+condFile+' poolcond/'+localCondFile)
for x in ['mc08.10741_0-1.singlepart_singlepiplusminus_logE.recon.ESD.e342_s439_14.2.21_outofcluster_athena.pool.root',
'mc08.10741_0-1.singlepart_singlepiplusminus_logE.recon.ESD.e342_s439_14.2.21_weights_from_inv_weights_athena.pool.root',
'mc08.10741_0-2.singlepart_singlepiplusminuszero_logE.recon.ESD.e342_s439_14.2.21_classify_noefrac_athena.pool.root',
'mc08.107412.singlepart_singlepizero_logE.recon.ESD.e342_s439_14.2.21_outofcluster_athena.pool.root']:
if not os.access(x,os.R_OK):
os.system('cp -f /afs/cern.ch/user/m/menke/public/AtlasOffline-14.2.21/'+x+' .')
os.system('pool_insertFileToCatalog -u xmlcatalog_file:poolcond/'+localCondFile+' '+x)
if not os.access('HadDMCoeff2_v21_1a.pool.root',os.R_OK):
os.system('cp -f /afs/ipp-garching.mpg.de/home/g/gdp/public/HadDMCoeff2/HadDMCoeff2_v21_1a.pool.root .')
os.system('pool_insertFileToCatalog -u xmlcatalog_file:poolcond/'+localCondFile+' HadDMCoeff2_v21_1a.pool.root')
else:
# change the DB for weights
def __call__( self, fn, *args, **kw ):
"""Include <fn> in the current scope by executing it globally."""
# some basic sanity check first
if isinstance(fn, str) and len(fn) == 0:
raise IncludeError("can not 'include' empty filenames")
# don't include file if not allowed (has to be exact match in py code)
if fn in self._once:
self.msg.debug( 'file "%s" is blocked; not included', fn )
return
# locate the file
name = FindFile( os.path.expanduser( os.path.expandvars( fn ) ), optionsPath, os.R_OK )
if not name:
name = FindFile( os.path.basename( fn ), optionsPath, os.R_OK )
if name:
self.msg.warning( 'using %s instead of %s', name, fn )
else:
raise IncludeError( 'include file %s can not be found' % fn )
self.msg.debug( 'located %s as %s', fn, name )
# print if 'show' is set to non-null
show = self._show
if 'show' in kw:
show = kw[ 'show' ]
# notify of start of file inclusion
if show:
self.msg.info( 'including file "%s" with ID %d', fn, self.fid )
else:
self.msg.info( 'including file "%s"', fn )
self._fcurrent = name
# actual inclusion
if show and self._doTrace( name ):
# traced
_filecache[ name ] = open( name, 'r' ).readlines()
_linecache[ name ] = 0, self.fid
self.fid += 1
from past.builtins import execfile
sys.settrace( self._trace_include )
execfile( name, self._workspace, self._workspace )
sys.settrace( sys._getframe(0).f_trace )
# finish file printout
ncur, fid = _linecache[ name ]
buf = _filecache[ name ]
for i in range( ncur, len(buf) ):
self._oneline( fid, i, silentMarker, buf )
del _filecache[ name ]
del _linecache[ name ]
self.msg.info( 'end of "%s"', fn )
else:
# non-traced
#execfile( name, self._workspace, self._workspace )
exec(compile(open(name).read(), name, 'exec'), self._workspace, self._workspace)
if hasattr( self, '_collect' ):
if not self._collect % 10:
import gc
gc.collect()
else:
self._collect += 1