def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
self.transientAttrs = set(transientAttrs) | {'_readVersion',}
self.toPublicAttrs = set(toPublicAttrs)
from RingerCore.Configure import checkForUnusedVars
checkForUnusedVars( kw, self._logger.warning )
def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
self.transientAttrs = set(transientAttrs) | {'_readVersion',}
self.toPublicAttrs = set(toPublicAttrs)
from RingerCore.util import checkForUnusedVars
checkForUnusedVars( kw, self._logger.warning )
def __init__(self, localCluster=None, **kw):
Logger.__init__(self, kw)
if localCluster in (None, NotSet):
localCluster = clusterManagerConf()
self.localCluster = AvailableManager.retrieve(localCluster)
if self.localCluster is ClusterManager.LSF:
self.prefix = LSFArgumentParser.prefix
prog = 'bsub'
elif self.localCluster is ClusterManager.PBS:
self.prefix = PBSJobArgumentParser.prefix
from socket import gethostname
prog = 'qsub'
# Work on environment
if "service1" in gethostname():
import re
THEANO_FLAGS = os.getenv('THEANO_FLAGS')
m = re.search('cxx=([^,]*)(,*)?', THEANO_FLAGS)
if m:
CXX_THEANO_FLAGS = m.group(1)
os.environ['THEANO_FLAGS'] = re.sub(
'cxx=[^,]*', "cxx=ssh service1 " + CXX_THEANO_FLAGS,
THEANO_FLAGS)
self._debug("Set THEANO_FLAGS to: %s",
os.getenv('THEANO_FLAGS'))
else:
self._warning("THEANO FLAGS continues equal to: %s",
os.getenv('THEANO_FLAGS'))
else:
self._fatal(
"Not implemented LocalClusterNamespace for cluster manager %s",
AvailableManager.retrieve(self.localCluster),
NotImplementedError)
JobSubmitNamespace.__init__(self, prog=prog)
def __setstate__(self, d):
"""
Add logger to object if it doesn't have one:
"""
v = d.pop('ignoreAttrs')
self.__dict__['ignoreAttrs'] = [re.compile(s) for s in v]
Logger.__setstate__(self,d)
def __init__(self, *args, **kw):
"""
Set values using min:incr:max and matlabFlag as explained in class
documentation.
"""
from RingerCore.Configure import checkForUnusedVars
Logger.__init__(self, kw)
self._matlabFlag = kw.pop('matlabFlag', False )
checkForUnusedVars( kw, self._warning )
del kw
if len(args) > 3:
raise ValueError("Input more than 3 values, format should be [min=0:[incr=1:]]max")
if len(args) == 0 or (args[0] in ([],tuple(),)):
# Create empty looping bounds
self._vec = [0,-1,1]
return
if isinstance(args[0], LoopingBounds ):
self._vec = args[0]._vec
oldFlag = self._matlabFlag
self._matlabFlag = args[0]._matlabFlag
if oldFlag != self._matlabFlag:
if self._matlabFlag:
self.turnMatlabFlagOn()
else:
self.turnMatlabFlagOff()
return
elif isinstance(args[0], list ):
self._vec = args[0]
else:
self._vec = list(args)
if len(self._vec) == 1:
self._vec.append( self._vec[0] + 1 )
if not self._matlabFlag:
self._vec[1] -= 1
self._vec[0] = 1 if self._matlabFlag else 0
elif len(self._vec) == 2:
if self._matlabFlag:
self._vec[1] += 1
else:
tmp = self._vec[1]
if self._matlabFlag:
if tmp > 0:
self._vec[1] = self._vec[2] + 1
else:
self._vec[1] = self._vec[2] - 1
else:
self._vec[1] = self._vec[2]
self._vec[2] = tmp
if len(self._vec) == 3 and self._vec[2] == 0:
raise ValueError("Attempted to create looping bounds without increment.")
def __init__(self, **kw): Logger.__init__(self, kw) self.__useFortran = kw.pop( 'useFortran', False ) self.order = 'F' if self.__useFortran else 'C' self.pdim = 0 if self.__useFortran else 1 self.odim = 1 if self.__useFortran else 0 self.fp_dtype = np.dtype( kw.pop( 'fp_dtype', np.float64 ) ) self.int_dtype = np.dtype( kw.pop( 'int_dtype', np.int64 ) ) self.scounter_dtype = np.dtype( kw.pop( 'scounter_dtype', np.uint8 ) ) self.flag_dtype = np.dtype( kw.pop( 'flag_dtype', np.int8 ) ) checkForUnusedVars(kw)
def __init__(self, **kw):
Logger.__init__(self, kw)
self.__useFortran = kw.pop('useFortran', False)
self.order = 'F' if self.__useFortran else 'C'
self.pdim = 0 if self.__useFortran else 1
self.odim = 1 if self.__useFortran else 0
self.fp_dtype = np.dtype(kw.pop('fp_dtype', np.float64))
self.int_dtype = np.dtype(kw.pop('int_dtype', np.int64))
self.scounter_dtype = np.dtype(kw.pop('scounter_dtype', np.uint8))
self.flag_dtype = np.dtype(kw.pop('flag_dtype', np.int8))
checkForUnusedVars(kw)
def __init__(self, outputFile, **kw):
Logger.__init__(self, kw)
if not outputFile.endswith(".root"):
outputFile += ".root"
# Create TFile object to hold everything
from ROOT import TFile
self._file = TFile(outputFile, "recreate")
self._currentDir = ""
self._objects = dict()
self._dirs = list()
def __init__(self, *args, **kw):
"""
Set values using min:incr:max and matlabFlag as explained in class
documentation.
"""
from RingerCore.util import checkForUnusedVars
Logger.__init__(self, kw)
self._matlabFlag = kw.pop('matlabFlag', False )
checkForUnusedVars( kw, self._logger.warning )
del kw
if len(args) > 3:
raise ValueError("Input more than 3 values, format should be [min=0:[incr=1:]]max")
if len(args) == 0:
raise ValueError("No input. Format should be [min=0,[incr=1,]]max")
if isinstance(args[0], LoopingBounds ):
self._vec = args[0]._vec
oldFlag = self._matlabFlag
self._matlabFlag = args[0]._matlabFlag
if oldFlag != self._matlabFlag:
if self._matlabFlag:
self.turnMatlabFlagOn()
else:
self.turnMatlabFlagOff()
return
elif isinstance(args[0], list ):
self._vec = args[0]
else:
self._vec = list(args)
if len(self._vec) == 1:
self._vec.append( self._vec[0] + 1 )
if not self._matlabFlag:
self._vec[1] -= 1
self._vec[0] = 1 if self._matlabFlag else 0
elif len(self._vec) == 2:
if self._matlabFlag:
self._vec[1] += 1
else:
tmp = self._vec[1]
if self._matlabFlag:
if tmp > 0:
self._vec[1] = self._vec[2] + 1
else:
self._vec[1] = self._vec[2] - 1
else:
self._vec[1] = self._vec[2]
self._vec[2] = tmp
if len(self._vec) == 3 and self._vec[2] == 0:
raise ValueError("Attempted to create looping bounds without increment.")
def sourceEnvFile():
"""
Emulate source new_env_file.sh on python environment.
"""
try:
from RingerCore.Logger import Logger
logger = Logger.getModuleLogger(__name__)
import os, sys
global loadedEnvFile
if not loadedEnvFile:
with open(os.path.expandvars('$ROOTCOREBIN/../FastNetTool/cmt/new_env_file.sh'),'r') as f:
lines = f.readlines()
lineparser = re.compile(r'test "\$\{(?P<shellVar>[A-Z1-9]*)#\*(?P<addedPath>\S+)\}" = "\$\{(?P=shellVar)\}" && export (?P=shellVar)=\$(?P=shellVar):(?P=addedPath) || true')
for line in lines:
m = lineparser.match(line)
if m:
shellVar = m.group('shellVar')
if shellVar != 'PYTHONPATH':
continue
addedPath = os.path.expandvars(m.group('addedPath'))
if not addedPath:
logger.warning("Couldn't retrieve added path on line \"%s\".", line)
continue
if not os.path.exists(addedPath):
logger.warning("Couldn't find following path \"%s\".", addedPath)
continue
if not addedPath in os.environ[shellVar]:
sys.path.append(addedPath)
logger.info("Successfully added path: \"%s\".", line)
loadedEnvFile=True
except IOError:
raise RuntimeError("Cannot find new_env_file.sh, did you forget to set environment or compile the package?")
def __init__(self, ignoreAttrs = set(), toProtectedAttrs = set(), ignoreRawChildren = False, **kw ):
"""
-> ignoreAttrs: not consider this attributes on the dictionary values.
-> toProtectedAttrs: change public attributes to protected or private
attributes. That is, suppose the dictionary value is 'val' and the class
value should be _val or __val, then add toProtectedAttrs = ['_val'] or
'__val'.
-> ignoreRawChildren: Do not attempt to conver raw children to higher level object.
"""
Logger.__init__(self, kw)
ignoreAttrs = list(set(ignoreAttrs) | RawDictCnv.baseAttrs)
import re
self.ignoreAttrs = [re.compile(ignoreAttr) for ignoreAttr in ignoreAttrs]
self.toProtectedAttrs = set(toProtectedAttrs)
self.ignoreRawChildren = ignoreRawChildren
from RingerCore.Configure import checkForUnusedVars
checkForUnusedVars( kw, self._logger.warning )
def __init__(self, ignoreAttrs = set(), toProtectedAttrs = set(), ignoreRawChildren = False, **kw ):
"""
-> ignoreAttrs: not consider this attributes on the dictionary values.
-> toProtectedAttrs: change public attributes to protected or private
attributes. That is, suppose the dictionary value is 'val' and the class
value should be _val or __val, then add toProtectedAttrs = ['_val'] or
'__val'.
-> ignoreRawChildren: Do not attempt to conver raw children to higher level object.
"""
Logger.__init__(self, kw)
ignoreAttrs = list(set(ignoreAttrs) | RawDictCnv.baseAttrs)
import re
self.ignoreAttrs = [re.compile(ignoreAttr) for ignoreAttr in ignoreAttrs]
self.toProtectedAttrs = set(toProtectedAttrs)
self.ignoreRawChildren = ignoreRawChildren
from RingerCore.util import checkForUnusedVars
checkForUnusedVars( kw, self._logger.warning )
def __init__(self, d={}, **kw):
if None in self._contextManager._acceptedTypes:
self._contextManager._acceptedTypes = TexObject,
if (not isinstance(self, TexObjectCollection)
and not hasattr(self, '_preamble')
and not hasattr(self, '_enclosure')
and not hasattr(self, '_body')
and not hasattr(self, '_footer')
and not hasattr(self, '_appendix')):
raise TexException(
self, 'Class %s does not write any tex code.' %
self.__class__.__name__)
d.update(kw)
Logger.__init__(self, d)
if hasattr(self, '_body'):
self._body = formatTex(self._body, retrieve_kw(d, 'textWidth', 80))
self._stream = kw.pop('stream', tss)
self._keywords = {
key: val
for key, val in d.iteritems() if not key.startswith('_')
}
self._keywords.update({
key: val
for key, val in self.__dict__.iteritems()
if not key.startswith('_')
})
if 'star' in self._keywords and self._keywords['star']:
self._keywords['star'] = '*'
else:
self._keywords['star'] = ''
if hasattr(self, '_assertVars'):
for key in self._assertVars:
if not key in self._keywords:
raise TexException(self, "Assert var %s failed." % key)
gcc.set(self)
self._contextManaged = d.pop('_contextManaged', True)
self._context = self._contextManager()
self._isInContext = self._context is not None
if (self._isInContext
and isinstance(self._context, TexObjectCollection)
and self._contextManaged):
self._context += self
if self._isInContext:
self._stream = self._context._stream
def __init__(self, prog=None, **kw):
Logger.__init__(self, kw)
if prog is not None:
self.prog = prog
else:
try:
self.prog = self.__class__.prog
except AttributeError:
raise AttributeError(
"Not specified class (%s) prog attribute!" %
self.__class__.__name__)
if not hasattr(self, 'prefix'):
try:
self.prefix = self.__class__.ParserClass.prefix
except AttributeError:
raise AttributeError(
"Not specified class (%s) ParserClass attribute!" %
self.__class__.__name__)
argparse.Namespace.__init__(self)
self.fcn = os.system
def __init__(self, tes, **kw):
Logger.__init__( self, **kw )
import logging
self.baseDir = kw.pop('baseDir','')
self.bucketName = kw.pop('bucketName','')
self.pidName = kw.pop('pidName','tight')
self._level = kw.pop('level',logging.INFO)
self.appName = kw.pop('appName','TrigAnalysisTool')
checkForUnusedVars(kw)
del kw
self._logger.info('Start wrapper between python and c++ core.')
self._trigAnalysisTool = ROOT.TrigAnalysisTool(self.appName,
self.bucketName,
self.baseDir,
self.pidName,
self._level)
for te in tes:
self._trigAnalysisTool.push_back( te() )
self._logger.info('TrigAnalysisTool has been created.' )
def watchLock(filename):
logger = Logger.getModuleLogger("watchLock")
lockFileName = os.path.join(
os.path.join(os.path.dirname(filename),
'.' + os.path.basename(filename) + '.lock'))
firstMsg = True
while os.path.exists(lockFileName):
if firstMsg:
logger.warning("Waiting other process to unlock file %s...",
lockFileName)
firstMsg = False
sleep(1)
lockFile = LockFile(lockFileName)
return lockFile
def __getstate__(self):
"""
Makes logger invisible for pickle
"""
odict = Logger.__getstate__(self)
#def getStr(i):
# if isinstance(i, re
if 'ignoreAttrs' in odict:
s = odict['ignoreAttrs']
def getStr(c):
try:
return c.pattern
except AttributeError:
return c
odict['ignoreAttrs'] = [getStr(v) for v in s]
return odict
def sourceEnvFile():
"""
Emulate source new_env_file.sh on python environment.
"""
try:
from RingerCore.Logger import Logger
logger = Logger.getModuleLogger(__name__)
import os, sys
global loadedEnvFile
if not loadedEnvFile:
with open(
os.path.expandvars(
'$ROOTCOREBIN/../FastNetTool/cmt/new_env_file.sh'),
'r') as f:
lines = f.readlines()
lineparser = re.compile(
r'test "\$\{(?P<shellVar>[A-Z1-9]*)#\*(?P<addedPath>\S+)\}" = "\$\{(?P=shellVar)\}" && export (?P=shellVar)=\$(?P=shellVar):(?P=addedPath) || true'
)
for line in lines:
m = lineparser.match(line)
if m:
shellVar = m.group('shellVar')
if shellVar != 'PYTHONPATH':
continue
addedPath = os.path.expandvars(m.group('addedPath'))
if not addedPath:
logger.warning(
"Couldn't retrieve added path on line \"%s\".",
line)
continue
if not os.path.exists(addedPath):
logger.warning(
"Couldn't find following path \"%s\".",
addedPath)
continue
if not addedPath in os.environ[shellVar]:
sys.path.append(addedPath)
logger.info("Successfully added path: \"%s\".",
line)
loadedEnvFile = True
except IOError:
raise RuntimeError(
"Cannot find new_env_file.sh, did you forget to set environment or compile the package?"
)
__all__ = ['RawDictStreamable', 'RawDictStreamer', 'RawDictCnv', 'mangle_attr',
'LoggerStreamable', 'LoggerRawDictStreamer', 'checkAttrOrSetDefault',
'isRawDictFormat', 'retrieveRawDict']
from RingerCore.Logger import Logger
mLogger = Logger.getModuleLogger( __name__ )
def mangle_attr(source, attr):
"""
Simulate python private attritubutes mangling. Taken from:
http://stackoverflow.com/a/7789483/1162884
"""
# return public attrs unchanged
if not attr.startswith("__") or attr.endswith("__") or '.' in attr:
return attr
# if source is an object, get the class
if not hasattr(source, "__bases__"):
source = source.__class__
# mangle attr
return "_%s%s" % (source.__name__.lstrip("_"), attr)
class RawDictStreamer( Logger ):
"""
This is the default streamer class, responsible of converting python classes
to raw dictionaries.
"""
def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
def _print_message(self, message, file=None):
from RingerCore.Logger import Logger
logger = Logger.getModuleLogger(self.__class__.__name__)
if message:
logger.info('\n' + message)
def __init__(self, d = {}, **kw):
Logger.__init__(self, d, **kw)
def __init__(self, outputFile):
Logger.__init__(self)
from RingerCore.FileIO import ensureExtension
if not outputFile:
raise TexException(self, 'Cannot stream to empty file path.')
self.outputFile = ensureExtension(outputFile, self._outputExtension)
# Make outputs not usable by user
parser.add_argument('--outputs', action='store_const',
required = False, default = '"tuningData*"', const = '"tuningData*"',
dest = 'grid_outputs',
help = argparse.SUPPRESS )
# Force secondary to be reusable:
parser.add_argument('--reusableSecondary', action='store_const',
required = False, default = 'BKG', const = 'BKG', dest = 'grid_reusableSecondary',
help = """Allow reuse secondary dataset.""")
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
from RingerCore.Logger import Logger
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Treat special argument
if len(args.reference) > 2:
mainLogger.fatal("--reference set to multiple values: %r" % args.reference, ValueError)
if len(args.reference) is 1:
args.reference.append( args.reference[0] )
if args.operation != 'Offline' and not args.treePath:
mainLogger.fatal("If operation is not set to Offline, it is needed to set the TreePath manually.")
if ( len(args.inDS_BKG) > 1 or len(args.grid_inDS) > 1):
mainLogger.fatal("Cannot use multiple datasets in this version.", NotImplementedError)
from subprocess import check_output
# TODO For now we only support one container for signal and one for background.
# We need to change it so that we can input multiple containers. This can be
def __init__(self, prog = 'prun', **kw): Logger.__init__( self, kw ) LoggerNamespace.__init__( self, **kw ) self.prog = prog
__all__ = ['RawDictStreamable', 'RawDictStreamer', 'RawDictCnv', 'mangle_attr',
'LoggerStreamable', 'LoggerRawDictStreamer', 'checkAttrOrSetDefault',
'isRawDictFormat', 'retrieveRawDict']
from RingerCore.Logger import Logger
mLogger = Logger.getModuleLogger( __name__ )
def mangle_attr(source, attr):
"""
Simulate python private attritubutes mangling. Taken from:
http://stackoverflow.com/a/7789483/1162884
"""
# return public attrs unchanged
if not attr.startswith("__") or attr.endswith("__") or '.' in attr:
return attr
# if source is an object, get the class
if not hasattr(source, "__bases__"):
source = source.__class__
# mangle attr
return "_%s%s" % (source.__name__.lstrip("_"), attr)
class RawDictStreamer( Logger ):
"""
This is the default streamer class. Overload this method to deal with
special cases.
"""
def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
def __init__(self, d = {}, **kw):
Logger.__init__(self, d, **kw)
def __init__(self, **kw):
self._choice = NotSet
if not hasattr(self, 'name'):
self.name = self.__class__.__name__.lstrip('_')
self.name = self.name.replace('Configure', '')
Logger.__init__(self, kw, logName=self.name)
dest = 'grid_outputs',
help = argparse.SUPPRESS )
# Force secondary to be reusable:
parser.add_argument('--reusableSecondary', action='store_const',
required = False, default = 'BKG', const = 'BKG', dest = 'grid_reusableSecondary',
help = """Allow reuse secondary dataset.""")
import sys
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
# Retrieve parser args:
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
from RingerCore.Logger import Logger
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Treat special argument
if len(args.reference) > 2:
mainLogger.fatal("--reference set to multiple values: %r" % args.reference, ValueError)
if len(args.reference) is 1:
args.reference.append( args.reference[0] )
if args.operation != 'Offline' and not args.treePath:
mainLogger.fatal("If operation is not set to Offline, it is needed to set the TreePath manually.")
if ( len(args.inDS_BKG) > 1 or len(args.grid_inDS) > 1):
mainLogger.fatal("Cannot use multiple datasets in this version.", NotImplementedError)
from subprocess import check_output
# TODO For now we only support one container for signal and one for background.
# We need to change it so that we can input multiple containers. This can be
from RingerCore.Logger import Logger, LoggingLevel
from TuningTools.TuningJob import TuningJob
from TuningTools.PreProc import *
from TuningTools.TuningJob import fixPPCol
from TuningTools.coreDef import coreConf, TuningToolCores
from TuningTools.TuningJob import ReferenceBenchmark, ReferenceBenchmarkCollection, BatchSizeMethod
from RingerCore.Configure import Development
import logging
import argparse
start = timer()
Development.set(True)
coreConf.set(TuningToolCores.keras)
#coreConf.set(TuningToolCores.FastNet)
mainLogger = Logger.getModuleLogger("job")
parser = argparse.ArgumentParser(description='', add_help=False)
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--data',
action='store',
dest='data',
required=True,
help="The input tuning files.")
parser.add_argument('-g',
'--gpu',
action='store',
dest='gpu',
required=False,