def sourceEnvFile():
"""
Emulate source new_env_file.sh on python environment.
"""
try:
from RingerCore.Logger import Logger
logger = Logger.getModuleLogger(__name__)
import os, sys
global loadedEnvFile
if not loadedEnvFile:
with open(os.path.expandvars('$ROOTCOREBIN/../FastNetTool/cmt/new_env_file.sh'),'r') as f:
lines = f.readlines()
lineparser = re.compile(r'test "\$\{(?P<shellVar>[A-Z1-9]*)#\*(?P<addedPath>\S+)\}" = "\$\{(?P=shellVar)\}" && export (?P=shellVar)=\$(?P=shellVar):(?P=addedPath) || true')
for line in lines:
m = lineparser.match(line)
if m:
shellVar = m.group('shellVar')
if shellVar != 'PYTHONPATH':
continue
addedPath = os.path.expandvars(m.group('addedPath'))
if not addedPath:
logger.warning("Couldn't retrieve added path on line \"%s\".", line)
continue
if not os.path.exists(addedPath):
logger.warning("Couldn't find following path \"%s\".", addedPath)
continue
if not addedPath in os.environ[shellVar]:
sys.path.append(addedPath)
logger.info("Successfully added path: \"%s\".", line)
loadedEnvFile=True
except IOError:
raise RuntimeError("Cannot find new_env_file.sh, did you forget to set environment or compile the package?")
def watchLock(filename):
logger = Logger.getModuleLogger("watchLock")
lockFileName = os.path.join(
os.path.join(os.path.dirname(filename),
'.' + os.path.basename(filename) + '.lock'))
firstMsg = True
while os.path.exists(lockFileName):
if firstMsg:
logger.warning("Waiting other process to unlock file %s...",
lockFileName)
firstMsg = False
sleep(1)
lockFile = LockFile(lockFileName)
return lockFile
def sourceEnvFile():
"""
Emulate source new_env_file.sh on python environment.
"""
try:
from RingerCore.Logger import Logger
logger = Logger.getModuleLogger(__name__)
import os, sys
global loadedEnvFile
if not loadedEnvFile:
with open(
os.path.expandvars(
'$ROOTCOREBIN/../FastNetTool/cmt/new_env_file.sh'),
'r') as f:
lines = f.readlines()
lineparser = re.compile(
r'test "\$\{(?P<shellVar>[A-Z1-9]*)#\*(?P<addedPath>\S+)\}" = "\$\{(?P=shellVar)\}" && export (?P=shellVar)=\$(?P=shellVar):(?P=addedPath) || true'
)
for line in lines:
m = lineparser.match(line)
if m:
shellVar = m.group('shellVar')
if shellVar != 'PYTHONPATH':
continue
addedPath = os.path.expandvars(m.group('addedPath'))
if not addedPath:
logger.warning(
"Couldn't retrieve added path on line \"%s\".",
line)
continue
if not os.path.exists(addedPath):
logger.warning(
"Couldn't find following path \"%s\".",
addedPath)
continue
if not addedPath in os.environ[shellVar]:
sys.path.append(addedPath)
logger.info("Successfully added path: \"%s\".",
line)
loadedEnvFile = True
except IOError:
raise RuntimeError(
"Cannot find new_env_file.sh, did you forget to set environment or compile the package?"
)
__all__ = ['RawDictStreamable', 'RawDictStreamer', 'RawDictCnv', 'mangle_attr',
'LoggerStreamable', 'LoggerRawDictStreamer', 'checkAttrOrSetDefault',
'isRawDictFormat', 'retrieveRawDict']
from RingerCore.Logger import Logger
mLogger = Logger.getModuleLogger( __name__ )
def mangle_attr(source, attr):
"""
Simulate python private attritubutes mangling. Taken from:
http://stackoverflow.com/a/7789483/1162884
"""
# return public attrs unchanged
if not attr.startswith("__") or attr.endswith("__") or '.' in attr:
return attr
# if source is an object, get the class
if not hasattr(source, "__bases__"):
source = source.__class__
# mangle attr
return "_%s%s" % (source.__name__.lstrip("_"), attr)
class RawDictStreamer( Logger ):
"""
This is the default streamer class. Overload this method to deal with
special cases.
"""
def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
from RingerCore.Logger import Logger, LoggingLevel
from TuningTools.TuningJob import TuningJob
from TuningTools.PreProc import *
from TuningTools.TuningJob import fixPPCol
from TuningTools.coreDef import coreConf, TuningToolCores
from TuningTools.TuningJob import ReferenceBenchmark, ReferenceBenchmarkCollection, BatchSizeMethod
from RingerCore.Configure import Development
import logging
import argparse
start = timer()
Development.set(True)
coreConf.set(TuningToolCores.keras)
#coreConf.set(TuningToolCores.FastNet)
mainLogger = Logger.getModuleLogger("job")
parser = argparse.ArgumentParser(description='', add_help=False)
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--data',
action='store',
dest='data',
required=True,
help="The input tuning files.")
parser.add_argument('-g',
'--gpu',
action='store',
dest='gpu',
required=False,
# Make outputs not usable by user
parser.add_argument('--outputs', action='store_const',
required = False, default = '"tuningData*"', const = '"tuningData*"',
dest = 'grid_outputs',
help = argparse.SUPPRESS )
# Force secondary to be reusable:
parser.add_argument('--reusableSecondary', action='store_const',
required = False, default = 'BKG', const = 'BKG', dest = 'grid_reusableSecondary',
help = """Allow reuse secondary dataset.""")
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
from RingerCore.Logger import Logger
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Treat special argument
if len(args.reference) > 2:
mainLogger.fatal("--reference set to multiple values: %r" % args.reference, ValueError)
if len(args.reference) is 1:
args.reference.append( args.reference[0] )
if args.operation != 'Offline' and not args.treePath:
mainLogger.fatal("If operation is not set to Offline, it is needed to set the TreePath manually.")
if ( len(args.inDS_BKG) > 1 or len(args.grid_inDS) > 1):
mainLogger.fatal("Cannot use multiple datasets in this version.", NotImplementedError)
from subprocess import check_output
# TODO For now we only support one container for signal and one for background.
# We need to change it so that we can input multiple containers. This can be
dest = 'grid_outputs',
help = argparse.SUPPRESS )
# Force secondary to be reusable:
parser.add_argument('--reusableSecondary', action='store_const',
required = False, default = 'BKG', const = 'BKG', dest = 'grid_reusableSecondary',
help = """Allow reuse secondary dataset.""")
import sys
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
# Retrieve parser args:
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
from RingerCore.Logger import Logger
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Treat special argument
if len(args.reference) > 2:
mainLogger.fatal("--reference set to multiple values: %r" % args.reference, ValueError)
if len(args.reference) is 1:
args.reference.append( args.reference[0] )
if args.operation != 'Offline' and not args.treePath:
mainLogger.fatal("If operation is not set to Offline, it is needed to set the TreePath manually.")
if ( len(args.inDS_BKG) > 1 or len(args.grid_inDS) > 1):
mainLogger.fatal("Cannot use multiple datasets in this version.", NotImplementedError)
from subprocess import check_output
# TODO For now we only support one container for signal and one for background.
# We need to change it so that we can input multiple containers. This can be
def _print_message(self, message, file=None):
from RingerCore.Logger import Logger
logger = Logger.getModuleLogger(self.__class__.__name__)
if message:
logger.info('\n' + message)
__all__ = ['RawDictStreamable', 'RawDictStreamer', 'RawDictCnv', 'mangle_attr',
'LoggerStreamable', 'LoggerRawDictStreamer', 'checkAttrOrSetDefault',
'isRawDictFormat', 'retrieveRawDict']
from RingerCore.Logger import Logger
mLogger = Logger.getModuleLogger( __name__ )
def mangle_attr(source, attr):
"""
Simulate python private attritubutes mangling. Taken from:
http://stackoverflow.com/a/7789483/1162884
"""
# return public attrs unchanged
if not attr.startswith("__") or attr.endswith("__") or '.' in attr:
return attr
# if source is an object, get the class
if not hasattr(source, "__bases__"):
source = source.__class__
# mangle attr
return "_%s%s" % (source.__name__.lstrip("_"), attr)
class RawDictStreamer( Logger ):
"""
This is the default streamer class, responsible of converting python classes
to raw dictionaries.
"""
def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
