def __init__(self, w, b, **kwargs): Logger.__init__(self) self.layer = retrieve_kw(kwargs, 'Layer' ,0 ) self.funcName = retrieve_kw(kwargs, 'funcName' ,None ) checkForUnusedVars(kwargs) self.W = np.matrix(w) self.b = np.transpose( np.matrix(b) )
def __init__(self, rawDict, logger=None):
Logger.__init__(self, logger=logger)
self._u = 0.0
try: #Retrieve nodes information
self._etbin = rawDict['configuration']['etBin']
self._etabin= rawDict['configuration']['etaBin']
self._nodes = rawDict['discriminator']['nodes']
w = rawDict['discriminator']['weights']
b = rawDict['discriminator']['bias']
except KeyError:
self._logger.fatal("Not be able to retrieve the params from the dictionary")
self._nLayers = len(self._nodes)-1
self._layers = list()
for layer in range(self._nLayers):
W=[]; B=[]
for count in range(self._nodes[layer]*self._nodes[layer+1]):
W.append(w.pop(0))
W=np.array(W).reshape(self._nodes[layer+1], self._nodes[layer]).tolist()
for count in range(self._nodes[layer+1]):
B.append(b.pop(0))
# Create hidden layers
self._layers.append( Layer(W,B,Layer=layer,funcName='tansig') )
def __init__( self, outputFile, **kw ):
Logger.__init__(self,kw)
if not outputFile.endswith('.root'):
outputFile += '.root'
# Use this property to rebuild the storegate from a root file
self._restoreStoreGate=retrieve_kw(kw,'restoreStoreGate',False)
filterDirs=retrieve_kw(kw,'filterDirs', None)
#Create TFile object to hold everything
from ROOT import TFile
outputFile = expandPath( outputFile )
if self._restoreStoreGate:
import os.path
if not os.path.exists( outputFile ):
raise ValueError("File '%s' does not exist" % outputFile)
self._file = TFile( outputFile, "read")
else:
self._file = TFile( outputFile, "recreate")
self._currentDir = ""
self._objects = dict()
self._dirs = list()
import os
self._outputFile = os.path.abspath(outputFile)
if self._restoreStoreGate:
retrievedObjs = self.__restore(self._file, filterDirs=filterDirs)
for name, obj in retrievedObjs:
self._dirs.append(name)
self._objects[name]=obj
def __init__( self, logger = None ):
"""
Load TuningTools C++ library and set logger
"""
# Retrieve python logger
Logger.__init__( self, logger = logger)
self._store = None
def __init__(self, fileList, logger=None):
Logger.__init__(self, logger=logger)
from RingerCore import StoreGate
for f in fileList:
self._store.append(StoreGate(f, restoreStoreGate=True))
paths = self._store[-1].getDirs()
#TODO: this shold be a property
# Only for egamma trigger staff
def check_egamma_paths(p):
paths = []
for s in p:
if 'Egamma' in s:
if not 'icalomedium' in s:
paths.append(s)
return paths
# filter egamma only
paths = check_egamma_paths(paths)
resume1 = self.resume(paths, 'Efficiency')
try:
resume2 = self.resume(paths, 'Emulation')
except:
resume2 = None
self._logger.warning(
'There is no emulation dirname in this file.')
# hold the resume paths
self._resume.append({'Efficiency': resume1, 'Emulation': resume2})
# display all chains
self._effReader = Efficiency()
def __init__(self):
Logger.__init__(self)
try:# Check if rucio was set
import os
os.system('rucio --version')
except RuntimeError:
self._fatal('Rucio was not set! please setup this!')
raise RuntimeError('Rucio command not found!')
def __init__(self, w, b, **kw):
Logger.__init__(self, kw)
self.layer = kw.pop("Layer", 0)
self.func = kw.pop("Func", "tansig")
checkForUnusedVars(kw, self._logger.warning)
del kw
self.W = np.matrix(w)
self.b = np.transpose(np.matrix(b))
def __init__(self):
Logger.__init__(self)
try:# Check if rucio was set
import os
os.system('rucio --version')
except RuntimeError:
self._logger.fatal('Rucio was not set! please setup this!')
raise RuntimeError('Rucio command not found!')
def __init__( self, **kw ):
Logger.__init__( self, kw )
self.references = ReferenceBenchmarkCollection( [] )
self.doPerf = retrieve_kw( kw, 'doPerf', True )
self.batchMethod = BatchSizeMethod.retrieve(
retrieve_kw( kw, 'batchMethod', BatchSizeMethod.MinClassSize \
if not 'batchSize' in kw or kw['batchSize'] is NotSet else BatchSizeMethod.Manual )
)
self.batchSize = retrieve_kw( kw, 'batchSize', NotSet )
epochs = retrieve_kw( kw, 'epochs', 10000 )
maxFail = retrieve_kw( kw, 'maxFail', 50 )
self.useTstEfficiencyAsRef = retrieve_kw( kw, 'useTstEfficiencyAsRef', False )
self._core, self._coreEnum = retrieve_core()
self.sortIdx = None
if self._coreEnum is TuningToolCores.ExMachina:
self.trainOptions = dict()
self.trainOptions['algorithmName'] = retrieve_kw( kw, 'algorithmName', 'rprop' )
self.trainOptions['print'] = retrieve_kw( kw, 'showEvo', True )
self.trainOptions['networkArch'] = retrieve_kw( kw, 'networkArch', 'feedforward' )
self.trainOptions['costFunction'] = retrieve_kw( kw, 'costFunction', 'sp' )
self.trainOptions['shuffle'] = retrieve_kw( kw, 'shuffle', True )
self.trainOptions['nEpochs'] = epochs
self.trainOptions['nFails'] = maxFail
self.doMultiStop = False
elif self._coreEnum is TuningToolCores.FastNet:
seed = retrieve_kw( kw, 'seed', None )
self._core = self._core( level = LoggingLevel.toC(self.level), seed = seed )
self._core.trainFcn = retrieve_kw( kw, 'algorithmName', 'trainrp' )
self._core.showEvo = retrieve_kw( kw, 'showEvo', 50 )
self._core.multiStop = retrieve_kw( kw, 'doMultiStop', True )
self._core.epochs = epochs
self._core.maxFail = maxFail
else:
self._logger.fatal("TuningWrapper not implemented for %s" % TuningToolCores.tostring(self._coreEnum))
checkForUnusedVars(kw, self._logger.debug )
del kw
# Set default empty values:
if self._coreEnum is TuningToolCores.ExMachina:
self._emptyData = npCurrent.fp_array([])
elif self._coreEnum is TuningToolCores.FastNet:
self._emptyData = list()
self._emptyHandler = None
if self._coreEnum is TuningToolCores.ExMachina:
self._emptyTarget = npCurrent.fp_array([[]]).reshape(
npCurrent.access( pidx=1,
oidx=0 ) )
elif self._coreEnum is TuningToolCores.FastNet:
self._emptyTarget = None
# Set holders:
self._trnData = self._emptyData
self._valData = self._emptyData
self._tstData = self._emptyData
self._trnHandler = self._emptyHandler
self._valHandler = self._emptyHandler
self._tstHandler = self._emptyHandler
self._trnTarget = self._emptyTarget
self._valTarget = self._emptyTarget
self._tstTarget = self._emptyTarget
def __init__(self, name):
Logger.__init__(self)
self._name = name
self._basepath = str()
self._wtd = StatusWatchDog.DISABLE
self._status = StatusTool.ENABLE
self._initialized = StatusTool.NOT_INITIALIZED
self._finalized = StatusTool.NOT_FINALIZED
self._ids = []
def __init__(self, outputDS):
Logger.__init__(self)
from ROOT import TFile
# Check if the type is correct
if not '.root' in outputDS: outputDS += '.root'
self._file = TFile(outputDS, 'recreate')
self._outputDS = outputDS
self._logger.info( ('Create root file with name: %s') \
%(outputDS) )
def __init__(self, **kw):
#Retrive python logger
logger = kw.pop('logger', None)
self._label = kw.pop('label', '')
Logger.__init__(self, logger=logger)
self._obj = []
self._idxCorr = None
self.best = 0
self.worst = 0
def __init__(self, outputDS):
Logger.__init__(self)
from ROOT import TFile
# Check if the type is correct
if not '.root' in outputDS: outputDS+='.root'
self._file = TFile( outputDS, 'recreate' )
self._outputDS=outputDS
self._logger.info( ('Create root file with name: %s') \
%(outputDS) )
def __init__( self, data, target, batch_size=32, secondaryPP=None, shuffle=True, **kw): super(Sequence, self).__init__() Logger.__init__(self,**kw) self._data = data self._target = target self.batch_size = batch_size self._secondaryPP=secondaryPP self.shuffle = shuffle self.on_epoch_end()
def __init__(self, crossvalFileName, monFileName, **kw):
from ROOT import TFile, gROOT
gROOT.ProcessLine("gErrorIgnoreLevel = kFatal;");
#Set all global setting from ROOT style plot!
SetTuningStyle()
Logger.__init__(self, kw)
#Hold all information abount the monitoring root file
self._infoObjs = list()
try:#Protection
self._logger.info('Reading monRootFile (%s)',monFileName)
self._rootObj = TFile(monFileName, 'read')
except RuntimeError:
self._logger.fatal('Could not open root monitoring file.')
from RingerCore import load
try:#Protection
self._logger.info('Reading crossvalFile (%s)',crossvalFileName)
crossvalObj = load(crossvalFileName)
except RuntimeError:
self._logger.fatal('Could not open pickle summary file.')
#Loop over benchmarks
for benchmarkName in crossvalObj.keys():
#Must skip if ppchain collector
if benchmarkName == 'infoPPChain': continue
#Add summary information into MonTuningInfo helper class
self._infoObjs.append( TuningMonitoringInfo( benchmarkName, crossvalObj[benchmarkName] ) )
self._logger.info('Creating MonTuningInfo for %s and the iterator object [et=%d, eta=%d]',
benchmarkName,self._infoObjs[-1].etbin(), self._infoObjs[-1].etabin())
#Loop over all benchmarks
#Always be the same bin for all infoObjs
etabin = self._infoObjs[0].etabin()
etbin = self._infoObjs[0].etbin()
#Reading the data rings from path or object
refFile = kw.pop('refFile', None)
if refFile:
from TuningTools import TuningDataArchieve
TDArchieve = TuningDataArchieve(refFile)
self._logger.info(('Reading perf file with name %s')%(refFile))
try:
with TDArchieve as data:
self._data = (data['signal_patterns'][etbin][etabin], data['background_patterns'][etbin][etabin])
except RuntimeError:
self._logger.fatal('Could not open the patterns data file.')
raise RuntimeError('Could not open the patterns data file.')
else:
patterns = kw.pop('patterns', None)
if patterns:
self._data = (patterns['signal_patterns'][etbin][etabin], patterns['background_patterns'][etbin][etabin])
else:
raise RuntimeError('You must pass the ref file as parameter. abort!')
def __init__(self, filePath = None, **kw):
"""
Either specify the file path where the file should be read or the data
which should be appended to it
with SubsetGeneratorArchieve("/path/to/file") as data:
BLOCK
SubsetGeneratorArchieve( "file/path", subsetCol= SubsetGeneratorPatternCollection([...]) )
"""
Logger.__init__(self, kw)
self._filePath = filePath
self._subsetCol = kw.pop( 'subsetCol', None )
checkForUnusedVars( kw, self._warning )
def __init__(self, **kw):
Logger.__init__(self, **kw)
self._maxPileupLinearCorrectionValue = retrieve_kw(
kw, 'maxPileupLinearCorrectionValue', 100)
self._removeOutputTansigTF = retrieve_kw(kw, 'removeOutputTansigTF',
True)
self._useEtaVar = retrieve_kw(kw, 'useEtaVar', False)
self._useLumiVar = retrieve_kw(kw, 'useLumiVar', False)
self._toPython = retrieve_kw(kw, 'toPython', True)
self._doPileupCorrection = retrieve_kw(kw, 'doPileupCorrection', True)
def __init__(self, crossvalFileName, monFileName, **kw):
from ROOT import TFile, gROOT
gROOT.ProcessLine("gErrorIgnoreLevel = kFatal;")
#Set all global setting from ROOT style plot!
SetTuningStyle()
Logger.__init__(self, kw)
#Hold all information abount the monitoring root file
self._infoObjs = list()
try: #Protection
self._logger.info('Reading monRootFile (%s)', monFileName)
self._rootObj = TFile(monFileName, 'read')
except RuntimeError:
self._logger.fatal('Could not open root monitoring file.')
from RingerCore import load
try: #Protection
self._logger.info('Reading crossvalFile (%s)', crossvalFileName)
crossvalObj = load(crossvalFileName)
except RuntimeError:
self._logger.fatal('Could not open pickle summary file.')
#Loop over benchmarks
for benchmarkName in crossvalObj.keys():
#Must skip if ppchain collector
if benchmarkName == 'infoPPChain': continue
#Add summary information into MonTuningInfo helper class
self._infoObjs.append(
TuningMonitoringInfo(benchmarkName,
crossvalObj[benchmarkName]))
self._logger.info(
'Creating MonTuningInfo for %s and the iterator object [et=%d, eta=%d]',
benchmarkName, self._infoObjs[-1].etbinidx(),
self._infoObjs[-1].etabinidx())
#Loop over all benchmarks
#Always be the same bin for all infoObjs
etabinidx = self._infoObjs[0].etabinidx()
etbinidx = self._infoObjs[0].etbinidx()
#Reading the data rings from path or object
dataPath = kw.pop('dataPath', None)
if dataPath:
from TuningTools import TuningDataArchieve
self._logger.info(
('Reading data tuning file with name %s') % (dataPath))
TDArchieve = TuningDataArchieve.load(dataPath,
etBinIdx=etbinidx,
etaBinIdx=etabinidx,
loadEfficiencies=False)
self._data = (TDArchieve.signalPatterns,
TDArchieve.backgroundPatterns)
else:
self._logger.fatal(
'You must pass the ref file as parameter. abort!')
def __init__(self, filePath=None, **kw):
"""
Either specify the file path where the file should be read or the data
which should be appended to it
with CrosValidArchieve("/path/to/file") as data:
BLOCK
CrosValidArchieve( "file/path", crossValid = CrossValid() )
"""
Logger.__init__(self, kw)
self._filePath = filePath
self.crossValid = kw.pop('crossValid', None)
checkForUnusedVars(kw, self._warning)
def __init__(self, filePath = None, **kw):
"""
Either specify the file path where the file should be read or the data
which should be appended to it
with CrosValidArchieve("/path/to/file") as data:
BLOCK
CrosValidArchieve( "file/path", crossValid = CrossValid() )
"""
Logger.__init__(self, kw)
self._filePath = filePath
self.crossValid = kw.pop( 'crossValid', None )
checkForUnusedVars( kw, self._logger.warning )
def __init__(self, rawFile, benchmark, neuron, sort, init, etBinIdx,
etaBinIdx, **kw):
Logger.__init__(self, **kw)
self._rawFile = rawFile
self._benchmark = benchmark
self._reference = benchmark.split('_')[-1]
self._neuron = neuron
self._sort = sort
self._init = init
self._etBinIdx = etBinIdx
self._etaBinIdx = etaBinIdx
### Hold all ROOT objects
self._obj = {}
### Current frame version
self._version = 2
def __init__(self):
Logger.__init__(self)
# Do not change this if you dont know what are you doing
# standard vortex configuration approach
self._form = [
[72, 73, 74, 75, 76, 77, 78, 79, 80, 81],
[71, 42, 43, 44, 45, 46, 47, 48, 49, 82],
[70, 41, 20, 21, 22, 23, 24, 25, 50, 83],
[69, 40, 19, 6, 7, 8, 9, 26, 51, 84],
[68, 39, 18, 5, 0, 1, 10, 27, 52, 85],
[67, 38, 17, 4, 3, 2, 11, 28, 53, 86],
[66, 37, 16, 15, 14, 13, 12, 29, 54, 87],
[65, 36, 35, 34, 33, 32, 31, 30, 55, 88],
[64, 63, 62, 61, 60, 59, 58, 57, 56, 89],
[99, 98, 97, 96, 95, 94, 93, 92, 91, 90],
]
self._shape = (10, 10)
def __init__(self, filePath = None, **kw):
"""
Either specify the file path where the file should be read or the data
which should be appended to it:
with TuningJobConfigArchieve("/path/to/file") as data:
BLOCK
TuningJobConfigArchieve( "file/path", neuronBounds = ...,
sortBounds = ...,
initBounds = ... )
"""
Logger.__init__(self, kw)
self._filePath = filePath
self.neuronBounds = kw.pop('neuronBounds', None)
self._sortBounds = kw.pop('sortBounds', None)
self._initBounds = kw.pop('initBounds', None)
checkForUnusedVars( kw, self._logger.warning )
def __init__(self,
model=None,
trnData=None,
valData=None,
tstData=None,
references=None,
display=None):
callbacks.History.__init__(self)
Logger.__init__(self)
self.model = model
self.trnData = trnData
self.valData = valData
self.tstData = tstData
self.trnRoc = Roc()
self.valRoc = Roc()
self.tstRoc = Roc()
self.references = references
self.display = display
def __init__(self, filePath = None, **kw):
"""
Either specify the file path where the file should be read or the data
which should be appended to it:
with TuningJobConfigArchieve("/path/to/file") as data:
BLOCK
TuningJobConfigArchieve( "file/path", neuronBounds = ...,
sortBounds = ...,
initBounds = ... )
"""
Logger.__init__(self, kw)
self._filePath = filePath
self.neuronBounds = kw.pop('neuronBounds', None)
self._sortBounds = kw.pop('sortBounds', None)
self._initBounds = kw.pop('initBounds', None)
checkForUnusedVars( kw, self._warning )
def __init__(self, net, **kw):
Logger.__init__(self, kw)
train = kw.pop("train", None)
checkForUnusedVars(kw, self._logger.warning)
del kw
# Extract the information from c++ wrapper code
self.nNodes = []
self.numberOfLayers = net.getNumLayers()
self.dataTrain = None
# Hold the train evolution information
if train:
self.dataTrain = DataTrainEvolution(train)
self.layers = self.__retrieve(net)
self._logger.debug("The Neural object was created.")
def __init__( self, discriminator = None, dataCurator = None, pileupRef = None
, limits = None, maxCorr = None, frMargin = None ):
Logger.__init__( self )
self._discr = discriminator
self._dataCurator = dataCurator
self._baseLabel = ''
# Decision making parameters:
self.pileupRef = PileupReference.retrieve( pileupRef )
self.limits = limits
self.maxCorr = maxCorr
if frMargin is not None: self.frMargin = [1.] + frMargin if frMargin[0] != 1. else frMargin
self._pileupLabel = PileupReference.label( self.pileupRef )
self._pileupShortLabel = PileupReference.shortlabel( self.pileupRef )
self.subset = None
if self.pileupRef is PileupReference.avgmu:
#limits = [0,26,60]
self.limits = [15,37.5,60] if limits is None else limits
elif self.pileupRef is PileupReference.nvtx:
self.limits = [0,13,30] if limits is None else limits
# Other transient attributes:
self.sgnOut = None
self.bkgOut = None
self.sgnHist = None
self.bkgHist = None
self.sgnCorrData = None
self.bkgCorrDataList = None
self._effSubset = None
self._effOutput = None
self._ol = 12. if self._discr.removeOutputTansigTF else 1.
# Decision taking parameters:
self.intercept = None
self.slope = None
self.slopeRange = None
# Thresholds:
self.thres = None
self.rawThres = None
# Performance:
self.perf = None
self.rawPerf = None
# Bin information:
self.etBinIdx = self._dataCurator.etBinIdx
self.etaBinIdx = self._dataCurator.etaBinIdx
self.etBin = self._dataCurator.etBin.tolist()
self.etaBin = self._dataCurator.etaBin.tolist()
def __init__(self,
display=1,
doMultiStop=False,
patience=25,
save_the_best=True,
val_generator=None,
**kw):
# initialize all base objects
super(Callback, self).__init__()
Logger.__init__(self, **kw)
# number of max fails
self._patience = {'max': patience, 'score': 0, 'loss': 0}
self._display = display
self._save_the_best = save_the_best
# used to hold the best configs
self._best_weights = None
# used to hold the SP value
self._current_score = 0.0
self._val_generator = val_generator
def __init__(self, filename, **kw):
Logger.__init__(self,kw)
self._title = kw.pop('title', 'Tuning Report')
self._institute = kw.pop('institute', 'Universidade Federal do Rio de Janeiro (UFRJ)')
checkForUnusedVars( kw, self._logger.warning )
import socket
self._machine = socket.gethostname()
import getpass
self._author = getpass.getuser()
from time import gmtime, strftime
self._data = strftime("%Y-%m-%d %H:%M:%S", gmtime())
#Create output file
self._pfile = open(filename+'.tex','w')
# Import soma beamer contants
from BeamerTemplates import BeamerConstants as bconst
self._pfile.write( bconst.beginDocument )
pname = self._author+'$@$'+self._machine
self._pfile.write( (bconst.beginHeader) % \
(self._title, self._title, pname, self._institute) )
self._pfile.write( bconst.beginTitlePage )
def __init__( self, dataCurator, d, **kw ):
Logger.__init__( self, kw )
d.update( kw ); del kw
# Define discriminator type (for now we only have one discriminator type):
# FIXME This chould be configured using coreDef
try:
try:
from libTuningTools import DiscriminatorPyWrapper
except ImportError:
from libTuningToolsLib import DiscriminatorPyWrapper
except ImportError:
self._fatal("Cannot use FastNet DiscriminatorPyWrapper: library is not available!")
self.dataCurator = dataCurator
# Discriminator parameters:
self.DiscrClass = DiscriminatorPyWrapper
self.removeOutputTansigTF = retrieve_kw(d, 'removeOutputTansigTF', True )
# Decision making parameters:
self.thresEtBins = retrieve_kw( d, 'thresEtBins', None)
self.thresEtaBins = retrieve_kw( d, 'thresEtaBins', None)
# TODO There is no need to require this from user
self.method = DecisionMakingMethod.retrieve( retrieve_kw(d, 'decisionMakingMethod', DecisionMakingMethod.LHLinearApproach ) )
# Pileup limits specification
self.pileupRef = None
self.pileupLimits = retrieve_kw( d, 'pileupLimits', None)
self.maxCorr = retrieve_kw( d, 'maxCorr', None)
self.frMargin = retrieve_kw( d, 'frMargin', [1.05, 1.1, 1.15, 1.25, 1.5, 2.0])
if self.method is DecisionMakingMethod.LHLinearApproach:
self._info("Using limits: %r", self.pileupLimits)
if not 'pileupRef' in d:
self._fatal("pileupRef must specified in order to correctly label the xaxis.")
if 'pileupRef' in d:
self.pileupRef = PileupReference.retrieve( retrieve_kw( d, 'pileupRef' ) )
# Linear LH Threshold Correction methods
#self.maxFRMultipler = retrieve_kw( d, 'maxFRMultipler', None)
#checkForUnusedVars( d )
# TODO: Add fix fraction allowing to load a module and a table of values
if self.method is DecisionMakingMethod.LHLinearApproach:
try:
import ElectronIDDevelopment
except ImportError:
self._warning("Using a standalone version of the pile-up fitting version which may not be the latest.")
def __init__(self, filename, **kw):
Logger.__init__(self,kw)
self._title = kw.pop('title', 'Tuning Report')
self._institute = kw.pop('institute', 'Universidade Federal do Rio de Janeiro (UFRJ)')
checkForUnusedVars( kw, self._logger.warning )
import socket
self._machine = socket.gethostname()
import getpass
self._author = getpass.getuser()
from time import gmtime, strftime
self._data = strftime("%Y-%m-%d %H:%M:%S", gmtime())
#Create output file
self._pfile = open(filename+'.tex','w')
from BeamerTemplates import BeamerConstants as bconst
self._pfile.write( bconst.beginDocument )
pname = self._author+'$@$'+self._machine
self._pfile.write( (bconst.beginHeader) % \
(self._title, self._title, pname, self._institute) )
self._pfile.write( bconst.beginTitlePage )
def __init__(self, **kw):
Logger.__init__(self, kw)
# Retrieve all information needed
self._fList = retrieve_kw(kw, 'inputFiles', NotSet)
self._ofile = retrieve_kw(kw, 'outputFile', "histos.root")
self._treePath = retrieve_kw(kw, 'treePath', NotSet)
self._dataframe = retrieve_kw(kw, 'dataframe',
DataframeEnum.SkimmedNtuple)
self._nov = retrieve_kw(kw, 'nov', -1)
self._fList = csvStr2List(self._fList)
self._fList = expandFolders(self._fList)
# Loading libraries
if ROOT.gSystem.Load('libTuningTools') < 0:
self._fatal("Could not load TuningTools library", ImportError)
self._containersSvc = {}
self._storegateSvc = NotSet
import random
import time
random.seed(time.time())
# return a random number
self._id = random.randrange(100000)
except:
logger.warning('%sc not found',pypath)
try:
logger.info('PathResolver: %s',pypath)
return obj__Module__
except KeyError:
logger.fatal('Key %s not found in this module. Abort')
from RingerCore import Logger, LoggingLevel
import argparse
mainLogger = Logger.getModuleLogger("ConvertToPickle")
parser = argparse.ArgumentParser(description = '', add_help = False)
parser = argparse.ArgumentParser()
parser.add_argument('-i','--inputFiles', action='store',
dest='inputFiles', required = True, nargs='+',
help = "The input files that will be used to generate the pic")
import sys,os
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
dest = 'grid_outputs',
help = argparse.SUPPRESS )
# Hide forceStaged and make it always be true
parser.add_argument('--forceStaged', action='store_const',
required = False, dest = 'grid_forceStaged', default = True,
const = True, help = argparse.SUPPRESS)
# Hide forceStagedSecondary and make it always be true
parser.add_argument('--forceStagedSecondary', action='store_const',
required = False, dest = 'grid_forceStagedSecondary', default = True,
const = True, help = argparse.SUPPRESS)
parser.add_argument('--mergeOutput', action='store_const',
required = False, default = True, const = True,
dest = 'grid_mergeOutput',
help = argparse.SUPPRESS)
mainLogger = Logger.getModuleLogger(__name__)
import sys
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
printArgs( args, mainLogger.debug )
args.grid_allowTaskDuplication = True
# Set primary dataset number of files:
import os.path
def __init__(self, **kw):
Logger.__init__(self, kw)
printArgs(kw, self._debug)
self._nSorts = None
self._nBoxes = None
self._nTrain = None
self._nValid = None
self._nTest = None
self._seed = None
self._method = CrossValidMethod.retrieve(
retrieve_kw(kw, 'method', CrossValidMethod.Standard))
if self._method is CrossValidMethod.Standard:
self._nSorts = retrieve_kw(kw, 'nSorts', 50)
self._nBoxes = retrieve_kw(kw, 'nBoxes', 10)
self._nTrain = retrieve_kw(kw, 'nTrain', 6)
self._nValid = retrieve_kw(kw, 'nValid', 4)
self._nTest = retrieve_kw(
kw, 'nTest', self._nBoxes - (self._nTrain + self._nValid))
self._seed = retrieve_kw(kw, 'seed', None)
checkForUnusedVars(kw, self._warning)
# Check if variables are ok:
if (not self._nTest is None) and self._nTest < 0:
self._fatal("Number of test clusters is lesser than zero",
ValueError)
totalSum = self._nTrain + self._nValid + (self._nTest) if self._nTest else \
self._nTrain + self._nValid
if totalSum != self._nBoxes:
self._fatal(
"Sum of train, validation and test boxes doesn't match.",
ValueError)
np.random.seed(self._seed)
# Test number of possible combinations (N!/((N-K)!(K)!) is greater
# than the required sorts. If number of sorts (greater than half of the
# possible cases) is close to the number of combinations, generate all
# possible combinations and then gather the number of needed sorts.
# However, as calculating factorial can be heavy, we don't do this if the
# number of boxes is large.
self._sort_boxes_list = []
useRandomCreation = True
from math import factorial
if self._nBoxes < 201:
totalPossibilities = ( factorial( self._nBoxes ) ) / \
( factorial( self._nTrain ) * \
factorial( self._nValid ) * \
factorial( self._nTest ) )
if self._nSorts > (totalPossibilities / 2):
useRandomCreation = False
if useRandomCreation:
count = 0
while True:
random_boxes = np.random.permutation(self._nBoxes)
random_boxes = tuple(
chain(
sorted(random_boxes[0:self._nTrain]),
sorted(random_boxes[self._nTrain:self._nTrain +
self._nValid]),
sorted(random_boxes[self._nTrain +
self._nValid:])))
# Make sure we are not appending same sort again:
if not random_boxes in self._sort_boxes_list:
self._sort_boxes_list.append(random_boxes)
count += 1
if count == self._nSorts:
break
else:
self._sort_boxes_list = list(
combinations_taken_by_multiple_groups(
range(self._nBoxes),
(self._nTrain, self._nValid, self._nTest)))
for i in range(totalPossibilities - self._nSorts):
self._sort_boxes_list.pop(
np.random.random_integers(0, totalPossibilities))
elif self._method is CrossValidMethod.JackKnife:
self._nBoxes = retrieve_kw(kw, 'nBoxes', 10)
checkForUnusedVars(kw, self._warning)
self._nSorts = self._nBoxes
self._nTrain = self._nBoxes - 1
self._nValid = 1
self._nTest = 0
self._sort_boxes_list = list(
combinations_taken_by_multiple_groups(range(self._nBoxes), (
9,
1,
)))
elif self._method is CrossValidMethod.StratifiedKFold:
self._nBoxes = retrieve_kw(kw, 'nBoxes', 10)
self._shuffle = retrieve_kw(kw, 'shuffle', False)
checkForUnusedVars(kw, self._logger.warning)
self._nSorts = self._nBoxes
self._nTrain = self._nBoxes - 1
self._nValid = 1
self._nTest = 0
def __init__(self): Logger.__init__(self) self._eventStack = [] self._toolStack = []
#!/usr/bin/env python
from rDev.Event import EventLooper
from rDev.dataframe import ElectronCandidate
from TuningTools.dataframe.EnumCollection import Dataframe as DataframeEnum
from RingerCore import LoggingLevel, Logger, csvStr2List, expandFolders
import argparse
mainLogger = Logger.getModuleLogger("job")
parser = argparse.ArgumentParser(description='', add_help=False)
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--data',
action='store',
dest='data',
required=True,
nargs='+',
help="The input tuning files.")
import sys, os
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# Take all files
paths = csvStr2List(args.data)
paths = expandFolders(paths)
from RingerCore import load, save, appendToFileName
def __init__(self, obj, **kw): from pprint import pprint Logger.__init__(self, kw)
def __init__(self, **kw ):
Logger.__init__( self, kw )
printArgs( kw, self._logger.debug )
self._nSorts = None
self._nBoxes = None
self._nTrain = None
self._nValid = None
self._nTest = None
self._seed = None
self._method = CrossValidMethod.retrieve( retrieve_kw( kw, 'method', CrossValidMethod.Standard ) )
if self._method is CrossValidMethod.Standard:
self._nSorts = retrieve_kw( kw, 'nSorts', 50 )
self._nBoxes = retrieve_kw( kw, 'nBoxes', 10 )
self._nTrain = retrieve_kw( kw, 'nTrain', 6 )
self._nValid = retrieve_kw( kw, 'nValid', 4 )
self._nTest = retrieve_kw( kw, 'nTest', self._nBoxes - ( self._nTrain + self._nValid ) )
self._seed = retrieve_kw( kw, 'seed', None )
checkForUnusedVars( kw, self._logger.warning )
# Check if variables are ok:
if (not self._nTest is None) and self._nTest < 0:
self._logger.fatal("Number of test clusters is lesser than zero", ValueError)
totalSum = self._nTrain + self._nValid + (self._nTest) if self._nTest else \
self._nTrain + self._nValid
if totalSum != self._nBoxes:
self._logger.fatal("Sum of train, validation and test boxes doesn't match.", ValueError)
np.random.seed(self._seed)
# Test number of possible combinations (N!/((N-K)!(K)!) is greater
# than the required sorts. If number of sorts (greater than half of the
# possible cases) is close to the number of combinations, generate all
# possible combinations and then gather the number of needed sorts.
# However, as calculating factorial can be heavy, we don't do this if the
# number of boxes is large.
self._sort_boxes_list = []
useRandomCreation = True
from math import factorial
if self._nBoxes < 201:
totalPossibilities = ( factorial( self._nBoxes ) ) / \
( factorial( self._nTrain ) * \
factorial( self._nValid ) * \
factorial( self._nTest ) )
if self._nSorts > (totalPossibilities / 2):
useRandomCreation = False
if useRandomCreation:
count = 0
while True:
random_boxes = np.random.permutation(self._nBoxes)
random_boxes = tuple(chain(sorted(random_boxes[0:self._nTrain]),
sorted(random_boxes[self._nTrain:self._nTrain+self._nValid]),
sorted(random_boxes[self._nTrain+self._nValid:])))
# Make sure we are not appending same sort again:
if not random_boxes in self._sort_boxes_list:
self._sort_boxes_list.append( random_boxes )
count += 1
if count == self._nSorts:
break
else:
self._sort_boxes_list = list(
combinations_taken_by_multiple_groups(range(self._nBoxes),
(self._nTrain,
self._nVal,
self._nTest)))
for i in range(totalPossibilities - self._nSorts):
self._sort_boxes_list.pop( np.random_integers(0, totalPossibilities) )
elif self._method is CrossValidMethod.JackKnife:
self._nBoxes = retrieve_kw( kw, 'nBoxes', 10 )
checkForUnusedVars( kw, self._logger.warning )
self._nSorts = self._nBoxes
self._nTrain = self._nBoxes - 1
self._nValid = 1
self._nTest = 0
self._sort_boxes_list = list(
combinations_taken_by_multiple_groups(range(self._nBoxes), (9, 1,)) )
def __init__(self, input_, output): Logger.__init__( self ) self.f = TFile.Open(output, 'recreate') self.input_ = input_
#'e24_vloose_L1EM20VH',
#'e5_loose_idperf',
#'e5_lhloose_idperf',
#'e5_tight_idperf',
#'e5_lhtight_idperf',
'e24_medium_idperf_L1EM20VH',
'e24_lhmedium_idperf_L1EM20VH'
]
parser = argparse.ArgumentParser()
parser.add_argument('--inFolderList', nargs='+', required=True,
help = "Input container to retrieve data")
parser.add_argument('--signalDS', action='store_true',
help = "Whether the dataset contains TPNtuple")
parser.add_argument('--outfile', action='store', default="mergedOutput.root",
help = "Name of the output file")
parser.add_argument('--triggerList', nargs='+', default=defaultTrigList,
help = "Trigger list to keep on the filtered file.")
args=parser.parse_args()
mainLogger = Logger.getModuleLogger( __name__, LoggingLevel.INFO )
files = expandFolders( args.inFolderList )
rFile = RootFile( files, args.outfile )
rFile.dump('Offline/Egamma/Ntuple', ['electron'])
rFile.dump('Trigger/HLT/Egamma/Ntuple', args.triggerList)
if args.signalDS:
rFile.dump('Trigger/HLT/Egamma/TPNtuple', args.triggerList)
rFile.save()
def __init__( self, logger = None ): Logger.__init__( self, logger = logger )
#!/usr/bin/env python
# TODO Improve skeleton documentation
from timeit import default_timer as timer
start = timer()
DatasetLocationInput = '/afs/cern.ch/work/j/jodafons/public/validate_tuningtool/mc14_13TeV.147406.129160.sgn.offLikelihood.bkg.truth.trig.e24_lhmedium_L1EM20VH_etBin_0_etaBin_0.npz'
#try:
from RingerCore import Logger, LoggingLevel
mainLogger = Logger.getModuleLogger(__name__)
mainLogger.info("Entering main job.")
from TuningTools import TuningJob
tuningJob = TuningJob()
from TuningTools.PreProc import *
basepath = '/afs/cern.ch/work/j/jodafons/public'
tuningJob( DatasetLocationInput,
neuronBoundsCol = [15, 15],
sortBoundsCol = [0, 1],
initBoundsCol = 5,
#confFileList = basepath + '/user.wsfreund.config.nn5to20_sorts50_1by1_inits100_100by100/job.hn0015.s0040.il0000.iu0099.pic.gz',
#ppFileList = basepath+'/user.wsfreund.Norm1/ppFile_pp_Norm1.pic.gz',
#crossValidFile = basepath+'/user.wsfreund.CrossValid.50Sorts.seed_0/crossValid.pic.gz',
epochs = 100,
showEvo = 0,
parser = ArgumentParser(
description=
'Retrieve performance information from the Cross-Validation method on the GRID.',
parents=[crossValStatsJobParser, parentParser, ioGridParser, loggerParser],
conflict_handler='resolve')
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args(namespace=TuningToolGridNamespace('prun'))
args.setBExec(
'source ./buildthis.sh --grid --with-scipy --no-color || source ./buildthis.sh --grid --with-scipy --no-color'
)
mainLogger = Logger.getModuleLogger(__name__, args.output_level)
printArgs(args, mainLogger.debug)
# Set primary dataset number of files:
try:
# The input files can be send via a text file to avoid very large command lines?
mainLogger.info(("Retrieving files on the data container to separate "
"the jobs accordingly to each tunned bin reagion."))
from rucio.client import DIDClient
from rucio.common.exception import DataIdentifierNotFound
didClient = DIDClient()
parsedDataDS = args.grid__inDS.split(':')
did = parsedDataDS[-1]
if len(parsedDataDS) > 1:
scope = parsedDataDS
else:
def __init__(self, net, **kw):
Logger.__init__(self, kw)