def trnData(self, release=False): if coreConf() is TuningToolCores.FastNet: ret = self.__separate_patterns(self._trnData, self._trnTarget) elif coreConf() is TuningToolCores.keras: ret = self._trnData if release: self.release() return ret
def setTestData(self, data, target=None): """ Set test dataset of the tuning method. """ if self._merged: data_calo = data[0] data_track = data[1] if coreConf() is TuningToolCores.keras: if target is None: data_calo, target = self.__concatenate_patterns(data_calo) data_track, _ = self.__concatenate_patterns(data_track) _checkData(data_calo, target) _checkData(data_track, target) data = [data_calo, data_track] self._tstData = data self._tstTarget = target self._historyCallback.tstData = (data, target) elif coreConf() is TuningToolCores.FastNet: self._fatal( "Expert Neural Networks not implemented for FastNet core") else: if coreConf() is TuningToolCores.keras: if target is None: data, target = self.__concatenate_patterns(data) _checkData(data, target) self._tstData = data self._tstTarget = target self._historyCallback.tstData = (data, target) elif coreConf() is TuningToolCores.FastNet: self._tstData = data self._core.setValData(data)
def setTrainData(self, data, target=None): """ Set train dataset of the tuning method. """ if self._merged: data_calo = data[0] data_track = data[1] self._sgnSize = data_calo[0].shape[npCurrent.odim] self._bkgSize = data_track[1].shape[npCurrent.odim] if coreConf() is TuningToolCores.keras: if target is None: data_calo, target = self.__concatenate_patterns(data_calo) data_track, _ = self.__concatenate_patterns(data_track) _checkData(data_calo, target) _checkData(data_track, target) data = [data_calo, data_track] self._trnData = data self._trnTarget = target self._historyCallback.trnData = (data, target) elif coreConf() is TuningToolCores.FastNet: self._fatal( "Expert Neural Networks not implemented for FastNet core") else: self._sgnSize = data[0].shape[npCurrent.odim] self._bkgSize = data[1].shape[npCurrent.odim] if coreConf() is TuningToolCores.keras: if target is None: data, target = self.__concatenate_patterns(data) _checkData(data, target) self._trnData = data self._trnTarget = target self._historyCallback.trnData = (data, target) elif coreConf() is TuningToolCores.FastNet: self._trnData = data self._core.setTrainData(data)
def valData(self, release=False): if coreConf() is TuningToolCores.keras: ret = self.__separate_patterns(self._valData, self._valTarget) elif coreConf() is TuningToolCores.Keras: ret = self._valData if release: self.release() return ret
def doMultiStop(self): """ External access to doMultiStop """ if coreConf() is TuningToolCores.keras: return self._multiStop elif coreConf() is TuningToolCores.FastNet: return self._core.multiStop
def batchSize(self): """ External access to batchSize """ if coreConf() is TuningToolCores.keras: return self.trainOptions.get('batchSize', None) elif coreConf() is TuningToolCores.FastNet: return self._core.batchSize
def __batchSize(self, val): """ Internal access to batchSize """ if coreConf() is TuningToolCores.keras: self.trainOptions['batchSize'] = val elif coreConf() is TuningToolCores.FastNet: self._core.batchSize = val self._debug('Set batchSize to %d', val)
def batchSize(self, val): """ External access to batchSize """ if val is not NotSet: self.batchMethod = BatchSizeMethod.Manual if coreConf() is TuningToolCores.keras: self.trainOptions['batchSize'] = val elif coreConf() is TuningToolCores.FastNet: self._core.batchSize = val self._debug('Set batchSize to %d', val)
def __expDiscr_to_dict(self, model): """ Transform expert discriminators to dictionary """ if coreConf() is TuningToolCores.keras: ow, ob = model.get_layer(name='merge_dense_2').get_weights() hw, hb = model.get_layer(name='merge_dense_1').get_weights() chw, chb = model.get_layer(name='calo_dense_2').get_weights() thw, thb = model.get_layer(name='track_dense_2').get_weights() discrDict = { 'nodes': [[chw.shape[0], thw.shape[0]], [chw.shape[1], thw.shape[1]], hw.shape[0], hw.shape[1], ow.shape[1]], 'weights': { 'output_layer': ow, 'merged_hidden_layer': hw, 'calo_layer': chw, 'track_layer': thw }, 'bias': { 'output_layer': ob, 'merged_hidden_layer': hb, 'calo_layer': chb, 'track_layer': thb } } self._debug('Extracted discriminator to raw dictionary.') return discrDict
def newff(self, nodes, funcTrans=NotSet): """ Creates new feedforward neural network """ self._debug('Initalizing newff...') if coreConf() is TuningToolCores.ExMachina: if funcTrans is NotSet: funcTrans = ['tanh', 'tanh'] self._model = self._core.FeedForward(nodes, funcTrans, 'nw') elif coreConf() is TuningToolCores.FastNet: if funcTrans is NotSet: funcTrans = ['tansig', 'tansig'] if not self._core.newff(nodes, funcTrans, self._core.trainFcn): self._fatal("Couldn't allocate new feed-forward!") elif coreConf() is TuningToolCores.keras: from keras.models import Sequential from keras.layers.core import Dense, Dropout, Activation model = Sequential() model.add( Dense(nodes[0], input_dim=nodes[0], init='identity', trainable=False, name='dense_1')) model.add(Activation('linear')) model.add( Dense(nodes[1], input_dim=nodes[0], init='uniform', name='dense_2')) model.add(Activation('tanh')) model.add(Dense(nodes[2], init='uniform', name='dense_3')) model.add(Activation('tanh')) model.compile(loss=self.trainOptions['costFunction'], optimizer=self.trainOptions['optmin_alg'], metrics=self.trainOptions['metrics']) #keras.callbacks.History() #keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1) self._model = model self._historyCallback.model = model
def __discr_to_dict(self, model): """ Transform discriminators to dictionary """ if coreConf() is TuningToolCores.keras: hw, hb = model.get_layer(name='dense_2').get_weights() ow, ob = model.get_layer(name='dense_3').get_weights() discrDict = { 'nodes': npCurrent.int_array([hw.shape[0], hw.shape[1], ow.shape[1]]), 'weights': np.concatenate( [hw.reshape(-1, order='F'), ow.reshape(-1, order='F')]), 'bias': np.concatenate( [hb.reshape(-1, order='F'), ob.reshape(-1, order='F')]), } elif coreConf() is TuningToolCores.FastNet: n = [] w = [] b = [] for l in range(model.getNumLayers()): n.append(model.getNumNodes(l)) for l in range(len(n) - 1): for j in range(n[l + 1]): for k in range(n[l]): w.append(model.getWeight(l, j, k)) b.append(model.getBias(l, j)) discrDict = { 'nodes': npCurrent.int_array(n), 'weights': npCurrent.fp_array(w), 'bias': npCurrent.fp_array(b) } self._debug('Extracted discriminator to raw dictionary.') return discrDict
def __dict_to_discr(self, discrDict, appendage=None, pruneLastLayer=True): """ Transform dictionaries of networks into discriminators. """ nodes = discrDict['nodes'] weights = discrDict['weights'] bias = discrDict['bias'] if coreConf() is TuningToolCores.keras: from keras.models import Sequential from keras.layers.core import Dense, Dropout, Activation model = Sequential() names = ['dense_1', 'dense_2', 'dense_3'] if appendage: for i in range( len(names) - 1 if pruneLastLayer else len(names)): names[i] = '%s_%s' % (appendage, names[i]) model.add( Dense(nodes[0], input_dim=nodes[0], kernel_initializer='identity', trainable=False, name=names[0])) model.add(Activation('linear')) model.add( Dense(nodes[1], input_dim=nodes[0], trainable=False, kernel_initializer='uniform', name=names[1])) model.add(Activation('tanh')) w1 = weights[0:(nodes[0] * nodes[1])] w1 = w1.reshape((nodes[0], nodes[1]), order='F') b1 = bias[0:nodes[1]] model.get_layer(name=names[1]).set_weights((w1, b1)) if not pruneLastLayer: model.add( Dense(nodes[2], kernel_initializer='uniform', trainable=False, name=names[2])) model.add(Activation('tanh')) w2 = weights[(nodes[0] * nodes[1]):(nodes[0] * nodes[1] + nodes[1] * nodes[2])] w2 = w2.reshape((nodes[1], nodes[2]), order='F') b2 = bias[nodes[1]:nodes[1] + nodes[2]] model.layers[-2].set_weights((w2, b2)) return model
def setSortIdx(self, sort): if coreConf() is TuningToolCores.FastNet: if self.doMultiStop and self.useTstEfficiencyAsRef: if not len(self.references) == 3 or \ not self.references[0].reference == ReferenceBenchmark.SP or \ not self.references[1].reference == ReferenceBenchmark.Pd or \ not self.references[2].reference == ReferenceBenchmark.Pf: self._fatal( "The tuning wrapper references are not correct!") self.sortIdx = sort self._core.det = self.references[1].getReference( ds=Dataset.Validation, sort=sort) self._core.fa = self.references[2].getReference( ds=Dataset.Validation, sort=sort) self._info( 'Set multiStop target [sort:%d | Sig_Eff(%%) = %r, Bkg_Eff(%%) = %r].', sort, self._core.det * 100., self._core.fa * 100.)
def __enter__(self): # Open file: jobConfig = load(self._filePath) try: if type(jobConfig) is dict: if jobConfig['type'] != self._type: self._fatal(("Input jobConfig file is not from jobConfig " "type.")) # Read configuration file to retrieve pre-processing, if jobConfig['version'] >= 1: neuronBounds = MatlabLoopingBounds( jobConfig['neuronBounds']) sortBounds = PythonLoopingBounds(jobConfig['sortBounds']) initBounds = PythonLoopingBounds(jobConfig['initBounds']) modelBounds = [None for _ in neuronBounds()] if jobConfig['version'] == 2: modelBounds = jobConfig['modelBounds'] if modelBounds and coreConf() is TuningToolCores.keras: from keras.models import model_from_json import json for idx, model in enumerate(modelBounds): modelBounds[idx] = model_from_json( json.dumps(model, separators=( ',', ':'))) if model else None else: self._fatal("Unknown job configuration version") elif type(jobConfig ) is list: # zero version file (without versioning # control): neuronBounds = MatlabLoopingBounds( [jobConfig[0], jobConfig[0]]) sortBounds = MatlabLoopingBounds(jobConfig[1]) initBounds = MatlabLoopingBounds(jobConfig[2]) modelBounds = None else: self._fatal("Unknown file type entered for config file.") except RuntimeError, e: self._fatal(("Couldn't read configuration file '%s': Reason:" "\n\t %s" % (self._filePath, e)))
def _getCoreFramework(self): if coreConf.configured(): return coreConf() else: return None
def setReferences(self, references): # Make sure that the references are a collection of ReferenceBenchmark references = ReferenceBenchmarkCollection(references) if len(references) == 0: self._fatal("Reference collection must be not empty!", ValueError) if coreConf() is TuningToolCores.ExMachina: self._info("Setting reference target to MSE.") if len(references) != 1: self._logger.error( "Ignoring other references as ExMachina currently works with MSE." ) references = references[:1] self.references = references ref = self.references[0] if ref.reference != ReferenceBenchmark.MSE: self._fatal("Tuning using MSE and reference is not MSE!") elif coreConf() is TuningToolCores.FastNet: if self.doMultiStop: self.references = ReferenceBenchmarkCollection([None] * 3) # This is done like this for now, to prevent adding multiple # references. However, this will be removed when the FastNet is # made compatible with multiple references retrievedSP = False retrievedPD = False retrievedPF = False for ref in references: if ref.reference is ReferenceBenchmark.SP: if not retrievedSP: retrievedSP = True self.references[0] = ref else: self._warning( "Ignoring multiple Reference object: %s", ref) elif ref.reference is ReferenceBenchmark.Pd: if not retrievedPD: retrievedPD = True self.references[1] = ref self._core.det = self.references[1].getReference() else: self._warning( "Ignoring multiple Reference object: %s", ref) elif ref.reference is ReferenceBenchmark.Pf: if not retrievedPF: retrievedPF = True self.references[2] = ref self._core.fa = self.references[2].getReference() else: self._warning( "Ignoring multiple Reference object: %s", ref) self._info( 'Set multiStop target [Sig_Eff(%%) = %r, Bkg_Eff(%%) = %r].', self._core.det * 100., self._core.fa * 100.) else: self._info("Setting reference target to MSE.") if len(references) != 1: self._warning( "Ignoring other references when using FastNet with MSE." ) references = references[:1] self.references = references ref = self.references[0] if ref.reference != ReferenceBenchmark.MSE: self._fatal("Tuning using MSE and reference is not MSE!") elif coreConf() is TuningToolCores.keras: self.references = references self._info("keras will be using the following references:") #from TuningTools.keras_util.metrics import Efficiencies def addMetricToKeras(func, name, metrics): from keras import metrics as metrics_module setattr(metrics_module, name, func) metrics.append(name) for idx, bench in enumerate(self.references): self._info("Added %s", bench) self._historyCallback.references = references
def _getCoreFramework(self): if coreConf: return coreConf() else: return None
def __init__(self, **kw): Logger.__init__(self, kw) self.references = ReferenceBenchmarkCollection([]) coreframe = coreConf.core_framework() self.doPerf = retrieve_kw(kw, 'doPerf', True) self.batchMethod = BatchSizeMethod.retrieve( retrieve_kw( kw, 'batchMethod', BatchSizeMethod.MinClassSize \ if not 'batchSize' in kw or kw['batchSize'] is NotSet else BatchSizeMethod.Manual ) ) self.batchSize = retrieve_kw(kw, 'batchSize', NotSet) epochs = retrieve_kw(kw, 'epochs', 10000) maxFail = retrieve_kw(kw, 'maxFail', 50) self.useTstEfficiencyAsRef = retrieve_kw(kw, 'useTstEfficiencyAsRef', False) self._merged = retrieve_kw(kw, 'merged', False) self.networks = retrieve_kw(kw, 'networks', NotSet) self.sortIdx = None if coreConf() is TuningToolCores.FastNet: seed = retrieve_kw(kw, 'seed', None) self._core = coreframe(level=LoggingLevel.toC(self.level), seed=seed) self._core.trainFcn = retrieve_kw(kw, 'algorithmName', 'trainrp') self._core.showEvo = retrieve_kw(kw, 'showEvo', 50) self._core.multiStop = retrieve_kw(kw, 'doMultiStop', True) self._core.epochs = epochs self._core.maxFail = maxFail # TODO Add properties elif coreConf() is TuningToolCores.keras: self._core = coreframe from keras import callbacks from keras.optimizers import RMSprop, SGD from TuningTools.keras_util.callbacks import PerformanceHistory self.trainOptions = dict() self.trainOptions['optmin_alg'] = retrieve_kw( kw, 'optmin_alg', RMSprop(lr=0.001, rho=0.9, epsilon=1e-08)) #self.trainOptions['optmin_alg'] = retrieve_kw( kw, 'optmin_alg', SGD(lr=0.1, decay=1e-6, momentum=0.7) ) self.trainOptions['costFunction'] = retrieve_kw( kw, 'binary_crossentropy', 'mean_squared_error' ) # 'binary_crossentropy' #'mean_squared_error' # self.trainOptions['metrics'] = retrieve_kw(kw, 'metrics', [ 'accuracy', ]) self.trainOptions['shuffle'] = retrieve_kw(kw, 'shuffle', True) self._multiStop = retrieve_kw(kw, 'doMultiStop', True) self.trainOptions['nEpochs'] = epochs self.trainOptions['nFails'] = maxFail #self._earlyStopping = callbacks.EarlyStopping( monitor='val_Tuning_L2Calo_SP_sp_value' # val_loss, self.trainOptions['metrics'][0] FIXME This must change # , patience=self.trainOptions['nFails'] # , verbose=0 # , mode='max') self._earlyStopping = callbacks.EarlyStopping( monitor='val_loss' # val_acc , patience=self.trainOptions['nFails'], verbose=0, mode='auto') self._historyCallback = PerformanceHistory( display=retrieve_kw(kw, 'showEvo', 50)) else: self._fatal("TuningWrapper not implemented for %s", coreConf) checkForUnusedVars(kw, self._debug) del kw # Set default empty values: if coreConf() is TuningToolCores.keras: self._emptyData = npCurrent.fp_array([]) elif coreConf() is TuningToolCores.FastNet: self._emptyData = list() self._emptyHandler = None if coreConf() is TuningToolCores.keras: self._emptyTarget = npCurrent.fp_array([[]]).reshape( npCurrent.access(pidx=1, oidx=0)) elif coreConf() is TuningToolCores.FastNet: self._emptyTarget = None # Set holders: self._trnData = self._emptyData self._valData = self._emptyData self._tstData = self._emptyData self._trnHandler = self._emptyHandler self._valHandler = self._emptyHandler self._tstHandler = self._emptyHandler self._trnTarget = self._emptyTarget self._valTarget = self._emptyTarget self._tstTarget = self._emptyTarget
def __call__(self, **kw): """ Create a collection of tuning job configuration files at the output folder. """ # Cross validation configuration outputFolder = retrieve_kw(kw, 'outputFolder', 'jobConfig') neuronBounds = retrieve_kw(kw, 'neuronBounds', SeqLoopingBounds(5, 20)) sortBounds = retrieve_kw(kw, 'sortBounds', PythonLoopingBounds(50)) nInits = retrieve_kw(kw, 'nInits', 100) # Output configuration nNeuronsPerJob = retrieve_kw(kw, 'nNeuronsPerJob', 1) nSortsPerJob = retrieve_kw(kw, 'nSortsPerJob', 1) nInitsPerJob = retrieve_kw(kw, 'nInitsPerJob', 100) compress = retrieve_kw(kw, 'compress', True) prefix = retrieve_kw(kw, 'prefix', 'job') # keras input model models = retrieve_kw(kw, 'models', None) # for keras only if models and coreConf() is TuningToolCores.keras: if not type(models) is list: models = [models] import json from keras.models import Sequential for idx, model in enumerate(models): if type(model) is str: models[idx] = json.loads(model) elif type(model) is Sequential: models[idx] = json.loads(model.to_json()) elif type(model) is dict: continue else: self._logger.fatal('model type is not supported') neuronBounds = [1, len(models)] if 'level' in kw: self.level = kw.pop('level') # Make sure that bounds variables are LoopingBounds objects: if not isinstance(neuronBounds, SeqLoopingBounds): neuronBounds = SeqLoopingBounds(neuronBounds) if not isinstance(sortBounds, SeqLoopingBounds): sortBounds = PythonLoopingBounds(sortBounds) # and delete it to avoid mistakes: checkForUnusedVars(kw, self._warning) del kw if nInits < 1: self._fatal(("Cannot require zero or negative initialization " "number."), ValueError) # Do some checking in the arguments: nNeurons = len(neuronBounds) nSorts = len(sortBounds) if not nSorts: self._fatal("Sort bounds is empty.") if nNeuronsPerJob > nNeurons: self._warning( ("The number of neurons per job (%d) is " "greater then the total number of neurons (%d), changing it " "into the maximum possible value."), nNeuronsPerJob, nNeurons) nNeuronsPerJob = nNeurons if nSortsPerJob > nSorts: self._warning( ("The number of sorts per job (%d) is " "greater then the total number of sorts (%d), changing it " "into the maximum possible value."), nSortsPerJob, nSorts) nSortsPerJob = nSorts # Create the output folder: mkdir_p(outputFolder) # Create the windows in which each job will loop upon: neuronJobsWindowList = \ CreateTuningJobFiles._retrieveJobLoopingBoundsCol( neuronBounds, nNeuronsPerJob ) sortJobsWindowList = \ CreateTuningJobFiles._retrieveJobLoopingBoundsCol( sortBounds, nSortsPerJob ) initJobsWindowList = \ CreateTuningJobFiles._retrieveJobLoopingBoundsCol( \ PythonLoopingBounds( nInits ), \ nInitsPerJob ) # Loop over windows and create the job configuration for idx, neuronWindowBounds in enumerate(neuronJobsWindowList()): for sortWindowBounds in sortJobsWindowList(): for initWindowBounds in initJobsWindowList(): modelWindowBounds = [] self._debug( ('Retrieved following job configuration ' '(bounds.vec) : ' '[ neuronBounds=%s, sortBounds=%s, initBounds=%s]'), neuronWindowBounds.formattedString('hn'), sortWindowBounds.formattedString('s'), initWindowBounds.formattedString('i')) fulloutput = '{outputFolder}/{prefix}.{neuronStr}.{sortStr}.{initStr}'.format( outputFolder=outputFolder, prefix=prefix, neuronStr=neuronWindowBounds.formattedString('hn'), sortStr=sortWindowBounds.formattedString('s'), initStr=initWindowBounds.formattedString('i')) for neuron in neuronWindowBounds: modelWindowBounds.append(models[neuron - 1] if models else None) savedFile = TuningJobConfigArchieve( fulloutput, modelBounds=modelWindowBounds, neuronBounds=neuronWindowBounds, sortBounds=sortWindowBounds, initBounds=initWindowBounds).save(compress) self._info('Saved job option configuration at path: %s', savedFile)
def trainC_Exp(self): """ Train expert feedforward neural network """ if coreConf() is TuningToolCores.ExMachina: self._fatal("Expert Neural Networks not implemented for ExMachina") elif coreConf() is TuningToolCores.FastNet: self._fatal("Expert Neural Networks not implemented for FastNet") elif coreConf() is TuningToolCores.keras: from copy import deepcopy # Set batch size: if self.batchMethod is BatchSizeMethod.MinClassSize: self.__batchSize(self._bkgSize if self._sgnSize > self._bkgSize else self._sgnSize) elif self.batchMethod is BatchSizeMethod.HalfSizeSignalClass: self.__batchSize(self._sgnSize // 2) elif self.batchMethod is BatchSizeMethod.OneSample: self.__batchSize(1) references = ['SP', 'Pd', 'Pf'] # Holder of the discriminators: tunedDiscrList = [] tuningInfo = {} for idx, ref in enumerate(references): rawDictTempl = {'discriminator': None, 'benchmark': None} history = self._model[ref].fit( self._trnData, self._trnTarget, epochs=self.trainOptions['nEpochs'], batch_size=self.batchSize, callbacks=[self._historyCallback, self._earlyStopping] #, callbacks = [self._earlyStopping] , verbose=0, validation_data=(self._valData, self._valTarget), shuffle=self.trainOptions['shuffle']) # Retrieve raw network rawDictTempl['discriminator'] = self.__expDiscr_to_dict( self._model[ref]) rawDictTempl['benchmark'] = self.references[idx] tunedDiscrList.append(deepcopy(rawDictTempl)) tuningInfo[ref] = DataTrainEvolution(history).toRawObj() try: from sklearn.metrics import roc_curve except ImportError: # FIXME Can use previous function that we used here as an alternative raise ImportError( "sklearn is not available, please install it.") # Retrieve performance: opRoc, tstRoc = Roc(), Roc() for idx, tunedDiscrDict in enumerate(tunedDiscrList): discr = tunedDiscrDict['discriminator'] if self.doPerf: self._debug('Retrieving performance for %s networks.' % (ref)) # propagate inputs: trnOutput = self._model[ref].predict(self._trnData) valOutput = self._model[ref].predict(self._valData) tstOutput = self._model[ref].predict( self._tstData ) if self._tstData else npCurrent.fp_array([]) try: allOutput = np.concatenate( [trnOutput, valOutput, tstOutput]) allTarget = np.concatenate([ self._trnTarget, self._valTarget, self._tstTarget ]) except ValueError: allOutput = np.concatenate([trnOutput, valOutput]) allTarget = np.concatenate( [self._trnTarget, self._valTarget]) # Retrieve Rocs: opRoc(allOutput, allTarget) if self._tstData: tstRoc(tstOutput, self._tstTarget) else: tstRoc(valOutput, self._valTarget) # Add rocs to output information # TODO Change this to raw object tunedDiscrDict['summaryInfo'] = { 'roc_operation': opRoc.toRawObj(), 'roc_test': tstRoc.toRawObj() } for ref2 in self.references: opPoint = opRoc.retrieve(ref2) tstPoint = tstRoc.retrieve(ref2) # Print information: self._info( '%s NETWORKS Operation (%s): sp = %f, pd = %f, pf = %f, thres = %f', ref, ref2.name, opPoint.sp_value, opPoint.pd_value, opPoint.pf_value, opPoint.thres_value) self._info( '%s NETWORKS Test (%s): sp = %f, pd = %f, pf = %f, thres = %f', ref, ref2.name, tstPoint.sp_value, tstPoint.pd_value, tstPoint.pf_value, tstPoint.thres_value) self._info("Finished trainC_Exp for %s networks." % (ref)) self._debug("Finished trainC_Exp on python side.") return tunedDiscrList, tuningInfo
def train_c(self): """ Train feedforward neural network """ from copy import deepcopy # Holder of the discriminators: tunedDiscrList = [] tuningInfo = {} # Set batch size: if self.batchMethod is BatchSizeMethod.MinClassSize: self.__batchSize(self._bkgSize if self._sgnSize > self._bkgSize else self._sgnSize) elif self.batchMethod is BatchSizeMethod.HalfSizeSignalClass: self.__batchSize(self._sgnSize // 2) elif self.batchMethod is BatchSizeMethod.OneSample: self.__batchSize(1) rawDictTempl = {'discriminator': None, 'benchmark': None} if coreConf() is TuningToolCores.keras: history = self._model.fit( self._trnData, self._trnTarget #, epochs = self.trainOptions['nEpochs'] , batch_size=self.batchSize #, callbacks = [self._historyCallback, self._earlyStopping] , callbacks=[self._earlyStopping], verbose=0, validation_data=(self._valData, self._valTarget), shuffle=self.trainOptions['shuffle']) # Retrieve raw network rawDictTempl['discriminator'] = self.__discr_to_dict(self._model) rawDictTempl['benchmark'] = self.references[0] tunedDiscrList.append(deepcopy(rawDictTempl)) tuningInfo = DataTrainEvolution(history).toRawObj() try: from sklearn.metrics import roc_curve except ImportError: # FIXME Can use previous function that we used here as an alternative raise ImportError( "sklearn is not available, please install it.") elif coreConf() is TuningToolCores.FastNet: self._debug('executing train_c') [discriminatorPyWrapperList, trainDataPyWrapperList] = self._core.train_c() self._debug('finished train_c') # Transform model tolist of dict if self.doMultiStop: for idx, discr in enumerate(discriminatorPyWrapperList): rawDictTempl['discriminator'] = self.__discr_to_dict(discr) rawDictTempl['benchmark'] = self.references[idx] # FIXME This will need to be improved if set to tune for multiple # Pd and Pf values. tunedDiscrList.append(deepcopy(rawDictTempl)) else: rawDictTempl['discriminator'] = self.__discr_to_dict( discriminatorPyWrapperList[0]) rawDictTempl['benchmark'] = self.references[0] if self.useTstEfficiencyAsRef and self.sortIdx is not None: rawDictTempl['sortIdx'] = self.sortIdx tunedDiscrList.append(deepcopy(rawDictTempl)) tuningInfo = DataTrainEvolution(trainDataPyWrapperList).toRawObj() # TODO # cores # Retrieve performance: opRoc, tstRoc = Roc(), Roc() for idx, tunedDiscrDict in enumerate(tunedDiscrList): discr = tunedDiscrDict['discriminator'] if self.doPerf: self._debug('Retrieving performance.') if coreConf() is TuningToolCores.keras: # propagate inputs: trnOutput = self._model.predict(self._trnData) valOutput = self._model.predict(self._valData) tstOutput = self._model.predict( self._tstData ) if self._tstData else npCurrent.fp_array([]) try: allOutput = np.concatenate( [trnOutput, valOutput, tstOutput]) allTarget = np.concatenate([ self._trnTarget, self._valTarget, self._tstTarget ]) except ValueError: allOutput = np.concatenate([trnOutput, valOutput]) allTarget = np.concatenate( [self._trnTarget, self._valTarget]) # Retrieve Rocs: opRoc(allOutput, allTarget) if self._tstData: tstRoc(tstOutput, self._tstTarget) else: tstRoc(valOutput, self._valTarget) elif coreConf() is TuningToolCores.FastNet: perfList = self._core.valid_c( discriminatorPyWrapperList[idx]) opRoc(perfList[1]) tstRoc(perfList[0]) # Add rocs to output information # TODO Change this to raw object tunedDiscrDict['summaryInfo'] = { 'roc_operation': opRoc.toRawObj(), 'roc_test': tstRoc.toRawObj() } for ref in self.references: if coreConf() is TuningToolCores.FastNet: # FastNet won't loop on this, this is just looping for keras right now ref = tunedDiscrDict['benchmark'] opPoint = opRoc.retrieve(ref) tstPoint = tstRoc.retrieve(ref) # Print information: self._info( 'Operation (%s): sp = %f, pd = %f, pf = %f, thres = %f', ref.name, opPoint.sp_value, opPoint.pd_value, opPoint.pf_value, opPoint.thres_value) self._info( 'Test (%s): sp = %f, pd = %f, pf = %f, thres = %f', ref.name, tstPoint.sp_value, tstPoint.pd_value, tstPoint.pf_value, tstPoint.thres_value) if coreConf() is TuningToolCores.FastNet: break self._debug("Finished train_c on python side.") return tunedDiscrList, tuningInfo
def newExpff(self, nodes, et, eta, sort, funcTrans=NotSet): """ Creates new feedfoward neural network from expert calorimeter and tracking networks. """ self._debug('Initalizing newExpff...') models = {} if coreConf() is TuningToolCores.ExMachina: self._fatal("Expert Neural Networks not implemented for ExMachina") elif coreConf() is TuningToolCores.FastNet: self._fatal("Expert Neural Networks not implemented for FastNet") elif coreConf() is TuningToolCores.keras: from keras.models import Sequential from keras.layers import Merge from keras.layers.core import Dense, Dropout, Activation references = ['Pd', 'Pf', 'SP'] if len(self.networks[1][et][eta]) == 1: ref = 'SP' track_n = [ self.__dict_to_discr( self.networks[1][et][eta][ref]['sort_%1.3i' % (sort)], 'track', pruneLastLayer=True) ] else: track_n = {} for ref in references: track_n[ref] = self.__dict_to_discr( self.networks[1][et][eta][ref]['sort_%1.3i' % (sort)], 'track', pruneLastLayer=True) calo_nn = {} # from RingerCore import keyboard # keyboard() for ref in references: calo_nn = self.__dict_to_discr( self.networks[0][et][eta][ref]['sort_%1.3i' % (sort)], 'calo', pruneLastLayer=True) ## Extracting last layers if len(track_n) == 1: from copy import deepcopy track_nn = deepcopy(track_n[0]) else: track_nn = track_n[ref] merg_layer = Merge([calo_nn, track_nn], mode='concat', concat_axis=-1, name='merge_layer') ## Merged Model model = Sequential() model.add(merg_layer) names = ['merge_dense_1', 'merge_dense_2'] # NOTE: verify if there is nedd of a 'merge_dense_0' with identity model.add( Dense(nodes[1], kernel_initializer='uniform', name=names[0])) model.add(Activation('tanh')) model.add( Dense(nodes[2], kernel_initializer='uniform', input_dim=nodes[1], trainable=True, name=names[1])) model.add(Activation('tanh')) model.compile(loss=self.trainOptions['costFunction'], optimizer=self.trainOptions['optmin_alg'], metrics=self.trainOptions['metrics']) models[ref] = model self._model = models # FIXME: check historycallback compatibility self._historyCallback.model = models