def trainModel(self, nepochs, batchsize, stop_patience=300, lr_factor=0.5, lr_patience=2, lr_epsilon=0.003, lr_cooldown=6, lr_minimum=0.000001, maxqsize=20, **trainargs): #make sure tokens don't expire from tokenTools import checkTokens, renew_token_process from thread import start_new_thread checkTokens() start_new_thread(renew_token_process, ()) self.train_data.setBatchSize(batchsize) self.val_data.setBatchSize(batchsize) self.keras_model.save(self.outputDir + 'KERAS_check_last_model.h5') from DeepJet_callbacks import DeepJet_callbacks callbacks = DeepJet_callbacks(stop_patience=stop_patience, lr_factor=lr_factor, lr_patience=lr_patience, lr_epsilon=lr_epsilon, lr_cooldown=lr_cooldown, lr_minimum=lr_minimum, outputDir=self.outputDir) nepochs = nepochs - self.trainedepoches self.keras_model.fit_generator( self.train_data.generator(), steps_per_epoch=self.train_data.getNBatchesPerEpoch(), epochs=nepochs, callbacks=callbacks.callbacks, validation_data=self.val_data.generator(), validation_steps=self.val_data.getNBatchesPerEpoch(), #)#, max_q_size=maxqsize, **trainargs) self.saveModel("KERAS_model.h5") return self.keras_model, callbacks.history, callbacks #added callbacks
def on_epoch_begin(self, epoch, logs=None): from tokenTools import checkTokens checkTokens(self.cutofftime_hours)
def trainModel(self, nepochs, batchsize, stop_patience=300, lr_factor=0.5, lr_patience=2, lr_epsilon=0.003, lr_cooldown=6, lr_minimum=0.000001, maxqsize=20, checkperiod=10, **trainargs): #make sure tokens don't expire from tokenTools import checkTokens, renew_token_process from thread import start_new_thread print('starting afs backgrounder') checkTokens() start_new_thread(renew_token_process, ()) self.train_data.setBatchSize(batchsize) self.val_data.setBatchSize(batchsize) averagesamplesperfile = self.train_data.getAvEntriesPerFile() samplespreread = maxqsize * batchsize nfilespre = max(int(samplespreread / averagesamplesperfile), 2) nfilespre += 1 #if nfilespre>15:nfilespre=15 print('best pre read: ' + str(nfilespre) + ' a: ' + str(averagesamplesperfile)) print('total sample size: ' + str(self.train_data.nsamples)) print('max files open: ', self.train_data.maxFilesOpen) #exit() if self.train_data.maxFilesOpen < 0: self.train_data.maxFilesOpen = nfilespre self.val_data.maxFilesOpen = min(int(nfilespre / 2), 1) #self.keras_model.save(self.outputDir+'KERAS_check_last_model.h5') print('setting up callbacks') from DeepJet_callbacks import DeepJet_callbacks callbacks = DeepJet_callbacks(self.keras_model, self.train_data, stop_patience=stop_patience, lr_factor=lr_factor, lr_patience=lr_patience, lr_epsilon=lr_epsilon, lr_cooldown=lr_cooldown, lr_minimum=lr_minimum, outputDir=self.outputDir, checkperiod=checkperiod) nepochs = nepochs - self.trainedepoches print('starting training') self.keras_model.fit_generator( self.train_data.generator(), steps_per_epoch=self.train_data.getNBatchesPerEpoch(), epochs=nepochs, callbacks=callbacks.callbacks, validation_data=self.val_data.generator(), validation_steps=self.val_data.getNBatchesPerEpoch(), #)#, max_q_size=maxqsize, **trainargs) self.saveModel("KERAS_model.h5") import copy #reset all file reads etc tmpdc = copy.deepcopy(self.train_data) del self.train_data self.train_data = tmpdc return self.keras_model, callbacks.history
testd.isTrain = False traind.isTrain = True print 'split to %d train samples and %d test samples' % (ntrainepoch, nvalepoch) #for bookkeeping traind.setBatchSize(config_args['batchsize']) testd.setBatchSize(500) traind.writeToFile(outputDir+'trainsamples.dc') testd.writeToFile( outputDir+'valsamples.dc') #make sure tokens don't expire from tokenTools import checkTokens, renew_token_process from thread import start_new_thread checkTokens() start_new_thread(renew_token_process,()) print 'training' try: model.fit_generator( modifier(traind.generator()), verbose=1, steps_per_epoch = traind.getNBatchesPerEpoch(), epochs = config_args['nepochs'], callbacks = callbacks.callbacks, validation_data = modifier(testd.generator()), validation_steps = testd.getNBatchesPerEpoch(), max_q_size = config_args['maxqsize'], #maximum size for the generator queue class_weight = class_weight, )