def trainModel(self,
nepochs,
batchsize,
stop_patience=300,
lr_factor=0.5,
lr_patience=2,
lr_epsilon=0.003,
lr_cooldown=6,
lr_minimum=0.000001,
maxqsize=20,
**trainargs):
#make sure tokens don't expire
from tokenTools import checkTokens, renew_token_process
from thread import start_new_thread
checkTokens()
start_new_thread(renew_token_process, ())
self.train_data.setBatchSize(batchsize)
self.val_data.setBatchSize(batchsize)
self.keras_model.save(self.outputDir + 'KERAS_check_last_model.h5')
from DeepJet_callbacks import DeepJet_callbacks
callbacks = DeepJet_callbacks(stop_patience=stop_patience,
lr_factor=lr_factor,
lr_patience=lr_patience,
lr_epsilon=lr_epsilon,
lr_cooldown=lr_cooldown,
lr_minimum=lr_minimum,
outputDir=self.outputDir)
nepochs = nepochs - self.trainedepoches
self.keras_model.fit_generator(
self.train_data.generator(),
steps_per_epoch=self.train_data.getNBatchesPerEpoch(),
epochs=nepochs,
callbacks=callbacks.callbacks,
validation_data=self.val_data.generator(),
validation_steps=self.val_data.getNBatchesPerEpoch(), #)#,
max_q_size=maxqsize,
**trainargs)
self.saveModel("KERAS_model.h5")
return self.keras_model, callbacks.history, callbacks #added callbacks
def on_epoch_begin(self, epoch, logs=None):
from tokenTools import checkTokens
checkTokens(self.cutofftime_hours)
def trainModel(self,
nepochs,
batchsize,
stop_patience=300,
lr_factor=0.5,
lr_patience=2,
lr_epsilon=0.003,
lr_cooldown=6,
lr_minimum=0.000001,
maxqsize=20,
checkperiod=10,
**trainargs):
#make sure tokens don't expire
from tokenTools import checkTokens, renew_token_process
from thread import start_new_thread
print('starting afs backgrounder')
checkTokens()
start_new_thread(renew_token_process, ())
self.train_data.setBatchSize(batchsize)
self.val_data.setBatchSize(batchsize)
averagesamplesperfile = self.train_data.getAvEntriesPerFile()
samplespreread = maxqsize * batchsize
nfilespre = max(int(samplespreread / averagesamplesperfile), 2)
nfilespre += 1
#if nfilespre>15:nfilespre=15
print('best pre read: ' + str(nfilespre) + ' a: ' +
str(averagesamplesperfile))
print('total sample size: ' + str(self.train_data.nsamples))
print('max files open: ', self.train_data.maxFilesOpen)
#exit()
if self.train_data.maxFilesOpen < 0:
self.train_data.maxFilesOpen = nfilespre
self.val_data.maxFilesOpen = min(int(nfilespre / 2), 1)
#self.keras_model.save(self.outputDir+'KERAS_check_last_model.h5')
print('setting up callbacks')
from DeepJet_callbacks import DeepJet_callbacks
callbacks = DeepJet_callbacks(self.keras_model,
self.train_data,
stop_patience=stop_patience,
lr_factor=lr_factor,
lr_patience=lr_patience,
lr_epsilon=lr_epsilon,
lr_cooldown=lr_cooldown,
lr_minimum=lr_minimum,
outputDir=self.outputDir,
checkperiod=checkperiod)
nepochs = nepochs - self.trainedepoches
print('starting training')
self.keras_model.fit_generator(
self.train_data.generator(),
steps_per_epoch=self.train_data.getNBatchesPerEpoch(),
epochs=nepochs,
callbacks=callbacks.callbacks,
validation_data=self.val_data.generator(),
validation_steps=self.val_data.getNBatchesPerEpoch(), #)#,
max_q_size=maxqsize,
**trainargs)
self.saveModel("KERAS_model.h5")
import copy
#reset all file reads etc
tmpdc = copy.deepcopy(self.train_data)
del self.train_data
self.train_data = tmpdc
return self.keras_model, callbacks.history
testd.isTrain = False
traind.isTrain = True
print 'split to %d train samples and %d test samples' % (ntrainepoch, nvalepoch)
#for bookkeeping
traind.setBatchSize(config_args['batchsize'])
testd.setBatchSize(500)
traind.writeToFile(outputDir+'trainsamples.dc')
testd.writeToFile( outputDir+'valsamples.dc')
#make sure tokens don't expire
from tokenTools import checkTokens, renew_token_process
from thread import start_new_thread
checkTokens()
start_new_thread(renew_token_process,())
print 'training'
try:
model.fit_generator(
modifier(traind.generator()),
verbose=1,
steps_per_epoch = traind.getNBatchesPerEpoch(),
epochs = config_args['nepochs'],
callbacks = callbacks.callbacks,
validation_data = modifier(testd.generator()),
validation_steps = testd.getNBatchesPerEpoch(),
max_q_size = config_args['maxqsize'], #maximum size for the generator queue
class_weight = class_weight,
)
