data_type='train',
labels=labels)
genVal = DataGenerator(imagesMeta=data.valMeta,
GTMeta=data.trainGTMeta,
cfg=cfg,
data_type='val',
labels=labels)
genTest = DataGenerator(imagesMeta=data.testMeta,
GTMeta=data.testGTMeta,
cfg=cfg,
data_type='test',
labels=labels)
if True:
# Save config
utils.saveConfig(cfg)
utils.saveSplit(cfg, list(data.trainMeta.keys()),
list(data.valMeta.keys()))
# Create model
print('Creating model...')
model = Fast_HO_RCNN(cfg)
trainer = model_trainer(model=model,
genTrain=genTrain,
genVal=genVal,
genTest=genTest,
task=cfg.task)
trainer.compileModel(cfg)
if True:
def main():
# loading config file ...
cfgPath = sys.argv[1] if len(sys.argv) > 1 else './config.toml'
cfg = loadConfig(cfgPath)
try:
# ... and unpacking variables
dictget = lambda d, *k: [d[i] for i in k]
dataStats = cfg['data_stats']
modelParams = cfg['model_params']
trainCSV, testCSV = dictget(cfg['database'], 'train', 'test')
seqLength, stepSize = dictget(cfg['model_params'], 'seqLength',
'stepSize')
modelArch, modelDir, modelName = dictget(cfg['model_arch'],
'modelArch', 'modelDir',
'modelName')
optimizer, lossFunc, metricFuncs = dictget(cfg['training_params'],
'optimizer', 'lossFunc',
'metricFuncs')
lr, epochs, batchSize, patience, = dictget(cfg['training_params'],
'learningRate', 'epochs',
'batchSize', 'patience')
except KeyError as err:
print("\n\nERROR: not all parameters defined in config.toml : ", err)
print("Exiting ... \n\n")
sys.exit(1)
print("Loading training data ...")
xTrain, yTrain, stats = getData(trainCSV,
seqLength=seqLength,
stepSize=stepSize,
stats=dataStats)
print("Training Data Shape : ", xTrain.shape, "\n")
print("Loading testing data ...")
xTest, yTest, stats = getData(testCSV,
seqLength=seqLength,
stepSize=stepSize,
stats=dataStats)
print("Testing Data Shape : ", xTest.shape, "\n")
yTrain = np.expand_dims(
yTrain, -1) # adding extra axis as model expects 2 axis in the output
yTest = np.expand_dims(yTest, -1)
print("Compiling Model")
opt = getOptimizer(optimizer, lr)
model = makeModel(modelArch, modelParams, verbose=True)
model.compile(loss=lossFunc, optimizer=opt, metrics=metricFuncs)
# setting up directories
modelFolder = os.path.join(modelDir, modelName)
weightsFolder = os.path.join(modelFolder, "weights")
bestModelPath = os.path.join(weightsFolder, "best.hdf5")
ensureDir(bestModelPath)
saveConfig(cfgPath, modelFolder)
# callbacks
monitorMetric = 'val_loss'
check1 = ModelCheckpoint(os.path.join(weightsFolder,
modelName + "_{epoch:03d}.hdf5"),
monitor=monitorMetric,
mode='auto')
check2 = ModelCheckpoint(bestModelPath,
monitor=monitorMetric,
save_best_only=True,
mode='auto')
check3 = EarlyStopping(monitor=monitorMetric,
min_delta=0.01,
patience=patience,
verbose=0,
mode='auto')
check4 = CSVLogger(os.path.join(modelFolder,
modelName + '_trainingLog.csv'),
separator=',',
append=True)
check5 = ReduceLROnPlateau(monitor=monitorMetric,
factor=0.1,
patience=patience // 3,
verbose=1,
mode='auto',
min_delta=0.001,
cooldown=0,
min_lr=1e-10)
cb = [check2, check3, check4, check5]
if cfg['training_params']['saveAllWeights']:
cb.append(check1)
print("Starting Training ...")
model.fit(x=xTrain,
y=yTrain,
batch_size=batchSize,
epochs=epochs,
verbose=1,
callbacks=cb,
validation_data=(xTest, yTest),
shuffle=True)