def present_results(self, logs):
"""
Plot accuracy and loss figures
Do this for last model which is result of all training epochs. This model is probably overtrained.
Do this for best model which is the one with highest success metric.
:param logs:
:return:
"""
_, validation_generator = self.generators
present_results_generator(self.work_dir,
self.model,
logs,
validation_generator,
self.VALID_DATA_COUNT,
classes=self.class_indices,
suffix='last',
train_top_epochs=(self.config.top_epochs if
self.train_top else None))
best_model_weights = self.get_best_trained_model_weights()
best_model = self.load_model_from_file(best_model_weights)
present_results_generator(self.work_dir,
best_model,
None,
validation_generator,
self.VALID_DATA_COUNT,
classes=self.class_indices,
suffix='best')
def train(work_dir, data_dir,weights_file, seq_length, image_dim,batch_size, lr, optimizer):
if os.path.exists(work_dir):
print("Work dir exists. Choose another!")
else:
os.makedirs(work_dir)
os.makedirs(work_dir + '/logs')
os.makedirs(work_dir + '/checkpoints')
# save configuration parameters
with open(work_dir + "/params.txt", 'w') as f:
f.write('\n')
f.write('DATA_DIR=' + str(data_dir))
f.write('\n')
f.write('BATCH_SIZE=' + str(batch_size))
f.write('\n')
f.write('DIMENSION=' + str(image_dim))
f.write('\n')
f.write('LR=' + str(lr))
f.write('\n')
f.write('OPTIMIZER=' + str(optimizer))
f.write('\n')
csv_file_path=data_dir+'/data.csv'
generator=SequenceImageGenerator(rescale=1.0/255)
train_generator=generator.flow_from_csv(csv_file_path,True,batch_size=batch_size,target_size=image_dim,nb_seq=seq_length)
val_generator=generator.flow_from_csv(csv_file_path,False,batch_size=1,target_size=image_dim,nb_seq=seq_length)
NB_TRAIN_SAMPLES=train_generator.samples
NB_VAL_SAMPLES=val_generator.samples
EPOCHS=2000
callbacks=get_callbacks(work_dir)
model = get_model(weights_file,seq_length,image_dim,train_generator.num_class)
#model.fit_generator(train_generator,NB_TRAIN_SAMPLES/batch_size,epochs=10,metrics=['accuracy'], callbacks=[],validation_data=val_generator,validation_steps=NB_VAL_SAMPLES)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
history=model.fit_generator(train_generator, NB_TRAIN_SAMPLES / batch_size, epochs=EPOCHS, callbacks=callbacks,
validation_data=val_generator, validation_steps=NB_VAL_SAMPLES)
present_results_generator(work_dir,model,history,val_generator,NB_VAL_SAMPLES,classes=val_generator.classes)
def main():
work_dir = '/mnt/sda2/dev_root/work2.1/merged/lstm_c3d/4'
model_files = [
'/mnt/sda2/dev_root/work2/c3d/7t-c3d__16_112_adam_b8_1lr1e6/checkpoints/w.014-0.3507-2.81.hdf5',
'/mnt/sda2/dev_root/work2/lstm/2_lstm_40_224/checkpoints/w.029-0.4563-1.58.hdf5'
]
csv_file_path_c3d = '/mnt/sda2/dev_root/dataset/AFEW-Processed/AFEW_PFaces_16_112_small/data.csv'
csv_file_path_lstm = '/mnt/sda2/dev_root/dataset/AFEW-Processed/AFEW_Features_vggface_fc7_224/data.csv'
class_weight_dict = None
seq_length_lstm = 40
seq_length_c3d = 16
VAL_BATCH_SIZE = 1
nb_epoch = 1000
nb_classes = 7
batch_size = 32
image_dim = 112
lr = 0.0001
if os.path.exists(work_dir):
raise Exception("Work dir exists. Choose another!")
else:
os.makedirs(work_dir)
os.makedirs(work_dir + '/logs')
os.makedirs(work_dir + '/checkpoints')
# save configuration parameters
with open(work_dir + "/params.txt", 'w') as f:
f.write('\n')
f.write('BATCH_SIZE=' + str(batch_size))
f.write('\n')
f.write('DIMENSION=' + str(image_dim))
f.write('\n')
f.write('LR=' + str(lr))
f.write('\n')
f.write('OPTIMIZER=' + 'sgd')
f.write('\n')
models = []
#for model_file in model_files:
# model = load_model(model_file)
# models.append(model)
merged_model = get_merge_dense_only_model()
sgd = SGD(lr=lr, momentum=0.9, decay=0, nesterov=False)
merged_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
#image_data_generator = ImageDataGenerator(rescale=1.0/255)
#image_generator=SequenceImageIGenerator(seq_length_c3d,image_data_generator,(image_dim,image_dim))
#feature__generator = SequenceFeatureIGenerator(seq_length_lstm)
#merged_generator_train=MergeGenerator([image_generator,feature__generator],[csv_file_path_c3d,csv_file_path_lstm],True,batch_size)
#merged_generator_valid=MergeGenerator([image_generator,feature__generator],[csv_file_path_c3d,csv_file_path_lstm],False,VAL_BATCH_SIZE)
feature_generator = NonSequenceFeatureIGenerator()
train_generator = feature_generator.flow_from_csv_file(
'/mnt/sda2/dev_root/dataset/combined/merged/lstmc3d/data.csv', 32,
True)
val_generator = feature_generator.flow_from_csv_file(
'/mnt/sda2/dev_root/dataset/combined/merged/lstmc3d/data.csv', 1,
False)
history = merged_model.fit_generator(
train_generator,
steps_per_epoch=len(train_generator.data) / batch_size,
validation_data=val_generator,
validation_steps=len(val_generator.data) / VAL_BATCH_SIZE,
epochs=nb_epoch,
callbacks=get_callbacks(work_dir),
class_weight=class_weight_dict)
list_of_model_files = glob.glob(work_dir + '/checkpoints/*.hdf5')
latest_model_file = max(list_of_model_files, key=os.path.getctime)
latest_model = load_model(latest_model_file)
present_results_generator(work_dir,
latest_model,
history,
val_generator,
len(val_generator.data),
classes=val_generator.class_indices)
return
def main():
work_dir = '/mnt/sda2/dev_root/work2.1/merged/lstm_c3d/13-merge-data-top-only-model-single-layer-adagrad'
model_files = [
'/mnt/sda2/dev_root/work2/c3d/7t-c3d__16_112_adam_b8_1lr1e6/checkpoints/w.014-0.3507-2.81.hdf5',
'/mnt/sda2/dev_root/work2/lstm/2_lstm_40_224/checkpoints/w.029-0.4563-1.58.hdf5'
]
csv_file_path_c3d = '/mnt/sda2/dev_root/dataset/AFEW-Processed/AFEW_PFaces_16_112_small/data.csv'
csv_file_path_lstm = '/mnt/sda2/dev_root/dataset/AFEW-Processed/AFEW_Features_vggface_fc7_224/data.csv'
class_weight_dict = None
seq_length_lstm = 40
seq_length_c3d = 16
VAL_BATCH_SIZE = 1
nb_epoch = 1000
nb_classes = 7
batch_size = 32
image_dim = 112
lr = 0.001
optimizer = SGD(lr=lr, momentum=0.9, decay=0.0005)
optimizer = 'adagrad'
if os.path.exists(work_dir):
print("Work dir exists. Choose another!")
raise Exception("Work dir exists. Choose another!")
else:
os.makedirs(work_dir)
os.makedirs(work_dir + '/logs')
os.makedirs(work_dir + '/checkpoints')
# save configuration parameters
with open(work_dir + "/params.txt", 'w') as f:
f.write('\n')
f.write('BATCH_SIZE=' + str(batch_size))
f.write('\n')
f.write('DIMENSION=' + str(image_dim))
f.write('\n')
f.write('LR=' + str(lr))
f.write('\n')
f.write('OPTIMIZER=' + str(optimizer))
f.write('\n')
models = []
for model_file in model_files:
model = load_model(model_file)
models.append(model)
merged_model = merge_models(nb_classes)
merged_model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
with open(work_dir + "/params.txt", 'a') as f:
merged_model.summary(print_fn=lambda x: f.write(x + '\n'))
image_data_generator = ImageDataGenerator(rescale=1.0 / 255)
image_generator = SequenceImageIGenerator(seq_length_c3d,
image_data_generator,
(image_dim, image_dim))
feature__generator = SequenceFeatureIGenerator(seq_length_lstm)
merged_generator_train = InFlightMergeGenerator(
models, [image_generator, feature__generator],
[csv_file_path_c3d, csv_file_path_lstm], True, batch_size).flow()
merged_generator_valid = InFlightMergeGenerator(
models, [image_generator, feature__generator],
[csv_file_path_c3d, csv_file_path_lstm], False, VAL_BATCH_SIZE).flow()
image_generator_valid = merged_generator_valid.igenerators[0]
feature_generator_valid = merged_generator_valid.igenerators[1]
c3d_model = models[0]
lstm_model = models[1]
#present_results_generator(work_dir, c3d_model, None, image_generator_valid, len(merged_generator_valid.data),classes=merged_generator_valid.class_indices, suffix='c3d')
#present_results_generator(work_dir, lstm_model, None, feature_generator_valid, len(merged_generator_valid.data),classes=merged_generator_valid.class_indices, suffix='lstm')
history = merged_model.fit_generator(
merged_generator_train,
steps_per_epoch=len(merged_generator_train.data) / batch_size,
validation_data=merged_generator_valid,
validation_steps=len(merged_generator_valid.data) / VAL_BATCH_SIZE,
epochs=nb_epoch,
callbacks=get_callbacks(work_dir),
class_weight=class_weight_dict)
present_results_generator(work_dir,
merged_model,
history,
merged_generator_valid,
len(merged_generator_valid.data),
classes=merged_generator_valid.class_indices,
suffix='last')
present_results_generator(work_dir,
c3d_model,
None,
image_generator_valid,
len(merged_generator_valid.data),
classes=merged_generator_valid.class_indices,
suffix='c3d-post')
present_results_generator(work_dir,
lstm_model,
None,
feature_generator_valid,
len(merged_generator_valid.data),
classes=merged_generator_valid.class_indices,
suffix='lstm-post')
list_of_model_files = glob.glob(work_dir + '/checkpoints/*.hdf5')
latest_model_file = max(list_of_model_files, key=os.path.getctime)
latest_model = load_model(latest_model_file)
present_results_generator(work_dir,
latest_model,
None,
merged_generator_valid,
len(merged_generator_valid.data),
classes=merged_generator_valid.class_indices)
return
def train(work_dir, data_dir, seq_length, batch_size, lr, optimizer):
if os.path.exists(work_dir):
raise Exception("Work dir exists. Choose another!")
else:
os.makedirs(work_dir)
os.makedirs(work_dir + '/logs')
os.makedirs(work_dir + '/checkpoints')
# save configuration parameters
with open(work_dir + "/params.txt", 'w') as f:
f.write('\n')
f.write('DATA_DIR=' + str(data_dir))
f.write('\n')
f.write('BATCH_SIZE=' + str(batch_size))
f.write('\n')
f.write('SEQ_LENGTH=' + str(seq_length))
f.write('\n')
f.write('LR=' + str(lr))
f.write('\n')
f.write('OPTIMIZER=' + str(optimizer))
f.write('\n')
csv_file_path = data_dir + '/data.csv'
generator = SequenceFeatureGenerator()
train_generator = generator.flow_from_csv(csv_file_path,
True,
batch_size=batch_size,
nb_seq=seq_length)
val_generator = generator.flow_from_csv(csv_file_path,
False,
batch_size=1,
nb_seq=seq_length)
NB_TRAIN_SAMPLES = train_generator.samples
NB_VAL_SAMPLES = val_generator.samples
EPOCHS = 2000
callbacks = get_callbacks(work_dir)
model = get_model(train_generator.num_class,
(seq_length, train_generator.nb_feature))
for layer in model.layers:
layer.trainable = True
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
history = model.fit_generator(train_generator,
NB_TRAIN_SAMPLES / batch_size,
epochs=EPOCHS,
callbacks=callbacks,
validation_data=val_generator,
validation_steps=NB_VAL_SAMPLES)
present_results_generator(work_dir,
model,
history,
val_generator,
NB_VAL_SAMPLES,
classes=val_generator.class_indices.keys())
def train(source_dir, work_root_dir, data_type, seq_length, model, saved_model=None,
concat=False, image_shape=None, load_to_memory=False):
if not os.path.exists(work_root_dir):
os.makedirs(work_root_dir)
os.makedirs(work_root_dir+'/checkpoints')
os.makedirs(work_root_dir+'/logs')
# Set variables.
nb_epoch = 2000
batch_size = 8
data_file=source_dir+'/data.csv'
# Helper: TensorBoard
tb = TensorBoard(log_dir=work_root_dir+'/logs')
# Helper: Save the model.
checkpointer = ModelCheckpoint(save_weights_only=False,
filepath=work_root_dir + '/checkpoints/w.{epoch:03d}-{val_acc:.4f}-{val_loss:.2f}.hdf5',
verbose=1,
save_best_only=True, monitor='val_acc')
# Helper: Stop when we stop learning.
early_stopper = EarlyStopping(patience=20, monitor='val_acc')
# Helper: Save results.
timestamp = time.time()
csv_logger = CSVLogger(work_root_dir+'/logs/' + model + '-' + 'training-' + \
str(timestamp) + '.log')
# Get the data and process it.
if image_shape is None:
features_length = determine_feature_count(source_dir)
data = DataSet(
data_file=data_file,
sequence_dir=source_dir,
seq_length=seq_length,
class_limit=None
,given_classes=None
)
else:
features_length = None
data = DataSet(
data_file=data_file,
sequence_dir=source_dir,
seq_length=seq_length,
class_limit=None,
image_shape=image_shape
, given_classes=None
)
# Get samples per epoch.
# Multiply by 0.7 to attempt to guess how much of data.data is the train set.
steps_per_epoch = (len(data.data) * 0.7) // batch_size
X=None
y=None
X_test=None
y_test=None
generator=None
val_generator=None
if load_to_memory:
# Get data.
X, y = data.get_all_sequences_in_memory(True, data_type, concat)
X_test, y_test = data.get_all_sequences_in_memory(False, data_type, concat)
print ("Train samples %d, test samples %d"%(len(X),len(X_test)))
else:
# Get generators.
generator = data.frame_generator(batch_size, True, data_type, concat)
val_generator = data.frame_generator(1, False, data_type, concat)
# Get the model.
rm = ResearchModels(len(data.classes), model, seq_length, saved_model,features_length=features_length,dimension=image_shape)
# Fit!
if load_to_memory:
# Use standard fit.
history=rm.model.fit(
X,
y,
batch_size=batch_size,
validation_data=(X_test, y_test),
verbose=1,
callbacks=[checkpointer, tb, early_stopper, csv_logger],
epochs=nb_epoch)
else:
# Use fit generator.
history = rm.model.fit_generator(
generator=generator,
steps_per_epoch=steps_per_epoch,
epochs=nb_epoch,
verbose=1,
callbacks=[checkpointer, tb, early_stopper, csv_logger],
validation_data=val_generator,
validation_steps=365)
if val_generator:
_, test=data.split_train_test()
present_results_generator(work_root_dir, rm.model, history, val_generator, len(test),classes=data.classes)
else:
present_results(work_root_dir, rm.model,history, X_test=X_test, Y_test=y_test, classes=data.classes)