def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'inceptionv3',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# Create + compile model, load saved weights if they exist
saved_weights = None
# saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
model, keras_layer_names = rgb_create_model(
classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
model_name=params['model'],
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = None
ucf_weights = "a"
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if ucf_weights is None:
print("Loading MConvNet weights: ")
if params['model'] == "resnet50":
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
)
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading Keras weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_rgb_params_names.pkl",
"../models/kinetics_keras/tsn_rgb_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5"
)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Make spltis
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
# TODO Don't forget to change your names :)
filter_type = "gauss"
bestModelPath = "../models/rgb_kininit_" + filter_type + \
"_" + params['model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/rgb_kininit_train_" + filter_type + \
"_plot_" + params['model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/rgb_kininit_val_" + filter_type + \
"_plot_" + params['model'] + "_" + time_str + ".csv"
with tf.device('/gpu:0'): # NOTE Not using multi gpu
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
# Load and train
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
filter_type,
soft_sigmoid=soft_sigmoid)
y_t = []
y_t.append(
to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# TODO Repeat samples of unrepresented classes?
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
# global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
filter_type,
soft_sigmoid=soft_sigmoid)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
# Average over all validation chunks
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
# We consider accuracy as the average accuracy over the three
# types of accuracy
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training RGB-stream ',
message='Training RGB with following params: ' +
str(params),
login='******',
password='******')
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'resnet50',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# http://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_class_weight.html
# Logistic Regression in Rare Events Data, Gary King, Langche Zeng
# Keras needs a dict though
# From documentation: class_weight: Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only).
# This can be useful to tell the model to "pay more attention" to samples from an under-represented class.
y = labels_to_numpy(labels_train)
penalizing_method = 'balanced'
mu = 0.7
# penalizing_method = 'weighted_log'
class_weights = np.zeros(30)
for i in y:
class_weights[i] += 1
max_class = max(class_weights)
for i in range(len(class_weights)):
if class_weights[i] != 0.0:
if penalizing_method == 'balanced':
print(
str(i) + " " + str(class_weights[i]) + " " +
str(len(y) / (class_weights[i])))
#class_weights[i] = len(y) / (class_weights[i])
class_weights[i] = max_class / (class_weights[i])
elif penalizing_method == 'weighted_log':
print(
str(i) + " " + str(class_weights[i]) + " " +
str(math.log(mu * len(y) / (class_weights[i]))))
class_weights[i] = math.log(mu * len(y) / (class_weights[i]))
else:
print(str(i) + " " + str(class_weights[i]) + " inf ")
class_weights[i] = 0.0
g = sns.barplot(x=[str(i) for i in range(len(class_weights))],
y=class_weights)
plt.xticks(rotation=-90)
plt.title("Class weights " + penalizing_method)
plt.grid(True)
plt.show()
class_dictionary = {}
print(len(class_weights))
for i in range(len(class_weights)):
class_dictionary[i] = class_weights[i]
print(class_dictionary)
it = iter(class_weights)
seclist = [
utils.POSE_CLASSES, utils.OBJ_HUMAN_CLASSES, utils.HUMAN_HUMAN_CLASSES
]
class_lists = [list(islice(it, 0, i)) for i in seclist]
print(class_lists)
# Create + compile model, load saved weights if they exist
saved_weights = None
# saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
model, keras_layer_names = rgb_create_model(
classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
model_name=params['model'],
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = ""
ucf_weights = "../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if ucf_weights is None:
print("Loading MConvNet weights: ")
if params['model'] == "resnet50":
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
)
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading Keras weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_rgb_params_names.pkl",
"../models/kinetics_keras/tsn_rgb_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5"
)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Make spltis
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
# TODO Don't forget to change your names :)
filter_type = "gauss"
bestModelPath = "../models/rgb_weightsfinal_" + filter_type + "_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/rgb_weightsfinal_train_" + filter_type + "_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/rgb_weightsfinal_val_" + filter_type + "_plot_" + params[
'model'] + "_" + time_str + ".csv"
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
# Load and train
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
filter_type,
soft_sigmoid=soft_sigmoid)
y_t = []
y_t.append(
to_categorical(y_train_pose, num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
class_weight=class_lists,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print(
"Epoch " + str(epoch) + " chunk " + str(epoch_chunks_count) +
" (" + str(elapsed) + ") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) + "," +
str(history.history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy'][0] +
history.history['pred_human_human_categorical_accuracy'][0]
) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
# global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
filter_type,
soft_sigmoid=soft_sigmoid)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
# NOTE Can't have weights on validation
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
# Average over all validation chunks
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
# We consider accuracy as the average accuracy over the three
# types of accuracy
acc = (loss_acc_list[4] + loss_acc_list[5] + loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5], loss_acc_list[6],
loss_acc_list[0], loss_acc_list[1], loss_acc_list[2],
loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(
from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training RGB-stream with class_weights ' +
penalizing_method,
message='Training RGB with following params: ' + str(params),
login='******',
password='******')
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
K.clear_session()
# Load list of action classes and separate them (from _stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'nb_epochs': 100,
'model': 'resnet50',
'email': True,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
train=True) # IDs for training
partition['validation'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
train=True) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv")
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv")
# Create + compile model, load saved weights if they exist
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
flow_weights = "../models/flow_resnet50_1806281901.hdf5"
pose_weights = "../models/pose_alexnet_1808310209.hdf5"
rgb_dir = "/media/pedro/actv-ssd/gauss_"
flow_dir = "/media/pedro/actv-ssd/flow_"
pose_dir = "/media/pedro/actv-ssd/pose_rgb_crop"
time_str = time.strftime("%y%m%d%H%M", time.localtime())
bestModelPath = "../models/fusion_pose_gauss_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/fusion_pose_train_gauss_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/fusion_pose_val_gauss_plot_" + params[
'model'] + "_" + time_str + ".csv"
nsmodel = FusionPoseModel(classes['label_id'], rgb_weights, flow_weights,
pose_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
modelpath = None
if modelpath is not None:
print("Loading previous weights")
model.load_weights(modelpath)
print("Training set size: " + str(len(partition['train'])))
# Load splits
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
with tf.device('/gpu:0'):
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val_rgb = x_val_flow = x_val_pose = y_val_pose = y_val_object = y_val_human = x_train_rgb = x_train_pose = x_train_flow = y_train_pose = y_train_object = y_train_human = None
x_train_rgb, x_train_flow, x_train_pose, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
10,
pose_dir,
rgb_dir,
flow_dir,
"rgb",
"train",
train=True)
y_train_pose = to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES)
y_train_object = utils.to_binary_vector(
y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human')
y_train_human = utils.to_binary_vector(
y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human')
print("Loaded, running through CNN")
history = model.fit(
[x_train_rgb, x_train_flow, x_train_pose],
[y_train_pose, y_train_object, y_train_human],
batch_size=params['batch_size'],
epochs=1,
verbose=0)
elapsed = timeit.default_timer() - start_time
# learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
print("Validating data: ")
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val_rgb = x_val_flow = x_val_pose = y_val_pose = y_val_object = y_val_human = x_train_rgb = x_train_pose = x_train_flow = y_train_pose = y_train_object = y_train_human = None
x_val_rgb, x_val_flow, x_val_pose, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
10,
pose_dir,
flow_dir,
"rgb",
"val",
train=True)
y_val_pose = to_categorical(y_val_pose,
num_classes=utils.POSE_CLASSES)
y_val_object = utils.to_binary_vector(
y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human')
y_val_human = utils.to_binary_vector(
y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human')
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
[x_val_rgb, x_val_flow, x_val_pose],
[y_val_pose, y_val_object, y_val_human],
batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
cc_addr_list=[],
subject='Finished training pose fusion',
message='Training fusion with following params: ' +
str(params),
login='******',
password='******')
def main():
# root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 10,
'nb_epochs': 157,
'model': "inceptionv3",
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
minValLoss = 9999990.0
soft_sigmoid = True
warp = False # TODO Use warped (camera motion corrected) flow or not
crop = False # TODO Use crop flow or not
encoding = "rgb" # TODO Are flows stored as rgb or as 2 grayscales
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# Create + compile model, load saved weights if they exist
# saved_weights = "saved_models/RGB_Stream_Softmax_inceptionv3.hdf5"
saved_weights = "../models/flow_kineticsinit_inceptionv3_1808282350.hdf5"
ucf_weights = "../models/ucf_keras/keras-ucf101-TVL1flow-" + params[
'model'] + "-newsplit.hdf5" # Outdated model
# ucf_weights = None
# ucf_weights = None
model, keras_layer_names = flow_create_model(
classes=classes['label_id'],
model_name=params['model'],
soft_sigmoid=soft_sigmoid,
image_shape=(224, 224),
opt_flow_len=10,
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = None
ucf_weights = ""
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading kinetics weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_flow_params_names.pkl",
"../models/kinetics_keras/tsn_flow_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-flow-inceptionv3.hdf5"
)
if ucf_weights is None:
print("Loading UCF weights: ")
if params['model'] == "resnet50":
# TODO Better initialization, average UCF models overt he 3 splits provided
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-TVL1flow-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-TVL1flow-resnet50-newsplit.hdf5"
)
else:
if ucf_weights != "":
model.load_weights(ucf_weights)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Load first train_size of partition{'train'}
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
if crop is True:
bestModelPath = "../models/flowcrop_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flowcrop_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flowcrop_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
else:
if warp is True:
bestModelPath = "../models/flow_warp_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flow_warp_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flow_warp_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
else:
bestModelPath = "../models/flow_kineticsinit_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flow_kineticsinit_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flow_kineticsinit_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
first_epoch = True
with tf.device('/gpu:0'): # TODO Multi GPU
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
if epoch > 0:
first_epoch = False
for trainIDS in train_splits:
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
5,
first_epoch,
encoding=encoding,
soft_sigmoid=soft_sigmoid,
crop=True)
y_t = []
y_t.append(
to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
5,
first_epoch,
encoding=encoding,
soft_sigmoid=soft_sigmoid,
crop=True)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training OF-stream ',
message='Training OF with following params: ' +
str(params),
login='******',
password='******')