def main():
K.clear_session()
root_dir = '../../data/AVA/files/'
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
split = 'test'
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir + "AVA_" +
split.title() + "_Custom_Corrected.csv",
soft_sigmoid=True)
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "test_outputs/random/output_test_random_" + time_str + ".csv"
print("Test set size: " + str(len(partition['test'])))
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for row in partition['test']:
row = row.split("@")
video_name = row[0]
timestamp = row[1]
# action = row[6]
bb_topx = row[2]
bb_topy = row[3]
bb_botx = row[4]
bb_boty = row[5]
# Generate a random pose guess
rand_pose = randint(1, 10)
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
rand_pose)
output_file.write("%s\n" % line)
# Generate between 0 to 3 random human-object guesses
rand_ho = randint(0, 3)
for r in range(rand_ho):
rand_humanobject = randint(11, 22)
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
rand_humanobject)
output_file.write("%s\n" % line)
# Generate between 0 to 3 random human-human guesses
rand_hh = randint(0, 3)
for r in range(rand_hh):
rand_humanhuman = randint(23, 30)
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
rand_humanhuman)
output_file.write("%s\n" % line)
def main():
root_dir = '../../data/AVA/files/'
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
# Parameters for training
params = {
'dim': (300, 300),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'nb_epochs': 200,
'model': 'alexnet',
'email': True,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
store_predictions = True
minValLoss = 9999990.0
split = "test"
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir + "AVA_" +
split.title() + "_Custom_Corrected.csv",
soft_sigmoid=True)
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "output_test_pose_" + time_str + ".csv"
# Load trained model
pose_weights = "../models/pose_alexnet_1808310209.hdf5"
model = pose_create_model(classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
image_shape=params['dim'],
model_name=params['model'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
model.load_weights(pose_weights)
print("Test set size: " + str(len(partition['test'])))
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**11)
# Test directories where pre-processed test files are
pose_dir = "/media/pedro/actv-ssd/pose_" + split + "/"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
test_predictions = []
with tf.device('/gpu:0'):
for testIDS in test_splits:
# TODO Technically it shouldnt return labels here (these are ground truth)
x_test_pose, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
split,
filter_type,
soft_sigmoid=True,
train=False)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)))
predictions = model.predict(x_test_pose,
batch_size=params['batch_size'],
verbose=1)
if store_predictions is True:
# print(predictions[0][0])
# print(predictions[1][0])
# print(predictions[2][0])
# tarr = np.hstack((np.vstack(predictions[0]), np.vstack(predictions[1]), np.vstack(predictions[2])))
test_predictions.append(predictions)
# Convert predictions to readable output and perform majority voting
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
test_chunks_count += 1
if store_predictions is True:
#tp = np.vstack(test_predictions)
# print(tp.shape)
with open("thresholds/pose/predictions_pose_" + time_str + ".pickle",
'wb') as handle:
pickle.dump(test_predictions,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished prediction for ' + filter_type,
message='Testing pose with following params: ' +
str(params),
login='******',
password='******')
from rgb_data import get_AVA_set
import voting
import pickle
# File with predictions
stream = "fusion"
filter_type = "avg_fovea"
filename = "thresholds/context_fusion/predictions_" + stream + "_" + filter_type + "_1809281055.pickle"
with open(filename, 'rb') as handle:
predictions = pickle.load(handle)
root_dir = '../../data/AVA/files/'
# Load groundtruth (test or val set)
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
partition = get_AVA_set(classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=True)
test_splits = utils.make_chunks(original_list=partition,
size=len(partition),
chunk_size=2**11)
# Voting
thresh_values = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
for thresh in thresh_values:
print("Testing threshold " + str(thresh) + " ")
pose_votes = {}
obj_votes = {}
human_votes = {}
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'inceptionv3',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# Create + compile model, load saved weights if they exist
saved_weights = None
# saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
model, keras_layer_names = rgb_create_model(
classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
model_name=params['model'],
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = None
ucf_weights = "a"
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if ucf_weights is None:
print("Loading MConvNet weights: ")
if params['model'] == "resnet50":
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
)
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading Keras weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_rgb_params_names.pkl",
"../models/kinetics_keras/tsn_rgb_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5"
)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Make spltis
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
# TODO Don't forget to change your names :)
filter_type = "gauss"
bestModelPath = "../models/rgb_kininit_" + filter_type + \
"_" + params['model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/rgb_kininit_train_" + filter_type + \
"_plot_" + params['model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/rgb_kininit_val_" + filter_type + \
"_plot_" + params['model'] + "_" + time_str + ".csv"
with tf.device('/gpu:0'): # NOTE Not using multi gpu
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
# Load and train
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
filter_type,
soft_sigmoid=soft_sigmoid)
y_t = []
y_t.append(
to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# TODO Repeat samples of unrepresented classes?
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
# global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
filter_type,
soft_sigmoid=soft_sigmoid)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
# Average over all validation chunks
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
# We consider accuracy as the average accuracy over the three
# types of accuracy
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training RGB-stream ',
message='Training RGB with following params: ' +
str(params),
login='******',
password='******')
def main():
K.clear_session()
root_dir = '../../data/AVA/files/'
# K.clear_session()
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 150,
'model': 'resnet50',
'sendmail': True,
'gen_type': 'test'
}
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Test_Custom_Corrected.csv",
train=False)
filter_type = "gauss"
flowcrop = True
time_str = time.strftime("%y%m%d%H%M", time.localtime())
if flowcrop:
result_csv = "test_outputs/two-streams/output_" + params[
'gen_type'] + "_2stream_flowcrop_" + filter_type + "_" + time_str + ".csv"
else:
result_csv = "test_outputs/two-streams/output_" + params[
'gen_type'] + "_2stream_" + filter_type + "_" + time_str + ".csv"
# Load trained model
# rgb_weights = "../models/rgb_fovea_resnet50_1806301953.hdf5"
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
# rgb_weights = "../models/rgb_crop_resnet50_1806300210.hdf5"
# rgb_weights = "../models/rgb_rgb_resnet50_1807060914.hdf5"
# flow_weights = "../models/flow_resnet50_1806281901.hdf5"
flow_weights = "../models/flowcrop_resnet50_1807180022.hdf5"
nsmodel = TwoStreamModel(classes['label_id'], rgb_weights, flow_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
# two_stream_weights = "../models/two_stream_fusion_elfovresnet50_1807030015.hdf5"
# two_stream_weights = "../models/two_stream_fusion_rgbbaseline_rgb_resnet50_1809051930.hdf5"
# two_stream_weights = "../models/two_stream_fusion_flowcrop_crop_crop_resnet50_1809162007.hdf5"
# two_stream_weights = "../models/two_stream_fusion_flowcrop_fovea_fovea_resnet50_1809091554.hdf5"
two_stream_weights = "../models/two_stream_fusion_flowcrop_gauss_resnet50_1809012354.hdf5"
model.load_weights(two_stream_weights)
print("Test set size: " + str(len(partition['test'])))
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**10)
# Test directories where pre-processed test files are
rgb_dir = "/media/pedro/actv-ssd/" + filter_type + "_"
if flowcrop is False:
flow_dir = "/media/pedro/actv-ssd/flow_"
else:
flow_dir = "/media/pedro/actv-ssd/flowcrop_"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
for testIDS in test_splits:
x_test_rgb, x_test_flow, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
10,
rgb_dir,
flow_dir,
"test",
"rgb",
train=False,
crop=flowcrop)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)) + ":")
# Convert predictions to readable output and perform majority voting
predictions = model.predict([x_test_rgb, x_test_flow],
batch_size=params['batch_size'],
verbose=1)
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
x_test_rgb = None
x_test_flow = None
test_chunks_count += 1
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['sendmail']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished ' + params['gen_type'] +
' prediction for two stream.',
message='Testing fusion with following params: ' +
str(params),
login='******',
password='******')
def main():
K.clear_session()
root_dir = '../../data/AVA/files/'
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
# params = {'dim': (224, 224), 'batch_size': 32,
# 'n_classes': len(classes['label_id']), 'n_channels': 20,
# 'shuffle': False, 'nb_epochs': 200, 'model': 'resnet50', 'email': False,
# 'freeze_all': True, 'conv_fusion': False}
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 10,
'nb_epochs': 157,
'model': "inceptionv3",
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
crop = False # TODO Use crop flow or not
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Test_Custom_Corrected.csv",
soft_sigmoid=True)
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "test_outputs/kinetics_init/output_test_flow_kineticsinit_" + time_str + ".csv"
# Load trained model
# flow_weights = "../models/flowcrop_resnet50_1807180022.hdf5"
flow_weights = "../models/flow_kineticsinit_inceptionv3_1808290834.hdf5"
model, keras_layer_names = flow_create_model(
classes=classes['label_id'],
model_name=params['model'],
soft_sigmoid=True,
image_shape=(224, 224),
opt_flow_len=10,
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=True)
model.load_weights(flow_weights)
print("Test set size: " + str(len(partition['test'])))
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**10)
# Test directories where pre-processed test files are
#flow_dir = "/media/pedro/actv-ssd/flowcrop_test/"
flow_dir = "/media/pedro/actv-ssd/flow_test/"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
with tf.device('/gpu:0'):
for testIDS in test_splits:
# TODO Technically it shouldnt return labels here (these are ground truth)
#x_test_flow, y_test_pose, y_test_object, y_test_human = load_split(testIDS, None, params['dim'], params['n_channels'], "test", 10, False, encoding="rgb", soft_sigmoid=True, crop=crop)
x_test_flow, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
"test",
5,
False,
encoding="rgb",
soft_sigmoid=True,
crop=crop)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)))
predictions = model.predict(x_test_flow,
batch_size=params['batch_size'],
verbose=1)
# Convert predictions to readable output and perform majority voting
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
x_test_flow = None
test_chunks_count += 1
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished prediction for flow (crop)',
message='Testing flow with following params: ' +
str(params),
login='******',
password='******')
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
K.clear_session()
# Load list of action classes and separate them (from _stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'nb_epochs': 100,
'model': 'resnet50',
'email': True,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
train=True) # IDs for training
partition['validation'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
train=True) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv")
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv")
# Create + compile model, load saved weights if they exist
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
flow_weights = "../models/flow_resnet50_1806281901.hdf5"
pose_weights = "../models/pose_alexnet_1808310209.hdf5"
rgb_dir = "/media/pedro/actv-ssd/gauss_"
flow_dir = "/media/pedro/actv-ssd/flow_"
pose_dir = "/media/pedro/actv-ssd/pose_rgb_crop"
time_str = time.strftime("%y%m%d%H%M", time.localtime())
bestModelPath = "../models/fusion_pose_gauss_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/fusion_pose_train_gauss_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/fusion_pose_val_gauss_plot_" + params[
'model'] + "_" + time_str + ".csv"
nsmodel = FusionPoseModel(classes['label_id'], rgb_weights, flow_weights,
pose_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
modelpath = None
if modelpath is not None:
print("Loading previous weights")
model.load_weights(modelpath)
print("Training set size: " + str(len(partition['train'])))
# Load splits
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
with tf.device('/gpu:0'):
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val_rgb = x_val_flow = x_val_pose = y_val_pose = y_val_object = y_val_human = x_train_rgb = x_train_pose = x_train_flow = y_train_pose = y_train_object = y_train_human = None
x_train_rgb, x_train_flow, x_train_pose, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
10,
pose_dir,
rgb_dir,
flow_dir,
"rgb",
"train",
train=True)
y_train_pose = to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES)
y_train_object = utils.to_binary_vector(
y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human')
y_train_human = utils.to_binary_vector(
y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human')
print("Loaded, running through CNN")
history = model.fit(
[x_train_rgb, x_train_flow, x_train_pose],
[y_train_pose, y_train_object, y_train_human],
batch_size=params['batch_size'],
epochs=1,
verbose=0)
elapsed = timeit.default_timer() - start_time
# learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
print("Validating data: ")
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val_rgb = x_val_flow = x_val_pose = y_val_pose = y_val_object = y_val_human = x_train_rgb = x_train_pose = x_train_flow = y_train_pose = y_train_object = y_train_human = None
x_val_rgb, x_val_flow, x_val_pose, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
10,
pose_dir,
flow_dir,
"rgb",
"val",
train=True)
y_val_pose = to_categorical(y_val_pose,
num_classes=utils.POSE_CLASSES)
y_val_object = utils.to_binary_vector(
y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human')
y_val_human = utils.to_binary_vector(
y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human')
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
[x_val_rgb, x_val_flow, x_val_pose],
[y_val_pose, y_val_object, y_val_human],
batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
cc_addr_list=[],
subject='Finished training pose fusion',
message='Training fusion with following params: ' +
str(params),
login='******',
password='******')
def main():
GPU = False
CPU = True
num_cores = 8
if GPU:
num_GPU = 1
num_CPU = 1
if CPU:
num_CPU = 1
num_GPU = 0
# config = tf.ConfigProto(intra_op_parallelism_threads=num_cores, inter_op_parallelism_threads=num_cores, allow_soft_placement=True,
# device_count={'CPU': num_CPU, 'GPU': num_GPU})
# session = tf.Session(config=config)
# K.set_session(session)
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'resnet50',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
print(classes)
oversampling_train, oversampling_train_classes = oversampling(
classes, root_dir, "AVA_Train_Custom_Corrected.csv")
oversampling_val, oversampling_val_classes = oversampling(
classes, root_dir, "AVA_Val_Custom_Corrected.csv")
undersampling_train, undersampling_train_classes = undersampling(
classes, root_dir, "AVA_Train_Custom_Corrected.csv",
oversampling_train_classes)
undersampling_val, undersampling_val_classes = undersampling(
classes, root_dir, "AVA_Val_Custom_Corrected.csv",
oversampling_val_classes)
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
print(len(partition['train']))
undersampling_train = list(set(undersampling_train))
print(len(oversampling_train))
print(len(undersampling_train))
print(len(undersampling_train + oversampling_train))
print(1.0 * len(partition['train'] + oversampling_train) /
len(partition['train']))
bestsample = undersampling_train + oversampling_train
sys.exit(0)
# Labels
# labels_train = get_AVA_labels(classes, partition, "train", filename=root_dir + "AVA_Train_Custom_Corrected.csv", soft_sigmoid=soft_sigmoid)
# labels_val = get_AVA_labels(classes, partition, "validation", filename=root_dir + "AVA_Val_Custom_Corrected.csv", soft_sigmoid=soft_sigmoid)
original_train_size = len(partition['train'])
print("Training set size pre augmentation: " + str(original_train_size))
partition['train'] = partition['train'] + aug_train
print("Training set size pos augmentation: " +
str(len(partition['train'])) + " --> " +
str(100.0 * (len(partition['train']) - original_train_size) /
original_train_size) + " % increase")
original_val_size = len(partition['validation'])
print("validation set size pre augmentation: " + str(original_train_size))
partition['validation'] = partition['validation'] + aug_train
print("Validation set size pos augmentation: " +
str(len(partition['validation'])) + " --> " +
str(100.0 * (len(partition['validation']) - original_val_size) /
original_val_size) + " % increase")
img = cv2.imread(
"/media/pedro/actv-ssd/gauss_train/-5KQ66BBWC4_902_0.077_0.151_0.283_0.811/frames1.jpg"
)
print(img.shape)
if random.random() < 0.5:
flip_img = np.fliplr(img)
crop_rand_val = random.randrange(0, 5, 1) / 10.0
scale_rand_val = random.randrange(7, 8, 1) / 10.0
# print(crop_rand_val)
# print(scale_rand_val)
seq = iaa.Sequential(
[ # horizontal flips
iaa.Scale((scale_rand_val, 1.0)),
iaa.CropAndPad(percent=(0, crop_rand_val),
pad_mode=["edge"]) # random crops
],
random_order=True) # apply augmenters in random order
flipped = seq.augment_image(flip_img)
plt.imshow(flipped)
plt.show()
def main():
GPU = False
CPU = True
num_cores = 8
if GPU:
num_GPU = 1
num_CPU = 1
if CPU:
num_CPU = 1
num_GPU = 0
config = tf.ConfigProto(intra_op_parallelism_threads=num_cores,
inter_op_parallelism_threads=num_cores,
allow_soft_placement=True,
device_count={
'CPU': num_CPU,
'GPU': num_GPU
})
session = tf.Session(config=config)
K.set_session(session)
root_dir = '../../data/AVA/files/'
K.clear_session()
# Load list of action classes and separate them (from utilsm)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 150,
'model': 'resnet50',
'sendmail': False,
'gen_type': 'test'
}
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Test_Custom_Corrected.csv",
train=False)
filter_type = "gauss"
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "output_" + params[
'gen_type'] + "_fusion_context_" + filter_type + "_" + time_str + ".csv"
# Load trained model
# rgb_weights = "../models/rgb_fovea_resnet50_1806301953.hdf5"
# rgb_weights = "../models/rgb_crop_resnet50_1806300210.hdf5"
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
flow_weights = "../models/flow_resnet50_1806281901.hdf5"
context_weights = "../models/TW1/context_mlp128.hdf5"
# fusion_context_weights = "../models/fusion_context_fusion_fovea_resnet50_1807061835.hdf5"
# fusion_context_weights = "../models/fusion_context_fusion_crop_resnet50_1807181004.hdf5"
fusion_context_weights = "../models/fusion_context_fusion_gaussian_resnet50_1807210918.hdf5"
nsmodel = FusionContextModel(classes['label_id'], rgb_weights,
flow_weights, context_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
model.load_weights(fusion_context_weights)
print("Test set size: " + str(len(partition['test'])))
print(
"Building context dictionary from context file (these should be generated)..."
)
Xfilename = root_dir + "context_files/XContext_test_pastfuture.csv"
test_context_rows = {}
with open(Xfilename) as csvDataFile:
csvReader = csv.reader(csvDataFile)
for row in csvReader:
rkey = row[0] + "_" + row[1].lstrip("0") + \
"@" + str(row[2]) + "@" + str(row[3]) + "@" + str(row[4]) + "@" + str(row[5])
test_context_rows[rkey] = row[6]
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**10)
# Test directories where pre-processed test files are
rgb_dir = "/media/pedro/actv-ssd/" + filter_type + "_"
flow_dir = "/media/pedro/actv-ssd/flow_"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
print("Starting testing:")
with tf.device('/gpu:0'):
for testIDS in test_splits:
x_test_rgb, x_test_flow, x_test_context, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
10,
test_context_rows,
rgb_dir,
flow_dir,
"rgb",
"test",
train=False)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)) + ":")
# Convert predictions to readable output and perform majority voting
predictions = model.predict(
[x_test_rgb, x_test_flow, x_test_context],
batch_size=params['batch_size'],
verbose=1)
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
x_test_rgb = None
x_test_flow = None
x_test_context = None
test_chunks_count += 1
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['sendmail']:
utils.sendemail(
from_addr='*****@*****.**',
to_addr_list=[
'*****@*****.**', '*****@*****.**'
],
subject='Finished ' + params['gen_type'] +
' prediction for three stream.',
message='Testing fusion with following params: ' + str(params),
login='******',
password='******')
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'resnet50',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# http://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_class_weight.html
# Logistic Regression in Rare Events Data, Gary King, Langche Zeng
# Keras needs a dict though
# From documentation: class_weight: Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only).
# This can be useful to tell the model to "pay more attention" to samples from an under-represented class.
y = labels_to_numpy(labels_train)
penalizing_method = 'balanced'
mu = 0.7
# penalizing_method = 'weighted_log'
class_weights = np.zeros(30)
for i in y:
class_weights[i] += 1
max_class = max(class_weights)
for i in range(len(class_weights)):
if class_weights[i] != 0.0:
if penalizing_method == 'balanced':
print(
str(i) + " " + str(class_weights[i]) + " " +
str(len(y) / (class_weights[i])))
#class_weights[i] = len(y) / (class_weights[i])
class_weights[i] = max_class / (class_weights[i])
elif penalizing_method == 'weighted_log':
print(
str(i) + " " + str(class_weights[i]) + " " +
str(math.log(mu * len(y) / (class_weights[i]))))
class_weights[i] = math.log(mu * len(y) / (class_weights[i]))
else:
print(str(i) + " " + str(class_weights[i]) + " inf ")
class_weights[i] = 0.0
g = sns.barplot(x=[str(i) for i in range(len(class_weights))],
y=class_weights)
plt.xticks(rotation=-90)
plt.title("Class weights " + penalizing_method)
plt.grid(True)
plt.show()
class_dictionary = {}
print(len(class_weights))
for i in range(len(class_weights)):
class_dictionary[i] = class_weights[i]
print(class_dictionary)
it = iter(class_weights)
seclist = [
utils.POSE_CLASSES, utils.OBJ_HUMAN_CLASSES, utils.HUMAN_HUMAN_CLASSES
]
class_lists = [list(islice(it, 0, i)) for i in seclist]
print(class_lists)
# Create + compile model, load saved weights if they exist
saved_weights = None
# saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
model, keras_layer_names = rgb_create_model(
classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
model_name=params['model'],
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = ""
ucf_weights = "../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if ucf_weights is None:
print("Loading MConvNet weights: ")
if params['model'] == "resnet50":
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
)
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading Keras weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_rgb_params_names.pkl",
"../models/kinetics_keras/tsn_rgb_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5"
)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Make spltis
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
# TODO Don't forget to change your names :)
filter_type = "gauss"
bestModelPath = "../models/rgb_weightsfinal_" + filter_type + "_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/rgb_weightsfinal_train_" + filter_type + "_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/rgb_weightsfinal_val_" + filter_type + "_plot_" + params[
'model'] + "_" + time_str + ".csv"
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
# Load and train
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
filter_type,
soft_sigmoid=soft_sigmoid)
y_t = []
y_t.append(
to_categorical(y_train_pose, num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
class_weight=class_lists,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print(
"Epoch " + str(epoch) + " chunk " + str(epoch_chunks_count) +
" (" + str(elapsed) + ") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) + "," +
str(history.history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy'][0] +
history.history['pred_human_human_categorical_accuracy'][0]
) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
# global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
filter_type,
soft_sigmoid=soft_sigmoid)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
# NOTE Can't have weights on validation
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
# Average over all validation chunks
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
# We consider accuracy as the average accuracy over the three
# types of accuracy
acc = (loss_acc_list[4] + loss_acc_list[5] + loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5], loss_acc_list[6],
loss_acc_list[0], loss_acc_list[1], loss_acc_list[2],
loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(
from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training RGB-stream with class_weights ' +
penalizing_method,
message='Training RGB with following params: ' + str(params),
login='******',
password='******')
def main():
# root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 10,
'nb_epochs': 157,
'model': "inceptionv3",
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
minValLoss = 9999990.0
soft_sigmoid = True
warp = False # TODO Use warped (camera motion corrected) flow or not
crop = False # TODO Use crop flow or not
encoding = "rgb" # TODO Are flows stored as rgb or as 2 grayscales
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# Create + compile model, load saved weights if they exist
# saved_weights = "saved_models/RGB_Stream_Softmax_inceptionv3.hdf5"
saved_weights = "../models/flow_kineticsinit_inceptionv3_1808282350.hdf5"
ucf_weights = "../models/ucf_keras/keras-ucf101-TVL1flow-" + params[
'model'] + "-newsplit.hdf5" # Outdated model
# ucf_weights = None
# ucf_weights = None
model, keras_layer_names = flow_create_model(
classes=classes['label_id'],
model_name=params['model'],
soft_sigmoid=soft_sigmoid,
image_shape=(224, 224),
opt_flow_len=10,
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = None
ucf_weights = ""
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading kinetics weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_flow_params_names.pkl",
"../models/kinetics_keras/tsn_flow_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-flow-inceptionv3.hdf5"
)
if ucf_weights is None:
print("Loading UCF weights: ")
if params['model'] == "resnet50":
# TODO Better initialization, average UCF models overt he 3 splits provided
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-TVL1flow-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-TVL1flow-resnet50-newsplit.hdf5"
)
else:
if ucf_weights != "":
model.load_weights(ucf_weights)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Load first train_size of partition{'train'}
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
if crop is True:
bestModelPath = "../models/flowcrop_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flowcrop_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flowcrop_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
else:
if warp is True:
bestModelPath = "../models/flow_warp_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flow_warp_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flow_warp_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
else:
bestModelPath = "../models/flow_kineticsinit_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flow_kineticsinit_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flow_kineticsinit_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
first_epoch = True
with tf.device('/gpu:0'): # TODO Multi GPU
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
if epoch > 0:
first_epoch = False
for trainIDS in train_splits:
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
5,
first_epoch,
encoding=encoding,
soft_sigmoid=soft_sigmoid,
crop=True)
y_t = []
y_t.append(
to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
5,
first_epoch,
encoding=encoding,
soft_sigmoid=soft_sigmoid,
crop=True)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training OF-stream ',
message='Training OF with following params: ' +
str(params),
login='******',
password='******')