def main(args):
print(args)
config = Config('inference')
config.gpu_list = [0]
config.num_gpu = 1
model = get_model(config)
# for img_path in args.input:
img_path = args.input
print("Prepare to evaluate %s" % img_path)
result = evaluate(model, img_path, config)
print("result: ", result)
def main(args):
config = Config('inference')
print(config)
val_data = get_data("val", config)
model = get_model(config)
vis = Visualizer(config)
print("Prepare to validate model...")
pare_acc, pare_cm, sub_acc, sub_cm, corr_label, err_level = validate(
model, val_data, config, vis)
msg = 'parent-image validation accuracy:{}\n'.format(pare_acc)
msg += 'sub-image validation accuracy:{}\n'.format(sub_acc)
msg += 'validation scene confusion matrix:\n{}\n'.format(pare_cm)
msg += 'validation sub confusion matrix:\n{}\n'.format(sub_cm)
msg += 'number of correct labels in a scene:\n{}\n'.format(corr_label)
msg += 'number of error levels in a scene:\n{}\n'.format(err_level)
print("Validation Finish!", msg)
vis.log(msg, 'val_result', log_file=config.val_result)
print("save best validation result into " + config.val_result)
def train(configuration):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = repr(configuration.GPU)
####################################################################################################################
# Interpret configuration arguments.
####################################################################################################################
are_test_labels_available = configuration.are_test_labels_available
task_name = "task" + repr(configuration.task)
id_to_partition, partition_to_id = get_partition(
configuration.challenge_folder)
partition_to_chunk = get_partition_to_chunk(
partition_to_id, configuration.tf_records_folder)
train_size = len(partition_to_chunk["train"])
devel_size = len(partition_to_chunk["devel"])
test_size = len(partition_to_chunk["test"])
print(train_size, devel_size, test_size)
train_steps_per_epoch = train_size // configuration.train_batch_size
devel_steps_per_epoch = devel_size // configuration.devel_batch_size
test_steps_per_epoch = test_size // configuration.test_batch_size
print(train_steps_per_epoch, devel_steps_per_epoch, test_steps_per_epoch)
tf_records_folder = configuration.tf_records_folder
output_folder = configuration.output_folder
make_dirs_safe(output_folder)
if configuration.task == 1:
targets = ["arousal", "valence"]
number_of_targets = len(targets)
elif configuration.task == 2:
targets = ["arousal", "valence", "topic"]
number_of_targets = 16
elif configuration.task == 3:
targets = ["arousal", "valence", "trustworthiness"]
number_of_targets = len(targets)
else:
raise ValueError("Invalid task selection.")
feature_names = [k for k, v in configuration.use_data.items() if v is True]
method_string = "_" + repr(configuration.task) + "_" + \
"_".join([k for k, v in configuration.use_data.items() if v is True]) + "_" + \
configuration.model_type + "_" + \
repr(configuration.hidden_units) + "_" + \
repr(configuration.use_attention) + "_" + \
repr(configuration.initial_learning_rate) + "_" + \
repr(configuration.GPU)
method_string = configuration.output_folder + "/" + method_string
saver_paths = dict()
for target in targets:
saver_paths[target] = method_string + "_" + target + "_" + repr(
configuration.GPU)
####################################################################################################################
# Form computational graph.
####################################################################################################################
g = tf.Graph()
with g.as_default():
with tf.Session() as sess:
############################################################################################################
# Get dataset iterators.
############################################################################################################
dataset_train = data_provider.get_split(
tf_records_folder,
is_training=True,
task_name=task_name,
split_name="train",
are_test_labels_available=are_test_labels_available,
id_to_partition=id_to_partition,
feature_names=feature_names,
batch_size=configuration.train_batch_size,
seq_length=configuration.full_seq_length,
buffer_size=(train_steps_per_epoch + 1) // 4)
dataset_devel = data_provider.get_split(
tf_records_folder,
is_training=False,
task_name=task_name,
split_name="devel",
are_test_labels_available=are_test_labels_available,
id_to_partition=id_to_partition,
feature_names=feature_names,
batch_size=configuration.devel_batch_size,
seq_length=configuration.full_seq_length,
buffer_size=(devel_steps_per_epoch + 1) // 4)
dataset_test = data_provider.get_split(
tf_records_folder,
is_training=False,
task_name=task_name,
split_name="test",
are_test_labels_available=are_test_labels_available,
id_to_partition=id_to_partition,
feature_names=feature_names,
batch_size=configuration.test_batch_size,
seq_length=configuration.full_seq_length,
buffer_size=(test_steps_per_epoch + 1) // 4)
iterator_train = tf.data.Iterator.from_structure(
dataset_train.output_types, dataset_train.output_shapes)
iterator_devel = tf.data.Iterator.from_structure(
dataset_devel.output_types, dataset_devel.output_shapes)
iterator_test = tf.data.Iterator.from_structure(
dataset_test.output_types, dataset_test.output_shapes)
next_element_train = iterator_train.get_next()
next_element_devel = iterator_devel.get_next()
next_element_test = iterator_test.get_next()
init_op_train = iterator_train.make_initializer(dataset_train)
init_op_devel = iterator_devel.make_initializer(dataset_devel)
init_op_test = iterator_test.make_initializer(dataset_test)
############################################################################################################
# Define placeholders.
############################################################################################################
batch_size_tensor = tf.placeholder(tf.int32)
sequence_length_tensor = tf.placeholder(tf.int32)
support_train = tf.placeholder(tf.float32, (None, None, 1))
if task_name == "task2":
topic_train = tf.placeholder(tf.float32, (None, 10))
if task_name == "task3":
trustworthiness_train = tf.placeholder(tf.float32,
(None, None, 1))
if task_name in ["task1", "task3"]:
arousal_train = tf.placeholder(tf.float32, (None, None, 1))
valence_train = tf.placeholder(tf.float32, (None, None, 1))
elif task_name == "task2":
arousal_train = tf.placeholder(tf.float32, (None, 3))
valence_train = tf.placeholder(tf.float32, (None, 3))
else:
raise ValueError
step_id_train = tf.placeholder(tf.int32, (None, None, 1))
chunk_id_train = tf.placeholder(tf.int32, (None, None, 1))
recording_id_train = tf.placeholder(tf.int32, (None, None, 1))
tf_placeholder_train_dict = dict()
for feature_name in feature_names:
tf_placeholder_train_dict[feature_name] = tf.placeholder(
tf.float32, (None, None, FEATURE_NUM[feature_name]))
############################################################################################################
# Define model graph and get model.
############################################################################################################
with tf.variable_scope("Model"):
input_data_train = dict()
for feature_name in feature_names:
if configuration.use_data[feature_name]:
input_data_train[
feature_name] = tf_placeholder_train_dict[
feature_name]
if task_name in ["task1", "task3"]:
use_pooling = False
elif task_name == "task2":
use_pooling = True
else:
raise ValueError
pred_train = core.get_model(
input_data_dict=input_data_train,
support=support_train,
model_type=configuration.model_type,
batch_size=batch_size_tensor,
number_of_outputs=number_of_targets,
orig_seq_length=sequence_length_tensor,
hidden_units=configuration.hidden_units,
use_attention=configuration.use_attention,
use_pooling=use_pooling)
############################################################################################################
# Define loss function.
############################################################################################################
tensor_shape_train = [batch_size_tensor, sequence_length_tensor]
flattened_size_train = tensor_shape_train[0] * tensor_shape_train[1]
if task_name in ["task1", "task3"]:
pred_arousal_train = pred_train[:, :, 0]
pred_valence_train = pred_train[:, :, 1]
if task_name == "task3":
pred_trustworthiness_train = pred_train[:, :, 2]
single_pred_arousal_train = core.flatten_data(
pred_arousal_train, flattened_size_train)
single_pred_valence_train = core.flatten_data(
pred_valence_train, flattened_size_train)
if task_name == "task3":
single_pred_trustworthiness_train = core.flatten_data(
pred_trustworthiness_train, flattened_size_train)
single_true_support_train = core.flatten_data(
support_train, flattened_size_train)
single_true_arousal_train = core.flatten_data(
arousal_train, flattened_size_train)
single_true_valence_train = core.flatten_data(
valence_train, flattened_size_train)
if task_name == "task3":
single_true_trustworthiness_train = core.flatten_data(
trustworthiness_train, flattened_size_train)
elif task_name == "task2":
pred_arousal_train = pred_train[:, 0:3]
pred_valence_train = pred_train[:, 3:6]
pred_topic_train = pred_train[:, 6:16]
single_pred_arousal_train = pred_arousal_train
single_pred_valence_train = pred_valence_train
single_pred_topic_train = pred_topic_train
single_true_support_train = core.flatten_data(
support_train, flattened_size_train)
single_true_arousal_train = arousal_train
single_true_valence_train = valence_train
single_true_topic_train = topic_train
else:
raise ValueError
if task_name == "task1":
loss = core.loss_function_task_1(
pred_arousal=single_pred_arousal_train,
true_arousal=single_true_arousal_train,
pred_valence=single_pred_valence_train,
true_valence=single_true_valence_train,
support=single_true_support_train)
elif task_name == "task2":
loss = core.loss_function_task_2(
pred_arousal=single_pred_arousal_train,
true_arousal=single_true_arousal_train,
pred_valence=single_pred_valence_train,
true_valence=single_true_valence_train,
pred_topic=single_pred_topic_train,
true_topic=single_true_topic_train,
support=single_true_support_train)
elif task_name == "task3":
loss = core.loss_function_task_3(
pred_arousal=single_pred_arousal_train,
true_arousal=single_true_arousal_train,
pred_valence=single_pred_valence_train,
true_valence=single_true_valence_train,
pred_trustworthiness=single_pred_trustworthiness_train,
true_trustworthiness=single_true_trustworthiness_train,
support=single_true_support_train)
else:
raise NotImplementedError
vars = tf.trainable_variables()
model_vars = [v for v in vars if v.name.startswith("Model")]
saver_dict = dict()
for target in targets:
saver_dict[target] = tf.train.Saver(
{v.name: v
for v in model_vars})
total_loss = tf.reduce_sum(loss)
optimizer = tf.train.AdamOptimizer(
configuration.initial_learning_rate)
gradients, variables = zip(*optimizer.compute_gradients(loss))
gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
optimizer = optimizer.apply_gradients(zip(gradients, variables))
############################################################################################################
# Initialize variables and perform experiment.
############################################################################################################
sess.run(tf.global_variables_initializer())
############################################################################################################
# Train base model.
############################################################################################################
current_patience = 0
print("Start training base model.")
print("Fresh base model.")
for ee, epoch in enumerate(range(configuration.num_epochs)):
print("EPOCH:", epoch + 1)
input_feed_dict = {
batch_size_tensor: "batch_size",
sequence_length_tensor: "sequence_length",
arousal_train: "arousal",
valence_train: "valence",
support_train: "support"
}
if task_name == "task2":
input_feed_dict[topic_train] = "topic"
if task_name == "task3":
input_feed_dict[trustworthiness_train] = "trustworthiness"
for feature_name in feature_names:
input_feed_dict[
tf_placeholder_train_dict[feature_name]] = feature_name
run_epoch = core.RunEpoch(
sess=sess,
partition="train",
are_test_labels_available=are_test_labels_available,
init_op=init_op_train,
steps_per_epoch=train_steps_per_epoch,
next_element=next_element_train,
batch_size=configuration.train_batch_size,
seq_length=configuration.full_seq_length,
input_gaussian_noise=configuration.input_gaussian_noise,
optimizer=optimizer,
loss=total_loss,
pred=pred_train,
input_feed_dict=input_feed_dict,
targets=targets,
task_name=task_name)
train_items, train_subject_to_id = run_epoch.run_epoch()
if task_name == "task1":
train_measures = core.get_measures_task_1(train_items)
elif task_name == "task2":
train_measures = core.get_measures_task_2(train_items)
elif task_name == "task3":
train_measures = core.get_measures_task_3(train_items)
else:
raise NotImplementedError
print(method_string)
if task_name == "task1":
print("Train CCC:", train_measures["arousal"]["ccc"],
train_measures["valence"]["ccc"],
train_measures["ccc"])
print("Train CC:", train_measures["arousal"]["cc"],
train_measures["valence"]["cc"],
train_measures["cc"])
print("Train MAE:", train_measures["arousal"]["mae"],
train_measures["valence"]["mae"],
train_measures["mae"])
elif task_name == "task2":
print("Train UAR:",
train_measures["arousal"]["macro-recall"],
train_measures["valence"]["macro-recall"],
train_measures["topic"]["macro-recall"])
print("Train F1:", train_measures["arousal"]["micro-f1"],
train_measures["valence"]["micro-f1"],
train_measures["topic"]["micro-f1"])
print("Train TOT:", train_measures["arousal"]["score"],
train_measures["valence"]["score"],
train_measures["topic"]["score"])
elif task_name == "task3":
print("Train CCC:",
train_measures["trustworthiness"]["ccc"])
print("Train CC:",
train_measures["trustworthiness"]["cc"])
print("Train MAE:",
train_measures["trustworthiness"]["mae"])
else:
raise NotImplementedError
if ee == 0:
best_performance_dict = dict()
if task_name == "task1":
for target in targets:
best_performance_dict[target] = -1.0
elif task_name == "task2":
for target in targets:
best_performance_dict[target] = dict()
for measure_name in [
"macro-recall", "micro-f1", "score"
]:
best_performance_dict[target][
measure_name] = -1.0
elif task_name == "task3":
best_performance_dict["trustworthiness"] = -1.0
else:
raise NotImplementedError
if (ee) % configuration.val_every_n_epoch == 0:
input_feed_dict = {
batch_size_tensor: "batch_size",
sequence_length_tensor: "sequence_length",
arousal_train: "arousal",
valence_train: "valence",
support_train: "support"
}
if task_name == "task2":
input_feed_dict[topic_train] = "topic"
if task_name == "task3":
input_feed_dict[
trustworthiness_train] = "trustworthiness"
for feature_name in feature_names:
input_feed_dict[tf_placeholder_train_dict[
feature_name]] = feature_name
run_epoch = core.RunEpoch(
sess=sess,
partition="devel",
are_test_labels_available=are_test_labels_available,
init_op=init_op_devel,
steps_per_epoch=devel_steps_per_epoch,
next_element=next_element_devel,
batch_size=configuration.devel_batch_size,
seq_length=configuration.full_seq_length,
input_gaussian_noise=configuration.
input_gaussian_noise,
optimizer=None,
loss=None,
pred=pred_train,
input_feed_dict=input_feed_dict,
targets=targets,
task_name=task_name)
devel_items, devel_subject_to_id = run_epoch.run_epoch()
if task_name == "task1":
devel_measures = core.get_measures_task_1(devel_items)
elif task_name == "task2":
devel_measures = core.get_measures_task_2(devel_items)
elif task_name == "task3":
devel_measures = core.get_measures_task_3(devel_items)
else:
raise NotImplementedError
if task_name == "task1":
print("Devel CCC:", devel_measures["arousal"]["ccc"],
devel_measures["valence"]["ccc"],
devel_measures["ccc"])
print("Devel CC:", devel_measures["arousal"]["cc"],
devel_measures["valence"]["cc"],
devel_measures["cc"])
print("Devel MAE:", devel_measures["arousal"]["mae"],
devel_measures["valence"]["mae"],
devel_measures["mae"])
elif task_name == "task2":
print("Devel UAR:",
devel_measures["arousal"]["macro-recall"],
devel_measures["valence"]["macro-recall"],
devel_measures["topic"]["macro-recall"])
print("Devel F1:",
devel_measures["arousal"]["micro-f1"],
devel_measures["valence"]["micro-f1"],
devel_measures["topic"]["micro-f1"])
print("Devel TOT:", devel_measures["arousal"]["score"],
devel_measures["valence"]["score"],
devel_measures["topic"]["score"])
elif task_name == "task3":
print("Devel CCC:",
devel_measures["trustworthiness"]["ccc"])
print("Devel CC:",
devel_measures["trustworthiness"]["cc"])
print("Devel MAE:",
devel_measures["trustworthiness"]["mae"])
else:
raise NotImplementedError
noticed_improvement = False
if task_name == "task1":
for target in targets:
if best_performance_dict[target] < devel_measures[
target]["ccc"]:
best_performance_dict[target] = devel_measures[
target]["ccc"]
saver_dict[target].save(
sess, saver_paths[target])
noticed_improvement = True
elif task_name == "task2":
for target in targets:
if best_performance_dict[target][
"score"] < devel_measures[target]["score"]:
for measure_name in [
"macro-recall", "micro-f1", "score"
]:
best_performance_dict[target][
measure_name] = devel_measures[target][
measure_name]
saver_dict[target].save(
sess, saver_paths[target])
noticed_improvement = True
elif task_name == "task3":
if best_performance_dict[
"trustworthiness"] < devel_measures[
"trustworthiness"]["ccc"]:
best_performance_dict[
"trustworthiness"] = devel_measures[
"trustworthiness"]["ccc"]
saver_dict["trustworthiness"].save(
sess, saver_paths["trustworthiness"])
noticed_improvement = True
else:
raise NotImplementedError
if noticed_improvement:
current_patience = 0
else:
current_patience += 1
if current_patience > configuration.patience:
break
else:
pass
test_measures_dict = dict()
test_items_dict = dict()
for target in targets:
if task_name == "task3":
if target not in [
"trustworthiness",
]:
continue
saver_dict[target].restore(sess, saver_paths[target])
input_feed_dict = {
batch_size_tensor: "batch_size",
sequence_length_tensor: "sequence_length",
arousal_train: "arousal",
valence_train: "valence",
support_train: "support"
}
if task_name == "task2":
input_feed_dict[topic_train] = "topic"
if task_name == "task3":
input_feed_dict[trustworthiness_train] = "trustworthiness"
for feature_name in feature_names:
input_feed_dict[
tf_placeholder_train_dict[feature_name]] = feature_name
run_epoch = core.RunEpoch(
sess=sess,
partition="test",
are_test_labels_available=are_test_labels_available,
init_op=init_op_test,
steps_per_epoch=test_steps_per_epoch,
next_element=next_element_test,
batch_size=configuration.test_batch_size,
seq_length=configuration.full_seq_length,
input_gaussian_noise=configuration.input_gaussian_noise,
optimizer=None,
loss=None,
pred=pred_train,
input_feed_dict=input_feed_dict,
targets=targets,
task_name=task_name)
test_items, test_subject_to_id = run_epoch.run_epoch()
if are_test_labels_available:
if task_name == "task1":
test_measures = core.get_measures_task_1(test_items)
elif task_name == "task2":
test_measures = core.get_measures_task_2(test_items)
elif task_name == "task3":
test_measures = core.get_measures_task_3(test_items)
else:
raise NotImplementedError
test_measures_dict[target] = test_measures
test_items_dict[target] = test_items
if task_name == "task1":
print("Best devel CCC:", best_performance_dict["arousal"],
best_performance_dict["valence"],
(best_performance_dict["arousal"] +
best_performance_dict["valence"]) / 2.0)
if are_test_labels_available:
print("Test CCC:",
test_measures_dict["arousal"]["arousal"]["ccc"],
test_measures_dict["valence"]["valence"]["ccc"],
(test_measures_dict["arousal"]["arousal"]["ccc"] +
test_measures_dict["valence"]["valence"]["ccc"]) /
2.0)
print("Test CC:",
test_measures_dict["arousal"]["arousal"]["cc"],
test_measures_dict["valence"]["valence"]["cc"],
(test_measures_dict["arousal"]["arousal"]["cc"] +
test_measures_dict["valence"]["valence"]["cc"]) /
2.0)
print("Test MAE:",
test_measures_dict["arousal"]["arousal"]["mae"],
test_measures_dict["valence"]["valence"]["mae"],
(test_measures_dict["arousal"]["arousal"]["mae"] +
test_measures_dict["valence"]["valence"]["mae"]) /
2.0)
elif task_name == "task2":
print("Best devel CCC:",
best_performance_dict["arousal"]["score"],
best_performance_dict["valence"]["score"],
(best_performance_dict["arousal"]["score"] +
best_performance_dict["valence"]["score"]) / 2.0,
best_performance_dict["topic"]["score"])
if are_test_labels_available:
print(
"Test UAR:", test_measures_dict["arousal"]["arousal"]
["macro-recall"], test_measures_dict["valence"]
["valence"]["macro-recall"],
test_measures_dict["topic"]["topic"]["macro-recall"])
print("Test F1:",
test_measures_dict["arousal"]["arousal"]["micro-f1"],
test_measures_dict["valence"]["valence"]["micro-f1"],
test_measures_dict["topic"]["topic"]["micro-f1"])
print(
"Test TOT:", 0.66 *
test_measures_dict["arousal"]["arousal"]["micro-f1"] +
0.34 * test_measures_dict["arousal"]["arousal"]
["macro-recall"], 0.66 *
test_measures_dict["valence"]["valence"]["micro-f1"] +
0.34 * test_measures_dict["valence"]["valence"]
["macro-recall"],
0.66 * test_measures_dict["topic"]["topic"]["micro-f1"]
+ 0.34 *
test_measures_dict["topic"]["topic"]["macro-recall"])
elif task_name == "task3":
print("Best devel CCC:",
best_performance_dict["trustworthiness"])
if are_test_labels_available:
print(
"Test CCC:", test_measures_dict["trustworthiness"]
["trustworthiness"]["ccc"])
print(
"Test CC:", test_measures_dict["trustworthiness"]
["trustworthiness"]["cc"])
print(
"Test MAE:", test_measures_dict["trustworthiness"]
["trustworthiness"]["mae"])
else:
raise NotImplementedError
if task_name == "task1":
results = dict()
results["method_string"] = method_string
results["arousal"] = dict()
results["valence"] = dict()
results["arousal"]["best_devel_ccc"] = best_performance_dict[
"arousal"]
results["valence"]["best_devel_ccc"] = best_performance_dict[
"valence"]
if are_test_labels_available:
results["arousal"]["test_ccc"] = test_measures_dict[
"arousal"]["arousal"]["ccc"]
results["valence"]["test_ccc"] = test_measures_dict[
"valence"]["valence"]["ccc"]
results["arousal"]["test_cc"] = test_measures_dict[
"arousal"]["arousal"]["cc"]
results["valence"]["test_cc"] = test_measures_dict[
"valence"]["valence"]["cc"]
results["arousal"]["test_mae"] = test_measures_dict[
"arousal"]["arousal"]["mae"]
results["valence"]["test_mae"] = test_measures_dict[
"valence"]["valence"]["mae"]
results["arousal"]["test_true"] = test_items_dict[
"arousal"].arousal.true
results["valence"]["test_true"] = test_items_dict[
"valence"].valence.true
results["arousal"]["test_pred"] = test_items_dict[
"arousal"].arousal.pred
results["valence"]["test_pred"] = test_items_dict[
"valence"].valence.pred
print("Saving test predictions at:", method_string)
np.save(method_string + "/arousal_test_pred.npy",
test_items_dict["arousal"].arousal.pred)
np.save(method_string + "/valence_test_pred.npy",
test_items_dict["valence"].valence.pred)
elif task_name == "task2":
results = dict()
results["method_string"] = method_string
results["arousal"] = dict()
results["valence"] = dict()
results["emotion"] = dict()
results["topic"] = dict()
results["arousal"][
"best_devel_macro_recall"] = best_performance_dict[
"arousal"]["macro-recall"]
results["arousal"][
"best_devel_micro_f1"] = best_performance_dict["arousal"][
"micro-f1"]
results["arousal"]["best_devel_score"] = best_performance_dict[
"arousal"]["score"]
results["valence"][
"best_devel_macro_recall"] = best_performance_dict[
"valence"]["macro-recall"]
results["valence"][
"best_devel_micro_f1"] = best_performance_dict["valence"][
"micro-f1"]
results["valence"]["best_devel_score"] = best_performance_dict[
"valence"]["score"]
results["emotion"]["best_devel_macro_recall"] = (
best_performance_dict["arousal"]["macro-recall"] +
best_performance_dict["valence"]["macro-recall"]) / 2.0
results["emotion"]["best_devel_micro_f1"] = (
best_performance_dict["arousal"]["micro-f1"] +
best_performance_dict["valence"]["micro-f1"]) / 2.0
results["emotion"]["best_devel_score"] = (
best_performance_dict["arousal"]["score"] +
best_performance_dict["valence"]["score"]) / 2.0
results["topic"][
"best_devel_macro_recall"] = best_performance_dict[
"topic"]["macro-recall"]
results["topic"][
"best_devel_micro_f1"] = best_performance_dict["topic"][
"micro-f1"]
results["topic"]["best_devel_score"] = best_performance_dict[
"topic"]["score"]
if are_test_labels_available:
results["arousal"][
"test_macro_recall"] = test_measures_dict["arousal"][
"arousal"]["macro-recall"]
results["valence"][
"test_macro_recall"] = test_measures_dict["valence"][
"valence"]["macro-recall"]
results["emotion"]["test_macro_recall"] = (
test_measures_dict["arousal"]["arousal"]
["macro-recall"] + test_measures_dict["valence"]
["valence"]["macro-recall"]) / 2.0
results["topic"]["test_macro_recall"] = test_measures_dict[
"valence"]["valence"]["macro-recall"]
results["arousal"]["test_micro_f1"] = test_measures_dict[
"arousal"]["arousal"]["micro-f1"]
results["valence"]["test_micro_f1"] = test_measures_dict[
"valence"]["valence"]["micro-f1"]
results["emotion"]["test_micro_f1"] = (
test_measures_dict["arousal"]["arousal"]["micro-f1"] +
test_measures_dict["valence"]["valence"]["micro-f1"]
) / 2.0
results["topic"]["test_micro_f1"] = test_measures_dict[
"valence"]["valence"]["micro-f1"]
results["arousal"]["test_score"] = 0.66 * results["arousal"]["test_micro_f1"] + 0.34 * \
results["arousal"]["test_macro_recall"]
results["valence"]["test_score"] = 0.66 * results["valence"]["test_micro_f1"] + 0.34 * \
results["valence"]["test_macro_recall"]
results["emotion"]["test_score"] = (
results["arousal"]["test_score"] +
results["valence"]["test_score"]) / 2.0
results["topic"]["test_score"] = 0.66 * results["topic"][
"test_micro_f1"] + 0.34 * results["topic"][
"test_macro_recall"]
results["arousal"]["test_true"] = test_items_dict[
"arousal"].arousal.true
results["valence"]["test_true"] = test_items_dict[
"valence"].valence.true
results["topic"]["test_true"] = test_items_dict[
"topic"].topic.true
results["arousal"]["test_pred"] = test_items_dict[
"arousal"].arousal.pred
results["valence"]["test_pred"] = test_items_dict[
"valence"].valence.pred
results["topic"]["test_pred"] = test_items_dict[
"topic"].topic.pred
print("Saving test predictions at:", method_string)
np.save(method_string + "/arousal_test_pred.npy",
test_items_dict["arousal"].arousal.pred)
np.save(method_string + "/valence_test_pred.npy",
test_items_dict["valence"].valence.pred)
np.save(method_string + "/topic_test_pred.npy",
test_items_dict["topic"].topic.pred)
elif task_name == "task3":
results = dict()
results["method_string"] = method_string
results["trustworthiness"] = dict()
results["trustworthiness"][
"best_devel_ccc"] = best_performance_dict[
"trustworthiness"]
if are_test_labels_available:
results["trustworthiness"][
"test_ccc"] = test_measures_dict["trustworthiness"][
"trustworthiness"]["ccc"]
results["trustworthiness"]["test_cc"] = test_measures_dict[
"trustworthiness"]["trustworthiness"]["cc"]
results["trustworthiness"][
"test_mae"] = test_measures_dict["trustworthiness"][
"trustworthiness"]["mae"]
results["trustworthiness"]["test_true"] = test_items_dict[
"trustworthiness"].arousal.true
results["trustworthiness"]["test_pred"] = test_items_dict[
"trustworthiness"].arousal.pred
print("Saving test predictions at:", method_string)
np.save(
method_string + "/trustworthiness_test_pred.npy",
test_items_dict["trustworthiness"].trustworthiness.pred)
else:
raise NotImplementedError
return results
def _make_caricatures(
video_dir,
out_dir,
amp=5,
mode='full',
size='Small',
batch_size=1,
basemodel_name='resnet18',
):
os.makedirs(outdir, exist_ok=True)
transform = transforms.get_transform(split='val',
normalize=False,
rescale=False)
dataset = VideoFolder(datadir, step=1, transform=transform)
dataset_orig = VideoFolder(datadir, step=1)
# model = core.get_model(f'SeriesPretrainedFrozen{size}ManipulatorAttnDetector')
model = core.get_model(f'SeriesPretrainedFrozen{size}ManipulatorDetector')
ckpt_file = CHECKPOINTS[basemodel_name]
ckpt = torch.load(ckpt_file)
state_dict = mutils.remove_prefix(ckpt['state_dict'])
model.detector_model.load_state_dict(state_dict, strict=False)
model = model.to(device)
model.eval()
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=batch_size,
shuffle=False,
num_workers=4,
pin_memory=False,
drop_last=False,
)
dataloader_orig = torch.utils.data.DataLoader(
dataset_orig,
batch_size=batch_size,
shuffle=False,
num_workers=4,
pin_memory=False,
drop_last=False,
)
for i, ((names, frames, target),
(_, frames_orig, _)) in enumerate(zip(dataloader,
dataloader_orig)):
print(f'Processing [{i} / {len(dataloader)}]: {", ".join(names)}')
if i == 0:
continue
model = model.cpu()
model = model.to(device)
frames = frames.to(device)
attn_map = None
model.zero_grad()
if mode.lower() == 'gradcam':
normalize_attn = True
gcam_model = vutils.grad_cam.GradCAM(model.detector_model.model)
out, attn_map = gcam_forward(gcam_model, frames)
attn_map.detach_()
attn_map = attn_map.cpu()
del gcam_model
with torch.no_grad():
if mode == 'attn':
normalize_attn = True
out, attn_map = model.detector_model(frames)
del frames
frames_orig = frames_orig.to(device)
cari = model.manipulate(
frames_orig,
amp=torch.tensor(amp),
attn_map=attn_map,
normalize_attn=normalize_attn,
)
cari = cari.cpu()
del frames_orig
torch.cuda.empty_cache()
for n, (name, c) in enumerate(zip(names, cari)):
c = c.permute(1, 2, 3, 0)
outname = f'{name.replace(".mp4", "")}_cari_{size}_{mode}_amp{amp}' + '.mp4'
outfile = os.path.join(outdir, outname)
pretorched.data.utils.async_array_to_video(c, outfile)
if attn_map is not None:
am = normalize(attn_map[n]).cpu()
attn_outname = (
f'{name.replace(".mp4", "")}_attn_{size}_{mode}_amp{amp}' +
'.mp4')
attn_outfile = os.path.join(outdir, attn_outname)
if mode not in ['gradcam']:
pretorched.data.utils.async_array_to_video(
255 * am.unsqueeze(-1).repeat(1, 1, 1, 3),
attn_outfile,
)
heatmap_outname = (
f'{name.replace(".mp4", "")}_heatmap_{size}_{mode}_amp{amp}'
+ '.mp4')
heatmap = [
vutils.grad_cam.apply_heatmap(a, cc)
for a, cc in zip(am.numpy(), c.numpy())
]
heatmap_outfile = os.path.join(outdir, heatmap_outname)
pretorched.data.utils.async_array_to_video(
heatmap,
heatmap_outfile,
)
def main():
global args
args = parser.parse_args()
print(args)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# create model
model = MagNet().cuda()
fake_model = core.get_model('FrameModel').to(device)
gcam_model = grad_cam.GradCAM(fake_model.model)
# model = torch.nn.DataParallel(model).cuda()
print(model)
# load checkpoint
if os.path.isfile(args.load_ckpt):
print("=> loading checkpoint '{}'".format(args.load_ckpt))
checkpoint = torch.load(args.load_ckpt)
args.start_epoch = checkpoint['epoch']
# to load state_dict trained with DataParallel to model without DataParallel
new_state_dict = OrderedDict()
state_dict = checkpoint['state_dict']
for k, v in state_dict.items():
name = k[7:]
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
print(f'=> loaded checkpoint {args.load_ckpt} (epoch {checkpoint["epoch"]})')
else:
print("=> no checkpoint found at '{}'".format(args.load_ckpt))
assert False
if os.path.isfile(args.fake_ckpt):
print(f"=> loading checkpoint {args.fake_ckpt}")
fake_ckpt = torch.load(args.fake_ckpt, map_location='cpu')
fake_model.load_state_dict(mutils.remove_prefix(fake_ckpt['state_dict']))
print(f'=> no checkpoint found at {args.fake_ckpt}')
assert False
# check saving directory
save_dir = args.save_dir
if not os.path.exists(save_dir):
os.makedirs(save_dir)
print(save_dir)
# cudnn enable
cudnn.benchmark = True
# data loader
dataset_mag = ImageFromFolderTest(
args.video_path,
mag=args.amp,
mode=args.mode,
num_data=args.num_data,
preprocessing=False,
)
data_loader = data.DataLoader(
dataset_mag,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
drop_last=True,
)
# generate frames
mag_frames = []
attn_maps = []
model.eval()
# static mode or dynamic mode
if args.mode == 'static' or args.mode == 'dynamic':
for i, (xa, xb, amp_factor) in enumerate(data_loader):
if i % 10 == 0:
print('processing sample %d' % i)
amp_factor = amp_factor.unsqueeze(1).unsqueeze(1).unsqueeze(1)
xa = xa.cuda()
xb = xb.cuda()
amp_factor = amp_factor.cuda()
y_hat, _, _, _ = model(xa, xb, xb, amp_factor)
if i == 0:
# back to image scale (0-255)
tmp = xa.permute(0, 2, 3, 1).cpu().detach().numpy()
tmp = np.clip(tmp, -1.0, 1.0)
tmp = ((tmp + 1.0) * 127.5).astype(np.uint8)
mag_frames.append(tmp)
# back to image scale (0-255)
y_hat = y_hat.permute(0, 2, 3, 1).cpu().detach().numpy()
y_hat = np.clip(y_hat, -1.0, 1.0)
y_hat = ((y_hat + 1.0) * 127.5).astype(np.uint8)
mag_frames.append(y_hat)
else:
# temporal mode (difference of IIR)
# copy filter coefficients and follow codes from https://github.com/12dmodel/deep_motion_mag
filter_b = [args.fh - args.fl, args.fl - args.fh]
filter_a = [-1.0 * (2.0 - args.fh - args.fl), (1.0 - args.fl) * (1.0 - args.fh)]
x_state = []
y_state = []
for i, (xa, xb, amp_factor) in enumerate(data_loader):
if i % 10 == 0:
print('processing sample %d' % i)
amp_factor = amp_factor.unsqueeze(1).unsqueeze(1).unsqueeze(1)
xa = xa.cuda()
xb = xb.cuda()
amp_factor = amp_factor.cuda()
gcam_model.model.zero_grad()
probs, idx = gcam_model.forward(xb)
gcam_model.backward(idx=idx[0])
attn_map = gcam_model.generate(target_layer='layer4')
model.manipulator.attn_map = attn_map
vb, mb = model.encoder(xb)
x_state.insert(0, mb.detach())
while len(x_state) < len(filter_b):
x_state.insert(0, mb.detach())
if len(x_state) > len(filter_b):
x_state = x_state[: len(filter_b)]
y = torch.zeros_like(mb)
for i in range(len(x_state)):
y += x_state[i] * filter_b[i]
for i in range(len(y_state)):
y -= y_state[i] * filter_a[i]
y_state.insert(0, y.detach())
if len(y_state) > len(filter_a):
y_state = y_state[: len(filter_a)]
mb_m = model.manipulator(0.0, y, amp_factor)
mb_m += mb - y
y_hat = model.decoder(vb, mb_m)
if i == 0:
# back to image scale (0-255)
tmp = xa.permute(0, 2, 3, 1).cpu().detach().numpy()
tmp = np.clip(tmp, -1.0, 1.0)
tmp = ((tmp + 1.0) * 127.5).astype(np.uint8)
mag_frames.append(tmp)
# back to image scale (0-255)
y_hat = y_hat.permute(0, 2, 3, 1).cpu().detach().numpy()
y_hat = np.clip(y_hat, -1.0, 1.0)
y_hat = ((y_hat + 1.0) * 127.5).astype(np.uint8)
mag_frames.append(y_hat)
# save frames
mag_frames = np.concatenate(mag_frames, 0)
out_name = os.path.join(
save_dir,
'%s_%s_%s.mp4' % (os.path.basename(args.video_path), args.mode, args.amp),
)
vidwrite(mag_frames, out_name)
def process(i, frames, frames_orig, names, basemodel_name, size, amp, mode):
model = core.get_model(f'SeriesPretrainedFrozen{size}ManipulatorDetector')
device = f'cuda'
ckpt_file = CHECKPOINTS[basemodel_name]
ckpt = torch.load(ckpt_file)
state_dict = mutils.remove_prefix(ckpt['state_dict'])
model.detector_model.load_state_dict(state_dict, strict=False)
model = model.to(device)
model.eval()
frames = frames.to(device)
do_mag = True
attn_map = None
model.zero_grad()
if mode.lower() == 'gradcam':
normalize_attn = True
gcam_model = vutils.grad_cam.GradCAM(model.detector_model.model)
frames = model.detector_model.norm(frames)
do_mag, attn_map = gcam_forward(gcam_model, frames)
print(do_mag)
attn_map.detach_()
attn_map = attn_map.cpu()
del gcam_model
with torch.no_grad():
if mode == 'attn':
normalize_attn = True
out, attn_map = model.detector_model(frames)
del frames
torch.cuda.empty_cache()
frames_orig = frames_orig.to(device)
torch.cuda.empty_cache()
a = torch.tensor(amp, requires_grad=False)
# NOTE: Cannot use a batch size larger than 1!
if len(do_mag) == 1:
do_mag = do_mag[0]
if do_mag:
print('doing mag')
cari = model.manipulate(
frames_orig,
amp=a,
attn_map=attn_map,
normalize_attn=normalize_attn,
)
else:
print('skipping mag')
cari = frames_orig
del model
cari = cari.cpu()
del frames_orig
torch.cuda.empty_cache()
for n, (name, c) in enumerate(zip(names, cari)):
c = c.permute(1, 2, 3, 0)
outname = f'{name.replace(".mp4", "")}_cari_{size}_{mode}_amp{amp}' + '.mp4'
outfile = os.path.join(outdir, outname)
pretorched.data.utils.async_array_to_video(c, outfile)
if attn_map is not None:
am = attn_map[n]
am = am.cpu()
am = normalize(am)
attn_outname = (
f'{name.replace(".mp4", "")}_attn_{size}_{mode}_amp{amp}' +
'.mp4')
attn_outfile = os.path.join(outdir, attn_outname)
if mode not in ['gradcam']:
pretorched.data.utils.async_array_to_video(
255 * am.unsqueeze(-1).repeat(1, 1, 1, 3),
attn_outfile,
)
heatmap_outname = (
f'{name.replace(".mp4", "")}_heatmap_{size}_{mode}_amp{amp}' +
'.mp4')
heatmap = [
vutils.grad_cam.apply_heatmap(a, cc)
for a, cc in zip(am.numpy(), c.numpy())
]
heatmap_outfile = os.path.join(outdir, heatmap_outname)
pretorched.data.utils.async_array_to_video(
heatmap,
heatmap_outfile,
)
del heatmap
del a
del c
del am
def main_worker(gpu, ngpus_per_node, args):
global best_acc1
args.gpu = gpu
dir_path = os.path.dirname(os.path.realpath(__file__))
args.weights_dir = os.path.join(dir_path, args.weights_dir)
args.logs_dir = os.path.join(dir_path, args.logs_dir)
args.results_dir = os.path.join(dir_path, args.results_dir)
os.makedirs(args.weights_dir, exist_ok=True)
os.makedirs(args.logs_dir, exist_ok=True)
os.makedirs(args.results_dir, exist_ok=True)
save_name = 'HumanHeatvol_' + core.name_from_args(args)
print(f'Starting: {save_name}')
args.log_file = os.path.join(args.logs_dir, save_name + '.json')
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
dist.init_process_group(
backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank,
)
model = core.get_model(
args.model_name,
args.basemodel_name,
num_classes=args.num_classes,
pretrained=args.pretrained,
init_name=args.init,
)
input_size = model.input_size[-1]
args.normalize = core.do_normalize(model)
args.rescale = core.do_rescale(model)
if args.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
args.batch_size = int(args.batch_size / ngpus_per_node)
args.num_workers = int(
(args.num_workers + ngpus_per_node - 1) / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
else:
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.gpu is not None:
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
else:
# DataParallel will divide and allocate batch_size to all available GPUs
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# Define loss function (criterion) and optimizer
criterions = {
'ce': nn.CrossEntropyLoss().cuda(args.gpu),
'kl': nn.KLDivLoss(reduction='batchmean').cuda(args.gpu),
'cc': core.CorrCoefLoss(),
}
loss_weights = {
'ce': args.ce_weight,
'kl': args.kl_weight,
'cc': args.cc_weight,
}
print(loss_weights)
optimizer = core.get_optimizer(
model,
args.optimizer,
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
)
scheduler = core.get_scheduler(optimizer, args.scheduler)
# optionally initial model with some weight (e.g. for finetuning)
if args.init_weights:
if os.path.isfile(args.init_weights):
print("=> Initializing from checkpoint '{}'".format(
args.init_weights))
if args.gpu is None:
checkpoint = torch.load(args.init_weights)
else:
# Map model to be loaded to specified single gpu.
# loc = 'cuda:{}'.format(args.gpu)
loc = 'cpu'
checkpoint = torch.load(args.init_weights, map_location=loc)
# if args.gpu is not None:
# best_acc1 may be from a checkpoint from a different GPU
# best_acc1 = best_acc1.to(args.gpu)
model.load_state_dict(checkpoint['state_dict'])
try:
optimizer.load_state_dict(checkpoint['optimizer'])
except Exception:
pass
print("=> Initialized from checkpoint '{}' (epoch {})".format(
args.init_weights, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.init_weights))
torch.cuda.empty_cache()
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
if args.gpu is None:
checkpoint = torch.load(args.resume)
else:
# Map model to be loaded to specified single gpu.
# loc = 'cuda:{}'.format(args.gpu)
loc = 'cpu'
checkpoint = torch.load(args.resume, map_location=loc)
args.start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_acc1']
# if args.gpu is not None:
# best_acc1 may be from a checkpoint from a different GPU
# best_acc1 = best_acc1.to(args.gpu)
model.load_state_dict(checkpoint['state_dict'])
try:
optimizer.load_state_dict(checkpoint['optimizer'])
except Exception:
pass
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
torch.cuda.empty_cache()
cudnn.benchmark = True
# Data loading code
dataloaders = core.get_heatvol_dataloaders(
args.dataset,
args.data_root,
dataset_type=args.dataset_type,
record_set_type=args.record_set_type,
sampler_type=args.sampler_type,
segment_count=args.segment_count,
batch_size=args.batch_size,
num_workers=args.num_workers,
distributed=args.distributed,
size=input_size,
clip_length=args.clip_length,
frame_step=args.frame_step,
normalize=args.normalize,
rescale=args.rescale,
)
train_loader, val_loader = dataloaders['train'], dataloaders['val']
train_sampler = train_loader.sampler
if args.evaluate:
if args.resume:
train_args = vars(checkpoint.get('args'))
train_dataset = train_args['dataset']
else:
train_args = {}
train_dataset = 'None'
results_file = os.path.join(
args.results_dir,
f'Train_{train_dataset}_Eval_{args.dataset}_' + save_name +
'.json',
)
print(f'Evaluating: {results_file}')
acc, loss = validate(val_loader, model, criterions, loss_weights, args)
with open(results_file, 'w') as f:
json.dump(
{
'acc': acc,
'loss': loss,
'dataset': args.dataset,
'checkpoint_file': args.resume,
**vars(args),
'train_args': train_args,
},
f,
indent=4,
)
return
logger = loggers.TensorBoardLogger(args.logs_dir,
name=save_name,
rank=args.rank,
version=args.version)
history = defaultdict(list)
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
display = is_rank0(args, ngpus_per_node)
# Train for one epoch.
train_acc1, train_loss = train(
train_loader,
model,
criterions,
loss_weights,
optimizer,
logger,
epoch,
args,
display,
)
history['acc'].append(train_acc1.item())
history['loss'].append(train_loss)
# Evaluate on validation set.
val_acc1, val_loss = validate(val_loader, model, criterions,
loss_weights, args, display)
history['val_acc'].append(val_acc1)
history['val_loss'].append(val_loss)
history['epoch'].append(epoch + 1)
logger.log_metrics(
{
'EpochAccuracy/train': train_acc1,
'EpochLoss/train': train_loss,
'EpochAccuracy/val': val_acc1,
'EpochLoss/val': val_loss,
},
step=epoch + 1,
)
# Update the learning rate.
if type(scheduler).__name__ == 'ReduceLROnPlateau':
scheduler.step(val_loss)
else:
scheduler.step()
# remember best acc@1 and save checkpoint
is_best = val_acc1 > best_acc1
best_acc1 = max(val_acc1, best_acc1)
if is_rank0(args, ngpus_per_node):
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'model_name': args.model_name,
'basemodel_name': args.basemodel_name,
'state_dict': model.state_dict(),
'best_acc1': best_acc1,
'optimizer': optimizer.state_dict(),
'history': history,
'args': args,
},
is_best,
filename=os.path.join(args.weights_dir, save_name),
)
with open(args.log_file, 'w') as f:
json.dump(history, f, indent=4)
def test_ResManipulatorAttnDetector(frames3D_small):
model = core.get_model('ResPretrainedManipulatorAttnDetector')
model = model.to(device)
out = model(frames3D_small)
print(out.shape)
def test_caricature_model(frames3D_small):
model = core.get_model('GradCamCaricatureModel').to(device)
# frames = frames3D_small[0]
out, attn_maps = model(frames3D_small[0:1])
assert tuple(out.size()) == (1, 3, 16, 224, 224)
def test_get_model(model_name, basemodel_name, frames3D_small):
model = core.get_model(model_name, basemodel_name)
model = model.to(device)
out = model(frames3D_small)
assert tuple(out.size()) == (BATCH_SIZE, 2)