# load config
# t0001: human_anger, t0002: human_fear, t0003: monkey_anger, t0004: monkey_fear --> plot cnn_output
# t0005: human_anger, t0006: human_fear, t0007: monkey_anger, t0008: monkey_fear --> plot difference, stride3, highlight max
# t0009: human_anger, t0010: human_fear, t0011: monkey_anger, t0012: monkey_fear --> plot difference, first, highlight max
# t0013: human_anger, t0014: human_fear, t0015: monkey_anger, t0016: monkey_fear --> plot difference, first, reduce max
# t0017: human_anger --> plot difference, stride3, reduce max
# t0100: human_anger --> plot maximum
# t0104: human_anger --> plot weighted average
# t0108: human_anger --> plot 10 biggest values (maximum10)
config = load_config("norm_base_animate_cnn_response_t0001.json", path="configs/norm_base_config")
# load images
images,_ = load_data(config, train=config["dataset"])
# load model
normbase = NormBase(config,(224,224,3))
# calculate vector and options for plot
if config["plot_option"]=='cnn_output':
# plot cnn_response
vector_plot = normbase.evaluate_v4(images, flatten=False)
elif config["plot_option"]=='cnn_output_difference':
# take difference between response and reference, reference has different options
response = normbase.evaluate_v4(images, flatten=False)
if config["difference_option"]=='first':
reference = response[0,...]
elif 'stride' in config["difference_option"]:
stride_length=int(config["difference_option"][6:])
reference = np.roll(response, shift=stride_length, axis=0)
reference[:stride_length,...] = response[:stride_length,...]
elif config["difference_option"]=="reference":
congig_path = '../../configs/norm_base_config'
# config_name = 'norm_base_monkey_test.json'
config_name = 'norm_base_affectNet_sub8_4000.json'
config_file_path = os.path.join(congig_path, config_name)
print("config_file_path", config_file_path)
# load norm_base_config file
with open(config_file_path) as json_file:
config = json.load(json_file)
# load data
data = load_data(config, train=False, sort_by=['image'])
print("[Data] -- Data loaded --")
# create model
norm_base = NormBase(config, input_shape=(224, 224, 3))
# "load" model
load_folder = os.path.join("../../models/saved", config['save_name'])
r = np.load(os.path.join(os.path.join(load_folder, "ref_vector.npy")))
t = np.load(os.path.join(os.path.join(load_folder, "tuning_vector.npy")))
norm_base.set_ref_vector(r)
norm_base.set_tuning_vector(t)
print("[MODEL] Set ref vector", np.shape(r))
print("[MODEL] Set tuning vector", np.shape(t))
# predict tuning vector
# it_resp = norm_base.predict(data)
_, it_resp, _ = norm_base.evaluate(data)
print("shape it_resp", np.shape(it_resp))
plt.plot(it_resp)
Example of use
2021/01/04
- model is trained in separate steps (specified in config)
"""
config = load_config("norm_base_plotDirections_t0012.json")
save_name = config["sub_folder"]
retrain = False
# model
try:
if retrain:
raise IOError("retrain = True")
norm_base = NormBase(config,
input_shape=(224, 224, 3),
save_name=save_name)
except IOError:
norm_base = NormBase(config, input_shape=(224, 224, 3))
norm_base.fit(load_data(config,
train=config["train_dim_ref_tun_ref"][0]),
fit_dim_red=True,
fit_ref=False,
fit_tun=False)
norm_base.fit(load_data(config,
train=config["train_dim_ref_tun_ref"][1]),
fit_dim_red=False,
fit_ref=True,
fit_tun=False)
norm_base.fit(load_data(config,
# load configuration
config_name = "norm_base_investigate_PCA_m0001.json"
config = load_config(config_name)
# fit models with each condition
avatars = ["human_orig", "monkey_orig", "all_orig"]
# avatars = ["human_orig"]
indexes = []
# pca_threshold = [300, 300, 1500]
pca_threshold = [600, 600, 2000]
for i, avatar in enumerate(avatars):
# modify condition according to loop
config['train_avatar'] = avatar
# define and train norm base model
norm_base = NormBase(config, input_shape=(224, 224, 3))
norm_base.pca.var_threshold = pca_threshold[i]
norm_base.fit(load_data(config, train=True),
fit_dim_red=True,
fit_ref=False,
fit_tun=False)
# get index from the feature having the most variance
predict_v4 = norm_base.v4_predict
var_predict = np.std(predict_v4, axis=0)
index = np.flip(np.argsort(var_predict))[:config['PCA']]
# save index
indexes.append(np.array(index))
indexes = np.array(indexes)
from models.NormBase import NormBase
# t0001: 2-norm t0002: 1-norm t0003: simplified t0004: direction-only t0005: expressitivity-direction
# t0006: 2-norm-monkey_morph
do_reverse = False
do_normalize = False
config = load_config("norm_base_reproduce_ICANN_t0015.json",
path="configs/norm_base_config")
save_name = config["sub_folder"]
save_folder = os.path.join("models/saved", config['save_name'], save_name)
accuracy = np.load(os.path.join(save_folder, "accuracy.npy"))
it_resp = np.load(os.path.join(save_folder, "it_resp.npy"))
labels = np.load(os.path.join(save_folder, "labels.npy"))
norm_base = NormBase(config, input_shape=(224, 224, 3))
print("accuracy", accuracy)
print("it_resp.shape", it_resp.shape)
print("labels.shape", labels.shape)
# colors 0=Neutral=Black, 1=Threat=Yellow, 2=Fear=Blue, 3=LipSmacking=Red
colors = config['colors']
titles = config['condition']
seq_length = config['seq_length']
n_cat = config['n_category']
labels = config['labels']
if config['use_ggplot'] is not None:
use_ggplot = config['use_ggplot']
if use_ggplot:
config['tau_y'] = 2 # 15
config['tau_d'] = 2
# --------------------------------------------------------------------------------------------------------------------
# train model
# load training data
train_data = load_data(config)
train_data[0] = segment_sequence(train_data[0], config['train_seq_start_idx'],
config['seq_length'])
train_data[1] = segment_sequence(train_data[1], config['train_seq_start_idx'],
config['seq_length'])
print("[LOAD] Shape train_data segmented", np.shape(train_data[0]))
print("[LOAD] Shape train_label segmented", np.shape(train_data[1]))
# fit and save model
norm_base = NormBase(config, input_shape=(224, 224, 3))
expr_neurons, face_neurons, differentiators = norm_base.fit(
train_data, get_differentiator=True, get_it_resp=True)
norm_base.save()
# --------------------------------------------------------------------------------------------------------------------
# plot
norm_base.plot_it_neurons_per_sequence(face_neurons,
title="01_train",
save_folder=os.path.join(
"models/saved",
config['config_name']),
normalize=True)
# re-arrange condition from fear lip_smack threat to threat fear lip_smack
norm_base.plot_differentiators(differentiators,
title="02_train",
load_model = True
if load_model:
load_NB_model = True
fit_dim_red = False
else:
load_NB_model = False
fit_dim_red = True
# set read-out layer
config["v4_layer"] = "block4_conv3"
# declare model
model = NormBase(
config,
input_shape=tuple(config['input_shape']),
load_NB_model=load_NB_model
) # load model set to True so we get the semantic dictionary loaded
if not load_model:
# fit model
data = load_data(config, train=True)
model.fit(
data,
fit_dim_red=
fit_dim_red, # if it is the first time to run this script change this to True -> time consuming
fit_ref=True,
fit_tun=True)
model.save()
# show semantic index
save_path = os.path.join("models/saved", config["config_name"])
if not os.path.exists(save_path):
os.mkdir(save_path)
# load and define model
v4_model = load_extraction_model(config,
input_shape=tuple(config["input_shape"]))
v4_model = tf.keras.Model(inputs=v4_model.input,
outputs=v4_model.get_layer(
config['v4_layer']).output)
size_ft = tuple(np.shape(v4_model.output)[1:3])
print("[LOAD] size_ft", size_ft)
print("[LOAD] Model loaded")
print()
nb_model = NormBase(config, tuple(config['input_shape']))
# -------------------------------------------------------------------------------------------------------------------
# train
# load data
data = load_data(config)
# predict
preds = v4_model.predict(data[0], verbose=1)
print("[TRAIN] shape prediction", np.shape(preds))
# get feature maps that mimic a semantic selection pipeline
# keep only highest IoU semantic score
eyebrow_preds = preds[..., best_eyebrow_IoU_ft]
print("shape eyebrow semantic feature selection", np.shape(eyebrow_preds))
lips_preds = preds[..., best_lips_IoU_ft]
# ----------------------------------------------------------------------------------------------------------------------
# 4: test model with monkey avatar
# ----------------------------------------------------------------------------------------------------------------------
# 5: test transfer learning, predict on monkey avatar using the human avatar training
# ----------------------------------------------------------------------------------------------------------------------
# 6: show that the vector are orthogonal
# ----------------------------------------------------------------------------------------------------------------------
# 7: train PCA using both avatars
config_name = 'NB_PCA_human_monkey_c2c3_m0001.json'
config = load_config(config_name, path='configs/norm_base_config')
# declare model
model = NormBase(config, input_shape=tuple(config['input_shape']))
# train model
model.fit(load_data(config, train=True),
fit_dim_red=True,
fit_ref=True,
fit_tun=True)
model.save_NB_model(config)
# ----------------------------------------------------------------------------------------------------------------------
# 8: train training vector on human avatar with PCA index from both avatars
# ----------------------------------------------------------------------------------------------------------------------
# 9: predict on monkey avatar using the PCA trained on both avatars and tuning vector on human avatar
# ----------------------------------------------------------------------------------------------------------------------
# 10: show that vectors are at exactly 45°
]
# load config
configs = []
for config_name in config_names:
configs.append(load_config(config_name))
# train models/load model
# fit reference and tuning vector
norm_base_list = []
for config in configs:
try:
if retrain:
raise IOError("retrain = True")
norm_base = NormBase(config,
input_shape=(224, 224, 3),
save_name=config["sub_folder"])
except IOError:
norm_base = NormBase(config, input_shape=(224, 224, 3))
dataset = load_data(config, train=config["dataset"])
norm_base.fit(dataset) # no feature selection
norm_base.save_model(config, config["sub_folder"])
norm_base_list.append(norm_base)
# extract vectors
ref_human = norm_base_list[0].r
tun1_human, tun2_human = norm_base_list[0].t_mean[[1, 2]]
cat1_human = ref_human + tun1_human
cat2_human = ref_human + tun2_human
ref_monkey = norm_base_list[1].r
save_path = os.path.join("models/saved", config["config_name"])
if not os.path.exists(save_path):
os.mkdir(save_path)
# load and define model
v4_model = load_extraction_model(config,
input_shape=tuple(config["input_shape"]))
v4_model = tf.keras.Model(inputs=v4_model.input,
outputs=v4_model.get_layer(
config['v4_layer']).output)
size_ft = tuple(np.shape(v4_model.output)[1:3])
print("[LOAD] size_ft", size_ft)
print("[LOAD] Model loaded")
print()
nb_model = NormBase(config, tuple(config['input_shape']))
# --------------------------------------------------------------------------------------------------------------------
# build test case
labels = np.arange(5) # build one prediction per category
preds = np.zeros((5, 5, 5, 2))
preds[0, 2, 2, :] = 1
preds[1, 1, 1, 0] = 1
preds[1, 4, 2, 1] = 1
preds[2, 0, 0, 0] = 0
preds[2, 3, 2, 1] = 1
print("[TRAIN] preds", np.shape(preds))
print("preds[1]")
print(preds[1, ..., 0])
print("preds[1]")
print(preds[1, ..., 1])
test the naive PCA selection on the original morphing space dataset from the eLife paper
run: python -m projects.facial_shape_expression_recognition_transfer.01b_NB_morphing_space
"""
train_model = False
predict_it_resp = False
# load config
config_name = 'NB_morphing_space_m0001.json'
config = load_config(config_name, path='configs/norm_base_config')
# --------------------------------------------------------------------------------------------------------------------
# declare model
model = NormBase(config,
input_shape=tuple(config['input_shape']),
load_NB_model=True)
# train model
if train_model:
print("[FIT] Train model]")
data = load_data(config, train=True)
model.fit(data, fit_dim_red=True, fit_ref=True, fit_tun=True)
model.save()
# --------------------------------------------------------------------------------------------------------------------
# predict model
if predict_it_resp:
data = load_data(config, train=False)
print("[Test] Predict model")
accuracy, it_resp, labels = model.evaluate(data)
selection_human_expression = [
frame_neutral_human_expression, frame_expression_human_expression,
150 + frame_neutral_human_expression,
150 + frame_expression_human_expression
]
selection_monkey_expression = [
frame_neutral_monkey_expression, frame_expression_monkey_expression,
150 + frame_neutral_monkey_expression,
150 + frame_expression_monkey_expression
]
images = np.concatenate([
images_human_fear[selection_human_expression],
images_monkey_threat[selection_monkey_expression]
]) # (8, 224, 224, 3)
# calculate response
norm_base = NormBase(config, (224, 224, 3))
response = norm_base.evaluate_v4(images, flatten=False) # (8, 28, 28, 256)
# reduce response to selected feature map
response = response[..., n_feature_map] # (8, 28, 28)
response = np.expand_dims(response, axis=-1) # (8,28,28,1)
# calculate arrays based on position for different modes
position_array_list = []
position_xy_list = []
for i, mode in enumerate(mode_list):
position_array_list.append(
calculate_position(response, mode=mode, return_mode="array"))
position_xy_list.append(
calculate_position(response, mode=mode, return_mode="xy float"))
# create plot
test a face transfer using norm base mechanism
run: python -m projects.facial_shape_expression_recognition_transfer.03_face_part_semantic_feature_map_selection
"""
# load config
config_name = 'NB_morph_space_semantic_pattern_m0002.json'
config = load_config(config_name, path='configs/norm_base_config')
full_train = False
# --------------------------------------------------------------------------------------------------------------------
# train model
if full_train:
# declare model
model = NormBase(config, input_shape=tuple(config['input_shape']))
# load data
data = load_data(config)
# fit model
face_neurons = model.fit(data)
# save model
model.save()
else:
model = NormBase(config, input_shape=tuple(config['input_shape']), load_NB_model=True)
# load data
data = load_data(config)
# plot_sequence(np.array(data[0]).astype(np.uint8), video_name='01_train_sequence.mp4',
import os
from utils.load_config import load_config
from utils.load_data import load_data
from models.NormBase import NormBase
from utils.plot_cnn_output import plot_cnn_output
from utils.calculate_position import calculate_position
# load config
config = load_config("norm_base_config_plot_frequency_time_t0001.json")
# load data --> 150 x 224 x 224 x 3
images,_ = load_data(config, train=1)
# load NormBase
norm_base = NormBase(config, (224,224,3))
# (train NormBase)--> maybe necessary, but not atm
# get_preds --> 150 x 512, x and y concatenated
positions = norm_base.get_preds(images)
# calculate fft over time and leave out constant term
rfft = np.fft.rfft(positions, axis=0)
rfft_abs = np.abs(rfft)
power_spectrum = np.square(rfft_abs)[1:]
#calculate frequencies and leave out constant term
frequencies = np.fft.rfftfreq(positions.shape[0], d=1/30)[1:]
print("frequencies", frequencies)
# plot fft discarding constant offset (at zero frequency)
save_path = os.path.join("models/saved", config["config_name"])
if not os.path.exists(save_path):
os.mkdir(save_path)
# load and define model
v4_model = load_extraction_model(config,
input_shape=tuple(config["input_shape"]))
v4_model = tf.keras.Model(inputs=v4_model.input,
outputs=v4_model.get_layer(
config['v4_layer']).output)
size_ft = tuple(np.shape(v4_model.output)[1:3])
print("[LOAD] size_ft", size_ft)
print("[LOAD] Model loaded")
print()
nb_model_eyebrow = NormBase(config, tuple(config['input_shape']))
nb_model_lips = NormBase(config, tuple(config['input_shape']))
# -------------------------------------------------------------------------------------------------------------------
# train
# load data
data = load_data(config)
# predict
preds = v4_model.predict(data[0], verbose=1)
print("[TRAIN] shape prediction", np.shape(preds))
# get feature maps that mimic a semantic selection pipeline
# keep only highest IoU semantic score
eyebrow_preds = preds[..., best_eyebrow_IoU_ft]
print("shape eyebrow semantic feature selection", np.shape(eyebrow_preds))
def evaluate_model(config, save_name, legacy=False):
"""
This function runs and evaluates the model with given config.
The model is saved to models/saved/config['save_name']/save_name.
If already calculated the result is loaded instead of calculated.
:param config:
:param save_name:
:param legacy: set to True to use old version
:return:
"""
if not os.path.exists(os.path.join("models/saved", config['save_name'])):
os.mkdir(os.path.join("models/saved", config['save_name']))
# folder for save and load
save_folder = os.path.join("models/saved", config['save_name'], save_name)
if not os.path.exists(save_folder):
os.mkdir(save_folder)
if not legacy:
try:
# load results if available
accuracy = np.load(os.path.join(save_folder, "accuracy.npy"))
it_resp = np.load(os.path.join(save_folder, "it_resp.npy"))
labels = np.load(os.path.join(save_folder, "labels.npy"))
except IOError:
try:
norm_base = NormBase(config,
input_shape=(224, 224, 3),
save_name=save_name)
except IOError:
norm_base = NormBase(config, input_shape=(224, 224, 3))
data_train = load_data(config)
norm_base.fit(data_train)
norm_base.save_model(config, save_name)
data_test = load_data(config, train=False)
accuracy, it_resp, labels = norm_base.evaluate(data_test)
return accuracy, it_resp, labels
else:
try:
# load results if available
accuracy = np.load(os.path.join(save_folder, "accuracy.npy"))
it_resp = np.load(os.path.join(save_folder, "it_resp.npy"))
labels = np.load(os.path.join(save_folder, "labels.npy"))
print("[MODEL] it_resp is available and is loaded from {}".format(
save_folder))
# load vectors if available
ref_vector = np.load(os.path.join(save_folder, "ref_vector.npy"))
tun_vector = np.load(os.path.join(save_folder,
"tuning_vector.npy"))
except IOError:
# calculate results if not available
print("[LOOP] start training")
# create model
norm_base = NormBase(config, input_shape=(224, 224, 3))
try:
# load vectors if available
ref_vector = np.load(
os.path.join(save_folder, "ref_vector.npy"))
tun_vector = np.load(
os.path.join(save_folder, "tuning_vector.npy"))
print(
"[MODEL] ref_vector and tun_vector are available and loaded from {}"
.format(save_folder))
norm_base.set_ref_vector(ref_vector)
norm_base.set_tuning_vector(tun_vector)
print("[MODEL] Set ref vector", np.shape(ref_vector))
print("[MODEL] Set tuning vector", np.shape(tun_vector))
except IOError:
# calculate vectors if not available
# load train data
data_train = load_data(config)
print("[Data] -- Data loaded --")
# train model
norm_base.fit(data_train, batch_size=config['batch_size'])
ref_vector = norm_base.r
tun_vector = norm_base.t
# save model
np.save(os.path.join(save_folder, "ref_vector"), ref_vector)
np.save(os.path.join(save_folder, "tuning_vector"), tun_vector)
print("[LOOP] start prediction")
# load test data
data_test = load_data(config, train=False, sort_by=['image'])
print("[Data] -- Data loaded --")
# evaluate
accuracy, it_resp, labels = norm_base.evaluate(data_test)
np.save(os.path.join(save_folder, "accuracy"), accuracy)
np.save(os.path.join(save_folder, "it_resp"), it_resp)
np.save(os.path.join(save_folder, "labels"), labels)
return accuracy, it_resp, labels, ref_vector, tun_vector
"norm_base_investigate_layer_t0003.json"
]
# load config
configs = []
for config_name in config_names:
configs.append(load_config(config_name))
# train models/load model
norm_base_list = []
for config in configs:
try:
if retrain:
raise IOError("retrain = True")
norm_base = NormBase(config,
input_shape=(224, 224, 3),
save_name=config["sub_folder"])
except IOError:
norm_base = NormBase(config, input_shape=(224, 224, 3))
norm_base.fit(load_data(config,
train=config["train_dim_ref_tun_ref"][0]),
fit_dim_red=True,
fit_ref=False,
fit_tun=False)
norm_base.save_model(config, config["sub_folder"])
norm_base_list.append(norm_base)
# extract PCA
pca1_human = norm_base_list[0].pca.components_[0]
pca1_monkey = norm_base_list[1].pca.components_[0]
congig_path = '../../configs/norm_base_config'
# config_name = 'norm_base_monkey_test.json'
config_name = 'norm_base_affectNet_sub8_4000.json'
config_file_path = os.path.join(congig_path, config_name)
print("config_file_path", config_file_path)
# load norm_base_config file
with open(config_file_path) as json_file:
config = json.load(json_file)
# load data
data = load_data(config)
print("[Data] -- Data loaded --")
# create model
norm_base = NormBase(config, input_shape=(224, 224, 3))
# norm_base.print_v4_summary()
# train model
m, n = norm_base.fit(data, batch_size=config['batch_size'])
print("shape m", np.shape(m))
print("shape n", np.shape(n))
# save model
save_folder = os.path.join("../../models/saved", config['save_name'])
if not os.path.exists(save_folder):
os.mkdir(save_folder)
np.save(os.path.join(save_folder, "ref_vector"), m)
np.save(os.path.join(save_folder, "tuning_vector"), n)