def load_dataset_old(config):
if config['dataset'] == 'IDIAP':
(xtr, ytr_rad), (xval, yval_rad), (xte, yte_rad) = load_idiap_part(
config['data_path'], config['net_output'])
ytr_deg = np.rad2deg(ytr_rad)
yval_deg = np.rad2deg(yval_rad)
yte_deg = np.rad2deg(yte_rad)
elif (config['dataset'] == 'CAVIAR-o') or (config['dataset']
== 'CAVIAR-c'):
(xtr, ytr_deg), (xval, yval_deg), (xte, yte_deg) = load_caviar(
config['data_path'])
else:
raise ValueError("invalid dataset name!")
ytr_bit = deg2bit(ytr_deg)
yval_bit = deg2bit(yval_deg)
yte_bit = deg2bit(yte_deg)
return (xtr, ytr_bit, ytr_deg), (xval, yval_bit, yval_deg), (xte, yte_bit,
yte_deg)
def evaluate(self, x, ytrue_deg, data_part, return_per_image=False):
ytrue_bit = deg2bit(ytrue_deg)
ypreds = self.model.predict(x)
results = dict()
vmmix_mu, vmmix_kappas, vmmix_probs = self.parse_output_np(ypreds)
vmmix_mu_rad = np.deg2rad(bit2deg_multi(vmmix_mu))
samples = sample_von_mises_mixture_multi(vmmix_mu_rad, vmmix_kappas, vmmix_probs, n_samples=100)
point_preds = maximum_expected_utility(np.rad2deg(samples))
maad_errs = maad_from_deg(point_preds, ytrue_deg)
results['maad_loss'] = float(np.mean(maad_errs))
results['maad_sem'] = float(sem(maad_errs))
log_likelihoods = self._von_mises_mixture_log_likelihood_np(ytrue_bit, ypreds)
results['log_likelihood_mean'] = float(np.mean(log_likelihoods))
results['log_likelihood_sem'] = float(sem(log_likelihoods, axis=None))
print("MAAD error (%s) : %f pm %fSEM" % (data_part,
results['maad_loss'],
results['maad_sem']))
print("log-likelihood (%s) : %f pm %fSEM" % (data_part,
results['log_likelihood_mean'],
results['log_likelihood_sem']))
if return_per_image:
results['point_preds'] = bit2deg(deg2bit(point_preds))
results['maad'] = maad_errs
results['log_likelihood'] = log_likelihoods
return results
def convert_and_pad_angle_data(y):
""" In case there is only 1 component, we augment data with fake tilt and roll components
"""
y_pan_bit = angles.deg2bit(y)
y_tilt_bit = angles.deg2bit(np.zeros_like(y))
y_roll_bit = angles.deg2bit(np.zeros_like(y))
return np.hstack([y_pan_bit, y_tilt_bit, y_roll_bit])
def evaluate_multi(self, x, ytrue_deg, data_part, n_samples=50, verbose=1):
ytrue_bit = deg2bit(ytrue_deg)
results = dict()
preds = self.get_multiple_predictions(x,
ytrue_bit,
n_samples=n_samples)
results['elbo'] = np.mean(preds['elbo'])
results['elbo_sem'] = sem(np.mean(preds['elbo'], axis=1))
results['kl_div'] = np.mean(preds['kl_div'])
results['kl_div_sem'] = sem(np.mean(preds['kl_div'], axis=1))
ypreds = self.decoder_model.predict(x)
ypreds_bit = ypreds[:, 0:2]
kappa_preds = ypreds[:, 2:]
ypreds_deg = bit2deg(ypreds_bit)
loss = maad_from_deg(ytrue_deg, preds['maxutil_deg_dec'])
results['maad_loss'] = np.mean(loss)
results['maad_loss_sem'] = sem(loss, axis=None)
importance_loglikelihoods = importance_loglikelihood(
preds['mu_encoder'], preds['log_sigma_encoder'], preds['mu_prior'],
preds['log_sigma_prior'], preds['u_encoder'], preds['mu_bit'],
preds['kappa'], ytrue_bit)
results['importance_log_likelihood'] = np.mean(
importance_loglikelihoods)
results['importance_log_likelihood_sem'] = sem(
importance_loglikelihoods, axis=None)
if verbose:
print("MAAD error (%s) : %f ± %fSEM" %
(data_part, results['maad_loss'], results['maad_loss_sem']))
print("ELBO (%s) : %f ± %fSEM" %
(data_part, results['elbo'], results['elbo_sem']))
print(
"Approx Log-Likelihood, importance sampling (%s) : %f ± %fSEM"
% (data_part, results['importance_log_likelihood'],
results['importance_log_likelihood_sem']))
print("KL-div (%s) : %f ± %fSEM" %
(data_part, results['kl_div'], results['kl_div_sem']))
return results
def evaluate(self, x, ytrue_deg, data_part, return_per_image=False):
ytrue_bit = deg2bit(ytrue_deg)
ypreds = self.model.predict(x)
ypreds_bit = ypreds[:, 0:2]
ypreds_deg = bit2deg(ypreds_bit)
if self.predict_kappa:
kappa_preds = ypreds[:, 2:]
else:
kappa_preds = np.ones([ytrue_deg.shape[0], 1
]) * self.fixed_kappa_value
loss = maad_from_deg(ypreds_deg, ytrue_deg)
results = dict()
results['maad_loss'] = float(np.mean(loss))
results['maad_loss_sem'] = float(sem(loss))
print("MAAD error (%s) : %f pm %fSEM" %
(data_part, results['maad_loss'], results['maad_loss_sem']))
results['mean_kappa'] = float(np.mean(kappa_preds))
results['std_kappa'] = float(np.std(kappa_preds))
log_likelihoods = von_mises_log_likelihood_np(ytrue_bit, ypreds_bit,
kappa_preds)
results['log_likelihood_mean'] = float(np.mean(log_likelihoods))
results['log_likelihood_sem'] = float(sem(log_likelihoods, axis=None))
print("log-likelihood (%s) : %f pm %fSEM" %
(data_part, results['log_likelihood_mean'],
results['log_likelihood_sem']))
if return_per_image:
results['point_preds'] = bit2deg(deg2bit(ypreds_deg))
results['maad'] = loss
results['log_likelihood'] = log_likelihoods
return results
def load_dataset(name, data_path, part=None, return_info=False):
if name == 'IDIAP':
(xtr, ytr_rad), (xval, yval_rad), (xte, yte_rad) = load_idiap_part(
data_path, part)
ytr_deg = np.rad2deg(ytr_rad)
yval_deg = np.rad2deg(yval_rad)
yte_deg = np.rad2deg(yte_rad)
elif (name == 'CAVIAR-o') or (name == 'CAVIAR-c'):
(xtr, ytr_deg, info_tr), (xval, yval_deg,
info_val), (xte, yte_deg,
info_te) = load_caviar(data_path)
elif name == 'TownCentre':
(xtr, ytr_deg,
info_tr), (xval, yval_deg,
info_val), (xte, yte_deg,
info_te) = load_towncentre(data_path)
else:
raise ValueError("invalid dataset name!")
ytr_bit = deg2bit(ytr_deg)
yval_bit = deg2bit(yval_deg)
yte_bit = deg2bit(yte_deg)
if return_info:
return (xtr, ytr_bit, ytr_deg, info_tr), (xval, yval_bit, yval_deg,
info_val), (xte, yte_bit,
yte_deg, info_te)
else:
return (xtr, ytr_bit, ytr_deg), (xval, yval_bit,
yval_deg), (xte, yte_bit, yte_deg)
def evaluate(self, x, ytrue_deg, data_part, verbose=1):
ytrue_bit = deg2bit(ytrue_deg)
results = dict()
cvae_preds = self.full_model.predict([x, ytrue_bit])
elbo, ll, kl = self._cvae_elbo_loss_np(ytrue_bit, cvae_preds)
results['elbo'] = np.mean(elbo)
results['elbo_sem'] = sem(elbo)
results['kl'] = np.mean(kl)
results['kl_sem'] = sem(kl)
results['log_likelihood'] = np.mean(ll)
results['log_likelihood_loss_sem'] = sem(ll)
ypreds = self.decoder_model.predict(x)
ypreds_bit = ypreds[:, 0:2]
kappa_preds = ypreds[:, 2:]
ypreds_deg = bit2deg(ypreds_bit)
loss = maad_from_deg(ytrue_deg, ypreds_deg)
results['maad_loss'] = np.mean(loss)
results['maad_loss_sem'] = sem(loss)
# log_likelihood_loss = von_mises_log_likelihood_np(ytrue_bit, ypreds_bit, kappa_preds,
# input_type='biternion')
# results['log_likelihood'] = np.mean(log_likelihood_loss)
# results['log_likelihood_loss_sem'] = sem(log_likelihood_loss)
if verbose:
print("MAAD error (%s) : %f ± %fSEM" %
(data_part, results['maad_loss'], results['maad_loss_sem']))
print("ELBO (%s) : %f ± %fSEM" %
(data_part, results['elbo'], results['elbo_sem']))
print("KL-div (%s) : %f ± %fSEM" %
(data_part, results['kl'], results['kl_sem']))
# print("log-likelihood (%s) : %f±%fSEM" % (data_part,
# results['log_likelihood'],
# results['log_likelihood_loss_sem']))
return results
def main():
n_u = 8
exp_id = get_experiment_id()
root_log_dir = 'logs/cvae/'
experiment_dir = os.path.join(root_log_dir, exp_id)
os.mkdir(experiment_dir)
xtr, ytr_deg, xval, yval_deg, xte, yte_deg = load_towncentre(
'data/TownCentre.pkl.gz', canonical_split=True, verbose=1)
# xtr, ytr_deg = aug_data(xtr, ytr_deg)
# xval, yval_deg = aug_data(xval, yval_deg)
# xte, yte_deg = aug_data(xval, yval_deg)
ytr_bit = deg2bit(ytr_deg)
yval_bit = deg2bit(yval_deg)
yte_bit = deg2bit(yte_deg)
image_height, image_width, n_channels = xtr.shape[1:]
phi_shape = yte_bit.shape[1]
best_trial_id = 0
n_trials = 10
results = dict()
overall_best_ckpt_path = os.path.join(
experiment_dir, 'cvae.full_model.overall_best.weights.hdf5')
for tid in range(0, n_trials):
n_epochs = 100
batch_size = 10
print("TRIAL %d" % tid)
trial_dir = os.path.join(experiment_dir, str(tid))
os.mkdir(trial_dir)
cvae_best_ckpt_path = os.path.join(
trial_dir, 'cvae.full_model.trial_%d.best.weights.hdf5' % tid)
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=trial_dir)
train_csv_log = os.path.join(trial_dir, 'train.csv')
csv_callback = keras.callbacks.CSVLogger(train_csv_log,
separator=',',
append=False)
model_ckpt_callback = keras.callbacks.ModelCheckpoint(
cvae_best_ckpt_path,
monitor='val_loss',
mode='min',
save_best_only=True,
save_weights_only=True,
period=1,
verbose=1)
cvae_model = CVAE(image_height=image_height,
image_width=image_width,
n_channels=n_channels,
n_hidden_units=n_u,
kl_weight=0.7)
cvae_bestloglike_ckpt_path = os.path.join(
trial_dir,
'cvae.full_model.trial_%d.best_likelihood.weights.hdf5' % tid)
eval_callback = EvalCVAEModel(xval, yval_deg, 'validation', cvae_model,
cvae_bestloglike_ckpt_path)
cvae_model.full_model.fit([xtr, ytr_bit], [ytr_bit],
batch_size=batch_size,
epochs=n_epochs,
validation_data=([xval, yval_bit], yval_bit),
callbacks=[
tensorboard_callback, csv_callback,
model_ckpt_callback, eval_callback
])
cvae_model.evaluate(xtr, ytr_deg, 'train')
cvae_model.evaluate(xval, yval_deg, 'validation')
cvae_model.evaluate(xte, yte_deg, 'test')
best_model = CVAE(image_height=image_height,
image_width=image_width,
n_channels=n_channels,
n_hidden_units=n_u)
best_model.full_model.load_weights(cvae_bestloglike_ckpt_path)
trial_results = dict()
trial_results['ckpt_path'] = cvae_best_ckpt_path
trial_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
trial_results['validation'] = best_model.evaluate(
xval, yval_deg, 'validation')
trial_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results[tid] = trial_results
if tid > 0:
if trial_results['validation']['importance_log_likelihood'] > \
results[best_trial_id]['validation']['importance_log_likelihood']:
best_trial_id = tid
print(
"Better validation log-likelihood achieved, current best trial: %d"
% best_trial_id)
shutil.copy(results[best_trial_id]['ckpt_path'],
overall_best_ckpt_path)
print("Loading best model (trial_id = %d)" % best_trial_id)
best_ckpt_path = results[best_trial_id]['ckpt_path']
shutil.copy(best_ckpt_path, overall_best_ckpt_path)
best_model = CVAE(image_height=image_height,
image_width=image_width,
n_channels=n_channels,
n_hidden_units=n_u)
best_model.full_model.load_weights(overall_best_ckpt_path)
best_results = dict()
best_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
best_results['validation'] = best_model.evaluate(xval, yval_deg,
'validation')
best_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results['best'] = best_results
results_yml_file = os.path.join(experiment_dir, 'results.yml')
with open(results_yml_file, 'w') as results_yml_file:
yaml.dump(results, results_yml_file, default_flow_style=False)
def train():
config_path = sys.argv[1]
with open(config_path, 'r') as f:
config = yaml.load(f)
root_log_dir = config['root_log_dir']
data_path = config['data_path']
experiment_name = '_'.join(
[config['experiment_name'],
get_experiment_id()])
if not os.path.exists(root_log_dir):
os.mkdir(root_log_dir)
experiment_dir = os.path.join(root_log_dir, experiment_name)
os.mkdir(experiment_dir)
shutil.copy(config_path, experiment_dir)
xtr, ytr_deg, xval, yval_deg, xte, yte_deg = load_towncentre(
'data/TownCentre.pkl.gz', canonical_split=True, verbose=1)
image_height, image_width = xtr.shape[1], xtr.shape[2]
ytr_bit = deg2bit(ytr_deg)
yval_bit = deg2bit(yval_deg)
yte_bit = deg2bit(yte_deg)
net_output = config['net_output']
if net_output == 'biternion':
ytr = ytr_bit
yval = yval_bit
elif net_output == 'degrees':
ytr = ytr_deg
yval = yval_deg
else:
raise ValueError("net_output should be 'biternion' or 'degrees'")
predict_kappa = config['predict_kappa']
fixed_kappa_value = config['fixed_kappa_value']
if config['loss'] == 'cosine':
print("using cosine loss..")
loss_te = cosine_loss_tf
elif config['loss'] == 'von_mises':
print("using von-mises loss..")
loss_te = von_mises_loss_tf
elif config['loss'] == 'mad':
print("using mad loss..")
loss_te = mad_loss_tf
elif config['loss'] == 'vm_likelihood':
print("using likelihood loss..")
if predict_kappa:
loss_te = von_mises_neg_log_likelihood_keras
else:
def _von_mises_neg_log_likelihood_keras_fixed(y_true, y_pred):
mu_pred = y_pred[:, 0:2]
kappa_pred = tf.ones([tf.shape(y_pred[:, 2:])[0], 1
]) * fixed_kappa_value
return -K.mean(
von_mises_log_likelihood_tf(y_true, mu_pred, kappa_pred,
net_output))
loss_te = _von_mises_neg_log_likelihood_keras_fixed
else:
raise ValueError(
"loss should be 'mad','cosine','von_mises' or 'vm_likelihood'")
optimizer = get_optimizer(config['optimizer_params'])
best_trial_id = 0
n_trials = 1
results = dict()
for tid in range(0, n_trials):
print("TRIAL %d" % tid)
trial_dir = os.path.join(experiment_dir, str(tid))
os.mkdir(trial_dir)
print("logs could be found at %s" % trial_dir)
vgg_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=fixed_kappa_value)
vgg_model.model.compile(loss=loss_te, optimizer=optimizer)
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=trial_dir)
train_csv_log = os.path.join(trial_dir, 'train.csv')
csv_callback = keras.callbacks.CSVLogger(train_csv_log,
separator=',',
append=False)
best_model_weights_file = os.path.join(
trial_dir, 'vgg_bit_' + config['loss'] + '_town.best.weights.h5')
model_ckpt_callback = keras.callbacks.ModelCheckpoint(
best_model_weights_file,
monitor='val_loss',
mode='min',
save_best_only=True,
save_weights_only=True,
period=1,
verbose=1)
vgg_model.model.fit(x=xtr,
y=ytr,
batch_size=config['batch_size'],
epochs=config['n_epochs'],
verbose=1,
validation_data=(xval, yval),
callbacks=[
tensorboard_callback, csv_callback,
model_ckpt_callback
])
# final_model_ckpt_file = os.path.join(trial_dir, 'vgg_bit_' + config['loss'] + '_town.final.weigths.h5')
# vgg_model.model.save_weights(final_model_ckpt_file)
best_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=fixed_kappa_value)
best_model.model.load_weights(best_model_weights_file)
trial_results = dict()
trial_results['ckpt_path'] = best_model_weights_file
trial_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
trial_results['validation'] = best_model.evaluate(
xval, yval_deg, 'validation')
trial_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results[tid] = trial_results
if tid > 0:
if trial_results['validation']['log_likelihood_mean'] > \
results[best_trial_id]['validation']['log_likelihood_mean']:
best_trial_id = tid
print(
"Better validation loss achieved, current best trial: %d" %
best_trial_id)
print("evaluating model..")
best_ckpt_path = results[best_trial_id]['ckpt_path']
overall_best_ckpt_path = os.path.join(
experiment_dir, 'vgg.full_model.overall_best.weights.hdf5')
shutil.copy(best_ckpt_path, overall_best_ckpt_path)
best_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=fixed_kappa_value)
best_model.model.load_weights(overall_best_ckpt_path)
best_results = dict()
best_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
best_results['validation'] = best_model.evaluate(xval, yval_deg,
'validation')
best_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results['best'] = best_results
results_yml_file = os.path.join(experiment_dir, 'results.yml')
with open(results_yml_file, 'w') as results_yml_file:
yaml.dump(results, results_yml_file, default_flow_style=False)
return
def main():
exp_id = get_experiment_id()
root_log_dir = 'logs/vmmix/'
if not os.path.exists(root_log_dir):
os.mkdir(root_log_dir)
experiment_dir = os.path.join(root_log_dir, exp_id)
os.mkdir(experiment_dir)
xtr, ytr_deg, xval, yval_deg, xte, yte_deg = load_towncentre(
'data/TownCentre.pkl.gz', canonical_split=True, verbose=1)
# xtr, ytr_deg = aug_data(xtr, ytr_deg)
# xval, yval_deg = aug_data(xval, yval_deg)
# xte, yte_deg = aug_data(xval, yval_deg)
ytr_bit = deg2bit(ytr_deg)
yval_bit = deg2bit(yval_deg)
yte_bit = deg2bit(yte_deg)
image_height, image_width, n_channels = xtr.shape[1:]
phi_shape = yte_bit.shape[1]
best_trial_id = 0
n_trials = 5
results = dict()
n_epochs = 100
batch_size = 32
n_components = 5
for tid in range(0, n_trials):
print("TRIAL %d" % tid)
trial_dir = os.path.join(experiment_dir, str(tid))
os.mkdir(trial_dir)
vmmix_best_ckpt_path = os.path.join(
trial_dir, 'vmmix.full_model.trial_%d.best.weights.hdf5' % tid)
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=trial_dir)
train_csv_log = os.path.join(trial_dir, 'train.csv')
csv_callback = keras.callbacks.CSVLogger(train_csv_log,
separator=',',
append=False)
model_ckpt_callback = keras.callbacks.ModelCheckpoint(
vmmix_best_ckpt_path,
monitor='val_loss',
mode='min',
save_best_only=True,
save_weights_only=True,
period=1,
verbose=1)
vggmix_model = BiternionVGGMixture(image_height=image_height,
image_width=image_width,
n_channels=n_channels,
n_components=n_components)
vggmix_model.model.fit(xtr,
ytr_bit,
batch_size=batch_size,
epochs=n_epochs,
validation_data=(xval, yval_bit),
callbacks=[
tensorboard_callback, csv_callback,
model_ckpt_callback
])
best_model = BiternionVGGMixture(image_height=image_height,
image_width=image_width,
n_channels=n_channels,
n_components=n_components)
best_model.model.load_weights(vmmix_best_ckpt_path)
trial_results = dict()
trial_results['ckpt_path'] = vmmix_best_ckpt_path
trial_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
trial_results['validation'] = best_model.evaluate(
xval, yval_deg, 'validation')
trial_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results[tid] = trial_results
if tid > 0:
if trial_results['validation']['log_likelihood_mean'] > \
results[best_trial_id]['validation']['log_likelihood_mean']:
best_trial_id = tid
print(
"Better validation loss achieved, current best trial: %d" %
best_trial_id)
print("Loading best model (trial_id = %d)" % best_trial_id)
best_ckpt_path = results[best_trial_id]['ckpt_path']
overall_best_ckpt_path = os.path.join(
experiment_dir, 'vmmix.full_model.overall_best.weights.hdf5')
shutil.copy(best_ckpt_path, overall_best_ckpt_path)
best_model = BiternionVGGMixture(image_height=image_height,
image_width=image_width,
n_channels=n_channels,
n_components=n_components)
best_model.model.load_weights(overall_best_ckpt_path)
best_results = dict()
best_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
best_results['validation'] = best_model.evaluate(xval, yval_deg,
'validation')
best_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results['best'] = best_results
results_yml_file = os.path.join(experiment_dir, 'results.yml')
with open(results_yml_file, 'w') as results_yml_file:
yaml.dump(results, results_yml_file, default_flow_style=False)