def _convert_angles(data):
gaze_spherical = cart_to_spherical(data['gaze_xyz'])
data['yaw_rad'] = gaze_spherical[:, 1]
data['yaw_deg'] = np.rad2deg(data['yaw_rad'])
data['yaw_bit'] = rad2bit(data['yaw_rad'])
data['pitch_rad'] = gaze_spherical[:, 2]
data['pitch_deg'] = np.rad2deg(data['pitch_rad'])
data['pitch_bit'] = rad2bit(data['pitch_rad'])
def pdf(self, x, angle='azimuth', kappa=None):
vals = np.arange(0, 2 * np.pi, 0.01)
n_images = x.shape[0]
x_vals_tiled = np.ones(n_images)
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = \
self.unpack_preds(self.model.predict(x))
if angle == 'azimuth':
mu_preds_bit = az_preds_bit
kappa_preds = az_preds_kappa
gamma = self.az_gamma
if self.loss_type == 'cosine':
kappa_preds = np.ones([x.shape[0], 1]) * kappa
pdf_vals = np.zeros([n_images, len(vals)])
for xid, xval in enumerate(vals):
x_bit = rad2bit(x_vals_tiled * xval)
pdf_vals[:, xid] = P_UNIFORM*gamma + \
(1-gamma)*np.exp(np.squeeze(von_mises_log_likelihood_np(x_bit, mu_preds_bit, kappa_preds)))
return vals, pdf_vals
def pdf(self, x, x_vals):
n_images = x.shape[0]
x_vals_tiled = np.ones(n_images)
preds = self.model.predict(x)
mu_preds_bit = preds[:, 0:2]
if self.predict_kappa:
kappa_preds = preds[:, 2:]
else:
kappa_preds = np.ones([x.shape[0], 1]) * self.fixed_kappa_value
log_likelihoods = np.zeros([n_images, len(x_vals)])
for xid, xval in enumerate(x_vals):
x_bit = rad2bit(x_vals_tiled * xval)
log_likelihoods[:, xid] = np.exp(
np.squeeze(
von_mises_log_likelihood_np(x_bit, mu_preds_bit,
kappa_preds)))
return log_likelihoods
def pdf(self, x, x_vals):
""" Compute probability density function on a circle given images
Parameters
----------
x: numpy array of shape [n_images, image_width, image_height, n_channels]
angles in biternion (cos, sin) representation that will be used to compute likelihood
x_vals: numpy array of shape [n_points]
angles (in rads) at which pdf values were computed
Returns
-------
pdfs: numpy array of shape [n_images, n_components, n_points]
array containing pdf values for each CVAE sample on circle [0, 2pi] for each values
acc_pdf: numpy array of shape [n_images, n_points]
array containing accumulated pdf value on circle [0, 2pi] for each values
"""
n_images = x.shape[0]
x_vals_tiled = np.ones(n_images)
preds = self.model.predict(x)
mu_preds, kappa_preds, component_probs = self.parse_output_np(preds)
component_probs = np.tile(component_probs.reshape([n_images, self.n_components, 1]), [1, 1, len(x_vals)])
vm_pdfs = np.zeros([n_images, self.n_components, len(x_vals)])
for xid, xval in enumerate(x_vals):
for cid in range(0, self.n_components):
x_bit = rad2bit(x_vals_tiled*xval)
vm_pdfs[:, cid, xid] = np.exp(np.squeeze(von_mises_log_likelihood_np(x_bit,
mu_preds[:, cid, :],
kappa_preds[:, cid])))
acc_pdf = np.sum((component_probs*vm_pdfs), axis=1)
return vm_pdfs, acc_pdf, component_probs
def pdf_importance(self, x, y_bit, x_vals, n_samples=10):
""" Compute probability density function on a circle given images
Parameters
----------
x: numpy array of shape [n_images, image_width, image_height, n_channels]
angles in biternion (cos, sin) representation that will be used to compute likelihood
x_vals: numpy array of shape [n_points]
angles (in rads) at which pdf values were computed
Returns
-------
pdfs: numpy array of shape [n_images, n_samples, n_points]
array containing pdf values for each CVAE sample on circle [0, 2pi] for each values
acc_pdf: numpy array of shape [n_images, n_points]
array containing accumulated pdf value on circle [0, 2pi] for each values
"""
n_images = x.shape[0]
x_vals_tiled = np.ones(n_images)
decoder_preds = self.get_multiple_decoder_predictions(
x, n_samples=n_samples)
vm_pdfs = np.zeros([n_images, n_samples, len(x_vals)])
for xid, xval in enumerate(x_vals):
for sid in range(0, n_samples):
x_bit = rad2bit(x_vals_tiled * xval)
vm_pdfs[:, sid, xid] = np.exp(
np.squeeze(
von_mises_log_likelihood_np(
x_bit, decoder_preds['mu_bit'][:, sid, :],
decoder_preds['kappa'][:, sid])))
acc_pdf = np.mean(vm_pdfs, axis=1)
return vm_pdfs, acc_pdf
def pdf(self, x, gamma=1.0e-1, angle='azimuth', step=0.01):
"""
:param x: input images
:param gamma: weight of a default uniform distribution added to mixture
:param angle: azimuth, elevation or tilt
:param step: step of pdf
:return: points at (0, 2pi) and corresponding pdf values
"""
vals = np.arange(0, 2 * np.pi, step)
n_images = x.shape[0]
x_vals_tiled = np.ones(n_images)
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = \
self.unpack_all_preds(self.model.predict(x))
if angle == 'azimuth':
mu_preds_bit = az_preds_bit
kappa_preds = az_preds_kappa
gamma = self.az_gamma
elif angle == 'elevation':
mu_preds_bit = el_preds_bit
kappa_preds = el_preds_kappa
gamma = self.el_gamma
elif angle == 'tilt':
mu_preds_bit = ti_preds_bit
kappa_preds = ti_preds_kappa
gamma = self.ti_gamma
pdf_vals = np.zeros([n_images, len(vals)])
for xid, xval in enumerate(vals):
x_bit = rad2bit(x_vals_tiled * xval)
pdf_vals[:, xid] = np.exp(
self.log_likelihood(x_bit,
mu_preds_bit,
kappa_preds,
gamma,
angle=angle,
verbose=0)[0])
return vals, pdf_vals
def pdf(self, x, gamma, angle='azimuth'):
vals = np.arange(0, 2*np.pi, 0.01)
n_images = x.shape[0]
x_vals_tiled = np.ones(n_images)
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = \
self.unpack_all_preds(self.model.predict(x))
if angle == 'azimuth':
mu_preds_bit = az_preds_bit
kappa_preds = az_preds_kappa
gamma = self.az_gamma
pdf_vals = np.zeros([n_images, len(vals)])
for xid, xval in enumerate(vals):
x_bit = rad2bit(x_vals_tiled*xval)
pdf_vals[:, xid] = np.exp(self.log_likelihood(x_bit,
mu_preds_bit, kappa_preds, gamma, angle=angle, verbose=0)[0])
return vals, pdf_vals
def train():
config_path = sys.argv[1]
with open(config_path, 'r') as f:
config = yaml.load(f)
root_log_dir = config['root_log_dir']
data_path = config['data_path']
experiment_name = '_'.join(
[config['experiment_name'],
get_experiment_id()])
if not os.path.exists(root_log_dir):
os.mkdir(root_log_dir)
experiment_dir = os.path.join(root_log_dir, experiment_name)
os.mkdir(experiment_dir)
shutil.copy(config_path, experiment_dir)
(xtr, ptr_rad, ttr_rad, rtr_rad, names_tr), \
(xval, pval_rad, tval_rad, rval_rad, names_val), \
(xte, pte_rad, tte_rad, rte_rad, names_te) = load_idiap('data//IDIAP.pkl')
image_height, image_width = xtr.shape[1], xtr.shape[2]
net_output = config['net_output']
if net_output == 'pan':
ytr = rad2bit(ptr_rad)
yval = rad2bit(pval_rad)
yte = rad2bit(pte_rad)
ytr_deg = np.rad2deg(ptr_rad)
yval_deg = np.rad2deg(pval_rad)
yte_deg = np.rad2deg(pte_rad)
elif net_output == 'tilt':
ytr = rad2bit(ttr_rad)
yval = rad2bit(tval_rad)
yte = rad2bit(tte_rad)
ytr_deg = np.rad2deg(ttr_rad)
yval_deg = np.rad2deg(tval_rad)
yte_deg = np.rad2deg(tte_rad)
elif net_output == 'roll':
ytr = rad2bit(rtr_rad)
yval = rad2bit(rval_rad)
yte = rad2bit(rte_rad)
ytr_deg = np.rad2deg(rtr_rad)
yval_deg = np.rad2deg(rval_rad)
yte_deg = np.rad2deg(rte_rad)
else:
raise ValueError("net_output should be 'pan', 'tilt' or 'roll'")
net_output = config['net_output']
predict_kappa = config['predict_kappa']
fixed_kappa_value = config['fixed_kappa_value']
if config['loss'] == 'cosine':
print("using cosine loss..")
loss_te = cosine_loss_tf
elif config['loss'] == 'von_mises':
print("using von-mises loss..")
loss_te = von_mises_loss_tf
elif config['loss'] == 'mad':
print("using mad loss..")
loss_te = mad_loss_tf
elif config['loss'] == 'vm_likelihood':
print("using likelihood loss..")
if predict_kappa:
loss_te = von_mises_neg_log_likelihood_keras
else:
def _von_mises_neg_log_likelihood_keras_fixed(y_true, y_pred):
mu_pred = y_pred[:, 0:2]
kappa_pred = tf.ones([tf.shape(y_pred[:, 2:])[0], 1
]) * fixed_kappa_value
return -K.mean(
von_mises_log_likelihood_tf(y_true, mu_pred, kappa_pred))
loss_te = _von_mises_neg_log_likelihood_keras_fixed
else:
raise ValueError(
"loss should be 'mad','cosine','von_mises' or 'vm_likelihood'")
best_trial_id = 0
n_trials = config['n_trials']
batch_size = config['batch_size']
learning_rate = config['optimizer_params']['learning_rate']
results = dict()
for tid in range(0, n_trials):
print("TRIAL %d // %d" % (tid, n_trials))
print("batch_size: %d" % batch_size)
print("learning_rate: %f" % learning_rate)
trial_dir = os.path.join(experiment_dir, str(tid))
os.mkdir(trial_dir)
print("logs could be found at %s" % trial_dir)
vgg_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=fixed_kappa_value)
optimizer = keras.optimizers.Adam(epsilon=1.0e-07,
lr=learning_rate,
decay=0.0)
vgg_model.model.compile(loss=loss_te, optimizer=optimizer)
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=trial_dir)
train_csv_log = os.path.join(trial_dir, 'train.csv')
csv_callback = keras.callbacks.CSVLogger(train_csv_log,
separator=',',
append=False)
best_model_weights_file = os.path.join(
trial_dir, 'vgg_bit_' + config['loss'] + '_town.best.weights.h5')
model_ckpt_callback = keras.callbacks.ModelCheckpoint(
best_model_weights_file,
monitor='val_loss',
mode='min',
save_best_only=True,
save_weights_only=True,
period=1,
verbose=1)
vgg_model.model.save_weights(best_model_weights_file)
vgg_model.model.fit(x=xtr,
y=ytr,
batch_size=batch_size,
epochs=config['n_epochs'],
verbose=1,
validation_data=(xval, yval),
callbacks=[
tensorboard_callback, csv_callback,
model_ckpt_callback
])
best_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=fixed_kappa_value)
best_model.model.load_weights(best_model_weights_file)
trial_results = dict()
trial_results['learning_rate'] = float(learning_rate)
trial_results['batch_size'] = float(batch_size)
trial_results['ckpt_path'] = best_model_weights_file
trial_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
trial_results['validation'] = best_model.evaluate(
xval, yval_deg, 'validation')
trial_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results[tid] = trial_results
if tid > 0:
if trial_results['validation']['log_likelihood_mean'] > \
results[best_trial_id]['validation']['log_likelihood_mean']:
best_trial_id = tid
print(
"Better validation loss achieved, current best trial: %d" %
best_trial_id)
print("loading best model..")
best_ckpt_path = results[best_trial_id]['ckpt_path']
overall_best_ckpt_path = os.path.join(
experiment_dir, 'vgg.full_model.overall_best.weights.hdf5')
shutil.copy(best_ckpt_path, overall_best_ckpt_path)
print("finetuning kappa values..")
best_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=fixed_kappa_value)
best_model.model.load_weights(overall_best_ckpt_path)
best_kappa = fixed_kappa_value
if not predict_kappa:
best_kappa = finetune_kappa(xval, yval, best_model)
print("best kappa: %f" % best_kappa)
best_model = vgg.BiternionVGG(image_height=image_height,
image_width=image_width,
n_channels=3,
predict_kappa=predict_kappa,
fixed_kappa_value=best_kappa)
best_model.model.load_weights(overall_best_ckpt_path)
best_results = dict()
best_results['learning_rate'] = results[best_trial_id]['learning_rate']
best_results['batch_size'] = results[best_trial_id]['batch_size']
print("evaluating best model..")
best_results['train'] = best_model.evaluate(xtr, ytr_deg, 'train')
best_results['validation'] = best_model.evaluate(xval, yval_deg,
'validation')
best_results['test'] = best_model.evaluate(xte, yte_deg, 'test')
results['best'] = best_results
results_yml_file = os.path.join(experiment_dir, 'results.yml')
with open(results_yml_file, 'w') as results_yml_file:
yaml.dump(results, results_yml_file, default_flow_style=False)
return
