def evaluate(self, x, y_true, verbose=1, return_full=False):
y_pred = self.model.predict(np.asarray(x))
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = \
self.unpack_all_preds(y_pred)
az_preds_deg, el_preds_deg, ti_preds_deg = self.convert_to_deg_preds(
y_pred)
az_true_bit, el_true_bit, ti_true_bit = self.unpack_target(y_true)
az_true_deg = bit2deg(az_true_bit)
el_true_deg = bit2deg(el_true_bit)
ti_true_deg = bit2deg(ti_true_bit)
az_aads, az_maad, az_sem = self.maad(az_true_deg,
az_preds_deg,
'azimuth',
verbose=verbose)
el_aads, el_maad, el_sem = self.maad(el_true_deg,
el_preds_deg,
'elevation',
verbose=verbose)
ti_aads, ti_maad, ti_sem = self.maad(ti_true_deg,
ti_preds_deg,
'tilt',
verbose=verbose)
az_lls, az_ll_mean, az_ll_sem = self.log_likelihood(az_true_bit,
az_preds_bit,
az_preds_kappa,
self.az_gamma,
'azimuth',
verbose=verbose)
el_lls, el_ll_mean, el_ll_sem = self.log_likelihood(el_true_bit,
el_preds_bit,
el_preds_kappa,
self.el_gamma,
'elevation',
verbose=verbose)
ti_lls, el_ll_mean, ti_ll_sem = self.log_likelihood(ti_true_bit,
ti_preds_bit,
ti_preds_kappa,
self.ti_gamma,
'tilt',
verbose=verbose)
lls = az_lls + el_lls + ti_lls
ll_mean = np.mean(lls)
ll_sem = stats.sem(lls)
maad_mean = np.mean([az_maad, el_maad, ti_maad])
print("MAAD TOTAL: %2.2f+-%2.2fSE" % (maad_mean, az_sem))
print("Log-likelihood TOTAL: %2.2f+-%2.2fSE" % (ll_mean, ll_sem))
if return_full:
return maad_mean, ll_mean, ll_sem, lls
else:
return maad_mean, ll_mean, ll_sem
def convert_to_deg_preds(self, y_pred):
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = self.unpack_all_preds(y_pred)
az_preds_deg_lst = np.vstack([bit2deg(az_preds_bit[sid]) for sid in range(0, self.n_samples)]).T
az_preds_deg = maximum_expected_utility(az_preds_deg_lst)
el_preds_deg_lst = np.vstack([bit2deg(el_preds_bit[sid]) for sid in range(0, self.n_samples)]).T
el_preds_deg = maximum_expected_utility(el_preds_deg_lst)
ti_preds_deg_lst = np.vstack([bit2deg(ti_preds_bit[sid]) for sid in range(0, self.n_samples)]).T
ti_preds_deg = maximum_expected_utility(ti_preds_deg_lst)
return az_preds_deg, el_preds_deg, ti_preds_deg
def evaluate(self, x, ytrue_deg, data_part, return_per_image=False):
ytrue_bit = deg2bit(ytrue_deg)
ypreds = self.model.predict(x)
results = dict()
vmmix_mu, vmmix_kappas, vmmix_probs = self.parse_output_np(ypreds)
vmmix_mu_rad = np.deg2rad(bit2deg_multi(vmmix_mu))
samples = sample_von_mises_mixture_multi(vmmix_mu_rad, vmmix_kappas, vmmix_probs, n_samples=100)
point_preds = maximum_expected_utility(np.rad2deg(samples))
maad_errs = maad_from_deg(point_preds, ytrue_deg)
results['maad_loss'] = float(np.mean(maad_errs))
results['maad_sem'] = float(sem(maad_errs))
log_likelihoods = self._von_mises_mixture_log_likelihood_np(ytrue_bit, ypreds)
results['log_likelihood_mean'] = float(np.mean(log_likelihoods))
results['log_likelihood_sem'] = float(sem(log_likelihoods, axis=None))
print("MAAD error (%s) : %f pm %fSEM" % (data_part,
results['maad_loss'],
results['maad_sem']))
print("log-likelihood (%s) : %f pm %fSEM" % (data_part,
results['log_likelihood_mean'],
results['log_likelihood_sem']))
if return_per_image:
results['point_preds'] = bit2deg(deg2bit(point_preds))
results['maad'] = maad_errs
results['log_likelihood'] = log_likelihoods
return results
def save_detections_for_official_eval(self, x, save_path):
# det path example: '/home/sprokudin/RenderForCNN/view_estimation/vp_test_results/aeroplane_pred_view.txt'
y_pred = self.model.predict(np.asarray(x))
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = self.unpack_preds(
y_pred)
az_preds_deg = bit2deg(az_preds_bit)
el_preds_deg = bit2deg(el_preds_bit)
ti_preds_deg = bit2deg(ti_preds_bit)
y_pred = np.vstack([az_preds_deg, el_preds_deg, ti_preds_deg]).T
np.savetxt(save_path, y_pred, delimiter=' ', fmt='%i')
print("evaluation data saved to %s" % save_path)
return
def evaluate_multi(self, x, ytrue_deg, data_part, n_samples=50, verbose=1):
ytrue_bit = deg2bit(ytrue_deg)
results = dict()
preds = self.get_multiple_predictions(x,
ytrue_bit,
n_samples=n_samples)
results['elbo'] = np.mean(preds['elbo'])
results['elbo_sem'] = sem(np.mean(preds['elbo'], axis=1))
results['kl_div'] = np.mean(preds['kl_div'])
results['kl_div_sem'] = sem(np.mean(preds['kl_div'], axis=1))
ypreds = self.decoder_model.predict(x)
ypreds_bit = ypreds[:, 0:2]
kappa_preds = ypreds[:, 2:]
ypreds_deg = bit2deg(ypreds_bit)
loss = maad_from_deg(ytrue_deg, preds['maxutil_deg_dec'])
results['maad_loss'] = np.mean(loss)
results['maad_loss_sem'] = sem(loss, axis=None)
importance_loglikelihoods = importance_loglikelihood(
preds['mu_encoder'], preds['log_sigma_encoder'], preds['mu_prior'],
preds['log_sigma_prior'], preds['u_encoder'], preds['mu_bit'],
preds['kappa'], ytrue_bit)
results['importance_log_likelihood'] = np.mean(
importance_loglikelihoods)
results['importance_log_likelihood_sem'] = sem(
importance_loglikelihoods, axis=None)
if verbose:
print("MAAD error (%s) : %f ± %fSEM" %
(data_part, results['maad_loss'], results['maad_loss_sem']))
print("ELBO (%s) : %f ± %fSEM" %
(data_part, results['elbo'], results['elbo_sem']))
print(
"Approx Log-Likelihood, importance sampling (%s) : %f ± %fSEM"
% (data_part, results['importance_log_likelihood'],
results['importance_log_likelihood_sem']))
print("KL-div (%s) : %f ± %fSEM" %
(data_part, results['kl_div'], results['kl_div_sem']))
return results
def evaluate(self, x, ytrue_deg, data_part, return_per_image=False):
ytrue_bit = deg2bit(ytrue_deg)
ypreds = self.model.predict(x)
ypreds_bit = ypreds[:, 0:2]
ypreds_deg = bit2deg(ypreds_bit)
if self.predict_kappa:
kappa_preds = ypreds[:, 2:]
else:
kappa_preds = np.ones([ytrue_deg.shape[0], 1
]) * self.fixed_kappa_value
loss = maad_from_deg(ypreds_deg, ytrue_deg)
results = dict()
results['maad_loss'] = float(np.mean(loss))
results['maad_loss_sem'] = float(sem(loss))
print("MAAD error (%s) : %f pm %fSEM" %
(data_part, results['maad_loss'], results['maad_loss_sem']))
results['mean_kappa'] = float(np.mean(kappa_preds))
results['std_kappa'] = float(np.std(kappa_preds))
log_likelihoods = von_mises_log_likelihood_np(ytrue_bit, ypreds_bit,
kappa_preds)
results['log_likelihood_mean'] = float(np.mean(log_likelihoods))
results['log_likelihood_sem'] = float(sem(log_likelihoods, axis=None))
print("log-likelihood (%s) : %f pm %fSEM" %
(data_part, results['log_likelihood_mean'],
results['log_likelihood_sem']))
if return_per_image:
results['point_preds'] = bit2deg(deg2bit(ypreds_deg))
results['maad'] = loss
results['log_likelihood'] = log_likelihoods
return results
def evaluate(self, x, ytrue_deg, data_part, verbose=1):
ytrue_bit = deg2bit(ytrue_deg)
results = dict()
cvae_preds = self.full_model.predict([x, ytrue_bit])
elbo, ll, kl = self._cvae_elbo_loss_np(ytrue_bit, cvae_preds)
results['elbo'] = np.mean(elbo)
results['elbo_sem'] = sem(elbo)
results['kl'] = np.mean(kl)
results['kl_sem'] = sem(kl)
results['log_likelihood'] = np.mean(ll)
results['log_likelihood_loss_sem'] = sem(ll)
ypreds = self.decoder_model.predict(x)
ypreds_bit = ypreds[:, 0:2]
kappa_preds = ypreds[:, 2:]
ypreds_deg = bit2deg(ypreds_bit)
loss = maad_from_deg(ytrue_deg, ypreds_deg)
results['maad_loss'] = np.mean(loss)
results['maad_loss_sem'] = sem(loss)
# log_likelihood_loss = von_mises_log_likelihood_np(ytrue_bit, ypreds_bit, kappa_preds,
# input_type='biternion')
# results['log_likelihood'] = np.mean(log_likelihood_loss)
# results['log_likelihood_loss_sem'] = sem(log_likelihood_loss)
if verbose:
print("MAAD error (%s) : %f ± %fSEM" %
(data_part, results['maad_loss'], results['maad_loss_sem']))
print("ELBO (%s) : %f ± %fSEM" %
(data_part, results['elbo'], results['elbo_sem']))
print("KL-div (%s) : %f ± %fSEM" %
(data_part, results['kl'], results['kl_sem']))
# print("log-likelihood (%s) : %f±%fSEM" % (data_part,
# results['log_likelihood'],
# results['log_likelihood_loss_sem']))
return results
def evaluate(self, x, y_true, kappas=None, verbose=1, return_full=False):
y_pred = self.model.predict(np.asarray(x))
az_preds_bit, az_preds_kappa, el_preds_bit, el_preds_kappa, ti_preds_bit, ti_preds_kappa = self.unpack_preds(
y_pred)
az_preds_deg = bit2deg(az_preds_bit)
el_preds_deg = bit2deg(el_preds_bit)
ti_preds_deg = bit2deg(ti_preds_bit)
az_true_bit, el_true_bit, ti_true_bit = self.unpack_target(y_true)
az_true_deg = bit2deg(az_true_bit)
el_true_deg = bit2deg(el_true_bit)
ti_true_deg = bit2deg(ti_true_bit)
az_aads, az_maad, az_sem = self.maad(az_true_deg,
az_preds_deg,
'azimuth',
verbose=verbose)
el_aads, el_maad, el_sem = self.maad(el_true_deg,
el_preds_deg,
'elevation',
verbose=verbose)
ti_aads, ti_maad, ti_sem = self.maad(ti_true_deg,
ti_preds_deg,
'tilt',
verbose=verbose)
if self.loss_type == 'cosine':
print("cosine loss, using fixed kappas : %s" % str(kappas))
az_kappa, el_kappa, ti_kappa = kappas
az_preds_kappa = np.ones([y_true.shape[0], 1]) * az_kappa
el_preds_kappa = np.ones([y_true.shape[0], 1]) * el_kappa
ti_preds_kappa = np.ones([y_true.shape[0], 1]) * ti_kappa
az_lls, az_ll_mean, az_ll_sem = self.log_likelihood(az_true_bit,
az_preds_bit,
az_preds_kappa,
self.az_gamma,
'azimuth',
verbose=verbose)
el_lls, el_ll_mean, el_ll_sem = self.log_likelihood(el_true_bit,
el_preds_bit,
el_preds_kappa,
self.el_gamma,
'elevation',
verbose=verbose)
ti_lls, el_ll_mean, ti_ll_sem = self.log_likelihood(ti_true_bit,
ti_preds_bit,
ti_preds_kappa,
self.ti_gamma,
'tilt',
verbose=verbose)
lls = az_lls + el_lls + ti_lls
ll_mean = np.mean(lls)
ll_sem = stats.sem(lls)
maad_mean = np.mean([az_maad, el_maad, ti_maad])
print("MAAD TOTAL: %2.2f+-%2.2fSE" % (maad_mean, az_sem))
print("Log-likelihood TOTAL: %2.2f+-%2.2fSE" % (ll_mean, ll_sem))
if return_full:
return maad_mean, ll_mean, ll_sem, lls
else:
return maad_mean, ll_mean, ll_sem