def evaluate(self, x, ytrue_deg, data_part, return_per_image=False): ytrue_bit = deg2bit(ytrue_deg) ypreds = self.model.predict(x) results = dict() vmmix_mu, vmmix_kappas, vmmix_probs = self.parse_output_np(ypreds) vmmix_mu_rad = np.deg2rad(bit2deg_multi(vmmix_mu)) samples = sample_von_mises_mixture_multi(vmmix_mu_rad, vmmix_kappas, vmmix_probs, n_samples=100) point_preds = maximum_expected_utility(np.rad2deg(samples)) maad_errs = maad_from_deg(point_preds, ytrue_deg) results['maad_loss'] = float(np.mean(maad_errs)) results['maad_sem'] = float(sem(maad_errs)) log_likelihoods = self._von_mises_mixture_log_likelihood_np(ytrue_bit, ypreds) results['log_likelihood_mean'] = float(np.mean(log_likelihoods)) results['log_likelihood_sem'] = float(sem(log_likelihoods, axis=None)) print("MAAD error (%s) : %f pm %fSEM" % (data_part, results['maad_loss'], results['maad_sem'])) print("log-likelihood (%s) : %f pm %fSEM" % (data_part, results['log_likelihood_mean'], results['log_likelihood_sem'])) if return_per_image: results['point_preds'] = bit2deg(deg2bit(point_preds)) results['maad'] = maad_errs results['log_likelihood'] = log_likelihoods return results
def maad(self, y_true_deg, y_pred_deg, angle='', verbose=1): aads = maad_from_deg(y_true_deg, y_pred_deg) maad = np.mean(aads) sem = stats.sem(aads) if verbose: print("MAAD %s : %2.2f+-%2.2fSE" % (angle, maad, sem)) return aads, maad, sem
def maad(self, y_true_deg, y_pred_deg, angle='', verbose=1): """ Compute Mean Absolute Angular Deviation between ground truth and predictions (in degrees) """ aads = maad_from_deg(y_true_deg, y_pred_deg) maad = np.mean(aads) sem = stats.sem(aads) if verbose: print("MAAD %s : %2.2f+-%2.2fSE" % (angle, maad, sem)) return aads, maad, sem
def evaluate_multi(self, x, ytrue_deg, data_part, n_samples=50, verbose=1): ytrue_bit = deg2bit(ytrue_deg) results = dict() preds = self.get_multiple_predictions(x, ytrue_bit, n_samples=n_samples) results['elbo'] = np.mean(preds['elbo']) results['elbo_sem'] = sem(np.mean(preds['elbo'], axis=1)) results['kl_div'] = np.mean(preds['kl_div']) results['kl_div_sem'] = sem(np.mean(preds['kl_div'], axis=1)) ypreds = self.decoder_model.predict(x) ypreds_bit = ypreds[:, 0:2] kappa_preds = ypreds[:, 2:] ypreds_deg = bit2deg(ypreds_bit) loss = maad_from_deg(ytrue_deg, preds['maxutil_deg_dec']) results['maad_loss'] = np.mean(loss) results['maad_loss_sem'] = sem(loss, axis=None) importance_loglikelihoods = importance_loglikelihood( preds['mu_encoder'], preds['log_sigma_encoder'], preds['mu_prior'], preds['log_sigma_prior'], preds['u_encoder'], preds['mu_bit'], preds['kappa'], ytrue_bit) results['importance_log_likelihood'] = np.mean( importance_loglikelihoods) results['importance_log_likelihood_sem'] = sem( importance_loglikelihoods, axis=None) if verbose: print("MAAD error (%s) : %f ± %fSEM" % (data_part, results['maad_loss'], results['maad_loss_sem'])) print("ELBO (%s) : %f ± %fSEM" % (data_part, results['elbo'], results['elbo_sem'])) print( "Approx Log-Likelihood, importance sampling (%s) : %f ± %fSEM" % (data_part, results['importance_log_likelihood'], results['importance_log_likelihood_sem'])) print("KL-div (%s) : %f ± %fSEM" % (data_part, results['kl_div'], results['kl_div_sem'])) return results
def evaluate(self, x, ytrue_deg, data_part): ypreds_deg = np.squeeze(self.model.predict(x)) loss = maad_from_deg(ypreds_deg, ytrue_deg) results = dict() results['maad_loss'] = float(np.mean(loss)) results['maad_loss_sem'] = float(sem(loss, axis=None)) print("MAAD error (%s) : %f ± %fSEM" % (data_part, results['maad_loss'], results['maad_loss_sem'])) return results
def evaluate(self, x, ytrue_deg, data_part, verbose=1): ytrue_bit = deg2bit(ytrue_deg) results = dict() cvae_preds = self.full_model.predict([x, ytrue_bit]) elbo, ll, kl = self._cvae_elbo_loss_np(ytrue_bit, cvae_preds) results['elbo'] = np.mean(elbo) results['elbo_sem'] = sem(elbo) results['kl'] = np.mean(kl) results['kl_sem'] = sem(kl) results['log_likelihood'] = np.mean(ll) results['log_likelihood_loss_sem'] = sem(ll) ypreds = self.decoder_model.predict(x) ypreds_bit = ypreds[:, 0:2] kappa_preds = ypreds[:, 2:] ypreds_deg = bit2deg(ypreds_bit) loss = maad_from_deg(ytrue_deg, ypreds_deg) results['maad_loss'] = np.mean(loss) results['maad_loss_sem'] = sem(loss) # log_likelihood_loss = von_mises_log_likelihood_np(ytrue_bit, ypreds_bit, kappa_preds, # input_type='biternion') # results['log_likelihood'] = np.mean(log_likelihood_loss) # results['log_likelihood_loss_sem'] = sem(log_likelihood_loss) if verbose: print("MAAD error (%s) : %f ± %fSEM" % (data_part, results['maad_loss'], results['maad_loss_sem'])) print("ELBO (%s) : %f ± %fSEM" % (data_part, results['elbo'], results['elbo_sem'])) print("KL-div (%s) : %f ± %fSEM" % (data_part, results['kl'], results['kl_sem'])) # print("log-likelihood (%s) : %f±%fSEM" % (data_part, # results['log_likelihood'], # results['log_likelihood_loss_sem'])) return results
def evaluate(self, x, ytrue_deg, data_part, return_per_image=False): ytrue_bit = deg2bit(ytrue_deg) ypreds = self.model.predict(x) ypreds_bit = ypreds[:, 0:2] ypreds_deg = bit2deg(ypreds_bit) if self.predict_kappa: kappa_preds = ypreds[:, 2:] else: kappa_preds = np.ones([ytrue_deg.shape[0], 1 ]) * self.fixed_kappa_value loss = maad_from_deg(ypreds_deg, ytrue_deg) results = dict() results['maad_loss'] = float(np.mean(loss)) results['maad_loss_sem'] = float(sem(loss)) print("MAAD error (%s) : %f pm %fSEM" % (data_part, results['maad_loss'], results['maad_loss_sem'])) results['mean_kappa'] = float(np.mean(kappa_preds)) results['std_kappa'] = float(np.std(kappa_preds)) log_likelihoods = von_mises_log_likelihood_np(ytrue_bit, ypreds_bit, kappa_preds) results['log_likelihood_mean'] = float(np.mean(log_likelihoods)) results['log_likelihood_sem'] = float(sem(log_likelihoods, axis=None)) print("log-likelihood (%s) : %f pm %fSEM" % (data_part, results['log_likelihood_mean'], results['log_likelihood_sem'])) if return_per_image: results['point_preds'] = bit2deg(deg2bit(ypreds_deg)) results['maad'] = loss results['log_likelihood'] = log_likelihoods return results