def get_percentiles_from_file(directory, percentiles):
history = get_values_from_file(directory)
angular_metrics = []
angular_errors_history = []
for entry in history:
angular_errors = geodesic_error(entry["gt_params"], entry["opt_params"])
metric = np.mean(angular_errors)
angular_metrics.append(metric)
angular_errors_history.append(np.mean(angular_errors, axis=1))
index = angular_metrics.index(np.min(angular_metrics))
print(index)
best_iter_errors = np.array(angular_errors_history[index])
percentile_bins = []
for percentile in percentiles:
num_in_percentile = np.sum(best_iter_errors < percentile)
percentile_bins.append(num_in_percentile)
percentile_bins = np.array(percentile_bins, dtype=float) / float(best_iter_errors.shape[0])
print(percentile_bins)
np.save("sgd_ROC.npy", percentile_bins)
#np.save("adam_ROC.npy", percentile_bins)
#np.save("deep_opt_ROC.npy", percentile_bins)
plt.plot(percentiles, percentile_bins)
plt.xlabel("Angular error (radians)", fontsize=16, fontweight="bold")
plt.ylabel("Percentage of samples", fontsize=16, fontweight="bold")
plt.xticks(fontsize=14, fontweight='bold')
plt.yticks(fontsize=14, fontweight='bold')
plt.show()
def store_results(self, epoch, data, preds):
data_dict = {
self.input_names[i]: value
for i, value in enumerate(data)
}
preds_dict = {
self.output_names[i]: value
for i, value in enumerate(preds)
}
# Calculate values to store
gt_params = data_dict["gt_params"]
self.gt_params_log.write(gt_params)
optlearner_params = preds_dict["learned_params"]
self.opt_params_log.write(optlearner_params)
delta_d_hat = preds_dict["delta_d_hat"]
self.delta_d_hat_log.write(delta_d_hat)
delta_angle = geodesic_error(gt_params, optlearner_params)
delta_angle = np.mean(delta_angle, axis=0)
self.delta_angle_log.write(delta_angle)
delta_d = preds_dict["delta_d"]
self.delta_d_log.write(delta_d)
def get_per_param_metrics_from_file(directory):
history = get_values_from_file(directory)
angular_metric_history = []
for entry in history:
angular_errors = geodesic_error(entry["gt_params"], entry["opt_params"])
angular_metric = np.mean(angular_errors, axis=0)
angular_metric_history.append(angular_metric)
return angular_metric_history
def get_metrics_from_file(directory):
history = get_values_from_file(directory)
mse_history = []
median_history = []
ang_metric_history = []
ang_median_history = []
for entry in history:
median = np.median(np.square(entry["delta_d"]))
mse = np.mean(np.square(entry["delta_d"]))
mse_history.append(mse)
median_history.append(median)
angular_errors = geodesic_error(entry["gt_params"],
entry["opt_params"])
angular_metric = np.mean(angular_errors)
angular_metric_median = np.median(angular_errors)
ang_metric_history.append(angular_metric)
ang_median_history.append(angular_metric_median)
return mse_history, median_history, ang_metric_history, ang_median_history
def get_metrics_from_file(directory):
history = get_values_from_file(directory)
mse_history = []
median_history = []
percentile_history = []
ang_metric_history = []
ang_median_history = []
ang_percentile_history = []
for entry in history:
delta_d = entry["gt_params"] - entry["opt_params"]
median = np.median(np.square(delta_d))
mse = np.mean(np.square(delta_d))
percentile = np.percentile(np.square(delta_d), q=10, interpolation="higher")
mse_history.append(mse)
median_history.append(median)
percentile_history.append(percentile)
angular_errors = geodesic_error(entry["gt_params"], entry["opt_params"])
joints_to_exclude = []
#joints_to_exclude = [7, 8, 10, 11, 20, 21, 22, 23]
#joints_to_exclude = [7, 8, 10, 11, 12, 15, 20, 21, 22, 23]
angular_errors_filtered = angular_errors[:, [i for i in range(24) if i not in joints_to_exclude]]
#print("selected joints: " + str(np.mean(angular_errors_filtered)))
#exit(1)
angular_errors = angular_errors_filtered
angular_metric = np.mean(angular_errors)
#print("all joints: " + str(angular_metric) + "\n")
angular_metric_median = np.median(angular_errors)
ang_percentile = np.percentile(angular_errors, q=10, interpolation="higher")
#ang_perc_per_joint = np.percentile(angular_errors, q=10, interpolation="higher", axis=0)
#print(ang_perc_per_joint)
#exit(1)
ang_metric_history.append(angular_metric)
ang_median_history.append(angular_metric_median)
ang_percentile_history.append(ang_percentile)
return mse_history, median_history, percentile_history, ang_metric_history, ang_median_history, ang_percentile_history
def on_epoch_end(self, epoch, logs=None):
if epoch % self.period == 0:
data, _ = self.generator.yield_data(epoch)
data_dict = {"embedding_index": np.array(data[0]), "gt_params": np.array(data[1]), "gt_pc": np.array(data[2])}
if self.ARCHITECTURE == "PeriodicOptLearnerArchitecture":
data_dict["params_to_train"] = np.array(data[3])
if self.ARCHITECTURE == "NewDeepConv1DOptLearnerArchitecture":
data_dict["trainable_params"] = np.array(data[3])
preds = self.model.predict(data_dict)
# Process outputs to be easy to read
metrics_names = self.model.metrics_names[:-1]
output_names = [metric[:-5] for i, metric in enumerate(metrics_names) if i > 0]
preds_dict = {output_name: preds[i] for i, output_name in enumerate(output_names)}
# Calculate values to store
delta_d_hat = preds_dict["delta_d_hat"]
#delta_d_hat_mean = np.mean(delta_d_hat, axis=0)
delta_d_hat_samples = delta_d_hat[:self.num_samples]
self.delta_d_hat_log.write('epoch {:05d}\n'.format(epoch + 1))
self.delta_d_hat_log.write(str(delta_d_hat_samples) + "\n")
gt_params = data_dict["gt_params"]
optlearner_params = preds_dict["learned_params"]
delta_angle = geodesic_error(gt_params, optlearner_params)
delta_angle = np.mean(delta_angle, axis=0)
self.delta_angle_log.write('epoch {:05d}\n'.format(epoch + 1))
self.delta_angle_log.write(str(delta_angle) + "\n")
delta_d = preds_dict["delta_d"]
#delta_d_mean = np.mean(delta_d, axis=0)
delta_d_samples = delta_d[:self.num_samples]
self.delta_d_log.write('epoch {:05d}\n'.format(epoch + 1))
self.delta_d_log.write(str(delta_d_samples) + "\n")
delta_d_abs = np.mean(abs(preds_dict["delta_d"]), axis=0)
#print("Delta_d_abs: " + str(delta_d_abs))
self.delta_d_abs_log.write('epoch {:05d}\n'.format(epoch + 1))
self.delta_d_abs_log.write(str(delta_d_abs) + "\n")
params_angle = geodesic_error(delta_d, delta_d_hat)
params_angle = np.mean(params_angle, axis=0)
self.params_angle_log.write('epoch {:05d}\n'.format(epoch + 1))
self.params_angle_log.write(str(params_angle) + "\n")
if "params_mse" in preds_dict.keys():
params_mse = np.mean(preds_dict["params_mse"], axis=0)
print("Params MSE: " + str(params_mse))
self.params_mse_log.write('epoch {:05d}\n'.format(epoch + 1))
self.params_mse_log.write(str(params_mse) + "\n")
params_mspe = np.mean(preds_dict["params_mse"]/(delta_d_abs + self.epsilon), axis=0)
print("Params MSPE: " + str(params_mspe))
self.params_mspe_log.write('epoch {:05d}\n'.format(epoch + 1))
self.params_mspe_log.write(str(params_mspe) + "\n")
if "gt_cross_product" in preds_dict.keys():
gt_normals = preds_dict["gt_cross_product"][:self.num_samples]
#print("GT normals: " + str(gt_normals))
self.gt_normals_log.write('epoch {:05d}\n'.format(epoch + 1))
self.gt_normals_log.write(str(gt_normals) + "\n")
opt_normals = preds_dict["opt_cross_product"][:self.num_samples]
#print("Opt. normals: " + str(opt_normals))
self.opt_normals_log.write('epoch {:05d}\n'.format(epoch + 1))
self.opt_normals_log.write(str(opt_normals) + "\n")
mse_normals = np.mean(np.square(gt_normals - opt_normals), axis=-1)[:self.num_samples]
#print("mse. normals: " + str(mse_normals))
self.mse_normals_log.write('epoch {:05d}\n'.format(epoch + 1))
self.mse_normals_log.write(str(mse_normals) + "\n")
network_mse_normals = preds_dict["diff_angle_mse"][:self.num_samples]
#print("Cross normals MSE: " + str(network_mse_normals))
self.network_mse_normals_log.write('epoch {:05d}\n'.format(epoch + 1))
self.network_mse_normals_log.write(str(network_mse_normals) + "\n")