def gen_data(param_trainable, smpl, data_samples=10000, save_dir=None):
""" Generate random body poses """
X_params = 0.2 * 2 * (np.random.rand(data_samples, 85) - 0.5)
k = np.pi
for param, trainable in param_trainable.items():
if trainable:
param_int = int(param[6:8])
X_params[:,
param_int] = 2 * k * (np.random.rand(data_samples) - 0.5)
X_pcs = []
X_silh = []
for params in X_params:
# Render the point cloud
pc = smpl.set_params(beta=params[72:82],
pose=params[0:72],
trans=params[82:85])
X_pcs.append(pc)
# Now render the silhouette from this
silh = Mesh(pointcloud=pc).render_silhouette(show=False)
X_silh.append(silh)
X_pcs = np.array(X_pcs)
X_silh = np.array(X_silh)
if save_dir is not None:
# Save the generated data in the given location
param_dir = save_dir + "smpl_params/"
pc_dir = save_dir + "pointclouds/"
silh_dir = save_dir + "silhouettes/"
os.system('mkdir ' + param_dir)
os.system('mkdir ' + pc_dir)
os.system('mkdir ' + silh_dir)
faces = smpl.faces
for i in range(data_samples):
sample_id = "sample_{:05d}".format(i + 1)
params = X_params[i]
pc = X_pcs[i]
silh = X_silh[i]
np.savetxt(param_dir + sample_id + ".csv", params, delimiter=",")
print_mesh(pc_dir + sample_id + ".obj", pc, faces)
cv2.imwrite(silh_dir + sample_id + ".png", silh.astype("uint8"))
return X_params, X_pcs, X_silh
def on_epoch_begin(self, epoch, logs=None):
import cv2
outputs = self.model.predict(self.data)
#print(outputs)
gt_params = self.data[1]
pred_params = outputs[0]
print("GT and predicted parameters are equal: " +
str(np.allclose(gt_params, pred_params)))
#print(gt_params)
#print(pred_params)
#exit(1)
gt_pc = self.data[2]
#print(gt_pc)
pred_pc = outputs[2]
right_pred = Mesh(pointcloud=pred_pc[0]).render_silhouette(show=False)
cv2.imwrite("right_pred.png", right_pred.astype("uint8"))
wrong_pred = Mesh(pointcloud=pred_pc[10]).render_silhouette(show=False)
cv2.imwrite("wrong_pred.png", wrong_pred.astype("uint8"))
print_mesh("wrong_mesh.obj", pred_pc[10], faces)
print_mesh("right_mesh.obj", pred_pc[0], faces)
#print(pred_pc)
print("GT and predicted point clouds are equal: " +
str(np.allclose(gt_pc, pred_pc)))
close = np.array([
np.int(np.allclose(gt_pc[i], pred_pc[i]))
for i in range(gt_pc.shape[0])
])
#print([np.allclose(gt_pc[i], pred_pc[i]) for i in range(gt_pc.shape[0])])
not_close = np.array([not value for value in close])
close_sum = np.sum(close)
#print("Num close pc: " + str(close_sum))
not_close_gt = gt_pc[not_close]
not_close_pred = pred_pc[not_close]
#print(not_close_pred[0])
#exit(1)
diff_not_close = not_close_gt - not_close_pred
import pandas as pd
diff_df = pd.DataFrame(diff_not_close[10])
diff_df.to_csv("diff.csv")
dist_not_close = np.sum(np.square(diff_not_close), axis=-1)
dist_df = pd.DataFrame(dist_not_close[10])
dist_df.to_csv("dist.csv")
mean_not_close = np.mean(dist_not_close, axis=1)
#print(mean_not_close)
gt_example = not_close_gt[10]
pred_example = not_close_pred[10]
gt_silh = Mesh(pointcloud=gt_example).render_silhouette(show=False)
pred_silh = Mesh(pointcloud=pred_example).render_silhouette(show=False)
diff_silh = (gt_silh != pred_silh) * 255
all_silh = np.concatenate([gt_silh, pred_silh, diff_silh])
cv2.imwrite("silh_comp.png", all_silh.astype("uint8"))
#exit(1)
euc_dist = np.square(np.subtract(gt_pc, pred_pc))
#print(euc_dist[0])
euc_dist_summed = np.sum(euc_dist, axis=-1)
#print(euc_dist_summed[0])
actual_loss = np.mean(euc_dist_summed, axis=1)
#print(actual_loss)
actual_loss_sum = np.sum(actual_loss, axis=0)
#print(actual_loss_sum)
mean_loss = np.mean(actual_loss)
print("Calculated mean loss: {}".format(mean_loss))
print("Model mean loss: {}".format(np.mean(outputs[3])))
def on_epoch_end(self, epoch, logs=None):
""" Store the model loss and accuracy at the end of every epoch, and store a model prediction on data """
#print("Callback called at epoch " + str(epoch))
epoch = int(epoch)
if logs is not None:
self.epoch_log.write(
json.dumps({'epoch': epoch})[:-1] + ", " +
json.dumps(logs)[1:] + '\n')
if (epoch + 1) % self.period == 0 or epoch == 0 or epoch == -1:
# Predict on all of the given input parameters
for data_type, data in self.input_data.items():
if data[0][0] is not None:
print("Saving to directory: \n{}\n".format(self.pred_path))
# Predict on these input parameters
#print("data value: " + str(data))
data_dict = {
"embedding_index": np.array(data[0]),
"gt_params": np.array(data[1]),
"gt_pc": np.array(data[2])
}
preds = self.model.predict(
data_dict) #, batch_size=len(data[0]))
print(str(data_type))
print("------------------------------------")
metrics_names = self.model.metrics_names[:-2]
output_names = [
metric[:-5] for i, metric in enumerate(metrics_names)
if i > 0
]
preds_dict = {
output_name: preds[i]
for i, output_name in enumerate(output_names)
}
#print(preds_dict)
#exit(1)
self.delta_d_log.write('epoch {:05d}\n'.format(epoch + 1))
param_diff_sines = np.abs(
np.sin(0.5 * (data[1] - preds_dict["learned_params"])))
delta_d_diff_sines = np.abs(
np.sin(0.5 * (preds_dict["delta_d"] -
preds_dict["delta_d_hat"])))
trainable_diff_sines = []
for parameter in self.trainable_params:
param_int = int(parameter[6:])
trainable_diff_sines.append(
param_diff_sines[:, param_int])
print("Parameter: " + str(parameter))
print("GT SMPL: " + str(data[1][:, param_int]))
print("Parameters: " +
str(preds_dict["learned_params"][:, param_int]))
print("Parameter ang. MAE: " +
str(param_diff_sines[:, param_int]))
print("Delta_d: " +
str(preds_dict["delta_d"][:, param_int]))
print("Delta_d_hat: " +
str(preds_dict["delta_d_hat"][:, param_int]))
print("Difference sine: " +
str(delta_d_diff_sines[:, param_int]))
#print("Delta_d_hat loss: " + str(preds_dict["delta_d_hat_mse"]))
#print("Difference sine (direct): " + str(np.sin(preds_dict["delta_d"] - preds_dict["delta_d_hat"])[:, param_int]))
#print("Difference sine (from normals): " + str(preds_dict["diff_angles"]))
print("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
#self.delta_d_log.write('parameter: ' + str(parameter) + '\n' + 'Delta_d: ' + str(preds[6][:, param_int]) + '\n')
self.delta_d_log.write('parameter: ' + str(parameter) +
'\n' + 'Delta_d: ' +
str(preds[7][:, param_int]) +
'\n')
avg_diff_sines = np.mean(trainable_diff_sines, axis=0)
# Track resets
BLOCK_SIZE = self.data_samples / self.RESET_PERIOD
#print("BLOCK_SIZE " + str(BLOCK_SIZE))
BLOCKS = self.examples // BLOCK_SIZE
#print("BLOCKS " + str(BLOCKS))
if (epoch - 1) < 0 or self.testing:
was_reset = [False, False, False, False, False]
else:
INDEX = (epoch - 1) % self.RESET_PERIOD
#print("INDEX " + str(INDEX))
was_reset = [entry == INDEX for entry in BLOCKS]
#print("was_reset " + str(was_reset))
#exit(1)
silh_comp_list = []
for i, learned_pc in enumerate(preds[2], 1):
# Store the learned mesh
print_mesh(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.pred_pc_{:03d}.obj".format(
data_type, epoch + 1, i)), learned_pc,
self.smpl.faces)
pred_silhouette = Mesh(
pointcloud=learned_pc).render_silhouette(
show=False)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.pred_silh_{:03d}.png".format(data_type, epoch + 1, i)), pred_silhouette)
if self.gt_silhouettes[data_type] is not None:
# Store predicted silhouette and the difference between it and the GT silhouette
#gt_silhouette = (self.gt_silhouettes[data_type][i-1] * 255).astype("uint8")
gt_silhouette = self.gt_silhouettes[data_type][
i - 1].astype("uint8")
#print("gt_silhouette shape: " + str(gt_silhouette.shape))
gt_silhouette = gt_silhouette.reshape(
(gt_silhouette.shape[0],
gt_silhouette.shape[1]))
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.gt_silh_{:03d}.png".format(data_type, epoch + 1, i)), gt_silhouette)
diff_silh = (gt_silhouette !=
pred_silhouette) * 255
#diff_silh = abs(gt_silhouette - pred_silhouette)
#print(diff_silh.shape)
#cv2.imshow("Diff silh", diff_silh)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.diff_silh_{:03d}.png".format(data_type, epoch + 1, i)), diff_silh.astype("uint8"))
silh_comp = np.concatenate(
[gt_silhouette, pred_silhouette, diff_silh],
axis=1)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:05d}.silh_comp_{:03d}.png".format(data_type, epoch + 1, i)), silh_comp.astype("uint8"))
if was_reset[i - 1]:
# Grey the image
silh_comp /= 2
# Convert to rgb and write the difference sine to the image
silh_comp_rgb = np.zeros(
(silh_comp.shape[0], silh_comp.shape[1], 3))
for c in range(3):
silh_comp_rgb[:, :, c] = silh_comp
# Write to the image
font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCorner = (550, 30)
fontScale = 0.6
fontColor = (0, 0, 255)
lineType = 2
cv2.putText(
silh_comp_rgb, "Ang. MAE: {0:.3f}".format(
avg_diff_sines[i - 1]), bottomLeftCorner,
font, fontScale, fontColor, lineType)
# Add image to list
silh_comp_list.append(silh_comp_rgb)
# Save the predicted point cloud relative to the GT point cloud
print_mesh(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.gt_pc_{:03d}.obj".format(
data_type, epoch + 1, i)), data[2][i - 1],
self.smpl.faces)
print_point_clouds(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.comparison_{:03d}.obj".format(
data_type, epoch + 1, i)),
[learned_pc, data[2][i - 1]], [(255, 0, 0),
(0, 255, 0)])
if len(silh_comp_list) > 0:
silh_comps_rgb = np.concatenate(silh_comp_list, axis=0)
font = cv2.FONT_HERSHEY_SIMPLEX
topLeftCorner = (30, 30)
fontScale = 1
fontColor = (0, 0, 255)
lineType = 2
if self.testing:
text = "Iteration "
else:
text = "Epoch "
cv2.putText(silh_comps_rgb, text + str(epoch + 1),
topLeftCorner, font, fontScale, fontColor,
lineType)
cv2.imwrite(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.silh_comps.png".format(
data_type, epoch + 1)),
silh_comps_rgb.astype("uint8"))
def on_epoch_end(self, epoch, logs=None):
""" Store the model loss and accuracy at the end of every epoch, and store a model prediction on data """
self.epoch_log.write(
json.dumps({
'epoch': epoch,
'loss': logs['loss']
}) + '\n')
if (epoch + 1) % self.period == 0 or epoch == 0:
# Predict on all of the given silhouettes
for data_type, data in self.pred_data.items():
if data is not None:
if not isinstance(data, list) or type(data) == np.array:
data = np.array(data)
data = data.reshape(
(1, data.shape[0], data.shape[1], data.shape[2]))
#for i, silhouette in enumerate(data):
# # Save silhouettes
# silhouette *= 255
# cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.gt_silh_{:03d}.png".format(data_type, epoch + 1, i)), silhouette.astype("uint8"))
preds = self.model.predict(data)
#print("Predictions: " + str(preds))
for i, pred in enumerate(preds[1], 1):
#self.smpl.set_params(pred[:72].reshape((24, 3)), pred[72:82], pred[82:])
#self.smpl.save_to_obj(os.path.join(self.pred_path, "{}_pred_{:03d}.obj".format(data_type, i)))
#print_mesh(os.path.join(self.pred_path, "epoch.{:03d}.{}_gt_{:03d}.obj".format(epoch, data_type, i)), gt[i-1], smpl.faces)
print_mesh(
os.path.join(
self.pred_path,
"{}_epoch.{:03d}.pred_{:03d}.obj".format(
data_type, epoch + 1, i)), pred,
self.smpl.faces)
# Store predicted silhouette and the difference between it and the GT silhouette
gt_silhouette = (data[i - 1] *
255).astype("uint8").reshape(
data.shape[1], data.shape[2])
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.gt_silh_{:03d}.png".format(data_type, epoch + 1, i)), gt_silhouette)
pred_silhouette = Mesh(
pointcloud=pred).render_silhouette(show=False)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.pred_silh_{:03d}.png".format(data_type, epoch + 1, i)), pred_silhouette)
diff_silh = abs(gt_silhouette - pred_silhouette)
#print(diff_silh.shape)
#cv2.imshow("Diff silh", diff_silh)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.diff_silh_{:03d}.png".format(data_type, epoch + 1, i)), diff_silh.astype("uint8"))
silh_comp = np.concatenate(
[gt_silhouette, pred_silhouette, diff_silh])
cv2.imwrite(
os.path.join(
self.pred_path,
"{}_epoch.{:03d}.silh_comp_{:03d}.png".format(
data_type, epoch + 1, i)),
silh_comp.astype("uint8"))
if self.gt_pc[data_type] is not None:
print_mesh(
os.path.join(
self.pred_path,
"{}_epoch.{:03d}.gt_pc_{:03d}.obj".format(
data_type, epoch + 1, i)),
self.gt_pc[data_type], self.smpl.faces)
print_point_clouds(
os.path.join(
self.pred_path,
"{}_epoch.{:03d}.comparison_{:03d}.obj".
format(data_type, epoch + 1, i)),
[pred, self.gt_pc[data_type]], [(255, 0, 0),
(0, 255, 0)])
if self.visualise:
# Show a random sample
rand_index = np.random.randint(low=0,
high=len(data)) + 1
mesh = Mesh(filepath=os.path.join(
self.pred_path, "{}_epoch.{:03d}.pred_{:03d}.obj".
format(data_type, epoch + 1, rand_index)))
# Show the true silhouette
true_silh = data[rand_index - 1]
true_silh = true_silh.reshape(true_silh.shape[:-1])
plt.imshow(true_silh, cmap='gray')
plt.title("True {} silhouette {:03d}".format(
data_type, rand_index))
plt.show()
# Show the predicted silhouette and mesh
mesh.render_silhouette(
title="Predicted {} silhouette {:03d}".format(
data_type, rand_index))
diff_silh = cv2.imread(
"{}_epoch.{:03d}.diff_silh_{:03d}.png".format(
data_type, epoch + 1, rand_index))
cv2.imshow(
"Predicted {} silhouette {:03d}".format(
data_type, rand_index), diff_silh)
try:
mesh.render3D()
except Exception:
pass
def on_epoch_end(self, epoch, logs=None):
""" Store the model loss and accuracy at the end of every epoch, and store a model prediction on data """
#print("Callback called at epoch " + str(epoch))
epoch = int(epoch)
if logs is not None:
self.epoch_log.write(json.dumps({'epoch': epoch})[:-1] + ", " + json.dumps(logs)[1:] + '\n')
#if (epoch + 1) % self.period == 0 or epoch == 0 or epoch == -1:
if epoch % self.period == 0 or epoch == -1:
# Predict on all of the given input parameters
for data_type, data in self.input_data.items():
if data[0][0] is not None or self.generator_paths[data_type] is not None:
print("Saving to directory: \n{}\n".format(self.pred_path))
# Predict on these input parameters
#print("data value: " + str(data))
gen_path = self.generator_paths[data_type]
additional_input = None
if gen_path is not None:
gen_path = gen_path + "cb_samples_E{}.npz".format(epoch)
try:
with np.load(gen_path, allow_pickle=True) as temp_data:
print(temp_data.keys())
if "trainable_params" in temp_data.keys():
data = [temp_data["indices"], temp_data["params"], temp_data["pcs"], temp_data["trainable_params"]]
additional_input = "trainable_params"
elif "params_to_train" in temp_data.keys():
data = [temp_data["indices"], temp_data["params"], temp_data["pcs"], temp_data["params_to_train"]]
additional_input = "params_to_train"
else:
data = [temp_data["indices"], temp_data["params"], temp_data["pcs"]]
except Exception as e:
print("Skipping - load failed with exception '{}'".format(e))
return None
data_dict = {"embedding_index": np.array(data[0]), "gt_params": np.array(data[1]), "gt_pc": np.array(data[2])}
if self.ARCHITECTURE == "PeriodicOptLearnerArchitecture":
additional_input = "params_to_train"
if self.ARCHITECTURE == "NewDeepConv1DOptLearnerArchitecture":
additional_input = "trainable_params"
if additional_input is not None:
data_dict[additional_input] = np.array(data[3])
preds = self.model.predict(data_dict) #, batch_size=len(data[0]))
print(str(data_type))
print("------------------------------------")
#metrics_names = self.model.metrics_names[:-2]
metrics_names = self.model.metrics_names[:-1]
#print(metrics_names)
output_names = [metric[:-5] for i, metric in enumerate(metrics_names) if i > 0]
preds_dict = {output_name: preds[i] for i, output_name in enumerate(output_names)}
#print(preds_dict)
#exit(1)
#print("GT SMPL for first example: " + str(data[1][0]))
#print("Diff for first example: " + str(data[1][0] - preds_dict["learned_params"][0]))
param_diff_sines = np.abs(np.sin(0.5*(data[1] - preds_dict["learned_params"])))
delta_d_diff_sines = np.abs(np.sin(0.5*(preds_dict["delta_d"] - preds_dict["delta_d_hat"])))
trainable_diff_sines = []
for i, parameter in enumerate(self.trainable_params):
param_int = int(parameter[6:8])
trainable_diff_sines.append(param_diff_sines[:, param_int])
print("Parameter: " + str(parameter))
print("GT SMPL: " + str(data[1][:, param_int]))
print("Parameters: " + str(preds_dict["learned_params"][:, param_int]))
print("Parameter ang. MAE: " + str(param_diff_sines[:, param_int]))
if "delta_d_hat_mu" in preds_dict.keys():
print("Delta_d_hat_mu: " + str(preds_dict["delta_d_hat_mu"][:, param_int])) # ProbCNN architecture only
print("Delta_d_hat_sigma: " + str(preds_dict["delta_d_hat_sigma"][:, param_int])) # ProbCNN architecture only
print("Delta_d: " + str(preds_dict["delta_d"][:, param_int]))
print("Delta_d_hat: " + str(preds_dict["delta_d_hat"][:, param_int]))
#print("Delta_d_hat: " + str(preds_dict["delta_d_hat"][:, i+1]))
print("Difference sine: " + str(delta_d_diff_sines[:, param_int]))
#print("Difference sine: " + str(delta_d_diff_sines[:, i]))
#print("Delta_d_hat loss: " + str(preds_dict["delta_d_hat_mse"]))
#print("Difference sine (direct): " + str(np.sin(preds_dict["delta_d"] - preds_dict["delta_d_hat"])[:, param_int]))
#print("Difference sine (from normals): " + str(preds_dict["diff_angles"]))
print("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
#print("Predictions for first example: " + str(preds_dict["delta_d_hat"][0]))
if "rot3d_pose" in preds_dict.keys():
#print("rot3d_pose trace: " + str(np.trace(preds_dict["rot3d_pose"][0], axis1=1, axis2=2)))
#print("rot3d_delta_d_pose trace: " + str(np.trace(preds_dict["rot3d_delta_d_pose"][0], axis1=1, axis2=2)))
#print("rot3d_pose: " + str(preds_dict["rot3d_pose"][0])) # RotConv1d architecture only
#print("rot3d_delta_d_pose: " + str(preds_dict["rot3d_delta_d_pose"][0])) # RotConv1d architecture only
print("rot3d_pose error: " + str(preds_dict["rot3d_pose"][0] - preds_dict["rot3d_delta_d_pose"][0])) # RotConv1d architecture only
#pass
if "mapped_pose" in preds_dict.keys():
#print("mapped_pose: " + str(preds_dict["mapped_pose"][0])) # RotConv1d architecture only
#print("mapped_delta_d_pose: " + str(preds_dict["mapped_delta_d_pose"][0])) # RotConv1d architecture only
print("mapped_pose error: " + str(preds_dict["mapped_pose"][0] - preds_dict["mapped_delta_d_pose"][0])) # RotConv1d architecture only
pass
if "rodrigues_delta_d_pose" in preds_dict.keys():
#print("rodrigues pose error: " + str(preds_dict["rodrigues_delta_d_pose"][0] - preds_dict["delta_d_pose_vec"][0]))
pass
avg_diff_sines = np.mean(trainable_diff_sines, axis=0)
if epoch == -1:
# print parameters to file
gt_example_parameters = data[1]
pred_example_parameters = preds_dict["learned_params"]
diff_example_parameters = gt_example_parameters - pred_example_parameters
param_save_dir = self.pred_path + "/example_parameters/"
os.system('mkdir ' + str(param_save_dir))
np.savetxt(param_save_dir + "gt_params.txt", gt_example_parameters)
np.savetxt(param_save_dir + "pred_params.txt", pred_example_parameters)
np.savetxt(param_save_dir + "diff.txt", diff_example_parameters)
# Track resets
BLOCK_SIZE = self.data_samples / self.RESET_PERIOD
#print("BLOCK_SIZE " + str(BLOCK_SIZE))
BLOCKS = self.examples // BLOCK_SIZE
#print("BLOCKS " + str(BLOCKS))
#if (epoch - 1) < 0 or self.testing:
if epoch < 0 or self.testing:
was_reset = [False for _ in BLOCKS]
else:
#INDEX = (epoch - 1) % self.RESET_PERIOD
INDEX = epoch % self.RESET_PERIOD
#print("INDEX " + str(INDEX))
was_reset = [entry == INDEX for entry in BLOCKS]
#print("was_reset " + str(was_reset))
#exit(1)
silh_comp_list = []
for i, learned_pc in enumerate(preds[2], 1):
# Store the learned mesh
print_mesh(os.path.join(self.pred_path, "{}_epoch.{:05d}.pred_pc_{:03d}.obj".format(data_type, epoch + 1, i)), learned_pc, self.smpl.faces)
pred_silhouette = Mesh(pointcloud=learned_pc).render_silhouette(show=False)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.pred_silh_{:03d}.png".format(data_type, epoch + 1, i)), pred_silhouette)
gt_silhouette = Mesh(pointcloud=data_dict["gt_pc"][i-1]).render_silhouette(show=False)
# Store predicted silhouette and the difference between it and the GT silhouette
#gt_silhouette = (self.gt_silhouettes[data_type][i-1] * 255).astype("uint8")
#gt_silhouette = self.gt_silhouettes[data_type][i-1].astype("uint8")
#print("gt_silhouette shape: " + str(gt_silhouette.shape))
#gt_silhouette = gt_silhouette.reshape((gt_silhouette.shape[0], gt_silhouette.shape[1]))
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.gt_silh_{:03d}.png".format(data_type, epoch + 1, i)), gt_silhouette)
diff_silh = (gt_silhouette != pred_silhouette)*255
#diff_silh = abs(gt_silhouette - pred_silhouette)
#print(diff_silh.shape)
#cv2.imshow("Diff silh", diff_silh)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.diff_silh_{:03d}.png".format(data_type, epoch + 1, i)), diff_silh.astype("uint8"))
silh_comp = np.concatenate([gt_silhouette, pred_silhouette, diff_silh], axis=1)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:05d}.silh_comp_{:03d}.png".format(data_type, epoch + 1, i)), silh_comp.astype("uint8"))
if was_reset[i-1]:
# Grey the image
silh_comp /= 2
# Convert to rgb and write the difference sine to the image
silh_comp_rgb = np.zeros((silh_comp.shape[0], silh_comp.shape[1], 3))
for c in range(3):
silh_comp_rgb[:, :, c] = silh_comp
# Write to the image
font = cv2.FONT_HERSHEY_SIMPLEX
ang_mae = (550,30)
normals_loss = (550,240)
gt_main_rot = (0, 70)
pred_main_rot = (0, 50)
delta_d_hat_pos = (0, 90)
fontScale = 0.6
fontColor = (0,0,255)
lineType = 2
cv2.putText(silh_comp_rgb, "Ang. MAE: {0:.3f}".format(avg_diff_sines[i-1]),
ang_mae,
font,
fontScale,
fontColor,
lineType)
cv2.putText(silh_comp_rgb, "Norm. Loss: {0:.3f}".format(np.mean(preds_dict["diff_angle_mse"][i-1])),
normals_loss,
font,
fontScale,
fontColor,
lineType)
cv2.putText(silh_comp_rgb, "Main rot.: " +str(preds_dict["learned_params"][i-1, 0:3]),
pred_main_rot,
font,
fontScale,
fontColor,
lineType)
cv2.putText(silh_comp_rgb, "GT Main rot.: " +str(data[1][i-1, 0:3]),
gt_main_rot,
font,
fontScale,
fontColor,
lineType)
cv2.putText(silh_comp_rgb, "delta_d_hat: " +str(preds_dict["delta_d_hat"][i-1, 0:3]),
delta_d_hat_pos,
font,
fontScale,
fontColor,
lineType)
# Add image to list
silh_comp_list.append(silh_comp_rgb)
# Save the predicted point cloud relative to the GT point cloud
print_mesh(os.path.join(self.pred_path, "{}_epoch.{:05d}.gt_pc_{:03d}.obj".format(data_type, epoch + 1, i)), data[2][i-1], self.smpl.faces)
print_point_clouds(os.path.join(self.pred_path, "{}_epoch.{:05d}.comparison_{:03d}.obj".format(data_type, epoch + 1, i)), [learned_pc, data[2][i-1]], [(255,0,0),(0,255,0)])
if len(silh_comp_list) > 0:
silh_comps_rgb = np.concatenate(silh_comp_list, axis=0)
font = cv2.FONT_HERSHEY_SIMPLEX
topLeftCorner = (30,30)
fontScale = 1
fontColor = (0,0,255)
lineType = 2
if self.testing:
text = "Iteration "
else:
text = "Epoch "
cv2.putText(silh_comps_rgb, text + str(epoch + 1),
topLeftCorner,
font,
fontScale,
fontColor,
lineType)
cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:05d}.silh_comps.png".format(data_type, epoch + 1)), silh_comps_rgb.astype("uint8"))
def on_epoch_end(self, epoch, logs=None):
""" Store the model loss and accuracy at the end of every epoch, and store a model prediction on data """
if logs is not None:
self.epoch_log.write(
json.dumps({
'epoch': epoch,
'loss': logs['loss']
}) + '\n')
self.param_log.write(
json.dumps({
'epoch': epoch,
'delta_d_mse_loss': logs['delta_d_mse_loss']
}) + '\n')
self.mesh_log.write(
json.dumps(
{
'epoch': epoch,
'pc_mean_euc_dist_loss': logs['pc_mean_euc_dist_loss']
}) + '\n')
epoch = int(epoch)
if (epoch + 1) % self.period == 0 or epoch == 0:
# Predict on all of the given input parameters
for data_type, data in self.input_data.items():
if data[0][0] is not None:
print("Saving to directory '{}'".format(self.pred_path))
# Predict on these input parameters
#print("data value: " + str(data))
data_dict = {
"embedding_index": np.array(data[0]),
"gt_params": np.array(data[1]),
"gt_pc": np.array(data[2])
}
#print("embedding index size: " + str(data_dict["embedding_index"].shape))
#print("gt_params size: " + str(data_dict["gt_params"].shape))
#print("gt_pc size: " + str(data_dict["gt_pc"].shape))
#exit(1)
preds = self.model.predict(
data_dict) #, batch_size=len(data[0]))
print(str(data_type))
print("------------------------------------")
#print("GT SMPL: " + str(data[1][:,0:3]))
#print("Parameters: " + str(preds[0][:,0:3]))
#print("Delta_d: " + str(preds[6][:,0:3]))
#print("Delta_d_hat: " + str(preds[7][:,0:3]))
# print("epoch: " + str(epoch + 1))
# exit(1)
self.delta_d_log.write('epoch {:05d}\n'.format(epoch + 1))
for parameter in self.trainable_params:
param_int = int(parameter[6:])
print("Parameter: " + str(parameter))
print("GT SMPL: " + str(data[1][:, param_int]))
print("Parameters: " + str(preds[0][:, param_int]))
print("Delta_d: " + str(preds[6][:, param_int]))
print("Delta_d_hat: " + str(preds[7][:, param_int]))
print("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
self.delta_d_log.write('parameter: ' + str(parameter) +
'\n' + 'Delta_d: ' +
str(preds[6][:, param_int]) +
'\n')
for i, learned_pc in enumerate(preds[2], 1):
# Store the learned mesh
print_mesh(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.pred_pc_{:03d}.obj".format(
data_type, epoch + 1, i)), learned_pc,
self.smpl.faces)
pred_silhouette = Mesh(
pointcloud=learned_pc).render_silhouette(
show=False)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.pred_silh_{:03d}.png".format(data_type, epoch + 1, i)), pred_silhouette)
if self.gt_silhouettes[data_type] is not None:
# Store predicted silhouette and the difference between it and the GT silhouette
#gt_silhouette = (self.gt_silhouettes[data_type][i-1] * 255).astype("uint8")
gt_silhouette = self.gt_silhouettes[data_type][
i - 1].astype("uint8")
#print("gt_silhouette shape: " + str(gt_silhouette.shape))
gt_silhouette = gt_silhouette.reshape(
(gt_silhouette.shape[0],
gt_silhouette.shape[1]))
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.gt_silh_{:03d}.png".format(data_type, epoch + 1, i)), gt_silhouette)
diff_silh = abs(gt_silhouette - pred_silhouette)
#print(diff_silh.shape)
#cv2.imshow("Diff silh", diff_silh)
#cv2.imwrite(os.path.join(self.pred_path, "{}_epoch.{:03d}.diff_silh_{:03d}.png".format(data_type, epoch + 1, i)), diff_silh.astype("uint8"))
silh_comp = np.concatenate(
[gt_silhouette, pred_silhouette, diff_silh],
axis=1)
cv2.imwrite(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.silh_comp_{:03d}.png".
format(data_type, epoch + 1, i)),
silh_comp.astype("uint8"))
# Save the predicted point cloud relative to the GT point cloud
print_mesh(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.gt_pc_{:03d}.obj".format(
data_type, epoch + 1, i)), data[2][i - 1],
self.smpl.faces)
print_point_clouds(
os.path.join(
self.pred_path,
"{}_epoch.{:05d}.comparison_{:03d}.obj".format(
data_type, epoch + 1, i)),
[learned_pc, data[2][i - 1]], [(255, 0, 0),
(0, 255, 0)])
