def plot_vae(model, device, image, reconstruction, encoding, directory, epoch,
plot_count):
model.eval()
with torch.no_grad():
''' Plotting p(z) and q(z/x)'''
scatter = scatter_plot(encoding, directory, epoch, plot_count)
''' Plotting reconstructions '''
fig, ax = plt.subplots(nrows=1,
ncols=2,
figsize=(6, 6),
sharex=True,
sharey=True)
fig.suptitle("Reconstructions using z_image (encoding)", y=1.04)
ax[0].set_title("Input")
ax[0].imshow(image[:6].view(-1, model.input_image_size), cmap='gray')
ax[0].axis("off")
ax[1].set_title("Reconstruction")
if model.decoder_out_channels > model.in_channels:
ax[1].imshow(reconstruction[:6].argmax(dim=1).contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
else:
ax[1].imshow(reconstruction[:6].contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[1].axis("off")
recon = fig2data(fig)
fig.savefig(directory + "/recon-" + str(epoch) + "-" + str(plot_count))
plt.close()
''' Sampling from z'''
fig, ax = plt.subplots(1, figsize=(6, 6))
fig.suptitle("Sampling from Normal(0,1) Z")
random_encoding = torch.randn(encoding[:6].shape).to(device)
output_random_encoding = model.get_reconstruction(random_encoding)
if model.decoder_out_channels > model.in_channels:
ax.imshow(output_random_encoding[:6].argmax(
dim=1).contiguous().view(-1, model.input_image_size).detach(),
cmap='gray')
else:
ax.imshow(output_random_encoding[:6].contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
ax.axis("off")
normal_recon = fig2data(fig)
fig.savefig(directory + "/normal_sampling-" + str(epoch) + "-" +
str(plot_count))
return scatter, recon, normal_recon
def _find_lane_loc_hist(binary_warped, window_h=None, debug=True):
"""
Find initial location of the lane lines on the bottom part of the image.
:param binary_warped: binary or 0-1 mask of the lane lines
:param window_h: height of the processing window
:return: (leftx_base, rightx_base) - locations of the left and right lane lines
"""
if window_h is None:
window_h = binary_warped.shape[0] // 2
histogram = np.sum(binary_warped[-window_h:, :], axis=0)
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0] // 2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
if log.getEffectiveLevel() == logging.DEBUG and debug:
fig, plot = utils.plot_for_img(binary_warped)
plot.imshow(np.flip(utils.mask_to_3ch(binary_warped), axis=0),
origin='lower')
plot.plot(histogram)
cv2.imshow('histogram', utils.fig2data(fig))
cv2.waitKey()
return leftx_base, rightx_base
def visualize_aux_error(self, x: Tensor, **kwargs) -> Tensor:
"""
Given an input image x, returns the reconstructed image
:param x: (Tensor) [B x C x H x W]
:return: (Tensor) [B x C x H x W]
"""
mu, log_var = self.encode(x)
bs = mu.shape[0]
all_spacing = []
figure = plt.figure(figsize=(6, 6))
for i in np.arange(7,25):
spacing = self.aux_network(mu.clone().detach())
num = torch.ones_like(spacing[:,0])*i
# est_loss = spacing[:,2] + 1/torch.exp(num*spacing[:,0] + spacing[:,1])
est_loss = spacing[:,2] + (spacing[:,0]/num)
# print(i, spacing[0])
all_spacing.append(est_loss)
all_spacing = torch.stack(all_spacing, dim=1).detach().cpu().numpy()
y = np.arange(7,25)
for i in range(bs):
plt.plot(y, all_spacing[i,:], label=str(i+1))
plt.legend(loc='upper right')
img = fig2data(figure)
return img
def sampling_error(self, x: Tensor, **kwargs) -> Tensor:
"""
Given an input image x, returns the reconstructed image
:param x: (Tensor) [B x C x H x W]
:return: (Tensor) [B x C x H x W]
"""
error = []
figure = plt.figure(figsize=(6, 6))
bs = x.shape[0]
for i in range(7,25):
self.redo_features(i)
results = self.forward(x)
recons = results[0][:, :3, :, :]
input_batch = results[1]
recon_loss = self.gaussian_pyramid_loss(recons, input_batch)
print(recon_loss)
error.append(recon_loss)
etn = torch.stack(error, dim=1).numpy()
np.savetxt('sample_error.csv', etn, delimiter=',')
y = np.arange(7,25)
for i in range(bs):
plt.plot(y, etn[i,:], label=str(i+1))
plt.legend(loc='upper right')
img = fig2data(figure)
return img
def show_traj(step):
global global_trajectory, global_trajectory_real
max_x = 30.
max_y = 30.
max_speed = 15.0
trajectory = global_trajectory
real_trajectory = global_trajectory_real
fig = plt.figure(figsize=(7, 7))
ax1 = fig.add_subplot(111)
x = trajectory['x']
y = trajectory['y']
real_x = real_trajectory['x']
real_y = real_trajectory['y']
ax1.plot(x, y, label='trajectory', color='b', linewidth=5)
ax1.plot(real_x,
real_y,
label='real-trajectory',
color='b',
linewidth=5,
linestyle='--')
ax1.set_xlabel('Forward/(m)')
ax1.set_ylabel('Sideways/(m)')
ax1.set_xlim([0., max_x])
ax1.set_ylim([-max_y, max_y])
plt.legend(loc='lower right')
real_t = max_x * global_trajectory_real['ts_list']
vx = trajectory['vx']
vy = trajectory['vy']
real_vx = real_trajectory['vx']
real_vy = real_trajectory['vy']
ax2 = ax1.twinx()
ax2.plot(real_t, vx, label='vx', color='tab:cyan', linewidth=2)
ax2.plot(real_t, vy, label='vy', color='tab:purple', linewidth=2)
ax2.plot(real_t,
real_vx,
label='real-vx',
color='r',
linewidth=2,
linestyle='--')
ax2.plot(real_t,
real_vy,
label='real-vy',
color='g',
linewidth=2,
linestyle='--')
ax2.set_ylabel('Velocity/(m/s)')
ax2.set_ylim([-max_speed, max_speed])
plt.legend(loc='lower left')
plt.close('all')
img = fig2data(fig)
cv2.imwrite(
'result/output/%s/' % opt.dataset_name + str(step) + '_curve.png', img)
def show_traj(step):
global global_trajectory, global_trajectory_real
max_x = 30.
max_y = 30.
max_speed = 12.0
trajectory = global_trajectory
real_trajectory = global_trajectory_real
fig = plt.figure(figsize=(7, 7))
ax1 = fig.add_subplot(111)
x = trajectory['x']
y = trajectory['y']
x_var = trajectory['x_var']
y_var = trajectory['y_var']
real_x = real_trajectory['x']
real_y = real_trajectory['y']
ax1.plot(x, y, label='trajectory', color = 'b', linewidth=5)
ax1.fill_betweenx(y, x-x_var, x+x_var, color="crimson", alpha=0.4)
ax1.fill_between(x, y-y_var, y+y_var, color="cyan", alpha=0.4)
ax1.plot(real_x, real_y, label='real-trajectory', color = 'b', linewidth=5, linestyle='--')
ax1.set_xlabel('Forward/(m)')
ax1.set_ylabel('Sideways/(m)')
ax1.set_xlim([0., max_x])
ax1.set_ylim([-max_y, max_y])
plt.legend(loc='lower right')
t = max_x*np.arange(0, 1.0, 1./x.shape[0])
real_t = max_x/opt.max_t*global_trajectory_real['ts_list']
a = trajectory['a']
vx = trajectory['vx']
vy = trajectory['vy']
real_a = real_trajectory['a_list']
real_vx = real_trajectory['vx']
real_vy = real_trajectory['vy']
v = np.sqrt(np.power(vx, 2), np.power(vy, 2))
angle = np.arctan2(vy, vx)/np.pi*max_speed
real_v = np.sqrt(np.power(real_vx, 2), np.power(real_vy, 2))
real_angle = np.arctan2(real_vy, real_vx)/np.pi*max_speed
ax2 = ax1.twinx()
ax2.plot(t, v, label='speed', color = 'r', linewidth=2)
ax2.plot(t, a, label='acc', color = 'y', linewidth=2)
ax2.plot(t, angle, label='angle', color = 'g', linewidth=2)
# real
ax2.plot(real_t, real_v, label='real-speed', color = 'r', linewidth=2, linestyle='--')
ax2.plot(real_t, real_a, label='real-acc', color = 'y', linewidth=2, linestyle='--')
ax2.plot(real_t, real_angle, label='real-angle', color = 'g', linewidth=2, linestyle='--')
ax2.set_ylabel('Velocity/(m/s)')
ax2.set_ylim([-max_speed, max_speed])
plt.legend(loc='lower left')
plt.close('all')
img = fig2data(fig)
cv2.imwrite('result/output/%s/' % opt.dataset_name+str(step)+'_curve.png', img)
def plot_pixelcnn(model, device, image, reconstruction, directory, epoch,
plot_count, data_mean, data_std):
''' Plotting reconstructions '''
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(6, 6))
fig.suptitle("Reconstructions using z_image (encoding)", y=1.04)
ax[0].set_title("Input")
ax[0].imshow(image[:6].view(-1, model.input_image_size), cmap='gray')
ax[0].axis("off")
ax[1].set_title("Reconstruction")
ax[1].imshow(reconstruction[:6].argmax(dim=1).contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[1].axis("off")
recon = fig2data(fig)
fig.savefig(directory + "/recon-" + str(epoch) + "-" + str(plot_count))
''' Sampling from z'''
sample = torch.zeros(image[:6].shape).to(device) - data_mean / data_std
argmax_from_sampling, sample_from_sampling = generate_only_pixelcnn(
sample, model, data_mean, data_std)
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(6, 6))
fig.suptitle("Sampling from Normal Z", y=1.04)
ax[0].set_title("Sample (max)")
ax[0].imshow(argmax_from_sampling.argmax(dim=1).contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[0].axis("off")
ax[1].set_title("Sample")
ax[1].imshow(sample_from_sampling.view(-1,
model.input_image_size).detach(),
cmap='gray')
ax[1].axis("off")
normal_recon = fig2data(fig)
fig.savefig(directory + "/sample-" + str(epoch) + "-" + str(plot_count))
return None, recon, normal_recon
def scatter_plot(encoding, directory, epoch, plot_count):
X = data(encoding[:, 0, 0, 0])
Y = data(encoding[:, 1, 0, 0])
randn = np.random.randn(X.shape[0], 2)
x_lim = np.absolute([X.min(), X.max()]).max() + 4
y_lim = np.absolute([Y.min(), Y.max()]).max() + 4
fig, ax = plt.subplots(nrows=1,
ncols=2,
figsize=(6, 6),
sharex=True,
sharey=True)
fig.suptitle("Plotting p(z) and q(z/x)")
ax[0].set_xlim(-x_lim, x_lim)
ax[0].set_ylim(-y_lim, y_lim)
ax[0].set_title("p(z)")
ax[0].scatter(randn[:, 0], randn[:, 1], alpha=0.1)
ax[1].set_title("q(z/x)")
ax[1].scatter(X, Y, alpha=0.1)
fig.savefig(directory + "/scatter-" + str(epoch) + "-" + str(plot_count))
return fig2data(fig)
def plot_fitted_lane(self):
"""
Create plot of the left and right lines on top of the binary mask.
:return: bgr image
"""
left_fit, right_fit = self.lane_fit
if self.lane_mask is not None:
out_img = utils.mask_to_3ch(self.lane_mask)
else:
out_img = np.zeros((self.img_shape[1], self.img_shape[0], 3),
dtype=np.uint8)
# Generate x and y values for plotting
ploty = np.linspace(0, self.img_shape[1] - 1, self.img_shape[1])
try:
left_fitx = utils.polyval(left_fit, ploty)
right_fitx = utils.polyval(right_fit, ploty)
except TypeError:
# Avoids an error if `left` and `right_fit` are still none or incorrect
log.warning('The function failed to fit a line!')
left_fitx = 1 * ploty**2 + 1 * ploty
right_fitx = 1 * ploty**2 + 1 * ploty
## Visualization ##
if self.line_seg is not None:
leftx, lefty, rightx, righty = self.line_seg
# Colors in the left and right lane regions
out_img[lefty, leftx] = [255, 0, 0]
out_img[righty, rightx] = [0, 0, 255]
# Plots the left and right polynomials on the lane lines
fig, plot = utils.plot_for_img(out_img)
plot.plot(left_fitx, ploty, color='yellow')
plot.plot(right_fitx, ploty, color='yellow')
plot.imshow(out_img)
plt_img = utils.fig2data(fig)
return plt_img
def show_traj(save=False):
global global_trajectory
max_x = 30.
max_y = 30.
max_speed = 12.0
while True:
trajectory = global_trajectory
fig = plt.figure()
ax1 = fig.add_subplot(111)
x = trajectory['x']
y = trajectory['y']
ax1.plot(x, y, label='trajectory', color = 'b', linewidth=3)
ax1.set_xlabel('Tangential/(m)')
ax1.set_ylabel('Normal/(m)')
ax1.set_xlim([0., max_x])
ax1.set_ylim([-max_y, max_y])
plt.legend(loc='lower right')
t = max_x*np.arange(0, 1.0, 1./x.shape[0])
a = trajectory['a']
vx = trajectory['vx']
vy = trajectory['vy']
v = np.sqrt(np.power(vx, 2), np.power(vy, 2))
angle = np.arctan2(vy, vx)/np.pi*max_speed
ax2 = ax1.twinx()
ax2.plot(t, v, label='speed', color = 'r', linewidth=2)
ax2.plot(t, a, label='acc', color = 'y', linewidth=2)
ax2.plot(t, angle, label='angle', color = 'g', linewidth=2)
ax2.set_ylabel('Velocity/(m/s)')
ax2.set_ylim([-max_speed, max_speed])
plt.legend(loc='upper right')
if not save:
plt.show()
else:
image = fig2data(fig)
plt.close('all')
return image
def map(self,
plot_filename=None,
beta=0.9,
max_iter=1000,
lr=0.5,
lr_decay=200,
early_stop=100):
""" map y into the area
"""
imgs = []
dh = 0
idx = None
running_median, previous_median = [], 0
for i in range(max_iter):
# find nearest p for each e
idx = np.argmin(self.dist, axis=0)
# calculate total mass of each cell
mass_p = np.bincount(idx, minlength=self.K) / self.N0
# gradient descent with momentum and decay
dh = beta * dh + (1 - beta) * (mass_p - self.weight_p)
if i != 0 and i % lr_decay == 0:
lr *= 0.9
self.dist += lr * dh[:, None]
# plot to gif, TODO this is time consuming, got a better way?
if plot_filename and i % 10 == 0:
fig = utils.plot_map(self.x, idx / (self.K - 1))
img = utils.fig2data(fig)
imgs.append(img)
# check if converge
max_change = np.max((mass_p - self.weight_p) / self.weight_p)
if max_change.size > 1:
max_change = max_change[0]
max_change *= 100
if self.verbose and ((i < 100 and i % 10 == 0) or i % 100 == 0):
print("{0:d}: mass diff {1:.2f}%".format(i, max_change))
if max_change < 1:
if self.verbose:
print("{0:d}: mass diff {1:.2f}%".format(i, max_change))
break
if early_stop > 0:
# early stop if loss does not decrease TODO better way to early stop?
running_median.append(max_change)
if len(running_median) >= early_stop:
if previous_median != 0 and \
np.abs(np.median(np.asarray(running_median)) - previous_median) / previous_median < 0.02:
if self.verbose:
print("loss saturated, early stopped")
break
else:
previous_median = np.median(np.asarray(running_median))
running_median = []
if max_change <= 1:
break
if plot_filename and imgs:
imageio.mimsave(plot_filename, imgs, fps=4)
# labels come from y
pred_label_x = self.label_y[idx] if self.label_y is not None else None
# update coordinates of y
bincount = np.bincount(idx, minlength=self.K)
if 0 in bincount:
print(
'Empty cluster found, optimal transport probably did not converge\nTry larger lr or max_iter'
)
# return
for i in range(self.y.shape[1]):
# update p to the centroid of their correspondences
self.y[:, i] = np.bincount(
idx, weights=self.x[:, i], minlength=self.K) / bincount
return idx, pred_label_x
# Read next frame
st, im = cap.read()
if im is None:
break
if transpose:
im = imutils.rotate_bound(im, 90)
# Get class and distance to closest neighbour
class1, dist1, class2, dist2 = data[frame, 1:]
class1 = utils.choose_class(int(class1))
class2 = utils.choose_class(int(class2))
# Display new frame and forward plots
ax.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
ax1.set_xlim(frame, frame + 60)
ax1.set_title('Eucl. | ' + class1)
ax2.set_xlim(frame, frame + 60)
ax2.set_title('Mahal. | ' + class2)
# Get figure as image and save it to video
img = utils.fig2data(fig)
out.write(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
frame += 1
if frame % 36 == 0:
fig.clf()
ax, ax1, ax2 = build_figure()
cap.release()
out.release()
def plot_pixelvae(model, device, image, reconstruction, encoding, directory,
epoch, plot_count, data_mean, data_std):
model.eval()
z_image = model.get_z_image(encoding[:6])
sample = torch.zeros(image[:6].shape).to(device)
argmax_from_no_teacher_forcing, sample_from_no_teacher_forcing = generate(
z_image, sample, model, data_mean, data_std)
random_encoding = torch.randn(encoding[:6].shape).to(device)
random_z_image = model.get_z_image(random_encoding)
argmax_z_from_no_teacher_forcing, sample_z_from_no_teacher_forcing = generate(
random_z_image, sample, model, data_mean, data_std)
z_encoding_image_concat = torch.cat([random_z_image, image[:6]], dim=1)
''' Plotting p(z) and q(z/x) '''
scatter = scatter_plot(encoding, directory, epoch, plot_count)
''' Plotting reconstruction from z_image '''
fig, ax = plt.subplots(nrows=1,
ncols=5,
figsize=(8, 8),
sharex=True,
sharey=True)
fig.suptitle("Reconstructions using z_image (encoding)", y=1.04)
ax[0].set_title("Input")
ax[0].imshow(image[:6].view(-1, model.input_image_size), cmap='gray')
ax[0].axis("off")
ax[1].set_title("real z_image")
ax[1].imshow(z_image.contiguous().view(-1,
model.input_image_size).detach(),
cmap='gray')
ax[1].axis("off")
ax[2].set_title("Recon w tf (max)")
ax[2].imshow(reconstruction[:6].argmax(dim=1).contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[2].axis("off")
ax[3].set_title("Recon w/o tf (max)")
ax[3].imshow(argmax_from_no_teacher_forcing.argmax(dim=1).view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[3].axis("off")
ax[4].set_title("Recon w/o tf (sample)")
ax[4].imshow(sample_from_no_teacher_forcing.view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[4].axis("off")
recon = fig2data(fig)
fig.savefig(directory + "/recon_z_-" + str(epoch) + "-" + str(plot_count))
plt.close()
''' Plotting reconstructions from normal distribution'''
fig, ax = plt.subplots(nrows=1,
ncols=5,
figsize=(8, 8),
sharex=True,
sharey=True)
fig.suptitle("Reconstructions sampling from normal (encoding)", y=1.04)
ax[0].set_title("Input")
ax[0].imshow(image[:6].view(-1, model.input_image_size), cmap='gray')
ax[0].axis("off")
ax[1].set_title("rand z_image")
ax[1].imshow(random_z_image.contiguous().view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[1].axis("off")
ax[2].set_title("Recon w tf")
ax[2].imshow(model.run_pixelcnn(z_encoding_image_concat).argmax(
dim=1).contiguous().view(-1, model.input_image_size).detach(),
cmap='gray')
ax[2].axis("off")
ax[3].set_title("Recon w/o tf (max)")
ax[3].imshow(argmax_z_from_no_teacher_forcing.argmax(dim=1).view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[3].axis("off")
ax[4].set_title("Recon w/o tf (sample)")
ax[4].imshow(sample_z_from_no_teacher_forcing.view(
-1, model.input_image_size).detach(),
cmap='gray')
ax[4].axis("off")
normal_recon = fig2data(fig)
fig.savefig(directory + "/normal-recon-" + str(epoch) + "-" +
str(plot_count))
return scatter, recon, normal_recon