def plot_occs(static_map, h, ax1, ax2, ax3, traj_gt, traj1, traj2, occs_gt, occs1, occs2): num_points = static_map.shape[0] seq_length = 12 for ii, ped in enumerate(traj_gt): cols1, index1 = on_occupied(traj1, ii, static_map, num_points, seq_length, minimum_distance=.1) cols2, index2 = on_occupied(traj2, ii, static_map, num_points, seq_length, minimum_distance=.1) pixels1 = get_pixels_from_world(traj1[ii], h) pixels2 = get_pixels_from_world(traj2[ii], h) if cols1 > 0: plot_occ_pix(ax2, pixels1[index1]) occs1 += 1 if cols2 > 0: plot_occ_pix(ax3, pixels2[index2]) occs2 += 1 return occs_gt, occs1, occs2
def plot_col_pix(ax, traj, index_agend_1, index_agend_2, index_time, h): pixels_gt1 = get_pixels_from_world(traj[index_agend_1], h) pixels_gt2 = get_pixels_from_world(traj[index_agend_2], h) ax.scatter(pixels_gt1[index_time][0], pixels_gt1[index_time][1], marker='*', color='red', s=100) ax.scatter(pixels_gt2[index_time][0], pixels_gt2[index_time][1], marker='*', color='green', s=100)
def plot_pixel(ax, trajectory, person, h, a=1, last=False, first=False, intermediate=True, size=10, colors=None, linestyle='-', label=False): if colors is None: colors = np.random.rand(trajectory.size(0), 3) pixels_obs = get_pixels_from_world(trajectory[person], h) if intermediate: ax.plot(pixels_obs[:, 0], pixels_obs[:, 1], marker='.', color=colors[person, :], markersize=1, alpha=a, linestyle=linestyle) ax.quiver(pixels_obs[-1, 0], pixels_obs[-1, 1], pixels_obs[-1, 0] - pixels_obs[-2, 0], pixels_obs[-2, 1] - pixels_obs[-1, 1], color=colors[person, :]) if last: ax.scatter(pixels_obs[-1, 0], pixels_obs[-1, 1], marker='*', color=colors[person, :], s=20) ax.text(pixels_obs[0, 0] + 10, pixels_obs[0, 1] - 10, color=colors[person, :], s=str(person), fontsize=15) if first: ax.scatter(pixels_obs[0, 0], pixels_obs[0, 1], marker='p', color=colors[person, :], s=size)
def visualize_attention_weights(scene_name, encoded_image_size, attention_weights, curr_end_pos, ax1, ax2, counter=0): """ Function to visualize the attention weights on their relative scene image during inference time (training or testing). :param scene_name: the name of the SDD scene from which the attention weights were computed :param encoded_image_size: the width/height dimension of the scene image used as input for the Attention Encoder. The image should be a squared image, thus the dimension should be (encoded_image_size, encoded_image_size) :param attention_weights: the weights computed by the attention module :param curr_end_pos: the current positions of all agents in a scene """ ped_id = 0 grid_size = 8.0 # 'upscaling_factor' is used to increment the size of the scene image (to make it better visualizable) as well as # the agents' positions coordinates do adapt them to the new image size upscaling_factor = 1 ax1.cla() ax2.cla() # 'dataset_path' represents the path with the SDD scene folders inside dataset_path = get_root_dir() + "/datasets/safegan_dataset/SDD/" # Load the raw scene image on which the attention weights will be plotted. # Here I suppose they are inside a folder called "segmented_scenes" image_original = Image.open(get_root_dir() + "/datasets/safegan_dataset/SDD/" + scene_name + "/reference.jpg") original_image_size = Image.open(dataset_path + scene_name + "/annotated_boundaries.jpg").size # Increase the dimension of the raw scene image image_original = image_original.resize([ original_image_size[0] * upscaling_factor, original_image_size[1] * upscaling_factor ], Image.LANCZOS) # In order to plot the agents's coordinates on the scene image it is necessary to load the homography matrix of that scene # and then to convert the world coordinates into pixel values h_matrix = pd.read_csv(dataset_path + scene_name + '/{}_homography.txt'.format(scene_name), delim_whitespace=True, header=None).values pixels = get_pixels_from_world(curr_end_pos, h_matrix, True) #pixels = pixels * (encoded_image_size * upscaling_factor / original_image_size[0], # encoded_image_size * upscaling_factor / original_image_size[1]) # Here it is necessary to resize also the pixel coordinates of the agents' positions, according to the upscaling # factor and the original dimension of the scene image (that I take from the image with the annotated boundary points) original_image_size = Image.open(dataset_path + scene_name + "/annotated_boundaries.jpg").size # w, h = image_original.size col = np.round(pixels[ped_id][0]) row = np.round(pixels[ped_id][1]) grid_left_upper_corner = curr_end_pos - torch.tensor( [grid_size / 2.0, grid_size / 2.0]).expand_as(curr_end_pos).to(device) pixels_grid = get_pixels_from_world(grid_left_upper_corner, h_matrix, True) col_grid = (col - np.round(pixels_grid[ped_id][0])) row_grid = (row - np.round(pixels_grid[ped_id][1])) if row - row_grid > 0 and row + row_grid < h and col - col_grid > 0 and col + col_grid < w: image = image_original.crop( (col - col_grid, row - row_grid, col + col_grid, row + row_grid)) else: image = image_original.crop((w // 2 - col_grid, h // 2 - row_grid, w // 2 + col_grid, h // 2 + row_grid)) #image = image.resize([20 * upscaling_factor, 20 * upscaling_factor], Image.LANCZOS) # Resize the attention weights dimension to match it with the dimension of the upscaled raw scene image. # To expand the attention weights I use skimage that allows us to also smooth the pixel values during the expansion attention_weights = attention_weights.view( -1, encoded_image_size, encoded_image_size).detach().cpu().numpy() upscaling_factor = image.size[0] / encoded_image_size alpha = skimage.transform.pyramid_expand(attention_weights[ped_id], upscale=upscaling_factor, sigma=8) #pixels = np.expand_dims( np.array([w//2, h//2]), axis=0) rect = patches.Rectangle((pixels_grid[ped_id, 0], pixels_grid[ped_id, 1]), 2 * col_grid, 2 * row_grid, linewidth=1, edgecolor='white', facecolor='none') # Plot raw scene image, the agents' positions and the attention weights ax1.imshow(image_original) ax1.scatter(pixels[:, 0], pixels[:, 1], marker='.', color="b") ax1.scatter(pixels[ped_id, 0], pixels[ped_id, 1], marker='X', color="r") ax1.add_patch(rect) ax1.axis('off') ax2.imshow(image) ax2.imshow(np.flipud(alpha), alpha=0.7) plt.set_cmap(cm.Greys_r) ax2.axis('off') directory = get_root_dir() + '/results/plots/SDD/safeGAN_DP/attention' files = len(os.listdir(directory)) plt.savefig(directory + '/frame_{}.png'.format(files + 1)) plt.draw() plt.waitforbuttonpress()
def get_pixels(o, g, p, annotated_points, h1): op = get_pixels_from_world(o, h1) pg = get_pixels_from_world(g, h1) pp = get_pixels_from_world(p, h1) ap = get_pixels_from_world(annotated_points, h1) return op, pg, pp, ap
def evaluate_test_pixel_fde(data_set, model_name, selected_scene=None, selected_batch=-1): pred_traj_gt = load_pickle('pred_traj_gt', selected_scene, selected_batch, data_set, model_name) seq_start_end = load_pickle('seq_start_end', selected_scene, selected_batch, data_set, model_name) scene_name_list = load_pickle('scene_name_list', selected_scene, selected_batch, data_set, model_name) pred_traj_fake1_list = load_pickle('pred_traj_fake1_list', selected_scene, selected_batch, data_set, model_name) pred_traj_fake2_list = load_pickle('pred_traj_fake2_list', selected_scene, selected_batch, data_set, model_name) homography_list = load_pickle('homography_list', selected_scene, selected_batch, data_set, model_name) num_samples = len(pred_traj_fake1_list) ade1 = [] ade2 = [] scene_name = np.unique(scene_name_list) print(scene_name) for s in range(num_samples): # seq_len, batch, 2 traj1_pixels, traj2_pixels, traj_gt_pixels = [], [], [] total_traj = pred_traj_gt.size(1) for i, (start, end) in enumerate(seq_start_end): # get homography of current scene h = homography_list[i] num_ped = end - start # calculate pixels for a sample in a scene pixels_t1 = get_pixels_from_world( pred_traj_fake1_list[s][:, start:end].contiguous().view(-1, 2), h) pixels_t2 = get_pixels_from_world( pred_traj_fake2_list[s][:, start:end].contiguous().view(-1, 2), h) pixels_gt = get_pixels_from_world( pred_traj_gt[:, start:end].contiguous().view(-1, 2), h) t1 = torch.from_numpy(pixels_t1).view(-1, num_ped, 2) # time, peds in scene, 2 t2 = torch.from_numpy(pixels_t2).view(-1, num_ped, 2) tg = torch.from_numpy(pixels_gt).view(-1, num_ped, 2) traj1_pixels.append(t1) traj2_pixels.append(t2) traj_gt_pixels.append(tg) # concatignate along batch dimension traj1_pixels = torch.cat(traj1_pixels, dim=1) traj2_pixels = torch.cat(traj2_pixels, dim=1) traj_gt_pixels = torch.cat(traj_gt_pixels, dim=1) # (seq_len, batch, 2) for each sample we calculate displacement error ade1.append( final_displacement_error(traj1_pixels.view(-1, total_traj, 2)[-1], traj_gt_pixels.view(-1, total_traj, 2)[-1], mode='raw')) ade2.append( final_displacement_error(traj2_pixels.view(-1, total_traj, 2)[-1], traj_gt_pixels.view(-1, total_traj, 2)[-1], mode='raw')) pred_len = pred_traj_gt.size(0) ade1 = evaluate_helper(ade1, seq_start_end) / (total_traj) ade2 = evaluate_helper(ade2, seq_start_end) / (total_traj) return ade1, ade2
def evaluate_trajectory_quality(data_set, scene, model_name, batch=5, selection=-1): obs_traj = load_pickle('obs_traj', scene, batch, data_set, model_name) pred_traj_gt = load_pickle('pred_traj_gt', scene, batch, data_set, model_name) seq_start_end = load_pickle('seq_start_end', scene, batch, data_set, model_name) pred_traj_fake1_list = load_pickle('pred_traj_fake1_list', scene, batch, data_set, model_name) pred_traj_fake2_list = load_pickle('pred_traj_fake2_list', scene, batch, data_set, model_name) homography_list = load_pickle('homography_list', scene, batch, data_set, model_name) photo_list = load_pickle('photo_list', scene, batch, data_set, model_name) annotated_points_list = load_pickle('annotated_points_list', scene, batch, data_set, model_name) scene_name_list = load_pickle('scene_name_list', scene, batch, data_set, model_name) fig, ((ax1, ax14), (ax15, ax4)) = plt.subplots(2, 2, figsize=(32, 32), num=1) num_samples = len(pred_traj_fake1_list) for i, (start, end) in enumerate(seq_start_end): print(batch * len(seq_start_end) + i) if not (selection == -1 or batch * len(seq_start_end) + i == selection): continue start = start.item() end = end.item() num_ped = end - start photo = photo_list[i] h = homography_list[i] annotated_points = annotated_points_list[i] annotated_points = get_pixels_from_world(annotated_points, h) subsample = annotated_points.shape[0] // 500 plt.cla() traj_obs = obs_traj.permute(1, 0, 2)[start:end] traj_gt = pred_traj_gt.permute(1, 0, 2)[start:end] scene_name = np.unique(scene_name_list) print(scene_name) #if not (scene_name == scene).all(): # return 0, 0 plot_photo(ax1, photo, 'model1') plot_photo(ax4, photo, 'model2') for p in range(np.minimum(num_ped, 5)): plot_pixel(ax1, traj_obs, p, h, a=1, last=False, first=False, intermediate=True, size=10, colors=colors) plot_pixel(ax1, traj_gt, p, h, a=.1, last=True, first=False, intermediate=False, size=10, colors=colors) plot_pixel(ax4, traj_gt, p, h, a=1, last=True, first=False, intermediate=True, size=10, colors=colors) plot_photo(ax14, photo, 'model1') plot_photo(ax15, photo, 'model2') for sample in range(1): traj_pred1 = pred_traj_fake1_list[sample + 0].permute(1, 0, 2)[start:end] traj_pred2 = pred_traj_fake1_list[sample + 1].permute(1, 0, 2)[start:end] traj_pred3 = pred_traj_fake1_list[sample + 2].permute(1, 0, 2)[start:end] traj_pred11 = pred_traj_fake2_list[sample + 0].permute( 1, 0, 2)[start:end] traj_pred22 = pred_traj_fake2_list[sample + 1].permute( 1, 0, 2)[start:end] traj_pred33 = pred_traj_fake2_list[sample + 2].permute( 1, 0, 2)[start:end] for p in range(np.minimum(num_ped, 3)): if True: #p == 0 or p==1 or p==2: plot_pixel(ax14, traj_pred1, p, h, a=1, last=False, first=False, size=10, colors=colors) plot_pixel(ax14, traj_pred2, p, h, a=1, last=False, first=False, size=10, colors=colors) plot_pixel(ax14, traj_pred3, p, h, a=1, last=False, first=False, size=10, colors=colors) plot_pixel(ax15, traj_pred11, p, h, a=1, last=False, first=False, size=10, colors=colors, linestyle='-') plot_pixel(ax15, traj_pred22, p, h, a=1, last=False, first=False, size=10, colors=colors, linestyle='-') plot_pixel(ax15, traj_pred33, p, h, a=1, last=False, first=False, size=10, colors=colors, linestyle='-') #ax3.scatter(annotated_points[::subsample, 0], annotated_points[::subsample, 1], marker='.', color='white', s=1) #_, _, _ = plot_cols(ax2, ax3, ax4, traj_gt, traj_pred1, traj_pred2, cols_gt, cols1, cols2, h) #_, _, _ = plot_occs(annotated_points, h, ax2, ax3, ax4, traj_gt, traj1, traj2, cols_gt, cols1, cols2) plt.waitforbuttonpress() plt.draw() #plt.pause(.001) directory = '/results/plots/{}/'.format(data_set) frame = batch * len(seq_start_end) + i save_sample(fig, ax1, directory, model_name, scene, model_variant='OBS', frame=frame, sample='-') save_sample(fig, ax4, directory, model_name, scene, model_variant='GT', frame=frame, sample='-') save_sample(fig, ax14, directory, model_name, scene, model_variant='DP', frame=frame, sample='123') save_sample(fig, ax15, directory, model_name, scene, model_variant='DP_SP', frame=frame, sample='123') return 0, 0