def main(): try: os.makedirs(args.figure_directory) except: pass #============================================================================== # Utilities #============================================================================== def read_files(directory): filenames = [] files = os.listdir(directory) # ipdb.set_trace() for filename in files: if filename.endswith(".h5"): filenames.append(filename) filenames.sort() dataset_images = [] dataset_viewpoints = [] for i in range(len(filenames)): F = h5py.File(os.path.join(directory, filenames[i])) tmp_images = list(F["images"]) tmp_viewpoints = list(F["viewpoints"]) dataset_images.extend(tmp_images) dataset_viewpoints.extend(tmp_viewpoints) # for i in range(len(filenames)): # images_npy_path = os.path.join(directory, "images", filenames[i]) # viewpoints_npy_path = os.path.join(directory, "viewpoints", filenames[i]) # tmp_images = np.load(images_npy_path) # tmp_viewpoints = np.load(viewpoints_npy_path) # assert tmp_images.shape[0] == tmp_viewpoints.shape[0] # dataset_images.extend(tmp_images) # dataset_viewpoints.extend(tmp_viewpoints) dataset_images = np.array(dataset_images) dataset_viewpoints = np.array(dataset_viewpoints) dataset = list() for i in range(len(dataset_images)): item = { 'image': dataset_images[i], 'viewpoint': dataset_viewpoints[i] } dataset.append(item) return dataset def to_device(array): # if using_gpu: array = cuda.to_gpu(array) return array def fill_observations_axis(observation_images): axis_observations_image = np.full( (3, image_shape[1], total_observations_per_scene * image_shape[2]), black_color, dtype=np.float32) num_current_obs = len(observation_images) total_obs = total_observations_per_scene width = image_shape[2] x_start = width * (total_obs - num_current_obs) // 2 for obs_image in observation_images: x_end = x_start + width axis_observations_image[:, :, x_start:x_end] = obs_image x_start += width return axis_observations_image def compute_camera_angle_at_frame(t): return t * 2 * math.pi / (fps * 2) def rotate_query_viewpoint(horizontal_angle_rad, camera_distance, camera_position_y): camera_position = np.array([ camera_distance * math.sin(horizontal_angle_rad), # x camera_position_y, camera_distance * math.cos(horizontal_angle_rad), # z ]) center = np.array((0, camera_position_y, 0)) camera_direction = camera_position - center yaw, pitch = compute_yaw_and_pitch(camera_direction) query_viewpoints = xp.array( ( camera_position[0], camera_position[1], camera_position[2], math.cos(yaw), math.sin(yaw), math.cos(pitch), math.sin(pitch), ), dtype=np.float32, ) query_viewpoints = xp.broadcast_to(query_viewpoints, (1, ) + query_viewpoints.shape) return query_viewpoints def render(representation, camera_distance, camera_position_y, total_frames, animation_frame_array, rotate_camera=True): # viewpoint_file = open('viewpoints.txt','w') for t in range(0, total_frames): artist_array = [ axis_observations.imshow(cv2.cvtColor( make_uint8(axis_observations_image), cv2.COLOR_BGR2RGB), interpolation="none", animated=True) ] horizontal_angle_rad = compute_camera_angle_at_frame(t) if rotate_camera == False: horizontal_angle_rad = compute_camera_angle_at_frame(0) query_viewpoints = rotate_query_viewpoint(horizontal_angle_rad, camera_distance, camera_position_y) generated_images = model.generate_image(query_viewpoints, representation)[0] generated_images = chainer.backends.cuda.to_cpu(generated_images) generated_images = make_uint8(generated_images) generated_images = cv2.cvtColor(generated_images, cv2.COLOR_BGR2RGB) artist_array.append( axis_generation.imshow(generated_images, interpolation="none", animated=True)) animation_frame_array.append(artist_array) def render_wVar(representation, camera_distance, camera_position_y, total_frames, animation_frame_array, no_of_samples, rotate_camera=True, wVariance=True): # highest_var = 0.0 # with open("queries.txt",'w') as file_wviews, open("variance.txt",'w') as file_wvar: for t in range(0, total_frames): artist_array = [ axis_observations.imshow(cv2.cvtColor( make_uint8(axis_observations_image), cv2.COLOR_BGR2RGB), interpolation="none", animated=True) ] horizontal_angle_rad = compute_camera_angle_at_frame(t) if rotate_camera == False: horizontal_angle_rad = compute_camera_angle_at_frame(0) query_viewpoints = rotate_query_viewpoint(horizontal_angle_rad, camera_distance, camera_position_y) # q_x, q_y, q_z, _, _, _, _ = query_viewpoints[0] # file_wviews.writelines("".join(str(q_x))+", "+ # "".join(str(q_y))+", "+ # "".join(str(q_z))+"\n") generated_images = cp.squeeze( cp.array( model.generate_images(query_viewpoints, representation, no_of_samples))) # ipdb.set_trace() var_image = cp.var(generated_images, axis=0) mean_image = cp.mean(generated_images, axis=0) mean_image = make_uint8( np.squeeze(chainer.backends.cuda.to_cpu(mean_image))) mean_image_rgb = cv2.cvtColor(mean_image, cv2.COLOR_BGR2RGB) var_image = chainer.backends.cuda.to_cpu(var_image) # grayscale r, g, b = var_image gray_var_image = 0.2989 * r + 0.5870 * g + 0.1140 * b # thresholding Otsu's method # thresh = threshold_otsu(gray_var_image) # var_binary = gray_var_image > thresh ## hill climb algorthm for searching highest variance # cur_var = np.mean(gray_var_image) # if cur_var>highest_var: # highest_var = cur_var # if wVariance==True: # print('highest variance: '+str(highest_var)+', viewpoint: '+str(query_viewpoints[0])) # highest_var_vp = query_viewpoints[0] # file_wvar.writelines('highest variance: '+str(highest_var)+', viewpoint: '+str(highest_var_vp)+'\n') # else: # pass artist_array.append( axis_generation_var.imshow(gray_var_image, cmap=plt.cm.gray, interpolation="none", animated=True)) artist_array.append( axis_generation_mean.imshow(mean_image_rgb, interpolation="none", animated=True)) animation_frame_array.append(artist_array) # if wVariance==True: # print('final highest variance: '+str(highest_var)+', viewpoint: '+str(highest_var_vp)) # file_wvar.writelines('final highest variance: '+str(highest_var)+', viewpoint: '+str(highest_var_vp)+'\n') # else: # pass # file_wviews.close() # file_wvar.close() # loading dataset & model cuda.get_device(args.gpu_device).use() xp = cp hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) chainer.serializers.load_hdf5(args.snapshot_file, model) model.to_gpu() total_observations_per_scene = 4 fps = 30 black_color = -0.5 image_shape = (3, ) + hyperparams.image_size axis_observations_image = np.zeros( (3, image_shape[1], total_observations_per_scene * image_shape[2]), dtype=np.float32) #============================================================================== # Visualization #============================================================================== plt.style.use("dark_background") fig = plt.figure(figsize=(6, 7)) plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95) # fig.suptitle("GQN") axis_observations = fig.add_subplot(2, 1, 1) axis_observations.axis("off") axis_observations.set_title("observations") axis_generation = fig.add_subplot(2, 1, 2) axis_generation.axis("off") axis_generation.set_title("Rendered Predictions") axis_generation_var = fig.add_subplot(2, 2, 3) axis_generation_var.axis("off") axis_generation_var.set_title("Variance Render") axis_generation_mean = fig.add_subplot(2, 2, 4) axis_generation_mean.axis("off") axis_generation_mean.set_title("Mean Render") # iterator dataset = read_files(args.dataset_directory) file_number = 1 with chainer.no_backprop_mode(): iterator = chainer.iterators.SerialIterator(dataset, batch_size=1) # ipdb.set_trace() for i in tqdm(range(len(iterator.dataset))): animation_frame_array = [] images, viewpoints = np.array([ iterator.dataset[i]["image"] ]), np.array([iterator.dataset[i]["viewpoint"]]) camera_distance = np.mean( np.linalg.norm(viewpoints[:, :, :3], axis=2)) camera_position_y = np.mean(viewpoints[:, :, 1]) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) images = preprocess_images(images) batch_index = 0 total_views = images.shape[1] random_observation_view_indices = list(range(total_views)) random.shuffle(random_observation_view_indices) random_observation_view_indices = random_observation_view_indices[: total_observations_per_scene] observed_images = images[batch_index, random_observation_view_indices] observed_viewpoints = viewpoints[batch_index, random_observation_view_indices] observed_images = to_device(observed_images) observed_viewpoints = to_device(observed_viewpoints) # Scene encoder representation = model.compute_observation_representation( observed_images[None, :1], observed_viewpoints[None, :1]) # Update figure observation_index = random_observation_view_indices[0] observed_image = images[batch_index, observation_index] axis_observations_image = fill_observations_axis([observed_image]) # Neural rendering # render(representation, camera_distance, camera_position_y, # fps * 2, animation_frame_array) render_wVar(representation, camera_distance, camera_position_y, fps * 2, animation_frame_array, 100) for n in range(total_observations_per_scene): observation_indices = random_observation_view_indices[:n + 1] axis_observations_image = fill_observations_axis( images[batch_index, observation_indices]) # Scene encoder representation = model.compute_observation_representation( observed_images[None, :n + 1], observed_viewpoints[None, :n + 1]) # Neural rendering # render(representation, camera_distance, camera_position_y, # fps // 2, animation_frame_array,rotate_camera=False) render_wVar(representation, camera_distance, camera_position_y, fps // 2, animation_frame_array, 100, rotate_camera=False, wVariance=False) # Scene encoder with all given observations representation = model.compute_observation_representation( observed_images[None, :total_observations_per_scene + 1], observed_viewpoints[None, :total_observations_per_scene + 1]) # Neural rendering # render(representation, camera_distance, camera_position_y, # fps * 6, animation_frame_array) render_wVar(representation, camera_distance, camera_position_y, fps * 6, animation_frame_array, 100) anim = animation.ArtistAnimation( fig, animation_frame_array, interval=1 / fps, # originally 1/fps blit=True, repeat_delay=0) anim.save("{}/observations_{}.gif".format(args.figure_directory, file_number), writer="imagemagick", fps=10) # ipdb.set_trace() # anim.save( # "{}/rooms_ring_camera_observations_{}.mp4".format( # args.figure_directory, file_number), # writer='ffmpeg', # fps=10) file_number += 1
def main(): try: os.makedirs(args.figure_directory) except: pass xp = np using_gpu = args.gpu_device >= 0 if using_gpu: cuda.get_device(args.gpu_device).use() xp = cp dataset = gqn.data.Dataset(args.dataset_directory) meter = Meter() assert meter.load(args.snapshot_directory) hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) assert model.load(args.snapshot_directory, meter.epoch) if using_gpu: model.to_gpu() total_observations_per_scene = 4 fps = 30 black_color = -0.5 image_shape = (3, ) + hyperparams.image_size axis_observations_image = np.zeros( (3, image_shape[1], total_observations_per_scene * image_shape[2]), dtype=np.float32) #============================================================================== # Utilities #============================================================================== def to_device(array): if using_gpu: array = cuda.to_gpu(array) return array def fill_observations_axis(observation_images): axis_observations_image = np.full( (3, image_shape[1], total_observations_per_scene * image_shape[2]), black_color, dtype=np.float32) num_current_obs = len(observation_images) total_obs = total_observations_per_scene width = image_shape[2] x_start = width * (total_obs - num_current_obs) // 2 for obs_image in observation_images: x_end = x_start + width axis_observations_image[:, :, x_start:x_end] = obs_image x_start += width return axis_observations_image def compute_camera_angle_at_frame(t): horizontal_angle_rad = 2 * t * math.pi / (fps * 2) + math.pi / 4 y_rad_top = math.pi / 3 y_rad_bottom = -math.pi / 3 y_rad_range = y_rad_bottom - y_rad_top if t < fps * 1.5: vertical_angle_rad = y_rad_top elif fps * 1.5 <= t and t < fps * 2.5: interp = (t - fps * 1.5) / fps vertical_angle_rad = y_rad_top + interp * y_rad_range elif fps * 2.5 <= t and t < fps * 4: vertical_angle_rad = y_rad_bottom elif fps * 4.0 <= t and t < fps * 5: interp = (t - fps * 4.0) / fps vertical_angle_rad = y_rad_bottom - interp * y_rad_range else: vertical_angle_rad = y_rad_top return horizontal_angle_rad, vertical_angle_rad def rotate_query_viewpoint(horizontal_angle_rad, vertical_angle_rad): camera_direction = np.array([ math.sin(horizontal_angle_rad), # x math.sin(vertical_angle_rad), # y math.cos(horizontal_angle_rad), # z ]) camera_direction = args.camera_distance * camera_direction / np.linalg.norm( camera_direction) yaw, pitch = compute_yaw_and_pitch(camera_direction) query_viewpoints = xp.array( ( camera_direction[0], camera_direction[1], camera_direction[2], math.cos(yaw), math.sin(yaw), math.cos(pitch), math.sin(pitch), ), dtype=np.float32, ) query_viewpoints = xp.broadcast_to(query_viewpoints, (1, ) + query_viewpoints.shape) return query_viewpoints #============================================================================== # Visualization #============================================================================== plt.style.use("dark_background") fig = plt.figure(figsize=(6, 7)) plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95) # fig.suptitle("GQN") axis_observations = fig.add_subplot(2, 1, 1) axis_observations.axis("off") axis_observations.set_title("observations") axis_generation = fig.add_subplot(2, 1, 2) axis_generation.axis("off") axis_generation.set_title("neural rendering") #============================================================================== # Generating animation #============================================================================== file_number = 1 random.seed(0) np.random.seed(0) with chainer.no_backprop_mode(): for subset in dataset: iterator = gqn.data.Iterator(subset, batch_size=1) for data_indices in iterator: animation_frame_array = [] observed_image_array = xp.full( (total_observations_per_scene, ) + image_shape, black_color, dtype=np.float32) observed_viewpoint_array = xp.zeros( (total_observations_per_scene, 7), dtype=np.float32) # shape: (batch, views, height, width, channels) images, viewpoints = subset[data_indices] # (batch, views, height, width, channels) -> (batch, views, channels, height, width) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) images = preprocess_images(images) batch_index = 0 #------------------------------------------------------------------------------ # Generate images with a single observation #------------------------------------------------------------------------------ observation_index = 0 # Scene encoder observed_image = images[batch_index, observation_index] observed_viewpoint = viewpoints[batch_index, observation_index] observed_image_array[observation_index] = to_device( observed_image) observed_viewpoint_array[observation_index] = to_device( observed_viewpoint) representation = model.compute_observation_representation( observed_image_array[None, :observation_index + 1], observed_viewpoint_array[None, :observation_index + 1]) # Update figure axis_observations_image = fill_observations_axis( [observed_image]) # Rotate camera for t in range(fps, fps * 6): artist_array = [ axis_observations.imshow( make_uint8(axis_observations_image), interpolation="none", animated=True) ] horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame( t) query_viewpoints = rotate_query_viewpoint( horizontal_angle_rad, vertical_angle_rad) generated_images = model.generate_image( query_viewpoints, representation)[0] artist_array.append( axis_generation.imshow(make_uint8(generated_images), interpolation="none", animated=True)) animation_frame_array.append(artist_array) #------------------------------------------------------------------------------ # Add observations #------------------------------------------------------------------------------ for n in range(total_observations_per_scene): axis_observations_image = fill_observations_axis( images[batch_index, :n + 1]) # Scene encoder representation = model.compute_observation_representation( observed_image_array[None, :n + 1], observed_viewpoint_array[None, :n + 1]) for t in range(fps // 2): artist_array = [ axis_observations.imshow( make_uint8(axis_observations_image), interpolation="none", animated=True) ] horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame( 0) query_viewpoints = rotate_query_viewpoint( horizontal_angle_rad, vertical_angle_rad) generated_images = model.generate_image( query_viewpoints, representation)[0] artist_array.append( axis_generation.imshow( make_uint8(generated_images), interpolation="none", animated=True)) animation_frame_array.append(artist_array) #------------------------------------------------------------------------------ # Generate images with all observations #------------------------------------------------------------------------------ # Scene encoder representation = model.compute_observation_representation( observed_image_array[None, :total_observations_per_scene + 1], observed_viewpoint_array[ None, :total_observations_per_scene + 1]) # Rotate camera for t in range(0, fps * 6): artist_array = [ axis_observations.imshow( make_uint8(axis_observations_image), interpolation="none", animated=True) ] horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame( t) query_viewpoints = rotate_query_viewpoint( horizontal_angle_rad, vertical_angle_rad) generated_images = model.generate_image( query_viewpoints, representation)[0] artist_array.append( axis_generation.imshow(make_uint8(generated_images), interpolation="none", animated=True)) animation_frame_array.append(artist_array) #------------------------------------------------------------------------------ # Write to file #------------------------------------------------------------------------------ anim = animation.ArtistAnimation(fig, animation_frame_array, interval=1 / fps, blit=True, repeat_delay=0) # anim.save( # "{}/shepard_matzler_observations_{}.gif".format( # args.figure_directory, file_number), # writer="imagemagick", # fps=fps) anim.save("{}/shepard_matzler_observations_{}.mp4".format( args.figure_directory, file_number), writer="ffmpeg", fps=fps) file_number += 1
def main(): meter_train = Meter() assert meter_train.load(args.snapshot_directory) #============================================================================== # Selecting the GPU #============================================================================== xp = np gpu_device = args.gpu_device using_gpu = gpu_device >= 0 if using_gpu: cuda.get_device(gpu_device).use() xp = cp #============================================================================== # Dataset #============================================================================== dataset_test = Dataset(args.test_dataset_directory) #============================================================================== # Hyperparameters #============================================================================== hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) print(hyperparams, "\n") #============================================================================== # Model #============================================================================== model = Model(hyperparams) assert model.load(args.snapshot_directory, meter_train.epoch) if using_gpu: model.to_gpu() #============================================================================== # Pixel-variance annealing #============================================================================== variance_scheduler = PixelVarianceScheduler() assert variance_scheduler.load(args.snapshot_directory) print(variance_scheduler, "\n") #============================================================================== # Algorithms #============================================================================== def encode_scene(images, viewpoints): # (batch, views, height, width, channels) -> (batch, views, channels, height, width) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) # Sample number of views total_views = images.shape[1] num_views = random.choice(range(1, total_views + 1)) # Sample views observation_view_indices = list(range(total_views)) random.shuffle(observation_view_indices) observation_view_indices = observation_view_indices[:num_views] observation_images = preprocess_images( images[:, observation_view_indices]) observation_query = viewpoints[:, observation_view_indices] representation = model.compute_observation_representation( observation_images, observation_query) # Sample query view query_index = random.choice(range(total_views)) query_images = preprocess_images(images[:, query_index]) query_viewpoints = viewpoints[:, query_index] # Transfer to gpu if necessary query_images = to_device(query_images, gpu_device) query_viewpoints = to_device(query_viewpoints, gpu_device) return representation, query_images, query_viewpoints def estimate_ELBO(query_images, z_t_param_array, pixel_mean, pixel_log_sigma): # KL Diverge, pixel_ln_varnce kl_divergence = 0 for params_t in z_t_param_array: mean_z_q, ln_var_z_q, mean_z_p, ln_var_z_p = params_t normal_q = chainer.distributions.Normal(mean_z_q, log_scale=ln_var_z_q) normal_p = chainer.distributions.Normal(mean_z_p, log_scale=ln_var_z_p) kld_t = chainer.kl_divergence(normal_q, normal_p) kl_divergence += cf.sum(kld_t) kl_divergence = kl_divergence / args.batch_size # Negative log-likelihood of generated image batch_size = query_images.shape[0] num_pixels_per_batch = np.prod(query_images.shape[1:]) normal = chainer.distributions.Normal(query_images, log_scale=pixel_log_sigma) log_px = cf.sum(normal.log_prob(pixel_mean)) / batch_size negative_log_likelihood = -log_px # Empirical ELBO ELBO = log_px - kl_divergence # https://arxiv.org/abs/1604.08772 Section.2 # https://www.reddit.com/r/MachineLearning/comments/56m5o2/discussion_calculation_of_bitsdims/ bits_per_pixel = -(ELBO / num_pixels_per_batch - np.log(256)) / np.log(2) return ELBO, bits_per_pixel, negative_log_likelihood, kl_divergence #============================================================================== # Test the model #============================================================================== meter = Meter() pixel_log_sigma = xp.full((args.batch_size, 3) + hyperparams.image_size, math.log(variance_scheduler.standard_deviation), dtype="float32") with chainer.no_backprop_mode(): for subset_index, subset in enumerate(dataset_test): iterator = Iterator(subset, batch_size=args.batch_size) for data_indices in iterator: images, viewpoints = subset[data_indices] # Scene encoder representation, query_images, query_viewpoints = encode_scene( images, viewpoints) # Compute empirical ELBO (z_t_param_array, pixel_mean) = model.sample_z_and_x_params_from_posterior( query_images, query_viewpoints, representation) (ELBO, bits_per_pixel, negative_log_likelihood, kl_divergence) = estimate_ELBO(query_images, z_t_param_array, pixel_mean, pixel_log_sigma) mean_squared_error = cf.mean_squared_error( query_images, pixel_mean) # Logging meter.update(ELBO=float(ELBO.data), bits_per_pixel=float(bits_per_pixel.data), negative_log_likelihood=float( negative_log_likelihood.data), kl_divergence=float(kl_divergence.data), mean_squared_error=float(mean_squared_error.data)) if subset_index % 100 == 0: print(" Subset {}/{}:".format( subset_index + 1, len(dataset_test), )) print(" {}".format(meter)) print(" Test:") print(" {} - done in {:.3f} min".format( meter, meter.elapsed_time, ))
def gqn_process(): # load model my_gpu = args.gpu_device if my_gpu < 0: xp = np else: cuda.get_device(args.gpu_device).use() xp = cp hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) chainer.serializers.load_hdf5(args.snapshot_file, model) if my_gpu > -1: model.to_gpu() chainer.print_runtime_info() observed_viewpoint, observed_image, offset = data_recv.get() observed_viewpoint = np.expand_dims(np.expand_dims( np.asarray(observed_viewpoint).astype(np.float32), axis=0), axis=0) observed_image = np.expand_dims(np.expand_dims( np.asarray(observed_image).astype(np.float32), axis=0), axis=0) offset = np.asarray(offset) camera_distance = np.mean( np.linalg.norm(observed_viewpoint[:, :, :3], axis=2)) camera_position_z = np.mean(observed_viewpoint[:, :, 1]) observed_image = observed_image.transpose( (0, 1, 4, 2, 3)).astype(np.float32) observed_image = preprocess_images(observed_image) # create representation and generate uncertainty map of environment [1000 viewpoints?] total_frames = 10 representation = model.compute_observation_representation( observed_image, observed_viewpoint) # get predictions highest_var = 0.0 no_of_samples = 20 highest_var_vp = 0 try: for i in range(0, total_frames): horizontal_angle_rad = compute_camera_angle_at_frame( i, total_frames) query_viewpoints = rotate_query_viewpoint(horizontal_angle_rad, camera_distance, camera_position_z, xp) generated_images = xp.squeeze( xp.array( model.generate_images(query_viewpoints, representation, no_of_samples))) var_image = xp.var(generated_images, axis=0) # var_image = chainer.backends.cuda.to_cpu(var_image) # grayscale # r,g,b = var_image # gray_var_image = 0.2989*r+0.5870*g+0.1140*b current_var = xp.mean(var_image) if highest_var == 0: highest_var = current_var highest_var_vp = query_viewpoints[0] elif current_var > highest_var: highest_var = current_var highest_var_vp = query_viewpoints[0] except KeyboardInterrupt: logging.warning('interrupt') # return next viewpoint and unit vector of end effector based on highest uncertainty found in the uncertainty map _x, _y, _z, _, _, _, _ = highest_var_vp _yaw, _pitch = compute_yaw_and_pitch([_x, _y, _z]) next_viewpoint = [_x, _y, _z, _yaw, _pitch] next_viewpoint = [chainer.backends.cuda.to_cpu(x) for x in next_viewpoint] next_viewpoint = [float(x) for x in next_viewpoint] data_send.put(next_viewpoint)
def main(): try: os.makedirs(args.figure_directory) except: pass # loading dataset & model cuda.get_device(args.gpu_device).use() xp=cp hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) chainer.serializers.load_hdf5(args.snapshot_file, model) model.to_gpu() total_observations_per_scene = 4 fps = 30 black_color = -0.5 image_shape = (3, ) + hyperparams.image_size axis_observations_image = np.zeros( (3, image_shape[1], total_observations_per_scene * image_shape[2]), dtype=np.float32) #============================================================================== # Utilities #============================================================================== def read_files(directory): filenames = [] files = os.listdir(directory) # ipdb.set_trace() for filename in files: if filename.endswith(".h5"): filenames.append(filename) filenames.sort() dataset_images = [] dataset_viewpoints = [] for i in range(len(filenames)): F = h5py.File(os.path.join(directory,filenames[i])) tmp_images = list(F["images"]) tmp_viewpoints = list(F["viewpoints"]) dataset_images.extend(tmp_images) dataset_viewpoints.extend(tmp_viewpoints) # for i in range(len(filenames)): # images_npy_path = os.path.join(directory, "images", filenames[i]) # viewpoints_npy_path = os.path.join(directory, "viewpoints", filenames[i]) # tmp_images = np.load(images_npy_path) # tmp_viewpoints = np.load(viewpoints_npy_path) # assert tmp_images.shape[0] == tmp_viewpoints.shape[0] # dataset_images.extend(tmp_images) # dataset_viewpoints.extend(tmp_viewpoints) dataset_images = np.array(dataset_images) dataset_viewpoints = np.array(dataset_viewpoints) dataset = list() for i in range(len(dataset_images)): item = {'image':dataset_images[i],'viewpoint':dataset_viewpoints[i]} dataset.append(item) return dataset def to_device(array): # if using_gpu: array = cuda.to_gpu(array) return array def fill_observations_axis(observation_images): axis_observations_image = np.full( (3, image_shape[1], total_observations_per_scene * image_shape[2]), black_color, dtype=np.float32) num_current_obs = len(observation_images) total_obs = total_observations_per_scene width = image_shape[2] x_start = width * (total_obs - num_current_obs) // 2 for obs_image in observation_images: x_end = x_start + width axis_observations_image[:, :, x_start:x_end] = obs_image x_start += width return axis_observations_image def compute_camera_angle_at_frame(t): return t * 2 * math.pi / (fps * 2) def rotate_query_viewpoint(horizontal_angle_rad, camera_distance, camera_position_y): camera_position = np.array([ camera_distance * math.sin(horizontal_angle_rad), # x camera_position_y, camera_distance * math.cos(horizontal_angle_rad), # z ]) center = np.array((0, camera_position_y, 0)) camera_direction = camera_position - center yaw, pitch = compute_yaw_and_pitch(camera_direction) query_viewpoints = xp.array( ( camera_position[0], camera_position[1], camera_position[2], math.cos(yaw), math.sin(yaw), math.cos(pitch), math.sin(pitch), ), dtype=np.float32, ) query_viewpoints = xp.broadcast_to(query_viewpoints, (1, ) + query_viewpoints.shape) return query_viewpoints def render(representation, camera_distance, camera_position_y, total_frames, animation_frame_array, rotate_camera=True): for t in range(0, total_frames): artist_array = [ axis_observations.imshow( make_uint8(axis_observations_image), interpolation="none", animated=True) ] horizontal_angle_rad = compute_camera_angle_at_frame(t) if rotate_camera == False: horizontal_angle_rad = compute_camera_angle_at_frame(0) query_viewpoints = rotate_query_viewpoint( horizontal_angle_rad, camera_distance, camera_position_y) generated_images = model.generate_image(query_viewpoints, representation)[0] artist_array.append( axis_generation.imshow( make_uint8(generated_images), interpolation="none", animated=True)) animation_frame_array.append(artist_array) #============================================================================== # Visualization #============================================================================== plt.style.use("dark_background") fig = plt.figure(figsize=(6, 7)) plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95) # fig.suptitle("GQN") axis_observations = fig.add_subplot(2, 1, 1) axis_observations.axis("off") axis_observations.set_title("observations") axis_generation = fig.add_subplot(2, 1, 2) axis_generation.axis("off") axis_generation.set_title("neural rendering") #============================================================================== # Generating animation #============================================================================== dataset = read_files(args.dataset_directory) file_number = 1 random.seed(0) np.random.seed(0) with chainer.no_backprop_mode(): iterator = chainer.iterators.SerialIterator(dataset,batch_size=1) for i in range(len(iterator.dataset)): animation_frame_array = [] # shape: (batch, views, height, width, channels) images, viewpoints = np.array([iterator.dataset[i]["image"]]),np.array([iterator.dataset[i]["viewpoint"]]) camera_distance = np.mean( np.linalg.norm(viewpoints[:, :, :3], axis=2)) camera_position_y = np.mean(viewpoints[:, :, 1]) # (batch, views, height, width, channels) -> (batch, views, channels, height, width) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) images = preprocess_images(images) batch_index = 0 total_views = images.shape[1] random_observation_view_indices = list(range(total_views)) random.shuffle(random_observation_view_indices) random_observation_view_indices = random_observation_view_indices[: total_observations_per_scene] #------------------------------------------------------------------------------ # Observations #------------------------------------------------------------------------------ observed_images = images[batch_index, random_observation_view_indices] observed_viewpoints = viewpoints[ batch_index, random_observation_view_indices] observed_images = to_device(observed_images) observed_viewpoints = to_device(observed_viewpoints) #------------------------------------------------------------------------------ # Generate images with a single observation #------------------------------------------------------------------------------ # Scene encoder representation = model.compute_observation_representation( observed_images[None, :1], observed_viewpoints[None, :1]) # Update figure observation_index = random_observation_view_indices[0] observed_image = images[batch_index, observation_index] axis_observations_image = fill_observations_axis( [observed_image]) # Neural rendering render(representation, camera_distance, camera_position_y, fps * 2, animation_frame_array) #------------------------------------------------------------------------------ # Add observations #------------------------------------------------------------------------------ for n in range(total_observations_per_scene): observation_indices = random_observation_view_indices[:n + 1] axis_observations_image = fill_observations_axis( images[batch_index, observation_indices]) # Scene encoder representation = model.compute_observation_representation( observed_images[None, :n + 1], observed_viewpoints[None, :n + 1]) # Neural rendering render( representation, camera_distance, camera_position_y, fps // 2, animation_frame_array, rotate_camera=False) #------------------------------------------------------------------------------ # Generate images with all observations #------------------------------------------------------------------------------ # Scene encoder representation = model.compute_observation_representation( observed_images[None, :total_observations_per_scene + 1], observed_viewpoints[None, :total_observations_per_scene + 1]) # Neural rendering render(representation, camera_distance, camera_position_y, fps * 4, animation_frame_array) #------------------------------------------------------------------------------ # Write to file #------------------------------------------------------------------------------ anim = animation.ArtistAnimation( fig, animation_frame_array, interval=1 / fps, blit=True, repeat_delay=0) # anim.save( # "{}/shepard_matzler_observations_{}.gif".format( # args.figure_directory, file_number), # writer="imagemagick", # fps=fps) anim.save( "{}/rooms_ring_camera_observations_{}.mp4".format( args.figure_directory, file_number), writer="ffmpeg", fps=fps) file_number += 1
def main(): try: os.makedirs(args.figure_directory) except: pass xp = np using_gpu = args.gpu_device >= 0 if using_gpu: cuda.get_device(args.gpu_device).use() xp = cp dataset = gqn.data.Dataset(args.dataset_directory) meter = Meter() assert meter.load(args.snapshot_directory) hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) assert model.load(args.snapshot_directory, meter.epoch) if using_gpu: model.to_gpu() #============================================================================== # Visualization #============================================================================== plt.figure(figsize=(12, 16)) axis_observation_1 = plt.subplot2grid((4, 3), (0, 0)) axis_observation_2 = plt.subplot2grid((4, 3), (0, 1)) axis_observation_3 = plt.subplot2grid((4, 3), (0, 2)) axis_predictions = plt.subplot2grid((4, 3), (1, 0), rowspan=3, colspan=3) axis_observation_1.axis("off") axis_observation_2.axis("off") axis_observation_3.axis("off") axis_predictions.set_xticks([], []) axis_predictions.set_yticks([], []) axis_observation_1.set_title("Observation 1", fontsize=22) axis_observation_2.set_title("Observation 2", fontsize=22) axis_observation_3.set_title("Observation 3", fontsize=22) axis_predictions.set_title("Neural Rendering", fontsize=22) axis_predictions.set_xlabel("Yaw", fontsize=22) axis_predictions.set_ylabel("Pitch", fontsize=22) #============================================================================== # Generating images #============================================================================== num_views_per_scene = 3 num_yaw_pitch_steps = 10 image_width, image_height = hyperparams.image_size prediction_images = make_uint8( np.full((num_yaw_pitch_steps * image_width, num_yaw_pitch_steps * image_height, 3), 0)) file_number = 1 random.seed(0) np.random.seed(0) with chainer.no_backprop_mode(): for subset in dataset: iterator = gqn.data.Iterator(subset, batch_size=1) for data_indices in iterator: # shape: (batch, views, height, width, channels) # range: [-1, 1] images, viewpoints = subset[data_indices] camera_distance = np.mean( np.linalg.norm(viewpoints[:, :, :3], axis=2)) # (batch, views, height, width, channels) -> (batch, views, channels, height, width) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) images = preprocess_images(images) batch_index = 0 #------------------------------------------------------------------------------ # Observations #------------------------------------------------------------------------------ total_views = images.shape[1] random_observation_view_indices = list(range(total_views)) random.shuffle(random_observation_view_indices) random_observation_view_indices = random_observation_view_indices[: num_views_per_scene] observed_images = images[:, random_observation_view_indices] observed_viewpoints = viewpoints[:, random_observation_view_indices] representation = model.compute_observation_representation( observed_images, observed_viewpoints) axis_observation_1.imshow( make_uint8(observed_images[batch_index, 0])) axis_observation_2.imshow( make_uint8(observed_images[batch_index, 1])) axis_observation_3.imshow( make_uint8(observed_images[batch_index, 2])) y_angle_rad = math.pi / 2 for pitch_loop in range(num_yaw_pitch_steps): camera_y = math.sin(y_angle_rad) x_angle_rad = math.pi for yaw_loop in range(num_yaw_pitch_steps): camera_direction = np.array([ math.sin(x_angle_rad), camera_y, math.cos(x_angle_rad) ]) camera_direction = camera_distance * camera_direction / np.linalg.norm( camera_direction) yaw, pitch = compute_yaw_and_pitch(camera_direction) query_viewpoints = xp.array( ( camera_direction[0], camera_direction[1], camera_direction[2], math.cos(yaw), math.sin(yaw), math.cos(pitch), math.sin(pitch), ), dtype=np.float32, ) query_viewpoints = xp.broadcast_to( query_viewpoints, (1, ) + query_viewpoints.shape) generated_images = model.generate_image( query_viewpoints, representation)[0] yi_start = pitch_loop * image_height yi_end = (pitch_loop + 1) * image_height xi_start = yaw_loop * image_width xi_end = (yaw_loop + 1) * image_width prediction_images[yi_start:yi_end, xi_start:xi_end] = make_uint8( generated_images) x_angle_rad -= 2 * math.pi / num_yaw_pitch_steps y_angle_rad -= math.pi / num_yaw_pitch_steps axis_predictions.imshow(prediction_images) plt.savefig("{}/shepard_metzler_predictions_{}.png".format( args.figure_directory, file_number)) file_number += 1
def main(): start_time = time.time() writer = SummaryWriter('/GQN/chainer-gqn/tensor-log') try: os.makedirs(args.figure_directory) except: pass xp = np using_gpu = args.gpu_device >= 0 if using_gpu: cuda.get_device(args.gpu_device).use() xp = cp dataset = gqn.data.Dataset(args.dataset_directory) meter = Meter() assert meter.load(args.snapshot_directory) hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) assert model.load(args.snapshot_directory, meter.epoch) if using_gpu: model.to_gpu() total_observations_per_scene = 4 fps = 30 black_color = -0.5 image_shape = (3, ) + hyperparams.image_size axis_observations_image = np.zeros( (3, image_shape[1], total_observations_per_scene * image_shape[2]), dtype=np.float32) #============================================================================== # Utilities #============================================================================== def to_device(array): if using_gpu: array = cuda.to_gpu(array) return array def fill_observations_axis(observation_images): axis_observations_image = np.full( (3, image_shape[1], total_observations_per_scene * image_shape[2]), black_color, dtype=np.float32) num_current_obs = len(observation_images) total_obs = total_observations_per_scene width = image_shape[2] x_start = width * (total_obs - num_current_obs) // 2 for obs_image in observation_images: x_end = x_start + width axis_observations_image[:, :, x_start:x_end] = obs_image x_start += width return axis_observations_image def compute_camera_angle_at_frame(t): horizontal_angle_rad = 2 * t * math.pi / (fps * 2) + math.pi / 4 y_rad_top = math.pi / 3 y_rad_bottom = -math.pi / 3 y_rad_range = y_rad_bottom - y_rad_top if t < fps * 1.5: vertical_angle_rad = y_rad_top elif fps * 1.5 <= t and t < fps * 2.5: interp = (t - fps * 1.5) / fps vertical_angle_rad = y_rad_top + interp * y_rad_range elif fps * 2.5 <= t and t < fps * 4: vertical_angle_rad = y_rad_bottom elif fps * 4.0 <= t and t < fps * 5: interp = (t - fps * 4.0) / fps vertical_angle_rad = y_rad_bottom - interp * y_rad_range else: vertical_angle_rad = y_rad_top return horizontal_angle_rad, vertical_angle_rad def compute_vertical_rotation_at_frame(horizontal, vertical, t): # move horizontal view only horizontal_angle_rad = horizontal + (t - fps) * (math.pi / 64) vertical_angle_rad = vertical + 0 return horizontal_angle_rad, vertical_angle_rad def rotate_query_viewpoint(horizontal_angle_rad, vertical_angle_rad, camera_distance): camera_direction = np.array([ math.sin(horizontal_angle_rad), # x math.sin(vertical_angle_rad), # y math.cos(horizontal_angle_rad), # z ]) # removed linalg norm for observation purposes camera_direction = camera_distance * camera_direction # ipdb.set_trace() yaw, pitch = compute_yaw_and_pitch(camera_direction) query_viewpoints = xp.array( ( camera_direction[0], camera_direction[1], camera_direction[2], math.cos(yaw), math.sin(yaw), math.cos(pitch), math.sin(pitch), ), dtype=np.float32, ) query_viewpoints = xp.broadcast_to(query_viewpoints, (1, ) + query_viewpoints.shape) return query_viewpoints def render(representation, camera_distance, obs_viewpoint, start_t, end_t, animation_frame_array, savename=None, rotate_camera=True): all_var_bg = [] all_var = [] all_var_z = [] all_q_view = [] all_c = [] all_h = [] all_u = [] for t in range(start_t, end_t): artist_array = [ axis_observations.imshow(make_uint8(axis_observations_image), interpolation="none", animated=True) ] # convert x,y into radians?? # try reversing the camera direction calculation in rotate query viewpoint (impossible to reverse the linalg norm...) horizontal_angle_rad = np.arctan2(obs_viewpoint[0], obs_viewpoint[2]) vertical_angle_rad = np.arcsin(obs_viewpoint[1] / camera_distance) # xz_diagonal = np.sqrt(np.square(obs_viewpoint[0])+np.square(obs_viewpoint[2])) # vertical_angle_rad = np.arctan2(obs_viewpoint[1],xz_diagonal) # vertical_angle_rad = np.arcsin(obs_viewpoint[1]/camera_distance) # horizontal_angle_rad, vertical_angle_rad = 0,0 # ipdb.set_trace() horizontal_angle_rad, vertical_angle_rad = compute_vertical_rotation_at_frame( horizontal_angle_rad, vertical_angle_rad, t) if rotate_camera == False: horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame( 0) query_viewpoints = rotate_query_viewpoint(horizontal_angle_rad, vertical_angle_rad, camera_distance) # obtain generated images, as well as mean and variance before gaussian generated_images, var_bg, latent_z, ct = model.generate_multi_image( query_viewpoints, representation, 100) logging.info("retrieved variables, time elapsed: " + str(time.time() - start_time)) # cpu_generated_images = chainer.backends.cuda.to_cpu(generated_images) generated_images = np.squeeze(generated_images) latent_z = np.squeeze(latent_z) # ipdb.set_trace() ct = np.squeeze(ct) # ht = np.squeeze(np.asarray(ht)) # ut = np.squeeze(np.asarray(ut)) # obtain data from Chainer Variable and obtain mean var_bg = cp.mean(var_bg, axis=0) logging.info("variance of bg, time elapsed: " + str(time.time() - start_time)) var_z = cp.var(latent_z, axis=0) logging.info("variance of z, time elapsed: " + str(time.time() - start_time)) # ipdb.set_trace() # print(ct.shape()) var_c = cp.var(ct, axis=0) logging.info("variance of c, time elapsed: " + str(time.time() - start_time)) # var_h = cp.var(ht,axis=0) # var_u = cp.var(ut,axis=0) # write viewpoint and image variance to file gen_img_var = np.var(generated_images, axis=0) logging.info("calculated variance of gen images, time elapsed: " + str(time.time() - start_time)) all_var_bg.append((var_bg)[None]) all_var.append((gen_img_var)[None]) all_var_z.append((var_z)[None]) all_q_view.append( chainer.backends.cuda.to_cpu(horizontal_angle_rad)[None] * 180 / math.pi) all_c.append((var_c)[None]) logging.info("appending, time elapsed: " + str(time.time() - start_time)) # all_h.append(chainer.backends.cuda.to_cpu(var_h)[None]) # all_u.append(chainer.backends.cuda.to_cpu(var_u)[None]) # sample = generated_images[0] pred_mean = cp.mean(generated_images, axis=0) # artist_array.append( # axis_generation.imshow( # make_uint8(pred_mean), # interpolation="none", # animated=True)) # animation_frame_array.append(artist_array) all_var_bg = np.concatenate(chainer.backends.cuda.to_cpu(all_var_bg), axis=0) all_var = np.concatenate(chainer.backends.cuda.to_cpu(all_var), axis=0) all_var_z = np.concatenate(chainer.backends.cuda.to_cpu(all_var_z), axis=0) all_c = np.concatenate(chainer.backends.cuda.to_cpu(all_c), axis=0) # all_h = np.concatenate(all_h,axis=0) # all_u = np.concatenate(all_u,axis=0) logging.info("concatenating, time elapsed: " + str(time.time() - start_time)) with h5py.File(savename, "a") as f: f.create_dataset("variance_all_viewpoints", data=all_var) f.create_dataset("query_viewpoints", data=np.squeeze(np.asarray(all_q_view))) f.create_dataset("variance_b4_gaussian", data=all_var_bg) f.create_dataset("variance_of_z", data=all_var_z) f.create_dataset("c", data=all_c) # f.create_dataset("h",data=all_h) # f.create_dataset("u",data=all_u) logging.info("saving, time elapsed: " + str(time.time() - start_time)) #============================================================================== # Visualization #============================================================================== plt.style.use("dark_background") fig = plt.figure(figsize=(6, 7)) plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95) # fig.suptitle("GQN") axis_observations = fig.add_subplot(2, 1, 1) axis_observations.axis("off") axis_observations.set_title("observations") axis_generation = fig.add_subplot(2, 1, 2) axis_generation.axis("off") axis_generation.set_title("neural rendering") #============================================================================== # Generating animation #============================================================================== file_number = 1 random.seed(0) np.random.seed(0) logging.basicConfig( level=logging.INFO, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s' ) with chainer.no_backprop_mode(): for subset in dataset: iterator = gqn.data.Iterator(subset, batch_size=1) for data_indices in iterator: animation_frame_array = [] # shape: (batch, views, height, width, channels) images, viewpoints = subset[data_indices] camera_distance = np.mean( np.linalg.norm(viewpoints[:, :, :3], axis=2)) # (batch, views, height, width, channels) -> (batch, views, channels, height, width) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) images = preprocess_images(images) logging.info('preprocess ' + str(time.time() - start_time)) batch_index = 0 total_views = images.shape[1] random_observation_view_indices = list(range(total_views)) random.shuffle(random_observation_view_indices) random_observation_view_indices = random_observation_view_indices[: total_observations_per_scene] #------------------------------------------------------------------------------ # Observations #------------------------------------------------------------------------------ observed_images = images[batch_index, random_observation_view_indices] observed_viewpoints = viewpoints[ batch_index, random_observation_view_indices] observed_images = to_device(observed_images) observed_viewpoints = to_device(observed_viewpoints) #------------------------------------------------------------------------------ # Generate images with a single observation #------------------------------------------------------------------------------ # Scene encoder representation = model.compute_observation_representation( observed_images[None, :1], observed_viewpoints[None, :1]) # Update figure observation_index = random_observation_view_indices[0] observed_image = images[batch_index, observation_index] axis_observations_image = fill_observations_axis( [observed_image]) # save observed viewpoint filename = "{}/variance_{}.hdf5".format( args.figure_directory, file_number) if os.path.exists(filename): os.remove(filename) with h5py.File(filename, "a") as f: f.create_dataset("observed_viewpoint", data=chainer.backends.cuda.to_cpu( observed_viewpoints[0])) f.create_dataset( "obs_viewpoint_horizontal_angle", data=np.arcsin( chainer.backends.cuda.to_cpu( observed_viewpoints[0][0]) / camera_distance) * 180 / math.pi) logging.info('write 2 variables to hdf5 file, time elapsed: ' + str(time.time() - start_time)) obs_viewpoint = np.squeeze(observed_viewpoints[0]) # Neural rendering render(representation, camera_distance, observed_viewpoints[0], fps, fps * 6, animation_frame_array, savename=filename) logging.info( 'write 4 other variables to hdf5 file, time elapsed: ' + str(time.time() - start_time)) #------------------------------------------------------------------------------ # Write to file #------------------------------------------------------------------------------ # anim = animation.ArtistAnimation( # fig, # animation_frame_array, # interval=1 / fps, # blit=True, # repeat_delay=0) # anim.save( # "{}/shepard_metzler_observations_{}.gif".format( # args.figure_directory, file_number), # writer="imagemagick", # fps=fps) # anim.save( # "{}/shepard_metzler_observations_{}.mp4".format( # args.figure_directory, file_number), # writer="ffmpeg", # fps=2) if file_number == 20: break else: file_number += 1
def main(): try: os.makedirs(args.figure_directory) except: pass xp = np using_gpu = args.gpu_device >= 0 if using_gpu: cuda.get_device(args.gpu_device).use() xp = cp dataset = gqn.data.Dataset(args.dataset_directory, # use_ground_truth=True ) meter = Meter() assert meter.load(args.snapshot_directory) hyperparams = HyperParameters() assert hyperparams.load(args.snapshot_directory) model = Model(hyperparams) assert model.load(args.snapshot_directory, meter.epoch) if using_gpu: model.to_gpu() total_observations_per_scene = 4 fps = 30 black_color = -0.5 image_shape = (3, ) + hyperparams.image_size axis_observations_image = np.zeros( (3, image_shape[1], total_observations_per_scene * image_shape[2]), dtype=np.float32) #============================================================================== # Utilities #============================================================================== def to_device(array): if using_gpu: array = cuda.to_gpu(array) return array def fill_observations_axis(observation_images): axis_observations_image = np.full( (3, image_shape[1], total_observations_per_scene * image_shape[2]), black_color, dtype=np.float32) num_current_obs = len(observation_images) total_obs = total_observations_per_scene width = image_shape[2] x_start = width * (total_obs - num_current_obs) // 2 for obs_image in observation_images: x_end = x_start + width axis_observations_image[:, :, x_start:x_end] = obs_image x_start += width return axis_observations_image def compute_camera_angle_at_frame(t): horizontal_angle_rad = 2 * t * math.pi / (fps * 2) + math.pi / 4 y_rad_top = math.pi / 3 y_rad_bottom = -math.pi / 3 y_rad_range = y_rad_bottom - y_rad_top if t < fps * 1.5: vertical_angle_rad = y_rad_top elif fps * 1.5 <= t and t < fps * 2.5: interp = (t - fps * 1.5) / fps vertical_angle_rad = y_rad_top + interp * y_rad_range elif fps * 2.5 <= t and t < fps * 4: vertical_angle_rad = y_rad_bottom elif fps * 4.0 <= t and t < fps * 5: interp = (t - fps * 4.0) / fps vertical_angle_rad = y_rad_bottom - interp * y_rad_range else: vertical_angle_rad = y_rad_top return horizontal_angle_rad, vertical_angle_rad def rotate_query_viewpoint(horizontal_angle_rad, vertical_angle_rad, camera_distance): camera_direction = np.array([ math.sin(horizontal_angle_rad), # x math.sin(vertical_angle_rad), # y math.cos(horizontal_angle_rad), # z ]) camera_direction = camera_distance * camera_direction / np.linalg.norm( camera_direction) yaw, pitch = compute_yaw_and_pitch(camera_direction) query_viewpoints = xp.array( ( camera_direction[0], camera_direction[1], camera_direction[2], math.cos(yaw), math.sin(yaw), math.cos(pitch), math.sin(pitch), ), dtype=np.float32, ) query_viewpoints = xp.broadcast_to(query_viewpoints, (1, ) + query_viewpoints.shape) return query_viewpoints # added/modified def compute_horizontal_rotation_at_frame(t): '''This rotates the scene horizontally.''' horizontal_angle_rad = 2 * t * math.pi / (fps * 2) + math.pi / 4 vertical_angle_rad = 0 return horizontal_angle_rad, vertical_angle_rad def get_mse_image(ground_truth, predicted): '''Calculates MSE between ground truth and predicted observation, and returns an image.''' assert ground_truth.shape == predicted.shape mse_image = np.square(ground_truth - predicted) * 0.5 mse_image = np.concatenate(mse_image).astype(np.float32) mse_image = np.reshape(mse_image, (3, 64, 64)) return mse_image.transpose(1, 2, 0) def render(representation, camera_distance, start_t, end_t, gt_images, gt_viewpoints, animation_frame_array, rotate_camera=True): gt_images = np.squeeze(gt_images) gt_viewpoints = cp.reshape(cp.asarray(gt_viewpoints), (15, 1, 7)) idx = cp.argsort(cp.squeeze(gt_viewpoints)[:, 0]) gt_images = [ i for i, v in sorted(zip(gt_images, idx), key=operator.itemgetter(1)) ] gt_viewpoints = [ i for i, v in sorted(zip(gt_viewpoints, idx), key=operator.itemgetter(1)) ] count = 0 '''shows variance and mean images of 100 samples from the Gaussian.''' for t in range(start_t, end_t): artist_array = [ axis_observations.imshow(make_uint8(axis_observations_image), interpolation="none", animated=True) ] horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame( t) if rotate_camera == False: horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame( 0) query_viewpoints = rotate_query_viewpoint(horizontal_angle_rad, vertical_angle_rad, camera_distance) # shape 100x1x3x64x64, when Model is from model_testing.py generated_images = model.generate_image(query_viewpoints, representation, 100) # generate predicted from ground truth viewpoints predicted_images = model.generate_image(gt_viewpoints[count], representation, 1) # predicted_images = model.generate_image(query_viewpoints, representation,1) predicted_images = np.squeeze(predicted_images) image_mse = get_mse_image(gt_images[count], predicted_images) # when sampling with 100 cpu_generated_images = chainer.backends.cuda.to_cpu( generated_images) generated_images = np.squeeze(cpu_generated_images) # # cpu calculation # cpu_image_mean = np.mean(cpu_generated_images,axis=0) # cpu_image_std = np.std(cpu_generated_images,axis=0) # cpu_image_var = np.var(cpu_generated_images,axis=0) # image_mean = np.squeeze(chainer.backends.cuda.to_gpu(cpu_image_mean)) # image_std = chainer.backends.cuda.to_gpu(cpu_image_std) # image_var = np.squeeze(chainer.backends.cuda.to_gpu(cpu_image_var)) image_mean = cp.mean(cp.squeeze(generated_images), axis=0) image_var = cp.var(cp.squeeze(generated_images), axis=0) # convert to black and white. # grayscale r, g, b = image_var gray_image_var = 0.2989 * r + 0.5870 * g + 0.1140 * b # thresholding Otsu's method thresh = threshold_otsu(gray_image_var) var_binary = gray_image_var > thresh sample_image = np.squeeze(generated_images[0]) if count == 14: count = 0 elif (t - fps) % 10 == 0: count += 1 print("computed an image. Count =", count) artist_array.append( axis_generation_variance.imshow(var_binary, cmap=plt.cm.gray, interpolation="none", animated=True)) artist_array.append( axis_generation_mean.imshow(make_uint8(image_mean), interpolation="none", animated=True)) artist_array.append( axis_generation_sample.imshow(make_uint8(sample_image), interpolation="none", animated=True)) artist_array.append( axis_generation_mse.imshow(make_uint8(image_mse), cmap='gray', interpolation="none", animated=True)) animation_frame_array.append(artist_array) #============================================================================== # Visualization #============================================================================== plt.style.use("dark_background") fig = plt.figure(figsize=(6, 7)) plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95) # fig.suptitle("GQN") axis_observations = fig.add_subplot(3, 1, 1) axis_observations.axis("off") axis_observations.set_title("observations") axis_generation_mse = fig.add_subplot(3, 2, 3) axis_generation_mse.axis("off") axis_generation_mse.set_title("MSE") axis_generation_variance = fig.add_subplot(3, 2, 4) axis_generation_variance.axis("off") axis_generation_variance.set_title("Variance") axis_generation_mean = fig.add_subplot(3, 2, 5) axis_generation_mean.axis("off") axis_generation_mean.set_title("Mean") axis_generation_sample = fig.add_subplot(3, 2, 6) axis_generation_sample.axis("off") axis_generation_sample.set_title("Normal Rendering") #============================================================================== # Generating animation #============================================================================== file_number = 1 random.seed(0) np.random.seed(0) with chainer.no_backprop_mode(): for subset in dataset: iterator = gqn.data.Iterator(subset, batch_size=1) for data_indices in iterator: animation_frame_array = [] # shape: (batch, views, height, width, channels) images, viewpoints = subset[data_indices] # images, viewpoints, original images = subset[data_indices] camera_distance = np.mean( np.linalg.norm(viewpoints[:, :, :3], axis=2)) # (batch, views, height, width, channels) -> (batch, views, channels, height, width) images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32) images = preprocess_images(images) # (batch, views, height, width, channels) -> (batch, views, channels, height, width) # original_images = original_images.transpose((0, 1, 4, 2, 3)).astype(np.float32) # original_images = preprocess_images(original_images) batch_index = 0 total_views = images.shape[1] random_observation_view_indices = list(range(total_views)) random.shuffle(random_observation_view_indices) random_viewed_observation_indices = random_observation_view_indices[: total_observations_per_scene] #------------------------------------------------------------------------------ # Ground Truth #------------------------------------------------------------------------------ gt_images = images gt_viewpoints = viewpoints # gt_images = original_images #------------------------------------------------------------------------------ # Observations #------------------------------------------------------------------------------ observed_images = images[batch_index, random_viewed_observation_indices] observed_viewpoints = viewpoints[ batch_index, random_viewed_observation_indices] observed_images = to_device(observed_images) observed_viewpoints = to_device(observed_viewpoints) #------------------------------------------------------------------------------ # Generate images with a single observation #------------------------------------------------------------------------------ # Scene encoder representation = model.compute_observation_representation( observed_images[None, :1], observed_viewpoints[None, :1]) # Update figure observation_index = random_viewed_observation_indices[0] observed_image = images[batch_index, observation_index] axis_observations_image = fill_observations_axis( [observed_image]) # Neural rendering render(representation, camera_distance, fps, fps * 6, gt_images, gt_viewpoints, animation_frame_array) #------------------------------------------------------------------------------ # Add observations #------------------------------------------------------------------------------ for n in range(1, total_observations_per_scene): observation_indices = random_viewed_observation_indices[: n + 1] axis_observations_image = fill_observations_axis( images[batch_index, observation_indices]) # Scene encoder representation = model.compute_observation_representation( observed_images[None, :n + 1], observed_viewpoints[None, :n + 1]) # Neural rendering render(representation, camera_distance, 0, fps // 2, gt_images, gt_viewpoints, animation_frame_array, rotate_camera=False) #------------------------------------------------------------------------------ # Generate images with all observations #------------------------------------------------------------------------------ # Scene encoder representation = model.compute_observation_representation( observed_images[None, :total_observations_per_scene + 1], observed_viewpoints[None, :total_observations_per_scene + 1]) # Neural rendering render(representation, camera_distance, 0, fps * 6, gt_images, gt_viewpoints, animation_frame_array) #------------------------------------------------------------------------------ # Write to file #------------------------------------------------------------------------------ anim = animation.ArtistAnimation(fig, animation_frame_array, interval=1 / fps, blit=True, repeat_delay=0) # anim.save( # "{}/shepard_metzler_observations_{}.gif".format( # args.figure_directory, file_number), # writer="imagemagick", # fps=fps) anim.save("{}/shepard_metzler_observations_{}.mp4".format( args.figure_directory, file_number), writer="ffmpeg", fps=fps) print("video saved") file_number += 1