def load_depth(file_name): """Loads the depth using the sintel SDK and converts to torch.Tensor """ assert os.path.isfile(file_name), "Invalid file {}".format(file_name) import sintel_io depth = sintel_io.depth_read(file_name) return torch.from_numpy(depth).view(1, 1, *depth.shape).float()
def depth_read_and_extend(depth_path): depth = depth_read(depth_path) if depth.shape[0] % 32 != 0: depth = np.pad(depth, ((38, 38), (0, 0)), 'constant') if np.min(depth) < 0: depth += 5 return depth
def sintel_iter(rgb, depth, bs = 6): for i in range(0, len(depth), bs): b_r = rgb[i:i + bs] b_d = depth[i:i + bs] depths = np.exp(np.clip(np.concatenate([depth_read(d)[None,:,:,None] for d in b_d]), 0, 20)) rgbs = np.concatenate([open_rgb(r)[None,:,:] for r in b_r]) yield rgbs, depths
def load_depth(file_name): """Load the depth using the syntel SDK and converts to torch.Tensor.""" if not os.path.isfile(file_name): raise AssertionError(f"Invalid file {file_name}") import sintel_io depth = sintel_io.depth_read(file_name) return torch.from_numpy(depth).view(1, 1, *depth.shape).float()
def get_histo_for_discrete_depth(train_dir): histo = np.zeros(15, dtype=np.int32) for patch in os.listdir(train_dir): if patch.endswith('.dpt'): patch_path = path.join(train_dir, patch) dpt = depth_read(patch_path) dpt_histo = np.bincount(np.concatenate(dpt).astype(int), minlength=16)[1:] histo += dpt_histo return histo
def get_depth_histogram(depth_dir): dpt_list = list() for depth in os.listdir(depth_dir): path = os.path.join(depth_dir,depth) dpt = depth_read(path) dpt_list.append(dpt) dpt_list = np.concatenate(np.concatenate(dpt_list)) histo = np.histogram(dpt_list, np.arange(1,np.max(dpt_list))) plt.plot(histo[0])
def get_depth_stats(depth_dir): depth_min = 10000 depth_max = 0 for file in os.listdir(depth_dir): filepath = os.path.join(depth_dir, file) dpt = depth_read(filepath) if (np.max(dpt) > depth_max): depth_max = np.max(dpt) if (np.min(dpt) < depth_min): depth_min = np.min(dpt) return depth_min, depth_max
import numpy as np from matplotlib import pyplot as plt import sys sys.path.append('../') import sintel_io as sio # Test and display some real data folder_name = 'alley_1' frame_no = 1 #smaller than 10 DEPTHFILE = '/cluster/scratch/takmaza/CVL/MPI-Sintel-complete/training/depth/' + folder_name + '/frame_000' + str( frame_no) + '.dpt' CAMFILE = '/cluster/scratch/takmaza/CVL/MPI-Sintel-complete/training/camdata_left/' + folder_name + '/frame_0001.cam' # Load data depth = sio.depth_read(DEPTHFILE) I, E = sio.cam_read(CAMFILE) print(depth.shape) # Display data #plt.figure() #plt.imshow(depth,cmap='gray') #plt.title('depth') print(I) print(E) #plt.show()
def __init__(self, root, train=True, transform=None, target_transform=None, load_pickle=True, train_dir=None, label_dir=None, test_dir=None, add_noise=True): self.root = os.path.expanduser(root) self.transform = transform self.target_transform = target_transform self.train = train # training set or test set self.add_noise = add_noise if train: pickle_dir = path.join(train_dir, "pickle") else: pickle_dir = path.join(test_dir, "pickle") # now load the picked numpy arrays if self.train: self.train_data = [] self.train_labels = [] if load_pickle: for pckl in os.listdir(pickle_dir): if pckl.endswith(".pickle"): pckl_path = path.join(pickle_dir, pckl) pckl_file = open(pckl_path, 'rb') entry = pickle.load(pckl_file, encoding='latin1') self.train_data.append(entry['data']) self.train_labels.append(entry['labels']) pckl_file.close() self.train_data = np.concatenate(self.train_data) self.train_labels = np.concatenate(self.train_labels) else: for patch in os.listdir(train_dir): if patch.endswith('.png'): patch_path = path.join(train_dir, patch) base = path.splitext(patch)[0] label = base + '.png' label_path = path.join(label_dir, label) patch_file = Image.open(patch_path) patch_arr = np.array(patch_file) label_file = depth_read(label_path) label_arr = np.array(label_file) label_arr = label_arr + 4 # Fix from -4:10 to 0:14 if patch_arr.shape[0] == 436: patch_arr_extended = np.zeros((512, 1024, 3), dtype=np.uint8) patch_arr_extended[38:-38, :, :] = patch_arr patch_arr = patch_arr_extended label_arr_extended = np.zeros((512, 1024), dtype=np.uint8) label_arr_extended[38:-38, :] = label_arr label_arr = label_arr_extended #Make the classes 0-14 instead of 1-15 # label_arr -= 1 self.train_data.append(patch_arr) self.train_labels.append(label_arr) self.train_data = np.expand_dims(self.train_data, axis=0) self.train_labels = np.expand_dims(self.train_labels, axis=0) self.train_data = np.squeeze(self.train_data, axis=0) self.train_labels = np.squeeze(self.train_labels, axis=0) entry = dict() entry['data'] = self.train_data entry['labels'] = self.train_labels pckl_file = path.join(pickle_dir, 'train_seg.pickle') if not path.isdir(pickle_dir): os.mkdir(pickle_dir) with open(pckl_file, 'wb') as pckl: pickle.dump(entry, pckl) else: self.test_data = [] self.test_labels = [] if load_pickle: for pckl in os.listdir(pickle_dir): if pckl.endswith(".pickle"): pckl_path = path.join(pickle_dir, pckl) pckl_file = open(pckl_path, 'rb') entry = pickle.load(pckl_file, encoding='latin1') self.test_data.append(entry['data']) self.test_labels.append(entry['labels']) pckl_file.close() self.test_data = np.concatenate(self.test_data) self.test_labels = np.concatenate(self.test_labels) else: for patch in os.listdir(test_dir): if patch.endswith('.png'): patch_path = path.join(test_dir, patch) base = path.splitext(patch)[0] label = base + '.dpt' label_path = path.join(label_dir, label) patch_file = Image.open(patch_path) patch_arr = np.array(patch_file) label_file = depth_read(label_path) label_arr = np.array(label_file) label_arr = label_arr + 4 #Fix from -4:10 to 0:14 if patch_arr.shape[0] == 436: patch_arr_extended = np.zeros((512, 1024, 3), dtype=np.uint8) patch_arr_extended[38:-38, :, :] = patch_arr patch_arr = patch_arr_extended label_arr_extended = np.zeros((512, 1024), dtype=np.uint8) label_arr_extended[38:-38, :] = label_arr label_arr = label_arr_extended # Make the classes 0-14 instead of 1-15 # label_arr -= 1 self.test_data.append(patch_arr) self.test_labels.append(label_arr) self.test_data = np.expand_dims(self.test_data, axis=0) self.test_labels = np.expand_dims(self.test_labels, axis=0) self.test_data = np.squeeze(self.test_data, axis=0) self.test_labels = np.squeeze(self.test_labels, axis=0) entry = dict() entry['data'] = self.test_data entry['labels'] = self.test_labels pckl_file = path.join(pickle_dir, 'test_seg.pickle') if not path.isdir(pickle_dir): os.mkdir(pickle_dir) with open(pckl_file, 'wb') as pckl: pickle.dump(entry, pckl)
def load_dpt(self, dpt_path): dpt = np.float32(sintel_io.depth_read(dpt_path))+ 1.0 # dpt = fake_depth() + 1.0 #dpt = np.float32(skimage.transform.resize(dpt, (480, 854))) return dpt
def depth_loader(path): # return rp.readPFM(path) return depth_read(path)
def get_depth_histogram(depth_dir): all_d = list() for file in os.listdir(depth_dir): d = depth_read(os.path.join(depth_dir,file)) all_d.append(np.concatenate(d)) np.histogram(all_d)
def evaluate_exp3(): print("Start of generating of png files, attached to paper") synthia_list = get_file_list(synthia_dir) synthia_depth_list = filter_files(synthia_list, ['Stereo_Left', 'Depth', 'png']) synthia_rgb_list = filter_files(synthia_list, ['Stereo_Left','RGB','png']) synthia_rgb = open_rgb(synthia_rgb_list[230]) synthia_depth = open_depth(synthia_depth_list[230]) synthia_rgb = cv2.resize(synthia_rgb, tuple(synthia_size[::-1])) synthia_depth = cv2.resize(synthia_depth, tuple(synthia_size[::-1])) mat_f = h5py.File(nyu_dir) NYU_depth_list = mat_f['depths'] NYU_rgb_list = mat_f['images'] nyu_rgb = NYU_rgb_list[18][None,:,:,:].transpose((0,3,2,1))[0] nyu_depth = NYU_depth_list[18].T nyu_rgb = cv2.resize(nyu_rgb, tuple(synthia_size[::-1])) nyu_depth = cv2.resize(nyu_depth, tuple(synthia_size[::-1])) sintel_depth_list = sorted(get_file_list(sintel_depth_dir)) sintel_rgb_list = sorted(get_file_list(sintel_rgb_dir)) sintel_rgb = open_rgb(sintel_rgb_list[169]) sintel_depth = depth_read(sintel_depth_list[169]) sintel_rgb = cv2.resize(sintel_rgb, tuple(synthia_size[::-1])) sintel_depth = cv2.resize(sintel_depth, tuple(synthia_size[::-1])) mask_uni = get_mask(synthia_size, 0.4) mask_reg = get_regular_grid_mask(synthia_size) mask_grad_synthia = combined_mask(synthia_rgb, synthia_size, combined = False) mask_comb_synthia = combined_mask(synthia_rgb, synthia_size, combined = True) print("Mask shapes: ",mask_uni.shape, mask_reg.shape, mask_grad_synthia.shape, mask_comb_synthia.shape) mask_grad_nyu = combined_mask(nyu_rgb, synthia_size, combined = False) mask_comb_nyu = combined_mask(nyu_rgb, synthia_size, combined = True) mask_grad_sintel = combined_mask(sintel_rgb, synthia_size, combined = False) mask_comb_sintel = combined_mask(sintel_rgb, synthia_size, combined = True) print("Passed stage of generating masks") tf.reset_default_graph() input_rgb = tf.placeholder(tf.float32, shape=(None,None,None, 3)) target = tf.placeholder(tf.float32, shape=(None,None,None, 1)) mask_t = tf.placeholder(tf.float32, shape=(None,None,None,1)) d_flg = tf.placeholder(tf.bool) y_true, gr_input, d_input, m = preprocess(input_rgb, target, mask_t, synthia_size, eps, maxi) G_output = G_Depth(d_input, m, d_flg) sess = tf.InteractiveSession() saver = tf.train.Saver() #saver.restore(sess, tf.train.latest_checkpoint(checkpoint_dir2)) synthia_masks = [mask_grad_synthia, mask_grad_synthia, mask_grad_synthia, mask_comb_synthia] nyu_masks = [mask_reg, mask_uni, mask_grad_nyu, mask_comb_nyu] sintel_masks = [mask_reg, mask_uni, mask_grad_sintel, mask_comb_sintel] model_list = [checkpoint_dir5, checkpoint_dir6, checkpoint_dir7, checkpoint_dir2] modell_list = [checkpoint_dir3, checkpoint_dir4, checkpoint_dir7, checkpoint_dir2] synthia_depth_outcome = [] nyu_depth_outcome = [] sintel_depth_outcome = [] #getting depth for each of mask for i in zip(synthia_masks,nyu_masks,sintel_masks, model_list, modell_list): saver.restore(sess, tf.train.latest_checkpoint(i[4])) synthia_temp = sess.run(G_output, feed_dict={input_rgb:synthia_rgb[None,:,:,:], target:synthia_depth[None,:,:,None], mask_t:i[0], d_flg:False}) saver.restore(sess, tf.train.latest_checkpoint(i[3])) nyu_temp = sess.run(G_output, feed_dict={input_rgb:nyu_rgb[None,:,:,:], target:nyu_depth[None,:,:,None], mask_t:i[1], d_flg:False}) sintel_temp = sess.run(G_output, feed_dict={input_rgb:sintel_rgb[None,:,:,:], target:sintel_depth[None,:,:,None], mask_t:i[2], d_flg:False}) print(synthia_temp.shape) synthia_depth_outcome.append(synthia_temp[0,:,:,0]) nyu_depth_outcome.append(nyu_temp[0,:,:,0]) sintel_depth_outcome.append(sintel_temp[0,:,:,0]) synthia_depth_outcome.insert(0, np.log(synthia_depth)) nyu_depth_outcome.insert(0, np.log(nyu_depth)) sintel_depth_outcome.insert(0, np.log(sintel_depth)) synthia_upper = [synthia_rgb.astype(int), mask_reg[0,:,:,0]*np.log(synthia_depth), mask_uni[0,:,:,0]*np.log(synthia_depth), mask_grad_synthia[0,:,:,0], mask_comb_synthia[0,:,:,0]] nyu_upper = [nyu_rgb.astype(int), mask_reg[0,:,:,0]*np.log(nyu_depth), mask_uni[0,:,:,0]*np.log(nyu_depth), mask_grad_nyu[0,:,:,0], mask_comb_nyu[0,:,:,0]] sintel_upper = [sintel_rgb.astype(int), mask_reg[0,:,:,0]*np.log(sintel_depth), mask_uni[0,:,:,0]*np.log(sintel_depth), mask_grad_sintel[0,:,:,0], mask_comb_sintel[0,:,:,0]] synthia_upper.extend(synthia_depth_outcome) nyu_upper.extend(nyu_depth_outcome) sintel_upper.extend(sintel_depth_outcome) print("Before final generate of synthia") cmap = sns.cubehelix_palette(1, light=1, as_cmap=True,reverse=True) fig, axes = plt.subplots(nrows=2, ncols=5) fig.set_size_inches(30,8) plt.setp(axes, xticks=[], yticks=[]) fig.tight_layout() for i in range(2): for j in range(5): axes[i,j].imshow(synthia_upper[i*5 + j], cmap) fig.savefig('Results/Dense/synthia_results.png', bbox_inches='tight') print("Before final generate of nyu") fig, axes = plt.subplots(nrows=2, ncols=5) fig.set_size_inches(30,8) plt.setp(axes, xticks=[], yticks=[]) fig.tight_layout() for i in range(2): for j in range(5): axes[i,j].imshow(nyu_upper[i*5 + j], cmap) fig.savefig('Results/Dense/nyu_results.png', bbox_inches='tight') print("Before final generate of sintel") fig, axes = plt.subplots(nrows=2, ncols=5) fig.set_size_inches(30,8) plt.setp(axes, xticks=[], yticks=[]) fig.tight_layout() for i in range(2): for j in range(5): axes[i,j].imshow(sintel_upper[i*5 + j], cmap) fig.savefig('Results/Dense/sintel_results.png', bbox_inches='tight') print("Finished generating png-s") print("Start of generating error_maps") saver.restore(sess, tf.train.latest_checkpoint(checkpoint_dir2)) rgb_list = [synthia_rgb[None,:,:,:], nyu_rgb[None,:,:,:], sintel_rgb[None,:,:,:]] depth_list = [synthia_depth[None,:,:,None], nyu_depth[None,:,:,None], sintel_depth[None,:,:,None]] errs = [] for rgb, d in zip(rgb_list, depth_list): mask_c = get_grad_mask(rgb, synthia_size, combined=True) out_c = sess.run(G_output, feed_dict={input_rgb:rgb, target:d, mask_t:mask_c, d_flg:False})[0,:,:,0] out_gt = resize(np.log(np.clip(d[0,:,:,0], 1e-3, 1e4)), synthia_size, preserve_range=True) errs.append(get_colors(abs(out_gt - out_c))) fig, axes = plt.subplots(nrows=1, ncols=3) fig.set_size_inches(15,5) plt.setp(axes, xticks=[], yticks=[]) fig.tight_layout() for i in range(3): axes[i].imshow(errs[i]) fig.savefig('Results/Dense/errors_maps.png', bbox_inches='tight') sess.close()