def load_dreyeve_sample(sequence_dir, sample, mean_dreyeve_image, frames_per_seq=16, h=448, w=448, ): """ Function to load a dreyeve_sample. :param sequence_dir: string, sequence directory (e.g. 'Z:/DATA/04/'). :param sample: int, sample to load in (15, 7499). N.B. this is the sample where prediction occurs! :param mean_dreyeve_image: mean dreyeve image, subtracted to each frame. :param frames_per_seq: number of temporal frames for each sample :param h: h :param w: w :return: a dreyeve_sample like I, OF, SEG """ h_c = h_s = h // 4 w_c = w_s = h // 4 I_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32') I_s = np.zeros(shape=(1, frames_per_seq, h_s, w_s,3), dtype='float32') I_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32') SEG_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32') SEG_s = np.zeros(shape=(1,frames_per_seq, h_s, w_s,3), dtype='float32') SEG_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32') OF_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32') OF_s = np.zeros(shape=(1, frames_per_seq, h_s, w_s,3), dtype='float32') OF_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32') for fr in range(0, frames_per_seq): offset = sample - frames_per_seq + 1 + fr # tricky # read image x = read_image(join(sequence_dir,'Frames','{}.jpg'.format(offset)), channels_first=False, resize_dim=(h, w)) - mean_dreyeve_image I_s[0, fr, :, :,:] = resize_tensor(x,new_shape=(h_s, w_s)) # read semseg seg = resize_tensor(np.load(join(sequence_dir,'Seg', '{}.npz'.format(offset)))['arr_0'], new_shape=(h, w)) SEG_s[0,fr,:, :, :] = resize_tensor(seg, new_shape=(h_s, w_s)) # read of of = read_image(join(sequence_dir,'Optical', '{}.png'.format(offset + 1)), channels_first=False, resize_dim=(h, w)) of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean OF_s[0, fr, :, :, :] = resize_tensor(of, new_shape=(h_s, w_s)) I_ff[0,0, :, :,:] = x SEG_ff[0,0, :, :, :] = seg OF_ff[0, 0, :, :, :] = of return [I_ff, I_s,I_c,OF_ff,OF_s,OF_c,SEG_ff, SEG_s, SEG_c]
def load_dreyeve_sample( frames_list, sample, mean_dreyeve_image, frames_per_seq=16, h=448, w=448, ): h_c = h_s = h // 4 w_c = w_s = h // 4 I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32') I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32') I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32') for fr in range(frames_per_seq): offset = sample - frames_per_seq + 1 + fr # tricky x_in = cv2.resize(frames_list[offset], dsize=resize_dim_in[::-1], interpolation=cv2.INTER_LINEAR) x_in = np.transpose(x_in, (2, 0, 1)).astype(np.float32) x_in -= mean_dreyeve_image I_s[0, :, fr, :, :] = resize_tensor(x_in, new_size=(h_s, w_s)) x_disp = cv2.resize(frames_list[offset], dsize=resize_dim_disp[::-1], interpolation=cv2.INTER_LINEAR) I_ff[0, :, 0, :, :] = x_in return [I_ff, I_s, I_c], x_disp
def load_dreyeve_sample(sequence_dir, sample, mean_dreyeve_image, frames_per_seq=16, h=448, w=448): h_c = h_s = h // 4 w_c = w_s = h // 4 I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32') I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32') I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32') for fr in range(frames_per_seq): offset = sample - frames_per_seq + 1 + fr # read image x = read_image(join(sequence_dir, 'frames', '{}.jpg'.format(offset)), channels_first=True, resize_dim=(h, w)) - mean_dreyeve_image I_s[0, :, fr, :, :] = resize_tensor(x, new_size=(h_s, w_s)) I_ff[0, :, 0, :, :] = x return [I_ff, I_s, I_c]
def load_saliency_data_simo(signatures, nb_frames, image_size): """ Crop -> NSS with fixations Fine -> KLD with saliency maps and IG with fixations (baseline is saliency maps) :param signatures: sample signatures, previously evaluated. List of tuples like (num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long. :param nb_frames: number of temporal frames in each sample. :param image_size: tuple in the form (h,w). This refers to the fullframe image. :return: a tuple holding the fullframe and the cropped saliency. """ batchsize = len(signatures) h, w = image_size h_c = h // 4 w_c = w // 4 Y = np.zeros(shape=(batchsize, 2, h, w), dtype=np.float32) Y_c = np.zeros(shape=(batchsize, 1, h_c, w_c), dtype=np.float32) for b in range(0, batchsize): # retrieve the signature num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b] y_dir_sal = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency') y_dir_fix = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency_fix') # saliency y_sal = read_image(join(y_dir_sal, '{:06d}.png'.format(start + nb_frames - 1)), channels_first=True, color=False, resize_dim=image_size) / 255 y_fix = read_image(join(y_dir_fix, '{:06d}.png'.format(start + nb_frames - 1)), channels_first=True, color=False, resize_dim=image_size) / 255 # resize to (256, 256) before cropping y_before_crop = resize_tensor(np.expand_dims(y_fix, axis=0), new_shape=frame_size_before_crop) Y[b, 0, :, :] = y_sal Y[b, 1, :, :] = y_fix Y_c[b, 0, :, :] = crop_tensor(y_before_crop, indexes=(hc1, hc2, wc1, wc2))[0] if do_mirror: Y = Y[:, :, :, ::-1] Y_c = Y_c[:, :, :, ::-1] return [Y, Y_c]
def load_saliency_data(signatures, nb_frames, image_size, gt_type='fix'): """ Function to load a saliency batch. :param signatures: sample signatures, previously evaluated. List of tuples like (num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long. :param nb_frames: number of temporal frames in each sample. :param image_size: tuple in the form (h,w). This refers to the fullframe image. :param gt_type: choose among `sal` (old groundtruth) and `fix` (new groundtruth). :return: a tuple holding the fullframe and the cropped saliency. """ batchsize = len(signatures) h, w = image_size h_c = h // 4 w_c = w // 4 Y = np.zeros(shape=(batchsize, h, w, 1), dtype=np.float32) Y_c = np.zeros(shape=(batchsize, h_c, w_c, 1), dtype=np.float32) for b in range(0, batchsize): # retrieve the signature num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b] y_dir = join(dreyeve_dir, "DREYEVE_DATA", '{:02d}'.format(num_run), 'output_frames' if gt_type == 'sal' else 'output_frames') # saliency y = read_image(join(y_dir, '{}.jpg'.format(start + nb_frames - 1)), channels_first=False, color=False, resize_dim=image_size) # resize to (256, 256) before cropping y_before_crop = resize_tensor(np.expand_dims(y, axis=2), new_shape=frame_size_before_crop) Y[b, :, :, 0] = y Y_c[b, :, :, 0] = crop_tensor(y_before_crop, indexes=(hc1, hc2, wc1, wc2))[0] if do_mirror: Y = Y[:, :, :, ::-1] Y_c = Y_c[:, :, :, ::-1] return [Y, Y_c]
def load_batch_data(signatures, nb_frames, image_size, batch_type): """ Function to load a data batch. This is common for `image`, `optical_flow` and `semseg`. :param signatures: sample signatures, previously evaluated. List of tuples like (num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long. :param nb_frames: number of temporal frames in each sample. :param image_size: tuple in the form (h,w). This refers to the fullframe image. :param batch_type: choose among [`image`, `optical_flow`, `semseg`]. :return: a tuple holding the fullframe, the small and the cropped batch. """ assert batch_type in ['image', 'optical_flow', 'semseg' ], 'Unknown batch type: {}'.format(batch_type) batchsize = len(signatures) h, w = image_size h_s = h_c = h // 4 w_s = w_c = w // 4 if batch_type == 'image': B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32) B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3), dtype=np.float32) B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3), dtype=np.float32) subdir = 'DREYEVE_DATA' # mean frame image, to subtract mean elif batch_type == 'optical_flow': B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32) B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3), dtype=np.float32) B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3), dtype=np.float32) subdir = 'optical_flow' elif batch_type == 'semseg': B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32) B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3), dtype=np.float32) B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3), dtype=np.float32) subdir = 'Segmentation' for b in range(batchsize): # retrieve the signature num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b] data_dir = join(dreyeve_dir, subdir, '{:02d}'.format(num_run)) mean_image = read_image(join(dreyeve_dir, 'DREYEVE_DATA', '{:02d}'.format(num_run), 'mean_frame.png'), channels_first=False, resize_dim=image_size) for offset in range(0, nb_frames): if batch_type == 'image': x = read_image(join(data_dir, 'Frames', '{}.jpg'.format(start + offset)), channels_first=False, resize_dim=image_size) - mean_image # resize to (256, 256) before cropping x_before_crop = resize_tensor(x, new_shape=frame_size_before_crop) B_s[b, offset, :, :, :] = resize_tensor(x, new_shape=(h_s, w_s)) B_c[b, offset, :, :, :] = crop_tensor(x_before_crop, indexes=(hc1, hc2, wc1, wc2)) elif batch_type == 'optical_flow': x = read_image(join(data_dir, '{}.png'.format(start + offset + 1)), channels_first=False, resize_dim=image_size) x -= np.mean(x, axis=(1, 2), keepdims=True) # remove mean # resize to (256, 256) before cropping x_before_crop = resize_tensor(x, new_shape=frame_size_before_crop) B_s[b, offset, :, :, :] = resize_tensor(x, new_shape=(h_s, w_s)) B_c[b, offset, :, :, :] = crop_tensor(x_before_crop, indexes=(hc1, hc2, wc1, wc2)) elif batch_type == 'semseg': x = resize_tensor(np.load( join(data_dir, '{}.npz'.format(start + offset)))['arr_0'], new_shape=image_size) # resize to (256, 256) before cropping x_before_crop = resize_tensor(x, new_shape=frame_size_before_crop) B_s[b, offset, :, :, :] = resize_tensor(x, new_shape=(h_s, w_s)) B_c[b, offset, :, :, :] = crop_tensor(x_before_crop, indexes=(hc1, hc2, wc1, wc2)) B_ff[b, 0, :, :, :] = x if do_mirror: B_ff = B_ff[:, :, :, :, ::-1] B_s = B_s[:, :, :, :, ::-1] B_c = B_c[:, :, :, :, ::-1] return [B_ff, B_s, B_c]
def roll(self): self.fr += 1 self.I_s, self.I_c, self.OF_s, self.OF_c, self.SEG_s, self.SEG_c, \ self.Il_s, self.Il_c, self.OFl_s, self.OFl_c, self.SEGl_s, self.SEGl_c, \ self.Ir_s, self.Ir_c, self.OFr_s, self.OFr_c, self.SEGr_s, self.SEGr_c = [ np.roll(x, -1, axis=2) for x in [self.I_s, self.I_c, self.OF_s, self.OF_c, self.SEG_s, self.SEG_c, self.Il_s, self.Il_c, self.OFl_s, self.OFl_c, self.SEGl_s, self.SEGl_c, self.Ir_s, self.Ir_c, self.OFr_s, self.OFr_c, self.SEGr_s, self.SEGr_c]] # read image x = read_image(join(self.sequence_dir, 'frames', '{:06d}.jpg'.format(self.fr)), channels_first=True, resize_dim=(self.h, self.w)) \ - self.mean_dreyeve_image xl = translate(x, pixels=116, side='left') xr = translate(x, pixels=116, side='right') self.I_s[0, :, -1, :, :] = resize_tensor(x, new_size=(self.h_s, self.w_s)) self.Il_s[0, :, -1, :, :] = resize_tensor(xl, new_size=(self.h_s, self.w_s)) self.Ir_s[0, :, -1, :, :] = resize_tensor(xr, new_size=(self.h_s, self.w_s)) # read of of = read_image(join(self.sequence_dir, 'optical_flow', '{:06d}.png'.format(self.fr + 1)), channels_first=True, resize_dim=(h, w)) of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean ofl = translate(of, pixels=116, side='left') ofr = translate(of, pixels=116, side='right') self.OF_s[0, :, -1, :, :] = resize_tensor(of, new_size=(self.h_s, self.w_s)) self.OFl_s[0, :, -1, :, :] = resize_tensor(ofl, new_size=(self.h_s, self.w_s)) self.OFr_s[0, :, -1, :, :] = resize_tensor(ofr, new_size=(self.h_s, self.w_s)) # read semseg seg = resize_tensor(np.load( join(self.sequence_dir, 'semseg', '{:06d}.npz'.format(self.fr)))['arr_0'][0], new_size=(h, w)) segl = translate(seg, pixels=116, side='left') segr = translate(seg, pixels=116, side='right') self.SEG_s[0, :, -1, :, :] = resize_tensor(seg, new_size=(self.h_s, self.w_s)) self.SEGl_s[0, :, -1, :, :] = resize_tensor(segl, new_size=(self.h_s, self.w_s)) self.SEGr_s[0, :, -1, :, :] = resize_tensor(segr, new_size=(self.h_s, self.w_s)) self.I_ff[0, :, 0, :, :] = x self.Il_ff[0, :, 0, :, :] = xl self.Ir_ff[0, :, 0, :, :] = xr self.OF_ff[0, :, 0, :, :] = of self.OFl_ff[0, :, 0, :, :] = ofl self.OFr_ff[0, :, 0, :, :] = ofr self.SEG_ff[0, :, 0, :, :] = seg self.SEGl_ff[0, :, 0, :, :] = segl self.SEGr_ff[0, :, 0, :, :] = segr y_sal = read_image(join(self.sequence_dir, 'saliency', '{:06d}.png'.format(self.fr)), channels_first=False, color=False) yl_sal = translate(y_sal, pixels=500, side='left') yr_sal = translate(y_sal, pixels=500, side='right') self.Y_sal[0, 0] = y_sal self.Yl_sal[0, 0] = yl_sal self.Yr_sal[0, 0] = yr_sal y_fix = read_image(join(self.sequence_dir, 'saliency_fix', '{:06d}.png'.format(self.fr)), channels_first=False, color=False) yl_fix = translate(y_fix, pixels=500, side='left') yr_fix = translate(y_fix, pixels=500, side='right') self.Y_fix[0, 0] = y_fix self.Yl_fix[0, 0] = yl_fix self.Yr_fix[0, 0] = yr_fix
def load_dreyeve_sample(sequence_dir, stop, mean_dreyeve_image): """ Function to load a dreyeve_sample. :param sequence_dir: string, sequence directory (e.g. 'Z:/DATA/04/'). :param stop: int, sample to load in (15, 7499). N.B. this is the sample where prediction occurs! :param mean_dreyeve_image: mean dreyeve image, subtracted to each frame. :param frames_per_seq: number of temporal frames for each sample :param h: h :param w: w :return: a dreyeve_sample like I, OF, SEG """ h_c = h_s = h // 4 w_c = w_s = h // 4 I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32') I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32') I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32') OF_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32') OF_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32') OF_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32') SEG_ff = np.zeros(shape=(1, 19, 1, h, w), dtype='float32') SEG_s = np.zeros(shape=(1, 19, frames_per_seq, h_s, w_s), dtype='float32') SEG_c = np.zeros(shape=(1, 19, frames_per_seq, h_c, w_c), dtype='float32') Y_sal = np.zeros(shape=(1, 1, h, w), dtype='float32') Y_fix = np.zeros(shape=(1, 1, h, w), dtype='float32') for fr in xrange(0, frames_per_seq): offset = stop - frames_per_seq + 1 + fr # tricky # read image x = read_image(join(sequence_dir, 'frames', '{:06d}.jpg'.format(offset)), channels_first=True, resize_dim=(h, w)) - mean_dreyeve_image I_s[0, :, fr, :, :] = resize_tensor(x, new_size=(h_s, w_s)) # read of of = read_image(join(sequence_dir, 'optical_flow', '{:06d}.png'.format(offset + 1)), channels_first=True, resize_dim=(h, w)) of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean OF_s[0, :, fr, :, :] = resize_tensor(of, new_size=(h_s, w_s)) # read semseg seg = resize_tensor(np.load( join(sequence_dir, 'semseg', '{:06d}.npz'.format(offset)))['arr_0'][0], new_size=(h, w)) SEG_s[0, :, fr, :, :] = resize_tensor(seg, new_size=(h_s, w_s)) I_ff[0, :, 0, :, :] = x OF_ff[0, :, 0, :, :] = of SEG_ff[0, :, 0, :, :] = seg Y_sal[0, 0] = read_image(join(sequence_dir, 'saliency', '{:06d}.png'.format(stop)), channels_first=False, color=False, resize_dim=(h, w)) Y_fix[0, 0] = read_image(join(sequence_dir, 'saliency_fix', '{:06d}.png'.format(stop)), channels_first=False, color=False, resize_dim=(h, w)) return [I_ff, I_s, I_c, OF_ff, OF_s, OF_c, SEG_ff, SEG_s, SEG_c], [Y_sal, Y_fix]
normalize(X[3][0, :, 0, :, :].transpose(1, 2, 0)), normalize( seg_to_colormap(np.argmax(X[6][0, :, 0, :, :], axis=0), channels_first=False)) ], layout=(3, 1), resize_dim=(720, 720)) y_stitch = stitch_together([ np.tile(normalize(Y_image[0].transpose(1, 2, 0)), reps=(1, 1, 3)), np.tile(normalize(Y_flow[0].transpose(1, 2, 0)), reps=(1, 1, 3)), np.tile(normalize(Y_semseg[0].transpose(1, 2, 0)), reps=(1, 1, 3)) ], layout=(3, 1), resize_dim=(720, 720)) y_tot = np.tile(normalize( resize_tensor(Y_dreyevenet[0], new_size=(720, 720)).transpose(1, 2, 0)), reps=(1, 1, 3)) cv2.imshow( 'prediction', stitch_together([x_stitch, y_stitch, y_tot], layout=(1, 3), resize_dim=(500, 1500))) cv2.waitKey(1)