def generator(b_s, root_path, args=None, output_size=(480, 640)): imgs_path = root_path + '/Images/' maps_path = root_path + '/FixMaps/' fixs_path = root_path + '/FixPts/' images = [ imgs_path + f for f in os.listdir(imgs_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] maps = [ maps_path + f for f in os.listdir(maps_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [fixs_path + f for f in os.listdir(fixs_path) if f.endswith('.mat')] images.sort() maps.sort() fixs.sort() if args.dreloss: maps_path_td = root_path + '/FixMaps_TD/' fixs_path_td = root_path + '/FixPts_TD/' maps_TD = [ maps_path_td + f for f in os.listdir(maps_path_td) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs_TD = [ fixs_path_td + f for f in os.listdir(fixs_path_td) if f.endswith('.mat') ] maps_TD.sort() fixs_TD.sort() counter = 0 while True: Y = preprocess_maps(maps[counter:counter + b_s], output_size[0], output_size[1]) Y_fix = preprocess_fixmaps(fixs[counter:counter + b_s], output_size[0], output_size[1]) X = preprocess_images(images[counter:counter + b_s], args.input_size[0], args.input_size[1], 'channels_first') if args.dreloss: DOF = preprocess_dof(maps[counter:counter + b_s], maps_TD[counter:counter + b_s], output_size[0], output_size[1]) yield [X], [ Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, DOF ] else: yield [X], [ Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix ] counter = (counter + b_s) % len(images)
def generator(b_s, root_path, phase_gen='train', args=None): imgs_path = root_path + '/Images/' maps_path = root_path + '/FixMaps/' fixs_path = root_path + '/FixPts/' images = [ imgs_path + f for f in os.listdir(imgs_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] maps = [ maps_path + f for f in os.listdir(maps_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [fixs_path + f for f in os.listdir(fixs_path) if f.endswith('.mat')] images.sort() maps.sort() fixs.sort() gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt)) counter = 0 while True: Y = preprocess_maps(maps[counter:counter + b_s], shape_r_out, shape_c_out) # Single-side clip, valid when train on ASD dataset if args.ssc and phase_gen == 'train': Y[Y < 0.1] = 0 Y_fix = preprocess_fixmaps(fixs[counter:counter + b_s], shape_r_out, shape_c_out) X = preprocess_images(images[counter:counter + b_s], shape_r, shape_c) yield [X, gaussian], [ Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, # DOF, ] counter = (counter + b_s) % len(images)
def generator(b_s, phase_gen='train'): if phase_gen == 'train': images = [ imgs_train_path + f for f in os.listdir(imgs_train_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] maps = [ maps_train_path + f for f in os.listdir(maps_train_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [ fixs_train_path + f for f in os.listdir(fixs_train_path) if f.endswith('.mat') ] elif phase_gen == 'val': images = [ imgs_val_path + f for f in os.listdir(imgs_val_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] maps = [ maps_val_path + f for f in os.listdir(maps_val_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [ fixs_val_path + f for f in os.listdir(fixs_val_path) if f.endswith('.mat') ] else: raise NotImplementedError images.sort() maps.sort() fixs.sort() gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt)) counter = 0 while True: Y = preprocess_maps(maps[counter:counter + b_s], shape_r_out, shape_c_out) Y_fix = preprocess_fixmaps(fixs[counter:counter + b_s], shape_r_out, shape_c_out) yield [ preprocess_images(images[counter:counter + b_s], shape_r, shape_c), gaussian ], [Y, Y, Y_fix] counter = (counter + b_s) % len(images)
def generator(video_b_s, image_b_s, phase_gen='train'): if phase_gen == 'train': videos = [ videos_train_path + f for videos_train_path in videos_train_paths for f in os.listdir(videos_train_path) if os.path.isdir(videos_train_path + f) ] images = [ imgs_path + 'images/' + f for f in os.listdir(imgs_path + 'images/') if f.endswith(('.jpg', '.jpeg', '.png')) ] fixationmaps = [ imgs_path + 'maps/' + f for f in os.listdir(imgs_path + 'maps/') if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [ imgs_path + 'fixation/maps/' + f for f in os.listdir(imgs_path + 'fixation/maps/') if f.endswith(('.mat')) ] images.sort() fixationmaps.sort() fixs.sort() image_train_data = [] for image, fixationmap, fix in zip(images, fixationmaps, fixs): annotation_data = { 'image': image, 'fixmap': fixationmap, 'fix': fix } image_train_data.append(annotation_data) random.shuffle(image_train_data) # videos.sort() random.shuffle(videos) loop = 1 video_counter = 0 image_counter = 0 while True: if loop % 2: Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3)) Ymaps = np.zeros((video_b_s, num_frames, shape_r_out, shape_c_out, 1)) + 0.01 Yfixs = np.zeros((video_b_s, num_frames, shape_r_out, shape_c_out, 1)) + 0.01 Img_Ymaps = np.zeros((video_b_s, num_frames, shape_r_attention, shape_c_attention, 1)) + 0.01 Img_Yfixs = np.zeros((video_b_s, num_frames, shape_r_attention, shape_c_attention, 1)) + 0.01 for i in range(0, video_b_s): video_path = videos[(video_counter + i) % len(videos)] images = [ video_path + frames_path + f for f in os.listdir(video_path + frames_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] maps = [ video_path + maps_path + f for f in os.listdir(video_path + maps_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [ video_path + fixs_path + f for f in os.listdir(video_path + fixs_path) if f.endswith('.mat') ] # images.sort() # maps.sort() # fixs.sort() start = random.randint(0, max(len(images) - num_frames, 0)) X = preprocess_images( images[start:min(start + num_frames, len(images))], shape_r, shape_c) Y = preprocess_maps( maps[start:min(start + num_frames, len(images))], shape_r_out, shape_c_out) Y_fix = preprocess_fixmaps( fixs[start:min(start + num_frames, len(images))], shape_r_out, shape_c_out) Xims[i, 0:X.shape[0], :] = np.copy(X) Ymaps[i, 0:Y.shape[0], :] = np.copy(Y) Yfixs[i, 0:Y_fix.shape[0], :] = np.copy(Y_fix) Xims[i, X.shape[0]:num_frames, :] = np.copy(X[-1, :, :]) Ymaps[i, Y.shape[0]:num_frames, :] = np.copy(Y[-1, :, :]) Yfixs[i, Y_fix.shape[0]:num_frames, :] = np.copy( Y_fix[-1, :, :]) yield Xims, [ Ymaps, Ymaps, Yfixs, Img_Ymaps, Img_Ymaps, Img_Yfixs ] # video_counter = (video_counter + video_b_s) % len(videos) loop = loop + 1 else: Xims = np.zeros((image_b_s, 1, shape_r, shape_c, 3)) Ymaps = np.zeros( (image_b_s, 1, shape_r_out, shape_c_out, 1)) + 0.01 Yfixs = np.zeros( (image_b_s, 1, shape_r_out, shape_c_out, 1)) + 0.01 Img_Ymaps = np.zeros((image_b_s, 1, shape_r_attention, shape_c_attention, 1)) + 0.01 Img_Yfixs = np.zeros((image_b_s, 1, shape_r_attention, shape_c_attention, 1)) + 0.01 for i in range(0, image_b_s): img_data = image_train_data[(image_counter + i) % len(image_train_data)] X = preprocess_images([img_data['image']], shape_r, shape_c) Y = preprocess_maps([img_data['fixmap']], shape_r_attention, shape_c_attention) Y_fix = preprocess_fixmaps([img_data['fix']], shape_r_attention, shape_c_attention) Xims[i, 0, :] = np.copy(X) Img_Ymaps[i, 0, :] = np.copy(Y) Img_Yfixs[i, 0, :] = np.copy(Y_fix) yield Xims, [ Ymaps, Ymaps, Yfixs, Img_Ymaps, Img_Ymaps, Img_Yfixs ] # # image_counter = (image_counter + image_b_s) % len(image_train_data) loop = loop + 1 elif phase_gen == 'val': videos = [ videos_val_path + f for videos_val_path in videos_val_paths for f in os.listdir(videos_val_path) if os.path.isdir(videos_val_path + f) ] random.shuffle(videos) video_counter = 0 while True: Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3)) Ymaps = np.zeros( (video_b_s, num_frames, shape_r_out, shape_c_out, 1)) + 0.01 Yfixs = np.zeros( (video_b_s, num_frames, shape_r_out, shape_c_out, 1)) + 0.01 Img_Ymaps = np.zeros((video_b_s, num_frames, shape_r_attention, shape_c_attention, 1)) + 0.01 Img_Yfixs = np.zeros((video_b_s, num_frames, shape_r_attention, shape_c_attention, 1)) + 0.01 for i in range(0, video_b_s): video_path = videos[(video_counter + i) % len(videos)] images = [ video_path + frames_path + f for f in os.listdir(video_path + frames_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] maps = [ video_path + maps_path + f for f in os.listdir(video_path + maps_path) if f.endswith(('.jpg', '.jpeg', '.png')) ] fixs = [ video_path + fixs_path + f for f in os.listdir(video_path + fixs_path) if f.endswith('.mat') ] start = random.randint(0, max(len(images) - num_frames, 0)) X = preprocess_images( images[start:min(start + num_frames, len(images))], shape_r, shape_c) Y = preprocess_maps( maps[start:min(start + num_frames, len(images))], shape_r_out, shape_c_out) Y_fix = preprocess_fixmaps( fixs[start:min(start + num_frames, len(images))], shape_r_out, shape_c_out) Xims[i, 0:X.shape[0], :] = np.copy(X) Ymaps[i, 0:Y.shape[0], :] = np.copy(Y) Yfixs[i, 0:Y_fix.shape[0], :] = np.copy(Y_fix) Xims[i, X.shape[0]:num_frames, :] = np.copy(X[-1, :, :]) Ymaps[i, Y.shape[0]:num_frames, :] = np.copy(Y[-1, :, :]) Yfixs[i, Y_fix.shape[0]:num_frames, :] = np.copy(Y_fix[-1, :, :]) yield Xims, [Ymaps, Ymaps, Yfixs, Img_Ymaps, Img_Ymaps, Img_Yfixs] # video_counter = (video_counter + video_b_s) % len(videos) else: raise NotImplementedError
def _VideoBatch_Generator(phase_gen='train'): ''' Args: VideoCap: cv2预读取视频的操作 batch_size: batch的大小 frame_num: 一个视频的连续帧 resize_shape: 输出视频帧的大小 Out: video_batch: (batch_size, frame_num, width, height, RGB) ''' VideoCap = cv2.VideoCapture(VideoName) #frame_width = VideoCap.get(3) #3:CV_CAP_PROP_FRAME_HEIGHT #frame_height = VideoCap.get(4) #4:CV_CAP_PROP_FRAME_HEIGHT if phase_gen == 'test': Video_frames = VideoCap.get(7) #7:CV_CAP_PROP_FRAME_COUNT print(Video_frames) batch_size = Video_frames // frame_num if not Video_frames % frame_num == 0: batch_size = batch_size + 1 print(batch_size) start_frame = 0 Video_Batch = np.zeros(shape=[1, 1, 720, 1080, 3]) for j in range(int(batch_size)): Video_Slice = np.zeros(shape=[1, 1, 720, 1080, 3]) if start_frame + frame_num > Video_frames: start_frame = Video_frames - frame_num stop_frame = start_frame + frame_num for i in range(start_frame, stop_frame): VideoCap.set(1, i) #1:CV_CAP_PROP_POS_FRAMES _, frame = VideoCap.read() #(frame_width, frame_height, BGR) frame = cv2.resize(frame, input_shape) frame = frame.astype(np.float32) frame = cv2.cvtColor( frame, cv2.COLOR_BGR2RGB) #(resize_width, resize_height, RGB) frame = frame[np.newaxis, np.newaxis, ...] if np.shape(Video_Slice)[1] == 1 and i == start_frame: Video_Slice = frame else: Video_Slice = np.concatenate((Video_Slice, frame), axis=1) print(np.shape(Video_Slice)) if np.shape(Video_Batch)[0] == 1 and start_frame == 0: Video_Batch = Video_Slice else: Video_Batch = np.concatenate((Video_Batch, Video_Slice), axis=0) print(np.shape(Video_Batch)) start_frame = stop_frame #training process 读取的是文件夹里的照片 # video_trainpaths里包含的是按数字标号保存的video,每个video文件家里三个文件夹 “images”() “maps” if phase_gen == 'train': videos = [ Video_train_paths + '/' + video_train_path for video_train_path in os.listdir(Video_train_paths) if os.path.isdir(Video_train_paths + '/' + video_train_path) ] videos.sort() random.shuffle(videos) range_num = 0 while True: Xims = np.zeros( (batch_size, frame_num, input_shape[0], input_shape[1], 3)) Ymaps = np.zeros((batch_size, frame_num, output_shape[0], output_shape[1], 1)) + 0.01 Yfixs = np.zeros((batch_size, frame_num, output_shape[0], output_shape[1], 1)) + 0.01 for i in range(batch_size): video = videos[(range_num * batch_size + i) % len(videos)] images = [ video + frames_path + f for f in os.listdir(video + frames_path) if f.endswith((".jpg", ".jpeg", ".png")) ] maps = [ video + maps_path + f for f in os.listdir(video + maps_path) if f.endswith((".jpg", ".jpeg", "png")) ] fixs = [ video + fixs_path + f for f in os.listdir(video + fixs_path) if f.endswith(".mat") ] start = np.random.choice(max(1, len(images) - frame_num)) X = preprocess_images( images[start:min((start + frame_num), len(images))], input_shape[0], input_shape[1]) Y = preprocess_maps( maps[start:min((start + frame_num), len(images))], output_shape[0], output_shape[1]) Y_fix = preprocess_fixmaps( fixs[start:min((start + frame_num), len(images))], output_shape[0], output_shape[1]) Xims[i, 0:np.shape(X)[0], :] = np.copy(X) Ymaps[i, 0:np.shape(Y)[0], :] = np.copy(Y) Yfixs[i, 0:np.shape(Y_fix)[0], :] = np.copy(Y_fix) Xims[i, X.shape[0]:frame_num, :] = np.copy(X[-1, :, :]) Ymaps[i, Y.shape[0]:frame_num, :] = np.copy(Y[-1, :, :]) Yfixs[i, Y_fix.shape[0]:frame_num, :] = np.copy(Y_fix[-1, :, :]) yield Xims, Ymaps, Yfixs range_num = range_num + 1
def generator_data(video_b_s, phase_gen='train'): num_frames = input_t if phase_gen == 'train': train_pth = os.path.join(train_data_pth, '*', 'images') images_seq = glob.glob(train_pth) datas = [] for image_pth in images_seq: images = sorted(glob.glob(image_pth + '/*')) segs = [xx.replace('images', 'semantic') for xx in images] maps = [xx.replace('images', 'maps') for xx in images] fixs = [xx.replace('images', 'fixation/maps') for xx in images] fixs = [xx.replace('.png', '.mat') for xx in fixs] for jj in range(len(images) - input_t): datas.append((images[jj:jj + input_t], segs[jj:jj + input_t], [ maps[jj + input_t], ], [ fixs[jj + input_t], ])) counts = 0 print('len -> train data :', len(datas), 'batch_data ->:', len(datas) // video_b_s) random.shuffle(datas) while True: if counts >= (len(datas) - video_b_s): random.shuffle(datas) counts = counts % (len(datas) - video_b_s) Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3)) Xsegs = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3)) Ymaps = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001 Yfixs = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001 for i in range(0, video_b_s): X1 = preprocess_images(datas[counts][0], shape_r, shape_c) X2 = preprocess_images(datas[counts][1], shape_r, shape_c) Y = preprocess_maps(datas[counts][2], shape_r_out, shape_c_out) Y_fix = preprocess_fixmaps(datas[counts][3], shape_r_out, shape_c_out) Xims[i, :] = np.copy(X1) Xsegs[i, :] = np.copy(X2) Ymaps[i, :] = np.copy(Y[0]) Yfixs[i, :] = np.copy(Y_fix[0]) # --------------------------------------------------- counts += 1 yield [Xims, Xsegs], [Ymaps, Ymaps, Yfixs] # else: val_pth = os.path.join(val_data_pth, '*', 'images') images_seq = glob.glob(val_pth) datas = [] for image_pth in images_seq: images = sorted(glob.glob(image_pth + '/*')) segs = [xx.replace('images', 'semantic') for xx in images] maps = [xx.replace('images', 'maps') for xx in images] fixs = [xx.replace('images', 'fixation/maps') for xx in images] fixs = [xx.replace('.png', '.mat') for xx in fixs] for jj in range(len(images) - input_t): datas.append((images[jj:jj + input_t], segs[jj:jj + input_t], [ maps[jj + input_t], ], [ fixs[jj + input_t], ])) counts = 0 print('len -> val data :', len(datas), 'batch_data ->:', len(datas) // video_b_s) random.shuffle(datas) while True: if counts >= (len(datas) - video_b_s): random.shuffle(datas) counts = counts % (len(datas) - video_b_s) Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3)) Xsegs = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3)) Ymaps = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001 Yfixs = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001 for i in range(0, video_b_s): X1 = preprocess_images(datas[counts][0], shape_r, shape_c) X2 = preprocess_images(datas[counts][1], shape_r, shape_c) Y = preprocess_maps(datas[counts][2], shape_r_out, shape_c_out) Y_fix = preprocess_fixmaps(datas[counts][3], shape_r_out, shape_c_out) Xims[i, :] = np.copy(X1) Xsegs[i, :] = np.copy(X2) Ymaps[i, :] = np.copy(Y[0]) Yfixs[i, :] = np.copy(Y_fix[0]) # --------------------------------------------------- counts += 1 yield [Xims, Xsegs], [Ymaps, Ymaps, Yfixs] #