def generator_test(b_s,
imgs_test_path,
large_scale_dataset=False,
group=0,
th=500,
args=None):
images = [
imgs_test_path + f for f in os.listdir(imgs_test_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
images.sort()
counter = 0
if large_scale_dataset:
assert group != 0
start = (group - 1) * th
end_ = min(group * th, len(images))
images = images[start:end_]
while True:
yield [
preprocess_images(images[counter:counter + b_s],
args.input_size[0], args.input_size[1])
]
counter = (counter + b_s) % len(images)
else:
while True:
yield [
preprocess_images(images[counter:counter + b_s],
args.input_size[0], args.input_size[1])
]
counter = (counter + b_s) % len(images)
def generator_test(b_s,
imgs_test_path,
large_scale_dataset=False,
group=0,
th=500):
images = [
imgs_test_path + f for f in os.listdir(imgs_test_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
images.sort()
counter = 0
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
if large_scale_dataset:
assert group != 0
start = (group - 1) * th
end_ = min(group * th, len(images))
images = images[start:end_]
images_extend = images + images
while True:
yield [
preprocess_images(images_extend[counter:counter + b_s],
shape_r, shape_c), gaussian
]
counter = (counter + b_s) % len(images)
else:
images_extend = images + images
while True:
yield [
preprocess_images(images_extend[counter:counter + b_s],
shape_r, shape_c), gaussian
]
counter = (counter + b_s) % len(images)
def generator(b_s, phase_gen='train'):
if phase_gen == 'train':
images = [
imgs_train_path + f for f in os.listdir(imgs_train_path)
if f.endswith('.jpg')
]
maps = [
maps_train_path + f for f in os.listdir(maps_train_path)
if f.endswith('.jpg')
]
elif phase_gen == 'val':
images = [
imgs_val_path + f for f in os.listdir(imgs_val_path)
if f.endswith('.jpg')
]
maps = [
maps_val_path + f for f in os.listdir(maps_val_path)
if f.endswith('.jpg')
]
else:
raise NotImplementedError
images.sort()
maps.sort()
counter = 0
while True:
yield preprocess_images(images[counter:counter + b_s], shape_r,
shape_c), preprocess_maps(
maps[counter:counter + b_s], shape_r_gt,
shape_c_gt)
counter = (counter + b_s) % len(images)
def generator(b_s, root_path, args=None, output_size=(480, 640)):
imgs_path = root_path + '/Images/'
maps_path = root_path + '/FixMaps/'
fixs_path = root_path + '/FixPts/'
images = [
imgs_path + f for f in os.listdir(imgs_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps = [
maps_path + f for f in os.listdir(maps_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [fixs_path + f for f in os.listdir(fixs_path) if f.endswith('.mat')]
images.sort()
maps.sort()
fixs.sort()
if args.dreloss:
maps_path_td = root_path + '/FixMaps_TD/'
fixs_path_td = root_path + '/FixPts_TD/'
maps_TD = [
maps_path_td + f for f in os.listdir(maps_path_td)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs_TD = [
fixs_path_td + f for f in os.listdir(fixs_path_td)
if f.endswith('.mat')
]
maps_TD.sort()
fixs_TD.sort()
counter = 0
while True:
Y = preprocess_maps(maps[counter:counter + b_s], output_size[0],
output_size[1])
Y_fix = preprocess_fixmaps(fixs[counter:counter + b_s], output_size[0],
output_size[1])
X = preprocess_images(images[counter:counter + b_s],
args.input_size[0], args.input_size[1],
'channels_first')
if args.dreloss:
DOF = preprocess_dof(maps[counter:counter + b_s],
maps_TD[counter:counter + b_s],
output_size[0], output_size[1])
yield [X], [
Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y,
Y_fix, DOF
]
else:
yield [X], [
Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix, Y, Y, Y_fix
]
counter = (counter + b_s) % len(images)
def compute_saliency(image_path):
if use_default_center_bias:
predictions = m.predict(
[preprocess_images([image_path], shape_r, shape_c),
gaussian])[0]
else:
predictions = m.predict(
[preprocess_images([image_path], shape_r, shape_c),
gaussian])[0]
original_image = cv2.imread(image_path, 0)
res = postprocess_predictions(
predictions[0][0],
original_image.shape[0],
original_image.shape[1],
do_default_smoothing=do_default_smoothing)
return res
def generator_test(b_s, imgs_test_path):
images = [imgs_test_path + f for f in os.listdir(imgs_test_path) if f.endswith('.jpg')]
images.sort()
counter = 0
while True:
yield preprocess_images(images[counter:counter + b_s], shape_r, shape_c)
counter = (counter + b_s) % len(images)
def compute_saliency(image_path):
if use_default_center_bias:
pred = model.predict_on_batch(
preprocess_images([image_path], shape_r, shape_c))
sm = pred[0][0]
else:
get_unbiased_output = K.function(
[model.layers[0].input,
K.learning_phase()], [model.layers[21].output])
pred = get_unbiased_output(
[preprocess_images([image_path], shape_r, shape_c), 0])
sm = pred[0][0][0]
original_image = cv2.imread(image_path, 0)
sm = postprocess_predictions(sm, original_image.shape[0],
original_image.shape[1])
return sm
def compute_saliency(img_path):
pred = model.predict_on_batch(
preprocess_images([img_path], shape_r, shape_c))
sm = pred[0][0]
original_image = cv2.imread(img_path, 0)
sm = postprocess_predictions(sm, original_image.shape[0],
original_image.shape[1])
return sm
def generator_test(b_s, imgs_test_path):
images = [imgs_test_path + f for f in os.listdir(imgs_test_path) if f.endswith(('.jpg', '.jpeg', '.png'))]
images.sort()
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
counter = 0
while True:
yield [preprocess_images(images[counter:counter + b_s], shape_r, shape_c), gaussian]
counter = (counter + b_s) % len(images)
def generator_test(b_s, images):
images.sort()
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
counter = 0
while True:
yield [
preprocess_images(images[counter:counter + b_s], shape_r, shape_c),
gaussian
]
counter = (counter + b_s) % len(images)
def generator(b_s, root_path, phase_gen='train', args=None):
imgs_path = root_path + '/Images/'
maps_path = root_path + '/FixMaps/'
fixs_path = root_path + '/FixPts/'
images = [
imgs_path + f for f in os.listdir(imgs_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps = [
maps_path + f for f in os.listdir(maps_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [fixs_path + f for f in os.listdir(fixs_path) if f.endswith('.mat')]
images.sort()
maps.sort()
fixs.sort()
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
counter = 0
while True:
Y = preprocess_maps(maps[counter:counter + b_s], shape_r_out,
shape_c_out)
# Single-side clip, valid when train on ASD dataset
if args.ssc and phase_gen == 'train':
Y[Y < 0.1] = 0
Y_fix = preprocess_fixmaps(fixs[counter:counter + b_s], shape_r_out,
shape_c_out)
X = preprocess_images(images[counter:counter + b_s], shape_r, shape_c)
yield [X, gaussian], [
Y,
Y,
Y_fix,
Y,
Y,
Y_fix,
Y,
Y,
Y_fix,
Y,
Y,
Y_fix,
# DOF,
]
counter = (counter + b_s) % len(images)
def get_test(video_test_path):
images = [
video_test_path + frames_path + f
for f in os.listdir(video_test_path + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
images.sort()
start = 0
while True:
Xims = np.zeros((1, num_frames, shape_r, shape_c, 3))
X = preprocess_images(
images[start:min(start + num_frames, len(images))], shape_r,
shape_c)
Xims[0, 0:min(len(images) - start, num_frames), :] = np.copy(X)
yield Xims #
start = min(start + num_frames, len(images))
def generator(b_s, phase_gen='train'):
if phase_gen == 'train':
images = [
imgs_train_path + f for f in os.listdir(imgs_train_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps = [
maps_train_path + f for f in os.listdir(maps_train_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [
fixs_train_path + f for f in os.listdir(fixs_train_path)
if f.endswith('.mat')
]
elif phase_gen == 'val':
images = [
imgs_val_path + f for f in os.listdir(imgs_val_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps = [
maps_val_path + f for f in os.listdir(maps_val_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [
fixs_val_path + f for f in os.listdir(fixs_val_path)
if f.endswith('.mat')
]
else:
raise NotImplementedError
images.sort()
maps.sort()
fixs.sort()
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
counter = 0
while True:
Y = preprocess_maps(maps[counter:counter + b_s], shape_r_out,
shape_c_out)
Y_fix = preprocess_fixmaps(fixs[counter:counter + b_s], shape_r_out,
shape_c_out)
yield [
preprocess_images(images[counter:counter + b_s], shape_r, shape_c),
gaussian
], [Y, Y, Y_fix]
counter = (counter + b_s) % len(images)
def generator(b_s, phase_gen='train'):
if phase_gen == 'train':
images = [imgs_train_path + f for f in os.listdir(imgs_train_path) if f.endswith('.jpg')]
maps = [maps_train_path + f for f in os.listdir(maps_train_path) if f.endswith('.jpg')]
elif phase_gen == 'val':
images = [imgs_val_path + f for f in os.listdir(imgs_val_path) if f.endswith('.jpg')]
maps = [maps_val_path + f for f in os.listdir(maps_val_path) if f.endswith('.jpg')]
else:
raise NotImplementedError
images.sort()
maps.sort()
counter = 0
while True:
yield preprocess_images(images[counter:counter + b_s], shape_r, shape_c), preprocess_maps(maps[counter:counter + b_s], shape_r_gt, shape_c_gt)
counter = (counter + b_s) % len(images)
def generator_test(b_s, imgs_test_path):
images = [imgs_test_path + "\\" + f for f in os.listdir(imgs_test_path) if f.endswith(('.jpg', '.jpeg', '.png'))]
images.sort()
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
#counter = 0
#while True:
# yield [preprocess_images(images[counter:counter + b_s], shape_r, shape_c), gaussian]
# counter = (counter + b_s) % len(images)
#Funciona para b_s = 1 NUEVA VERSION!
counter = 0
while counter < len(images):
print("Ejecutado generator_test para la imagen ", counter + 1)
yield [preprocess_images(images[counter:counter + b_s], shape_r, shape_c), gaussian]
counter = counter + 1
def get_test_td_prior_mopt(video_test_path):
images = [
video_test_path + frames_path + f
for f in os.listdir(video_test_path + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
optflws = [
video_test_path + optflw_path + f
for f in os.listdir(video_test_path + optflw_path)
if f.endswith(('.flo'))
]
# add one more optical flow
images.sort()
optflws.sort()
for j in range(0, opt_num + 1):
optflws.append(optflws[-1])
start = 0
while True:
Xims_ops = np.zeros((1, num_frames, shape_r, shape_c, 3 + 2 * opt_num))
gaussian = np.zeros((1, shape_r_gaus, shape_c_gaus, nb_gaussian))
data = [gaussian for i in range(0, num_frames)]
X = preprocess_images(
images[start:min(start + num_frames, len(images))], shape_r,
shape_c)
X_f = preprocess_optflws_multi(
optflws[start:min(start + num_frames + opt_num + 1, len(optflws))],
shape_r,
shape_c,
X.shape[0],
phase='test')
Xims_ops[0, 0:min(len(images) - start, num_frames), :, :,
0:3] = np.copy(X)
Xims_ops[0, 0:min(len(images) - start, num_frames), :, :,
3:] = np.copy(X_f)
data.append(Xims_ops)
yield data #
start = min(start + num_frames, len(images))
def get_test_td_simple_mopt(video_test_path):
images = [
video_test_path + frames_path + f
for f in os.listdir(video_test_path + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
optflws = [
video_test_path + optflw_path + f
for f in os.listdir(video_test_path + optflw_path)
if f.endswith(('.flo\n'))
]
# add one more optical flow
for j in range(0, opt_num + 1):
optflws.append(optflws[-1])
images.sort()
optflws.sort()
start = 0
# true???? note the val steps when using predictor
while True:
Xims_ops = np.zeros((num_frames, shape_r, shape_c, 3 + 2 * opt_num))
# Xops = np.zeros((1, num_frames, shape_r, shape_c, 2))
X = preprocess_images(
images[start:min(start + num_frames, len(images))], shape_r,
shape_c)
X_f = preprocess_optflws_multi(
optflws[start:min(start + num_frames + opt_num, len(optflws))],
shape_r,
shape_c,
X.shape[0],
phase='test')
Xims_ops[0:min(len(images) - start, num_frames), :, :,
0:3] = np.copy(X)
Xims_ops[0:min(len(images) - start, num_frames), :, :,
3:] = np.copy(X_f)
yield Xims_ops #
start = min(start + num_frames, len(images))
def _VideoBatch_Generator(phase_gen='train'):
'''
Args:
VideoCap: cv2预读取视频的操作
batch_size: batch的大小
frame_num: 一个视频的连续帧
resize_shape: 输出视频帧的大小
Out: video_batch: (batch_size, frame_num, width, height, RGB)
'''
VideoCap = cv2.VideoCapture(VideoName)
#frame_width = VideoCap.get(3) #3:CV_CAP_PROP_FRAME_HEIGHT
#frame_height = VideoCap.get(4) #4:CV_CAP_PROP_FRAME_HEIGHT
if phase_gen == 'test':
Video_frames = VideoCap.get(7) #7:CV_CAP_PROP_FRAME_COUNT
print(Video_frames)
batch_size = Video_frames // frame_num
if not Video_frames % frame_num == 0:
batch_size = batch_size + 1
print(batch_size)
start_frame = 0
Video_Batch = np.zeros(shape=[1, 1, 720, 1080, 3])
for j in range(int(batch_size)):
Video_Slice = np.zeros(shape=[1, 1, 720, 1080, 3])
if start_frame + frame_num > Video_frames:
start_frame = Video_frames - frame_num
stop_frame = start_frame + frame_num
for i in range(start_frame, stop_frame):
VideoCap.set(1, i) #1:CV_CAP_PROP_POS_FRAMES
_, frame = VideoCap.read() #（frame_width, frame_height, BGR）
frame = cv2.resize(frame, input_shape)
frame = frame.astype(np.float32)
frame = cv2.cvtColor(
frame,
cv2.COLOR_BGR2RGB) #（resize_width, resize_height, RGB）
frame = frame[np.newaxis, np.newaxis, ...]
if np.shape(Video_Slice)[1] == 1 and i == start_frame:
Video_Slice = frame
else:
Video_Slice = np.concatenate((Video_Slice, frame), axis=1)
print(np.shape(Video_Slice))
if np.shape(Video_Batch)[0] == 1 and start_frame == 0:
Video_Batch = Video_Slice
else:
Video_Batch = np.concatenate((Video_Batch, Video_Slice),
axis=0)
print(np.shape(Video_Batch))
start_frame = stop_frame
#training process 读取的是文件夹里的照片
# video_trainpaths里包含的是按数字标号保存的video，每个video文件家里三个文件夹 “images”（） “maps”
if phase_gen == 'train':
videos = [
Video_train_paths + '/' + video_train_path
for video_train_path in os.listdir(Video_train_paths)
if os.path.isdir(Video_train_paths + '/' + video_train_path)
]
videos.sort()
random.shuffle(videos)
range_num = 0
while True:
Xims = np.zeros(
(batch_size, frame_num, input_shape[0], input_shape[1], 3))
Ymaps = np.zeros((batch_size, frame_num, output_shape[0],
output_shape[1], 1)) + 0.01
Yfixs = np.zeros((batch_size, frame_num, output_shape[0],
output_shape[1], 1)) + 0.01
for i in range(batch_size):
video = videos[(range_num * batch_size + i) % len(videos)]
images = [
video + frames_path + f
for f in os.listdir(video + frames_path)
if f.endswith((".jpg", ".jpeg", ".png"))
]
maps = [
video + maps_path + f
for f in os.listdir(video + maps_path)
if f.endswith((".jpg", ".jpeg", "png"))
]
fixs = [
video + fixs_path + f
for f in os.listdir(video + fixs_path)
if f.endswith(".mat")
]
start = np.random.choice(max(1, len(images) - frame_num))
X = preprocess_images(
images[start:min((start + frame_num), len(images))],
input_shape[0], input_shape[1])
Y = preprocess_maps(
maps[start:min((start + frame_num), len(images))],
output_shape[0], output_shape[1])
Y_fix = preprocess_fixmaps(
fixs[start:min((start + frame_num), len(images))],
output_shape[0], output_shape[1])
Xims[i, 0:np.shape(X)[0], :] = np.copy(X)
Ymaps[i, 0:np.shape(Y)[0], :] = np.copy(Y)
Yfixs[i, 0:np.shape(Y_fix)[0], :] = np.copy(Y_fix)
Xims[i, X.shape[0]:frame_num, :] = np.copy(X[-1, :, :])
Ymaps[i, Y.shape[0]:frame_num, :] = np.copy(Y[-1, :, :])
Yfixs[i,
Y_fix.shape[0]:frame_num, :] = np.copy(Y_fix[-1, :, :])
yield Xims, Ymaps, Yfixs
range_num = range_num + 1
def generator(b_s, images, maps, shape_r, shape_c, shape_r_gt, shape_c_gt,shuffle=True):
counter = 0
while True:
yield preprocess_images(images[counter:counter + b_s], shape_r, shape_c), preprocess_maps(maps[counter:counter + b_s], shape_r_gt, shape_c_gt)
counter = (counter + b_s)
def generator_td_simple_mopt(phase_gen='train'):
if phase_gen == 'train':
videos_paths = videos_train_paths
elif phase_gen == 'test':
videos_paths = videos_val_paths
else:
raise NotImplementedError
videos = [
videos_path + f for videos_path in videos_paths
for f in os.listdir(videos_path) if os.path.isdir(videos_path + f)
]
random.shuffle(videos)
video_counter = 0
while True:
Xims_ops = np.zeros((num_frames, shape_r, shape_c, 3 + 2 * opt_num))
# Xops = np.zeros((video_b_s, num_frames, shape_r, shape_c, 2))
Ymaps = np.zeros((num_frames, shape_r_gaus, shape_c_gaus, 1)) + 0.01
Yfixs = np.zeros((num_frames, shape_r_gaus, shape_c_gaus, 1)) + 0.01
for i in range(0, video_b_s):
video_path = videos[(video_counter + i) % len(videos)]
images = [
video_path + frames_path + f
for f in os.listdir(video_path + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
images.sort()
optflws = [
video_path + optflw_path + f
for f in os.listdir(video_path + optflw_path)
if f.endswith('.flo')
]
optflws.sort()
if len(optflws) < 10:
continue
for j in range(0, opt_num + 1):
optflws.append(optflws[-1])
maps = [
video_path + maps_path + f
for f in os.listdir(video_path + maps_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps.sort()
fixs = [
video_path + fixs_path + f
for f in os.listdir(video_path + fixs_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs.sort()
start = random.randint(0, max(len(images) - num_frames * f_gap, 0))
X = preprocess_images(
images[start:min(start +
num_frames * f_gap, len(images)):f_gap],
shape_r, shape_c)
X_f = preprocess_optflws_multi(
optflws[start:min(start + num_frames * f_gap +
opt_num, len(optflws))], shape_r, shape_c,
X.shape[0]) # how to get filename list????
Y = preprocess_maps(
maps[start:min(start + num_frames * f_gap, len(images)):f_gap],
shape_r_gaus, shape_c_gaus)
Y_fix = preprocess_fixmaps_png(
fixs[start:min(start + num_frames, len(images)):f_gap],
shape_r_gaus, shape_c_gaus)
# print('Xims_ops:', Xims_ops.shape)
# print('X:', X.shape)
# print('X_f:', X_f.shape)
Xims_ops[0:X.shape[0], :, :, 0:3] = np.copy(X)
Xims_ops[0:X_f.shape[0], :, :, 3:] = np.copy(X_f)
Ymaps[0:Y.shape[0], :] = np.copy(Y)
Yfixs[0:Y_fix.shape[0], :] = np.copy(Y_fix)
Xims_ops[X.shape[0]:num_frames, :, :, 0:3] = np.copy(X[-1, :, :])
Xims_ops[X_f.shape[0]:num_frames, :, :,
3:] = np.copy(X_f[-1, :, :])
Ymaps[Y.shape[0]:num_frames, :] = np.copy(Y[-1, :, :])
Yfixs[Y_fix.shape[0]:num_frames, :] = np.copy(Y_fix[-1, :, :])
yield Xims_ops, \
[Ymaps, Ymaps, Yfixs, Ymaps,
Ymaps, Ymaps, Yfixs, Ymaps,
Ymaps, Ymaps, Yfixs, Ymaps,
Ymaps, Ymaps, Yfixs, Ymaps
] #
video_counter = (video_counter + video_b_s) % len(videos)
def generator_test(b_s, images):
counter = 0
while True:
im = preprocess_images(images[counter:counter + b_s], shape_r, shape_c)
yield im
counter = (counter + b_s) % len(images)
def generator(video_b_s, image_b_s, phase_gen='train'):
if phase_gen == 'train':
videos = [
videos_train_path + f for videos_train_path in videos_train_paths
for f in os.listdir(videos_train_path)
if os.path.isdir(videos_train_path + f)
]
images = [
imgs_path + 'images/' + f
for f in os.listdir(imgs_path + 'images/')
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixationmaps = [
imgs_path + 'maps/' + f for f in os.listdir(imgs_path + 'maps/')
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [
imgs_path + 'fixation/maps/' + f
for f in os.listdir(imgs_path + 'fixation/maps/')
if f.endswith(('.mat'))
]
images.sort()
fixationmaps.sort()
fixs.sort()
image_train_data = []
for image, fixationmap, fix in zip(images, fixationmaps, fixs):
annotation_data = {
'image': image,
'fixmap': fixationmap,
'fix': fix
}
image_train_data.append(annotation_data)
random.shuffle(image_train_data)
# videos.sort()
random.shuffle(videos)
loop = 1
video_counter = 0
image_counter = 0
while True:
if loop % 2:
Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3))
Ymaps = np.zeros((video_b_s, num_frames, shape_r_out,
shape_c_out, 1)) + 0.01
Yfixs = np.zeros((video_b_s, num_frames, shape_r_out,
shape_c_out, 1)) + 0.01
Img_Ymaps = np.zeros((video_b_s, num_frames, shape_r_attention,
shape_c_attention, 1)) + 0.01
Img_Yfixs = np.zeros((video_b_s, num_frames, shape_r_attention,
shape_c_attention, 1)) + 0.01
for i in range(0, video_b_s):
video_path = videos[(video_counter + i) % len(videos)]
images = [
video_path + frames_path + f
for f in os.listdir(video_path + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps = [
video_path + maps_path + f
for f in os.listdir(video_path + maps_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [
video_path + fixs_path + f
for f in os.listdir(video_path + fixs_path)
if f.endswith('.mat')
]
# images.sort()
# maps.sort()
# fixs.sort()
start = random.randint(0, max(len(images) - num_frames, 0))
X = preprocess_images(
images[start:min(start + num_frames, len(images))],
shape_r, shape_c)
Y = preprocess_maps(
maps[start:min(start + num_frames, len(images))],
shape_r_out, shape_c_out)
Y_fix = preprocess_fixmaps(
fixs[start:min(start + num_frames, len(images))],
shape_r_out, shape_c_out)
Xims[i, 0:X.shape[0], :] = np.copy(X)
Ymaps[i, 0:Y.shape[0], :] = np.copy(Y)
Yfixs[i, 0:Y_fix.shape[0], :] = np.copy(Y_fix)
Xims[i, X.shape[0]:num_frames, :] = np.copy(X[-1, :, :])
Ymaps[i, Y.shape[0]:num_frames, :] = np.copy(Y[-1, :, :])
Yfixs[i, Y_fix.shape[0]:num_frames, :] = np.copy(
Y_fix[-1, :, :])
yield Xims, [
Ymaps, Ymaps, Yfixs, Img_Ymaps, Img_Ymaps, Img_Yfixs
] #
video_counter = (video_counter + video_b_s) % len(videos)
loop = loop + 1
else:
Xims = np.zeros((image_b_s, 1, shape_r, shape_c, 3))
Ymaps = np.zeros(
(image_b_s, 1, shape_r_out, shape_c_out, 1)) + 0.01
Yfixs = np.zeros(
(image_b_s, 1, shape_r_out, shape_c_out, 1)) + 0.01
Img_Ymaps = np.zeros((image_b_s, 1, shape_r_attention,
shape_c_attention, 1)) + 0.01
Img_Yfixs = np.zeros((image_b_s, 1, shape_r_attention,
shape_c_attention, 1)) + 0.01
for i in range(0, image_b_s):
img_data = image_train_data[(image_counter + i) %
len(image_train_data)]
X = preprocess_images([img_data['image']], shape_r,
shape_c)
Y = preprocess_maps([img_data['fixmap']],
shape_r_attention, shape_c_attention)
Y_fix = preprocess_fixmaps([img_data['fix']],
shape_r_attention,
shape_c_attention)
Xims[i, 0, :] = np.copy(X)
Img_Ymaps[i, 0, :] = np.copy(Y)
Img_Yfixs[i, 0, :] = np.copy(Y_fix)
yield Xims, [
Ymaps, Ymaps, Yfixs, Img_Ymaps, Img_Ymaps, Img_Yfixs
] # #
image_counter = (image_counter +
image_b_s) % len(image_train_data)
loop = loop + 1
elif phase_gen == 'val':
videos = [
videos_val_path + f for videos_val_path in videos_val_paths
for f in os.listdir(videos_val_path)
if os.path.isdir(videos_val_path + f)
]
random.shuffle(videos)
video_counter = 0
while True:
Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3))
Ymaps = np.zeros(
(video_b_s, num_frames, shape_r_out, shape_c_out, 1)) + 0.01
Yfixs = np.zeros(
(video_b_s, num_frames, shape_r_out, shape_c_out, 1)) + 0.01
Img_Ymaps = np.zeros((video_b_s, num_frames, shape_r_attention,
shape_c_attention, 1)) + 0.01
Img_Yfixs = np.zeros((video_b_s, num_frames, shape_r_attention,
shape_c_attention, 1)) + 0.01
for i in range(0, video_b_s):
video_path = videos[(video_counter + i) % len(videos)]
images = [
video_path + frames_path + f
for f in os.listdir(video_path + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
maps = [
video_path + maps_path + f
for f in os.listdir(video_path + maps_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
fixs = [
video_path + fixs_path + f
for f in os.listdir(video_path + fixs_path)
if f.endswith('.mat')
]
start = random.randint(0, max(len(images) - num_frames, 0))
X = preprocess_images(
images[start:min(start + num_frames, len(images))],
shape_r, shape_c)
Y = preprocess_maps(
maps[start:min(start + num_frames, len(images))],
shape_r_out, shape_c_out)
Y_fix = preprocess_fixmaps(
fixs[start:min(start + num_frames, len(images))],
shape_r_out, shape_c_out)
Xims[i, 0:X.shape[0], :] = np.copy(X)
Ymaps[i, 0:Y.shape[0], :] = np.copy(Y)
Yfixs[i, 0:Y_fix.shape[0], :] = np.copy(Y_fix)
Xims[i, X.shape[0]:num_frames, :] = np.copy(X[-1, :, :])
Ymaps[i, Y.shape[0]:num_frames, :] = np.copy(Y[-1, :, :])
Yfixs[i,
Y_fix.shape[0]:num_frames, :] = np.copy(Y_fix[-1, :, :])
yield Xims, [Ymaps, Ymaps, Yfixs, Img_Ymaps, Img_Ymaps,
Img_Yfixs] #
video_counter = (video_counter + video_b_s) % len(videos)
else:
raise NotImplementedError
def process_test_data(test_pth):
Xims = np.zeros((1, len(test_pth), input_shape[0], input_shape[1], 3))
X = preprocess_images(test_pth, input_shape[0], input_shape[1])
Xims[0, 0:len(test_pth), :] = np.copy(X)
return Xims #
def generator_test(b_s, image_paths):
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
while True:
yield [preprocess_images(image_paths, shape_r, shape_c), gaussian]
def generator_data_DR(video_b_s, phase_gen='train'):
num_frames = input_t
if phase_gen == 'train':
train_pth = os.path.join(train_data_pth, '*', 'maps')
maps_seq = glob.glob(train_pth)
datas = []
for maps_pth in maps_seq:
maps = sorted(glob.glob(maps_pth + '/*'))[:-500]
images = [xx.replace('maps', 'images') for xx in maps]
segs = [xx.replace('images', 'semantic') for xx in images]
segs = [xx.replace('.jpg', '.png') for xx in segs]
# fixs = [xx.replace('images', 'fixation/maps') for xx in images]
# fixs = [xx.replace('.jpg', '.mat') for xx in fixs]
for jj in range(len(images) - input_t):
datas.append((
images[jj:jj + input_t],
segs[jj:jj + input_t],
[
maps[jj + input_t],
],
# [fixs[jj + input_t], ]
))
counts = 0
print('len -> train data :', len(datas), 'batch_data ->:',
len(datas) // video_b_s)
random.shuffle(datas)
while True:
if counts >= (len(datas) - video_b_s):
random.shuffle(datas)
counts = counts % (len(datas) - video_b_s)
Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3))
Xsegs = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3))
Ymaps = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001
# Yfixs = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001
for i in range(0, video_b_s):
X1 = preprocess_images(datas[counts][0], shape_r, shape_c)
X2 = preprocess_images(datas[counts][1], shape_r, shape_c)
Y = preprocess_maps(datas[counts][2], shape_r_out, shape_c_out)
# Y_fix = preprocess_fixmaps(datas[counts][3], shape_r_out, shape_c_out)
Xims[i, :] = np.copy(X1)
Xsegs[i, :] = np.copy(X2)
Ymaps[i, :] = np.copy(Y[0])
# Yfixs[i, :] = np.copy(Y_fix[0])
# ---------------------------------------------------
counts += 1
yield [Xims, Xsegs], [Ymaps, Ymaps] #
else:
val_pth = os.path.join(val_data_pth, '*', 'maps')
maps_seq = glob.glob(val_pth)
datas = []
for maps_pth in maps_seq:
maps = sorted(glob.glob(maps_pth + '/*'))[-500:]
images = [xx.replace('maps', 'images') for xx in maps]
segs = [xx.replace('images', 'semantic') for xx in images]
segs = [xx.replace('.jpg', '.png') for xx in segs]
# fixs = [xx.replace('images', 'fixation/maps') for xx in images]
# fixs = [xx.replace('.jpg', '.mat') for xx in fixs]
for jj in range(len(images) - input_t):
datas.append((
images[jj:jj + input_t],
segs[jj:jj + input_t],
[
maps[jj + input_t],
],
# [fixs[jj + input_t], ]
))
counts = 0
print('len -> val data :', len(datas), 'batch_data ->:',
len(datas) // video_b_s)
random.shuffle(datas)
while True:
if counts >= (len(datas) - video_b_s):
random.shuffle(datas)
counts = counts % (len(datas) - video_b_s)
Xims = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3))
Xsegs = np.zeros((video_b_s, num_frames, shape_r, shape_c, 3))
Ymaps = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001
# Yfixs = np.zeros((video_b_s, shape_r_out, shape_c_out, 1)) + 0.001
for i in range(0, video_b_s):
X1 = preprocess_images(datas[counts][0], shape_r, shape_c)
X2 = preprocess_images(datas[counts][1], shape_r, shape_c)
Y = preprocess_maps(datas[counts][2], shape_r_out, shape_c_out)
# Y_fix = preprocess_fixmaps(datas[counts][3], shape_r_out, shape_c_out)
Xims[i, :] = np.copy(X1)
Xsegs[i, :] = np.copy(X2)
Ymaps[i, :] = np.copy(Y[0])
# Yfixs[i, :] = np.copy(Y_fix[0])
# ---------------------------------------------------
counts += 1
yield [Xims, Xsegs], [Ymaps, Ymaps] #
