def load_saliency_data_simo(signatures, nb_frames, image_size):
"""
Crop -> NSS with fixations
Fine -> KLD with saliency maps and IG with fixations (baseline is saliency maps)
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:return: a tuple holding the fullframe and the cropped saliency.
"""
batchsize = len(signatures)
h, w = image_size
h_c = h // 4
w_c = w // 4
Y = np.zeros(shape=(batchsize, 2, h, w), dtype=np.float32)
Y_c = np.zeros(shape=(batchsize, 1, h_c, w_c), dtype=np.float32)
for b in range(0, batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
y_dir_sal = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency')
y_dir_fix = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency_fix')
# saliency
y_sal = read_image(join(y_dir_sal,
'{:06d}.png'.format(start + nb_frames - 1)),
channels_first=True,
color=False,
resize_dim=image_size) / 255
y_fix = read_image(join(y_dir_fix,
'{:06d}.png'.format(start + nb_frames - 1)),
channels_first=True,
color=False,
resize_dim=image_size) / 255
# resize to (256, 256) before cropping
y_before_crop = resize_tensor(np.expand_dims(y_fix, axis=0),
new_shape=frame_size_before_crop)
Y[b, 0, :, :] = y_sal
Y[b, 1, :, :] = y_fix
Y_c[b, 0, :, :] = crop_tensor(y_before_crop,
indexes=(hc1, hc2, wc1, wc2))[0]
if do_mirror:
Y = Y[:, :, :, ::-1]
Y_c = Y_c[:, :, :, ::-1]
return [Y, Y_c]
def load_saliency_data(signatures, nb_frames, image_size, gt_type='fix'):
"""
Function to load a saliency batch.
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:param gt_type: choose among `sal` (old groundtruth) and `fix` (new groundtruth).
:return: a tuple holding the fullframe and the cropped saliency.
"""
batchsize = len(signatures)
h, w = image_size
h_c = h // 4
w_c = w // 4
Y = np.zeros(shape=(batchsize, h, w, 1), dtype=np.float32)
Y_c = np.zeros(shape=(batchsize, h_c, w_c, 1), dtype=np.float32)
for b in range(0, batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
y_dir = join(dreyeve_dir, "DREYEVE_DATA", '{:02d}'.format(num_run),
'output_frames' if gt_type == 'sal' else 'output_frames')
# saliency
y = read_image(join(y_dir, '{}.jpg'.format(start + nb_frames - 1)),
channels_first=False,
color=False,
resize_dim=image_size)
# resize to (256, 256) before cropping
y_before_crop = resize_tensor(np.expand_dims(y, axis=2),
new_shape=frame_size_before_crop)
Y[b, :, :, 0] = y
Y_c[b, :, :, 0] = crop_tensor(y_before_crop,
indexes=(hc1, hc2, wc1, wc2))[0]
if do_mirror:
Y = Y[:, :, :, ::-1]
Y_c = Y_c[:, :, :, ::-1]
return [Y, Y_c]
def load_batch_data(signatures, nb_frames, image_size, batch_type):
"""
Function to load a data batch. This is common for `image`, `optical_flow` and `semseg`.
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:param batch_type: choose among [`image`, `optical_flow`, `semseg`].
:return: a tuple holding the fullframe, the small and the cropped batch.
"""
assert batch_type in ['image', 'optical_flow', 'semseg'
], 'Unknown batch type: {}'.format(batch_type)
batchsize = len(signatures)
h, w = image_size
h_s = h_c = h // 4
w_s = w_c = w // 4
if batch_type == 'image':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'DREYEVE_DATA'
# mean frame image, to subtract mean
elif batch_type == 'optical_flow':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'optical_flow'
elif batch_type == 'semseg':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'Segmentation'
for b in range(batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
data_dir = join(dreyeve_dir, subdir, '{:02d}'.format(num_run))
mean_image = read_image(join(dreyeve_dir, 'DREYEVE_DATA',
'{:02d}'.format(num_run),
'mean_frame.png'),
channels_first=False,
resize_dim=image_size)
for offset in range(0, nb_frames):
if batch_type == 'image':
x = read_image(join(data_dir, 'Frames',
'{}.jpg'.format(start + offset)),
channels_first=False,
resize_dim=image_size) - mean_image
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
elif batch_type == 'optical_flow':
x = read_image(join(data_dir,
'{}.png'.format(start + offset + 1)),
channels_first=False,
resize_dim=image_size)
x -= np.mean(x, axis=(1, 2), keepdims=True) # remove mean
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
elif batch_type == 'semseg':
x = resize_tensor(np.load(
join(data_dir, '{}.npz'.format(start + offset)))['arr_0'],
new_shape=image_size)
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
B_ff[b, 0, :, :, :] = x
if do_mirror:
B_ff = B_ff[:, :, :, :, ::-1]
B_s = B_s[:, :, :, :, ::-1]
B_c = B_c[:, :, :, :, ::-1]
return [B_ff, B_s, B_c]