def load_dreyeve_sample(sequence_dir, sample, mean_dreyeve_image, frames_per_seq=16, h=448, w=448, ):
"""
Function to load a dreyeve_sample.
:param sequence_dir: string, sequence directory (e.g. 'Z:/DATA/04/').
:param sample: int, sample to load in (15, 7499). N.B. this is the sample where prediction occurs!
:param mean_dreyeve_image: mean dreyeve image, subtracted to each frame.
:param frames_per_seq: number of temporal frames for each sample
:param h: h
:param w: w
:return: a dreyeve_sample like I, OF, SEG
"""
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32')
I_s = np.zeros(shape=(1, frames_per_seq, h_s, w_s,3), dtype='float32')
I_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32')
SEG_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32')
SEG_s = np.zeros(shape=(1,frames_per_seq, h_s, w_s,3), dtype='float32')
SEG_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32')
OF_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32')
OF_s = np.zeros(shape=(1, frames_per_seq, h_s, w_s,3), dtype='float32')
OF_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32')
for fr in range(0, frames_per_seq):
offset = sample - frames_per_seq + 1 + fr # tricky
# read image
x = read_image(join(sequence_dir,'Frames','{}.jpg'.format(offset)),
channels_first=False, resize_dim=(h, w)) - mean_dreyeve_image
I_s[0, fr, :, :,:] = resize_tensor(x,new_shape=(h_s, w_s))
# read semseg
seg = resize_tensor(np.load(join(sequence_dir,'Seg', '{}.npz'.format(offset)))['arr_0'],
new_shape=(h, w))
SEG_s[0,fr,:, :, :] = resize_tensor(seg, new_shape=(h_s, w_s))
# read of
of = read_image(join(sequence_dir,'Optical', '{}.png'.format(offset + 1)),
channels_first=False, resize_dim=(h, w))
of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean
OF_s[0, fr, :, :, :] = resize_tensor(of, new_shape=(h_s, w_s))
I_ff[0,0, :, :,:] = x
SEG_ff[0,0, :, :, :] = seg
OF_ff[0, 0, :, :, :] = of
return [I_ff, I_s,I_c,OF_ff,OF_s,OF_c,SEG_ff, SEG_s, SEG_c]
def load_dreyeve_sample(
frames_list,
sample,
mean_dreyeve_image,
frames_per_seq=16,
h=448,
w=448,
):
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
for fr in range(frames_per_seq):
offset = sample - frames_per_seq + 1 + fr # tricky
x_in = cv2.resize(frames_list[offset],
dsize=resize_dim_in[::-1],
interpolation=cv2.INTER_LINEAR)
x_in = np.transpose(x_in, (2, 0, 1)).astype(np.float32)
x_in -= mean_dreyeve_image
I_s[0, :, fr, :, :] = resize_tensor(x_in, new_size=(h_s, w_s))
x_disp = cv2.resize(frames_list[offset],
dsize=resize_dim_disp[::-1],
interpolation=cv2.INTER_LINEAR)
I_ff[0, :, 0, :, :] = x_in
return [I_ff, I_s, I_c], x_disp
def load_dreyeve_sample(sequence_dir,
sample,
mean_dreyeve_image,
frames_per_seq=16,
h=448,
w=448):
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
for fr in range(frames_per_seq):
offset = sample - frames_per_seq + 1 + fr
# read image
x = read_image(join(sequence_dir, 'frames', '{}.jpg'.format(offset)),
channels_first=True,
resize_dim=(h, w)) - mean_dreyeve_image
I_s[0, :, fr, :, :] = resize_tensor(x, new_size=(h_s, w_s))
I_ff[0, :, 0, :, :] = x
return [I_ff, I_s, I_c]
def load_saliency_data_simo(signatures, nb_frames, image_size):
"""
Crop -> NSS with fixations
Fine -> KLD with saliency maps and IG with fixations (baseline is saliency maps)
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:return: a tuple holding the fullframe and the cropped saliency.
"""
batchsize = len(signatures)
h, w = image_size
h_c = h // 4
w_c = w // 4
Y = np.zeros(shape=(batchsize, 2, h, w), dtype=np.float32)
Y_c = np.zeros(shape=(batchsize, 1, h_c, w_c), dtype=np.float32)
for b in range(0, batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
y_dir_sal = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency')
y_dir_fix = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency_fix')
# saliency
y_sal = read_image(join(y_dir_sal,
'{:06d}.png'.format(start + nb_frames - 1)),
channels_first=True,
color=False,
resize_dim=image_size) / 255
y_fix = read_image(join(y_dir_fix,
'{:06d}.png'.format(start + nb_frames - 1)),
channels_first=True,
color=False,
resize_dim=image_size) / 255
# resize to (256, 256) before cropping
y_before_crop = resize_tensor(np.expand_dims(y_fix, axis=0),
new_shape=frame_size_before_crop)
Y[b, 0, :, :] = y_sal
Y[b, 1, :, :] = y_fix
Y_c[b, 0, :, :] = crop_tensor(y_before_crop,
indexes=(hc1, hc2, wc1, wc2))[0]
if do_mirror:
Y = Y[:, :, :, ::-1]
Y_c = Y_c[:, :, :, ::-1]
return [Y, Y_c]
def load_saliency_data(signatures, nb_frames, image_size, gt_type='fix'):
"""
Function to load a saliency batch.
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:param gt_type: choose among `sal` (old groundtruth) and `fix` (new groundtruth).
:return: a tuple holding the fullframe and the cropped saliency.
"""
batchsize = len(signatures)
h, w = image_size
h_c = h // 4
w_c = w // 4
Y = np.zeros(shape=(batchsize, h, w, 1), dtype=np.float32)
Y_c = np.zeros(shape=(batchsize, h_c, w_c, 1), dtype=np.float32)
for b in range(0, batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
y_dir = join(dreyeve_dir, "DREYEVE_DATA", '{:02d}'.format(num_run),
'output_frames' if gt_type == 'sal' else 'output_frames')
# saliency
y = read_image(join(y_dir, '{}.jpg'.format(start + nb_frames - 1)),
channels_first=False,
color=False,
resize_dim=image_size)
# resize to (256, 256) before cropping
y_before_crop = resize_tensor(np.expand_dims(y, axis=2),
new_shape=frame_size_before_crop)
Y[b, :, :, 0] = y
Y_c[b, :, :, 0] = crop_tensor(y_before_crop,
indexes=(hc1, hc2, wc1, wc2))[0]
if do_mirror:
Y = Y[:, :, :, ::-1]
Y_c = Y_c[:, :, :, ::-1]
return [Y, Y_c]
def load_batch_data(signatures, nb_frames, image_size, batch_type):
"""
Function to load a data batch. This is common for `image`, `optical_flow` and `semseg`.
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:param batch_type: choose among [`image`, `optical_flow`, `semseg`].
:return: a tuple holding the fullframe, the small and the cropped batch.
"""
assert batch_type in ['image', 'optical_flow', 'semseg'
], 'Unknown batch type: {}'.format(batch_type)
batchsize = len(signatures)
h, w = image_size
h_s = h_c = h // 4
w_s = w_c = w // 4
if batch_type == 'image':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'DREYEVE_DATA'
# mean frame image, to subtract mean
elif batch_type == 'optical_flow':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'optical_flow'
elif batch_type == 'semseg':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'Segmentation'
for b in range(batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
data_dir = join(dreyeve_dir, subdir, '{:02d}'.format(num_run))
mean_image = read_image(join(dreyeve_dir, 'DREYEVE_DATA',
'{:02d}'.format(num_run),
'mean_frame.png'),
channels_first=False,
resize_dim=image_size)
for offset in range(0, nb_frames):
if batch_type == 'image':
x = read_image(join(data_dir, 'Frames',
'{}.jpg'.format(start + offset)),
channels_first=False,
resize_dim=image_size) - mean_image
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
elif batch_type == 'optical_flow':
x = read_image(join(data_dir,
'{}.png'.format(start + offset + 1)),
channels_first=False,
resize_dim=image_size)
x -= np.mean(x, axis=(1, 2), keepdims=True) # remove mean
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
elif batch_type == 'semseg':
x = resize_tensor(np.load(
join(data_dir, '{}.npz'.format(start + offset)))['arr_0'],
new_shape=image_size)
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
B_ff[b, 0, :, :, :] = x
if do_mirror:
B_ff = B_ff[:, :, :, :, ::-1]
B_s = B_s[:, :, :, :, ::-1]
B_c = B_c[:, :, :, :, ::-1]
return [B_ff, B_s, B_c]
def roll(self):
self.fr += 1
self.I_s, self.I_c, self.OF_s, self.OF_c, self.SEG_s, self.SEG_c, \
self.Il_s, self.Il_c, self.OFl_s, self.OFl_c, self.SEGl_s, self.SEGl_c, \
self.Ir_s, self.Ir_c, self.OFr_s, self.OFr_c, self.SEGr_s, self.SEGr_c = [
np.roll(x, -1, axis=2) for x in
[self.I_s, self.I_c, self.OF_s, self.OF_c, self.SEG_s, self.SEG_c,
self.Il_s, self.Il_c, self.OFl_s, self.OFl_c, self.SEGl_s, self.SEGl_c,
self.Ir_s, self.Ir_c, self.OFr_s, self.OFr_c, self.SEGr_s, self.SEGr_c]]
# read image
x = read_image(join(self.sequence_dir, 'frames', '{:06d}.jpg'.format(self.fr)), channels_first=True,
resize_dim=(self.h, self.w)) \
- self.mean_dreyeve_image
xl = translate(x, pixels=116, side='left')
xr = translate(x, pixels=116, side='right')
self.I_s[0, :, -1, :, :] = resize_tensor(x,
new_size=(self.h_s, self.w_s))
self.Il_s[0, :,
-1, :, :] = resize_tensor(xl, new_size=(self.h_s, self.w_s))
self.Ir_s[0, :,
-1, :, :] = resize_tensor(xr, new_size=(self.h_s, self.w_s))
# read of
of = read_image(join(self.sequence_dir, 'optical_flow',
'{:06d}.png'.format(self.fr + 1)),
channels_first=True,
resize_dim=(h, w))
of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean
ofl = translate(of, pixels=116, side='left')
ofr = translate(of, pixels=116, side='right')
self.OF_s[0, :,
-1, :, :] = resize_tensor(of, new_size=(self.h_s, self.w_s))
self.OFl_s[0, :,
-1, :, :] = resize_tensor(ofl,
new_size=(self.h_s, self.w_s))
self.OFr_s[0, :,
-1, :, :] = resize_tensor(ofr,
new_size=(self.h_s, self.w_s))
# read semseg
seg = resize_tensor(np.load(
join(self.sequence_dir, 'semseg',
'{:06d}.npz'.format(self.fr)))['arr_0'][0],
new_size=(h, w))
segl = translate(seg, pixels=116, side='left')
segr = translate(seg, pixels=116, side='right')
self.SEG_s[0, :,
-1, :, :] = resize_tensor(seg,
new_size=(self.h_s, self.w_s))
self.SEGl_s[0, :,
-1, :, :] = resize_tensor(segl,
new_size=(self.h_s, self.w_s))
self.SEGr_s[0, :,
-1, :, :] = resize_tensor(segr,
new_size=(self.h_s, self.w_s))
self.I_ff[0, :, 0, :, :] = x
self.Il_ff[0, :, 0, :, :] = xl
self.Ir_ff[0, :, 0, :, :] = xr
self.OF_ff[0, :, 0, :, :] = of
self.OFl_ff[0, :, 0, :, :] = ofl
self.OFr_ff[0, :, 0, :, :] = ofr
self.SEG_ff[0, :, 0, :, :] = seg
self.SEGl_ff[0, :, 0, :, :] = segl
self.SEGr_ff[0, :, 0, :, :] = segr
y_sal = read_image(join(self.sequence_dir, 'saliency',
'{:06d}.png'.format(self.fr)),
channels_first=False,
color=False)
yl_sal = translate(y_sal, pixels=500, side='left')
yr_sal = translate(y_sal, pixels=500, side='right')
self.Y_sal[0, 0] = y_sal
self.Yl_sal[0, 0] = yl_sal
self.Yr_sal[0, 0] = yr_sal
y_fix = read_image(join(self.sequence_dir, 'saliency_fix',
'{:06d}.png'.format(self.fr)),
channels_first=False,
color=False)
yl_fix = translate(y_fix, pixels=500, side='left')
yr_fix = translate(y_fix, pixels=500, side='right')
self.Y_fix[0, 0] = y_fix
self.Yl_fix[0, 0] = yl_fix
self.Yr_fix[0, 0] = yr_fix
def load_dreyeve_sample(sequence_dir, stop, mean_dreyeve_image):
"""
Function to load a dreyeve_sample.
:param sequence_dir: string, sequence directory (e.g. 'Z:/DATA/04/').
:param stop: int, sample to load in (15, 7499). N.B. this is the sample where prediction occurs!
:param mean_dreyeve_image: mean dreyeve image, subtracted to each frame.
:param frames_per_seq: number of temporal frames for each sample
:param h: h
:param w: w
:return: a dreyeve_sample like I, OF, SEG
"""
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
OF_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
OF_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
OF_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
SEG_ff = np.zeros(shape=(1, 19, 1, h, w), dtype='float32')
SEG_s = np.zeros(shape=(1, 19, frames_per_seq, h_s, w_s), dtype='float32')
SEG_c = np.zeros(shape=(1, 19, frames_per_seq, h_c, w_c), dtype='float32')
Y_sal = np.zeros(shape=(1, 1, h, w), dtype='float32')
Y_fix = np.zeros(shape=(1, 1, h, w), dtype='float32')
for fr in xrange(0, frames_per_seq):
offset = stop - frames_per_seq + 1 + fr # tricky
# read image
x = read_image(join(sequence_dir, 'frames',
'{:06d}.jpg'.format(offset)),
channels_first=True,
resize_dim=(h, w)) - mean_dreyeve_image
I_s[0, :, fr, :, :] = resize_tensor(x, new_size=(h_s, w_s))
# read of
of = read_image(join(sequence_dir, 'optical_flow',
'{:06d}.png'.format(offset + 1)),
channels_first=True,
resize_dim=(h, w))
of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean
OF_s[0, :, fr, :, :] = resize_tensor(of, new_size=(h_s, w_s))
# read semseg
seg = resize_tensor(np.load(
join(sequence_dir, 'semseg',
'{:06d}.npz'.format(offset)))['arr_0'][0],
new_size=(h, w))
SEG_s[0, :, fr, :, :] = resize_tensor(seg, new_size=(h_s, w_s))
I_ff[0, :, 0, :, :] = x
OF_ff[0, :, 0, :, :] = of
SEG_ff[0, :, 0, :, :] = seg
Y_sal[0, 0] = read_image(join(sequence_dir, 'saliency',
'{:06d}.png'.format(stop)),
channels_first=False,
color=False,
resize_dim=(h, w))
Y_fix[0, 0] = read_image(join(sequence_dir, 'saliency_fix',
'{:06d}.png'.format(stop)),
channels_first=False,
color=False,
resize_dim=(h, w))
return [I_ff, I_s, I_c, OF_ff, OF_s, OF_c, SEG_ff, SEG_s,
SEG_c], [Y_sal, Y_fix]
normalize(X[3][0, :, 0, :, :].transpose(1, 2, 0)),
normalize(
seg_to_colormap(np.argmax(X[6][0, :, 0, :, :], axis=0),
channels_first=False))
],
layout=(3, 1),
resize_dim=(720, 720))
y_stitch = stitch_together([
np.tile(normalize(Y_image[0].transpose(1, 2, 0)),
reps=(1, 1, 3)),
np.tile(normalize(Y_flow[0].transpose(1, 2, 0)),
reps=(1, 1, 3)),
np.tile(normalize(Y_semseg[0].transpose(1, 2, 0)),
reps=(1, 1, 3))
],
layout=(3, 1),
resize_dim=(720, 720))
y_tot = np.tile(normalize(
resize_tensor(Y_dreyevenet[0],
new_size=(720, 720)).transpose(1, 2, 0)),
reps=(1, 1, 3))
cv2.imshow(
'prediction',
stitch_together([x_stitch, y_stitch, y_tot],
layout=(1, 3),
resize_dim=(500, 1500)))
cv2.waitKey(1)