def main():
prediction_root = '/majinbu/public/DREYEVE/PREDICTIONS_RMDN'
gaussian_path = '/majinbu/public/DREYEVE/DATA/dreyeve_mean_train_gt_fix.png'
central_gaussian = read_image(gaussian_path,
channels_first=False,
color=False)
test_sequences = range(38, 74 + 1)
klds = []
for seq in test_sequences:
seq_root = join(prediction_root, '{:02d}'.format(seq), 'output')
for frame_idx in tqdm(
range(15, 7500, 5),
desc='Processing sequence {:02d}...'.format(seq)):
img = read_image(join(seq_root, '{:06d}.png'.format(frame_idx)),
channels_first=False,
color=False,
resize_dim=(1080, 1920))
klds.append(kld_numeric(img, central_gaussian))
print np.mean(klds)
def on_epoch_end(self, epoch, logs={}):
# create epoch folder
epoch_out_dir = join(self.out_dir_path, '{:02d}'.format(epoch))
if not exists(epoch_out_dir):
os.makedirs(epoch_out_dir)
# load batch
seqs, frs, X, Y = RMDN_batch(batchsize=batchsize, mode='val')
# predict batch
Z = self.model.predict(X)
for b in range(0, batchsize):
for t in range(0, T):
seq = seqs[b, t]
id = frs[b, t]
# load this frame image and gt
rgb_frame = read_image(join(DREYEVE_ROOT, '{:02d}'.format(seq), 'frames', '{:06d}.jpg'.format(id)),
channels_first=False, resize_dim=(h, w), dtype=np.uint8)
fixation_map = read_image(join(DREYEVE_ROOT, '{:02d}'.format(seq), 'saliency_fix',
'{:06d}.png'.format(id+1)),
channels_first=False, resize_dim=(h, w), dtype=np.uint8)
# extract this frame mixture
gmm = Z[b, t]
pred_map = gmm_to_probability_map(gmm=gmm, image_size=(h, w))
pred_map = normalize(pred_map)
pred_map = np.tile(np.expand_dims(pred_map, axis=-1), reps=(1, 1, 3))
# stitch
stitch = stitch_together([rgb_frame, pred_map, fixation_map], layout=(1, 3))
write_image(join(epoch_out_dir, '{:02d}_{:02d}.jpg'.format(b, t)), stitch)
def compute_metrics_for_mean_gt(sequences):
"""
Function to compute metrics using the mean training gt as prediction.
:param sequences: A list of sequences to consider.
"""
# some variables
gt_h, gt_w = 1080, 1920
pred_dir = 'Z:\\PREDICTIONS_MEAN_TRAIN_GT'
dreyeve_dir = 'Z:\\DATA'
mean_gt_sal = read_image(join(dreyeve_dir, 'dreyeve_mean_train_gt.png'),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
mean_gt_fix = read_image(join(dreyeve_dir,
'dreyeve_mean_train_gt_fix.png'),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
ig_baseline = read_image(join(dreyeve_dir,
'dreyeve_mean_train_gt_fix.png'),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
for seq in sequences:
print('Processing sequence {}'.format(seq))
# gt dirs
seq_gt_sal_dir = join(dreyeve_dir, '{:02d}'.format(seq), 'saliency')
seq_gt_fix_dir = join(dreyeve_dir, '{:02d}'.format(seq),
'saliency_fix')
print('Computing metrics...')
metric_saver = MetricSaver(pred_dir, seq, model='mean_gt')
for fr in tqdm(xrange(15, 7500 - 1, 5)):
# load gts
gt_sal = read_image(join(seq_gt_sal_dir,
'{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
gt_fix = read_image(join(seq_gt_fix_dir,
'{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
# feed the saver
metric_saver.feed(fr,
predictions=[mean_gt_sal, mean_gt_fix],
groundtruth=[gt_sal, gt_fix],
ig_baseline=ig_baseline)
# save mean values
metric_saver.save_mean_metrics()
def compute_metrics_for_old_model(sequences):
"""
Function to compute metrics out of the old model (2015).
:param sequences: A list of sequences to consider.
"""
# some variables
gt_h, gt_w = 1080, 1920
pred_dir = 'Z:\\PREDICTIONS\\architecture7'
dreyeve_dir = 'Z:\\DATA'
ig_baseline = read_image(join(dreyeve_dir,
'dreyeve_mean_train_gt_fix.png'),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
for seq in sequences:
print('Processing sequence {}'.format(seq))
# prediction dirs
seq_pred_dir = join(pred_dir, '{:02d}'.format(seq), 'output')
# gt dirs
seq_gt_sal_dir = join(dreyeve_dir, '{:02d}'.format(seq), 'saliency')
seq_gt_fix_dir = join(dreyeve_dir, '{:02d}'.format(seq),
'saliency_fix')
print('Computing metrics...')
metric_saver = MetricSaver(pred_dir, seq, model='old')
for fr in tqdm(xrange(15, 7500 - 1, 5)):
# load predictions
p = read_image(join(seq_pred_dir, '{:06}.png'.format(fr + 1)),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
# load gts
gt_sal = read_image(join(seq_gt_sal_dir,
'{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
gt_fix = read_image(join(seq_gt_fix_dir,
'{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
# feed the saver
metric_saver.feed(fr,
predictions=[p],
groundtruth=[gt_sal, gt_fix],
ig_baseline=ig_baseline)
# save mean values
metric_saver.save_mean_metrics()
def load_dreyeve_sample(sequence_dir, sample, mean_dreyeve_image, frames_per_seq=16, h=448, w=448, ):
"""
Function to load a dreyeve_sample.
:param sequence_dir: string, sequence directory (e.g. 'Z:/DATA/04/').
:param sample: int, sample to load in (15, 7499). N.B. this is the sample where prediction occurs!
:param mean_dreyeve_image: mean dreyeve image, subtracted to each frame.
:param frames_per_seq: number of temporal frames for each sample
:param h: h
:param w: w
:return: a dreyeve_sample like I, OF, SEG
"""
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32')
I_s = np.zeros(shape=(1, frames_per_seq, h_s, w_s,3), dtype='float32')
I_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32')
SEG_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32')
SEG_s = np.zeros(shape=(1,frames_per_seq, h_s, w_s,3), dtype='float32')
SEG_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32')
OF_ff = np.zeros(shape=(1, 1, h, w,3), dtype='float32')
OF_s = np.zeros(shape=(1, frames_per_seq, h_s, w_s,3), dtype='float32')
OF_c = np.zeros(shape=(1, frames_per_seq, h_c, w_c,3), dtype='float32')
for fr in range(0, frames_per_seq):
offset = sample - frames_per_seq + 1 + fr # tricky
# read image
x = read_image(join(sequence_dir,'Frames','{}.jpg'.format(offset)),
channels_first=False, resize_dim=(h, w)) - mean_dreyeve_image
I_s[0, fr, :, :,:] = resize_tensor(x,new_shape=(h_s, w_s))
# read semseg
seg = resize_tensor(np.load(join(sequence_dir,'Seg', '{}.npz'.format(offset)))['arr_0'],
new_shape=(h, w))
SEG_s[0,fr,:, :, :] = resize_tensor(seg, new_shape=(h_s, w_s))
# read of
of = read_image(join(sequence_dir,'Optical', '{}.png'.format(offset + 1)),
channels_first=False, resize_dim=(h, w))
of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean
OF_s[0, fr, :, :, :] = resize_tensor(of, new_shape=(h_s, w_s))
I_ff[0,0, :, :,:] = x
SEG_ff[0,0, :, :, :] = seg
OF_ff[0, 0, :, :, :] = of
return [I_ff, I_s,I_c,OF_ff,OF_s,OF_c,SEG_ff, SEG_s, SEG_c]
def load_saliency_data_simo(signatures, nb_frames, image_size):
"""
Crop -> NSS with fixations
Fine -> KLD with saliency maps and IG with fixations (baseline is saliency maps)
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:return: a tuple holding the fullframe and the cropped saliency.
"""
batchsize = len(signatures)
h, w = image_size
h_c = h // 4
w_c = w // 4
Y = np.zeros(shape=(batchsize, 2, h, w), dtype=np.float32)
Y_c = np.zeros(shape=(batchsize, 1, h_c, w_c), dtype=np.float32)
for b in range(0, batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
y_dir_sal = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency')
y_dir_fix = join(dreyeve_dir, '{:02d}'.format(num_run), 'saliency_fix')
# saliency
y_sal = read_image(join(y_dir_sal,
'{:06d}.png'.format(start + nb_frames - 1)),
channels_first=True,
color=False,
resize_dim=image_size) / 255
y_fix = read_image(join(y_dir_fix,
'{:06d}.png'.format(start + nb_frames - 1)),
channels_first=True,
color=False,
resize_dim=image_size) / 255
# resize to (256, 256) before cropping
y_before_crop = resize_tensor(np.expand_dims(y_fix, axis=0),
new_shape=frame_size_before_crop)
Y[b, 0, :, :] = y_sal
Y[b, 1, :, :] = y_fix
Y_c[b, 0, :, :] = crop_tensor(y_before_crop,
indexes=(hc1, hc2, wc1, wc2))[0]
if do_mirror:
Y = Y[:, :, :, ::-1]
Y_c = Y_c[:, :, :, ::-1]
return [Y, Y_c]
def __init__(self, dreyeve_root):
self.dreyeve_root = dreyeve_root
self.dreyeve_data_root = join(dreyeve_root, 'DATA')
self.subseq_file = join(dreyeve_root, 'subsequences.txt')
# load subsequences
self.subseqs = np.loadtxt(self.subseq_file, dtype=str)
# filter attentive
self.subseqs = self.subseqs[self.subseqs[:, -1] == 'k']
# cast to int
self.subseqs = np.int32(self.subseqs[:, :-1])
# filter test sequences
self.subseqs = np.array(
[seq for seq in self.subseqs if seq[0] in dreyeve_test_seq])
# filter too short sequences
self.subseqs = np.array(
[seq for seq in self.subseqs if seq[2] - seq[1] >= frames_per_seq])
self.len = len(self.subseqs)
self.counter = 0
# load mean dreyeve image
self.mean_dreyeve_image = read_image(join(self.dreyeve_data_root,
'dreyeve_mean_frame.png'),
channels_first=True,
resize_dim=(h, w))
def load_dreyeve_sample(sequence_dir,
sample,
mean_dreyeve_image,
frames_per_seq=16,
h=448,
w=448):
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
for fr in range(frames_per_seq):
offset = sample - frames_per_seq + 1 + fr
# read image
x = read_image(join(sequence_dir, 'frames', '{}.jpg'.format(offset)),
channels_first=True,
resize_dim=(h, w)) - mean_dreyeve_image
I_s[0, :, fr, :, :] = resize_tensor(x, new_size=(h_s, w_s))
I_ff[0, :, 0, :, :] = x
return [I_ff, I_s, I_c]
def load_sample_for_encoding(frames_dir, sample_number, temporal_window,
img_size):
"""
Loads a sample to be encoded by C3D.
:param frames_dir: directory where the frames are.
:param sample_number: the frame index.
:param temporal_window: the temporal window we consider (16 frames).
:param img_size: the size of images C3D has to be fed with.
:return: a ndarray having shape (1, 3, temporal_window, img_size[0], img_size[1].
"""
h, w = img_size
sample = np.zeros(shape=(3, temporal_window, h, w), dtype=np.float32)
# if sample_number==15 --> [0,15]
start = sample_number - temporal_window + 1
stop = sample_number + 1
for f in xrange(start, stop):
img = read_image(img_path=join(frames_dir, '{:06d}.jpg'.format(f)),
channels_first=True,
resize_dim=(h, w))
t_idx = f - start
sample[:, t_idx, :, :] = img
# add batch dimension
sample = np.expand_dims(sample, axis=0)
return sample
def load_saliency_data(signatures, nb_frames, image_size, gt_type='fix'):
"""
Function to load a saliency batch.
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:param gt_type: choose among `sal` (old groundtruth) and `fix` (new groundtruth).
:return: a tuple holding the fullframe and the cropped saliency.
"""
batchsize = len(signatures)
h, w = image_size
h_c = h // 4
w_c = w // 4
Y = np.zeros(shape=(batchsize, h, w, 1), dtype=np.float32)
Y_c = np.zeros(shape=(batchsize, h_c, w_c, 1), dtype=np.float32)
for b in range(0, batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
y_dir = join(dreyeve_dir, "DREYEVE_DATA", '{:02d}'.format(num_run),
'output_frames' if gt_type == 'sal' else 'output_frames')
# saliency
y = read_image(join(y_dir, '{}.jpg'.format(start + nb_frames - 1)),
channels_first=False,
color=False,
resize_dim=image_size)
# resize to (256, 256) before cropping
y_before_crop = resize_tensor(np.expand_dims(y, axis=2),
new_shape=frame_size_before_crop)
Y[b, :, :, 0] = y
Y_c[b, :, :, 0] = crop_tensor(y_before_crop,
indexes=(hc1, hc2, wc1, wc2))[0]
if do_mirror:
Y = Y[:, :, :, ::-1]
Y_c = Y_c[:, :, :, ::-1]
return [Y, Y_c]
def predict(filepath,mean_path):
frames_per_seq, h, w = 16, 448, 448
dreyevenet_model = DreyeveNet(frames_per_seq=frames_per_seq, h=h, w=w)
dreyevenet_model.load_weights('Best_Weights/Tune.h5')
mean_dreyeve_image = read_image(mean_path,channels_first=False, resize_dim=(h, w))
output_dir = join(filepath,'npz')
if not os.path.exists(output_dir):
os.mkdir(output_dir)
for sample in tqdm(range(0 + 15 , 749)):
X = load_dreyeve_sample(filepath, sample=sample, mean_dreyeve_image=mean_dreyeve_image,
frames_per_seq=frames_per_seq, h=h, w=w)#heeeerrrrrr
#Y_dreyevenet = dreyevenet_model.predict(X)[0] # get only [fine_out][remove batch]
Y_image = dreyevenet_model.predict(X)[0] # predict on image
#output folder here output will be npz files not photos
np.savez_compressed(output_dir + "/{}".format(sample),Y_image)
def RMDN_batch(batchsize, mode):
"""
Function to provide a batch for RMDN training.
:param batchsize: batchsize.
:param mode: choose among [`train`, `val`, `test`].
:return: a batch like sequence numbers, frame ids, X, Y
"""
assert mode in ['train', 'val', 'test'], 'Unknown mode {} for dreyeve batch loader'.format(mode)
if mode == 'train':
sequences = dreyeve_train_seq
allowed_frames = train_frame_range
elif mode == 'val':
sequences = dreyeve_train_seq
allowed_frames = val_frame_range
elif mode == 'test':
sequences = dreyeve_test_seq
allowed_frames = test_frame_range
seqs = np.zeros(shape=(batchsize, T), dtype=np.uint32)
frs = np.zeros(shape=(batchsize, T), dtype=np.uint32)
X = np.zeros(shape=(batchsize, T, encoding_dim), dtype=np.float32)
Y = np.zeros(shape=(batchsize, T, h, w))
for b in xrange(0, batchsize):
# sample a sequence
seq = np.random.choice(sequences)
seq_enc_dir = join(DREYEVE_ROOT, '{:02d}'.format(seq), 'c3d_encodings')
seq_gt_dir = join(DREYEVE_ROOT, '{:02d}'.format(seq), 'saliency_fix')
# choose a random sample
fr = np.random.choice(allowed_frames)
for offset in range(0, T):
c3d_encoding = np.load(join(seq_enc_dir, '{:06d}.npz'.format(fr+offset)))['arr_0']
gt = read_image(join(seq_gt_dir, '{:06d}.png'.format(fr+offset+1)), channels_first=False, color=False,
resize_dim=(h, w))
seqs[b, offset] = seq
frs[b, offset] = fr+offset
X[b, offset] = c3d_encoding
Y[b, offset] = gt
return seqs, frs, X, Y
def load(seq, frame):
"""
Function to load a 16-frames sequence to plot
:param seq: the sequence number
:param frame: the frame inside the sequence
:return: a stitched image to show
"""
small_size = (270, 480)
sequence_x_dir = join(dreyeve_dir, '{:02d}'.format(seq))
sequence_y_dir = join(pred_dir, '{:02d}'.format(seq))
# x
x_img = read_image(join(sequence_x_dir, 'frames',
'{:06d}.jpg'.format(frame)),
channels_first=False,
color_mode='BGR',
dtype=np.uint8)
x_img_small = cv2.resize(x_img, small_size[::-1])
of_img = read_image(join(sequence_x_dir, 'optical_flow',
'{:06d}.png'.format(frame + 1)),
channels_first=False,
color_mode='BGR',
resize_dim=small_size)
seg_img = seg_to_colormap(np.argmax(np.squeeze(
np.load(join(sequence_x_dir, 'semseg',
'{:06d}.npz'.format(frame)))['arr_0']),
axis=0),
channels_first=False)
seg_img = cv2.cvtColor(seg_img, cv2.COLOR_RGB2BGR)
# pred
image_p = normalize(
np.squeeze(
np.load(
join(sequence_y_dir, 'image_branch',
'{:06d}.npz'.format(frame)))['arr_0']))
flow_p = normalize(
np.squeeze(
np.load(
join(sequence_y_dir, 'flow_branch',
'{:06d}.npz'.format(frame)))['arr_0']))
semseg_p = normalize(
np.squeeze(
np.load(
join(sequence_y_dir, 'semseg_branch',
'{:06d}.npz'.format(frame)))['arr_0']))
dreyevenet_p = normalize(
np.squeeze(
np.load(
join(sequence_y_dir, 'dreyeveNet',
'{:06d}.npz'.format(frame)))['arr_0']))
image_p = cv2.resize(cv2.cvtColor(image_p, cv2.COLOR_GRAY2BGR),
small_size[::-1])
flow_p = cv2.resize(cv2.cvtColor(flow_p, cv2.COLOR_GRAY2BGR),
small_size[::-1])
semseg_p = cv2.resize(cv2.cvtColor(semseg_p, cv2.COLOR_GRAY2BGR),
small_size[::-1])
dreyevenet_p = cv2.resize(cv2.cvtColor(dreyevenet_p, cv2.COLOR_GRAY2BGR),
small_size[::-1])
s1 = stitch_together(
[x_img_small, of_img, seg_img, image_p, flow_p, semseg_p],
layout=(2, 3))
x_img = cv2.resize(x_img, dsize=(s1.shape[1], s1.shape[0]))
dreyevenet_p = cv2.resize(dreyevenet_p, dsize=(s1.shape[1], s1.shape[0]))
dreyevenet_p = cv2.applyColorMap(dreyevenet_p, cv2.COLORMAP_JET)
blend = cv2.addWeighted(x_img, 0.5, dreyevenet_p, 0.5, gamma=0)
stitch = stitch_together([s1, blend], layout=(2, 1), resize_dim=(720, 970))
return stitch
success, image = vidcap.read()
count = 0
success = True
while success:
frames.append(image)
success, image = vidcap.read()
print('Finished reading video!')
print('Now starting prediction...')
demo_dir = 'demo_images'
# load mean dreyeve image
mean_dreyeve_image = read_image(join(demo_dir, 'dreyeve_mean_frame.png'),
channels_first=True,
resize_dim=(h, w))
# get the models
image_branch = SaliencyBranch(input_shape=(3, frames_per_seq, h, w),
c3d_pretrained=True,
branch='image')
image_branch.compile(optimizer='adam', loss='kld')
if (args.gt_type == 'sage'):
image_branch.load_weights(
'../pretrained_models/dreyeve/sage_image_branch.h5'
) # load weights
else:
image_branch.load_weights(
'../pretrained_models/dreyeve/bdda_image_branch.h5'
def save_competitive_figures(out_dir,
competitive_list,
prediction_dir,
resize_dim=(540, 960)):
"""
Function that takes a list of competitive results and saves figures in a directory, for a future paper figure.
:param out_dir: directory where to save figures.
:param competitive_list: list of tuples like (sequence, frame, gap) ordered by decreasing gap.
:param prediction_dir: directory where our predictions stored.
:param resize_dim: optional resize dim for output figures.
"""
stitches_dir = join(out_dir, 'stitches')
if not os.path.exists(stitches_dir):
os.makedirs(stitches_dir)
for i, competitive_result in enumerate(competitive_list):
seq, frame, _ = competitive_result
frame = int(frame)
this_sample_dir = join(out_dir, '{:06d}'.format(i))
if not os.path.exists(this_sample_dir):
os.makedirs(this_sample_dir)
# read frame
im = read_image(join(dreyeve_root, 'DATA', '{:02d}'.format(seq),
'frames', '{:06d}.jpg'.format(frame)),
channels_first=False,
color_mode='BGR',
dtype=np.uint8,
resize_dim=resize_dim)
cv2.imwrite(
join(this_sample_dir,
'R{:02d}_frame_{:06d}_000.jpg'.format(seq, frame)), im)
# read gt and blend
gt = read_image(join(dreyeve_root, 'DATA', '{:02d}'.format(seq),
'saliency_fix', '{:06d}.png'.format(frame + 1)),
channels_first=False,
color=False,
dtype=np.float32,
resize_dim=resize_dim)
gt = blend_map(im, gt, factor=0.5)
cv2.imwrite(
join(this_sample_dir,
'R{:02d}_frame_{:06d}_001.jpg'.format(seq, frame)), gt)
# read pred and blend
my_pred = np.squeeze(
np.load(
join(dreyeve_root, prediction_dir, '{:02d}'.format(seq),
'dreyeveNet', '{:06d}.npz'.format(frame)))['arr_0'])
my_pred = cv2.resize(my_pred, dsize=resize_dim[::-1])
my_pred = blend_map(im, my_pred, factor=0.5)
cv2.imwrite(
join(this_sample_dir,
'R{:02d}_frame_{:06d}_002.jpg'.format(seq, frame)), my_pred)
to_stitch = [im, gt, my_pred]
for c, competitor in enumerate(
['image_branch', 'flow_branch', 'semseg_branch']):
# read competitor result
comp_pred = np.squeeze(
np.load(
join(dreyeve_root, prediction_dir, '{:02d}'.format(seq),
competitor, '{:06d}.npz'.format(frame)))['arr_0'])
comp_pred = cv2.resize(comp_pred, dsize=resize_dim[::-1])
comp_pred = blend_map(im, comp_pred, factor=0.5)
to_stitch.append(comp_pred)
cv2.imwrite(
join(
this_sample_dir, 'R{:02d}_frame_{:06d}_{:03d}.jpg'.format(
seq, frame, c + 3)), comp_pred)
stitch = stitch_together(to_stitch, layout=(1, len(to_stitch)))
cv2.imwrite(join(stitches_dir, '{:06d}.jpg'.format(i)), stitch)
def load_batch_data(signatures, nb_frames, image_size, batch_type):
"""
Function to load a data batch. This is common for `image`, `optical_flow` and `semseg`.
:param signatures: sample signatures, previously evaluated. List of tuples like
(num_run, start, hc1, hc2, wc1, wc2). The list is batchsize signatures long.
:param nb_frames: number of temporal frames in each sample.
:param image_size: tuple in the form (h,w). This refers to the fullframe image.
:param batch_type: choose among [`image`, `optical_flow`, `semseg`].
:return: a tuple holding the fullframe, the small and the cropped batch.
"""
assert batch_type in ['image', 'optical_flow', 'semseg'
], 'Unknown batch type: {}'.format(batch_type)
batchsize = len(signatures)
h, w = image_size
h_s = h_c = h // 4
w_s = w_c = w // 4
if batch_type == 'image':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'DREYEVE_DATA'
# mean frame image, to subtract mean
elif batch_type == 'optical_flow':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'optical_flow'
elif batch_type == 'semseg':
B_ff = np.zeros(shape=(batchsize, 1, h, w, 3), dtype=np.float32)
B_s = np.zeros(shape=(batchsize, nb_frames, h_s, w_s, 3),
dtype=np.float32)
B_c = np.zeros(shape=(batchsize, nb_frames, h_c, w_c, 3),
dtype=np.float32)
subdir = 'Segmentation'
for b in range(batchsize):
# retrieve the signature
num_run, start, hc1, hc2, wc1, wc2, do_mirror = signatures[b]
data_dir = join(dreyeve_dir, subdir, '{:02d}'.format(num_run))
mean_image = read_image(join(dreyeve_dir, 'DREYEVE_DATA',
'{:02d}'.format(num_run),
'mean_frame.png'),
channels_first=False,
resize_dim=image_size)
for offset in range(0, nb_frames):
if batch_type == 'image':
x = read_image(join(data_dir, 'Frames',
'{}.jpg'.format(start + offset)),
channels_first=False,
resize_dim=image_size) - mean_image
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
elif batch_type == 'optical_flow':
x = read_image(join(data_dir,
'{}.png'.format(start + offset + 1)),
channels_first=False,
resize_dim=image_size)
x -= np.mean(x, axis=(1, 2), keepdims=True) # remove mean
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
elif batch_type == 'semseg':
x = resize_tensor(np.load(
join(data_dir, '{}.npz'.format(start + offset)))['arr_0'],
new_shape=image_size)
# resize to (256, 256) before cropping
x_before_crop = resize_tensor(x,
new_shape=frame_size_before_crop)
B_s[b, offset, :, :, :] = resize_tensor(x,
new_shape=(h_s, w_s))
B_c[b, offset, :, :, :] = crop_tensor(x_before_crop,
indexes=(hc1, hc2, wc1,
wc2))
B_ff[b, 0, :, :, :] = x
if do_mirror:
B_ff = B_ff[:, :, :, :, ::-1]
B_s = B_s[:, :, :, :, ::-1]
B_c = B_c[:, :, :, :, ::-1]
return [B_ff, B_s, B_c]
def load_dreyeve_sample(sequence_dir, stop, mean_dreyeve_image):
"""
Function to load a dreyeve_sample.
:param sequence_dir: string, sequence directory (e.g. 'Z:/DATA/04/').
:param stop: int, sample to load in (15, 7499). N.B. this is the sample where prediction occurs!
:param mean_dreyeve_image: mean dreyeve image, subtracted to each frame.
:param frames_per_seq: number of temporal frames for each sample
:param h: h
:param w: w
:return: a dreyeve_sample like I, OF, SEG
"""
h_c = h_s = h // 4
w_c = w_s = h // 4
I_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
I_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
I_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
OF_ff = np.zeros(shape=(1, 3, 1, h, w), dtype='float32')
OF_s = np.zeros(shape=(1, 3, frames_per_seq, h_s, w_s), dtype='float32')
OF_c = np.zeros(shape=(1, 3, frames_per_seq, h_c, w_c), dtype='float32')
SEG_ff = np.zeros(shape=(1, 19, 1, h, w), dtype='float32')
SEG_s = np.zeros(shape=(1, 19, frames_per_seq, h_s, w_s), dtype='float32')
SEG_c = np.zeros(shape=(1, 19, frames_per_seq, h_c, w_c), dtype='float32')
Y_sal = np.zeros(shape=(1, 1, h, w), dtype='float32')
Y_fix = np.zeros(shape=(1, 1, h, w), dtype='float32')
for fr in xrange(0, frames_per_seq):
offset = stop - frames_per_seq + 1 + fr # tricky
# read image
x = read_image(join(sequence_dir, 'frames',
'{:06d}.jpg'.format(offset)),
channels_first=True,
resize_dim=(h, w)) - mean_dreyeve_image
I_s[0, :, fr, :, :] = resize_tensor(x, new_size=(h_s, w_s))
# read of
of = read_image(join(sequence_dir, 'optical_flow',
'{:06d}.png'.format(offset + 1)),
channels_first=True,
resize_dim=(h, w))
of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean
OF_s[0, :, fr, :, :] = resize_tensor(of, new_size=(h_s, w_s))
# read semseg
seg = resize_tensor(np.load(
join(sequence_dir, 'semseg',
'{:06d}.npz'.format(offset)))['arr_0'][0],
new_size=(h, w))
SEG_s[0, :, fr, :, :] = resize_tensor(seg, new_size=(h_s, w_s))
I_ff[0, :, 0, :, :] = x
OF_ff[0, :, 0, :, :] = of
SEG_ff[0, :, 0, :, :] = seg
Y_sal[0, 0] = read_image(join(sequence_dir, 'saliency',
'{:06d}.png'.format(stop)),
channels_first=False,
color=False,
resize_dim=(h, w))
Y_fix[0, 0] = read_image(join(sequence_dir, 'saliency_fix',
'{:06d}.png'.format(stop)),
channels_first=False,
color=False,
resize_dim=(h, w))
return [I_ff, I_s, I_c, OF_ff, OF_s, OF_c, SEG_ff, SEG_s,
SEG_c], [Y_sal, Y_fix]
norm_s_map = (s_map - np.min(s_map)) / (
(np.max(s_map) - np.min(s_map)) * 1.0)
return 255.0 * norm_s_map
if __name__ == '__main__':
img_shape = (1280, 720)
demo_dir = 'demo_images/'
img_name = 'demo_img.jpg'
im = read_image(join(demo_dir, 'frames', img_name),
channels_first=False,
color=True,
dtype=np.float32,
resize_dim=img_shape)
bdda_pred = read_image(join(demo_dir, 'preds', 'bdda_pred.jpg'),
channels_first=False,
color=False,
dtype=np.float32,
resize_dim=img_shape)
sage_pred = read_image(join(demo_dir, 'preds', 'sage_pred.jpg'),
channels_first=False,
color=False,
dtype=np.float32,
resize_dim=img_shape)
sample=16,
shape_c=shape_c,
shape_r=shape_r)
# predict sample
P_bdda = model_bdda.predict(X)
P_bdda = np.squeeze(P_bdda)
P_bdda = postprocess_predictions(P_bdda, shape_r, shape_c)
P_sage = model_sage.predict(X)
P_sage = np.squeeze(P_sage)
P_sage = postprocess_predictions(P_sage, shape_r, shape_c)
im = read_image(join(demo_dir, 'demo_img.jpg'), channels_first=True)
h, w = im.shape[1], im.shape[2]
bdda_pred = P_bdda
sage_pred = P_sage
bdda_pred = cv2.resize(bdda_pred, dsize=(w, h))
sage_pred = cv2.resize(sage_pred, dsize=(w, h))
im = normalize_map(im)
bdda_pred = normalize_map(bdda_pred)
sage_pred = normalize_map(sage_pred)
im = im.astype(np.uint8)
bdda_pred = bdda_pred.astype(np.uint8)
sage_pred = sage_pred.astype(np.uint8)
verbose = True
# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("--seq")
parser.add_argument("--pred_dir")
args = parser.parse_args()
assert args.seq is not None, 'Please provide a correct dreyeve sequence'
assert args.pred_dir is not None, 'Please provide a correct pred_dir'
dreyeve_dir = '/majinbu/public/DREYEVE/DATA' # local
# load mean dreyeve image
mean_dreyeve_image = read_image(join(dreyeve_dir,
'dreyeve_mean_frame.png'),
channels_first=True,
resize_dim=(h, w))
# get the models
dreyevenet_model = DreyeveNet(frames_per_seq=frames_per_seq, h=h, w=w)
dreyevenet_model.compile(optimizer='adam', loss='kld') # do we need this?
dreyevenet_model.load_weights(
'dreyevenet_model_central_crop.h5') # load weights
# set up pred directory
image_pred_dir = join(args.pred_dir, '{:02d}'.format(int(args.seq)),
'blend')
makedirs([image_pred_dir])
sequence_dir = join(dreyeve_dir, '{:02d}'.format(int(args.seq)))
def compute_metrics_for_new_model(sequences):
"""
Function to compute metrics out of the new model (2017).
:param sequences: A list of sequences to consider.
"""
# some variables
gt_h, gt_w = 1080, 1920
pred_dir = 'Z:\\PREDICTIONS_2017'
dreyeve_dir = 'Z:\\DATA'
ig_baseline = read_image(join(dreyeve_dir,
'dreyeve_mean_train_gt_fix.png'),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
for seq in sequences:
print('Processing sequence {}'.format(seq))
# prediction dirs
dir_pred_dreyevenet = join(pred_dir, '{:02d}'.format(seq),
'dreyeveNet')
dir_pred_image = join(pred_dir, '{:02d}'.format(seq), 'image_branch')
dir_pred_flow = join(pred_dir, '{:02d}'.format(seq), 'flow_branch')
dir_pred_seg = join(pred_dir, '{:02d}'.format(seq), 'semseg_branch')
# gt dirs
dir_gt_sal = join(dreyeve_dir, '{:02d}'.format(seq), 'saliency')
dir_gt_fix = join(dreyeve_dir, '{:02d}'.format(seq), 'saliency_fix')
print('Computing metrics...')
metric_saver = MetricSaver(pred_dir, seq, model='new')
ablation = AblationStudy(pred_dir, seq)
for fr in tqdm(xrange(15, 7500 - 1, 5)):
# load predictions
p_dreyeve = np.squeeze(
np.load(join(dir_pred_dreyevenet,
'{:06}.npz'.format(fr)))['arr_0'])
p_image = np.squeeze(
np.load(join(dir_pred_image, '{:06}.npz'.format(fr)))['arr_0'])
p_flow = np.squeeze(
np.load(join(dir_pred_flow, '{:06}.npz'.format(fr)))['arr_0'])
p_seg = np.squeeze(
np.load(join(dir_pred_seg, '{:06}.npz'.format(fr)))['arr_0'])
p_dreyeve = cv2.resize(p_dreyeve, dsize=(gt_h, gt_w)[::-1])
p_image = cv2.resize(p_image, dsize=(gt_h, gt_w)[::-1])
p_flow = cv2.resize(p_flow, dsize=(gt_h, gt_w)[::-1])
p_seg = cv2.resize(p_seg, dsize=(gt_h, gt_w)[::-1])
# load gts
gt_sal = read_image(join(dir_gt_sal, '{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
gt_fix = read_image(join(dir_gt_fix, '{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
# feed the saver
metric_saver.feed(fr,
predictions=[p_dreyeve, p_image, p_flow, p_seg],
groundtruth=[gt_sal, gt_fix],
ig_baseline=ig_baseline)
ablation.feed(fr,
predictions=[p_dreyeve, p_image, p_flow, p_seg],
groundtruth=[gt_sal, gt_fix],
ig_baseline=ig_baseline)
# save mean values
metric_saver.save_mean_metrics()
ablation.save_mean_metrics()
def roll(self):
self.fr += 1
self.I_s, self.I_c, self.OF_s, self.OF_c, self.SEG_s, self.SEG_c, \
self.Il_s, self.Il_c, self.OFl_s, self.OFl_c, self.SEGl_s, self.SEGl_c, \
self.Ir_s, self.Ir_c, self.OFr_s, self.OFr_c, self.SEGr_s, self.SEGr_c = [
np.roll(x, -1, axis=2) for x in
[self.I_s, self.I_c, self.OF_s, self.OF_c, self.SEG_s, self.SEG_c,
self.Il_s, self.Il_c, self.OFl_s, self.OFl_c, self.SEGl_s, self.SEGl_c,
self.Ir_s, self.Ir_c, self.OFr_s, self.OFr_c, self.SEGr_s, self.SEGr_c]]
# read image
x = read_image(join(self.sequence_dir, 'frames', '{:06d}.jpg'.format(self.fr)), channels_first=True,
resize_dim=(self.h, self.w)) \
- self.mean_dreyeve_image
xl = translate(x, pixels=116, side='left')
xr = translate(x, pixels=116, side='right')
self.I_s[0, :, -1, :, :] = resize_tensor(x,
new_size=(self.h_s, self.w_s))
self.Il_s[0, :,
-1, :, :] = resize_tensor(xl, new_size=(self.h_s, self.w_s))
self.Ir_s[0, :,
-1, :, :] = resize_tensor(xr, new_size=(self.h_s, self.w_s))
# read of
of = read_image(join(self.sequence_dir, 'optical_flow',
'{:06d}.png'.format(self.fr + 1)),
channels_first=True,
resize_dim=(h, w))
of -= np.mean(of, axis=(1, 2), keepdims=True) # remove mean
ofl = translate(of, pixels=116, side='left')
ofr = translate(of, pixels=116, side='right')
self.OF_s[0, :,
-1, :, :] = resize_tensor(of, new_size=(self.h_s, self.w_s))
self.OFl_s[0, :,
-1, :, :] = resize_tensor(ofl,
new_size=(self.h_s, self.w_s))
self.OFr_s[0, :,
-1, :, :] = resize_tensor(ofr,
new_size=(self.h_s, self.w_s))
# read semseg
seg = resize_tensor(np.load(
join(self.sequence_dir, 'semseg',
'{:06d}.npz'.format(self.fr)))['arr_0'][0],
new_size=(h, w))
segl = translate(seg, pixels=116, side='left')
segr = translate(seg, pixels=116, side='right')
self.SEG_s[0, :,
-1, :, :] = resize_tensor(seg,
new_size=(self.h_s, self.w_s))
self.SEGl_s[0, :,
-1, :, :] = resize_tensor(segl,
new_size=(self.h_s, self.w_s))
self.SEGr_s[0, :,
-1, :, :] = resize_tensor(segr,
new_size=(self.h_s, self.w_s))
self.I_ff[0, :, 0, :, :] = x
self.Il_ff[0, :, 0, :, :] = xl
self.Ir_ff[0, :, 0, :, :] = xr
self.OF_ff[0, :, 0, :, :] = of
self.OFl_ff[0, :, 0, :, :] = ofl
self.OFr_ff[0, :, 0, :, :] = ofr
self.SEG_ff[0, :, 0, :, :] = seg
self.SEGl_ff[0, :, 0, :, :] = segl
self.SEGr_ff[0, :, 0, :, :] = segr
y_sal = read_image(join(self.sequence_dir, 'saliency',
'{:06d}.png'.format(self.fr)),
channels_first=False,
color=False)
yl_sal = translate(y_sal, pixels=500, side='left')
yr_sal = translate(y_sal, pixels=500, side='right')
self.Y_sal[0, 0] = y_sal
self.Yl_sal[0, 0] = yl_sal
self.Yr_sal[0, 0] = yr_sal
y_fix = read_image(join(self.sequence_dir, 'saliency_fix',
'{:06d}.png'.format(self.fr)),
channels_first=False,
color=False)
yl_fix = translate(y_fix, pixels=500, side='left')
yr_fix = translate(y_fix, pixels=500, side='right')
self.Y_fix[0, 0] = y_fix
self.Yl_fix[0, 0] = yl_fix
self.Yr_fix[0, 0] = yr_fix
def save_competitive_figures(out_dir, competitive_list, prediction_dir, resize_dim=(540, 960)):
"""
Function that takes a list of competitive results and saves figures in a directory, for a future paper figure.
:param out_dir: directory where to save figures.
:param competitive_list: list of tuples like (sequence, frame, gap) ordered by decreasing gap.
:param prediction_dir: directory where our predictions stored.
:param resize_dim: optional resize dim for output figures.
"""
stitches_dir = join(out_dir, 'stitches')
if not os.path.exists(stitches_dir):
os.makedirs(stitches_dir)
for i, competitive_result in enumerate(competitive_list):
seq, frame, _ = competitive_result
frame = int(frame)
this_sample_dir = join(out_dir, '{:06d}'.format(i))
if not os.path.exists(this_sample_dir):
os.makedirs(this_sample_dir)
# read frame
im = read_image(join(dreyeve_root, 'DATA', '{:02d}'.format(seq), 'frames', '{:06d}.jpg'.format(frame)),
channels_first=False,
color_mode='BGR',
dtype=np.uint8,
resize_dim=resize_dim)
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_rgb.jpg'.format(seq, frame)), im)
# read flow
flow = read_image(join(dreyeve_root, 'DATA', '{:02d}'.format(seq), 'optical_flow', '{:06d}.png'.format(frame)),
channels_first=False,
color_mode='BGR',
dtype=np.uint8,
resize_dim=resize_dim)
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_flow.jpg'.format(seq, frame)), flow)
# read segmentation
semseg = np.squeeze(np.load(join(dreyeve_root, 'DATA', '{:02d}'.format(seq),
'semseg', '{:06d}.npz'.format(frame)))['arr_0'])
semseg_rgb = cv2.resize(cv2.cvtColor(seg_to_rgb(semseg), cv2.COLOR_RGB2BGR), dsize=(960, 540))
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_seg.jpg'.format(seq, frame)), semseg_rgb)
# read gt and blend
gt = read_image(join(dreyeve_root, 'DATA', '{:02d}'.format(seq), 'saliency_fix', '{:06d}.png'.format(frame+1)),
channels_first=False,
color=False,
dtype=np.float32,
resize_dim=resize_dim)
gt = blend_map(im, gt, factor=0.5)
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_gt.jpg'.format(seq, frame)), gt)
# read image pred and blend
image_pred = np.squeeze(np.load(join(dreyeve_root, prediction_dir, '{:02d}'.format(seq),
'image_branch', '{:06d}.npz'.format(frame)))['arr_0'])
image_pred = cv2.resize(image_pred, dsize=resize_dim[::-1])
image_pred = blend_map(im, image_pred, factor=0.5)
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_image_pred.jpg'.format(seq, frame)), image_pred)
# read image pred and blend
flow_pred = np.squeeze(np.load(join(dreyeve_root, prediction_dir, '{:02d}'.format(seq),
'flow_branch', '{:06d}.npz'.format(frame)))['arr_0'])
flow_pred = cv2.resize(flow_pred, dsize=resize_dim[::-1])
flow_pred = blend_map(flow, flow_pred, factor=0.5)
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_flow_pred.jpg'.format(seq, frame)), flow_pred)
# read image pred and blend
semseg_pred = np.squeeze(np.load(join(dreyeve_root, prediction_dir, '{:02d}'.format(seq),
'semseg_branch', '{:06d}.npz'.format(frame)))['arr_0'])
semseg_pred = cv2.resize(semseg_pred, dsize=resize_dim[::-1])
semseg_pred = blend_map(semseg_rgb, semseg_pred, factor=0.5)
cv2.imwrite(join(this_sample_dir, 'R{:02d}_frame_{:06d}_semseg_pred.jpg'.format(seq, frame)), semseg_pred)
stitch = [gt, im, image_pred, flow, flow_pred, semseg_rgb, semseg_pred]
cv2.imwrite(join(stitches_dir, '{:05d}.jpg'.format(i)),
np.concatenate(stitch, axis=1))
def compute_metrics_for_wang2015saliency(sequences):
"""
Function to compute metrics from model [2].
:param sequences: A list of sequences to consider.
"""
# some variables
gt_h, gt_w = 1080, 1920
pred_dir = 'Z:\\PREDICTIONS_wang2015saliency'
dreyeve_dir = 'Z:\\DATA'
pred_list = glob(
'Z:/CODE_EXPERIMENTS/EXP_VIDEOSALIENCY_COMPARISON/methods/SaliencySeg-master/code/final_results/**/final_saliency/*.bmp'
)
ig_baseline = read_image(join(dreyeve_dir,
'dreyeve_mean_train_gt_fix.png'),
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
print('Computing metrics...')
last_seq = 0
for index in tqdm(xrange(0, len(pred_list))):
pred_img = pred_list[index]
# recover sequence number
seq = int(
os.path.basename(os.path.dirname(
os.path.dirname(pred_img))).split('_')[1])
# if sequence is new, recover some variables
if seq != last_seq:
if 'metric_saver' in locals():
metric_saver.save_mean_metrics()
metric_saver = MetricSaver(pred_dir, seq, model='competitor')
last_seq = seq
# gt dirs
seq_gt_sal_dir = join(dreyeve_dir, '{:02d}'.format(seq),
'saliency')
seq_gt_fix_dir = join(dreyeve_dir, '{:02d}'.format(seq),
'saliency_fix')
# get frame number
fr = int(os.path.basename(pred_img).split('.')[0])
# load prediction
p = read_image(pred_img,
channels_first=False,
color=False,
resize_dim=(gt_h, gt_w))
# load gts
gt_sal = read_image(join(seq_gt_sal_dir, '{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
gt_fix = read_image(join(seq_gt_fix_dir, '{:06d}.png'.format(fr + 1)),
channels_first=False,
color=False)
# feed the saver
metric_saver.feed(fr,
predictions=[p],
groundtruth=[gt_sal, gt_fix],
ig_baseline=ig_baseline)
# save mean values
metric_saver.save_mean_metrics()
