def produce_maps(weight_path, checkpoint_path, imgs_test_path, output_folder):
#model = ml_net_model(img_cols=shape_c, img_rows=shape_r, downsampling_factor_product=10,weight_path=weight_path)
#sgd = SGD(lr=1e-3, decay=0.0005, momentum=0.9, nesterov=True)
#print("Compile ML-Net Model")
#model.compile(sgd, loss)
# path of output folder
if not os.path.exists(output_folder):
os.makedirs(output_folder)
nb_imgs_test = len(imgs_test_path)
print("Load weights ML-Net")
model = load_model(checkpoint_path) #loading from the finetuned model
#model.load_weights(checkpoint_path)
predictions = model.predict_generator(generator_test(b_s=1, imgs_test_path=imgs_test_path), nb_imgs_test)
prediction_paths = []
for pred, name in zip(predictions, imgs_test_path):
original_image = misc.imread(name)
if len(original_image.shape) > 2:
original_image = rgb2gray(original_image)
res = postprocess_predictions(pred[0], original_image.shape[0], original_image.shape[1])
im_name = re.split('/',imgs_test_path)[-1]
pred_name = output_folder + im_name
misc.imsave(pred_name, res.astype(int))
prediction_paths.append(pred_name)
return prediction_paths
def compute_saliency(image_path):
predictions = m.predict_generator(
generator_test(b_s, [image_path]), 1)[0]
original_image = cv2.imread(image_path, 0)
res = postprocess_predictions(predictions[0][0],
original_image.shape[0],
original_image.shape[1])
return res
def compute_saliency(img_path):
pred = model.predict_on_batch(
preprocess_images([img_path], shape_r, shape_c))
sm = pred[0][0]
original_image = cv2.imread(img_path, 0)
sm = postprocess_predictions(sm, original_image.shape[0],
original_image.shape[1])
return sm
def compute_SAMSal(rgbvid, filename):
# Check if currently exists
outname, ext = os.path.splitext(filename)
outname += '_sam.npy'
print('Checking for', outname)
if os.path.exists(outname):
salmap = np.load(outname)
if salmap.shape[-1] == rgbvid.shape[-1]:
print('file found')
return salmap
print('Saliency Map length incorrect. Recomputing')
else:
print(outname, 'does not exist. Computing from scratch')
x = Input((3, shape_r, shape_c))
x_maps = Input((nb_gaussian, shape_r_gt, shape_c_gt))
m = Model(input=[x, x_maps], output=sam_vgg([x, x_maps]))
print("Compiling SAM-VGG")
m.compile(RMSprop(lr=1e-4),
loss=[kl_divergence, correlation_coefficient, nss])
nb_imgs_test = rgbvid.shape[-1]
if nb_imgs_test % b_s != 0:
print(
"The number of test images should be a multiple of the batch size. Please change your batch size in config.py accordingly."
)
exit()
print("Loading SAM-VGG weights")
m.load_weights('./sam/weights/sam-vgg_salicon_weights.pkl')
print("Predicting saliency maps")
predictions = m.predict_generator(generator_from_mat(b_s=b_s, vid=rgbvid),
nb_imgs_test)[0]
H, W, _, n = rgbvid.shape
output = np.zeros((H, W, n))
for i, pred in enumerate(predictions):
res = postprocess_predictions(pred[0], H, W)
output[..., i] = normalize(logsal(res))
#name = 'frame' + str(i) + '.png'
#original_image = cv2.imread(imgs_test_path + name, 0)
#res = postprocess_predictions(pred[0], original_image.shape[0], original_image.shape[1])
#cv2.imwrite(output_folder + '%s' % name, res.astype(int))
outname, ext = os.path.splitext(filename)
outname += '_sam.npy'
np.save(outname, output)
return output
def make_predictions(model,imgs_test_path,output_folder):
nb_imgs_test = len(imgs_test_path)
predictions = model.predict_generator(generator_test(1, imgs_test_path), nb_imgs_test)
prediction_paths = []
for pred, name in zip(predictions, imgs_test_path):
original_image = misc.imread(name)
if len(original_image.shape) > 2:
original_image = rgb2gray(original_image)
res = postprocess_predictions(pred[0], original_image.shape[0], original_image.shape[1])
im_name = re.split('/',imgs_test_path)[-1]
pred_name = output_folder + im_name
misc.imsave(pred_name, res.astype(int))
prediction_paths.append(pred_name)
return prediction_paths
def saliencyattentivemodel(inputimage):
x = Input((3, shape_r, shape_c))
x_maps = Input((nb_gaussian, shape_r_gt, shape_c_gt))
# version (0 for SAM-VGG and 1 for SAM-ResNet)
if version == 0:
m = Model(input=[x, x_maps], output=sam_vgg([x, x_maps]))
print("Compiling SAM-VGG")
m.compile(RMSprop(lr=1e-4),
loss=[kl_divergence, correlation_coefficient, nss])
elif version == 1:
m = Model(input=[x, x_maps], output=sam_resnet([x, x_maps]))
print("Compiling SAM-ResNet")
m.compile(RMSprop(lr=1e-4),
loss=[kl_divergence, correlation_coefficient, nss])
else:
raise NotImplementedError
# Output Folder Path
output_folder = 'predictions/'
nb_imgs_test = 1 #len(file_names)
if nb_imgs_test % b_s != 0:
print(
"The number of test images should be a multiple of the batch size. Please change your batch size in config.py accordingly."
)
exit()
if version == 0:
print("Loading SAM-VGG weights")
m.load_weights('weights/sam-vgg_salicon_weights.pkl')
elif version == 1:
print("Loading SAM-ResNet weights")
m.load_weights('weights/sam-resnet_salicon_weights.pkl')
predictions = m.predict_generator(
generator_test_singleimage(b_s=b_s, image=inputimage), nb_imgs_test)[0]
print("predictions:", predictions[0])
outname = 'SalmapFromWrapper.jpg'
original_image = image
res = postprocess_predictions(predictions[0], original_image.shape[0],
original_image.shape[1])
#mport pdb; pdb.set_trace()
cv2.imwrite(output_folder + '%s' % outname, res.astype(int))
cv2.imshow('salmap', res.astype('uint8'))
cv2.waitKey(0)
return res.astype('uint8')
def compute_saliency(image_path):
if use_default_center_bias:
predictions = m.predict(
[preprocess_images([image_path], shape_r, shape_c),
gaussian])[0]
else:
predictions = m.predict(
[preprocess_images([image_path], shape_r, shape_c),
gaussian])[0]
original_image = cv2.imread(image_path, 0)
res = postprocess_predictions(
predictions[0][0],
original_image.shape[0],
original_image.shape[1],
do_default_smoothing=do_default_smoothing)
return res
def compute_saliency(image_path):
if use_default_center_bias:
pred = model.predict_on_batch(
preprocess_images([image_path], shape_r, shape_c))
sm = pred[0][0]
else:
get_unbiased_output = K.function(
[model.layers[0].input,
K.learning_phase()], [model.layers[21].output])
pred = get_unbiased_output(
[preprocess_images([image_path], shape_r, shape_c), 0])
sm = pred[0][0][0]
original_image = cv2.imread(image_path, 0)
sm = postprocess_predictions(sm, original_image.shape[0],
original_image.shape[1])
return sm
def call(videos_test_path, out_):
phase = 'test'
if phase == 'train':
x = Input(batch_shape=(None, None, shape_r, shape_c, 3))
stateful = False
else:
x = Input(batch_shape=(1, None, shape_r, shape_c, 3))
stateful = True
if phase == 'train':
if nb_train % video_b_s != 0 or nb_videos_val % video_b_s != 0:
print(
"The number of training and validation images should be a multiple of the batch size. "
"Please change your batch size in config.py accordingly.")
exit()
m = Model(inputs=x, outputs=acl_vgg(x, stateful))
print("Compiling ACL-VGG")
m.compile(Adam(lr=1e-4),
loss=[
kl_divergence, correlation_coefficient, nss,
kl_divergence, correlation_coefficient, nss
]) #
print("Training ACL-VGG")
m.fit_generator(generator(video_b_s=video_b_s, image_b_s=image_b_s),
nb_train,
epochs=nb_epoch,
validation_data=generator(video_b_s=video_b_s,
image_b_s=0,
phase_gen='val'),
validation_steps=nb_videos_val,
callbacks=[
EarlyStopping(patience=15),
ModelCheckpoint(
'acl-vgg.{epoch:02d}-{val_loss:.4f}.h5',
save_best_only=False),
LearningRateScheduler(schedule=schedule_vgg)
])
elif phase == "test":
videos = [
videos_test_path + f for f in os.listdir(videos_test_path)
if os.path.isdir(videos_test_path + f)
]
videos.sort()
nb_videos_test = len(videos)
m = Model(inputs=x, outputs=acl_vgg(x, stateful))
print("Loading ACL weights")
m.load_weights('ACL.h5')
for i in range(nb_videos_test):
output_folder = out_ + '/' + videos[i] + '/'
if not os.path.exists(output_folder):
os.makedirs(output_folder)
images_names = [
f for f in os.listdir(videos[i] + frames_path)
if f.endswith(('.jpg', '.jpeg', '.png'))
]
images_names.sort()
print("Predicting saliency maps for " + videos[i])
prediction = m.predict_generator(
get_test(video_test_path=videos[i]),
max(ceil(len(images_names) / num_frames), 2))
predictions = prediction[0]
for j in range(len(images_names)):
original_image = imread(videos[i] + frames_path +
images_names[j])
x, y = divmod(j, num_frames)
res = postprocess_predictions(predictions[x, y, :, :, 0],
original_image.shape[0],
original_image.shape[1])
imsave(output_folder + '%s' % images_names[j], res.astype(int))
m.reset_states()
else:
raise NotImplementedError
if f.endswith(('.jpg', '.jpeg', '.png'))
]
nb_imgs_test = len(file_names)
if nb_imgs_test % b_s != 0:
print(
"The number of test images should be a multiple of the batch size. Please change your batch size in config.py accordingly."
)
exit()
if version == 0:
print("Loading SAM-VGG weights")
m.load_weights('weights/sam-vgg_salicon_weights.pkl')
elif version == 1:
print("Loading SAM-ResNet weights")
m.load_weights('weights/sam-resnet_salicon_weights.pkl')
print("Predicting saliency maps for " + imgs_test_path)
predictions = m.predict_generator(
generator_test(b_s=b_s, images=file_names_with_path),
nb_imgs_test)[0]
for pred, name in zip(predictions, file_names):
original_image = cv2.imread(imgs_test_path + name, 0)
res = postprocess_predictions(pred[0], original_image.shape[0],
original_image.shape[1])
cv2.imwrite(output_folder + '%s' % name, res.astype(int))
else:
raise NotImplementedError
model.fit_generator(generator(b_s=b_s), nb_imgs_train, nb_epoch=nb_epoch,
validation_data=generator(b_s=b_s, phase_gen='val'), nb_val_samples=nb_imgs_val,
callbacks=[EarlyStopping(patience=5),
ModelCheckpoint('weights.mlnet.{epoch:02d}-{val_loss:.4f}.pkl', save_best_only=True)])
elif phase == "test":
# path of output folder
output_folder = ''
if len(sys.argv) < 2:
raise SyntaxError
imgs_test_path = sys.argv[2]
file_names = [f for f in os.listdir(imgs_test_path) if f.endswith('.jpg')]
file_names.sort()
nb_imgs_test = len(file_names)
print("Load weights ML-Net")
model.load_weights('mlnet_salicon_weights.pkl')
print("Predict saliency maps for " + imgs_test_path)
predictions = model.predict_generator(generator_test(b_s=1, imgs_test_path=imgs_test_path), nb_imgs_test)
for pred, name in zip(predictions, file_names):
original_image = cv2.imread(imgs_test_path + name, 0)
res = postprocess_predictions(pred[0], original_image.shape[0], original_image.shape[1])
cv2.imwrite(output_folder + '%s' % name, res.astype(int))
else:
raise NotImplementedError
random.seed(seed)
test_data = []
mypath = sys.argv[1]
testing_images = [sys.argv[1] + "/" + f for f in os.listdir(mypath)]
testing_images.sort()
for image in testing_images:
data = {'image': image}
test_data.append(data)
x = Input(batch_shape=(1, im_res, im_res, 3))
m = Model(inputs=x, outputs=sam_vgg(x))
print("Loading weights")
m.load_weights('PAGE-Net.h5')
print("Making prediction")
# Output Folder Path
saliency_output = sys.argv[2]
print "Saving saliency files in " + saliency_output
if not os.path.exists(saliency_output):
os.makedirs(saliency_output)
for data in test_data:
Ximg, original_image, img_name = get_test(data)
predictions = m.predict(Ximg, batch_size=1)
res_saliency = postprocess_predictions(predictions[9][0, :, :, 0],
original_image.shape[0],
original_image.shape[1])
imsave(saliency_output + '/%s.png' % img_name[0:-4],
res_saliency.astype(int))
"The number of test images should be a multiple of the batch size. Please change your batch size in config.py accordingly."
)
exit()
if version == 0:
print("Loading SAM-VGG weights")
m.load_weights('weights/sam-vgg_salicon_weights.pkl')
elif version == 1:
print("Loading SAM-ResNet weights")
m.load_weights('weights/sam-resnet_salicon_weights.pkl')
print("Predicting saliency maps for " + imgs_test_path)
predictions = m.predict_generator(
generator_test(b_s=b_s, imgs_test_path=imgs_test_path),
nb_imgs_test)[0]
for pred, name in zip(predictions, file_names):
original_image = cv2.imread(imgs_test_path + name, 0)
res = postprocess_predictions(
pred[0],
original_image.shape[0],
original_image.shape[1],
no_normalization=NO_NORMALIZATION,
no_blur=NO_BLUR)
f = h5py.File('{}/{}.h5'.format(output_folder, name), 'w')
f.create_dataset('saliency_map', data=res)
f.close()
# cv2.imwrite(output_folder + '%s' % name, res.astype(int))
else:
raise NotImplementedError
#m.load_weights('weights/sam-resnet_salicon2017_weights.pkl') #New version
m.load_weights_new('weights/sam-resnet_salicon2017_weights.pkl', reshape=True) #Final version
print("==============================================")
#Todo controlado hasta aqui\\\\\\\\\\\\\\\\\\\\\\\\\\
print("Predicting saliency maps for " + imgs_test_path)
'''https://stackoverflow.com/questions/58352326/running-the-tensorflow-2-0-code-gives-valueerror-tf-function-decorated-functio'''
#predictions = m.predict_generator(generator_test(b_s=b_s, imgs_test_path=imgs_test_path), nb_imgs_test)[0]
#predictions = m.predict(generator_test(b_s=b_s, imgs_test_path=imgs_test_path), nb_imgs_test)[0] #Nueva version
#predictions = m.predict(generator_test(b_s=b_s, imgs_test_path=imgs_test_path), nb_imgs_test) #Nueva version
predictions = m.predict(generator_test(b_s=b_s, imgs_test_path=imgs_test_path),steps = nb_imgs_test)[0] #Nueva version. Output shape = (1, 1, 480, 640)
print("Longitud de `predictions`: ", len(predictions))
#x = [preprocess_images(images[counter:counter + b_s], shape_r, shape_c), gaussian]
#predictions = m.predict(x,batch_size = nb_imgs_test)[0] #PRUEBAS
print("==============================================")
for pred, name in zip(predictions, file_names):
#pred = predictions[0]
#name = file_names[0]
print("Dibujando el saliency map de la imagen ", name)
#original_image = cv2.imread(imgs_test_path + name, 0)
original_image = cv2.imread(imgs_test_path + "/" + name, 0)
#res = postprocess_predictions(pred[0], original_image.shape[0], original_image.shape[1])
res = postprocess_predictions(pred, original_image.shape[0], original_image.shape[1]) #New version
cv2.imwrite(output_folder + '%s' % name, res.astype(int)) #res.shape (300, 450)
else:
raise NotImplementedError
print("Programa finalizado")
