def init_data(self,img_rows,img_cols,mode):
print('reading training data')
dataType = mode+'2014'
annFile = '/home/eric/Desktop/experiment/salicon/salicon-api/annotations/fixations_%s.json' %dataType
salicon = SALICON(annFile)
imgIds = salicon.getImgIds();
img = salicon.loadImgs(imgIds[:])
data = []
anno = []
for i in xrange(len(img)):
#loading img_data
I = cv2.imread('/home/eric/Desktop/experiment/salicon/salicon-api/images/'+mode+'/%s'%img[i]['file_name'])
I = cv2.resize(I, (img_cols, img_rows),interpolation = cv2.INTER_LINEAR)
data.append(I)
#loading annotations
annIds = salicon.getAnnIds(imgIds=img[i]['id'])
anns = salicon.loadAnns(annIds)
sal_map = salicon.showAnns(anns)
sal_map = cv2.resize(sal_map, (img_cols/32, img_rows/32),interpolation = cv2.INTER_LINEAR) #resize the saliency map
anno.append(sal_map)
data = np.array(data, dtype=np.int8)
data = data.transpose((0, 3, 1, 2))
anno = np.array(anno, dtype=np.float32)
anno = anno.reshape(anno.shape[0],(img_rows/32)*(img_cols/32)) #converting spatial saliency map into vectors
#normalize the saliency map for training with cross entropy
for i in xrange(len(anno)):
# cur_map = (cur_map-np.min(cur_map))/(np.max(cur_map)-np.min(cur_map)) #data v2
anno[i] = anno[i]/np.sum(anno[i]) #data v1
self.data = data
self.anno = anno
def convert_salicon(annotatation_path, save_path):
salicon = SALICON(annotatation_path)
imgIds = salicon.getImgIds()
if not os.path.exists(save_path):
os.makedirs(save_path)
# Load each image, create heatmap and store to disk.
for i, imgId in enumerate(imgIds):
img = salicon.loadImgs(imgId)[0]
if i % 1000 is 0:
print(i, img['file_name'])
annIds = salicon.getAnnIds(imgIds=img['id'])
anns = salicon.loadAnns(annIds)
sal_map = salicon.buildFixMap(anns, doBlur=True, sigma=FLAGS.sigma)
im = Image.fromarray(np.uint8(sal_map * 255))
im.save(save_path + img['file_name'])
# Cleanup, just to be sure.
del salicon
import skimage.io as io
import matplotlib.pyplot as plt
import cv2
import scipy.misc
# In[2]:
dataDir='/home/faiz/Desktop/datasets'
dataType='train2015r1'
annFile='%s/annotationsNew/fixations_%s.json'%(dataDir,dataType)
print annFile
# In[3]:
# initialize COCO api for instance annotations
salicon=SALICON(annFile)
# In[4]:
# get all images
save_dir = '/home/faiz/Desktop/saliency/train/'
imgIds = salicon.getImgIds();
print len(imgIds)
for each in range(0, len(imgIds)):
img = salicon.loadImgs(imgIds[each])[0]
# load and display image
#I = io.imread('%s/images/%s/%s'%(dataDir,dataType,img['file_name']))
# load and display instance annotations
#plt.imshow(I)
annIds = salicon.getAnnIds(imgIds=img['id'])
parser.add_argument("gpuid",type=int,
help="GPU ID")
args = parser.parse_args()
rst_blob_names = ['imdata','fmap']
im_means = [123.6800, 116.7790, 103.9390]
datum = caffe.proto.caffe_pb2.Datum()
#####################
# val datasets
#####################
##### For MIT
mit_impath = '/opt/CV_db/image_databases/pub/MIT1003/json_stimuli'
annFileS = '/opt/CV_db/Users/$(USER)/MIT1003/salformat/gtjson/mit1003.json'
mit_anns = SALICON(annFileS)
mit_ids = mit_anns.getImgIds()
mit_trnids = mit_ids[0:900]
mit_valids = mit_ids[900:]
mit_anns.dataset['type'] = 'fixations'
lmdbpath = '/dev/shm/$(USER)/MIT1003/data_origsize_900_103/lmdb'
lmdbfile = '{}_tr/imgs'.format(lmdbpath)
mit_trn_img_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
lmdbfile = '{}_tr/fmaps'.format(lmdbpath)
mit_trn_map_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
lmdbfile = '{}_val/imgs'.format(lmdbpath)
mit_val_img_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
lmdbfile = '{}_val/fmaps'.format(lmdbpath)
mit_val_map_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
lmdbfile = '{}/salicon/data/lmdb_sal-19_train2014/VGG_sal/fmaps_sal/lmdb'.format(
lmdbpath)
slcn_trn_map_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
lmdbfile = '{}/salicon/data/lmdb_sal-19_val2014/VGG_sal/imgs/lmdb'.format(
lmdbpath)
slcn_val_img_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
lmdbfile = '{}/salicon/data/lmdb_sal-19_val2014/VGG_sal/fmaps_sal/lmdb'.format(
lmdbpath)
slcn_val_map_csr = lmdb.open(lmdbfile, readonly=True).begin().cursor()
# For Salicon
dataDirS = '/opt/CV_db/image_databases/pub/SALICON'
dataTypeS = 'train2014'
annFileS = '%s/fixations_%s.json' % (dataDirS, dataTypeS)
slcn_trn = SALICON(annFileS)
slcn_trnids = slcn_trn.getImgIds()
dataDirS = '/opt/CV_db/image_databases/pub/SALICON'
dataTypeS = 'val2014'
annFileS = '%s/fixations_%s.json' % (dataDirS, dataTypeS)
slcn_val = SALICON(annFileS)
slcn_valids = slcn_val.getImgIds()
slcn_valtrn, slcn_valval = np.split(
np.random.permutation(len(slcn_valids)),
[4000]) # use 4000 for val for training, the rest for validation
# create dbs:
trn_dbs = {
'salicon_trn': {
'img_path': slcn_impath,
# In[11]:
dataDir = '.'
dataType = 'train2014examples'
algName = 'fake'
annFile = '%s/annotations/fixations_%s.json' % (dataDir, dataType)
subtypes = ['results', 'evalImgs', 'eval']
[resFile, evalImgsFile, evalFile] = [
'%s/results/fixations_%s_%s_%s.json' %
(dataDir, dataType, algName, subtype) for subtype in subtypes
]
# In[12]:
# create coco object and cocoRes object
salicon = SALICON(annFile)
saliconRes = salicon.loadRes(resFile)
# In[13]:
# create cocoEval object by taking coco and cocoRes
saliconEval = SALICONEval(salicon, saliconRes)
# evaluate on a subset of images by setting
# cocoEval.params['image_id'] = cocoRes.getImgIds()
saliconEval.params['image_id'] = saliconRes.getImgIds()
# evaluate results
saliconEval.evaluate()
# print output evaluation scores
def _get_SALICON(data_type, stimuli_url, stimuli_hash, fixation_url, fixation_hash, location):
from salicon.salicon import SALICON
with TemporaryDirectory(cleanup=True) as temp_dir:
download_and_check(stimuli_url,
os.path.join(temp_dir, 'stimuli.zip'),
stimuli_hash)
if fixation_url is not None:
download_and_check(fixation_url,
os.path.join(temp_dir, 'fixations.json'),
fixation_hash)
# Stimuli
annFile = os.path.join(temp_dir, 'fixations.json')
salicon = SALICON(annFile)
# get all images
imgIds = salicon.getImgIds()
images = salicon.loadImgs(imgIds)
print('Creating stimuli')
f = zipfile.ZipFile(os.path.join(temp_dir, 'stimuli.zip'))
namelist = f.namelist()
f.extractall(temp_dir, namelist)
del f
stimuli_src_location = os.path.join(temp_dir, data_type)
stimuli_target_location = os.path.join(location, 'stimuli') if location else None
filenames = [img['file_name'] for img in images]
stimuli = create_stimuli(stimuli_src_location, filenames, stimuli_target_location)
if fixation_url is not None:
# FixationTrains
print('Creating fixations')
ns = []
train_xs = []
train_ys = []
train_ts = []
train_subjects = []
for n, imgId in progressinfo(enumerate(imgIds)):
annIds = salicon.getAnnIds(imgIds=imgId)
anns = salicon.loadAnns(annIds)
for ann in anns:
fs = ann['fixations']
# SALICON annotations are 1-indexed, not 0-indexed.
xs = np.array([f[1]-1 for f in fs])
ys = np.array([f[0]-1 for f in fs])
ns.append(np.ones(len(xs), dtype=int)*n)
train_xs.append(xs)
train_ys.append(ys)
train_ts.append(range(len(xs)))
train_subjects.append(np.ones(len(xs), dtype=int)*ann['worker_id'])
xs = np.hstack(train_xs)
ys = np.hstack(train_ys)
ts = np.hstack(train_ts)
ns = np.hstack(ns)
subjects = np.hstack(train_subjects)
fixations = Fixations.FixationsWithoutHistory(xs, ys, ts, ns, subjects)
else:
fixations = None
if location:
dill.dump(stimuli, open(os.path.join(location, 'stimuli.pydat'), 'wb'))
if fixations is not None:
dill.dump(fixations, open(os.path.join(location, 'fixations.pydat'), 'wb'))
return stimuli, fixations
#get_ipython().magic(u'matplotlib inline')
from salicon.salicon import SALICON
import numpy as np
import skimage.io as io
import matplotlib.pyplot as plt
# In[2]:
dataDir = '..'
dataType = 'train2014examples'
annFile = '%s/annotations/fixations_%s.json' % (dataDir, dataType)
# In[3]:
# initialize COCO api for instance annotations
salicon = SALICON(annFile)
# In[4]:
# get all images
imgIds = salicon.getImgIds()
img = salicon.loadImgs(imgIds[np.random.randint(0, len(imgIds))])[0]
# In[6]:
# load and display image
I = io.imread('%s/images/%s/%s' % (dataDir, dataType, img['file_name']))
plt.figure()
plt.imshow(I)
plt.show()
# In[11]: dataDir='.' dataType='train2014examples' algName = 'fake' annFile='%s/annotations/fixations_%s.json'%(dataDir,dataType) subtypes=['results', 'evalImgs', 'eval'] [resFile, evalImgsFile, evalFile]= ['%s/results/fixations_%s_%s_%s.json'%(dataDir,dataType,algName,subtype) for subtype in subtypes] # In[12]: # create coco object and cocoRes object salicon = SALICON(annFile) saliconRes = salicon.loadRes(resFile) # In[13]: # create cocoEval object by taking coco and cocoRes saliconEval = SALICONEval(salicon, saliconRes) # evaluate on a subset of images by setting # cocoEval.params['image_id'] = cocoRes.getImgIds() saliconEval.params['image_id'] = saliconRes.getImgIds() # evaluate results saliconEval.evaluate()
# Demo demonstrating the algorithm result formats for SALICON
# select results type for demo
print 'Running demo for saliency predicition results'
# set appropriate files for given type of results
dataDir='..'
dataType='train2014examples'
algo = 'fake'
annFile='%s/annotations/fixations_%s.json'%(dataDir,dataType)
resFile='%s/results/fixations_%s_%s_results.json'%(dataDir,dataType,algo)
# In[3]:
# initialize COCO ground truth and results api's
salicon = SALICON(annFile)
saliconRes = salicon.loadRes(resFile)
# In[4]:
# visialuze ground truth and results side by side
imgIds = list(set([ann['image_id'] for ann in saliconRes.loadAnns(saliconRes.getAnnIds())]))
print imgIds
nImgs = len(imgIds)
imgId = imgIds[np.random.randint(nImgs)]
img = salicon.loadImgs(imgId)[0]
I = io.imread('%s/images/%s/%s'%(dataDir,dataType,img['file_name']))
# show ground truth labels
annIds = salicon.getAnnIds(imgIds=imgId)
# Demo demonstrating the algorithm result formats for SALICON
# select results type for demo
print 'Running demo for saliency predicition results'
# set appropriate files for given type of results
dataDir = '..'
dataType = 'train2014examples'
algo = 'fake'
annFile = '%s/annotations/fixations_%s.json' % (dataDir, dataType)
resFile = '%s/results/fixations_%s_%s_results.json' % (dataDir, dataType, algo)
# In[3]:
# initialize COCO ground truth and results api's
salicon = SALICON(annFile)
saliconRes = salicon.loadRes(resFile)
# In[4]:
# visialuze ground truth and results side by side
imgIds = list(
set([
ann['image_id'] for ann in saliconRes.loadAnns(saliconRes.getAnnIds())
]))
print imgIds
nImgs = len(imgIds)
imgId = imgIds[np.random.randint(nImgs)]
img = salicon.loadImgs(imgId)[0]
I = io.imread('%s/images/%s/%s' % (dataDir, dataType, img['file_name']))
def main():
img_rows, img_cols = 300, 400
#initializing salicon data
sal_anno = '/home/eric/Desktop/experiment/salicon/salicon-api/annotations/fixations_train2014.json'
salicon = SALICON(sal_anno)
#loading VQA data
vqa_dict = np.load('valid_data_train.npy')
question_bank = np.load('question_type.npy')
answer_bank = np.load('answer_type.npy')
vqa_dir = '/media/eric/New Volume/VQA/VQA_HAT/vqahat_train'
#defining data structure
metrics = ['cc', 'sim', 'kld', 'emd', 'spearmanr']
que_score = dict()
ans_score = dict()
overall_score = dict()
for question in question_bank:
que_score[question] = init_metrics(metrics)
for answer in answer_bank:
ans_score[answer] = init_metrics(metrics)
overall_score = init_metrics(metrics)
#main loop for comparing different attention maps
nan_count_q = dict()
nan_count_a = dict()
nan_corr_q = dict()
nan_corr_a = dict()
nan_count = 0
nan_corr = 0
for i in question_bank:
nan_count_q[i] = 0
nan_corr_q[i] = 0
for i in answer_bank:
nan_count_a[i] = 0
nan_corr_a[i] = 0
for cur_data in vqa_dict:
question_id = cur_data['question_id']
question_type = cur_data['question_type']
answer_type = cur_data['answer_type']
img_id = cur_data['img_id']
#load vqa attention map
vqa_img = os.path.join(vqa_dir, str(question_id) + '_1.png')
que_att_map = cv2.imread(vqa_img)
que_att_map = que_att_map[:, :, 0]
que_att_map = cv2.resize(que_att_map, (img_cols, img_rows),
interpolation=cv2.INTER_LINEAR)
que_att_map = que_att_map.astype('float32')
que_att_map /= 255
#load free-viewing attention map
annIds = salicon.getAnnIds(img_id)
anns = salicon.loadAnns(annIds)
fv_att_map = salicon.showAnns(anns)
fv_att_map = cv2.resize(fv_att_map, (img_cols, img_rows),
interpolation=cv2.INTER_LINEAR)
#computing scores for different metrics
cc = cal_cc_score(fv_att_map, que_att_map)
sim = cal_sim_score(fv_att_map, que_att_map)
kld = cal_kld_score(fv_att_map, que_att_map)
emd = cal_emd_score(fv_att_map, que_att_map)
rank_corr, _ = spearmanr(fv_att_map.reshape(-1),
que_att_map.reshape(-1))
#storing data in a naive way
if np.isnan(cc):
cc = 0
nan_count_q[question_type] += 1
nan_count_a[answer_type] += 1
nan_count += 1
if np.isnan(rank_corr):
rank_corr = 0
nan_corr_q[question_type] += 1
nan_corr_a[answer_type] += 1
nan_corr += 1
que_score[question_type]['cc'] += cc
que_score[question_type]['sim'] += sim
que_score[question_type]['spearmanr'] += rank_corr
que_score[question_type]['kld'] += kld
que_score[question_type]['emd'] += emd
que_score[question_type]['count'] += 1
ans_score[answer_type]['cc'] += cc
ans_score[answer_type]['sim'] += sim
ans_score[answer_type]['spearmanr'] += rank_corr
ans_score[answer_type]['kld'] += kld
ans_score[answer_type]['emd'] += emd
ans_score[answer_type]['count'] += 1
overall_score['cc'] += cc
overall_score['sim'] += sim
overall_score['spearmanr'] += rank_corr
overall_score['kld'] += kld
overall_score['emd'] += emd
overall_score['count'] += 1
#computing average score
for q_type in question_bank:
for cur_metric in metrics:
if cur_metric == 'cc':
que_score[q_type][cur_metric] /= que_score[q_type][
'count'] - nan_count_q[q_type]
elif cur_metric == 'spearmanr':
que_score[q_type][cur_metric] /= que_score[q_type][
'count'] - nan_corr_q[q_type]
else:
que_score[q_type][cur_metric] /= que_score[q_type]['count']
for a_type in answer_bank:
for cur_metric in metrics:
if cur_metric == 'cc':
ans_score[a_type][cur_metric] /= ans_score[a_type][
'count'] - nan_count_a[a_type]
elif cur_metric == 'spearmanr':
ans_score[a_type][cur_metric] /= ans_score[a_type][
'count'] - nan_corr_a[a_type]
else:
ans_score[a_type][cur_metric] /= ans_score[a_type]['count']
for cur_metric in metrics:
if cur_metric == 'cc':
overall_score[cur_metric] /= overall_score['count'] - nan_count
elif cur_metric == 'spearmanr':
overall_score[cur_metric] /= overall_score['count'] - nan_corr
else:
overall_score[cur_metric] /= overall_score['count']
np.save('question_score', que_score)
np.save('answer_score', ans_score)
np.save('overall_score', overall_score)