def vg_check_pre_stats():
client = MongoClient("mongodb://localhost:27017")
db = client.visual_genome_1_2
db_rel_train_all = db.relationships_all_train.find(no_cursor_timeout=True)
db_rel_test_all = db.relationships_all_test.find(no_cursor_timeout=True)
train_stats = {}
test_stats = {}
cnt = 0
zl.tick()
for db_rel in db_rel_train_all:
if cnt % 1000 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
for r in db_rel['relationships']:
name = r['predicate']
if name not in train_stats:
train_stats[name] = 0
train_stats[name] += 1
for db_rel in db_rel_test_all:
if cnt % 1000 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
for r in db_rel['relationships']:
name = r['predicate']
if name not in test_stats:
test_stats[name] = 0
test_stats[name] += 1
zl.save('output/train_pre_stats.pkl', train_stats)
zl.save('output/test_pre_stats.pkl', test_stats)
print zl.sort_dict_by_val(train_stats)
print zl.sort_dict_by_val(test_stats)
def vg_make_meta_visual_phrase():
m = h5py.File('data/vg1_2_meta.h5', 'r', driver='core')
h5f = h5py.File('data/vg1_2_vp_meta.h5')
triplets = {}
cnt = 0
zl.tick()
for k in m['gt/train'].keys():
if cnt % 1000 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
# sub_boxes = m['gt/train/%s/sub_boxes'%k][...]
# obj_boxes = m['gt/train/%s/obj_boxes'%k][...]
rlp_labels = m['gt/train/%s/rlp_labels' % k][...]
for i in xrange(rlp_labels.shape[0]):
# sub_box = sub_boxes[i]
# obj_box = obj_boxes[i]
rlp_label = rlp_labels[i]
# joint_bbox = [min(sub_bbox[0],obj_bbox[0]), min(sub_bbox[1],obj_bbox[1]),max(sub_bbox[2],obj_bbox[2]),max(sub_bbox[3],obj_bbox[3])]
s_lbl = zl.idx2name_cls(m, rlp_label[0])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
spo = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
# spo = '%d_%d_%d'%(rlp_label[0],rlp_label[1],rlp_label[2])
if spo not in triplets:
triplets[spo] = 0
triplets[spo] += 1
zl.save('output/pkl/triplets_train_vp.pkl', triplets)
triplets_sorted = zl.sort_dict_by_val(triplets)
triplets_ok = []
for k, v in triplets_sorted:
triplets_ok.append(k)
print k, v
triplets_ok = sorted(triplets_ok)
triplets_ok = ['__background__'] + triplets_ok
for i in xrange(len(triplets_ok)):
h5f['meta/tri/idx2name/%d' % i] = triplets_ok[i]
h5f['meta/tri/name2idx/%s' % triplets_ok[i]] = i
print len(triplets_ok)
def run_relation(model_type,iteration):
vgg_data = h5py.File('output/sg_vrd_2016_test.hdf5')
result = h5py.File('output/sg_vrd_2016_result_'+model_type+'_'+iteration+'.hdf5')
#if os.path.exists('output/sg_vrd_2016_result.hdf5'):
# os.remove('output/sg_vrd_2016_result.hdf5')
m = h5py.File('data/sg_vrd_meta.h5')
data_root='data/sg_vrd_2016/Data/sg_test_images/'
keep = 100
thresh = 0.0001
net = caffe.Net('models/sg_vrd/relation/test_'+model_type+'.prototxt','output/relation/vr/sg_vrd_relation_vgg16_'+model_type+'_iter_'+iteration+'.caffemodel',caffe.TEST)
#net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST)
cnt =0
zl.tick()
rel_types = {}
rel_types['p']=[]
rel_types['s']=[]
rel_types['v']=[]
rel_types['c']=[]
for k in m['meta/pre/name2idx'].keys():
idx = int(str(m['meta/pre/name2idx/'+k][...]))
r_type = m['meta/pre/name2idx/'+k].attrs['type']
rel_types[r_type].append(idx)
for imid in vgg_data.keys():
cnt+=1
print cnt,zl.tock()
zl.tick()
# if cnt%100==0:
# print cnt
classemes = vgg_data[imid]['classemes']
visuals = vgg_data[imid]['visuals']
locations = vgg_data[imid]['locations']
cls_confs = vgg_data[imid]['cls_confs']
# im = cv2.imread(data_root+imid+'.jpg')
# #print cls_confs
# # for box in locations:
# # b=box[:4].astype(np.int32)
# # cv2.rectangle(im,(b[0],b[1]),(b[2],b[3]),(255,0,0))
# w,h = im.shape[2],im.shape[1]
rlp_labels = []
rlp_confs = []
sub_boxes=[]
obj_boxes=[]
relation_vectors = []
for s in xrange(len(locations)):
for o in xrange(len(locations)):
if s==o:continue
sub = locations[s]
obj = locations[o]
sub_visual = visuals[s]
obj_visual = visuals[o]
sub_cls = cls_confs[s,0]
obj_cls = cls_confs[o,0]
sub_score = cls_confs[s,1]
obj_score = cls_confs[o,1]
sub_classme = classemes[s]
obj_classme = classemes[o]
if sub_score<0.01 or obj_score<0.01:continue
sub_loc_encoded = bbox_transform( np.array([obj[:4]]), np.array([sub[:4]]))[0]
obj_loc_encoded = bbox_transform( np.array([sub[:4]]), np.array([obj[:4]]))[0]
#sub_loc_encoded = bbox_transform(np.array([[0, 0, w, h]]), np.array([sub[:4]]))[0]
#obj_loc_encoded = bbox_transform(np.array([[0, 0, w, h]]), np.array([obj[:4]]))[0]
visual = np.hstack((sub_visual, obj_visual)).reshape(1,8192)
classeme = np.hstack((sub_classme, obj_classme)).reshape(1,202)
loc = np.hstack((sub_loc_encoded, obj_loc_encoded)).reshape(1,8)
if 'all' in model_type:
blob = {
'classeme':classeme,
'visual':visual,
'location':loc
}
elif 'visual' in model_type:
blob = {
'visual':visual,
}
elif 'classeme' in model_type:
blob = {
'classeme':classeme,
}
elif 'location' in model_type:
blob = {
'location':loc
}
#batch this
net.forward_all(**blob)
relation_score =net.blobs['relation_prob'].data[0].copy()
#l2_norm = relation_score/LA.norm(relation_score)
#relation_score=softmax(relation_score)
#relation_score/=LA.norm(relation_score)
#relation_score=softmax(relation_score)
# argmax = np.argmax(relation_score)
# rs = relation_score[argmax]
# predicate = argmax
# rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# # rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
# rlp_conf = rs+sub_score+obj_score#*sub_score*obj_score
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub[:4])
# obj_boxes.append(obj[:4])
# relation_vectors.append(relation_score)
# for i in xrange(70):
# rs = relation_score[i]
# if rs>0.0:
# predicate =i
# rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# # rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
# rlp_conf = rs
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub[:4])
# obj_boxes.append(obj[:4])
r_scores = {'s':{},'v':{},'c':{},'p':{}}
for i in xrange(70):
rs = relation_score[i]
if i in rel_types['s']:r_scores['s'][i] = rs
if i in rel_types['v']:r_scores['v'][i] = rs
if i in rel_types['c']:r_scores['c'][i] = rs
if i in rel_types['p']:r_scores['p'][i] = rs
r_scores['s'] = zl.sort_dict_by_val(r_scores['s'])
r_scores['v'] = zl.sort_dict_by_val(r_scores['v'])
r_scores['c'] = zl.sort_dict_by_val(r_scores['c'])
r_scores['p'] = zl.sort_dict_by_val(r_scores['p'])
for i,rs in r_scores['s'][:4]:
predicate =i
rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
rlp_conf = rs
rlp_confs.append(rlp_conf)
rlp_labels.append(rlp_label)
sub_boxes.append(sub[:4])
obj_boxes.append(obj[:4])
for i,rs in r_scores['v'][:4]:
predicate =i
rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
rlp_conf = rs
rlp_confs.append(rlp_conf)
rlp_labels.append(rlp_label)
sub_boxes.append(sub[:4])
obj_boxes.append(obj[:4])
for i,rs in r_scores['p'][:4]:
predicate =i
rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
rlp_conf = rs
rlp_confs.append(rlp_conf)
rlp_labels.append(rlp_label)
sub_boxes.append(sub[:4])
obj_boxes.append(obj[:4])
for i,rs in r_scores['c']:
predicate =i
rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
rlp_conf = rs
rlp_confs.append(rlp_conf)
rlp_labels.append(rlp_label)
sub_boxes.append(sub[:4])
obj_boxes.append(obj[:4])
# if rs>0.0:
# predicate =i
# rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
# # rlp_conf = rs+sub_score+obj_score#relation_score[predicate]
# rlp_conf = rs
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub[:4])
# obj_boxes.append(obj[:4])
result.create_dataset(imid+'/rlp_confs',dtype='float16', data=np.array(rlp_confs).astype(np.float16))
result.create_dataset(imid+'/sub_boxes',dtype='float16', data=np.array(sub_boxes).astype(np.float16))
result.create_dataset(imid+'/obj_boxes',dtype='float16', data=np.array(obj_boxes).astype(np.float16))
result.create_dataset(imid+'/rlp_labels',dtype='float16', data=np.array(rlp_labels).astype(np.float16))
def run_retrieval_vp_v2():
# h5_path = 'output/sg_vrd_2016_result.classeme.hdf5'
h5_path = 'output/precalc/vg1_2_vp2016_test_nms2_.4.hdf5'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
data_root = '/home/zawlin/data/data_vrd/vg_1.2/'
rlp_map = []
for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())):
tri = str(m_vp['meta/tri/idx2name'][str(i)][...])
s_lbl = tri.split('_')[0]
p_lbl = tri.split('_')[1]
o_lbl = tri.split('_')[2]
rlp_label = [
zl.name2idx_cls(m, s_lbl),
zl.name2idx_pre(m, p_lbl),
zl.name2idx_cls(m, o_lbl)
]
rlp_map.append(rlp_label)
rlp_map = np.array(rlp_map)
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['labels'] = h5_in[i]['labels'][...]
h5f[i]['confs'] = h5_in[i]['confs'][...]
h5f[i]['boxes'] = h5_in[i]['boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
total = 0
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
for k in xrange(len(retr_meta['rlp_labels'])):
total += retr_meta['counts'][k]
if k > 1000:
print total / 1000.
exit(0)
break
continue
cnt += 1
rlp_label = retr_meta['rlp_labels'][k]
gt_files = retr_meta['files'][k]
# print gt_files
# exit(0)
# for f in gt_files:
# impath= zl.imid2path(m,f)
# print impath
# im= cv2.imread(data_root+impath)
# cv2.imshow('im',im)
# cv2.waitKey(0)
results = {}
zl.tick()
ranks = []
for imid in imids:
labels = h5f[imid]['labels'] - 1
rlp_confs = h5f[imid]['confs']
rlp_labels = rlp_map[labels]
if rlp_labels.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
delay = 0
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
impath = zl.imid2path(m, imid)
if found and i >= 5:
break
# print gt_files
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < 5:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+impath)
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / 5 #total_gt
r_100 = tp_100 / 5 #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.tock())
def run_retrieval():
h5f = h5py.File('output/vg_results/vg1_2_2016_result_all_100000.all.hdf5')
#h5f = h5py.file('output/results/lu_method_results.hdf5')
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
# pre = 'teddy bear_sit on_blanket'
pre = 'cloth_next to_suitcase'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
sub_lbl = pre.split('_')[0]
pre_lbl = pre.split('_')[1]
obj_lbl = pre.split('_')[2]
sub_idx = zl.name2idx_cls(m, sub_lbl)
pre_idx = zl.name2idx_pre(m, pre_lbl)
obj_idx = zl.name2idx_cls(m, obj_lbl)
rlp_label = np.array([sub_idx, pre_idx, obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tick()
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
# print confs
# print np.average(confs)
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:20]:
sub_boxes = h5f[imid + '/sub_boxes'][...]
obj_boxes = h5f[imid + '/obj_boxes'][...]
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] != 0:
sub_boxes = sub_boxes[ind]
obj_boxes = obj_boxes[ind]
sb = sub_boxes[0]
ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
cv2.rectangle(im, (ob[0], ob[1]), (ob[2], ob[3]), (0, 0, 200), 1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_retrieval_vp():
h5f = h5py.File('output/precalc/vg1_2_vp2016_test_new.hdf5')
# print h5f[h5f.keys()[0]].keys()
# exit(0)
#h5f = h5py.file('output/results/lu_method_results.hdf5')
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
pre = 'teddy bear_sit on_blanket'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
r_idx = zl.name2idx_tri(m_vp, pre)
# sub_lbl = pre.split('_')[0]
# pre_lbl = pre.split('_')[1]
# obj_lbl = pre.split('_')[2]
# sub_idx = zl.name2idx_cls(m,sub_lbl)
# pre_idx = zl.name2idx_pre(m,pre_lbl)
# obj_idx = zl.name2idx_cls(m,obj_lbl)
# rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
# rlp_labels = h5f[imid+'/labels'][...]
rlp_confs = h5f[imid + '/confs'][:, r_idx]
ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[ind[:5]]
results[imid] = np.average(rlp_confs)
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tick()
indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg_vp/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:200]:
boxes = h5f[imid + '/boxes'][...]
rlp_labels = h5f[imid + '/labels'][...]
rlp_confs = h5f[imid + '/confs'][...]
# indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] != 0:
boxes = boxes[ind]
sb = boxes[0]
# ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_retrieval_vp_all():
h5f = h5py.File('output/precalc/vg1_2_vp2016_test_new.hdf5', 'r')
# print h5f[h5f.keys()[0]].keys()
# exit(0)
#h5f = h5py.file('output/results/lu_method_results.hdf5')
img_set_file = 'data/vg1_2_2016/ImageSets/test.txt'
imlist = {
line.strip().split(' ')[1]: line.strip().split(' ')[0]
for line in open(img_set_file)
}
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
# pre = 'teddy bear_sit on_blanket'
pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
r_idx = zl.name2idx_tri(m_vp, pre)
# sub_lbl = pre.split('_')[0]
# pre_lbl = pre.split('_')[1]
# obj_lbl = pre.split('_')[2]
# sub_idx = zl.name2idx_cls(m,sub_lbl)
# pre_idx = zl.name2idx_pre(m,pre_lbl)
# obj_idx = zl.name2idx_cls(m,obj_lbl)
# rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16)
results = {}
for imid in imids:
# if cnt>2000:
# break
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
# rlp_labels = h5f[imid+'/labels'][...]
if imid + '/scores' not in h5f:
print imid
continue
# exit(0)
rlp_confs = h5f[imid + '/scores'][:, r_idx]
# print np.argmax(rlp_confs)
# exit(0)
# ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[np.argmax(rlp_confs)]
# print rlp_confs
results[imid] = np.average(rlp_confs)
# print np.average(rlp_confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg_vp/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:200]:
if imid + '/scores' not in h5f:
continue
#rlp_labels = h5f[imid+'/labels'][...]
impath = imlist[imid]
imid_orig = impath.split('/')[1].replace('.jpg', '')
rlp_confs = h5f[imid + '/scores'][:, r_idx]
ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[ind[:1]]
boxes = h5f[imid + '/boxes'][...][ind, :]
# indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
#boxes = boxes[ind]
sb = boxes[0]
# ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid_orig))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_retrieval_zeroshot():
# h5_path = 'output/sg_vrd_2016_result_all_19500.hdf5'
# h5_path = 'output/sg_vrd_2016_result_diff_all_5000.hdf5'
# h5_path= 'output/results/lu_method_results.hdf5'
# h5_path = 'output/sg_vrd_2016_result.hdf5.dd'
# h5_path = 'output/results/lu_method_results_max.hdf5'
h5_path = 'output/results/lu_visual_method_results.hdf5'
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
zeroshots = m['meta/zeroshots'][...]
gt_cache_path = 'output/cache/sg_vrd_gt_cache.pkl'
gt_h5f = {}
np.random.seed(76)
if os.path.exists(gt_cache_path):
print 'load gt from cache'
gt_h5f = zl.load(gt_cache_path)
else:
print 'cacheing gt'
for k in m['gt/test']:
gt_h5f[k] = {}
sub_boxes = m['gt/test'][k]['sub_boxes'][...]
rlp_labels = m['gt/test'][k]['rlp_labels'][...]
obj_boxes = m['gt/test'][k]['obj_boxes'][...]
gt_h5f[k]['sub_boxes'] = sub_boxes
gt_h5f[k]['obj_boxes'] = obj_boxes
gt_h5f[k]['rlp_labels'] = rlp_labels
print 'caching gt done'
zl.save(gt_cache_path, gt_h5f)
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
# h5f = h5py.File('output/sg_vrd_2016_result.classeme.hdf5')
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['rlp_labels'] = h5_in[i]['rlp_labels'][...]
h5f[i]['rlp_confs'] = h5_in[i]['rlp_confs'][...]
h5f[i]['sub_boxes'] = h5_in[i]['sub_boxes'][...]
h5f[i]['obj_boxes'] = h5_in[i]['obj_boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
#h5f = h5py.file('output/results/lu_method_results.hdf5')
retr_meta = zl.load('output/pkl/vr_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
for k in xrange(len(retr_meta['rlp_labels'])):
# if k>1000:
# break
rlp_label = retr_meta['rlp_labels'][k]
if not np.any(np.all(zeroshots == rlp_label, axis=1)): continue
gt_files = retr_meta['files'][k]
cnt += 1
results = {}
zl.tic()
for imid in imids:
rlp_labels = h5f[imid]['rlp_labels']
rlp_confs = h5f[imid]['rlp_confs']
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.random.uniform(-1, 1) #np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
delay = 0
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
r_at_k = 5
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
if found and i >= r_at_k:
break
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < r_at_k:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+imid+'.jpg')
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / r_at_k #total_gt
r_100 = tp_100 / r_at_k #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.toc())
