def print_triplet_info():
m = h5py.File('/media/zawlin/ssd/Dropbox/proj/sg_vrd_meta.h5')
train_triplets, train_triplets_sorted = get_triplet_stats('train')
test_triplets, test_triplets_sorted = get_triplet_stats('test')
triplets, triplets_sorted = test_triplets, test_triplets_sorted
#triplets,triplets_sorted = train_triplets,train_triplets_sorted
for i in xrange(len(triplets_sorted)):
if triplets_sorted[i][0] in train_triplets: continue
spo = triplets_sorted[i][0].split('_')
s = zl.name2idx_cls(m, spo[0])
p = zl.name2idx_pre(m, spo[1])
o = zl.name2idx_cls(m, spo[2])
spo_str = '%s_%s_%s' % (str(s), str(p), str(o))
print '%s;%s;%s' % (triplets_sorted[i][0], spo_str,
triplets_sorted[i][1])
def save_zeroshots_to_meta():
m = h5py.File('/media/zawlin/ssd/Dropbox/proj/sg_vrd_meta.h5')
train_triplets, train_triplets_sorted = get_triplet_stats('train')
test_triplets, test_triplets_sorted = get_triplet_stats('test')
triplets, triplets_sorted = test_triplets, test_triplets_sorted
#triplets,triplets_sorted = train_triplets,train_triplets_sorted
zeroshots = []
for i in xrange(len(triplets_sorted)):
if triplets_sorted[i][0] in train_triplets: continue
spo = triplets_sorted[i][0].split('_')
s = zl.name2idx_cls(m, spo[0])
p = zl.name2idx_pre(m, spo[1])
o = zl.name2idx_cls(m, spo[2])
zeroshots.append([s, p, o])
spo_str = '%s_%s_%s' % (str(s), str(p), str(o))
print '%s;%s;%s' % (triplets_sorted[i][0], spo_str,
triplets_sorted[i][1])
# if triplets_sorted[i][0] not in train_triplets:
# print triplets_sorted[i][0],triplets_idx_sorted[i][0],triplets_sorted[i][1]
# if triplets_sorted[i][0] not in triplets:
# print test_triplets_sorted[i][0],test_triplets_sorted[i][1]
m.create_dataset('meta/zeroshots',
data=np.array(zeroshots).astype(np.int16))
def relation_make_meta():
spo_info = zl.load('output/spo_info_vg.pkl')
spo_list = []
spo_dict = {}
for k in spo_info.keys():
if spo_info[k] >= 5:
spo_list.append(k)
for spo in spo_list:
spo_dict[spo] = 0
spo_list = spo_dict
blacklist = [
'VG_100K/2363098',
'VG_100K_2/2402233',
'VG_100K/2365839',
'VG_100K_2/2398948',
'VG_100K/2315697',
'VG_100K_2/2403354',
]
client = MongoClient("mongodb://localhost:27017")
db = client.visual_genome_1_2
imdatas = {}
for imdata in db.image_data.find(no_cursor_timeout=True):
imid = imdata['image_id']
imdatas[imid] = imdata
db_results_train = db.relationships_all_train.find(no_cursor_timeout=True)
m = h5py.File('data/vg1_2_meta.h5')
cnt = 0
for doc in db_results_train:
if cnt % 1000 == 0:
print cnt
cnt += 1
imid = doc['image_id']
im_data = imdatas[imid]
if im_data['width'] < 100 or im_data['height'] < 100: continue
im_path_full = im_data['url'].replace(
'https://cs.stanford.edu/people/rak248/', '')
im_path_folder = im_path_full.split('/')[0]
im_path_file = im_path_full.split('/')[1]
im_index = im_path_folder + '/' + im_path_file.replace('.jpg', '')
if im_index in blacklist: continue
obj_boxes = []
sub_boxes = []
rlp_labels = []
imid = doc['image_id']
imdata = imdatas[imid]
for r in doc['relationships']:
pre = r['predicate']
sub_name = r['subject']['name']
obj_name = r['object']['name']
spo = sub_name + '_' + pre + '_' + obj_name
if spo in spo_list:
sidx = zl.name2idx_cls(m, sub_name)
oidx = zl.name2idx_cls(m, obj_name)
pidx = zl.name2idx_pre(m, pre)
ox1, oy1, ow, oh = r['object']['x'], r['object']['y'], r[
'object']['w'], r['object']['h']
sx1, sy1, sw, sh = r['subject']['x'], r['subject']['y'], r[
'subject']['w'], r['subject']['h']
ox2, oy2 = ox1 + ow, oy1 + oh
sx2, sy2 = sx1 + sw, sy1 + sh
rlp_labels.append([sidx, pidx, oidx])
sub_boxes.append([sx1, sy1, sx2, sy2])
obj_boxes.append([ox1, oy1, ox2, oy2])
m.create_dataset('gt/train/%s/rlp_labels' % imid,
data=np.array(rlp_labels).astype(np.int16))
m.create_dataset('gt/train/%s/sub_boxes' % imid,
data=np.array(sub_boxes).astype(np.int16))
m.create_dataset('gt/train/%s/obj_boxes' % imid,
data=np.array(obj_boxes).astype(np.int16))
db_results_test = db.relationships_all_test.find(no_cursor_timeout=True)
for doc in db_results_test:
if cnt % 1000 == 0:
print cnt
cnt += 1
imid = doc['image_id']
im_data = imdatas[imid]
if im_data['width'] < 100 or im_data['height'] < 100: continue
im_path_full = im_data['url'].replace(
'https://cs.stanford.edu/people/rak248/', '')
im_path_folder = im_path_full.split('/')[0]
im_path_file = im_path_full.split('/')[1]
im_index = im_path_folder + '/' + im_path_file.replace('.jpg', '')
if im_index in blacklist: continue
obj_boxes = []
sub_boxes = []
rlp_labels = []
imid = doc['image_id']
imdata = imdatas[imid]
for r in doc['relationships']:
pre = r['predicate']
sub_name = r['subject']['name']
obj_name = r['object']['name']
spo = sub_name + '_' + pre + '_' + obj_name
if spo in spo_list:
sidx = zl.name2idx_cls(m, sub_name)
oidx = zl.name2idx_cls(m, obj_name)
pidx = zl.name2idx_pre(m, pre)
ox1, oy1, ow, oh = r['object']['x'], r['object']['y'], r[
'object']['w'], r['object']['h']
sx1, sy1, sw, sh = r['subject']['x'], r['subject']['y'], r[
'subject']['w'], r['subject']['h']
ox2, oy2 = ox1 + ow, oy1 + oh
sx2, sy2 = sx1 + sw, sy1 + sh
rlp_labels.append([sidx, pidx, oidx])
sub_boxes.append([sx1, sy1, sx2, sy2])
obj_boxes.append([ox1, oy1, ox2, oy2])
m.create_dataset('gt/test/%s/rlp_labels' % imid,
data=np.array(rlp_labels).astype(np.int16))
m.create_dataset('gt/test/%s/sub_boxes' % imid,
data=np.array(sub_boxes).astype(np.int16))
m.create_dataset('gt/test/%s/obj_boxes' % imid,
data=np.array(obj_boxes).astype(np.int16))
def make_matlab_from_vp_nms2():
import scipy.io as sio
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_vp_meta.h5')
h5path = 'output/precalc/sg_vrd_2016_test_nms.7.hdf5'
h5f_nms = h5py.File(h5path)
imids = sorted(m['gt/test'].keys())
boxes_ours= np.empty((len(imids)),dtype=np.object)
rlp_labels_ours = np.empty((len(imids)),dtype=np.object)
rlp_confs_ours = np.empty((len(imids)),dtype=np.object)
for i in xrange(boxes_ours.shape[0]):
boxes_ours[i]=[]
rlp_labels_ours[i]=[]
rlp_confs_ours[i]=[]
cnt = 1
idx = 0
for imid in imids:
if cnt %100==0:
print cnt
cnt+=1
if imid not in h5f_nms:
rlp_labels_ours[idx]=[]
rlp_confs_ours[idx]=[]
boxes_ours[idx]=[]
idx += 1
continue
boxes = h5f_nms[imid]['boxes'][...]
confs = h5f_nms[imid]['confs'][...]
labels = h5f_nms[imid]['labels'][...]
ind = np.argsort(confs)[::-1]
if ind.shape[0]>100:
ind = ind[:100]
boxes_raw = boxes[ind]
confs = confs[ind]
labels = labels[ind]
boxes = []
rlp_confs = []
rlp_labels = []
for i in xrange(confs.shape[0]):
lbl = zl.idx2name_tri(m_vp,labels[i])
sub_lbl = lbl.split('_')[0]
pre_lbl = lbl.split('_')[1]
obj_lbl = lbl.split('_')[2]
rlp_label = [zl.name2idx_cls(m,sub_lbl),zl.name2idx_pre(m,pre_lbl),zl.name2idx_cls(m,obj_lbl)]
rlp_labels.append(np.array(rlp_label).astype(np.float64))
rlp_confs.append(np.array(confs[i]).astype(np.float64))
boxes.append(np.array(boxes_raw[i]))
# boxes = np.array(boxes).reshape((-1,4))
# rlp_labels = np.array(rlp_labels).reshape((-1,3))
# rlp_confs= np.array(rlp_confs)[:,np.newaxis]
rlp_confs_ours[idx] = rlp_confs
rlp_labels_ours[idx] = rlp_labels
boxes_ours[idx] = boxes
idx += 1
# boxes_ours.append([])
# boxes_ours=boxes_ours[:-1]
sio.savemat('output/sg_vrd_vp_results_.7.mat', {'bboxes_ours': boxes_ours,
'rlp_labels_ours':rlp_labels_ours,'rlp_confs_ours':rlp_confs_ours})#'relation_vectors':relation_vectors})
def run_retrieval_vp_v2():
# h5_path = 'output/sg_vrd_2016_result.classeme.hdf5'
h5_path = 'output/precalc/vg1_2_vp2016_test_nms2_.4.hdf5'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
data_root = '/home/zawlin/data/data_vrd/vg_1.2/'
rlp_map = []
for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())):
tri = str(m_vp['meta/tri/idx2name'][str(i)][...])
s_lbl = tri.split('_')[0]
p_lbl = tri.split('_')[1]
o_lbl = tri.split('_')[2]
rlp_label = [
zl.name2idx_cls(m, s_lbl),
zl.name2idx_pre(m, p_lbl),
zl.name2idx_cls(m, o_lbl)
]
rlp_map.append(rlp_label)
rlp_map = np.array(rlp_map)
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['labels'] = h5_in[i]['labels'][...]
h5f[i]['confs'] = h5_in[i]['confs'][...]
h5f[i]['boxes'] = h5_in[i]['boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
total = 0
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
for k in xrange(len(retr_meta['rlp_labels'])):
total += retr_meta['counts'][k]
if k > 1000:
print total / 1000.
exit(0)
break
continue
cnt += 1
rlp_label = retr_meta['rlp_labels'][k]
gt_files = retr_meta['files'][k]
# print gt_files
# exit(0)
# for f in gt_files:
# impath= zl.imid2path(m,f)
# print impath
# im= cv2.imread(data_root+impath)
# cv2.imshow('im',im)
# cv2.waitKey(0)
results = {}
zl.tick()
ranks = []
for imid in imids:
labels = h5f[imid]['labels'] - 1
rlp_confs = h5f[imid]['confs']
rlp_labels = rlp_map[labels]
if rlp_labels.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
delay = 0
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
impath = zl.imid2path(m, imid)
if found and i >= 5:
break
# print gt_files
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < 5:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+impath)
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / 5 #total_gt
r_100 = tp_100 / 5 #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.tock())
def run_retrieval():
h5f = h5py.File('output/vg_results/vg1_2_2016_result_all_100000.all.hdf5')
#h5f = h5py.file('output/results/lu_method_results.hdf5')
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
# pre = 'teddy bear_sit on_blanket'
pre = 'cloth_next to_suitcase'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
sub_lbl = pre.split('_')[0]
pre_lbl = pre.split('_')[1]
obj_lbl = pre.split('_')[2]
sub_idx = zl.name2idx_cls(m, sub_lbl)
pre_idx = zl.name2idx_pre(m, pre_lbl)
obj_idx = zl.name2idx_cls(m, obj_lbl)
rlp_label = np.array([sub_idx, pre_idx, obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tick()
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
# print confs
# print np.average(confs)
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:20]:
sub_boxes = h5f[imid + '/sub_boxes'][...]
obj_boxes = h5f[imid + '/obj_boxes'][...]
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] != 0:
sub_boxes = sub_boxes[ind]
obj_boxes = obj_boxes[ind]
sb = sub_boxes[0]
ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
cv2.rectangle(im, (ob[0], ob[1]), (ob[2], ob[3]), (0, 0, 200), 1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1