def save_train_vg():
prototxt = 'models/vg1_2/resnet50/test_cache.prototxt'
model = 'data/models/vrd_rfcn/vg_resnet50_rfcn_iter_112000.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, model, caffe.TEST)
m = h5py.File('data/vg1_2_meta.h5', 'r')
cache_hdf5 = h5py.File('output/vg_cache.h5', 'a')
cnt = 0
for k in m['gt/train'].keys():
cnt += 1
glog.info(cnt)
path = zl.imid2path(m, k)
im_path = C.get_vg_path_train(path)
im_detect(net, im_path)
data = net.blobs['conv_new_1'].data[...].astype(np.float16)
cache_hdf5.create_dataset('train/%s' % k + '/conv_new_1',
dtype='float16',
data=data)
def run_retrieval_vp_v2():
# h5_path = 'output/sg_vrd_2016_result.classeme.hdf5'
h5_path = 'output/precalc/vg1_2_vp2016_test_nms2_.4.hdf5'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
data_root = '/home/zawlin/data/data_vrd/vg_1.2/'
rlp_map = []
for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())):
tri = str(m_vp['meta/tri/idx2name'][str(i)][...])
s_lbl = tri.split('_')[0]
p_lbl = tri.split('_')[1]
o_lbl = tri.split('_')[2]
rlp_label = [
zl.name2idx_cls(m, s_lbl),
zl.name2idx_pre(m, p_lbl),
zl.name2idx_cls(m, o_lbl)
]
rlp_map.append(rlp_label)
rlp_map = np.array(rlp_map)
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['labels'] = h5_in[i]['labels'][...]
h5f[i]['confs'] = h5_in[i]['confs'][...]
h5f[i]['boxes'] = h5_in[i]['boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
total = 0
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
for k in xrange(len(retr_meta['rlp_labels'])):
total += retr_meta['counts'][k]
if k > 1000:
print total / 1000.
exit(0)
break
continue
cnt += 1
rlp_label = retr_meta['rlp_labels'][k]
gt_files = retr_meta['files'][k]
# print gt_files
# exit(0)
# for f in gt_files:
# impath= zl.imid2path(m,f)
# print impath
# im= cv2.imread(data_root+impath)
# cv2.imshow('im',im)
# cv2.waitKey(0)
results = {}
zl.tick()
ranks = []
for imid in imids:
labels = h5f[imid]['labels'] - 1
rlp_confs = h5f[imid]['confs']
rlp_labels = rlp_map[labels]
if rlp_labels.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
delay = 0
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
impath = zl.imid2path(m, imid)
if found and i >= 5:
break
# print gt_files
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < 5:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+impath)
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / 5 #total_gt
r_100 = tp_100 / 5 #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.tock())
def run_relation_visualization():
h5f_ours = h5py.File(
'output/vg_results/vg1_2_2016_result_all_100000.all.hdf5', 'r')
m = h5py.File('/media/zawlin/ssd/Dropbox/proj/vg1_2_meta.h5')
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f_ours.keys()
#random.shuffle(imids)
rel_types = {}
rel_types['p'] = []
rel_types['s'] = []
rel_types['v'] = []
rel_types['c'] = []
for k in m['meta/pre/name2idx'].keys():
idx = int(str(m['meta/pre/name2idx/' + k][...]))
r_type = m['meta/pre/name2idx/' + k].attrs['type']
rel_types[r_type].append(idx)
for imid in imids:
obj_boxes_ours = h5f_ours[imid + '/obj_boxes'][...]
sub_boxes_ours = h5f_ours[imid + '/sub_boxes'][...]
rlp_labels_ours = h5f_ours[imid + '/rlp_labels'][...].astype(np.int32)
rlp_confs_ours = h5f_ours[imid + '/rlp_confs'][...]
sorted_ind = np.argsort(rlp_confs_ours)[::-1]
rlp_confs_ours = rlp_confs_ours[sorted_ind]
obj_boxes_ours = obj_boxes_ours[sorted_ind]
sub_boxes_ours = sub_boxes_ours[sorted_ind]
rlp_labels_ours = rlp_labels_ours[sorted_ind]
if rlp_confs_ours.shape[0] <= 0: continue
ours_indices = {}
ours_indices['p'] = []
ours_indices['s'] = []
ours_indices['v'] = []
ours_indices['c'] = []
#map indices to type
for i in xrange(rlp_confs_ours.shape[0]):
pre_label = rlp_labels_ours[i][1]
if pre_label in rel_types['p']: ours_indices['p'].append(i)
if pre_label in rel_types['s']: ours_indices['s'].append(i)
if pre_label in rel_types['v']: ours_indices['v'].append(i)
if pre_label in rel_types['c']: ours_indices['c'].append(i)
im_path = zl.imid2path(m, imid)
im_orig = cv2.imread(data_root + im_path)
cv2.namedWindow('ctrl')
cv2.destroyWindow('ctrl')
cv2.namedWindow('ctrl')
ours_p_len = len(ours_indices['p']) - 1
ours_c_len = len(ours_indices['c']) - 1
ours_v_len = len(ours_indices['v']) - 1
ours_s_len = len(ours_indices['s']) - 1
ours_len = len(rlp_labels_ours) - 1
if ours_len > 0:
cv2.createTrackbar('idx_ours', 'ctrl', 0, ours_len, nothing)
if ours_p_len > 0:
cv2.createTrackbar('idx_ours_p', 'ctrl', 0, ours_p_len, nothing)
if ours_c_len > 0:
cv2.createTrackbar('idx_ours_c', 'ctrl', 0, ours_c_len, nothing)
if ours_v_len > 0:
cv2.createTrackbar('idx_ours_v', 'ctrl', 0, ours_v_len, nothing)
if ours_s_len > 0:
cv2.createTrackbar('idx_ours_s', 'ctrl', 0, ours_s_len, nothing)
cnt = 0
while True:
cv2.imshow('orig', im_orig)
if ours_len >= 0:
idx_ours = cv2.getTrackbarPos('idx_ours', 'ctrl')
im_ours = im_orig.copy()
s_box_ours = sub_boxes_ours[idx_ours]
o_box_ours = obj_boxes_ours[idx_ours]
conf = rlp_confs_ours[idx_ours]
sub_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][0])][...]
obj_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][2])][...]
pre_label_ours = m['meta/pre/idx2name/' +
str(rlp_labels_ours[idx_ours][1])][...]
r_label_ours = '%s %s %s' % (sub_label_ours, pre_label_ours,
obj_label_ours)
cv2.putText(im_ours, r_label_ours, (50, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)
cv2.rectangle(im_ours, (s_box_ours[0], s_box_ours[1]),
(s_box_ours[2], s_box_ours[3]), (0, 200, 0), 2)
cv2.rectangle(im_ours, (o_box_ours[0], o_box_ours[1]),
(o_box_ours[2], o_box_ours[3]), (0, 0, 200), 2)
cv2.imshow('im_ours', im_ours)
if ours_p_len >= 0:
idx_ours_p = cv2.getTrackbarPos('idx_ours_p', 'ctrl')
idx_ours = ours_indices['p'][idx_ours_p]
im_ours_p = im_orig.copy()
s_box_ours = sub_boxes_ours[idx_ours]
o_box_ours = obj_boxes_ours[idx_ours]
conf = rlp_confs_ours[idx_ours]
sub_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][0])][...]
obj_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][2])][...]
pre_label_ours = m['meta/pre/idx2name/' +
str(rlp_labels_ours[idx_ours][1])][...]
r_label_ours = '%s %s %s' % (sub_label_ours, pre_label_ours,
obj_label_ours)
cv2.putText(im_ours_p, r_label_ours, (50, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)
cv2.rectangle(im_ours_p, (s_box_ours[0], s_box_ours[1]),
(s_box_ours[2], s_box_ours[3]), (0, 200, 0), 2)
cv2.rectangle(im_ours_p, (o_box_ours[0], o_box_ours[1]),
(o_box_ours[2], o_box_ours[3]), (0, 0, 200), 2)
cv2.imshow('im_ours_p', im_ours_p)
if ours_v_len >= 0:
idx_ours_v = cv2.getTrackbarPos('idx_ours_v', 'ctrl')
idx_ours = ours_indices['v'][idx_ours_v]
im_ours_v = im_orig.copy()
s_box_ours = sub_boxes_ours[idx_ours]
o_box_ours = obj_boxes_ours[idx_ours]
conf = rlp_confs_ours[idx_ours]
sub_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][0])][...]
obj_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][2])][...]
pre_label_ours = m['meta/pre/idx2name/' +
str(rlp_labels_ours[idx_ours][1])][...]
r_label_ours = '%s %s %s' % (sub_label_ours, pre_label_ours,
obj_label_ours)
cv2.putText(im_ours_v, r_label_ours, (50, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)
cv2.rectangle(im_ours_v, (s_box_ours[0], s_box_ours[1]),
(s_box_ours[2], s_box_ours[3]), (0, 200, 0), 2)
cv2.rectangle(im_ours_v, (o_box_ours[0], o_box_ours[1]),
(o_box_ours[2], o_box_ours[3]), (0, 0, 200), 2)
cv2.imshow('im_ours_v', im_ours_v)
if ours_c_len >= 0:
idx_ours_c = cv2.getTrackbarPos('idx_ours_c', 'ctrl')
idx_ours = ours_indices['c'][idx_ours_c]
im_ours_c = im_orig.copy()
s_box_ours = sub_boxes_ours[idx_ours]
o_box_ours = obj_boxes_ours[idx_ours]
conf = rlp_confs_ours[idx_ours]
sub_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][0])][...]
obj_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][2])][...]
pre_label_ours = m['meta/pre/idx2name/' +
str(rlp_labels_ours[idx_ours][1])][...]
r_label_ours = '%s %s %s' % (sub_label_ours, pre_label_ours,
obj_label_ours)
cv2.putText(im_ours_c, r_label_ours, (50, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)
cv2.rectangle(im_ours_c, (s_box_ours[0], s_box_ours[1]),
(s_box_ours[2], s_box_ours[3]), (0, 200, 0), 2)
cv2.rectangle(im_ours_c, (o_box_ours[0], o_box_ours[1]),
(o_box_ours[2], o_box_ours[3]), (0, 0, 200), 2)
cv2.imshow('im_ours_c', im_ours_c)
if ours_s_len >= 0:
idx_ours_s = cv2.getTrackbarPos('idx_ours_s', 'ctrl')
idx_ours = ours_indices['s'][idx_ours_s]
im_ours_s = im_orig.copy()
s_box_ours = sub_boxes_ours[idx_ours]
o_box_ours = obj_boxes_ours[idx_ours]
conf = rlp_confs_ours[idx_ours]
sub_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][0])][...]
obj_label_ours = m['meta/cls/idx2name/' +
str(rlp_labels_ours[idx_ours][2])][...]
pre_label_ours = m['meta/pre/idx2name/' +
str(rlp_labels_ours[idx_ours][1])][...]
r_label_ours = '%s %s %s' % (sub_label_ours, pre_label_ours,
obj_label_ours)
cv2.putText(im_ours_s, r_label_ours, (50, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2)
cv2.rectangle(im_ours_s, (s_box_ours[0], s_box_ours[1]),
(s_box_ours[2], s_box_ours[3]), (0, 200, 0), 2)
cv2.rectangle(im_ours_s, (o_box_ours[0], o_box_ours[1]),
(o_box_ours[2], o_box_ours[3]), (0, 0, 200), 2)
cv2.imshow('im_ours_s', im_ours_s)
c = cv2.waitKey(1) & 0xFF
if c == ord(' '):
break
if c == ord('s'):
im_folder = 'output/results/examples/' + imid
if not os.path.exists('output/results/examples/' + imid):
os.makedirs('output/results/examples/' + imid)
if not os.path.exists('output/results/examples/' + imid +
'/orig_' + imid + '.jpg'):
cv2.imwrite(
'output/results/examples/' + imid + '/orig_' + imid +
'.jpg', im_orig)
if ours_v_len >= 0:
cv2.imwrite(
'output/results/examples/' + imid + '/ours_v_' + imid +
'_' + str(idx_ours) + '.jpg', im_ours_v)
if ours_p_len >= 0:
cv2.imwrite(
'output/results/examples/' + imid + '/ours_p_' + imid +
'_' + str(idx_ours) + '.jpg', im_ours_p)
if ours_c_len >= 0:
cv2.imwrite(
'output/results/examples/' + imid + '/ours_c_' + imid +
'_' + str(idx_ours) + '.jpg', im_ours_c)
if ours_s_len >= 0:
cv2.imwrite(
'output/results/examples/' + imid + '/ours_s_' + imid +
'_' + str(idx_ours) + '.jpg', im_ours_s)
cnt += 1
if c == 27:
exit(0)
def run_retrieval():
h5f = h5py.File('output/vg_results/vg1_2_2016_result_all_100000.all.hdf5')
#h5f = h5py.file('output/results/lu_method_results.hdf5')
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
# pre = 'teddy bear_sit on_blanket'
pre = 'cloth_next to_suitcase'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
sub_lbl = pre.split('_')[0]
pre_lbl = pre.split('_')[1]
obj_lbl = pre.split('_')[2]
sub_idx = zl.name2idx_cls(m, sub_lbl)
pre_idx = zl.name2idx_pre(m, pre_lbl)
obj_idx = zl.name2idx_cls(m, obj_lbl)
rlp_label = np.array([sub_idx, pre_idx, obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tick()
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
# print confs
# print np.average(confs)
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:20]:
sub_boxes = h5f[imid + '/sub_boxes'][...]
obj_boxes = h5f[imid + '/obj_boxes'][...]
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] != 0:
sub_boxes = sub_boxes[ind]
obj_boxes = obj_boxes[ind]
sb = sub_boxes[0]
ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
cv2.rectangle(im, (ob[0], ob[1]), (ob[2], ob[3]), (0, 0, 200), 1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_retrieval_vp():
h5f = h5py.File('output/precalc/vg1_2_vp2016_test_new.hdf5')
# print h5f[h5f.keys()[0]].keys()
# exit(0)
#h5f = h5py.file('output/results/lu_method_results.hdf5')
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
pre = 'teddy bear_sit on_blanket'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
r_idx = zl.name2idx_tri(m_vp, pre)
# sub_lbl = pre.split('_')[0]
# pre_lbl = pre.split('_')[1]
# obj_lbl = pre.split('_')[2]
# sub_idx = zl.name2idx_cls(m,sub_lbl)
# pre_idx = zl.name2idx_pre(m,pre_lbl)
# obj_idx = zl.name2idx_cls(m,obj_lbl)
# rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
# rlp_labels = h5f[imid+'/labels'][...]
rlp_confs = h5f[imid + '/confs'][:, r_idx]
ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[ind[:5]]
results[imid] = np.average(rlp_confs)
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tick()
indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg_vp/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:200]:
boxes = h5f[imid + '/boxes'][...]
rlp_labels = h5f[imid + '/labels'][...]
rlp_confs = h5f[imid + '/confs'][...]
# indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] != 0:
boxes = boxes[ind]
sb = boxes[0]
# ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_retrieval_vp_all():
h5f = h5py.File('output/precalc/vg1_2_vp2016_test_new.hdf5', 'r')
# print h5f[h5f.keys()[0]].keys()
# exit(0)
#h5f = h5py.file('output/results/lu_method_results.hdf5')
img_set_file = 'data/vg1_2_2016/ImageSets/test.txt'
imlist = {
line.strip().split(' ')[1]: line.strip().split(' ')[0]
for line in open(img_set_file)
}
data_root = '/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
# pre = 'teddy bear_sit on_blanket'
pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
r_idx = zl.name2idx_tri(m_vp, pre)
# sub_lbl = pre.split('_')[0]
# pre_lbl = pre.split('_')[1]
# obj_lbl = pre.split('_')[2]
# sub_idx = zl.name2idx_cls(m,sub_lbl)
# pre_idx = zl.name2idx_pre(m,pre_lbl)
# obj_idx = zl.name2idx_cls(m,obj_lbl)
# rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16)
results = {}
for imid in imids:
# if cnt>2000:
# break
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
# rlp_labels = h5f[imid+'/labels'][...]
if imid + '/scores' not in h5f:
print imid
continue
# exit(0)
rlp_confs = h5f[imid + '/scores'][:, r_idx]
# print np.argmax(rlp_confs)
# exit(0)
# ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[np.argmax(rlp_confs)]
# print rlp_confs
results[imid] = np.average(rlp_confs)
# print np.average(rlp_confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg_vp/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:200]:
if imid + '/scores' not in h5f:
continue
#rlp_labels = h5f[imid+'/labels'][...]
impath = imlist[imid]
imid_orig = impath.split('/')[1].replace('.jpg', '')
rlp_confs = h5f[imid + '/scores'][:, r_idx]
ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[ind[:1]]
boxes = h5f[imid + '/boxes'][...][ind, :]
# indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
#boxes = boxes[ind]
sb = boxes[0]
# ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid_orig))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def prep_pre(split_type):
if split_type != 'train' and split_type != 'test':
print 'error'
exit(0)
caffe.set_mode_gpu()
caffe.set_device(0)
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
cfg.TEST.HAS_RPN = False
net = caffe.Net(
'models/vg1_2/vgg16/faster_rcnn_end2end/test_jointbox.prototxt',
'output/models/vg1_2_vgg16_faster_rcnn_finetune_iter_145000.caffemodel',
caffe.TEST)
# net.name = os.path.splitext(os.path.basename(args.caffemodel))[0]
h5path = 'output/precalc/vg1_2_2016_predicate_exp_%s.hdf5' % split_type
h5f = h5py.File(h5path)
cnt = 0
for k in h5f.keys():
if cnt % 1000 == 0:
print cnt
cnt += 1
if 'sub_visual' not in h5f[k]:
print k
print len(h5f.keys())
exit(0)
root = 'data/vg1_2_2016/Data/%s/' % split_type
_t = {'im_detect': Timer(), 'misc': Timer()}
cnt = 0
thresh = .01
for imid in m['gt/%s' % split_type].keys():
cnt += 1
impath = zl.imid2path(m, imid)
fpath = os.path.join(root, impath)
im = cv2.imread(fpath)
if im == None:
print fpath
if cnt % 100 == 0:
print cnt
fpath = os.path.join(root, impath)
im = cv2.imread(fpath)
if im == None:
print fpath
gt_sub_boxes = m['gt/%s/%s/sub_boxes' % (split_type, imid)][...]
gt_obj_boxes = m['gt/%s/%s/obj_boxes' % (split_type, imid)][...]
gt_rlp_labels = m['gt/%s/%s/rlp_labels' % (split_type, imid)][...]
sub_boxes = []
obj_boxes = []
joint_boxes = []
boxes_batch = []
b_type = {}
b_idx = 0
sub_visual = []
obj_visual = []
joint_visual = []
pre_label = []
sub_cls = []
obj_cls = []
for i in xrange(gt_sub_boxes.shape[0]):
gt_sub_box = gt_sub_boxes[i][...]
gt_obj_box = gt_obj_boxes[i][...]
gt_rlp_label = gt_rlp_labels[i][...]
pre_idx = gt_rlp_label[1]
sub_cls_idx = gt_rlp_label[0]
obj_cls_idx = gt_rlp_label[2]
sub_cls.append(sub_cls_idx)
obj_cls.append(obj_cls_idx)
pre_label.append(pre_idx)
#print sub_lbl,predicate,obj_lbl
joint_bbox = [
min(gt_sub_box[0], gt_obj_box[0]),
min(gt_sub_box[1], gt_obj_box[1]),
max(gt_sub_box[2], gt_obj_box[2]),
max(gt_sub_box[3], gt_obj_box[3])
]
joint_boxes.append(joint_bbox)
sub_boxes.append(gt_sub_box)
obj_boxes.append(gt_obj_box)
# cv2.rectangle(im,(joint_bbox[0],joint_bbox[1]),(joint_bbox[2],joint_bbox[3]),(255,255,255),4)
# cv2.rectangle(im,(sub_bbox[0],sub_bbox[1]),(sub_bbox[2],sub_bbox[3]),(0,0,255),2)
# cv2.rectangle(im,(obj_bbox[0],obj_bbox[1]),(obj_bbox[2],obj_bbox[3]),(255,0,0),2)
for i in xrange(len(sub_boxes)):
boxes_batch.append(sub_boxes[i])
b_type[b_idx] = 's'
b_idx += 1
for i in xrange(len(obj_boxes)):
boxes_batch.append(obj_boxes[i])
b_type[b_idx] = 'o'
b_idx += 1
for i in xrange(len(joint_boxes)):
boxes_batch.append(joint_boxes[i])
b_type[b_idx] = 'j'
b_idx += 1
box_proposals = None
_t['im_detect'].tic()
score_raw, scores, fc7, boxes = im_detect(net, im,
np.array(boxes_batch))
for i in xrange(scores.shape[0]):
s_idx = np.argmax(scores[i, 1:]) + 1
cls_box = None
cls_box = boxes[i, s_idx * 4:(s_idx + 1) * 4]
if b_type[i] == 's':
sub_visual.append(fc7[i])
if b_type[i] == 'o':
obj_visual.append(fc7[i])
if b_type[i] == 'j':
joint_visual.append(fc7[i])
# cls_name = str(m['meta/cls/idx2name/' + str(s_idx)][...])
# if b_type[i] == 's':
# print str(m['meta/pre/idx2name/'+str(pre_label[i])][...])
# cv2.rectangle(im,(cls_box[0],cls_box[1]),(cls_box[2],cls_box[3]),(255,0,0),2)
# cv2.imshow('im',im)
_t['im_detect'].toc()
_t['misc'].tic()
sub_visual = np.array(sub_visual).astype(np.float16)
obj_visual = np.array(obj_visual).astype(np.float16)
joint_visual = np.array(joint_visual).astype(np.float16)
pre_label = np.array(pre_label).astype(np.int32)
sub_boxes = np.array(sub_boxes).astype(np.int32)
obj_boxes = np.array(obj_boxes).astype(np.int32)
sub_cls = np.array(sub_cls).astype(np.int32)
obj_cls = np.array(obj_cls).astype(np.int32)
h5f.create_dataset(imid + '/sub_visual',
dtype='float16',
data=sub_visual)
h5f.create_dataset(imid + '/obj_visual',
dtype='float16',
data=obj_visual)
h5f.create_dataset(imid + '/joint_visual',
dtype='float16',
data=joint_visual)
h5f.create_dataset(imid + '/pre_label',
dtype='float16',
data=pre_label)
h5f.create_dataset(imid + '/sub_boxes',
dtype='float16',
data=sub_boxes)
h5f.create_dataset(imid + '/obj_boxes',
dtype='float16',
data=obj_boxes)
h5f.create_dataset(imid + '/sub_cls', dtype='float16', data=sub_cls)
h5f.create_dataset(imid + '/obj_cls', dtype='float16', data=obj_cls)
_t['misc'].toc()
# rpn_rois = net.blobs['rois'].data
# pool5 = net.blobs['pool5'].data
# _t['misc'].tic()
# cnt += 1
print 'im_detect: {:d} {:.3f}s {:.3f}s' \
.format(cnt, _t['im_detect'].average_time,
_t['misc'].average_time)
def run_test_save_result():
caffe.set_mode_gpu()
caffe.set_device(0)
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
net = caffe.Net('models/vg1_2/vgg16/faster_rcnn_end2end/test.prototxt',
'output/models/vg1_2_vgg16_faster_rcnn_finetune_iter_120000.caffemodel',caffe.TEST)
h5path = 'output/precalc/vg1_2_2016_test.hdf5'
h5f = h5py.File(h5path)
root = 'data/vg1_2_2016/Data/test/'
_t = {'im_detect': Timer(), 'misc': Timer()}
cnt = 0
thresh = .25
for imid in m['gt/test'].keys():
cnt += 1
if imid in h5f:continue
impath = zl.imid2path(m,imid)
fpath = os.path.join(root, impath)
if not os.path.exists(fpath):
zl.copy_file('data/vg1_2_2016/Data/train/'+impath,fpath)
# print fpath
# exit(0)
im = cv2.imread(fpath)
if im == None:
print fpath
box_proposals = None
_t['im_detect'].tic()
score_raw, scores, fc7, boxes = im_detect(net, im, box_proposals)
_t['im_detect'].toc()
res_locations = []
res_visuals = []
res_classemes = []
res_cls_confs = []
_t['misc'].tic()
for j in xrange(1, 201):
inds = np.where(scores[:, j] > 0.01)[0]
cls_scores = scores[inds, j]
cls_boxes = boxes[inds, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
res_loc = cls_boxes
res_vis = fc7[inds]
res_classeme = scores[inds]
res_cls_conf = np.column_stack((np.zeros(cls_scores.shape[0]) + j, cls_scores))
keep = nms(cls_dets, .2,force_cpu=True) # nms threshold
cls_dets = cls_dets[keep, :]
res_loc = res_loc[keep]
res_vis = res_vis[keep]
res_classeme = res_classeme[keep]
res_cls_conf = res_cls_conf[keep]
res_classemes.extend(res_classeme)
res_visuals.extend(res_vis)
res_locations.extend(res_loc)
res_cls_confs.extend(res_cls_conf)
inds = np.where(np.array(res_cls_confs)[:, 1] > thresh)[0]
res_classemes = np.array(res_classemes)[inds]
res_visuals = np.array(res_visuals)[inds]
res_locations = np.array(res_locations)[inds]
res_cls_confs = np.array(res_cls_confs)[inds]
h5f.create_dataset(imid + '/classemes', dtype='float16', data=res_classemes.astype(np.float16))
h5f.create_dataset(imid + '/visuals', dtype='float16', data=res_visuals.astype(np.float16))
h5f.create_dataset(imid + '/locations', dtype='short', data=res_locations.astype(np.short))
h5f.create_dataset(imid + '/cls_confs', dtype='float16', data=res_cls_confs.astype(np.float16))
_t['misc'].toc()
print 'im_detect: {:d} {:.3f}s {:.3f}s' \
.format(cnt, _t['im_detect'].average_time,
_t['misc'].average_time)
