def test_matlab_engine():
import utils.zl_edge_box as zl_eb
zl.tic()
windows = zl_eb.get_windows('/home/zawlin/data/indoor.jpg')
print zl.toc()
windows = np.array(windows)
im = cv2.imread('/home/zawlin/data/indoor.jpg')
scores = [.1, .2, .3, .4, .5, .6, .7, .8, .9, .95, .99]
ims = []
for s in scores:
ims.append(im.copy())
for i in xrange(len(scores)):
s = scores[i]
im_render = ims[i]
for i in xrange(windows.shape[0]):
roi = windows[i].astype(np.int32)
if windows[i][4] > s:
cv2.rectangle(im_render, (roi[0], roi[1]), (roi[2], roi[3]),
(255, 0, 0), 1)
cv2.imwrite('/home/zawlin/data/iccv/iccv_thresh_%f.jpg' % s, im_render)
for i in xrange(windows.shape[0]):
b = windows[i].astype(np.int32)
cv2.rectangle(im, (b[0], b[1]), (b[2], b[3]), (255, 0, 0), 2)
cv2.imshow('drawn', im)
cv2.waitKey(0)
def test_coco_api():
coco = COCO('/media/zawlin/ssd/coco/annotations/instances_train2014.json')
cats = coco.loadCats(coco.getCatIds())
print dir(cats)
nms = [cat['name'] for cat in cats]
# catIds = coco.getCatIds(catNms=['person','dog','skateboard'])
# imgIds = coco.getImgIds(catIds=catIds )
imgs = coco.loadImgs()
for img in imgs:
im_path = C.coco_dir + 'train2014/' + img['file_name']
im = cv2.imread(im_path)
annIds = coco.getAnnIds(imgIds=img['id'], )
anns = coco.loadAnns(annIds)
for ann in anns:
bbox = np.array(ann['bbox']).astype(np.int32)
zl.tic()
cats = coco.loadCats(ann['category_id'])[0]['name']
print zl.toc()
bbox[2], bbox[3] = bbox[0] + bbox[2], bbox[1] + bbox[3]
cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
(255, 0, 0), 2)
cv2.putText(im, str(cats), (bbox[0], bbox[1]),
cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 255), 2)
cv2.imshow('im', im)
if cv2.waitKey(0) == 27:
exit(0)
def setup(self, bottom, top):
"""Setup the RoIDataLayer."""
self._cur_idx = 0
self.gt_labels = {}
self.meta = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r')
self.cache = h5py.File('output/sg_vrd_cache.h5','r')
self.cache_mem = {}
if os.path.exists('output/cache/sg_vrd_gt.pkl'):
self.gt_labels = zl.load('output/cache/sg_vrd_gt.pkl')
glog.info('loaded gt data from cache')
else:
glog.info( 'Preloading gt')
zl.tic()
for k in self.meta['gt/train'].keys():
rlp_labels = self.meta['gt/train/%s/rlp_labels'%k][...]
sub_boxes = self.meta['gt/train/%s/sub_boxes'%k][...].astype(np.float)
obj_boxes = self.meta['gt/train/%s/obj_boxes'%k][...].astype(np.float)
if sub_boxes.shape[0]>0:
zeros = np.zeros((sub_boxes.shape[0],1), dtype=np.float)
# first index is always zero since we do one image by one image
sub_boxes = np.concatenate((zeros, sub_boxes),axis=1)
obj_boxes = np.concatenate((zeros, obj_boxes),axis=1)
self.gt_labels[k] = {}
self.gt_labels[k]['rlp_labels']=rlp_labels
self.gt_labels[k]['sub_boxes']=sub_boxes
self.gt_labels[k]['obj_boxes']=obj_boxes
glog.info('done preloading gt %f'%zl.toc())
zl.save('output/cache/sg_vrd_gt.pkl',self.gt_labels)
self.imids = []
for k in self.gt_labels.keys():
self.imids.append(k)
self.imidx =0
random.shuffle(self.imids)
# parse the layer parameter string, which must be valid YAML
layer_params = yaml.load(self.param_str)
self._num_classes = layer_params['num_classes']
self._name_to_top_map = {}
# data blob: holds a batch of N images, each with 3 channels
idx = 0
top[idx].reshape(cfg.TRAIN.IMS_PER_BATCH, 1024,
50,50)
self._name_to_top_map['conv_new_1'] = idx
idx += 1
top[idx].reshape(1, 5, 1, 1)
self._name_to_top_map['sub_boxes'] = idx
idx += 1
top[idx].reshape(1, 5, 1, 1)
self._name_to_top_map['obj_boxes'] = idx
idx += 1
# labels blob: R categorical labels in [0, ..., K] for K foreground
# classes plus background
top[idx].reshape(1, 1, 1, 1)
self._name_to_top_map['labels'] = idx
def test_edge_box_py():
import edgebox.edge_boxes as eb
im = cv2.imread('/home/zawlin/data/indoor.jpg')
im = cv2.resize(im, (500, 640))
zl.tic()
windows = eb.get_windows(['/home/zawlin/data/indoor.jpg'], maxboxes=10)
windows = np.array(windows[0])
for i in xrange(windows.shape[0]):
b = windows[i].astype(np.int32)
cv2.rectangle(im, (b[1], b[0]), (b[3], b[2]), (255, 0, 0), 2)
cv2.imshow('drawn', im)
cv2.waitKey(0)
def test_edge_boxes_cpp_wrapper():
from edge_boxes_python import edge_boxes_python
eb = edge_boxes_python('edges/cpp/external/gop_1.3/data/sf.dat')
im = cv2.imread('/home/zawlin/data/indoor.jpg')
#im = cv2.resize(im,(500,640))
#im = zl.url_to_image('http://images1.mtv.com/uri/mgid:file:docroot:cmt.com:/sitewide/assets/img/artists/wesley_james/photo_gallery/didnt_i/8-Indoor-Scene-x600.jpg')
print 'detecting'
zl.tic()
bbs = eb.get_edge_boxes(im)
bbs = bbs[bbs[:, 4].argsort()[::-1]]
for i in xrange(10):
b = bbs[i]
cv2.rectangle(im, (b[0], b[1]), (b[2], b[3]), (255, 0, 0), 2)
cv2.imshow('drawn', im)
cv2.waitKey(0)
print zl.toc()
print bbs
def load_test():
h5path = '/media/zawlin/ssd/iccv2017/data/voc/gen_eb.h5'
h5f = h5py.File(h5path, driver='core')
data = {}
cnt = 0
zl.tic()
for i in h5f['train/']:
cnt += 1
if cnt % 100 == 0:
print zl.toc(), cnt
zl.tic()
data = h5f['train/%s' % i][...].astype(np.float32)
print data[:, -2]
idx = np.argsort(data[:, -2], axis=0)
print data[idx][::-1]
exit(0)
def setup(self, bottom, top):
self._cur_idx = 0
self.rdata = sio.loadmat('data/meta/vrd/annotation_train.mat',
struct_as_record=False,
squeeze_me=True)
self.meta = h5py.File('data/sg_vrd_meta.h5', 'r')
self.gt_labels = {}
if os.path.exists('output/cache/sg_vrd_gt.pkl'):
self.gt_labels = zl.load('output/cache/sg_vrd_gt.pkl')
glog.info('loaded gt data from cache')
else:
glog.info('Preloading gt')
zl.tic()
for k in self.meta['gt/train'].keys():
rlp_labels = self.meta['gt/train/%s/rlp_labels' % k][...]
sub_boxes = self.meta['gt/train/%s/sub_boxes' % k][...].astype(
np.float)
obj_boxes = self.meta['gt/train/%s/obj_boxes' % k][...].astype(
np.float)
if sub_boxes.shape[0] > 0:
zeros = np.zeros((sub_boxes.shape[0], 1), dtype=np.float)
# first index is always zero since we do one image by one image
sub_boxes = np.concatenate((zeros, sub_boxes), axis=1)
obj_boxes = np.concatenate((zeros, obj_boxes), axis=1)
self.gt_labels[k] = {}
self.gt_labels[k]['rlp_labels'] = rlp_labels
self.gt_labels[k]['sub_boxes'] = sub_boxes
self.gt_labels[k]['obj_boxes'] = obj_boxes
glog.info('done preloading gt %f' % zl.toc())
vgg_h5 = h5py.File("output/sg_vrd_2016_train_predicate_exp_train.hdf5",
'r', 'core')
if os.path.exists('output/cache/sg_vrd_2016_train.pkl'):
self.vgg_data = zl.load('output/cache/sg_vrd_2016_train.pkl')
glog.info('loaded train data from cache')
else:
glog.info('Preloading training data')
zl.tic()
self.vgg_data = {}
for k in vgg_h5.keys():
sub_visual = vgg_h5[k]['sub_classeme'][...]
obj_visual = vgg_h5[k]['obj_classeme'][...]
pre_label = vgg_h5[k]['pre_label'][...]
self.vgg_data[k] = {}
self.vgg_data[k]['sub_visual'] = sub_visual
self.vgg_data[k]['obj_visual'] = obj_visual
self.vgg_data[k]['pre_label'] = pre_label
glog.info('done preloading training data %f' % zl.toc())
zl.save('output/cache/sg_vrd_2016_train.pkl', self.vgg_data)
vgg_h5.close()
self.meta = h5py.File('data/sg_vrd_meta.h5', 'r', 'core')
layer_params = yaml.load(self.param_str)
self._batch_size = layer_params['batch_size']
self.train_data = []
self._name_to_top_map = {}
# data blob: holds a batch of N images, each with 3 channels
# top[0].reshape(self._batch_size, 4096 * 2 )
top[0].reshape(self._batch_size, 2 * 101)
top[1].reshape(self._batch_size)
# self._name_to_top_map['visual'] = 0
# self._name_to_top_map['classeme'] = 0
self._name_to_top_map['visual'] = 0
# self._name_to_top_map['location'] = 1
self._name_to_top_map['label'] = 1
def gen_obj_detection_results_from_hdf5_vgg(h5_path):
m = h5py.File('data/vg1_2_meta.h5', 'r', 'core')
h5f = h5py.File(h5_path, 'r')
thresh = 0.01
cnt = 0
zl.tic()
mat_pred_label = []
mat_pred_conf = []
mat_pred_bb = []
mat_gt_label = []
mat_gt_bb = []
imid2path = {}
impath2id = {}
imidx2id = {}
for i in m['gt/test/'].keys():
imid2path[str(i)] = str(m['meta/imid2path/%s' % i][...])[:-4]
for i in imid2path.keys():
path = imid2path[i]
impath2id[str(path)] = i
img_set_file = 'data/vg1_2_2016/ImageSets/test.txt'
imlist = {
line.strip().split(' ')[1]: line.strip().split(' ')[0]
for line in open(img_set_file)
}
for imid in imlist.keys():
path = imlist[imid]
if path in impath2id:
imidx2id[imid] = impath2id[path]
#for k in m['gt/test/'].keys():
for k in imlist.keys():
if k not in imidx2id:
continue
imid = imidx2id[k]
if cnt % 1000 == 0:
print cnt, zl.toc()
zl.tic()
cnt += 1
scores = h5f['%s/scores' % k][...]
boxes = h5f['%s/boxes' % k][...]
boxes_tosort = []
mat_pred_label_i = []
mat_pred_conf_i = []
mat_pred_bb_i = []
for j in xrange(1, 101):
inds = np.where(scores[:, j] > 0.001)[0]
cls_scores = scores[inds, j]
cls_boxes = boxes[inds, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
keep = nms(cls_dets, .2, force_cpu=True) # nms threshold
cls_dets = cls_dets[keep, :]
boxes_tosort.append(cls_dets)
for j in xrange(len(boxes_tosort)):
cls_dets = boxes_tosort[j]
for di in xrange(cls_dets.shape[0]):
# print 'here'
di = cls_dets[di]
score = di[-1]
cls_idx = j + 1
if score > 1:
score = 1
if score < thresh:
continue
res_line = '%s %d %f %d %d %d %d' % (k, cls_idx, score, di[0],
di[1], di[2], di[3])
mat_pred_label_i.append(cls_idx)
mat_pred_conf_i.append(score)
mat_pred_bb_i.append([di[0], di[1], di[2], di[3]])
mat_pred_label.append(mat_pred_label_i)
mat_pred_conf.append(mat_pred_conf_i)
mat_pred_bb.append(mat_pred_bb_i)
obj_boxes = m['gt/test/%s/obj_boxes' % imid][...]
sub_boxes = m['gt/test/%s/sub_boxes' % imid][...]
rlp_labels = m['gt/test/%s/rlp_labels' % imid][...]
mat_gt_label_i = []
mat_gt_bb_i = []
mat_gt_i = []
for gti in xrange(obj_boxes.shape[0]):
mat_gt_i.append([
rlp_labels[gti, 0], sub_boxes[gti, 0], sub_boxes[gti, 1],
sub_boxes[gti, 2], sub_boxes[gti, 3]
])
mat_gt_i.append([
rlp_labels[gti, 2], obj_boxes[gti, 0], obj_boxes[gti, 1],
obj_boxes[gti, 2], obj_boxes[gti, 3]
])
if len(mat_gt_i) > 0:
mat_gt_i = np.array(mat_gt_i)
mat_gt_i = zl.unique_arr(mat_gt_i)
for gti in xrange(mat_gt_i.shape[0]):
mat_gt_bb_i.append(mat_gt_i[gti, 1:])
mat_gt_label_i.append(mat_gt_i[gti, 0])
mat_gt_label.append(mat_gt_label_i)
mat_gt_bb.append(mat_gt_bb_i)
#outfile.write(res_line+'\n')
sio.savemat(
'output/vg_vrd_objs.mat', {
'pred_bb': mat_pred_bb,
'pred_conf': mat_pred_conf,
'pred_label': mat_pred_label,
'gt_bb': mat_gt_bb,
'gt_label': mat_gt_label
})
def run_retrieval_vp():
h5f = h5py.File('output/precalc/ours_retr.hdf5')
retr_meta = zl.load('output/pkl/vr_retr_meta.pkl')
# print h5f[h5f.keys()[0]].keys()
# exit(0)
#h5f = h5py.file('output/results/lu_method_results.hdf5')
# data_root='/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
pre = 'teddy bear_sit on_blanket'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
r_idx = zl.name2idx_tri(m_vp, pre)
# sub_lbl = pre.split('_')[0]
# pre_lbl = pre.split('_')[1]
# obj_lbl = pre.split('_')[2]
# sub_idx = zl.name2idx_cls(m,sub_lbl)
# pre_idx = zl.name2idx_pre(m,pre_lbl)
# obj_idx = zl.name2idx_cls(m,obj_lbl)
# rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.toc()
zl.tic()
cnt += 1
# rlp_labels = h5f[imid+'/labels'][...]
rlp_confs = h5f[imid + '/confs'][:, r_idx]
ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[ind[:5]]
results[imid] = np.average(rlp_confs)
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tic()
indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg_vp/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:200]:
boxes = h5f[imid + '/boxes'][...]
rlp_labels = h5f[imid + '/labels'][...]
rlp_confs = h5f[imid + '/confs'][...]
# indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] != 0:
boxes = boxes[ind]
sb = boxes[0]
# ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_relation(model_type, iteration):
cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test']
result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' +
iteration + '.hdf5')
m = h5py.File('data/sg_vrd_meta.h5')
#--------------cache boxes-----------------
h5_boxes = h5py.File('output/precalc/sg_vrd_objs.hdf5')
cache_boxes = {}
if os.path.exists('output/cache/sg_vrd_objs_test.pkl'):
cache_boxes = zl.load('output/cache/sg_vrd_objs_test.pkl')
glog.info('loaded obj data from cache')
else:
glog.info('Preloading obj')
zl.tic()
for k in h5_boxes['test'].keys():
boxes = h5_boxes['test/%s/boxes' % k][...]
cache_boxes[k] = boxes
glog.info('done preloading obj %f' % zl.toc())
zl.save('output/cache/sg_vrd_obj_test.pkl', cache_boxes)
#--------------cache boxes-----------------
#--------------cache old boxes-------------
h5_boxes = h5py.File('output/sg_vrd_2016_test.hdf5')
cache_old_boxes = {}
if os.path.exists('output/cache/sg_vrd_objs_test_vgg.pkl'):
cache_old_boxes = zl.load('output/cache/sg_vrd_objs_test_vgg.pkl')
glog.info('loaded obj data from cache')
else:
glog.info('Preloading obj')
zl.tic()
for k in h5_boxes.keys():
locations = h5_boxes['%s/locations' % k][...]
cls_confs = h5_boxes['%s/cls_confs' % k][...]
boxes = np.concatenate((locations, cls_confs[:, 1, np.newaxis],
cls_confs[:, 0, np.newaxis]),
axis=1)
cache_old_boxes[k] = boxes
glog.info('done preloading obj %f' % zl.toc())
zl.save('output/cache/sg_vrd_obj_test_vgg.pkl', cache_old_boxes)
#--------------cache old boxes-------------
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
keep = 100
thresh = 0.0001
prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt'
model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel'
net = caffe.Net(prototxt, model_path, caffe.TEST)
# sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data})
# exit(0)
#net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST)
cnt = 0
for imid in cache_h5.keys():
cnt += 1
if cnt % 10 == 0:
glog.info(cnt)
obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...]
sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...]
rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...]
rlp_labels = []
rlp_confs = []
sub_boxes = []
obj_boxes = []
#boxes = cache_boxes[imid]
boxes = cache_old_boxes[imid]
if boxes.shape[0] >= 2:
for s in xrange(boxes[:20].shape[0]):
for o in xrange(boxes[:20].shape[0]):
if s == o: continue
if boxes[s][4] < 0.01: continue
if boxes[o][4] < 0.01: continue
sbox = boxes[s][:4]
obox = boxes[o][:4]
rlp_labels.append([boxes[s, 5], -1, boxes[o, 5]])
rlp_confs.append(boxes[s, 4] + boxes[o, 4])
sub_boxes.append(sbox)
obj_boxes.append(obox)
if len(sub_boxes) <= 0: continue
#sub_box = np.array(sub_box)
#obj_box = np.array(obj_box)
im_path = C.get_sg_vrd_path_test(imid)
im_detect(net, im_path, sub_boxes, obj_boxes)
relation_prob = net.blobs['relation_prob'].data[...]
for r in xrange(relation_prob.shape[0]):
argmax = np.argmax(relation_prob[r, ...])
rs = relation_prob[r, argmax].squeeze()
rlp_labels[r][1] = argmax
rlp_confs[r] = rlp_confs[r] + rs
im = cv2.imread(data_root + imid + '.jpg')
for i in xrange(len(sub_boxes)):
sb = sub_boxes[i]
cv2.rectangle(im, (int(sb[0]), int(sb[1])),
(int(sb[2]), int(sb[3])), (255, 0, 0), 2)
pass
cv2.imshow('im', im)
#if cv2.waitKey(0)==27:
# exit(0)
#rlp_confs.append(rlp_conf)
#rlp_labels.append(rlp_label)
#sub_boxes.append(sub_box)
#obj_boxes.append(obj_box)
# for i in xrange(70):
# rs = relation_score[i]
# if rs>0.0:
# predicate =i
# #print relation_score[predicate]
# rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32)
# #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs)
# rlp_conf = rs#+sub_score+obj_score#relation_score[predicate]
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub_boxes_gt[s])
# obj_boxes.append(obj_boxes_gt[s])
result.create_dataset(imid + '/rlp_confs',
dtype='float16',
data=np.array(rlp_confs).astype(np.float16))
result.create_dataset(imid + '/sub_boxes',
dtype='float16',
data=np.array(sub_boxes).astype(np.float16))
result.create_dataset(imid + '/obj_boxes',
dtype='float16',
data=np.array(obj_boxes).astype(np.float16))
result.create_dataset(imid + '/rlp_labels',
dtype='float16',
data=np.array(rlp_labels).astype(np.float16))
def eval():
cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml')
#cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml')
imdb, roidb = combined_roidb('sg_vrd_2016_test')
import cv2
#h5f = h5py.File('/media/zawlin/ssd/iccv2017/data/voc/gen_eb.h5',driver='core')
h5path = 'data/sg_vrd_2016/EB/eb.h5'
h5f = h5py.File(h5path,driver='core')
h5_rois = {}
for i in h5f['test/']:
data=h5f['test/%s'%i][...].astype(np.float32)
idx = np.argsort(data[:,-1],axis=0)
data_sorted = data[idx][::-1]
data_sorted_idx = np.where((data_sorted[:,2]-data_sorted[:,0]>20) & (data_sorted[:,3]-data_sorted[:,1]>20))
data_sorted = data_sorted[data_sorted_idx]
#print data_sorted
h5_rois[i] = data_sorted[:1000,:4]
#cfg.TEST.HAS_RPN=False
net =None
#prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_eb_sigmoid.prototxt'
prototxt = 'models/sg_vrd/wsd/test_eb_wsddn_s.prototxt'
#model = 'data/rfcn_models/resnet50_rfcn_iter_1200.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_16000.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/eb_wsddn_s_iter_5000.caffemodel'
model = 'output/rfcn_end2end/sg_vrd_2016_train/eb_wsddn_s_iter_9400.caffemodel'
#model = 'data/rfcn_models/resnet50_rfcn_final.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_eb_sigx_iter_100000.caffemodel'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_600.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, model, caffe.TEST)
#prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_rpn.prototxt'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_1600.caffemodel'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel'
#net2 = caffe.Net(prototxt, model, caffe.TEST)
#net.params['conv_new_1_zl'][0].data[...] = net2.params['conv_new_1_zl'][0].data[...]
#net.params['conv_new_1_zl'][1].data[...] = net2.params['conv_new_1_zl'][1].data[...]
#net2 = None
net.name = 'resnet50_rfcn_iter_1200'
num_images = len(imdb.image_index)
num_images = 100
del imdb.image_index[num_images:]
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
output_dir = get_output_dir(imdb, net)
zl.tic()
# timers
_t = {'im_detect' : Timer(), 'misc' : Timer()}
max_per_image =200
thresh = 0.00001
cv2.namedWindow('im',0)
cnt = 0
for i in xrange(num_images):
# filter out any ground truth boxes
im_path = imdb.image_path_at(i)
im_name = im_path.split('/')[-1]
eb_roi = h5_rois[im_name]
im = cv2.imread(imdb.image_path_at(i))
_t['im_detect'].tic()
#scores, boxes = im_detect(net, im, box_proposals)
scores, boxes = im_detect_iccv(net, im, eb_roi)
#attention = net.blobs['attention'].data.squeeze()
#net.blobs['attention'].data
#scores = np.multiply(scores,attention)
_t['im_detect'].toc()
_t['misc'].tic()
# skip j = 0, because it's the background class
for j in xrange(1, imdb.num_classes):
if j == 15:
dfdfd=1
dfdfd += 1
inds = np.where(scores[:, j-1] > thresh)[0]
cls_scores = scores[inds, j-1]
if cfg.TEST.AGNOSTIC:
cls_boxes = boxes[inds, 1:]
else:
cls_boxes = boxes[inds, j*4:(j+1)*4]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
keep = nms(cls_dets, cfg.TEST.NMS,force_cpu=True)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
# Limit to max_per_image detections *over all classes*
if max_per_image > 0:
image_scores = np.hstack([all_boxes[j][i][:, -1]
for j in xrange(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in xrange(1, imdb.num_classes):
keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
all_boxes[j][i] = all_boxes[j][i][keep, :]
for j in xrange(1, imdb.num_classes):
cls_str = imdb.classes[j]
for roi in all_boxes[j][i]:
cv2.putText(im,cls_str,(roi[0],roi[1]),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,0),1)
cv2.rectangle(im,(roi[0],roi[1]),(roi[2],roi[3]),(0,0,255),1)
cnt += 1
cv2.imwrite('/home/zawlin/%d.jpg'%cnt,im)
cv2.imshow('vis',im)
cv2.waitKey(0)
_t['misc'].toc()
print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
.format(i + 1, num_images, _t['im_detect'].average_time,
_t['misc'].average_time)
det_file = os.path.join(output_dir, 'detections.pkl')
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)
print 'Evaluating detections'
imdb.evaluate_detections(all_boxes, output_dir)
print zl.toc()
def gen_recall():
cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml')
#cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml')
imdb, roidb = combined_roidb('sg_vrd_2016_test')
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
import cv2
h5path = 'data/sg_vrd_2016/EB/eb.h5'
h5f = h5py.File(h5path,driver='core')
h5_rois = {}
for i in h5f['test/']:
data=h5f['test/%s'%i][...].astype(np.float32)
idx = np.argsort(data[:,-1],axis=0)
data_sorted = data[idx][::-1]
data_sorted_idx = np.where((data_sorted[:,2]-data_sorted[:,0]>20) & (data_sorted[:,3]-data_sorted[:,1]>20))
data_sorted = data_sorted[data_sorted_idx]
#print data_sorted
h5_rois[i] = data_sorted[:4000,:4]
#cfg.TEST.HAS_RPN=False
net =None
#prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_eb_sigmoid.prototxt'
prototxt = 'models/sg_vrd/wsd/test_eb_wsddn_s.prototxt'
#model = 'data/rfcn_models/resnet50_rfcn_iter_1200.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_16000.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/eb_wsddn_s_iter_5000.caffemodel'
model = 'output/rfcn_end2end/sg_vrd_2016_train/eb_wsddn_s_iter_11000.caffemodel'
#model = 'data/rfcn_models/resnet50_rfcn_final.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_eb_sigx_iter_100000.caffemodel'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_600.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, model, caffe.TEST)
#prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_rpn.prototxt'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_1600.caffemodel'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel'
#net2 = caffe.Net(prototxt, model, caffe.TEST)
#net.params['conv_new_1_zl'][0].data[...] = net2.params['conv_new_1_zl'][0].data[...]
#net.params['conv_new_1_zl'][1].data[...] = net2.params['conv_new_1_zl'][1].data[...]
#net2 = None
net.name = 'resnet50_rfcn_iter_1200'
num_images = len(imdb.image_index)
#num_images = 100
#del imdb.image_index[num_images:]
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
output_dir = get_output_dir(imdb, net)
zl.tic()
# timers
_t = {'im_detect' : Timer(), 'misc' : Timer()}
max_per_image =20
thresh = 0.00001
cv2.namedWindow('im',0)
cnt = 0
mat_pred_label = []
mat_pred_conf = []
mat_pred_bb = []
mat_gt_label = []
mat_gt_bb = []
for i in xrange(num_images):
cnt+=1
# filter out any ground truth boxes
im_path = imdb.image_path_at(i)
im_name = im_path.split('/')[-1]
imid = im_name[:-4]
eb_roi = h5_rois[im_name]
im = cv2.imread(imdb.image_path_at(i))
_t['im_detect'].tic()
scores, boxes = im_detect_iccv(net, im, eb_roi)
_t['im_detect'].toc()
_t['misc'].tic()
boxes_tosort = []
for j in xrange(1, 101):
inds = np.where(scores[:, j-1] > 0.00001)[0]
cls_scores = scores[inds, j-1]
cls_boxes = boxes[inds, 1:]
#cls_boxes = boxes[inds, j * 4:(j + 1) * 4]
# cls_boxes = boxes[inds]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
keep = nms(cls_dets, .7, force_cpu=True) # nms threshold
# keep = nms_fast(cls_dets,.3)
cls_dets = cls_dets[keep, :]
boxes_tosort.append(cls_dets)
mat_pred_label_i = []
mat_pred_conf_i = []
mat_pred_bb_i = []
for j in xrange(len(boxes_tosort)):
cls_dets = boxes_tosort[j]
idx = np.argsort(cls_dets[:,-1],axis=0)[::-1]
cls_dets = cls_dets[idx]
if cls_dets.shape[0]>max_per_image:
cls_dets = cls_dets[:max_per_image,:]
for di in xrange(cls_dets.shape[0]):
# print 'here'
di = cls_dets[di]
score = di[-1]
cls_idx = j + 1
cls_name = zl.idx2name_cls(m,cls_idx)
#cls_name = str(m['meta/cls/idx2name/' + str(cls_idx)][...])
if score > 1:
score = 1
if score < thresh:
continue
cv2.rectangle(im,(di[0],di[1]),(di[2],di[3]),(255,0,0),2)
x, y = int(di[0]), int(di[1])
if x < 10:
x = 15
if y < 10:
y = 15
mat_pred_label_i.append(cls_idx)
mat_pred_conf_i.append(score)
mat_pred_bb_i.append([di[0],di[1],di[2],di[3]])
cv2.putText(im,cls_name,(x,y),cv2.FONT_HERSHEY_SIMPLEX,1.0,(0,0,255),2)
res_line = '%s %d %f %d %d %d %d'%(imid,cls_idx,score,di[0],di[1],di[2],di[3])
mat_pred_label.append(mat_pred_label_i)
mat_pred_conf.append(mat_pred_conf_i)
mat_pred_bb.append(mat_pred_bb_i)
obj_boxes = m['gt/test/%s/obj_boxes'%imid][...]
sub_boxes = m['gt/test/%s/sub_boxes'%imid][...]
rlp_labels = m['gt/test/%s/rlp_labels'%imid][...]
mat_gt_label_i = []
mat_gt_bb_i = []
mat_gt_i = []
for gti in xrange(obj_boxes.shape[0]):
mat_gt_i.append([rlp_labels[gti,0],sub_boxes[gti,0],sub_boxes[gti,1],sub_boxes[gti,2],sub_boxes[gti,3]])
mat_gt_i.append([rlp_labels[gti,2],obj_boxes[gti,0],obj_boxes[gti,1],obj_boxes[gti,2],obj_boxes[gti,3]])
if len(mat_gt_i)>0:
mat_gt_i = np.array(mat_gt_i)
mat_gt_i=zl.unique_arr(mat_gt_i)
for gti in xrange(mat_gt_i.shape[0]):
mat_gt_bb_i.append(mat_gt_i[gti,1:])
mat_gt_label_i.append(mat_gt_i[gti,0])
mat_gt_label.append(mat_gt_label_i)
mat_gt_bb.append(mat_gt_bb_i)
#matlab_gt.append(matlab_gt_i)
#now get gt
cv2.imshow('im',im)
if cv2.waitKey(0) == 27:
exit(0)
_t['misc'].toc()
print 'im_detect: {:d} {:.3f}s {:.3f}s' \
.format(cnt, _t['im_detect'].average_time,
_t['misc'].average_time)
sio.savemat('output/sg_vrd_objs.mat', {'pred_bb': mat_pred_bb,
'pred_conf':mat_pred_conf,
'pred_label':mat_pred_label,
'gt_bb':mat_gt_bb,
'gt_label':mat_gt_label
})
def run_retrieval_vp_v2():
# h5_path = 'output/sg_vrd_2016_result.classeme.hdf5'
h5_path = 'output/precalc/sg_vrd_2016_test_nms.7.hdf5'
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_vp_meta.h5', 'r',
'core')
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
data_root = '/home/zawlin/data/data_vrd/vrd/sg/Data/sg_test_images/'
rlp_map = []
for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())):
tri = str(m_vp['meta/tri/idx2name'][str(i)][...])
s_lbl = tri.split('_')[0]
p_lbl = tri.split('_')[1]
o_lbl = tri.split('_')[2]
rlp_label = [
zl.name2idx_cls(m, s_lbl),
zl.name2idx_pre(m, p_lbl),
zl.name2idx_cls(m, o_lbl)
]
rlp_map.append(rlp_label)
rlp_map = np.array(rlp_map)
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['labels'] = h5_in[i]['labels'][...]
h5f[i]['confs'] = h5_in[i]['confs'][...]
h5f[i]['boxes'] = h5_in[i]['boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
retr_meta = zl.load('output/pkl/vr_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
for k in xrange(len(retr_meta['rlp_labels'])):
if k > 1000:
break
cnt += 1
rlp_label = retr_meta['rlp_labels'][k]
gt_files = retr_meta['files'][k]
# print gt_files
# exit(0)
# for f in gt_files:
# impath= zl.imid2path(m,f)
# print impath
# im= cv2.imread(data_root+impath)
# cv2.imshow('im',im)
# cv2.waitKey(0)
results = {}
zl.tic()
ranks = []
for imid in imids:
labels = h5f[imid]['labels'] - 1
rlp_confs = h5f[imid]['confs']
rlp_labels = rlp_map[labels]
if rlp_labels.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
delay = 0
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
impath = zl.imid2path(m, imid)
if found and i >= 5:
break
# print gt_files
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < 5:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
if True:
cor_or_not = str(cor)
if cor: delay = 0
if delay == 0:
im = cv2.imread(data_root + impath)
cv2.putText(im, cor_or_not, (50, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
cv2.putText(im, lbl_str, (50, 80),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
cv2.putText(im, str(i), (50, 100),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
cv2.imshow('im', im)
c = cv2.waitKey(delay) & 0xFF
if c == 27:
exit(0)
if c == ord('s'):
delay = 1 - delay
if c == ord('c'):
delay = 1
r_50 = tp_50 / 5 #total_gt
r_100 = tp_100 / 5 #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.toc())
def run_retrieval_zeroshot():
# h5_path = 'output/sg_vrd_2016_result_all_19500.hdf5'
# h5_path = 'output/sg_vrd_2016_result_diff_all_5000.hdf5'
# h5_path= 'output/results/lu_method_results.hdf5'
# h5_path = 'output/sg_vrd_2016_result.hdf5.dd'
# h5_path = 'output/results/lu_method_results_max.hdf5'
h5_path = 'output/results/lu_visual_method_results.hdf5'
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
zeroshots = m['meta/zeroshots'][...]
gt_cache_path = 'output/cache/sg_vrd_gt_cache.pkl'
gt_h5f = {}
np.random.seed(76)
if os.path.exists(gt_cache_path):
print 'load gt from cache'
gt_h5f = zl.load(gt_cache_path)
else:
print 'cacheing gt'
for k in m['gt/test']:
gt_h5f[k] = {}
sub_boxes = m['gt/test'][k]['sub_boxes'][...]
rlp_labels = m['gt/test'][k]['rlp_labels'][...]
obj_boxes = m['gt/test'][k]['obj_boxes'][...]
gt_h5f[k]['sub_boxes'] = sub_boxes
gt_h5f[k]['obj_boxes'] = obj_boxes
gt_h5f[k]['rlp_labels'] = rlp_labels
print 'caching gt done'
zl.save(gt_cache_path, gt_h5f)
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
# h5f = h5py.File('output/sg_vrd_2016_result.classeme.hdf5')
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['rlp_labels'] = h5_in[i]['rlp_labels'][...]
h5f[i]['rlp_confs'] = h5_in[i]['rlp_confs'][...]
h5f[i]['sub_boxes'] = h5_in[i]['sub_boxes'][...]
h5f[i]['obj_boxes'] = h5_in[i]['obj_boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
#h5f = h5py.file('output/results/lu_method_results.hdf5')
retr_meta = zl.load('output/pkl/vr_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
for k in xrange(len(retr_meta['rlp_labels'])):
# if k>1000:
# break
rlp_label = retr_meta['rlp_labels'][k]
if not np.any(np.all(zeroshots == rlp_label, axis=1)): continue
gt_files = retr_meta['files'][k]
cnt += 1
results = {}
zl.tic()
for imid in imids:
rlp_labels = h5f[imid]['rlp_labels']
rlp_confs = h5f[imid]['rlp_confs']
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.random.uniform(-1, 1) #np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
delay = 0
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
r_at_k = 5
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
if found and i >= r_at_k:
break
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < r_at_k:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+imid+'.jpg')
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / r_at_k #total_gt
r_100 = tp_100 / r_at_k #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.toc())
def gen_obj_detection_results_from_hdf5_vr(h5_path):
m = h5py.File('data/sg_vrd_meta.h5', 'r', 'core')
h5f = h5py.File(h5_path, 'r')
thresh = 0.001
cnt = 0
zl.tic()
mat_pred_label = []
mat_pred_conf = []
mat_pred_bb = []
mat_gt_label = []
mat_gt_bb = []
imid2path = {}
impath2id = {}
imidx2id = {}
root = 'data/sg_vrd_2016/Data/sg_test_images/'
img_set_file = 'data/sg_vrd_2016/ImageSets/test.txt'
imlist = {
line.strip().split(' ')[1]: line.strip().split(' ')[0]
for line in open(img_set_file)
}
#for k in m['gt/test/'].keys():
for k in imlist.keys():
imid = str(m['db/testidx/%s' % k][...])
#imid = imlist[k]
if cnt % 1000 == 0:
print cnt, zl.toc()
zl.tic()
cnt += 1
scores = h5f['%s/scores' % k][...]
boxes = h5f['%s/boxes' % k][...]
im = cv2.imread(root + imid + '.jpg')
boxes_tosort = []
mat_pred_label_i = []
mat_pred_conf_i = []
mat_pred_bb_i = []
for j in xrange(1, 101):
inds = np.where(scores[:, j] > 0.001)[0]
cls_scores = scores[inds, j]
cls_boxes = boxes[inds, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
keep = nms(cls_dets, .2, force_cpu=True) # nms threshold
cls_dets = cls_dets[keep, :]
boxes_tosort.append(cls_dets)
for j in xrange(len(boxes_tosort)):
cls_dets = boxes_tosort[j]
for di in xrange(cls_dets.shape[0]):
# print 'here'
di = cls_dets[di]
score = di[-1]
cls_idx = j + 1
if score > 1:
score = 1
if score < thresh:
continue
mat_pred_label_i.append(cls_idx)
mat_pred_conf_i.append(score)
mat_pred_bb_i.append([di[0], di[1], di[2], di[3]])
#for i in xrange(5):
# bb = np.array(mat_pred_bb_i[i]).astype(np.int)
# cv2.rectangle(im,(bb[0],bb[1]),(bb[2],bb[3]),(255,0,0),2)
#cv2.imshow('im',im)
#if cv2.waitKey(0)==27:
# exit(0)
mat_pred_label.append(mat_pred_label_i)
mat_pred_conf.append(mat_pred_conf_i)
mat_pred_bb.append(mat_pred_bb_i)
obj_boxes = m['gt/test/%s/obj_boxes' % imid][...]
sub_boxes = m['gt/test/%s/sub_boxes' % imid][...]
rlp_labels = m['gt/test/%s/rlp_labels' % imid][...]
mat_gt_label_i = []
mat_gt_bb_i = []
mat_gt_i = []
for gti in xrange(obj_boxes.shape[0]):
mat_gt_i.append([
rlp_labels[gti, 0], sub_boxes[gti, 0], sub_boxes[gti, 1],
sub_boxes[gti, 2], sub_boxes[gti, 3]
])
mat_gt_i.append([
rlp_labels[gti, 2], obj_boxes[gti, 0], obj_boxes[gti, 1],
obj_boxes[gti, 2], obj_boxes[gti, 3]
])
if len(mat_gt_i) > 0:
mat_gt_i = np.array(mat_gt_i)
mat_gt_i = zl.unique_arr(mat_gt_i)
for gti in xrange(mat_gt_i.shape[0]):
mat_gt_bb_i.append(mat_gt_i[gti, 1:])
mat_gt_label_i.append(mat_gt_i[gti, 0])
mat_gt_label.append(mat_gt_label_i)
mat_gt_bb.append(mat_gt_bb_i)
#outfile.write(res_line+'\n')
sio.savemat(
'output/vg_vrd_objs.mat', {
'pred_bb': mat_pred_bb,
'pred_conf': mat_pred_conf,
'pred_label': mat_pred_label,
'gt_bb': mat_gt_bb,
'gt_label': mat_gt_label
})
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_1600.caffemodel'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel'
#net2 = caffe.Net(prototxt, model, caffe.TEST)
#net.params['conv_new_1_zl'][0].data[...] = net2.params['conv_new_1_zl'][0].data[...]
#net.params['conv_new_1_zl'][1].data[...] = net2.params['conv_new_1_zl'][1].data[...]
#net2 = None
net.name = 'resnet50_rfcn_iter_1200'
num_images = len(imdb.image_index)
#num_images = 100
#del imdb.image_index[num_images:]
#num_images = 10#len(imdb.image_index)
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
output_dir = get_output_dir(imdb, net)
zl.tic()
# timers
_t = {'im_detect' : Timer(), 'misc' : Timer()}
max_per_image = 50
thresh = 0.7
for i in xrange(num_images):
# filter out any ground truth boxes
im_path = imdb.image_path_at(i)
im_name = im_path.split('/')[-1]
eb_roi = h5_rois[im_name]
im = cv2.imread(imdb.image_path_at(i))
_t['im_detect'].tic()
#scores, boxes = im_detect(net, im, box_proposals)
scores, boxes = im_detect(net, im, eb_roi)
attention = net.blobs['attention'].data.squeeze()
def gen_recall():
cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml')
#cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml')
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
h5objpath= 'output/precalc/sg_vrd_obj_rerank_wsd.h5'
h5f_wsd = h5py.File(h5objpath,'a')
h5path = 'data/sg_vrd_2016/EB/eb.h5'
h5f = h5py.File(h5path,driver='core')
h5_rois = {}
for i in h5f['test/']:
data=h5f['test/%s'%i][...].astype(np.float32)
idx = np.argsort(data[:,-1],axis=0)
data_sorted = data[idx][::-1]
#data_sorted_idx = np.where((data_sorted[:,2]-data_sorted[:,0]>20) & (data_sorted[:,3]-data_sorted[:,1]>20))
#data_sorted = data_sorted[data_sorted_idx]
#print data_sorted
h5_rois[i[:-4]] = data_sorted[:4000,:4]
#cfg.TEST.HAS_RPN=False
net =None
#prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_eb_sigmoid.prototxt'
prototxt = 'models/sg_vrd/wsd/test_eb_wsddn_s.prototxt'
#model = 'data/rfcn_models/resnet50_rfcn_iter_1200.caffemodel'
root = 'data/sg_vrd_2016/Data/sg_test_images/'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_16000.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/eb_wsddn_s_iter_5000.caffemodel'
model = 'output/rfcn_end2end/sg_vrd_2016_train/eb_wsddn_s_iter_11000.caffemodel'
#model = 'data/rfcn_models/resnet50_rfcn_final.caffemodel'
#model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_eb_sigx_iter_100000.caffemodel'
#model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_600.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, model, caffe.TEST)
db_net = caffe.Net('data/models/dbox/test.prototxt','data/models/dbox/fast-dbox-slidwindow-multiscale.caffemodel',caffe.TEST)
net.name = 'resnet50_rfcn_iter_1200'
zl.tic()
# timers
_t = {'im_detect' : Timer(), 'misc' : Timer()}
max_per_image =5
thresh = 0.00001
cv2.namedWindow('im',0)
cnt = 0
for k in m['gt/test'].keys():
im_path = root+k+'.jpg'
im = cv2.imread(im_path)
imid = k
cnt+=1
# filter out any ground truth boxes
eb_roi = h5_rois[imid]
db_scores = dbox.im_obj_detect(db_net,im,eb_roi)
idx = np.argsort(db_scores[:,-1],axis=0)[::-1]
eb_roi = eb_roi[idx[:100]]
_t['im_detect'].tic()
scores, boxes = im_detect_iccv(net, im, eb_roi)
_t['im_detect'].toc()
_t['misc'].tic()
boxes_tosort = []
h5_tosave = []
for j in xrange(1, 101):
inds = np.where(scores[:, j-1] > 0.00001)[0]
cls_scores = scores[inds, j-1]
cls_boxes = boxes[inds, 1:]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
cls_idx = np.zeros((cls_dets.shape[0],1))+j
h5box = np.hstack((cls_boxes,cls_scores[:,np.newaxis],cls_idx))
if h5box.shape[0]>0:
h5_tosave.extend(h5box.tolist())
keep = nms(cls_dets, .2, force_cpu=True) # nms threshold
cls_dets = cls_dets[keep, :]
boxes_tosort.append(cls_dets)
h5_tosave = np.array(h5_tosave,dtype=np.float16)
if h5_tosave.shape[0]>max_per_image:
idx = np.argsort(h5_tosave[:,4],axis=0)[::-1]
h5_tosave = h5_tosave[idx[:max_per_image]]
for i in xrange(h5_tosave.shape[0]) :
di=h5_tosave[i]
x, y = int(di[0]), int(di[1])
if x < 10:
x = 15
if y < 10:
y = 15
score = di[-2]
if score > 1:
score = 1
if score < thresh:
continue
cls_idx = int(di[-1])
cls_name = zl.idx2name_cls(m,cls_idx)
cv2.rectangle(im,(di[0],di[1]),(di[2],di[3]),(255,0,0),2)
cv2.putText(im,cls_name,(x,y),cv2.FONT_HERSHEY_SIMPLEX,1.0,(0,0,255),2)
# 5 for vrd
# 10 for vg
#h5f_wsd.create_dataset('test/'+imid+'/boxes',dtype='float16', data=h5_tosave)
cv2.imshow('im',im)
if cv2.waitKey(0) == 27:
exit(0)
_t['misc'].toc()
print 'im_detect: {:d} {:.3f}s {:.3f}s' \
.format(cnt, _t['im_detect'].average_time,
_t['misc'].average_time)
