def save_train_resnet():
prototxt = 'models/sg_vrd/resnet50/test_cache.prototxt'
model = 'data/imagenet/ResNet-50-model.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(1)
net = caffe.Net(prototxt, model, caffe.TEST)
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r')
cache_hdf5 = h5py.File('output/precalc/sg_vrd_cache_resnet.h5', 'a')
cnt = 0
layer_name = 'res5c'
for k in m['gt/test'].keys():
cnt += 1
glog.info(cnt)
im_path = C.get_sg_vrd_path_test(k)
im_detect(net, im_path)
data = net.blobs[layer_name].data[...].astype(np.float16)
cache_hdf5.create_dataset('test/%s' % k + '/%s' % layer_name,
dtype='float16',
data=data)
def save_train_vgg():
prototxt = 'data/models/vgg/test.prototxt'
model = 'data/models/vgg/vgg16_faster_rcnn_finetune_iter_40000.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, model, caffe.TEST)
cfg.TEST.HAS_RPN = True
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r')
cache_hdf5 = h5py.File('output/precalc/sg_vrd_cache_vgg.h5', 'a')
cnt = 0
layer_name = 'conv5_3'
thresh = 0.2
for k in m['gt/test'].keys():
cnt += 1
glog.info(cnt)
im_path = C.get_sg_vrd_path_test(k)
im = cv2.imread(im_path)
scores, boxes = im_detect_orig(net, im)
#visualize(im,scores,boxes)
data = net.blobs[layer_name].data[...].astype(np.float16)
cache_hdf5.create_dataset('test/%s' % k + '/%s' % layer_name,
dtype='float16',
data=data)
def run():
prototxt = 'models/sg_vrd/rel_iccv/test_iccv_gt.prototxt'
obj_detector_model='data/models/vrd_rfcn/vrd_resnet50_rfcn_iter_70000.caffemodel'
relation_model='output/sg_vrd_rfcn/psroi_context_tri_sum_cached_iter_75500.caffemodel'
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, caffe.TEST)
net.copy_from(obj_detector_model)
net.copy_from(relation_model)
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r')
cnt = 0
root_img = '/home/zawlin/Dropbox/iccv17_hw/_results_from_zl/img/'
root_cls = '/home/zawlin/Dropbox/iccv17_hw/_results_from_zl/cls/'
import glog
for k in m['gt/test'].keys():
if not os.path.exists(root_img+k):
os.makedirs(root_img+k)
cnt += 1
glog.info(cnt)
if cnt >80:
break
rlp_labels = m['gt/test/%s/rlp_labels'%k][...]
sub_boxes = m['gt/test/%s/sub_boxes'%k][...].astype(np.float)
obj_boxes = m['gt/test/%s/obj_boxes'%k][...].astype(np.float)
if sub_boxes.shape[0]>0:
zeros = np.zeros((sub_boxes.shape[0],1), dtype=np.float)
# first index is always zero since we do one image by one image
sub_boxes = np.concatenate((zeros, sub_boxes),axis=1)
obj_boxes = np.concatenate((zeros, obj_boxes),axis=1)
im_path = C.get_sg_vrd_path_test(k)
im = cv2.imread(im_path)
for i in xrange(sub_boxes.shape[0]):
# sb = sub_boxes[i][1:].astype(np.int)
# ob = obj_boxes[i][1:].astype(np.int)
rlp = rlp_labels[i]
rel = zl.idx2name_cls(m,rlp[0])+' '+zl.idx2name_pre(m,rlp[1])+' '+zl.idx2name_cls(m,rlp[2])
x1,y1,x2,y2 = union(sub_boxes[i][1:],obj_boxes[i][1:])
cv2.rectangle(im,(x1,y1),(x2,y2),(255,0,0),2)
cv2.putText(im,rel,(x1,y1),cv2.FONT_HERSHEY_SIMPLEX,1.0,(0,0,255),2)
# cv2.rectangle(im,(sb[0],sb[1]),(sb[2],sb[3]),(255,0,0),2)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(255,0,0),2)
cv2.imshow('im',im)
cv2.imwrite(root_img+k+'/_.orig.jpg',im)
im_detect(net,im_path,sub_boxes,obj_boxes)
rfcn_sub_rel = net.blobs['rfcn_sub_rel'].data[0]
rfcn_obj_rel = net.blobs['rfcn_obj_rel'].data[0]
rfcn_union_rel = net.blobs['rfcn_union_rel'].data[0]
for pi in xrange(70):
index = pi
head, last = index * 9, (index + 1)*9
feat_sub = rfcn_sub_rel[head:last]
feat_obj= rfcn_obj_rel[head:last]
feat_union= rfcn_union_rel[head:last]
im_vis_sub = get_vis(feat_sub)
im_vis_obj = get_vis(feat_obj)
im_vis_union = get_vis(feat_union)
pre = zl.idx2name_pre(m,pi)
cv2.imwrite(root_img+k+'/%s_sub.jpg'%pre,im_vis_sub)
cv2.imwrite(root_img+k+'/%s_obj.jpg'%pre,im_vis_obj)
cv2.imwrite(root_img+k+'/%s_union.jpg'%pre,im_vis_union)
if not os.path.exists(root_cls+pre):
os.makedirs(root_cls+pre)
cv2.imwrite(root_cls+pre+'/%s_sub.jpg'%k,im_vis_sub)
cv2.imwrite(root_cls+pre+'/%s_obj.jpg'%k,im_vis_obj)
cv2.imwrite(root_cls+pre+'/%s_union.jpg'%k,im_vis_union)
#cv2.imshow(pre+'sub',im_vis_sub)
#cv2.imshow(pre+'obj',im_vis_obj)
#cv2.imshow(pre+'union',im_vis_union)
#if cv2.waitKey(0)==27:
# exit(0)
else:
#todo
#print nothing
pass
cv2.waitKey(0)
def run_relation(model_type, iteration):
cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test']
result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' +
iteration + '.hdf5')
m = h5py.File('data/sg_vrd_meta.h5')
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
keep = 100
thresh = 0.0001
prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt'
model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel'
net = caffe.Net(prototxt, model_path, caffe.TEST)
# sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data})
# exit(0)
#net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST)
cnt = 0
for imid in cache_h5.keys():
cnt += 1
if cnt % 10 == 0:
glog.info(cnt)
obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...]
sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...]
rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...]
rlp_labels = []
rlp_confs = []
sub_boxes = []
obj_boxes = []
for s in xrange(sub_boxes_gt.shape[0]):
im_path = C.get_sg_vrd_path_test(imid)
im_detect(net, im_path, sub_boxes_gt, obj_boxes_gt)
relation_prob = net.blobs['relation_prob'].data[...]
sub_cls = rlp_labels_gt[:, 0]
obj_cls = rlp_labels_gt[:, 2]
argmax = np.argmax(relation_prob[s, ...])
gt_pre = rlp_labels_gt[s, 1]
rs = relation_prob[s, argmax]
predicate = argmax
#rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32)
rlp_label = np.array([sub_cls[s], predicate,
obj_cls[s]]).astype(np.int32)
#print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs)
rlp_conf = rs #+sub_score+obj_score#relation_score[predicate]
rlp_confs.append(rlp_conf)
rlp_labels.append(rlp_label)
sub_boxes.append(sub_boxes_gt[s])
obj_boxes.append(obj_boxes_gt[s])
# for i in xrange(70):
# rs = relation_score[i]
# if rs>0.0:
# predicate =i
# #print relation_score[predicate]
# rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32)
# #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs)
# rlp_conf = rs#+sub_score+obj_score#relation_score[predicate]
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub_boxes_gt[s])
# obj_boxes.append(obj_boxes_gt[s])
result.create_dataset(imid + '/rlp_confs',
dtype='float16',
data=np.array(rlp_confs).astype(np.float16))
result.create_dataset(imid + '/sub_boxes',
dtype='float16',
data=np.array(sub_boxes).astype(np.float16))
result.create_dataset(imid + '/obj_boxes',
dtype='float16',
data=np.array(obj_boxes).astype(np.float16))
result.create_dataset(imid + '/rlp_labels',
dtype='float16',
data=np.array(rlp_labels).astype(np.float16))
def run_relation(model_type, iteration):
cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test']
result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' +
iteration + '.hdf5')
m = h5py.File('data/sg_vrd_meta.h5')
#--------------cache boxes-----------------
h5_boxes = h5py.File('output/precalc/sg_vrd_objs.hdf5')
cache_boxes = {}
if os.path.exists('output/cache/sg_vrd_objs_test.pkl'):
cache_boxes = zl.load('output/cache/sg_vrd_objs_test.pkl')
glog.info('loaded obj data from cache')
else:
glog.info('Preloading obj')
zl.tic()
for k in h5_boxes['test'].keys():
boxes = h5_boxes['test/%s/boxes' % k][...]
cache_boxes[k] = boxes
glog.info('done preloading obj %f' % zl.toc())
zl.save('output/cache/sg_vrd_obj_test.pkl', cache_boxes)
#--------------cache boxes-----------------
#--------------cache old boxes-------------
h5_boxes = h5py.File('output/sg_vrd_2016_test.hdf5')
cache_old_boxes = {}
if os.path.exists('output/cache/sg_vrd_objs_test_vgg.pkl'):
cache_old_boxes = zl.load('output/cache/sg_vrd_objs_test_vgg.pkl')
glog.info('loaded obj data from cache')
else:
glog.info('Preloading obj')
zl.tic()
for k in h5_boxes.keys():
locations = h5_boxes['%s/locations' % k][...]
cls_confs = h5_boxes['%s/cls_confs' % k][...]
boxes = np.concatenate((locations, cls_confs[:, 1, np.newaxis],
cls_confs[:, 0, np.newaxis]),
axis=1)
cache_old_boxes[k] = boxes
glog.info('done preloading obj %f' % zl.toc())
zl.save('output/cache/sg_vrd_obj_test_vgg.pkl', cache_old_boxes)
#--------------cache old boxes-------------
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
keep = 100
thresh = 0.0001
prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt'
model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel'
net = caffe.Net(prototxt, model_path, caffe.TEST)
# sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data})
# exit(0)
#net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST)
cnt = 0
for imid in cache_h5.keys():
cnt += 1
if cnt % 10 == 0:
glog.info(cnt)
obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...]
sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...]
rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...]
rlp_labels = []
rlp_confs = []
sub_boxes = []
obj_boxes = []
#boxes = cache_boxes[imid]
boxes = cache_old_boxes[imid]
if boxes.shape[0] >= 2:
for s in xrange(boxes[:20].shape[0]):
for o in xrange(boxes[:20].shape[0]):
if s == o: continue
if boxes[s][4] < 0.01: continue
if boxes[o][4] < 0.01: continue
sbox = boxes[s][:4]
obox = boxes[o][:4]
rlp_labels.append([boxes[s, 5], -1, boxes[o, 5]])
rlp_confs.append(boxes[s, 4] + boxes[o, 4])
sub_boxes.append(sbox)
obj_boxes.append(obox)
if len(sub_boxes) <= 0: continue
#sub_box = np.array(sub_box)
#obj_box = np.array(obj_box)
im_path = C.get_sg_vrd_path_test(imid)
im_detect(net, im_path, sub_boxes, obj_boxes)
relation_prob = net.blobs['relation_prob'].data[...]
for r in xrange(relation_prob.shape[0]):
argmax = np.argmax(relation_prob[r, ...])
rs = relation_prob[r, argmax].squeeze()
rlp_labels[r][1] = argmax
rlp_confs[r] = rlp_confs[r] + rs
im = cv2.imread(data_root + imid + '.jpg')
for i in xrange(len(sub_boxes)):
sb = sub_boxes[i]
cv2.rectangle(im, (int(sb[0]), int(sb[1])),
(int(sb[2]), int(sb[3])), (255, 0, 0), 2)
pass
cv2.imshow('im', im)
#if cv2.waitKey(0)==27:
# exit(0)
#rlp_confs.append(rlp_conf)
#rlp_labels.append(rlp_label)
#sub_boxes.append(sub_box)
#obj_boxes.append(obj_box)
# for i in xrange(70):
# rs = relation_score[i]
# if rs>0.0:
# predicate =i
# #print relation_score[predicate]
# rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32)
# #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs)
# rlp_conf = rs#+sub_score+obj_score#relation_score[predicate]
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub_boxes_gt[s])
# obj_boxes.append(obj_boxes_gt[s])
result.create_dataset(imid + '/rlp_confs',
dtype='float16',
data=np.array(rlp_confs).astype(np.float16))
result.create_dataset(imid + '/sub_boxes',
dtype='float16',
data=np.array(sub_boxes).astype(np.float16))
result.create_dataset(imid + '/obj_boxes',
dtype='float16',
data=np.array(obj_boxes).astype(np.float16))
result.create_dataset(imid + '/rlp_labels',
dtype='float16',
data=np.array(rlp_labels).astype(np.float16))