def save_train_resnet(): prototxt = 'models/sg_vrd/resnet50/test_cache.prototxt' model = 'data/imagenet/ResNet-50-model.caffemodel' caffe.set_mode_gpu() caffe.set_device(1) net = caffe.Net(prototxt, model, caffe.TEST) m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r') cache_hdf5 = h5py.File('output/precalc/sg_vrd_cache_resnet.h5', 'a') cnt = 0 layer_name = 'res5c' for k in m['gt/test'].keys(): cnt += 1 glog.info(cnt) im_path = C.get_sg_vrd_path_test(k) im_detect(net, im_path) data = net.blobs[layer_name].data[...].astype(np.float16) cache_hdf5.create_dataset('test/%s' % k + '/%s' % layer_name, dtype='float16', data=data)
def save_train_vgg(): prototxt = 'data/models/vgg/test.prototxt' model = 'data/models/vgg/vgg16_faster_rcnn_finetune_iter_40000.caffemodel' caffe.set_mode_gpu() caffe.set_device(0) net = caffe.Net(prototxt, model, caffe.TEST) cfg.TEST.HAS_RPN = True m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r') cache_hdf5 = h5py.File('output/precalc/sg_vrd_cache_vgg.h5', 'a') cnt = 0 layer_name = 'conv5_3' thresh = 0.2 for k in m['gt/test'].keys(): cnt += 1 glog.info(cnt) im_path = C.get_sg_vrd_path_test(k) im = cv2.imread(im_path) scores, boxes = im_detect_orig(net, im) #visualize(im,scores,boxes) data = net.blobs[layer_name].data[...].astype(np.float16) cache_hdf5.create_dataset('test/%s' % k + '/%s' % layer_name, dtype='float16', data=data)
def run(): prototxt = 'models/sg_vrd/rel_iccv/test_iccv_gt.prototxt' obj_detector_model='data/models/vrd_rfcn/vrd_resnet50_rfcn_iter_70000.caffemodel' relation_model='output/sg_vrd_rfcn/psroi_context_tri_sum_cached_iter_75500.caffemodel' caffe.set_mode_gpu() caffe.set_device(0) net = caffe.Net(prototxt, caffe.TEST) net.copy_from(obj_detector_model) net.copy_from(relation_model) m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r') cnt = 0 root_img = '/home/zawlin/Dropbox/iccv17_hw/_results_from_zl/img/' root_cls = '/home/zawlin/Dropbox/iccv17_hw/_results_from_zl/cls/' import glog for k in m['gt/test'].keys(): if not os.path.exists(root_img+k): os.makedirs(root_img+k) cnt += 1 glog.info(cnt) if cnt >80: break rlp_labels = m['gt/test/%s/rlp_labels'%k][...] sub_boxes = m['gt/test/%s/sub_boxes'%k][...].astype(np.float) obj_boxes = m['gt/test/%s/obj_boxes'%k][...].astype(np.float) if sub_boxes.shape[0]>0: zeros = np.zeros((sub_boxes.shape[0],1), dtype=np.float) # first index is always zero since we do one image by one image sub_boxes = np.concatenate((zeros, sub_boxes),axis=1) obj_boxes = np.concatenate((zeros, obj_boxes),axis=1) im_path = C.get_sg_vrd_path_test(k) im = cv2.imread(im_path) for i in xrange(sub_boxes.shape[0]): # sb = sub_boxes[i][1:].astype(np.int) # ob = obj_boxes[i][1:].astype(np.int) rlp = rlp_labels[i] rel = zl.idx2name_cls(m,rlp[0])+' '+zl.idx2name_pre(m,rlp[1])+' '+zl.idx2name_cls(m,rlp[2]) x1,y1,x2,y2 = union(sub_boxes[i][1:],obj_boxes[i][1:]) cv2.rectangle(im,(x1,y1),(x2,y2),(255,0,0),2) cv2.putText(im,rel,(x1,y1),cv2.FONT_HERSHEY_SIMPLEX,1.0,(0,0,255),2) # cv2.rectangle(im,(sb[0],sb[1]),(sb[2],sb[3]),(255,0,0),2) # cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(255,0,0),2) cv2.imshow('im',im) cv2.imwrite(root_img+k+'/_.orig.jpg',im) im_detect(net,im_path,sub_boxes,obj_boxes) rfcn_sub_rel = net.blobs['rfcn_sub_rel'].data[0] rfcn_obj_rel = net.blobs['rfcn_obj_rel'].data[0] rfcn_union_rel = net.blobs['rfcn_union_rel'].data[0] for pi in xrange(70): index = pi head, last = index * 9, (index + 1)*9 feat_sub = rfcn_sub_rel[head:last] feat_obj= rfcn_obj_rel[head:last] feat_union= rfcn_union_rel[head:last] im_vis_sub = get_vis(feat_sub) im_vis_obj = get_vis(feat_obj) im_vis_union = get_vis(feat_union) pre = zl.idx2name_pre(m,pi) cv2.imwrite(root_img+k+'/%s_sub.jpg'%pre,im_vis_sub) cv2.imwrite(root_img+k+'/%s_obj.jpg'%pre,im_vis_obj) cv2.imwrite(root_img+k+'/%s_union.jpg'%pre,im_vis_union) if not os.path.exists(root_cls+pre): os.makedirs(root_cls+pre) cv2.imwrite(root_cls+pre+'/%s_sub.jpg'%k,im_vis_sub) cv2.imwrite(root_cls+pre+'/%s_obj.jpg'%k,im_vis_obj) cv2.imwrite(root_cls+pre+'/%s_union.jpg'%k,im_vis_union) #cv2.imshow(pre+'sub',im_vis_sub) #cv2.imshow(pre+'obj',im_vis_obj) #cv2.imshow(pre+'union',im_vis_union) #if cv2.waitKey(0)==27: # exit(0) else: #todo #print nothing pass cv2.waitKey(0)
def run_relation(model_type, iteration): cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test'] result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' + iteration + '.hdf5') m = h5py.File('data/sg_vrd_meta.h5') data_root = 'data/sg_vrd_2016/Data/sg_test_images/' keep = 100 thresh = 0.0001 prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt' model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel' net = caffe.Net(prototxt, model_path, caffe.TEST) # sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data}) # exit(0) #net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST) cnt = 0 for imid in cache_h5.keys(): cnt += 1 if cnt % 10 == 0: glog.info(cnt) obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...] sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...] rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...] rlp_labels = [] rlp_confs = [] sub_boxes = [] obj_boxes = [] for s in xrange(sub_boxes_gt.shape[0]): im_path = C.get_sg_vrd_path_test(imid) im_detect(net, im_path, sub_boxes_gt, obj_boxes_gt) relation_prob = net.blobs['relation_prob'].data[...] sub_cls = rlp_labels_gt[:, 0] obj_cls = rlp_labels_gt[:, 2] argmax = np.argmax(relation_prob[s, ...]) gt_pre = rlp_labels_gt[s, 1] rs = relation_prob[s, argmax] predicate = argmax #rlp_label = np.array([sub_cls,predicate,obj_cls]).astype(np.int32) rlp_label = np.array([sub_cls[s], predicate, obj_cls[s]]).astype(np.int32) #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs) rlp_conf = rs #+sub_score+obj_score#relation_score[predicate] rlp_confs.append(rlp_conf) rlp_labels.append(rlp_label) sub_boxes.append(sub_boxes_gt[s]) obj_boxes.append(obj_boxes_gt[s]) # for i in xrange(70): # rs = relation_score[i] # if rs>0.0: # predicate =i # #print relation_score[predicate] # rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32) # #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs) # rlp_conf = rs#+sub_score+obj_score#relation_score[predicate] # rlp_confs.append(rlp_conf) # rlp_labels.append(rlp_label) # sub_boxes.append(sub_boxes_gt[s]) # obj_boxes.append(obj_boxes_gt[s]) result.create_dataset(imid + '/rlp_confs', dtype='float16', data=np.array(rlp_confs).astype(np.float16)) result.create_dataset(imid + '/sub_boxes', dtype='float16', data=np.array(sub_boxes).astype(np.float16)) result.create_dataset(imid + '/obj_boxes', dtype='float16', data=np.array(obj_boxes).astype(np.float16)) result.create_dataset(imid + '/rlp_labels', dtype='float16', data=np.array(rlp_labels).astype(np.float16))
def run_relation(model_type, iteration): cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test'] result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' + iteration + '.hdf5') m = h5py.File('data/sg_vrd_meta.h5') #--------------cache boxes----------------- h5_boxes = h5py.File('output/precalc/sg_vrd_objs.hdf5') cache_boxes = {} if os.path.exists('output/cache/sg_vrd_objs_test.pkl'): cache_boxes = zl.load('output/cache/sg_vrd_objs_test.pkl') glog.info('loaded obj data from cache') else: glog.info('Preloading obj') zl.tic() for k in h5_boxes['test'].keys(): boxes = h5_boxes['test/%s/boxes' % k][...] cache_boxes[k] = boxes glog.info('done preloading obj %f' % zl.toc()) zl.save('output/cache/sg_vrd_obj_test.pkl', cache_boxes) #--------------cache boxes----------------- #--------------cache old boxes------------- h5_boxes = h5py.File('output/sg_vrd_2016_test.hdf5') cache_old_boxes = {} if os.path.exists('output/cache/sg_vrd_objs_test_vgg.pkl'): cache_old_boxes = zl.load('output/cache/sg_vrd_objs_test_vgg.pkl') glog.info('loaded obj data from cache') else: glog.info('Preloading obj') zl.tic() for k in h5_boxes.keys(): locations = h5_boxes['%s/locations' % k][...] cls_confs = h5_boxes['%s/cls_confs' % k][...] boxes = np.concatenate((locations, cls_confs[:, 1, np.newaxis], cls_confs[:, 0, np.newaxis]), axis=1) cache_old_boxes[k] = boxes glog.info('done preloading obj %f' % zl.toc()) zl.save('output/cache/sg_vrd_obj_test_vgg.pkl', cache_old_boxes) #--------------cache old boxes------------- data_root = 'data/sg_vrd_2016/Data/sg_test_images/' keep = 100 thresh = 0.0001 prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt' model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel' net = caffe.Net(prototxt, model_path, caffe.TEST) # sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data}) # exit(0) #net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST) cnt = 0 for imid in cache_h5.keys(): cnt += 1 if cnt % 10 == 0: glog.info(cnt) obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...] sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...] rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...] rlp_labels = [] rlp_confs = [] sub_boxes = [] obj_boxes = [] #boxes = cache_boxes[imid] boxes = cache_old_boxes[imid] if boxes.shape[0] >= 2: for s in xrange(boxes[:20].shape[0]): for o in xrange(boxes[:20].shape[0]): if s == o: continue if boxes[s][4] < 0.01: continue if boxes[o][4] < 0.01: continue sbox = boxes[s][:4] obox = boxes[o][:4] rlp_labels.append([boxes[s, 5], -1, boxes[o, 5]]) rlp_confs.append(boxes[s, 4] + boxes[o, 4]) sub_boxes.append(sbox) obj_boxes.append(obox) if len(sub_boxes) <= 0: continue #sub_box = np.array(sub_box) #obj_box = np.array(obj_box) im_path = C.get_sg_vrd_path_test(imid) im_detect(net, im_path, sub_boxes, obj_boxes) relation_prob = net.blobs['relation_prob'].data[...] for r in xrange(relation_prob.shape[0]): argmax = np.argmax(relation_prob[r, ...]) rs = relation_prob[r, argmax].squeeze() rlp_labels[r][1] = argmax rlp_confs[r] = rlp_confs[r] + rs im = cv2.imread(data_root + imid + '.jpg') for i in xrange(len(sub_boxes)): sb = sub_boxes[i] cv2.rectangle(im, (int(sb[0]), int(sb[1])), (int(sb[2]), int(sb[3])), (255, 0, 0), 2) pass cv2.imshow('im', im) #if cv2.waitKey(0)==27: # exit(0) #rlp_confs.append(rlp_conf) #rlp_labels.append(rlp_label) #sub_boxes.append(sub_box) #obj_boxes.append(obj_box) # for i in xrange(70): # rs = relation_score[i] # if rs>0.0: # predicate =i # #print relation_score[predicate] # rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32) # #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs) # rlp_conf = rs#+sub_score+obj_score#relation_score[predicate] # rlp_confs.append(rlp_conf) # rlp_labels.append(rlp_label) # sub_boxes.append(sub_boxes_gt[s]) # obj_boxes.append(obj_boxes_gt[s]) result.create_dataset(imid + '/rlp_confs', dtype='float16', data=np.array(rlp_confs).astype(np.float16)) result.create_dataset(imid + '/sub_boxes', dtype='float16', data=np.array(sub_boxes).astype(np.float16)) result.create_dataset(imid + '/obj_boxes', dtype='float16', data=np.array(obj_boxes).astype(np.float16)) result.create_dataset(imid + '/rlp_labels', dtype='float16', data=np.array(rlp_labels).astype(np.float16))