def test_matlab_engine(): import utils.zl_edge_box as zl_eb zl.tic() windows = zl_eb.get_windows('/home/zawlin/data/indoor.jpg') print zl.toc() windows = np.array(windows) im = cv2.imread('/home/zawlin/data/indoor.jpg') scores = [.1, .2, .3, .4, .5, .6, .7, .8, .9, .95, .99] ims = [] for s in scores: ims.append(im.copy()) for i in xrange(len(scores)): s = scores[i] im_render = ims[i] for i in xrange(windows.shape[0]): roi = windows[i].astype(np.int32) if windows[i][4] > s: cv2.rectangle(im_render, (roi[0], roi[1]), (roi[2], roi[3]), (255, 0, 0), 1) cv2.imwrite('/home/zawlin/data/iccv/iccv_thresh_%f.jpg' % s, im_render) for i in xrange(windows.shape[0]): b = windows[i].astype(np.int32) cv2.rectangle(im, (b[0], b[1]), (b[2], b[3]), (255, 0, 0), 2) cv2.imshow('drawn', im) cv2.waitKey(0)
def test_coco_api(): coco = COCO('/media/zawlin/ssd/coco/annotations/instances_train2014.json') cats = coco.loadCats(coco.getCatIds()) print dir(cats) nms = [cat['name'] for cat in cats] # catIds = coco.getCatIds(catNms=['person','dog','skateboard']) # imgIds = coco.getImgIds(catIds=catIds ) imgs = coco.loadImgs() for img in imgs: im_path = C.coco_dir + 'train2014/' + img['file_name'] im = cv2.imread(im_path) annIds = coco.getAnnIds(imgIds=img['id'], ) anns = coco.loadAnns(annIds) for ann in anns: bbox = np.array(ann['bbox']).astype(np.int32) zl.tic() cats = coco.loadCats(ann['category_id'])[0]['name'] print zl.toc() bbox[2], bbox[3] = bbox[0] + bbox[2], bbox[1] + bbox[3] cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (255, 0, 0), 2) cv2.putText(im, str(cats), (bbox[0], bbox[1]), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 255), 2) cv2.imshow('im', im) if cv2.waitKey(0) == 27: exit(0)
def setup(self, bottom, top): """Setup the RoIDataLayer.""" self._cur_idx = 0 self.gt_labels = {} self.meta = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r') self.cache = h5py.File('output/sg_vrd_cache.h5','r') self.cache_mem = {} if os.path.exists('output/cache/sg_vrd_gt.pkl'): self.gt_labels = zl.load('output/cache/sg_vrd_gt.pkl') glog.info('loaded gt data from cache') else: glog.info( 'Preloading gt') zl.tic() for k in self.meta['gt/train'].keys(): rlp_labels = self.meta['gt/train/%s/rlp_labels'%k][...] sub_boxes = self.meta['gt/train/%s/sub_boxes'%k][...].astype(np.float) obj_boxes = self.meta['gt/train/%s/obj_boxes'%k][...].astype(np.float) if sub_boxes.shape[0]>0: zeros = np.zeros((sub_boxes.shape[0],1), dtype=np.float) # first index is always zero since we do one image by one image sub_boxes = np.concatenate((zeros, sub_boxes),axis=1) obj_boxes = np.concatenate((zeros, obj_boxes),axis=1) self.gt_labels[k] = {} self.gt_labels[k]['rlp_labels']=rlp_labels self.gt_labels[k]['sub_boxes']=sub_boxes self.gt_labels[k]['obj_boxes']=obj_boxes glog.info('done preloading gt %f'%zl.toc()) zl.save('output/cache/sg_vrd_gt.pkl',self.gt_labels) self.imids = [] for k in self.gt_labels.keys(): self.imids.append(k) self.imidx =0 random.shuffle(self.imids) # parse the layer parameter string, which must be valid YAML layer_params = yaml.load(self.param_str) self._num_classes = layer_params['num_classes'] self._name_to_top_map = {} # data blob: holds a batch of N images, each with 3 channels idx = 0 top[idx].reshape(cfg.TRAIN.IMS_PER_BATCH, 1024, 50,50) self._name_to_top_map['conv_new_1'] = idx idx += 1 top[idx].reshape(1, 5, 1, 1) self._name_to_top_map['sub_boxes'] = idx idx += 1 top[idx].reshape(1, 5, 1, 1) self._name_to_top_map['obj_boxes'] = idx idx += 1 # labels blob: R categorical labels in [0, ..., K] for K foreground # classes plus background top[idx].reshape(1, 1, 1, 1) self._name_to_top_map['labels'] = idx
def test_edge_boxes_cpp_wrapper(): from edge_boxes_python import edge_boxes_python eb = edge_boxes_python('edges/cpp/external/gop_1.3/data/sf.dat') im = cv2.imread('/home/zawlin/data/indoor.jpg') #im = cv2.resize(im,(500,640)) #im = zl.url_to_image('http://images1.mtv.com/uri/mgid:file:docroot:cmt.com:/sitewide/assets/img/artists/wesley_james/photo_gallery/didnt_i/8-Indoor-Scene-x600.jpg') print 'detecting' zl.tic() bbs = eb.get_edge_boxes(im) bbs = bbs[bbs[:, 4].argsort()[::-1]] for i in xrange(10): b = bbs[i] cv2.rectangle(im, (b[0], b[1]), (b[2], b[3]), (255, 0, 0), 2) cv2.imshow('drawn', im) cv2.waitKey(0) print zl.toc() print bbs
def load_test(): h5path = '/media/zawlin/ssd/iccv2017/data/voc/gen_eb.h5' h5f = h5py.File(h5path, driver='core') data = {} cnt = 0 zl.tic() for i in h5f['train/']: cnt += 1 if cnt % 100 == 0: print zl.toc(), cnt zl.tic() data = h5f['train/%s' % i][...].astype(np.float32) print data[:, -2] idx = np.argsort(data[:, -2], axis=0) print data[idx][::-1] exit(0)
cls_str = imdb.classes[j] for roi in all_boxes[j][i]: cv2.putText(im,cls_str,(roi[0],roi[1]),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,0),1) cv2.rectangle(im,(roi[0],roi[1]),(roi[2],roi[3]),(0,0,255),1) # Limit to max_per_image detections *over all classes* if max_per_image > 0: image_scores = np.hstack([all_boxes[j][i][:, -1] for j in xrange(1, imdb.num_classes)]) if len(image_scores) > max_per_image: image_thresh = np.sort(image_scores)[-max_per_image] for j in xrange(1, imdb.num_classes): keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0] all_boxes[j][i] = all_boxes[j][i][keep, :] #cv2.imshow('vis',im) #cv2.waitKey(0) _t['misc'].toc() print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \ .format(i + 1, num_images, _t['im_detect'].average_time, _t['misc'].average_time) det_file = os.path.join(output_dir, 'detections.pkl') with open(det_file, 'wb') as f: cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL) print 'Evaluating detections' imdb.evaluate_detections(all_boxes, output_dir) print zl.toc()
def setup(self, bottom, top): self._cur_idx = 0 self.rdata = sio.loadmat('data/meta/vrd/annotation_train.mat', struct_as_record=False, squeeze_me=True) self.meta = h5py.File('data/sg_vrd_meta.h5', 'r') self.gt_labels = {} if os.path.exists('output/cache/sg_vrd_gt.pkl'): self.gt_labels = zl.load('output/cache/sg_vrd_gt.pkl') glog.info('loaded gt data from cache') else: glog.info('Preloading gt') zl.tic() for k in self.meta['gt/train'].keys(): rlp_labels = self.meta['gt/train/%s/rlp_labels' % k][...] sub_boxes = self.meta['gt/train/%s/sub_boxes' % k][...].astype( np.float) obj_boxes = self.meta['gt/train/%s/obj_boxes' % k][...].astype( np.float) if sub_boxes.shape[0] > 0: zeros = np.zeros((sub_boxes.shape[0], 1), dtype=np.float) # first index is always zero since we do one image by one image sub_boxes = np.concatenate((zeros, sub_boxes), axis=1) obj_boxes = np.concatenate((zeros, obj_boxes), axis=1) self.gt_labels[k] = {} self.gt_labels[k]['rlp_labels'] = rlp_labels self.gt_labels[k]['sub_boxes'] = sub_boxes self.gt_labels[k]['obj_boxes'] = obj_boxes glog.info('done preloading gt %f' % zl.toc()) vgg_h5 = h5py.File("output/sg_vrd_2016_train_predicate_exp_train.hdf5", 'r', 'core') if os.path.exists('output/cache/sg_vrd_2016_train.pkl'): self.vgg_data = zl.load('output/cache/sg_vrd_2016_train.pkl') glog.info('loaded train data from cache') else: glog.info('Preloading training data') zl.tic() self.vgg_data = {} for k in vgg_h5.keys(): sub_visual = vgg_h5[k]['sub_classeme'][...] obj_visual = vgg_h5[k]['obj_classeme'][...] pre_label = vgg_h5[k]['pre_label'][...] self.vgg_data[k] = {} self.vgg_data[k]['sub_visual'] = sub_visual self.vgg_data[k]['obj_visual'] = obj_visual self.vgg_data[k]['pre_label'] = pre_label glog.info('done preloading training data %f' % zl.toc()) zl.save('output/cache/sg_vrd_2016_train.pkl', self.vgg_data) vgg_h5.close() self.meta = h5py.File('data/sg_vrd_meta.h5', 'r', 'core') layer_params = yaml.load(self.param_str) self._batch_size = layer_params['batch_size'] self.train_data = [] self._name_to_top_map = {} # data blob: holds a batch of N images, each with 3 channels # top[0].reshape(self._batch_size, 4096 * 2 ) top[0].reshape(self._batch_size, 2 * 101) top[1].reshape(self._batch_size) # self._name_to_top_map['visual'] = 0 # self._name_to_top_map['classeme'] = 0 self._name_to_top_map['visual'] = 0 # self._name_to_top_map['location'] = 1 self._name_to_top_map['label'] = 1
def gen_obj_detection_results_from_hdf5_vgg(h5_path): m = h5py.File('data/vg1_2_meta.h5', 'r', 'core') h5f = h5py.File(h5_path, 'r') thresh = 0.01 cnt = 0 zl.tic() mat_pred_label = [] mat_pred_conf = [] mat_pred_bb = [] mat_gt_label = [] mat_gt_bb = [] imid2path = {} impath2id = {} imidx2id = {} for i in m['gt/test/'].keys(): imid2path[str(i)] = str(m['meta/imid2path/%s' % i][...])[:-4] for i in imid2path.keys(): path = imid2path[i] impath2id[str(path)] = i img_set_file = 'data/vg1_2_2016/ImageSets/test.txt' imlist = { line.strip().split(' ')[1]: line.strip().split(' ')[0] for line in open(img_set_file) } for imid in imlist.keys(): path = imlist[imid] if path in impath2id: imidx2id[imid] = impath2id[path] #for k in m['gt/test/'].keys(): for k in imlist.keys(): if k not in imidx2id: continue imid = imidx2id[k] if cnt % 1000 == 0: print cnt, zl.toc() zl.tic() cnt += 1 scores = h5f['%s/scores' % k][...] boxes = h5f['%s/boxes' % k][...] boxes_tosort = [] mat_pred_label_i = [] mat_pred_conf_i = [] mat_pred_bb_i = [] for j in xrange(1, 101): inds = np.where(scores[:, j] > 0.001)[0] cls_scores = scores[inds, j] cls_boxes = boxes[inds, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \ .astype(np.float32, copy=False) keep = nms(cls_dets, .2, force_cpu=True) # nms threshold cls_dets = cls_dets[keep, :] boxes_tosort.append(cls_dets) for j in xrange(len(boxes_tosort)): cls_dets = boxes_tosort[j] for di in xrange(cls_dets.shape[0]): # print 'here' di = cls_dets[di] score = di[-1] cls_idx = j + 1 if score > 1: score = 1 if score < thresh: continue res_line = '%s %d %f %d %d %d %d' % (k, cls_idx, score, di[0], di[1], di[2], di[3]) mat_pred_label_i.append(cls_idx) mat_pred_conf_i.append(score) mat_pred_bb_i.append([di[0], di[1], di[2], di[3]]) mat_pred_label.append(mat_pred_label_i) mat_pred_conf.append(mat_pred_conf_i) mat_pred_bb.append(mat_pred_bb_i) obj_boxes = m['gt/test/%s/obj_boxes' % imid][...] sub_boxes = m['gt/test/%s/sub_boxes' % imid][...] rlp_labels = m['gt/test/%s/rlp_labels' % imid][...] mat_gt_label_i = [] mat_gt_bb_i = [] mat_gt_i = [] for gti in xrange(obj_boxes.shape[0]): mat_gt_i.append([ rlp_labels[gti, 0], sub_boxes[gti, 0], sub_boxes[gti, 1], sub_boxes[gti, 2], sub_boxes[gti, 3] ]) mat_gt_i.append([ rlp_labels[gti, 2], obj_boxes[gti, 0], obj_boxes[gti, 1], obj_boxes[gti, 2], obj_boxes[gti, 3] ]) if len(mat_gt_i) > 0: mat_gt_i = np.array(mat_gt_i) mat_gt_i = zl.unique_arr(mat_gt_i) for gti in xrange(mat_gt_i.shape[0]): mat_gt_bb_i.append(mat_gt_i[gti, 1:]) mat_gt_label_i.append(mat_gt_i[gti, 0]) mat_gt_label.append(mat_gt_label_i) mat_gt_bb.append(mat_gt_bb_i) #outfile.write(res_line+'\n') sio.savemat( 'output/vg_vrd_objs.mat', { 'pred_bb': mat_pred_bb, 'pred_conf': mat_pred_conf, 'pred_label': mat_pred_label, 'gt_bb': mat_gt_bb, 'gt_label': mat_gt_label })
def gen_obj_detection_results_from_hdf5_vr(h5_path): m = h5py.File('data/sg_vrd_meta.h5', 'r', 'core') h5f = h5py.File(h5_path, 'r') thresh = 0.001 cnt = 0 zl.tic() mat_pred_label = [] mat_pred_conf = [] mat_pred_bb = [] mat_gt_label = [] mat_gt_bb = [] imid2path = {} impath2id = {} imidx2id = {} root = 'data/sg_vrd_2016/Data/sg_test_images/' img_set_file = 'data/sg_vrd_2016/ImageSets/test.txt' imlist = { line.strip().split(' ')[1]: line.strip().split(' ')[0] for line in open(img_set_file) } #for k in m['gt/test/'].keys(): for k in imlist.keys(): imid = str(m['db/testidx/%s' % k][...]) #imid = imlist[k] if cnt % 1000 == 0: print cnt, zl.toc() zl.tic() cnt += 1 scores = h5f['%s/scores' % k][...] boxes = h5f['%s/boxes' % k][...] im = cv2.imread(root + imid + '.jpg') boxes_tosort = [] mat_pred_label_i = [] mat_pred_conf_i = [] mat_pred_bb_i = [] for j in xrange(1, 101): inds = np.where(scores[:, j] > 0.001)[0] cls_scores = scores[inds, j] cls_boxes = boxes[inds, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \ .astype(np.float32, copy=False) keep = nms(cls_dets, .2, force_cpu=True) # nms threshold cls_dets = cls_dets[keep, :] boxes_tosort.append(cls_dets) for j in xrange(len(boxes_tosort)): cls_dets = boxes_tosort[j] for di in xrange(cls_dets.shape[0]): # print 'here' di = cls_dets[di] score = di[-1] cls_idx = j + 1 if score > 1: score = 1 if score < thresh: continue mat_pred_label_i.append(cls_idx) mat_pred_conf_i.append(score) mat_pred_bb_i.append([di[0], di[1], di[2], di[3]]) #for i in xrange(5): # bb = np.array(mat_pred_bb_i[i]).astype(np.int) # cv2.rectangle(im,(bb[0],bb[1]),(bb[2],bb[3]),(255,0,0),2) #cv2.imshow('im',im) #if cv2.waitKey(0)==27: # exit(0) mat_pred_label.append(mat_pred_label_i) mat_pred_conf.append(mat_pred_conf_i) mat_pred_bb.append(mat_pred_bb_i) obj_boxes = m['gt/test/%s/obj_boxes' % imid][...] sub_boxes = m['gt/test/%s/sub_boxes' % imid][...] rlp_labels = m['gt/test/%s/rlp_labels' % imid][...] mat_gt_label_i = [] mat_gt_bb_i = [] mat_gt_i = [] for gti in xrange(obj_boxes.shape[0]): mat_gt_i.append([ rlp_labels[gti, 0], sub_boxes[gti, 0], sub_boxes[gti, 1], sub_boxes[gti, 2], sub_boxes[gti, 3] ]) mat_gt_i.append([ rlp_labels[gti, 2], obj_boxes[gti, 0], obj_boxes[gti, 1], obj_boxes[gti, 2], obj_boxes[gti, 3] ]) if len(mat_gt_i) > 0: mat_gt_i = np.array(mat_gt_i) mat_gt_i = zl.unique_arr(mat_gt_i) for gti in xrange(mat_gt_i.shape[0]): mat_gt_bb_i.append(mat_gt_i[gti, 1:]) mat_gt_label_i.append(mat_gt_i[gti, 0]) mat_gt_label.append(mat_gt_label_i) mat_gt_bb.append(mat_gt_bb_i) #outfile.write(res_line+'\n') sio.savemat( 'output/vg_vrd_objs.mat', { 'pred_bb': mat_pred_bb, 'pred_conf': mat_pred_conf, 'pred_label': mat_pred_label, 'gt_bb': mat_gt_bb, 'gt_label': mat_gt_label })
def run_relation(model_type, iteration): cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test'] result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' + iteration + '.hdf5') m = h5py.File('data/sg_vrd_meta.h5') #--------------cache boxes----------------- h5_boxes = h5py.File('output/precalc/sg_vrd_objs.hdf5') cache_boxes = {} if os.path.exists('output/cache/sg_vrd_objs_test.pkl'): cache_boxes = zl.load('output/cache/sg_vrd_objs_test.pkl') glog.info('loaded obj data from cache') else: glog.info('Preloading obj') zl.tic() for k in h5_boxes['test'].keys(): boxes = h5_boxes['test/%s/boxes' % k][...] cache_boxes[k] = boxes glog.info('done preloading obj %f' % zl.toc()) zl.save('output/cache/sg_vrd_obj_test.pkl', cache_boxes) #--------------cache boxes----------------- #--------------cache old boxes------------- h5_boxes = h5py.File('output/sg_vrd_2016_test.hdf5') cache_old_boxes = {} if os.path.exists('output/cache/sg_vrd_objs_test_vgg.pkl'): cache_old_boxes = zl.load('output/cache/sg_vrd_objs_test_vgg.pkl') glog.info('loaded obj data from cache') else: glog.info('Preloading obj') zl.tic() for k in h5_boxes.keys(): locations = h5_boxes['%s/locations' % k][...] cls_confs = h5_boxes['%s/cls_confs' % k][...] boxes = np.concatenate((locations, cls_confs[:, 1, np.newaxis], cls_confs[:, 0, np.newaxis]), axis=1) cache_old_boxes[k] = boxes glog.info('done preloading obj %f' % zl.toc()) zl.save('output/cache/sg_vrd_obj_test_vgg.pkl', cache_old_boxes) #--------------cache old boxes------------- data_root = 'data/sg_vrd_2016/Data/sg_test_images/' keep = 100 thresh = 0.0001 prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt' model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel' net = caffe.Net(prototxt, model_path, caffe.TEST) # sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data}) # exit(0) #net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST) cnt = 0 for imid in cache_h5.keys(): cnt += 1 if cnt % 10 == 0: glog.info(cnt) obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...] sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...] rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...] rlp_labels = [] rlp_confs = [] sub_boxes = [] obj_boxes = [] #boxes = cache_boxes[imid] boxes = cache_old_boxes[imid] if boxes.shape[0] >= 2: for s in xrange(boxes[:20].shape[0]): for o in xrange(boxes[:20].shape[0]): if s == o: continue if boxes[s][4] < 0.01: continue if boxes[o][4] < 0.01: continue sbox = boxes[s][:4] obox = boxes[o][:4] rlp_labels.append([boxes[s, 5], -1, boxes[o, 5]]) rlp_confs.append(boxes[s, 4] + boxes[o, 4]) sub_boxes.append(sbox) obj_boxes.append(obox) if len(sub_boxes) <= 0: continue #sub_box = np.array(sub_box) #obj_box = np.array(obj_box) im_path = C.get_sg_vrd_path_test(imid) im_detect(net, im_path, sub_boxes, obj_boxes) relation_prob = net.blobs['relation_prob'].data[...] for r in xrange(relation_prob.shape[0]): argmax = np.argmax(relation_prob[r, ...]) rs = relation_prob[r, argmax].squeeze() rlp_labels[r][1] = argmax rlp_confs[r] = rlp_confs[r] + rs im = cv2.imread(data_root + imid + '.jpg') for i in xrange(len(sub_boxes)): sb = sub_boxes[i] cv2.rectangle(im, (int(sb[0]), int(sb[1])), (int(sb[2]), int(sb[3])), (255, 0, 0), 2) pass cv2.imshow('im', im) #if cv2.waitKey(0)==27: # exit(0) #rlp_confs.append(rlp_conf) #rlp_labels.append(rlp_label) #sub_boxes.append(sub_box) #obj_boxes.append(obj_box) # for i in xrange(70): # rs = relation_score[i] # if rs>0.0: # predicate =i # #print relation_score[predicate] # rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32) # #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs) # rlp_conf = rs#+sub_score+obj_score#relation_score[predicate] # rlp_confs.append(rlp_conf) # rlp_labels.append(rlp_label) # sub_boxes.append(sub_boxes_gt[s]) # obj_boxes.append(obj_boxes_gt[s]) result.create_dataset(imid + '/rlp_confs', dtype='float16', data=np.array(rlp_confs).astype(np.float16)) result.create_dataset(imid + '/sub_boxes', dtype='float16', data=np.array(sub_boxes).astype(np.float16)) result.create_dataset(imid + '/obj_boxes', dtype='float16', data=np.array(obj_boxes).astype(np.float16)) result.create_dataset(imid + '/rlp_labels', dtype='float16', data=np.array(rlp_labels).astype(np.float16))
def eval(): cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml') #cfg_from_file('experiments/cfgs/rfcn_end2end_iccv_eb.yml') imdb, roidb = combined_roidb('sg_vrd_2016_test') import cv2 #h5f = h5py.File('/media/zawlin/ssd/iccv2017/data/voc/gen_eb.h5',driver='core') h5path = 'data/sg_vrd_2016/EB/eb.h5' h5f = h5py.File(h5path,driver='core') h5_rois = {} for i in h5f['test/']: data=h5f['test/%s'%i][...].astype(np.float32) idx = np.argsort(data[:,-1],axis=0) data_sorted = data[idx][::-1] data_sorted_idx = np.where((data_sorted[:,2]-data_sorted[:,0]>20) & (data_sorted[:,3]-data_sorted[:,1]>20)) data_sorted = data_sorted[data_sorted_idx] #print data_sorted h5_rois[i] = data_sorted[:1000,:4] #cfg.TEST.HAS_RPN=False net =None #prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_eb_sigmoid.prototxt' prototxt = 'models/sg_vrd/wsd/test_eb_wsddn_s.prototxt' #model = 'data/rfcn_models/resnet50_rfcn_iter_1200.caffemodel' #model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_16000.caffemodel' #model = 'output/rfcn_end2end/voc_0712_train/eb_wsddn_s_iter_5000.caffemodel' model = 'output/rfcn_end2end/sg_vrd_2016_train/eb_wsddn_s_iter_9400.caffemodel' #model = 'data/rfcn_models/resnet50_rfcn_final.caffemodel' #model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_eb_sigx_iter_100000.caffemodel' #model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_600.caffemodel' caffe.set_mode_gpu() caffe.set_device(0) net = caffe.Net(prototxt, model, caffe.TEST) #prototxt = 'models/pascal_voc/ResNet-50/rfcn_end2end/test_iccv_rpn.prototxt' #model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel' #model = 'output/rfcn_end2end/voc_0712_train/resnet50_rfcn_iter_1600.caffemodel' #model = 'data/rfcn_models_iccv/eb_resnet50_rfcn_iter_800.caffemodel' #net2 = caffe.Net(prototxt, model, caffe.TEST) #net.params['conv_new_1_zl'][0].data[...] = net2.params['conv_new_1_zl'][0].data[...] #net.params['conv_new_1_zl'][1].data[...] = net2.params['conv_new_1_zl'][1].data[...] #net2 = None net.name = 'resnet50_rfcn_iter_1200' num_images = len(imdb.image_index) num_images = 100 del imdb.image_index[num_images:] all_boxes = [[[] for _ in xrange(num_images)] for _ in xrange(imdb.num_classes)] output_dir = get_output_dir(imdb, net) zl.tic() # timers _t = {'im_detect' : Timer(), 'misc' : Timer()} max_per_image =200 thresh = 0.00001 cv2.namedWindow('im',0) cnt = 0 for i in xrange(num_images): # filter out any ground truth boxes im_path = imdb.image_path_at(i) im_name = im_path.split('/')[-1] eb_roi = h5_rois[im_name] im = cv2.imread(imdb.image_path_at(i)) _t['im_detect'].tic() #scores, boxes = im_detect(net, im, box_proposals) scores, boxes = im_detect_iccv(net, im, eb_roi) #attention = net.blobs['attention'].data.squeeze() #net.blobs['attention'].data #scores = np.multiply(scores,attention) _t['im_detect'].toc() _t['misc'].tic() # skip j = 0, because it's the background class for j in xrange(1, imdb.num_classes): if j == 15: dfdfd=1 dfdfd += 1 inds = np.where(scores[:, j-1] > thresh)[0] cls_scores = scores[inds, j-1] if cfg.TEST.AGNOSTIC: cls_boxes = boxes[inds, 1:] else: cls_boxes = boxes[inds, j*4:(j+1)*4] cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \ .astype(np.float32, copy=False) keep = nms(cls_dets, cfg.TEST.NMS,force_cpu=True) cls_dets = cls_dets[keep, :] all_boxes[j][i] = cls_dets # Limit to max_per_image detections *over all classes* if max_per_image > 0: image_scores = np.hstack([all_boxes[j][i][:, -1] for j in xrange(1, imdb.num_classes)]) if len(image_scores) > max_per_image: image_thresh = np.sort(image_scores)[-max_per_image] for j in xrange(1, imdb.num_classes): keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0] all_boxes[j][i] = all_boxes[j][i][keep, :] for j in xrange(1, imdb.num_classes): cls_str = imdb.classes[j] for roi in all_boxes[j][i]: cv2.putText(im,cls_str,(roi[0],roi[1]),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,0),1) cv2.rectangle(im,(roi[0],roi[1]),(roi[2],roi[3]),(0,0,255),1) cnt += 1 cv2.imwrite('/home/zawlin/%d.jpg'%cnt,im) cv2.imshow('vis',im) cv2.waitKey(0) _t['misc'].toc() print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \ .format(i + 1, num_images, _t['im_detect'].average_time, _t['misc'].average_time) det_file = os.path.join(output_dir, 'detections.pkl') with open(det_file, 'wb') as f: cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL) print 'Evaluating detections' imdb.evaluate_detections(all_boxes, output_dir) print zl.toc()
def run_retrieval_vp_v2(): # h5_path = 'output/sg_vrd_2016_result.classeme.hdf5' h5_path = 'output/precalc/sg_vrd_2016_test_nms.7.hdf5' m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core') m_vp = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_vp_meta.h5', 'r', 'core') cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1] data_root = '/home/zawlin/data/data_vrd/vrd/sg/Data/sg_test_images/' rlp_map = [] for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())): tri = str(m_vp['meta/tri/idx2name'][str(i)][...]) s_lbl = tri.split('_')[0] p_lbl = tri.split('_')[1] o_lbl = tri.split('_')[2] rlp_label = [ zl.name2idx_cls(m, s_lbl), zl.name2idx_pre(m, p_lbl), zl.name2idx_cls(m, o_lbl) ] rlp_map.append(rlp_label) rlp_map = np.array(rlp_map) if os.path.exists(cache_path): print 'load from cache' h5f = zl.load(cache_path) else: h5_in = h5py.File(h5_path, 'r') h5f = {} print 'preloading data' for i in h5_in: h5f[i] = {} h5f[i]['labels'] = h5_in[i]['labels'][...] h5f[i]['confs'] = h5_in[i]['confs'][...] h5f[i]['boxes'] = h5_in[i]['boxes'][...] zl.save(cache_path, h5f) print 'preloading data done' retr_meta = zl.load('output/pkl/vr_retr_meta.pkl') thresh = 0.0 images = {} imids = h5f.keys() results = {} cnt = 0 r_acc_100 = 0 r_acc_50 = 0 tp_total = 0 gt_total = 0 median = [] for k in xrange(len(retr_meta['rlp_labels'])): if k > 1000: break cnt += 1 rlp_label = retr_meta['rlp_labels'][k] gt_files = retr_meta['files'][k] # print gt_files # exit(0) # for f in gt_files: # impath= zl.imid2path(m,f) # print impath # im= cv2.imread(data_root+impath) # cv2.imshow('im',im) # cv2.waitKey(0) results = {} zl.tic() ranks = [] for imid in imids: labels = h5f[imid]['labels'] - 1 rlp_confs = h5f[imid]['confs'] rlp_labels = rlp_map[labels] if rlp_labels.shape[0] == 0: results[imid] = 0.0 continue indexor = np.arange(rlp_labels.shape[0]) ind = indexor[np.all(rlp_labels == rlp_label, axis=1)] if ind.shape[0] == 0: results[imid] = 0.0 continue confs = rlp_confs[ind] results[imid] = np.average(confs) results_sorted = zl.sort_dict_by_val(results) total_gt = len(gt_files) + 0.0 gt_total += total_gt + 0.0 tp_50 = 0. tp_100 = 0. found = False s_lbl = zl.idx2name_cls(m, rlp_label[0]) p_lbl = zl.idx2name_pre(m, rlp_label[1]) o_lbl = zl.idx2name_cls(m, rlp_label[2]) lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl) delay = 0 for i in xrange(len(results_sorted)): imid, v = results_sorted[i] impath = zl.imid2path(m, imid) if found and i >= 5: break # print gt_files cor = imid in gt_files if cor: if not found: found = True median.append(i) if i < 5: tp_100 += 1 tp_total += 1 if i < 50: tp_50 += 1 if True: cor_or_not = str(cor) if cor: delay = 0 if delay == 0: im = cv2.imread(data_root + impath) cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) cv2.imshow('im', im) c = cv2.waitKey(delay) & 0xFF if c == 27: exit(0) if c == ord('s'): delay = 1 - delay if c == ord('c'): delay = 1 r_50 = tp_50 / 5 #total_gt r_100 = tp_100 / 5 #total_gt r_acc_50 += r_50 r_acc_100 += r_100 med = np.median(median) print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt, r_acc_100 / cnt, med, zl.toc())
def run_retrieval_zeroshot(): # h5_path = 'output/sg_vrd_2016_result_all_19500.hdf5' # h5_path = 'output/sg_vrd_2016_result_diff_all_5000.hdf5' # h5_path= 'output/results/lu_method_results.hdf5' # h5_path = 'output/sg_vrd_2016_result.hdf5.dd' # h5_path = 'output/results/lu_method_results_max.hdf5' h5_path = 'output/results/lu_visual_method_results.hdf5' data_root = 'data/sg_vrd_2016/Data/sg_test_images/' m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core') zeroshots = m['meta/zeroshots'][...] gt_cache_path = 'output/cache/sg_vrd_gt_cache.pkl' gt_h5f = {} np.random.seed(76) if os.path.exists(gt_cache_path): print 'load gt from cache' gt_h5f = zl.load(gt_cache_path) else: print 'cacheing gt' for k in m['gt/test']: gt_h5f[k] = {} sub_boxes = m['gt/test'][k]['sub_boxes'][...] rlp_labels = m['gt/test'][k]['rlp_labels'][...] obj_boxes = m['gt/test'][k]['obj_boxes'][...] gt_h5f[k]['sub_boxes'] = sub_boxes gt_h5f[k]['obj_boxes'] = obj_boxes gt_h5f[k]['rlp_labels'] = rlp_labels print 'caching gt done' zl.save(gt_cache_path, gt_h5f) cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1] # h5f = h5py.File('output/sg_vrd_2016_result.classeme.hdf5') if os.path.exists(cache_path): print 'load from cache' h5f = zl.load(cache_path) else: h5_in = h5py.File(h5_path, 'r') h5f = {} print 'preloading data' for i in h5_in: h5f[i] = {} h5f[i]['rlp_labels'] = h5_in[i]['rlp_labels'][...] h5f[i]['rlp_confs'] = h5_in[i]['rlp_confs'][...] h5f[i]['sub_boxes'] = h5_in[i]['sub_boxes'][...] h5f[i]['obj_boxes'] = h5_in[i]['obj_boxes'][...] zl.save(cache_path, h5f) print 'preloading data done' #h5f = h5py.file('output/results/lu_method_results.hdf5') retr_meta = zl.load('output/pkl/vr_retr_meta.pkl') thresh = 0.0 images = {} imids = h5f.keys() results = {} cnt = 0 r_acc_100 = 0 r_acc_50 = 0 tp_total = 0 gt_total = 0 median = [] for k in xrange(len(retr_meta['rlp_labels'])): # if k>1000: # break rlp_label = retr_meta['rlp_labels'][k] if not np.any(np.all(zeroshots == rlp_label, axis=1)): continue gt_files = retr_meta['files'][k] cnt += 1 results = {} zl.tic() for imid in imids: rlp_labels = h5f[imid]['rlp_labels'] rlp_confs = h5f[imid]['rlp_confs'] if rlp_confs.shape[0] == 0: results[imid] = 0.0 continue indexor = np.arange(rlp_labels.shape[0]) ind = indexor[np.all(rlp_labels == rlp_label, axis=1)] if ind.shape[0] == 0: results[imid] = 0.0 continue confs = rlp_confs[ind] results[imid] = np.random.uniform(-1, 1) #np.average(confs) results_sorted = zl.sort_dict_by_val(results) total_gt = len(gt_files) + 0.0 gt_total += total_gt + 0.0 tp_50 = 0. tp_100 = 0. found = False delay = 0 s_lbl = zl.idx2name_cls(m, rlp_label[0]) p_lbl = zl.idx2name_pre(m, rlp_label[1]) o_lbl = zl.idx2name_cls(m, rlp_label[2]) lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl) r_at_k = 5 for i in xrange(len(results_sorted)): imid, v = results_sorted[i] if found and i >= r_at_k: break cor = imid in gt_files if cor: if not found: found = True median.append(i) if i < r_at_k: tp_100 += 1 tp_total += 1 if i < 50: tp_50 += 1 # if True: # cor_or_not = str(cor) # if cor :delay=0 # if delay ==0: # im = cv2.imread(data_root+imid+'.jpg') # cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) # cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) # cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) # cv2.imshow('im',im) # c = cv2.waitKey(delay)&0xFF # if c ==27: # exit(0) # if c == ord('s'): # delay = 1-delay # if c == ord('c'): # delay = 1 r_50 = tp_50 / r_at_k #total_gt r_100 = tp_100 / r_at_k #total_gt r_acc_50 += r_50 r_acc_100 += r_100 med = np.median(median) print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt, r_acc_100 / cnt, med, zl.toc())
def run_retrieval_vp(): h5f = h5py.File('output/precalc/ours_retr.hdf5') retr_meta = zl.load('output/pkl/vr_retr_meta.pkl') # print h5f[h5f.keys()[0]].keys() # exit(0) #h5f = h5py.file('output/results/lu_method_results.hdf5') # data_root='/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/' m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core') thresh = 0.0 filters = [] rois = [] images = {} imids = h5f.keys() results = {} cnt = 0 pre = 'teddy bear_sit on_blanket' # pre = 'paw_in front of_cat' # pre = 'racket_hold by_player' r_idx = zl.name2idx_tri(m_vp, pre) # sub_lbl = pre.split('_')[0] # pre_lbl = pre.split('_')[1] # obj_lbl = pre.split('_')[2] # sub_idx = zl.name2idx_cls(m,sub_lbl) # pre_idx = zl.name2idx_pre(m,pre_lbl) # obj_idx = zl.name2idx_cls(m,obj_lbl) # rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16) results = {} for imid in imids: if cnt % 100 == 0: print cnt, zl.toc() zl.tic() cnt += 1 # rlp_labels = h5f[imid+'/labels'][...] rlp_confs = h5f[imid + '/confs'][:, r_idx] ind = np.argsort(rlp_confs)[::-1] rlp_confs = rlp_confs[ind[:5]] results[imid] = np.average(rlp_confs) if rlp_confs.shape[0] == 0: results[imid] = 0.0 continue zl.tic() indexor = np.arange(rlp_labels.shape[0]) # ind = indexor[np.all(rlp_labels==rlp_label,axis=1)] ind = np.where(rlp_labels == r_idx)[0] if ind.shape[0] == 0: results[imid] = 0.0 continue confs = rlp_confs[ind] results[imid] = np.average(confs) results_sorted = zl.sort_dict_by_val(results) example_folder = 'output/examples_retr_vg_vp/%s/' % pre zl.make_dirs(example_folder) cnt = 0 for imid, v in results_sorted[:200]: boxes = h5f[imid + '/boxes'][...] rlp_labels = h5f[imid + '/labels'][...] rlp_confs = h5f[imid + '/confs'][...] # indexor = np.arange(rlp_labels.shape[0]) # ind = indexor[np.all(rlp_labels==rlp_label,axis=1)] ind = np.where(rlp_labels == r_idx)[0] if ind.shape[0] != 0: boxes = boxes[ind] sb = boxes[0] # ob = obj_boxes[0] im = cv2.imread(data_root + zl.imid2path(m, imid)) cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1) # cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1) cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im) #cv2.imshow('im',im) cnt += 1