def __init__(self, args): # load model model_prefix = osp.join(ROOT_DIR, 'output', args.dataset + '_' + args.splitBy, args.model_id) tic = time.time() infos = json.load(open(model_prefix + '.json')) model_path = model_prefix + '.pth' self.dataset = args.dataset self.model_opt = infos['opt'] self.word_to_ix = infos['word_to_ix'] self.ix_to_att = {ix: att for att, ix in infos['att_to_ix'].items()} self.model = self.load_matnet_model(model_path, self.model_opt) print('MatNet [%s_%s\'s %s] loaded in %.2f seconds.' % (args.dataset, args.splitBy, args.model_id, time.time() - tic)) # load mask r-cnn tic = time.time() args.imdb_name = self.model_opt['imdb_name'] args.net_name = self.model_opt['net_name'] args.tag = self.model_opt['tag'] args.iters = self.model_opt['iters'] self.mrcn = inference.Inference(args) self.imdb = self.mrcn.imdb print( 'Mask R-CNN: imdb[%s], tag[%s], id[%s_mask_rcnn_iter_%s] loaded in %.2f seconds.' % (args.imdb_name, args.tag, args.net_name, args.iters, time.time() - tic))
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if 'coco' or 'combined' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' else: print('No image directory prepared for ', args.dataset) sys.exit(0) # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) # load detections = [{det_id, box, image_id, category_id, category_name, score}] save_dir = osp.join('cache/detections', args.dataset + '_' + args.splitBy) detections = json.load(open(osp.join(save_dir, args.dets_file_name))) image_to_dets = {} for det in detections: image_id = det['image_id'] if image_id not in image_to_dets: image_to_dets[image_id] = [] image_to_dets[image_id] += [det] # run mask rcnn for i, image_id in enumerate(image_to_dets.keys()): dets = image_to_dets[image_id] img_path = osp.join( IMAGE_DIR, 'COCO_train2014_' + str(image_id).zfill(12) + '.jpg') assert osp.isfile(img_path), img_path boxes = np.array([det['box'] for det in dets], dtype=np.float32) boxes = xywh_to_xyxy(boxes) labels = [ mrcn.imdb._class_to_ind[det['category_name']] for det in dets ] labels = np.array(labels, dtype=np.int32) masks, rles = mrcn.boxes_to_masks(img_path, boxes, labels) # add rles to det for ix, det in enumerate(dets): det['rle'] = rles[ix] print('%s/%s done.' % (i, len(image_to_dets))) # save dets.json = [{det_id, box, image_id, score}] # to cache/detections/ save_path = osp.join(save_dir, args.dets_file_name[:-10] + '_masks.json') with open(save_path, 'w') as f: json.dump(detections, f)
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] IMAGE_DIR = 'data/images/mscoco/images/train2014' # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) # load detections = [{det_id, box, image_id, category_id, category_name, score}] save_dir = osp.join('cache/detections', args.dataset+'_'+args.splitBy) dets_file_name = 'matt_dets_%s_%s_%s_%d.json' % (args.m, args.tid, dataset_splitBy, args.top_N) with open(osp.join(save_dir, dets_file_name), 'r') as f: detections = json.load(f) image_to_dets = {} for det in detections: image_id = det['image_id'] if image_id not in image_to_dets: image_to_dets[image_id] = [] image_to_dets[image_id] += [det] # run mask rcnn pbar = tqdm(total=len(image_to_dets.keys()), ascii=True, ncols=120) for i, image_id in enumerate(image_to_dets.keys()): dets = image_to_dets[image_id] img_path = osp.join(IMAGE_DIR, 'COCO_train2014_'+str(image_id).zfill(12)+'.jpg') assert osp.isfile(img_path), img_path boxes = np.array([det['box'] for det in dets], dtype=np.float32) boxes = xywh_to_xyxy(boxes) labels = [mrcn.imdb._class_to_ind[det['category_name']] for det in dets] labels = np.array(labels, dtype=np.int32) masks, rles = mrcn.boxes_to_masks(img_path, boxes, labels) # add rles to det for ix, det in enumerate(dets): det['rle'] = rles[ix] pbar.update(1) pbar.close() # save dets.json = [{det_id, box, image_id, score}] # to cache/detections/ mask_file_name = 'matt_mask_%s_%s_%s_%d.json' % (args.m, args.tid, dataset_splitBy, args.top_N) save_path = osp.join(save_dir, mask_file_name) with open(save_path, 'w') as f: json.dump(detections, f)
def main(args): dataset_splitBy = args.dataset + '_' + args.splitBy if not osp.isdir(osp.join('cache/feats/', dataset_splitBy)): os.makedirs(osp.join('cache/feats/', dataset_splitBy)) # Image Directory if 'coco' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' else: IMAGE_DIR = f'/input/data_endovis2017_instances_cropped/{args.splitBy}/images' # print('No image directory prepared for ', args.dataset) # sys.exit(0) # load dataset data_json = osp.join('cache/prepro', dataset_splitBy, 'data.json') data_h5 = osp.join('cache/prepro', dataset_splitBy, 'data.h5') loader = Loader(data_json, data_h5) images = loader.images # args.dataset = args.dataset.replace('endovis2017', 'endovis_2017') # load mrcn model mrcn = inference.Inference(args) imdb = mrcn.imdb # args.dataset = args.dataset.replace('endovis_2017', 'endovis2017') # feats_h5 feats_dir = osp.join( 'cache/feats', dataset_splitBy, 'mrcn', '%s_%s_%s' % (args.net_name, args.imdb_name, args.tag)) if not osp.isdir(feats_dir): os.makedirs(feats_dir) # extract for i, image in enumerate(images): file_name = image['file_name'] img_path = osp.join(IMAGE_DIR, file_name) feat, im_info = mrcn.extract_head(img_path) feat = feat.data.cpu().numpy() # write feat_h5 = osp.join(feats_dir, str(image['image_id']) + '.h5') f = h5py.File(feat_h5, 'w') f.create_dataset('head', dtype=np.float32, data=feat) f.create_dataset('im_info', dtype=np.float32, data=im_info) f.close() if i % 10 == 0: print('%s/%s image_id[%s] size[%s] im_scale[%.2f] writen.' % (i + 1, len(images), image['image_id'], feat.shape, im_info[0][2])) print('Done.')
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if 'coco' or 'combined' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' else: print('No image directory prepared for ', args.dataset) sys.exit(0) # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) imdb = mrcn.imdb # import refer from refer import REFER data_root, dataset, splitBy = params['data_root'], params[ 'dataset'], params['splitBy'] refer = REFER(data_root, dataset, splitBy) cat_name_to_cat_ix = { category_name: category_id for category_id, category_name in refer.Cats.items() } # detect and prepare dets.json proposals = [] det_id = 0 cnt = 0 # # TEMPS DEBUG # # os.makedirs('cache/old_internals') # img_path = '/home/mwb/Datasets/mscoco/images/train2014/COCO_train2014_000000581857.jpg' # scores, boxes = mrcn.predict(img_path) # image_feat = mrcn.net._predictions['__temp_net_conv'].data.cpu().numpy() # roi_feats = mrcn.net._predictions['__temp_pool5'].data.cpu().numpy() # rois = mrcn.net._predictions['__temp_rois'].data.cpu().numpy()[:,1:] # head_feats = mrcn.net._predictions['__temp_head_feats'].data.cpu().numpy() # head_pool = mrcn.net._predictions['__temp_head_pool'].data.cpu().numpy() # print(image_feat.shape, roi_feats.shape, rois.shape, head_feats.shape, head_pool.shape) # np.save('cache/old_internals/image_feat.npy', image_feat) # np.save('cache/old_internals/roi_feats.npy', roi_feats) # np.save('cache/old_internals/rois.npy', rois) # np.save('cache/old_internals/head_feats.npy', head_feats) # np.save('cache/old_internals/head_pool.npy', head_pool) for image_id, image in refer.Imgs.items(): file_name = image['file_name'] img_path = osp.join(IMAGE_DIR, file_name) # predict scores, boxes = mrcn.predict(img_path) rois = mrcn.net._predictions['rois'].data.cpu().numpy( )[:, 1:] / mrcn._scale cnt += 1 print('%s/%s done.' % (cnt, len(refer.Imgs))) info = { 'image_id': image_id, 'rois': rois, 'scores': scores, 'boxes': boxes, 'roi_scores': mrcn.net._predictions['__roi_scores'].data.cpu().numpy() } torch.cuda.empty_cache() proposals.append(info) # save dets.json = [{det_id, box, image_id, score}] # to cache/detections/ save_path = osp.join( save_dir, '%s_%s_%s_proposals.pkl' % (args.net_name, args.imdb_name, args.tag)) with open(save_path, 'wb') as f: pickle.dump(proposals, f)
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if 'coco' in dataset_splitBy or 'combined' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' elif 'sunspot' in dataset_splitBy: IMAGE_DIR = 'data/images/SUNRGBD' else: print('No image directory prepared for ', args.dataset) sys.exit(0) # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) imdb = mrcn.imdb # import refer from refer import REFER data_root, dataset, splitBy = params['data_root'], params[ 'dataset'], params['splitBy'] refer = REFER(data_root, dataset, splitBy) cat_name_to_cat_ix = { category_name: category_id for category_id, category_name in refer.Cats.items() } # detect and prepare dets.json dets = [] det_id = 0 h5_id = 0 cnt = 0 for image_id, image in refer.Imgs.items(): file_name = image['file_name'] img_path = osp.join(IMAGE_DIR, file_name) # predict try: scores, boxes = mrcn.predict(img_path) except: print('Error on {}'.format(img_path)) sys.exit(0) # get cls_to_dets, class_name -> [xyxysc] (n, 5) cls_to_dets, num_dets = cls_to_detections(scores, boxes, imdb, args.nms_thresh, args.conf_thresh) # make sure num_dets > 0 for this image, otherwise we lower down the conf_thresh thresh = args.conf_thresh while num_dets == 0: thresh = float(thresh) - 0.1 cls_to_dets, num_dets = cls_to_detections(scores, boxes, imdb, args.nms_thresh, thresh) # add to dets for category_name, detections in cls_to_dets.items(): # detections is list of (n, 5), [xyxysc] for detection in detections.tolist(): x1, y1, x2, y2, sc = detection det = { 'det_id': det_id, 'h5_id': det_id, # we make h5_id == det_id 'box': [x1, y1, x2 - x1 + 1, y2 - y1 + 1], 'image_id': image_id, 'category_id': cat_name_to_cat_ix[category_name], 'category_name': category_name, 'score': sc } dets += [det] det_id += 1 cnt += 1 print('%s/%s done.' % (cnt, len(refer.Imgs))) # save dets.json = [{det_id, box, image_id, score}] # to cache/detections/ save_path = osp.join( save_dir, '%s_%s_%s_dets.json' % (args.net_name, args.imdb_name, args.tag)) with open(save_path, 'w') as f: json.dump(dets, f)
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if 'coco' or 'combined' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' else: print('No image directory prepared for ', args.dataset) sys.exit(0) # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) imdb = mrcn.imdb # import refer from refer import REFER data_root, dataset, splitBy = params['data_root'], params[ 'dataset'], params['splitBy'] refer = REFER(data_root, dataset, splitBy) cat_name_to_cat_ix = { category_name: category_id for category_id, category_name in refer.Cats.items() } # detect and prepare dets.json dets = [] det_id = 0 h5_id = 0 cnt = 0 for image_id, image in refer.Imgs.items(): file_name = image['file_name'] img_path = osp.join(IMAGE_DIR, file_name) # predict scores, boxes = mrcn.predict(img_path) cls_to_dets, num_dets = cls_to_detections(scores, boxes, imdb, args.nms_thresh, 0) img_det_list = [] for category_name, detections in cls_to_dets.items(): # detections is list of (n, 5), [xyxysc] for detection in detections.tolist(): x1, y1, x2, y2, sc = detection det = { 'box': [x1, y1, x2 - x1 + 1, y2 - y1 + 1], 'category_name': category_name, 'score': sc } img_det_list.append(det) sorted_img_det_list = sorted(img_det_list, key=lambda x: x['score'], reverse=True) for det in sorted_img_det_list[:300]: dets.append({ 'det_id': det_id, 'h5_id': det_id, # we make h5_id == det_id 'box': det['box'], 'image_id': image_id, 'category_id': cat_name_to_cat_ix[det['category_name']], 'category_name': det['category_name'], 'score': det['score'] }) det_id += 1 cnt += 1 print('%s/%s done.' % (cnt, len(refer.Imgs))) save_path = osp.join(save_dir, 'mrcn_dets_300_%s.json' % (dataset_splitBy)) with open(save_path, 'w') as f: json.dump(dets, f)
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if 'coco' or 'combined' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' else: print('No image directory prepared for ', args.dataset) sys.exit(0) # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) imdb = mrcn.imdb # import refer from refer import REFER data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy'] refer = REFER(data_root, dataset, splitBy) cat_name_to_cat_ix = {category_name: category_id for category_id, category_name in refer.Cats.items()} # detect and prepare dets.json proposals = [] det_id = 0 cnt = 0 # # TEMPS DEBUG # # os.makedirs('cache/old_internals') # img_path = '/home/mwb/Datasets/mscoco/images/train2014/COCO_train2014_000000581857.jpg' # scores, boxes = mrcn.predict(img_path) # image_feat = mrcn.net._predictions['__temp_net_conv'].data.cpu().numpy() # roi_feats = mrcn.net._predictions['__temp_pool5'].data.cpu().numpy() # rois = mrcn.net._predictions['__temp_rois'].data.cpu().numpy()[:,1:] # head_feats = mrcn.net._predictions['__temp_head_feats'].data.cpu().numpy() # head_pool = mrcn.net._predictions['__temp_head_pool'].data.cpu().numpy() # print(image_feat.shape, roi_feats.shape, rois.shape, head_feats.shape, head_pool.shape) # np.save('cache/old_internals/image_feat.npy', image_feat) # np.save('cache/old_internals/roi_feats.npy', roi_feats) # np.save('cache/old_internals/rois.npy', rois) # np.save('cache/old_internals/head_feats.npy', head_feats) # np.save('cache/old_internals/head_pool.npy', head_pool) val_image_ids = {refer.Refs[ref_id]['image_id'] for ref_id in refer.getRefIds(split='val') if refer.Refs[ref_id]['split'] == 'val'} print('val image num:', len(val_image_ids)) start = time() # for image_id, image in refer.Imgs.items(): for image_id in val_image_ids: image = refer.Imgs[image_id] file_name = image['file_name'] img_path = osp.join(IMAGE_DIR, file_name) # predict scores, boxes = mrcn.predict(img_path) rois = mrcn.net._predictions['rois'].data.cpu().numpy()[:,1:] / mrcn._scale cnt += 1 # print('%s/%s done.' % (cnt, len(refer.Imgs))) print('%s/%s done.' % (cnt, len(val_image_ids))) # info = { # 'image_id': image_id, # 'rois': rois, # 'scores': scores, # 'boxes': boxes, # 'roi_scores': mrcn.net._predictions['__roi_scores'].data.cpu().numpy() # } torch.cuda.empty_cache() # proposals.append(info) total_t = time() - start avg_t = total_t / len(val_image_ids) print('time: %.6f / %.6f = %.6f' % (total_t, len(val_image_ids), avg_t))