def build_coco_batches(dataset, setname, input_H, input_W): im_dir = '/data/ryli/datasets/coco/images' im_type = 'train2014' vocab_file = '/data/ryli/rmi_phrasecut/data/vocabulary_Gref.txt' data_folder = './data/' + dataset + '/' + setname + '_batch/' data_prefix = dataset + '_' + setname if not os.path.isdir(data_folder): os.makedirs(data_folder) if dataset == 'Gref': refer = REFER('/data/ryli/rmi_phrasecut/external/refer/data', dataset='refcocog', splitBy='google') elif dataset == 'unc': refer = REFER('/data/ryli/rmi_phrasecut/external/refer/data', dataset='refcoco', splitBy='unc') elif dataset == 'unc+': refer = REFER('/data/ryli/rmi_phrasecut/external/refer/data', dataset='refcoco+', splitBy='unc') else: raise ValueError('Unknown dataset %s' % dataset) refs = [ refer.Refs[ref_id] for ref_id in refer.Refs if refer.Refs[ref_id]['split'] == setname ] sent_data = [] encoder = setup_encoder() for ref in refs: for sentence in ref['sentences']: sent_data.append(sentence['sent'].decode('latin-1').strip()) encodings = encoder.encode(sent_data) n_batch = 0 for ref in refs: im_name = 'COCO_' + im_type + '_' + str(ref['image_id']).zfill(12) im = skimage.io.imread('%s/%s/%s.jpg' % (im_dir, im_type, im_name)) seg = refer.Anns[ref['ann_id']]['segmentation'] rle = cocomask.frPyObjects(seg, im.shape[0], im.shape[1]) mask = np.max(cocomask.decode(rle), axis=2).astype(np.float32) if 'train' in setname: mask = im_processing.resize_and_pad(mask, input_H, input_W) for sentence in ref['sentences']: print('saving batch %d' % (n_batch + 1)) sent = sentence['sent'] np.savez(file=data_folder + data_prefix + '_' + str(n_batch) + '.npz', im_name_batch=im_name, encoding_batch=encodings[n_batch], mask_batch=(mask > 0), sent_batch=[sent]) n_batch += 1
def build_coco_batches(dataset, setname, T, input_H, input_W): im_dir = '/data/ryli/datasets/coco/images' im_type = 'train2014' vocab_file = './data/vocabulary_Gref.txt' data_folder = './' + dataset + '/' + setname + '_batch/' data_prefix = dataset + '_' + setname if not os.path.isdir(data_folder): os.makedirs(data_folder) if dataset == 'Gref': refer = REFER('./external/refer/data', dataset='refcocog', splitBy='google') elif dataset == 'unc': refer = REFER('./external/refer/data', dataset='refcoco', splitBy='unc') elif dataset == 'unc+': refer = REFER('./external/refer/data', dataset='refcoco+', splitBy='unc') else: raise ValueError('Unknown dataset %s' % dataset) refs = [ refer.Refs[ref_id] for ref_id in refer.Refs if refer.Refs[ref_id]['split'] == setname ] vocab_dict = text_processing.load_vocab_dict_from_file(vocab_file) n_batch = 0 for ref in refs: im_name = 'COCO_' + im_type + '_' + str(ref['image_id']).zfill(12) im = skimage.io.imread('%s/%s/%s.jpg' % (im_dir, im_type, im_name)) seg = refer.Anns[ref['ann_id']]['segmentation'] rle = cocomask.frPyObjects(seg, im.shape[0], im.shape[1]) mask = np.max(cocomask.decode(rle), axis=2).astype(np.float32) if 'train' in setname: im = skimage.img_as_ubyte( im_processing.resize_and_pad(im, input_H, input_W)) mask = im_processing.resize_and_pad(mask, input_H, input_W) if im.ndim == 2: im = np.tile(im[:, :, np.newaxis], (1, 1, 3)) for sentence in ref['sentences']: print('saving batch %d' % (n_batch + 1)) sent = sentence['sent'] text = text_processing.preprocess_sentence(sent, vocab_dict, T) np.savez(file=data_folder + data_prefix + '_' + str(n_batch) + '.npz', text_batch=text, im_batch=im, mask_batch=(mask > 0), sent_batch=[sent]) n_batch += 1
def language_eval(pred, split, params): sys.path.insert(0, osp.join('pyutils', 'refer2')) if 'refcoco' in params['dataset']: image_root = params['coco_image_root'] else: image_root = params['gta_image_root'] from refer import REFER refer = REFER(params['data_root'], image_root, params['dataset'], params['splitBy'], old_version=False) sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation')) from refEvaluation import RefEvaluation eval_cider_r = params['dataset'] == 'refgta' refEval = RefEvaluation(refer, pred, eval_cider_r=eval_cider_r) refEval.evaluate() overall = {} for metric, score in refEval.eval.items(): overall[metric] = score print(overall) from crossEvaluation import CrossEvaluation ceval = CrossEvaluation(refer, pred) ceval.cross_evaluate() ceval.make_ref_to_evals() ref_to_evals = ceval.ref_to_evals ceval.Xscore('CIDEr') return overall
def __init__(self, data_dir="/projectnb/llamagrp/shawnlin/ref-exp-gen/dataset/refer2/refer/data", dataset="refcoco", split="train", splitBy="google", transforms=None): assert (split in ["train", "val", "test"]) assert os.path.exists(data_dir), \ "cannot find folder {}, please download refcoco data into this folder".format(data_dir) self.data_dir = data_dir self.dataset = dataset self.transforms = transforms self.split = split self.splitBy = splitBy self.refer = REFER(self.data_dir, self.dataset, self.splitBy) self.ref_ids = self.refer.getRefIds(split=self.split)[:] #self.filter_non_overlap = filter_non_overlap #self.filter_duplicate_rels = filter_duplicate_rels and self.split == 'train' # read in dataset from a h5 file and a dict (json) file #self.im_h5 = h5py.File(self.image_file, 'r') self.info = json.load(open("/projectnb/llamagrp/shawnlin/ref-exp-gen/graph-rcnn.pytorch/datasets/vg_bm/VG-SGG-dicts.json", 'r')) #self.im_refs = self.im_h5['images'] # image data reference #im_scale = self.im_refs.shape[2] # add background class self.info['label_to_idx']['__background__'] = 0 self.class_to_ind = self.info['label_to_idx'] self.ind_to_classes = sorted(self.class_to_ind, key=lambda k: self.class_to_ind[k]) #cfg.ind_to_class = self.ind_to_classes self.predicate_to_ind = self.info['predicate_to_idx'] self.predicate_to_ind['__background__'] = 0 self.ind_to_predicates = sorted(self.predicate_to_ind, key=lambda k: self.predicate_to_ind[k])
def main(params): dataset = params['dataset'] splitBy = params['splitBy'] split = params['split'] output_txt = params['output_txt'] # load refer sys.path.insert(0, 'refer') from refer import REFER refer = REFER('data', dataset, splitBy) # get refs that within the indicate split refs = [] for ref_id, ref in refer.Refs.items(): if split in ref['split']: # as long as ref['split'] contains string of split, e.g., testA or testB (contains 'test') refs += [ref] random.shuffle(refs) # write object's image paths txt_file = open(output_txt, 'w') for ref in refs: ref_img_path = ref['file_name'] txt_file.write(ref_img_path + '\n') txt_file.close() print '%s refs\' filenames written in %s.' % (len(refs), output_txt)
def run(params): # parse dataset_splitBy = params[ 'dataset_splitBy'] # in Lua, we simply use dataset denoting detaset_splitBy i, j = dataset_splitBy.find('_'), dataset_splitBy.rfind('_') dataset = dataset_splitBy[:i] splitBy = dataset_splitBy[i + 1:] if i == j else dataset_splitBy[i + 1:j] model_id = params['model_id'] split = params['split'] cache_path = params['cache_path'] num_sents = params['num_sents'] # load refer refer = REFER(REFER_DIR, dataset, splitBy) # load lang result Res = load_lang_result(cache_path, dataset_splitBy, model_id, split) # sentToParse sentToParse(Res, num_sents) # parseToAtts parseToAtts(Res, num_sents, dataset) # compute scores overall, refToEval = compute_score(Res, num_sents, refer, dataset) pprint(overall) # write results if not osp.isdir(osp.join(cache_path, 'parse', dataset_splitBy)): os.mkdir(osp.join(cache_path, 'parse', dataset_splitBy)) result_path = osp.join(cache_path, 'parse', dataset_splitBy, 'model_id' + model_id + '_' + split) with open(result_path + '_out.json', 'w') as outfile: json.dump({'overall': overall, 'refToEval': refToEval}, outfile)
def main(): ref_data_root = '<path/to/refer/data/folder>' all_refs = [] for dataset, split_bys in [ ('refcoco', ['google', 'unc']), ('refcoco+', ['unc']), ('refcocog', ['google', 'umd']) ]: for split_by in split_bys: refer = REFER(ref_data_root, dataset, split_by) for ref_id in refer.getRefIds(): ref = refer.Refs[ref_id] ann = refer.refToAnn[ref_id] ref['ann'] = ann ref['dataset'] = dataset ref['dataset_partition'] = split_by all_refs.append(ref) coco_annotations_file = '<path/to/instances_train2014.json>' coco = COCO(coco_annotations_file) ref_image_ids = set(x['image_id'] for x in all_refs) coco_anns = {image_id: {'info': coco.imgs[image_id], 'anns': coco.imgToAnns[image_id]} for image_id in ref_image_ids} out_file = '<path/to/refcoco.json>' with open(out_file, 'w') as f: json.dump({'ref': all_refs, 'coco_anns': coco_anns}, f)
def main(opt): # dataset_split_by data_root, dataset, split_by = opt.data_root, opt.dataset, opt.split_by # load refer refer = REFER(data_root, dataset, split_by) # transfer dataset to multi-branches trees = transfer_dataset(refer) # create vocab word_vocab = build_vocab(trees, 'tokens', opt.word_count_threshold) tag_vocab = build_vocab(trees, 'tags', opt.word_count_threshold) dep_vocab = build_vocab(trees, 'deps', opt.word_count_threshold) wtoi = {w: i for i, w in enumerate(word_vocab)} ttoi = {w: i for i, w in enumerate(tag_vocab)} dtoi = {w: i for i, w in enumerate(dep_vocab)} # prepare refs, images, anns, sentences and write to json refs, images, anns, sentences = prepare_data(refer, trees) json.dump({'refs': refs, 'images': images, 'anns': anns, 'sentences': sentences, 'word_to_ix': wtoi, 'tag_to_ix': ttoi, 'dep_to_ix': dtoi, 'cat_to_ix': {cat_name: cat_id for cat_id, cat_name in refer.Cats.items()}, }, open(osp.join('data/feats', dataset + '_' + split_by, opt.output_json), 'w')) print('%s written.' % osp.join('data/feats', opt.output_json)) torch.save(trees, osp.join('data/feats', dataset + '_' + split_by, opt.output_pth)) print('%s written.' % osp.join('data/feats', opt.output_pth))
def main(params): # dataset_splitBy data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy'] # max_length if params['max_length'] == None: if params['dataset'] in ['refcoco', 'refclef', 'refcoco+']: params['max_length'] = 10 params['topK'] = 50 elif params['dataset'] in ['refcocog']: params['max_length'] = 20 params['topK'] = 50 else: raise NotImplementedError # mkdir and write json file if not osp.isdir(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy)): os.makedirs(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy)) # load refer sys.path.insert(0, 'pyutils/refer') from refer import REFER refer = REFER(data_root, dataset, splitBy) # create vocab vocab, sentToFinal = build_vocab(refer, params) itow = {i: w for i, w in enumerate(vocab)} wtoi = {w: i for i, w in enumerate(vocab)} # check sentence length check_sentLength(sentToFinal) # create attribute vocab att2cnt, ref_to_att_wds = build_att_vocab(refer, params, ['r1','r2','r7']) itoa = {i: a for i, a in enumerate(att2cnt.keys())} atoi = {a: i for i, a in enumerate(att2cnt.keys())} # prepare refs, images, anns, sentences # and write json refs, images, anns, sentences = prepare_json(refer, sentToFinal, ref_to_att_wds, params) json.dump({'refs': refs, 'images': images, 'anns': anns, 'sentences': sentences, 'word_to_ix': wtoi, 'att_to_ix' : atoi, 'att_to_cnt': att2cnt, 'cat_to_ix': {cat_name: cat_id for cat_id, cat_name in refer.Cats.items()}, 'label_length': params['max_length'],}, open(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy, params['output_json']), 'w')) print('%s written.' % osp.join(ROOT_DIR, 'cache/prepro', params['output_json'])) # write h5 file which contains /sentences f = h5py.File(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy, params['output_h5']), 'w') L = encode_captions(sentences, wtoi, params) f.create_dataset("labels", dtype='int32', data=L) f.close() print('%s writtern.' % osp.join(ROOT_DIR, 'cache/prepro', params['output_h5']))
def main(params): # dataset_splitBy data_root, dataset, splitBy = params['data_root'], params[ 'dataset'], params['splitBy'] # mkdir and write json file if not osp.isdir('cache/prepro'): os.mkdir('cache/prepro') if not osp.isdir(osp.join('cache/prepro', dataset + '_' + splitBy)): os.mkdir(osp.join('cache/prepro', dataset + '_' + splitBy)) if not osp.isdir(osp.join('models', dataset + '_' + splitBy)): os.mkdir( osp.join('models', dataset + '_' + splitBy)) # we also mkdir model/dataset_splitBy here! # load refer sys.path.insert(0, osp.join(ROOT_DIR, 'pyutils/refer')) from refer import REFER refer = REFER(data_root, dataset, splitBy) # create vocab vocab, sentToFinal = build_vocab(refer, params) itow = {i + 1: w for i, w in enumerate(vocab)} # lua 1-based wtoi = {w: i + 1 for i, w in enumerate(vocab)} # lua 1-based # check sentence length check_sentLength(sentToFinal) # prepare refs, images, anns, sentences # and write json refs, images, anns, sentences = prepare_json(refer, sentToFinal, params) json.dump( { 'refs': refs, 'images': images, 'anns': anns, 'sentences': sentences, 'ix_to_word': itow, 'word_to_ix': wtoi, 'ix_to_cat': refer.Cats }, open( osp.join('cache/prepro', dataset + '_' + splitBy, params['output_json']), 'w')) print('%s written.' % osp.join('cache/prepro', params['output_json'])) # write h5 file which contains /sentences f = h5py.File( osp.join('cache/prepro', dataset + '_' + splitBy, params['output_h5']), 'w') seqz_L, zseq_L = encode_captions(sentences, wtoi, params) f.create_dataset("seqz_labels", dtype='uint32', data=seqz_L) f.create_dataset("zseq_labels", dtype='uint32', data=zseq_L) f.close() print('%s writtern.' % osp.join('cache/prepro', params['output_h5']))
def __init__(self, refcoco_dir, refcoco_images_dir, split='val'): self.image_dir = refcoco_images_dir mattnet_maskrcnn_detections_path = refcoco_dir.joinpath( 'detections/refcocog_umd/res101_coco_minus_refer_notime_dets.json') with open(mattnet_maskrcnn_detections_path) as f: mattnet_maskrcnn_detections = json.load(f) id2dets = {} for det in mattnet_maskrcnn_detections: image_id = det['image_id'] if image_id not in id2dets: id2dets[image_id] = [] id2dets[image_id].append(det) self.id2dets = id2dets print('Load mattnet detections from', mattnet_maskrcnn_detections_path) assert split in ['train', 'val', 'test'] workspace_dir = Path(__file__).resolve().parent.parent refcoco_util_dir = workspace_dir.joinpath('refcoco_utils') import sys sys.path.append(str(refcoco_util_dir)) from refer import REFER self.refer = REFER('refcocog', 'umd') ref_ids = self.refer.getRefIds(split=split) img_ids = [] image_fns = [] for ref_id in ref_ids: ref = self.refer.Refs[ref_id] img_id = ref['image_id'] if img_id not in img_ids: img_ids.append(img_id) fn_ann = ref['file_name'] # COCO_train2014_000000419645_398406.jpg # COCO_train2014_000000419645.jpg suffix = fn_ann.split('.')[-1] fname = '_'.join(fn_ann.split('_')[:-1]) + '.' + suffix image_fns.append(fname) self.image_ids = img_ids self.image_fns = image_fns
def main(params): # dataset_splitBy data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy'] target_save_dir = osp.join(params['save_dir']+'/prepro', dataset+'_'+splitBy) if params['old']: params['data_json'] = 'old'+params['data_json'] params['data_h5'] = 'old'+params['data_h5'] if 'refcoco' in dataset: image_root = params['coco_image_root'] else: image_root = params['gta_image_root'] # mkdir and write json file if not osp.isdir(target_save_dir): os.makedirs(target_save_dir) # load refer sys.path.insert(0, osp.join(ROOT_DIR, 'pyutils/refer2')) from refer import REFER refer = REFER(data_root, image_root, dataset, splitBy, old_version=params['old']) # create vocab vocab, sentToFinal = build_vocab(refer, params) itow = {i+1: w for i, w in enumerate(vocab)}# 0 : eos, len(itow)+1 : sos wtoi = {w: i+1 for i, w in enumerate(vocab)} # check sentence length check_sentLength(sentToFinal) # prepare refs, images, anns, sentences # and write json refs, images, anns, sentences = prepare_json(refer, sentToFinal, params) json.dump({'refs': refs, 'images': images, 'anns': anns, 'sentences': sentences, 'ix_to_word': itow, 'word_to_ix': wtoi, 'ix_to_cat': refer.Cats, 'sents_info':refer.Sents }, open(osp.join(target_save_dir, params['data_json']), 'w')) print ('%s written.' % osp.join(target_save_dir, params['data_json'])) # write h5 file which contains /sentences f = h5py.File(osp.join(target_save_dir, params['data_h5']), 'w') seqz_L, zseq_L = encode_captions(sentences, wtoi, params) f.create_dataset("seqz_labels", dtype='uint32', data=seqz_L) f.create_dataset("zseq_labels", dtype='uint32', data=zseq_L) f.close() print ('%s writtern.' % osp.join(target_save_dir, params['data_h5']))
def getCider(pred, split, params): sys.path.insert(0, osp.join('pyutils', 'refer2')) from refer import REFER refer = REFER(params['data_root'], '_', params['dataset'], params['splitBy'], old_version=False) sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation')) from refEvaluation import myRefEvaluation eval_cider_r = params['dataset'] == 'refgta' refEval = myRefEvaluation(refer, pred, eval_cider_r=eval_cider_r) refEval.evaluate() overall = {} for metric, score in refEval.eval.items(): overall[metric] = score cider_score = overall['CIDEr'] return cider_score
def load_sentToInfo(save_path='/data/unagi0/mtanaka/sentToInfo.json'): if osp.exists(save_path): with open(save_path) as f: data = json.load(f) else: print('sent info converting...') sys.path.insert(0, osp.join('./', 'pyutils/refer2')) from refer import REFER data_root = './dataset/anns/original' # contains refclef, refcoco, refcoco+, refcocog and images dataset = 'refgta' splitBy = 'utokyo' refer = REFER(data_root, dataset, splitBy) data = refer.Sents with open(save_path, 'w') as f: json.dump(data, f) return data
def __init__(self, refcoco_dir, refcoco_images_dir, coco_dir, split='val'): self.image_dir = refcoco_images_dir # coco_train_annFile = coco_dir.joinpath('annotations/instances_train2014.json') # self.coco = COCO(coco_train_annFile) assert split in ['train', 'val', 'test'] workspace_dir = Path(__file__).resolve().parent.parent refcoco_util_dir = workspace_dir.joinpath('refcoco_utils') import sys sys.path.append(str(refcoco_util_dir)) from refer import REFER self.refer = REFER('refcocog', 'umd') ref_ids = self.refer.getRefIds(split=split) id2dets = {} img_ids = [] image_fns = [] for ref_id in ref_ids: ref = self.refer.Refs[ref_id] img_id = ref['image_id'] if img_id not in img_ids: img_ids.append(img_id) fn_ann = ref['file_name'] # COCO_train2014_000000419645_398406.jpg # COCO_train2014_000000419645.jpg suffix = fn_ann.split('.')[-1] fname = '_'.join(fn_ann.split('_')[:-1]) + '.' + suffix image_fns.append(fname) detections = self.refer.imgToAnns[img_id] id2dets[img_id] = detections self.image_ids = img_ids self.image_fns = image_fns self.id2dets = id2dets
def getWord2count(): referPath = osp.join(root_path, 'MAttNet3', 'data/') refer = REFER(referPath, 'reverie4', 'reverie4') sentToTokens = refer.sentToTokens # count up the number of words word2count = {} for sent_id, tokens in sentToTokens.items(): for wd in tokens: word2count[wd] = word2count.get(wd, 0) + 1 # add category words category_names = list(refer.Cats.values()) + ['__background__'] for cat_name in category_names: for wd in cat_name.split(): if wd not in word2count or word2count[wd] <= 5: word2count[wd] = 1e5 return word2count
def main(params): # dataset_splitBy data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy'] # mkdir and write json file if not osp.isdir(osp.join('cache/sub_obj_wds', dataset + '_' + splitBy)): os.makedirs(osp.join('cache/sub_obj_wds', dataset + '_' + splitBy)) # load refer sys.path.insert(0, 'pyutils/refer') from refer import REFER refer = REFER(data_root, dataset, splitBy) # create subject and object sub_obj_wds = get_sub_obj(refer, params, ['r1', 'r4', 'r5', 'r6']) json.dump({'sub_obj_wds': sub_obj_wds}, open(osp.join('cache/sub_obj_wds', dataset + '_' + splitBy, params['output_json']), 'w')) print('sub_obj_wds written!')
def create_cache(**kwargs): data_root = kwargs.get('data_root') dataset = kwargs.get('dataset') splitBy = kwargs.get('splitBy') splits = kwargs.get('splits') refer = REFER(data_root, dataset, splitBy) # print stats about the given dataset print('dataset [%s_%s] contains: ' % (dataset, splitBy)) ref_ids = refer.getRefIds() image_ids = refer.getImgIds() print('%s expressions for %s refs in %s images.' % (len(refer.Sents), len(ref_ids), len(image_ids))) checkpoint_dir = osp.join('cache', 'prepro', ds + "_" + splitBy) if not osp.isdir(checkpoint_dir): os.makedirs(checkpoint_dir) for split in splits + ['train']: ref_ids = refer.getRefIds(split=split) print('%s refs are in split [%s].' % (len(ref_ids), split)) #have to sample various sentences and their tokens from here. data = [] for ref_id in ref_ids: ref = refer.Refs[ref_id] image_id = ref['image_id'] ref['image_info'] = refer.Imgs[image_id] sentences = ref.pop('sentences') ref.pop('sent_ids') coco_boxes_info = refer.imgToAnns[image_id] coco_boxes = [box_ann['bbox'] for box_ann in coco_boxes_info] gtbox = refer.refToAnn[ref_id]['bbox'] for sentence in sentences: entnew = copy.deepcopy(ref) entnew['boxes'] = coco_boxes entnew['sentence'] = sentence entnew['gtbox'] = gtbox data.append(entnew) data_json = osp.join('cache/prepro', ds + "_" + splitBy, split + '.json') with open(data_json, 'w') as f: json.dump(data, f)
def main(params): dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if not osp.isdir('cache/parsed_sents/' + dataset_splitBy): os.makedirs('cache/parsed_sents/' + dataset_splitBy) # load refer sys.path.insert(0, 'pyutils/refer') from refer import REFER refer = REFER(params['data_root'], params['dataset'], params['splitBy']) # parse sents sents = refer.Sents.values() parse_sents(sents, params) # save with open(osp.join('cache/parsed_sents/' + dataset_splitBy, 'sents.json'), 'w') as io: json.dump(sents, io)
def gen_ref_coco_data(): dataroot = "/projectnb/llamagrp/shawnlin/ref-exp-gen/dataset/refer2/refer/data" dataset = "refcoco" refer = REFER(dataroot, dataset, "google") ref_ids = refer.getRefIds(split="test")[:] print("total ref ids:", len(ref_ids)) for ref_id in ref_ids[:]: ref = refer.Refs[ref_id] img_id = ref["image_id"] ann_id = ref["ann_id"] img_path = os.path.join(refer.IMAGE_DIR, refer.Imgs[img_id]["file_name"]) img = cv2.imread(img_path) #im_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) ref_expr = "\n".join([s["raw"] for s in ref["sentences"]]) #print(ref_expr) #print("img_id", img_id) yield (img, ref_expr, img_id, ann_id, ref_id)
def __init__(self, split): self.refer = REFER(dataset='refcoco+', splitBy='unc') self.ref_ids = self.refer.getRefIds(split=split) self.image_embeds = np.load( os.path.join("data", "embeddings", "FINALImageEmbeddings.npy")) self.image_ids = list( np.load(os.path.join("data", "embeddings", "FINALImageIDs.npy"))) before_text_embeds = time.time() self.text_embeds = np.concatenate( (np.load( os.path.join("data", "embeddings", "FINALTextEmbeddings1of2.npy")), np.load( os.path.join("data", "embeddings", "FINALTextEmbeddings2of2.npy"))), axis=0) after_text_embeds = time.time() print("Text Embedding Time: ", after_text_embeds - before_text_embeds) assert (len(self.text_embeds) == 141564) assert (self.text_embeds[0].shape[1] == 3072) print('Found {} referred objects in {} split.'.format( len(self.ref_ids), split))
def setEvalRefs(self): self.evalRefs = [eval for refId, eval in self.refToEval.items()] if __name__ == '__main__': import os.path as osp import sys ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..')) sys.path.insert(0, osp.join(ROOT_DIR, 'lib', 'datasets')) from refer import REFER # load refer of dataset dataset = 'refcoco' refer = REFER(dataset, splitBy = 'google') # mimic some Res val_refIds = refer.getRefIds(split='test') ref_id = 49767 print "GD: %s" % refer.Refs[ref_id]['sentences'] Res = [{'ref_id': ref_id, 'sent': 'left bottle'}] # evaluate some refer expressions refEval = RefEvaluation(refer, Res) refEval.evaluate() # print output evaluation scores for metric, score in refEval.eval.items(): print '%s: %.3f'%(metric, score)
parser.add_argument('--split', default='testA') parser.add_argument('--model_id', default='0') parser.add_argument('--beam_size', default='10', type=str) parser.add_argument('--write_result', default=1, help='write result into cache_path') parser.add_argument( '--refer_dir', default='new_data', help='data or new_data, refer directory that stores annotations.') # argparse args = parser.parse_args() params = vars(args) print 'parsed input parameters:' print json.dumps(params, indent=2) # load refer sys.path.insert(0, osp.join('pyutils', 'refer2')) from refer import REFER refer = REFER(params['refer_dir'], params['dataset'], params['splitBy']) sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation')) from crossEvaluation import CrossEvaluation sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation')) from refEvaluation import RefEvaluation # call main main(params)
import numpy as np import os from refer import REFER import os.path as osp import cv2 import argparse parser = argparse.ArgumentParser(description='Data preparation') parser.add_argument('--data_root', type=str) # contains refclef, refcoco, refcoco+, refcocog and images parser.add_argument('--output_dir', type=str) parser.add_argument('--dataset', type=str, choices=['refcoco', 'refcoco+','refcocog','refclef'],default='refcoco') parser.add_argument('--split', type=str,default='umd') parser.add_argument('--generate_mask', action='store_true') args = parser.parse_args() # data_root # contains refclef, refcoco, refcoco+, refcocog and images refer = REFER(args.data_root, args.dataset, args.split) print ('dataset [%s_%s] contains: ' % (args.dataset, args.split)) ref_ids = refer.getRefIds() image_ids = refer.getImgIds() print ('%s expressions for %s refs in %s images.' % (len(refer.Sents), len(ref_ids), len(image_ids))) print('\nAmong them:') if args.dataset == 'refclef': if args.split == 'unc': splits = ['train', 'val', 'testA','testB','testC'] else: splits = ['train', 'val', 'test'] elif args.dataset == 'refcoco': splits = ['train', 'val', 'testA', 'testB'] elif args.dataset == 'refcoco+':
parser = argparse.ArgumentParser() parser.add_argument('--dataset', default='refcoco', help='refcoco or refcocog') args = parser.parse_args() params = vars(args) # set path refer_data_path = 'data/refer/data' coco_data_path = 'data/coco/annotations' refer_coco_data_path = 'data/refer_coco/annotations' ####################################################################### # accumulate refer_trainval_ids and refer_test_ids ####################################################################### from refer import REFER refcoco = REFER(refer_data_path, 'refcoco', 'unc') refcoco_excluded_image_ids = [] for _, ref in refcoco.Refs.items(): if ref['split'] in ['testA', 'testB', 'val']: refcoco_excluded_image_ids += [ref['image_id']] refcoco_excluded_image_ids = list(set(refcoco_excluded_image_ids)) print('In refcoco_unc, %s valtest image_ids will be excluded.' % len(refcoco_excluded_image_ids)) refcocog = REFER(refer_data_path, 'refcocog', 'umd') refcocog_excluded_image_ids = [] for _, ref in refcocog.Refs.items(): if ref['split'] in ['test', 'val']: refcocog_excluded_image_ids += [ref['image_id']] refcocog_excluded_image_ids = list(set(refcocog_excluded_image_ids)) print('In refcocog_umd, %s valtest image_ids will be excluded.' %
def main(args): # Image Directory params = vars(args) dataset_splitBy = params['dataset'] + '_' + params['splitBy'] if 'coco' or 'combined' in dataset_splitBy: IMAGE_DIR = 'data/images/mscoco/images/train2014' elif 'clef' in dataset_splitBy: IMAGE_DIR = 'data/images/saiapr_tc-12' else: print('No image directory prepared for ', args.dataset) sys.exit(0) # make save dir save_dir = osp.join('cache/detections', dataset_splitBy) if not osp.isdir(save_dir): os.makedirs(save_dir) print(save_dir) # get mrcn instance mrcn = inference.Inference(args) imdb = mrcn.imdb # import refer from refer import REFER data_root, dataset, splitBy = params['data_root'], params[ 'dataset'], params['splitBy'] refer = REFER(data_root, dataset, splitBy) cat_name_to_cat_ix = { category_name: category_id for category_id, category_name in refer.Cats.items() } # detect and prepare dets.json proposals = [] det_id = 0 cnt = 0 # # TEMPS DEBUG # # os.makedirs('cache/old_internals') # img_path = '/home/mwb/Datasets/mscoco/images/train2014/COCO_train2014_000000581857.jpg' # scores, boxes = mrcn.predict(img_path) # image_feat = mrcn.net._predictions['__temp_net_conv'].data.cpu().numpy() # roi_feats = mrcn.net._predictions['__temp_pool5'].data.cpu().numpy() # rois = mrcn.net._predictions['__temp_rois'].data.cpu().numpy()[:,1:] # head_feats = mrcn.net._predictions['__temp_head_feats'].data.cpu().numpy() # head_pool = mrcn.net._predictions['__temp_head_pool'].data.cpu().numpy() # print(image_feat.shape, roi_feats.shape, rois.shape, head_feats.shape, head_pool.shape) # np.save('cache/old_internals/image_feat.npy', image_feat) # np.save('cache/old_internals/roi_feats.npy', roi_feats) # np.save('cache/old_internals/rois.npy', rois) # np.save('cache/old_internals/head_feats.npy', head_feats) # np.save('cache/old_internals/head_pool.npy', head_pool) for image_id, image in refer.Imgs.items(): file_name = image['file_name'] img_path = osp.join(IMAGE_DIR, file_name) # predict scores, boxes = mrcn.predict(img_path) rois = mrcn.net._predictions['rois'].data.cpu().numpy( )[:, 1:] / mrcn._scale cnt += 1 print('%s/%s done.' % (cnt, len(refer.Imgs))) info = { 'image_id': image_id, 'rois': rois, 'scores': scores, 'boxes': boxes, 'roi_scores': mrcn.net._predictions['__roi_scores'].data.cpu().numpy() } torch.cuda.empty_cache() proposals.append(info) # save dets.json = [{det_id, box, image_id, score}] # to cache/detections/ save_path = osp.join( save_dir, '%s_%s_%s_proposals.pkl' % (args.net_name, args.imdb_name, args.tag)) with open(save_path, 'wb') as f: pickle.dump(proposals, f)
def __init__(self): self.refer = REFER(dataset='refcoco+', splitBy='unc') self.image_ids = list(self.refer.getImgIds()) print('Found {} images.'.format(len(self.image_ids)))
# set path refer_data_path = 'data/refer/data' coco_data_path = 'data/coco/annotations' refer_coco_data_path = 'data/refer_coco/annotations' ####################################################################### # accumulate refer_trainval_ids and refer_test_ids ####################################################################### from refer import REFER train_image_ids = [] val_image_ids = [] test_image_ids = [] if params['dataset'] == 'refcoco': refer = REFER(refer_data_path, params['dataset'], 'unc') elif params['dataset'] == 'refcocog': refer = REFER(refer_data_path, params['dataset'], 'umd') else: raise ValueError('[%s] not considered yet.' % params['dataset']) for _, ref in refer.Refs.items(): if 'test' in ref['split']: test_image_ids.append(ref['image_id']) elif 'val' in ref['split']: val_image_ids.append(ref['image_id']) elif 'train' in ref['split']: train_image_ids.append(ref['image_id']) else: raise ValueError('No such split') train_image_ids = list(set(train_image_ids))
params = vars(args) # parse dataset_splitBy = params[ 'dataset_splitBy'] # in Lua, we simply use dataset denoting detaset_splitBy i, j = dataset_splitBy.find('_'), dataset_splitBy.rfind('_') dataset = dataset_splitBy[:i] splitBy = dataset_splitBy[i + 1:] if i == j else dataset_splitBy[i + 1:j] file_name = params['model_id'] + '_' + params['split'] + '_beam' + params[ 'beam_size'] + '.json' result_path = osp.join('cache', 'lang', dataset_splitBy, file_name) # load refer and refToEvaluation sys.path.insert(0, osp.join('pyutils', 'refer')) from refer import REFER refer = REFER(params['refer_dir'], dataset, splitBy) # load predictions Res = json.load(open(result_path, 'r'))['predictions'] # [{'ref_id', 'sent'}] # evaluate sys.path.insert(0, osp.join('pyutils', 'refer', 'evaluation')) from refEvaluation import RefEvaluation refEval = RefEvaluation(refer, Res) refEval.evaluate() overall = {} for metric, score in refEval.eval.items(): overall[metric] = score print overall if params['write_result'] > 0:
def __init__(self): self.refer = REFER(dataset='refcoco+', splitBy='unc') self.ref_ids = self.refer.getRefIds()