def __init__(self, data_dir="/projectnb/llamagrp/shawnlin/ref-exp-gen/dataset/refer2/refer/data", dataset="refcoco", split="train", splitBy="google", transforms=None):
assert (split in ["train", "val", "test"])
assert os.path.exists(data_dir), \
"cannot find folder {}, please download refcoco data into this folder".format(data_dir)
self.data_dir = data_dir
self.dataset = dataset
self.transforms = transforms
self.split = split
self.splitBy = splitBy
self.refer = REFER(self.data_dir, self.dataset, self.splitBy)
self.ref_ids = self.refer.getRefIds(split=self.split)[:]
#self.filter_non_overlap = filter_non_overlap
#self.filter_duplicate_rels = filter_duplicate_rels and self.split == 'train'
# read in dataset from a h5 file and a dict (json) file
#self.im_h5 = h5py.File(self.image_file, 'r')
self.info = json.load(open("/projectnb/llamagrp/shawnlin/ref-exp-gen/graph-rcnn.pytorch/datasets/vg_bm/VG-SGG-dicts.json", 'r'))
#self.im_refs = self.im_h5['images'] # image data reference
#im_scale = self.im_refs.shape[2]
# add background class
self.info['label_to_idx']['__background__'] = 0
self.class_to_ind = self.info['label_to_idx']
self.ind_to_classes = sorted(self.class_to_ind, key=lambda k:
self.class_to_ind[k])
#cfg.ind_to_class = self.ind_to_classes
self.predicate_to_ind = self.info['predicate_to_idx']
self.predicate_to_ind['__background__'] = 0
self.ind_to_predicates = sorted(self.predicate_to_ind, key=lambda k:
self.predicate_to_ind[k])
def main():
ref_data_root = '<path/to/refer/data/folder>'
all_refs = []
for dataset, split_bys in [
('refcoco', ['google', 'unc']),
('refcoco+', ['unc']),
('refcocog', ['google', 'umd'])
]:
for split_by in split_bys:
refer = REFER(ref_data_root, dataset, split_by)
for ref_id in refer.getRefIds():
ref = refer.Refs[ref_id]
ann = refer.refToAnn[ref_id]
ref['ann'] = ann
ref['dataset'] = dataset
ref['dataset_partition'] = split_by
all_refs.append(ref)
coco_annotations_file = '<path/to/instances_train2014.json>'
coco = COCO(coco_annotations_file)
ref_image_ids = set(x['image_id'] for x in all_refs)
coco_anns = {image_id: {'info': coco.imgs[image_id],
'anns': coco.imgToAnns[image_id]}
for image_id in ref_image_ids}
out_file = '<path/to/refcoco.json>'
with open(out_file, 'w') as f:
json.dump({'ref': all_refs, 'coco_anns': coco_anns}, f)
def build_coco_batches(dataset, setname, input_H, input_W):
im_dir = '/data/ryli/datasets/coco/images'
im_type = 'train2014'
vocab_file = '/data/ryli/rmi_phrasecut/data/vocabulary_Gref.txt'
data_folder = './data/' + dataset + '/' + setname + '_batch/'
data_prefix = dataset + '_' + setname
if not os.path.isdir(data_folder):
os.makedirs(data_folder)
if dataset == 'Gref':
refer = REFER('/data/ryli/rmi_phrasecut/external/refer/data',
dataset='refcocog',
splitBy='google')
elif dataset == 'unc':
refer = REFER('/data/ryli/rmi_phrasecut/external/refer/data',
dataset='refcoco',
splitBy='unc')
elif dataset == 'unc+':
refer = REFER('/data/ryli/rmi_phrasecut/external/refer/data',
dataset='refcoco+',
splitBy='unc')
else:
raise ValueError('Unknown dataset %s' % dataset)
refs = [
refer.Refs[ref_id] for ref_id in refer.Refs
if refer.Refs[ref_id]['split'] == setname
]
sent_data = []
encoder = setup_encoder()
for ref in refs:
for sentence in ref['sentences']:
sent_data.append(sentence['sent'].decode('latin-1').strip())
encodings = encoder.encode(sent_data)
n_batch = 0
for ref in refs:
im_name = 'COCO_' + im_type + '_' + str(ref['image_id']).zfill(12)
im = skimage.io.imread('%s/%s/%s.jpg' % (im_dir, im_type, im_name))
seg = refer.Anns[ref['ann_id']]['segmentation']
rle = cocomask.frPyObjects(seg, im.shape[0], im.shape[1])
mask = np.max(cocomask.decode(rle), axis=2).astype(np.float32)
if 'train' in setname:
mask = im_processing.resize_and_pad(mask, input_H, input_W)
for sentence in ref['sentences']:
print('saving batch %d' % (n_batch + 1))
sent = sentence['sent']
np.savez(file=data_folder + data_prefix + '_' + str(n_batch) +
'.npz',
im_name_batch=im_name,
encoding_batch=encodings[n_batch],
mask_batch=(mask > 0),
sent_batch=[sent])
n_batch += 1
class RefDataset(Dataset):
def __init__(self, split):
self.refer = REFER(dataset='refcoco+', splitBy='unc')
self.ref_ids = self.refer.getRefIds(split=split)
self.image_embeds = np.load(
os.path.join("data", "embeddings", "FINALImageEmbeddings.npy"))
self.image_ids = list(
np.load(os.path.join("data", "embeddings", "FINALImageIDs.npy")))
before_text_embeds = time.time()
self.text_embeds = np.concatenate(
(np.load(
os.path.join("data", "embeddings",
"FINALTextEmbeddings1of2.npy")),
np.load(
os.path.join("data", "embeddings",
"FINALTextEmbeddings2of2.npy"))),
axis=0)
after_text_embeds = time.time()
print("Text Embedding Time: ", after_text_embeds - before_text_embeds)
assert (len(self.text_embeds) == 141564)
assert (self.text_embeds[0].shape[1] == 3072)
print('Found {} referred objects in {} split.'.format(
len(self.ref_ids), split))
def __len__(self):
return len(self.ref_ids)
def __getitem__(self, i):
ref_id = self.ref_ids[i]
ref = self.refer.loadRefs(ref_id)[0]
image_id = ref['image_id']
image = self.refer.Imgs[image_id]
image_idx = self.image_ids.index(image_id)
image_embed = self.image_embeds[image_idx, :, :, :]
height = image['height']
width = image['width']
bound_box = torch.Tensor(self.refer.getRefBox(ref_id))
bound_box[0] /= width
bound_box[1] /= height
bound_box[2] /= width
bound_box[3] /= height
#bound_box = bound_box.unsqueeze(dim=0)
#whole_file_name = ref['file_name']
#file_name = whole_file_name[:whole_file_name.rfind("_")]+".jpg"
sent = random.choice(ref['sentences'])
ref_expr = sent['raw']
text_id = sent['sent_id']
text_idx = text_id
text_embed = torch.from_numpy(self.text_embeds[text_idx])
return image_embed, text_embed, bound_box
def build_coco_batches(dataset, setname, T, input_H, input_W):
im_dir = '/data/ryli/datasets/coco/images'
im_type = 'train2014'
vocab_file = './data/vocabulary_Gref.txt'
data_folder = './' + dataset + '/' + setname + '_batch/'
data_prefix = dataset + '_' + setname
if not os.path.isdir(data_folder):
os.makedirs(data_folder)
if dataset == 'Gref':
refer = REFER('./external/refer/data',
dataset='refcocog',
splitBy='google')
elif dataset == 'unc':
refer = REFER('./external/refer/data',
dataset='refcoco',
splitBy='unc')
elif dataset == 'unc+':
refer = REFER('./external/refer/data',
dataset='refcoco+',
splitBy='unc')
else:
raise ValueError('Unknown dataset %s' % dataset)
refs = [
refer.Refs[ref_id] for ref_id in refer.Refs
if refer.Refs[ref_id]['split'] == setname
]
vocab_dict = text_processing.load_vocab_dict_from_file(vocab_file)
n_batch = 0
for ref in refs:
im_name = 'COCO_' + im_type + '_' + str(ref['image_id']).zfill(12)
im = skimage.io.imread('%s/%s/%s.jpg' % (im_dir, im_type, im_name))
seg = refer.Anns[ref['ann_id']]['segmentation']
rle = cocomask.frPyObjects(seg, im.shape[0], im.shape[1])
mask = np.max(cocomask.decode(rle), axis=2).astype(np.float32)
if 'train' in setname:
im = skimage.img_as_ubyte(
im_processing.resize_and_pad(im, input_H, input_W))
mask = im_processing.resize_and_pad(mask, input_H, input_W)
if im.ndim == 2:
im = np.tile(im[:, :, np.newaxis], (1, 1, 3))
for sentence in ref['sentences']:
print('saving batch %d' % (n_batch + 1))
sent = sentence['sent']
text = text_processing.preprocess_sentence(sent, vocab_dict, T)
np.savez(file=data_folder + data_prefix + '_' + str(n_batch) +
'.npz',
text_batch=text,
im_batch=im,
mask_batch=(mask > 0),
sent_batch=[sent])
n_batch += 1
def __init__(self, refcoco_dir, refcoco_images_dir, split='val'):
self.image_dir = refcoco_images_dir
mattnet_maskrcnn_detections_path = refcoco_dir.joinpath(
'detections/refcocog_umd/res101_coco_minus_refer_notime_dets.json')
with open(mattnet_maskrcnn_detections_path) as f:
mattnet_maskrcnn_detections = json.load(f)
id2dets = {}
for det in mattnet_maskrcnn_detections:
image_id = det['image_id']
if image_id not in id2dets:
id2dets[image_id] = []
id2dets[image_id].append(det)
self.id2dets = id2dets
print('Load mattnet detections from', mattnet_maskrcnn_detections_path)
assert split in ['train', 'val', 'test']
workspace_dir = Path(__file__).resolve().parent.parent
refcoco_util_dir = workspace_dir.joinpath('refcoco_utils')
import sys
sys.path.append(str(refcoco_util_dir))
from refer import REFER
self.refer = REFER('refcocog', 'umd')
ref_ids = self.refer.getRefIds(split=split)
img_ids = []
image_fns = []
for ref_id in ref_ids:
ref = self.refer.Refs[ref_id]
img_id = ref['image_id']
if img_id not in img_ids:
img_ids.append(img_id)
fn_ann = ref['file_name']
# COCO_train2014_000000419645_398406.jpg
# COCO_train2014_000000419645.jpg
suffix = fn_ann.split('.')[-1]
fname = '_'.join(fn_ann.split('_')[:-1]) + '.' + suffix
image_fns.append(fname)
self.image_ids = img_ids
self.image_fns = image_fns
def run(params):
# parse
dataset_splitBy = params[
'dataset_splitBy'] # in Lua, we simply use dataset denoting detaset_splitBy
i, j = dataset_splitBy.find('_'), dataset_splitBy.rfind('_')
dataset = dataset_splitBy[:i]
splitBy = dataset_splitBy[i + 1:] if i == j else dataset_splitBy[i + 1:j]
model_id = params['model_id']
split = params['split']
cache_path = params['cache_path']
num_sents = params['num_sents']
# load refer
refer = REFER(REFER_DIR, dataset, splitBy)
# load lang result
Res = load_lang_result(cache_path, dataset_splitBy, model_id, split)
# sentToParse
sentToParse(Res, num_sents)
# parseToAtts
parseToAtts(Res, num_sents, dataset)
# compute scores
overall, refToEval = compute_score(Res, num_sents, refer, dataset)
pprint(overall)
# write results
if not osp.isdir(osp.join(cache_path, 'parse', dataset_splitBy)):
os.mkdir(osp.join(cache_path, 'parse', dataset_splitBy))
result_path = osp.join(cache_path, 'parse', dataset_splitBy,
'model_id' + model_id + '_' + split)
with open(result_path + '_out.json', 'w') as outfile:
json.dump({'overall': overall, 'refToEval': refToEval}, outfile)
def language_eval(pred, split, params):
sys.path.insert(0, osp.join('pyutils', 'refer2'))
if 'refcoco' in params['dataset']:
image_root = params['coco_image_root']
else:
image_root = params['gta_image_root']
from refer import REFER
refer = REFER(params['data_root'],
image_root,
params['dataset'],
params['splitBy'],
old_version=False)
sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation'))
from refEvaluation import RefEvaluation
eval_cider_r = params['dataset'] == 'refgta'
refEval = RefEvaluation(refer, pred, eval_cider_r=eval_cider_r)
refEval.evaluate()
overall = {}
for metric, score in refEval.eval.items():
overall[metric] = score
print(overall)
from crossEvaluation import CrossEvaluation
ceval = CrossEvaluation(refer, pred)
ceval.cross_evaluate()
ceval.make_ref_to_evals()
ref_to_evals = ceval.ref_to_evals
ceval.Xscore('CIDEr')
return overall
def main(opt):
# dataset_split_by
data_root, dataset, split_by = opt.data_root, opt.dataset, opt.split_by
# load refer
refer = REFER(data_root, dataset, split_by)
# transfer dataset to multi-branches
trees = transfer_dataset(refer)
# create vocab
word_vocab = build_vocab(trees, 'tokens', opt.word_count_threshold)
tag_vocab = build_vocab(trees, 'tags', opt.word_count_threshold)
dep_vocab = build_vocab(trees, 'deps', opt.word_count_threshold)
wtoi = {w: i for i, w in enumerate(word_vocab)}
ttoi = {w: i for i, w in enumerate(tag_vocab)}
dtoi = {w: i for i, w in enumerate(dep_vocab)}
# prepare refs, images, anns, sentences and write to json
refs, images, anns, sentences = prepare_data(refer, trees)
json.dump({'refs': refs,
'images': images,
'anns': anns,
'sentences': sentences,
'word_to_ix': wtoi,
'tag_to_ix': ttoi,
'dep_to_ix': dtoi,
'cat_to_ix': {cat_name: cat_id for cat_id, cat_name in refer.Cats.items()}, },
open(osp.join('data/feats', dataset + '_' + split_by, opt.output_json), 'w'))
print('%s written.' % osp.join('data/feats', opt.output_json))
torch.save(trees, osp.join('data/feats', dataset + '_' + split_by, opt.output_pth))
print('%s written.' % osp.join('data/feats', opt.output_pth))
def main(params):
dataset = params['dataset']
splitBy = params['splitBy']
split = params['split']
output_txt = params['output_txt']
# load refer
sys.path.insert(0, 'refer')
from refer import REFER
refer = REFER('data', dataset, splitBy)
# get refs that within the indicate split
refs = []
for ref_id, ref in refer.Refs.items():
if split in ref['split']:
# as long as ref['split'] contains string of split, e.g., testA or testB (contains 'test')
refs += [ref]
random.shuffle(refs)
# write object's image paths
txt_file = open(output_txt, 'w')
for ref in refs:
ref_img_path = ref['file_name']
txt_file.write(ref_img_path + '\n')
txt_file.close()
print '%s refs\' filenames written in %s.' % (len(refs), output_txt)
def main(params):
# dataset_splitBy
data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy']
# max_length
if params['max_length'] == None:
if params['dataset'] in ['refcoco', 'refclef', 'refcoco+']:
params['max_length'] = 10
params['topK'] = 50
elif params['dataset'] in ['refcocog']:
params['max_length'] = 20
params['topK'] = 50
else:
raise NotImplementedError
# mkdir and write json file
if not osp.isdir(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy)):
os.makedirs(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy))
# load refer
sys.path.insert(0, 'pyutils/refer')
from refer import REFER
refer = REFER(data_root, dataset, splitBy)
# create vocab
vocab, sentToFinal = build_vocab(refer, params)
itow = {i: w for i, w in enumerate(vocab)}
wtoi = {w: i for i, w in enumerate(vocab)}
# check sentence length
check_sentLength(sentToFinal)
# create attribute vocab
att2cnt, ref_to_att_wds = build_att_vocab(refer, params, ['r1','r2','r7'])
itoa = {i: a for i, a in enumerate(att2cnt.keys())}
atoi = {a: i for i, a in enumerate(att2cnt.keys())}
# prepare refs, images, anns, sentences
# and write json
refs, images, anns, sentences = prepare_json(refer, sentToFinal, ref_to_att_wds, params)
json.dump({'refs': refs,
'images': images,
'anns': anns,
'sentences': sentences,
'word_to_ix': wtoi,
'att_to_ix' : atoi,
'att_to_cnt': att2cnt,
'cat_to_ix': {cat_name: cat_id for cat_id, cat_name in refer.Cats.items()},
'label_length': params['max_length'],},
open(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy, params['output_json']), 'w'))
print('%s written.' % osp.join(ROOT_DIR, 'cache/prepro', params['output_json']))
# write h5 file which contains /sentences
f = h5py.File(osp.join(ROOT_DIR, 'cache/prepro', dataset+'_'+splitBy, params['output_h5']), 'w')
L = encode_captions(sentences, wtoi, params)
f.create_dataset("labels", dtype='int32', data=L)
f.close()
print('%s writtern.' % osp.join(ROOT_DIR, 'cache/prepro', params['output_h5']))
def __init__(self, refcoco_dir, refcoco_images_dir, coco_dir, split='val'):
self.image_dir = refcoco_images_dir
# coco_train_annFile = coco_dir.joinpath('annotations/instances_train2014.json')
# self.coco = COCO(coco_train_annFile)
assert split in ['train', 'val', 'test']
workspace_dir = Path(__file__).resolve().parent.parent
refcoco_util_dir = workspace_dir.joinpath('refcoco_utils')
import sys
sys.path.append(str(refcoco_util_dir))
from refer import REFER
self.refer = REFER('refcocog', 'umd')
ref_ids = self.refer.getRefIds(split=split)
id2dets = {}
img_ids = []
image_fns = []
for ref_id in ref_ids:
ref = self.refer.Refs[ref_id]
img_id = ref['image_id']
if img_id not in img_ids:
img_ids.append(img_id)
fn_ann = ref['file_name']
# COCO_train2014_000000419645_398406.jpg
# COCO_train2014_000000419645.jpg
suffix = fn_ann.split('.')[-1]
fname = '_'.join(fn_ann.split('_')[:-1]) + '.' + suffix
image_fns.append(fname)
detections = self.refer.imgToAnns[img_id]
id2dets[img_id] = detections
self.image_ids = img_ids
self.image_fns = image_fns
self.id2dets = id2dets
class RefDataset(Dataset):
def __init__(self):
self.refer = REFER(dataset='refcoco+', splitBy='unc')
self.ref_ids = self.refer.getRefIds()
def __len__(self):
return len(self.ref_ids)
def __getitem__(self, i):
ref_id = self.ref_ids[i]
ref = self.refer.loadRefs(ref_id)[0]
for sent in ref['sentences']:
s = sent['raw']
sid = sent['sent_id']
return s, sid
def main(params):
# dataset_splitBy
data_root, dataset, splitBy = params['data_root'], params[
'dataset'], params['splitBy']
# mkdir and write json file
if not osp.isdir('cache/prepro'):
os.mkdir('cache/prepro')
if not osp.isdir(osp.join('cache/prepro', dataset + '_' + splitBy)):
os.mkdir(osp.join('cache/prepro', dataset + '_' + splitBy))
if not osp.isdir(osp.join('models', dataset + '_' + splitBy)):
os.mkdir(
osp.join('models', dataset + '_' +
splitBy)) # we also mkdir model/dataset_splitBy here!
# load refer
sys.path.insert(0, osp.join(ROOT_DIR, 'pyutils/refer'))
from refer import REFER
refer = REFER(data_root, dataset, splitBy)
# create vocab
vocab, sentToFinal = build_vocab(refer, params)
itow = {i + 1: w for i, w in enumerate(vocab)} # lua 1-based
wtoi = {w: i + 1 for i, w in enumerate(vocab)} # lua 1-based
# check sentence length
check_sentLength(sentToFinal)
# prepare refs, images, anns, sentences
# and write json
refs, images, anns, sentences = prepare_json(refer, sentToFinal, params)
json.dump(
{
'refs': refs,
'images': images,
'anns': anns,
'sentences': sentences,
'ix_to_word': itow,
'word_to_ix': wtoi,
'ix_to_cat': refer.Cats
},
open(
osp.join('cache/prepro', dataset + '_' + splitBy,
params['output_json']), 'w'))
print('%s written.' % osp.join('cache/prepro', params['output_json']))
# write h5 file which contains /sentences
f = h5py.File(
osp.join('cache/prepro', dataset + '_' + splitBy, params['output_h5']),
'w')
seqz_L, zseq_L = encode_captions(sentences, wtoi, params)
f.create_dataset("seqz_labels", dtype='uint32', data=seqz_L)
f.create_dataset("zseq_labels", dtype='uint32', data=zseq_L)
f.close()
print('%s writtern.' % osp.join('cache/prepro', params['output_h5']))
def create_cache(**kwargs):
data_root = kwargs.get('data_root')
dataset = kwargs.get('dataset')
splitBy = kwargs.get('splitBy')
splits = kwargs.get('splits')
refer = REFER(data_root, dataset, splitBy)
# print stats about the given dataset
print('dataset [%s_%s] contains: ' % (dataset, splitBy))
ref_ids = refer.getRefIds()
image_ids = refer.getImgIds()
print('%s expressions for %s refs in %s images.' %
(len(refer.Sents), len(ref_ids), len(image_ids)))
checkpoint_dir = osp.join('cache', 'prepro', ds + "_" + splitBy)
if not osp.isdir(checkpoint_dir): os.makedirs(checkpoint_dir)
for split in splits + ['train']:
ref_ids = refer.getRefIds(split=split)
print('%s refs are in split [%s].' % (len(ref_ids), split))
#have to sample various sentences and their tokens from here.
data = []
for ref_id in ref_ids:
ref = refer.Refs[ref_id]
image_id = ref['image_id']
ref['image_info'] = refer.Imgs[image_id]
sentences = ref.pop('sentences')
ref.pop('sent_ids')
coco_boxes_info = refer.imgToAnns[image_id]
coco_boxes = [box_ann['bbox'] for box_ann in coco_boxes_info]
gtbox = refer.refToAnn[ref_id]['bbox']
for sentence in sentences:
entnew = copy.deepcopy(ref)
entnew['boxes'] = coco_boxes
entnew['sentence'] = sentence
entnew['gtbox'] = gtbox
data.append(entnew)
data_json = osp.join('cache/prepro', ds + "_" + splitBy,
split + '.json')
with open(data_json, 'w') as f:
json.dump(data, f)
def gen_ref_coco_data():
dataroot = "/projectnb/llamagrp/shawnlin/ref-exp-gen/dataset/refer2/refer/data"
dataset = "refcoco"
refer = REFER(dataroot, dataset, "google")
ref_ids = refer.getRefIds(split="test")[:]
print("total ref ids:", len(ref_ids))
for ref_id in ref_ids[:]:
ref = refer.Refs[ref_id]
img_id = ref["image_id"]
ann_id = ref["ann_id"]
img_path = os.path.join(refer.IMAGE_DIR,
refer.Imgs[img_id]["file_name"])
img = cv2.imread(img_path)
#im_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
ref_expr = "\n".join([s["raw"] for s in ref["sentences"]])
#print(ref_expr)
#print("img_id", img_id)
yield (img, ref_expr, img_id, ann_id, ref_id)
class ImageLoader(Dataset):
def __init__(self):
self.refer = REFER(dataset='refcoco+', splitBy='unc')
self.image_ids = list(self.refer.getImgIds())
print('Found {} images.'.format(len(self.image_ids)))
def __len__(self):
return len(self.image_ids)
def __getitem__(self, i):
image_id = self.image_ids[i]
image = self.refer.Imgs[image_id]
return image['file_name'], image['id']
def main(params):
# dataset_splitBy
data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy']
target_save_dir = osp.join(params['save_dir']+'/prepro', dataset+'_'+splitBy)
if params['old']:
params['data_json'] = 'old'+params['data_json']
params['data_h5'] = 'old'+params['data_h5']
if 'refcoco' in dataset:
image_root = params['coco_image_root']
else:
image_root = params['gta_image_root']
# mkdir and write json file
if not osp.isdir(target_save_dir):
os.makedirs(target_save_dir)
# load refer
sys.path.insert(0, osp.join(ROOT_DIR, 'pyutils/refer2'))
from refer import REFER
refer = REFER(data_root, image_root, dataset, splitBy, old_version=params['old'])
# create vocab
vocab, sentToFinal = build_vocab(refer, params)
itow = {i+1: w for i, w in enumerate(vocab)}# 0 : eos, len(itow)+1 : sos
wtoi = {w: i+1 for i, w in enumerate(vocab)}
# check sentence length
check_sentLength(sentToFinal)
# prepare refs, images, anns, sentences
# and write json
refs, images, anns, sentences = prepare_json(refer, sentToFinal, params)
json.dump({'refs': refs,
'images': images,
'anns': anns,
'sentences': sentences,
'ix_to_word': itow,
'word_to_ix': wtoi,
'ix_to_cat': refer.Cats,
'sents_info':refer.Sents
},
open(osp.join(target_save_dir, params['data_json']), 'w'))
print ('%s written.' % osp.join(target_save_dir, params['data_json']))
# write h5 file which contains /sentences
f = h5py.File(osp.join(target_save_dir, params['data_h5']), 'w')
seqz_L, zseq_L = encode_captions(sentences, wtoi, params)
f.create_dataset("seqz_labels", dtype='uint32', data=seqz_L)
f.create_dataset("zseq_labels", dtype='uint32', data=zseq_L)
f.close()
print ('%s writtern.' % osp.join(target_save_dir, params['data_h5']))
def __init__(self, split):
self.refer = REFER(dataset='refcoco+', splitBy='unc')
self.ref_ids = self.refer.getRefIds(split=split)
self.image_embeds = np.load(
os.path.join("data", "embeddings", "FINALImageEmbeddings.npy"))
self.image_ids = list(
np.load(os.path.join("data", "embeddings", "FINALImageIDs.npy")))
before_text_embeds = time.time()
self.text_embeds = np.concatenate(
(np.load(
os.path.join("data", "embeddings",
"FINALTextEmbeddings1of2.npy")),
np.load(
os.path.join("data", "embeddings",
"FINALTextEmbeddings2of2.npy"))),
axis=0)
after_text_embeds = time.time()
print("Text Embedding Time: ", after_text_embeds - before_text_embeds)
assert (len(self.text_embeds) == 141564)
assert (self.text_embeds[0].shape[1] == 3072)
print('Found {} referred objects in {} split.'.format(
len(self.ref_ids), split))
def getCider(pred, split, params):
sys.path.insert(0, osp.join('pyutils', 'refer2'))
from refer import REFER
refer = REFER(params['data_root'], '_', params['dataset'], params['splitBy'], old_version=False)
sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation'))
from refEvaluation import myRefEvaluation
eval_cider_r = params['dataset'] == 'refgta'
refEval = myRefEvaluation(refer, pred, eval_cider_r=eval_cider_r)
refEval.evaluate()
overall = {}
for metric, score in refEval.eval.items():
overall[metric] = score
cider_score = overall['CIDEr']
return cider_score
def load_sentToInfo(save_path='/data/unagi0/mtanaka/sentToInfo.json'):
if osp.exists(save_path):
with open(save_path) as f:
data = json.load(f)
else:
print('sent info converting...')
sys.path.insert(0, osp.join('./', 'pyutils/refer2'))
from refer import REFER
data_root = './dataset/anns/original' # contains refclef, refcoco, refcoco+, refcocog and images
dataset = 'refgta'
splitBy = 'utokyo'
refer = REFER(data_root, dataset, splitBy)
data = refer.Sents
with open(save_path, 'w') as f:
json.dump(data, f)
return data
def getWord2count():
referPath = osp.join(root_path, 'MAttNet3', 'data/')
refer = REFER(referPath, 'reverie4', 'reverie4')
sentToTokens = refer.sentToTokens
# count up the number of words
word2count = {}
for sent_id, tokens in sentToTokens.items():
for wd in tokens:
word2count[wd] = word2count.get(wd, 0) + 1
# add category words
category_names = list(refer.Cats.values()) + ['__background__']
for cat_name in category_names:
for wd in cat_name.split():
if wd not in word2count or word2count[wd] <= 5:
word2count[wd] = 1e5
return word2count
def main(params):
# dataset_splitBy
data_root, dataset, splitBy = params['data_root'], params['dataset'], params['splitBy']
# mkdir and write json file
if not osp.isdir(osp.join('cache/sub_obj_wds', dataset + '_' + splitBy)):
os.makedirs(osp.join('cache/sub_obj_wds', dataset + '_' + splitBy))
# load refer
sys.path.insert(0, 'pyutils/refer')
from refer import REFER
refer = REFER(data_root, dataset, splitBy)
# create subject and object
sub_obj_wds = get_sub_obj(refer, params, ['r1', 'r4', 'r5', 'r6'])
json.dump({'sub_obj_wds': sub_obj_wds},
open(osp.join('cache/sub_obj_wds', dataset + '_' + splitBy, params['output_json']), 'w'))
print('sub_obj_wds written!')
def main(params):
dataset_splitBy = params['dataset'] + '_' + params['splitBy']
if not osp.isdir('cache/parsed_sents/' + dataset_splitBy):
os.makedirs('cache/parsed_sents/' + dataset_splitBy)
# load refer
sys.path.insert(0, 'pyutils/refer')
from refer import REFER
refer = REFER(params['data_root'], params['dataset'], params['splitBy'])
# parse sents
sents = refer.Sents.values()
parse_sents(sents, params)
# save
with open(osp.join('cache/parsed_sents/' + dataset_splitBy, 'sents.json'),
'w') as io:
json.dump(sents, io)
parser.add_argument('--split', default='testA')
parser.add_argument('--model_id', default='0')
parser.add_argument('--beam_size', default='10', type=str)
parser.add_argument('--write_result',
default=1,
help='write result into cache_path')
parser.add_argument(
'--refer_dir',
default='new_data',
help='data or new_data, refer directory that stores annotations.')
# argparse
args = parser.parse_args()
params = vars(args)
print 'parsed input parameters:'
print json.dumps(params, indent=2)
# load refer
sys.path.insert(0, osp.join('pyutils', 'refer2'))
from refer import REFER
refer = REFER(params['refer_dir'], params['dataset'], params['splitBy'])
sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation'))
from crossEvaluation import CrossEvaluation
sys.path.insert(0, osp.join('pyutils', 'refer2', 'evaluation'))
from refEvaluation import RefEvaluation
# call main
main(params)
def setEvalRefs(self):
self.evalRefs = [eval for refId, eval in self.refToEval.items()]
if __name__ == '__main__':
import os.path as osp
import sys
ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..'))
sys.path.insert(0, osp.join(ROOT_DIR, 'lib', 'datasets'))
from refer import REFER
# load refer of dataset
dataset = 'refcoco'
refer = REFER(dataset, splitBy = 'google')
# mimic some Res
val_refIds = refer.getRefIds(split='test')
ref_id = 49767
print "GD: %s" % refer.Refs[ref_id]['sentences']
Res = [{'ref_id': ref_id, 'sent': 'left bottle'}]
# evaluate some refer expressions
refEval = RefEvaluation(refer, Res)
refEval.evaluate()
# print output evaluation scores
for metric, score in refEval.eval.items():
print '%s: %.3f'%(metric, score)
import numpy as np
import os
from refer import REFER
import os.path as osp
import cv2
import argparse
parser = argparse.ArgumentParser(description='Data preparation')
parser.add_argument('--data_root', type=str) # contains refclef, refcoco, refcoco+, refcocog and images
parser.add_argument('--output_dir', type=str)
parser.add_argument('--dataset', type=str, choices=['refcoco', 'refcoco+','refcocog','refclef'],default='refcoco')
parser.add_argument('--split', type=str,default='umd')
parser.add_argument('--generate_mask', action='store_true')
args = parser.parse_args()
# data_root # contains refclef, refcoco, refcoco+, refcocog and images
refer = REFER(args.data_root, args.dataset, args.split)
print ('dataset [%s_%s] contains: ' % (args.dataset, args.split))
ref_ids = refer.getRefIds()
image_ids = refer.getImgIds()
print ('%s expressions for %s refs in %s images.' % (len(refer.Sents), len(ref_ids), len(image_ids)))
print('\nAmong them:')
if args.dataset == 'refclef':
if args.split == 'unc':
splits = ['train', 'val', 'testA','testB','testC']
else:
splits = ['train', 'val', 'test']
elif args.dataset == 'refcoco':
splits = ['train', 'val', 'testA', 'testB']
elif args.dataset == 'refcoco+':
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', default='refcoco', help='refcoco or refcocog')
args = parser.parse_args()
params = vars(args)
# set path
refer_data_path = 'data/refer/data'
coco_data_path = 'data/coco/annotations'
refer_coco_data_path = 'data/refer_coco/annotations'
#######################################################################
# accumulate refer_trainval_ids and refer_test_ids
#######################################################################
from refer import REFER
refcoco = REFER(refer_data_path, 'refcoco', 'unc')
refcoco_excluded_image_ids = []
for _, ref in refcoco.Refs.items():
if ref['split'] in ['testA', 'testB', 'val']:
refcoco_excluded_image_ids += [ref['image_id']]
refcoco_excluded_image_ids = list(set(refcoco_excluded_image_ids))
print('In refcoco_unc, %s valtest image_ids will be excluded.' %
len(refcoco_excluded_image_ids))
refcocog = REFER(refer_data_path, 'refcocog', 'umd')
refcocog_excluded_image_ids = []
for _, ref in refcocog.Refs.items():
if ref['split'] in ['test', 'val']:
refcocog_excluded_image_ids += [ref['image_id']]
refcocog_excluded_image_ids = list(set(refcocog_excluded_image_ids))
print('In refcocog_umd, %s valtest image_ids will be excluded.' %
def main(args):
# Image Directory
params = vars(args)
dataset_splitBy = params['dataset'] + '_' + params['splitBy']
if 'coco' or 'combined' in dataset_splitBy:
IMAGE_DIR = 'data/images/mscoco/images/train2014'
elif 'clef' in dataset_splitBy:
IMAGE_DIR = 'data/images/saiapr_tc-12'
else:
print('No image directory prepared for ', args.dataset)
sys.exit(0)
# make save dir
save_dir = osp.join('cache/detections', dataset_splitBy)
if not osp.isdir(save_dir):
os.makedirs(save_dir)
print(save_dir)
# get mrcn instance
mrcn = inference.Inference(args)
imdb = mrcn.imdb
# import refer
from refer import REFER
data_root, dataset, splitBy = params['data_root'], params[
'dataset'], params['splitBy']
refer = REFER(data_root, dataset, splitBy)
cat_name_to_cat_ix = {
category_name: category_id
for category_id, category_name in refer.Cats.items()
}
# detect and prepare dets.json
proposals = []
det_id = 0
cnt = 0
# # TEMPS DEBUG
# # os.makedirs('cache/old_internals')
# img_path = '/home/mwb/Datasets/mscoco/images/train2014/COCO_train2014_000000581857.jpg'
# scores, boxes = mrcn.predict(img_path)
# image_feat = mrcn.net._predictions['__temp_net_conv'].data.cpu().numpy()
# roi_feats = mrcn.net._predictions['__temp_pool5'].data.cpu().numpy()
# rois = mrcn.net._predictions['__temp_rois'].data.cpu().numpy()[:,1:]
# head_feats = mrcn.net._predictions['__temp_head_feats'].data.cpu().numpy()
# head_pool = mrcn.net._predictions['__temp_head_pool'].data.cpu().numpy()
# print(image_feat.shape, roi_feats.shape, rois.shape, head_feats.shape, head_pool.shape)
# np.save('cache/old_internals/image_feat.npy', image_feat)
# np.save('cache/old_internals/roi_feats.npy', roi_feats)
# np.save('cache/old_internals/rois.npy', rois)
# np.save('cache/old_internals/head_feats.npy', head_feats)
# np.save('cache/old_internals/head_pool.npy', head_pool)
for image_id, image in refer.Imgs.items():
file_name = image['file_name']
img_path = osp.join(IMAGE_DIR, file_name)
# predict
scores, boxes = mrcn.predict(img_path)
rois = mrcn.net._predictions['rois'].data.cpu().numpy(
)[:, 1:] / mrcn._scale
cnt += 1
print('%s/%s done.' % (cnt, len(refer.Imgs)))
info = {
'image_id': image_id,
'rois': rois,
'scores': scores,
'boxes': boxes,
'roi_scores':
mrcn.net._predictions['__roi_scores'].data.cpu().numpy()
}
torch.cuda.empty_cache()
proposals.append(info)
# save dets.json = [{det_id, box, image_id, score}]
# to cache/detections/
save_path = osp.join(
save_dir,
'%s_%s_%s_proposals.pkl' % (args.net_name, args.imdb_name, args.tag))
with open(save_path, 'wb') as f:
pickle.dump(proposals, f)
def setEvalRefs(self):
self.evalRefs = [eval for refId, eval in self.refToEval.items()]
if __name__ == '__main__':
import os.path as osp
import sys
ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..'))
sys.path.insert(0, osp.join(ROOT_DIR, 'lib', 'datasets'))
from refer import REFER
# load refer of dataset
dataset = 'refcoco'
refer = REFER(dataset, splitBy = 'google')
# mimic some Res
val_refIds = refer.getRefIds(split='test')
ref_id = 49767
print "GD: %s" % refer.Refs[ref_id]['sentences']
Res = [{'ref_id': ref_id, 'sent': 'left bottle'}]
# evaluate some refer expressions
refEval = RefEvaluation(refer, Res)
refEval.evaluate()
# print output evaluation scores
for metric, score in refEval.eval.items():
print '%s: %.3f'%(metric, score)
# set path
refer_data_path = 'data/refer/data'
coco_data_path = 'data/coco/annotations'
refer_coco_data_path = 'data/refer_coco/annotations'
#######################################################################
# accumulate refer_trainval_ids and refer_test_ids
#######################################################################
from refer import REFER
train_image_ids = []
val_image_ids = []
test_image_ids = []
if params['dataset'] == 'refcoco':
refer = REFER(refer_data_path, params['dataset'], 'unc')
elif params['dataset'] == 'refcocog':
refer = REFER(refer_data_path, params['dataset'], 'umd')
else:
raise ValueError('[%s] not considered yet.' % params['dataset'])
for _, ref in refer.Refs.items():
if 'test' in ref['split']:
test_image_ids.append(ref['image_id'])
elif 'val' in ref['split']:
val_image_ids.append(ref['image_id'])
elif 'train' in ref['split']:
train_image_ids.append(ref['image_id'])
else:
raise ValueError('No such split')
train_image_ids = list(set(train_image_ids))
