def __init__(self, dataset: VQADataset):
super().__init__()
self.raw_dataset = dataset
if args.tiny:
topk = TINY_IMG_NUM
elif args.fast:
topk = FAST_IMG_NUM
else:
topk = None
# Loading detection features to img_data
img_data = []
if 'train' in dataset.splits:
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/train2014_obj36.tsv',
topk=topk))
if 'valid' in dataset.splits:
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'minival' in dataset.splits:
# minival is 5K images in the intersection of MSCOCO valid and VG,
# which is used in evaluating LXMERT pretraining performance.
# It is saved as the top 5K features in val2014_obj36.tsv
if topk is None:
topk = 5000
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'nominival' in dataset.splits:
# nominival = mscoco val - minival
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'test' in dataset.name: # If dataset contains any test split
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/test2015_obj36.tsv',
topk=topk))
# Convert img list to dict
self.imgid2img = {}
for img_datum in img_data:
self.imgid2img[img_datum['img_id']] = img_datum
# Only kept the data with loaded image features
self.data = []
for datum in tqdm(self.raw_dataset.data,
ascii=True,
desc="Loading Image features"):
if datum['img_id'] in self.imgid2img:
self.data.append(datum)
self.raw_dataset.data = self.data
print("Use %d data in torch dataset" % (len(self.data)), flush=True)
print(flush=True)
self.atleast1 = False
def __init__(self,
dataset,
transform,
img_path=None,
tsv_path=None,
mode='train',
use_tsv=False,
num_boxes=None):
'''
dataset: rawdataset with flickr image and sentences
img_path: path for images to be loaded from
mode: 'training' or 'dev' or 'test'
img_list_fname: the file containing the list of images to be fed to the system
'''
super().__init__()
self.raw_dataset = dataset
self.img_data_map = {dp['image']: dp for dp in self.raw_dataset.data}
self.img_path = img_path
self.transform = transform
self.mode = mode
self.tsv = use_tsv
self.tsv_path = tsv_path
tsv_file = os.path.join(self.tsv_path, '{}.tsv'.format(mode))
img_data = load_obj_tsv(tsv_file, topk=30, use_tsv=self.tsv)
self.imgid2img = {}
for img_datum in img_data:
self.imgid2img[img_datum['img_id'] + '.jpg'] = img_datum
self.num_boxes = num_boxes
self.non_label = []
self.features = self.convert_example_to_feature()
def load_data(self, name, number):
# if name == 'testdev':
# path = "data/vg_gqa_imgfeat/gqa_testdev_obj36.tsv"
# else:
# path = "data/vg_gqa_imgfeat/vg_gqa_obj36.tsv"
path = os.path.join(IMGFEAT_ROOT, "vg_gqa_obj36-36.tsv")
key = "%s_%d" % (path, number)
if key not in self.key2data:
self.key2data[key] = load_obj_tsv(path, topk=number)
return self.key2data[key]
def __init__(self, dataset: NLVR2Dataset):
super().__init__()
self.raw_dataset = dataset
topk = 30
# Loading detection features to img_data
test_image_features_path = r"C:\Users\asmit\Desktop\Guided Research\lxmert\data\mscoco_imgfeat\val2014_obj36.tsv"
img_data = []
img_data.extend(load_obj_tsv(test_image_features_path, topk=topk))
# Creating a dictionary with img_id as the key for all the objects in tsv
self.imgid2img = {}
for img_datum in img_data:
self.imgid2img[img_datum['img_id']] = img_datum
count = len(self.imgid2img)
print("Before precossing keys : ", count)
# Change the image_ids so that they match with the ids of annotations
for key in list(self.imgid2img.keys()):
if count > 0:
temp_feat = self.imgid2img[key]
temp_id = key[13:]
temp_id = str(int(temp_id))
self.imgid2img[temp_id] = temp_feat
del self.imgid2img[key]
count -= 1
else:
break
print("After precossing keys : ", len(self.imgid2img))
# Filter out the dataset
# data is a list with annotations(only list of sentences) and image_features in two separate lists
self.data = {}
temp_data = {}
count = 0
count_temp = 0
for each_datum in self.raw_dataset.data:
str_key = str(each_datum)
if str_key in self.imgid2img:
for each_sent in self.raw_dataset.data[each_datum]:
temp_img_info = {}
temp_img_info.update(self.imgid2img[str_key])
sent = ""
sent = sent + each_sent
temp_img_info['uid'] = str_key
temp_img_info['sent'] = sent
temp_data[count] = temp_img_info
count += 1
count_temp += 1
self.data = temp_data
print("Use %d data in torch dataset" % (len(self.data)))
n_items = list(islice(self.data.items(), 10))
def __init__(self, dataset: VQADataset):
super().__init__()
self.raw_dataset = dataset
if args.tiny:
topk = TINY_IMG_NUM
elif args.fast:
topk = FAST_IMG_NUM
else:
topk = None
# Loading detection features to img_data
img_data = []
if 'train' in dataset.splits:
img_data.extend(
load_obj_tsv(os.path.join(DATA_ROOT,
'mutant_imgfeat/train_obj36.tsv'),
topk=topk))
img_data.extend(
load_obj_tsv(os.path.join(DATA_ROOT,
'mutant_imgfeat/valid_obj36.tsv'),
topk=topk))
img_data.extend(
load_obj_tsv(os.path.join(
DATA_ROOT, 'mscoco_imgfeat/train2014_obj36.tsv'),
topk=topk))
img_data.extend(
load_obj_tsv(os.path.join(DATA_ROOT,
'mscoco_imgfeat/val2014_obj36.tsv'),
topk=topk))
if 'valid' in dataset.splits:
print("VALID")
img_data.extend(
load_obj_tsv(os.path.join(
DATA_ROOT, 'mscoco_imgfeat/train2014_obj36.tsv'),
topk=topk))
img_data.extend(
load_obj_tsv(os.path.join(DATA_ROOT,
'mscoco_imgfeat/val2014_obj36.tsv'),
topk=topk))
if 'minival' in dataset.splits:
# minival is 5K images in the intersection of MSCOCO valid and VG,
# which is used in evaluating LXMERT pretraining performance.
# It is saved as the top 5K features in val2014_obj36.tsv
img_data.extend(
load_obj_tsv(os.path.join(
DATA_ROOT, 'mscoco_imgfeat/train2014_obj36.tsv'),
topk=topk))
img_data.extend(
load_obj_tsv(os.path.join(DATA_ROOT,
'mscoco_imgfeat/val2014_obj36.tsv'),
topk=topk))
if 'test' in dataset.name: # If dataset contains any test split
# img_data.extend(load_obj_tsv('data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk))
# img_data.extend(load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk))
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mscoco_imgfeat/test2015_obj36.tsv',
topk=topk))
# Convert img list to dict
self.imgid2img = {}
for img_datum in img_data:
self.imgid2img[img_datum['img_id']] = img_datum
### img_data IS DATA FROM THE TSV
### self.raw_dataset.data IS FROM THE JSON
# Only kept the data with loaded image features
self.data = []
valid_imgs = []
# count_orig = 0
# count_mutant = 0
# count_orig_not_key = 0
# count_orig_not_in_tsv = 0
for datum in self.raw_dataset.data:
# print("datum.keys()", datum.keys())
if 'minival' in dataset.splits:
if datum['img_id'] in self.imgid2img:
self.data.append(datum)
valid_imgs.append(datum['img_id'])
elif 'test' in dataset.splits:
if datum['img_id'] in self.imgid2img:
self.data.append(datum)
valid_imgs.append(datum['img_id'])
else:
if 'img_id' in datum and 'orig_img_id' in datum:
if datum['img_id'] in self.imgid2img and datum[
'orig_img_id'] in self.imgid2img:
self.data.append(datum)
valid_imgs.append(datum['img_id'])
valid_imgs.append(datum['orig_img_id'])
# if datum['orig_img_id'] in self.imgid2img:
# self.data.append(datum)
# valid_imgs.append(datum['orig_img_id'])
# count_orig += 1
# else:
# count_orig_not_in_tsv += 1
# else:
# count_orig_not_key += 1
# print("count_mutant, count_orig, count_orig_not_in_tsv, count_orig_not_key",
# count_mutant, count_orig, count_orig_not_in_tsv, count_orig_not_key,
# flush=True)
# self.raw_dataset.data=self.data
# Only keep images with loaded data
valid_imgs = set(valid_imgs)
all_imgs = set(self.imgid2img)
invalid_imgs = all_imgs - valid_imgs
for unwanted_key in invalid_imgs:
del self.imgid2img[unwanted_key]
print("Use %d data in torch dataset" % (len(self.data)), flush=True)
print(flush=True)
ans_embed = np.load(
"/home/tgokhale/code/vqa_mutant/mutant/answer_embs.npy") + 1e-8
ans_embed = torch.tensor(ans_embed)
self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1)
self.nlp = spacy.load('en_core_web_lg')
def __init__(self, dataset: VQADataset):
super().__init__()
self.raw_dataset = dataset
if args.tiny:
topk = TINY_IMG_NUM
elif args.fast:
topk = FAST_IMG_NUM
else:
topk = None
# Loading detection features to img_data
img_data = []
if 'train' in dataset.splits:
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mutant_imgfeat/train_obj36.tsv',
topk=topk))
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv',
topk=topk))
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv',
topk=topk))
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'valid' in dataset.splits:
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'minival' in dataset.splits:
# minival is 5K images in the intersection of MSCOCO valid and VG,
# which is used in evaluating LXMERT pretraining performance.
# It is saved as the top 5K features in val2014_obj36.tsv
# if topk is None:
# topk = 5000
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv',
topk=topk))
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'nominival' in dataset.splits:
# nominival = mscoco val - minival
img_data.extend(
load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv',
topk=topk))
if 'test' in dataset.name: # If dataset contains any test split
img_data.extend(
load_obj_tsv(
'/data/datasets/vqa_mutant/data/mscoco_imgfeat/test2015_obj36.tsv',
topk=topk))
# Convert img list to dict
self.imgid2img = {}
for img_datum in img_data:
self.imgid2img[img_datum['img_id']] = img_datum
# Only kept the data with loaded image features
self.data = []
for datum in tqdm(self.raw_dataset.data,
ascii=True,
desc="Loading Image features"):
if datum['img_id'] in self.imgid2img:
self.data.append(datum)
self.raw_dataset.data = self.data
print("Use %d data in torch dataset" % (len(self.data)), flush=True)
print(flush=True)
ans_embed = np.load(
"/data/datasets/vqa_mutant/data/vqa/mutant_l2a/answer_embs.npy"
) + 1e-8
ans_embed = torch.tensor(ans_embed)
self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1)
self.nlp = spacy.load('en_core_web_lg')
def __init__(self, dataset: VQADataset):
super().__init__()
self.raw_dataset = dataset
if args.tiny:
topk = TINY_IMG_NUM
elif args.fast:
topk = FAST_IMG_NUM
else:
topk = None
if args.mutant:
print("Mutant Data to be loaded",flush=True)
# Loading detection features to img_data
img_data = []
if 'train' in dataset.splits:
print("TRAIN")
if args.mutant:
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/train_obj36.tsv', topk=topk))
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk))
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk))
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk))
if 'valid' in dataset.splits:
print("VALID")
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk))
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk))
if 'minival' in dataset.splits:
print("MINIVAL")
# minival is 5K images in the intersection of MSCOCO valid and VG,
# which is used in evaluating LXMERT pretraining performance.
# It is saved as the top 5K features in val2014_obj36.tsv
# if topk is None:
# topk = 50000
if args.mutant:
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk))
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk))
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk))
# img_data.extend(load_obj_tsv('/scratch/tgokhale/mutant_notcrowd/valid_obj36.tsv', topk=100))
if 'nominival' in dataset.splits:
print("NOMINIVAL")
if 'test' in dataset.name: # If dataset contains any test split
print("TEST")
img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/test2015_obj36.tsv', topk=topk))
# Convert img list to dict
self.imgid2img = {}
for img_datum in img_data:
self.imgid2img[img_datum['img_id']] = img_datum
# Only kept the data with loaded image features
self.data = []
valid_imgs = []
# for datum in tqdm(self.raw_dataset.data,ascii=True,desc="Loading Image features"):
for datum in self.raw_dataset.data:
if datum['img_id'] in self.imgid2img:
self.data.append(datum)
valid_imgs.append(datum['img_id'])
# orig and mutant
self.raw_dataset.data = self.data
# Only keep images with loaded data
valid_imgs = set(valid_imgs)
all_imgs = set(self.imgid2img)
invalid_imgs = all_imgs - valid_imgs
for unwanted_key in invalid_imgs:
del self.imgid2img[unwanted_key]
self.color_qtypes = ['what color is the', 'what color are the',
'what color', 'what color is',
'what is the color of the']
print("Use %d data in torch dataset" % (len(self.data)),flush=True)
print(flush=True)
