def __init__(self, dataset: VQADataset): super().__init__() self.raw_dataset = dataset if args.tiny: topk = TINY_IMG_NUM elif args.fast: topk = FAST_IMG_NUM else: topk = None # Loading detection features to img_data img_data = [] if 'train' in dataset.splits: img_data.extend( load_obj_tsv('data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk)) if 'valid' in dataset.splits: img_data.extend( load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'minival' in dataset.splits: # minival is 5K images in the intersection of MSCOCO valid and VG, # which is used in evaluating LXMERT pretraining performance. # It is saved as the top 5K features in val2014_obj36.tsv if topk is None: topk = 5000 img_data.extend( load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'nominival' in dataset.splits: # nominival = mscoco val - minival img_data.extend( load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'test' in dataset.name: # If dataset contains any test split img_data.extend( load_obj_tsv('data/mscoco_imgfeat/test2015_obj36.tsv', topk=topk)) # Convert img list to dict self.imgid2img = {} for img_datum in img_data: self.imgid2img[img_datum['img_id']] = img_datum # Only kept the data with loaded image features self.data = [] for datum in tqdm(self.raw_dataset.data, ascii=True, desc="Loading Image features"): if datum['img_id'] in self.imgid2img: self.data.append(datum) self.raw_dataset.data = self.data print("Use %d data in torch dataset" % (len(self.data)), flush=True) print(flush=True) self.atleast1 = False
def __init__(self, dataset, transform, img_path=None, tsv_path=None, mode='train', use_tsv=False, num_boxes=None): ''' dataset: rawdataset with flickr image and sentences img_path: path for images to be loaded from mode: 'training' or 'dev' or 'test' img_list_fname: the file containing the list of images to be fed to the system ''' super().__init__() self.raw_dataset = dataset self.img_data_map = {dp['image']: dp for dp in self.raw_dataset.data} self.img_path = img_path self.transform = transform self.mode = mode self.tsv = use_tsv self.tsv_path = tsv_path tsv_file = os.path.join(self.tsv_path, '{}.tsv'.format(mode)) img_data = load_obj_tsv(tsv_file, topk=30, use_tsv=self.tsv) self.imgid2img = {} for img_datum in img_data: self.imgid2img[img_datum['img_id'] + '.jpg'] = img_datum self.num_boxes = num_boxes self.non_label = [] self.features = self.convert_example_to_feature()
def load_data(self, name, number): # if name == 'testdev': # path = "data/vg_gqa_imgfeat/gqa_testdev_obj36.tsv" # else: # path = "data/vg_gqa_imgfeat/vg_gqa_obj36.tsv" path = os.path.join(IMGFEAT_ROOT, "vg_gqa_obj36-36.tsv") key = "%s_%d" % (path, number) if key not in self.key2data: self.key2data[key] = load_obj_tsv(path, topk=number) return self.key2data[key]
def __init__(self, dataset: NLVR2Dataset): super().__init__() self.raw_dataset = dataset topk = 30 # Loading detection features to img_data test_image_features_path = r"C:\Users\asmit\Desktop\Guided Research\lxmert\data\mscoco_imgfeat\val2014_obj36.tsv" img_data = [] img_data.extend(load_obj_tsv(test_image_features_path, topk=topk)) # Creating a dictionary with img_id as the key for all the objects in tsv self.imgid2img = {} for img_datum in img_data: self.imgid2img[img_datum['img_id']] = img_datum count = len(self.imgid2img) print("Before precossing keys : ", count) # Change the image_ids so that they match with the ids of annotations for key in list(self.imgid2img.keys()): if count > 0: temp_feat = self.imgid2img[key] temp_id = key[13:] temp_id = str(int(temp_id)) self.imgid2img[temp_id] = temp_feat del self.imgid2img[key] count -= 1 else: break print("After precossing keys : ", len(self.imgid2img)) # Filter out the dataset # data is a list with annotations(only list of sentences) and image_features in two separate lists self.data = {} temp_data = {} count = 0 count_temp = 0 for each_datum in self.raw_dataset.data: str_key = str(each_datum) if str_key in self.imgid2img: for each_sent in self.raw_dataset.data[each_datum]: temp_img_info = {} temp_img_info.update(self.imgid2img[str_key]) sent = "" sent = sent + each_sent temp_img_info['uid'] = str_key temp_img_info['sent'] = sent temp_data[count] = temp_img_info count += 1 count_temp += 1 self.data = temp_data print("Use %d data in torch dataset" % (len(self.data))) n_items = list(islice(self.data.items(), 10))
def __init__(self, dataset: VQADataset): super().__init__() self.raw_dataset = dataset if args.tiny: topk = TINY_IMG_NUM elif args.fast: topk = FAST_IMG_NUM else: topk = None # Loading detection features to img_data img_data = [] if 'train' in dataset.splits: img_data.extend( load_obj_tsv(os.path.join(DATA_ROOT, 'mutant_imgfeat/train_obj36.tsv'), topk=topk)) img_data.extend( load_obj_tsv(os.path.join(DATA_ROOT, 'mutant_imgfeat/valid_obj36.tsv'), topk=topk)) img_data.extend( load_obj_tsv(os.path.join( DATA_ROOT, 'mscoco_imgfeat/train2014_obj36.tsv'), topk=topk)) img_data.extend( load_obj_tsv(os.path.join(DATA_ROOT, 'mscoco_imgfeat/val2014_obj36.tsv'), topk=topk)) if 'valid' in dataset.splits: print("VALID") img_data.extend( load_obj_tsv(os.path.join( DATA_ROOT, 'mscoco_imgfeat/train2014_obj36.tsv'), topk=topk)) img_data.extend( load_obj_tsv(os.path.join(DATA_ROOT, 'mscoco_imgfeat/val2014_obj36.tsv'), topk=topk)) if 'minival' in dataset.splits: # minival is 5K images in the intersection of MSCOCO valid and VG, # which is used in evaluating LXMERT pretraining performance. # It is saved as the top 5K features in val2014_obj36.tsv img_data.extend( load_obj_tsv(os.path.join( DATA_ROOT, 'mscoco_imgfeat/train2014_obj36.tsv'), topk=topk)) img_data.extend( load_obj_tsv(os.path.join(DATA_ROOT, 'mscoco_imgfeat/val2014_obj36.tsv'), topk=topk)) if 'test' in dataset.name: # If dataset contains any test split # img_data.extend(load_obj_tsv('data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk)) # img_data.extend(load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mscoco_imgfeat/test2015_obj36.tsv', topk=topk)) # Convert img list to dict self.imgid2img = {} for img_datum in img_data: self.imgid2img[img_datum['img_id']] = img_datum ### img_data IS DATA FROM THE TSV ### self.raw_dataset.data IS FROM THE JSON # Only kept the data with loaded image features self.data = [] valid_imgs = [] # count_orig = 0 # count_mutant = 0 # count_orig_not_key = 0 # count_orig_not_in_tsv = 0 for datum in self.raw_dataset.data: # print("datum.keys()", datum.keys()) if 'minival' in dataset.splits: if datum['img_id'] in self.imgid2img: self.data.append(datum) valid_imgs.append(datum['img_id']) elif 'test' in dataset.splits: if datum['img_id'] in self.imgid2img: self.data.append(datum) valid_imgs.append(datum['img_id']) else: if 'img_id' in datum and 'orig_img_id' in datum: if datum['img_id'] in self.imgid2img and datum[ 'orig_img_id'] in self.imgid2img: self.data.append(datum) valid_imgs.append(datum['img_id']) valid_imgs.append(datum['orig_img_id']) # if datum['orig_img_id'] in self.imgid2img: # self.data.append(datum) # valid_imgs.append(datum['orig_img_id']) # count_orig += 1 # else: # count_orig_not_in_tsv += 1 # else: # count_orig_not_key += 1 # print("count_mutant, count_orig, count_orig_not_in_tsv, count_orig_not_key", # count_mutant, count_orig, count_orig_not_in_tsv, count_orig_not_key, # flush=True) # self.raw_dataset.data=self.data # Only keep images with loaded data valid_imgs = set(valid_imgs) all_imgs = set(self.imgid2img) invalid_imgs = all_imgs - valid_imgs for unwanted_key in invalid_imgs: del self.imgid2img[unwanted_key] print("Use %d data in torch dataset" % (len(self.data)), flush=True) print(flush=True) ans_embed = np.load( "/home/tgokhale/code/vqa_mutant/mutant/answer_embs.npy") + 1e-8 ans_embed = torch.tensor(ans_embed) self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1) self.nlp = spacy.load('en_core_web_lg')
def __init__(self, dataset: VQADataset): super().__init__() self.raw_dataset = dataset if args.tiny: topk = TINY_IMG_NUM elif args.fast: topk = FAST_IMG_NUM else: topk = None # Loading detection features to img_data img_data = [] if 'train' in dataset.splits: img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mutant_imgfeat/train_obj36.tsv', topk=topk)) img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk)) img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk)) img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'valid' in dataset.splits: img_data.extend( load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'minival' in dataset.splits: # minival is 5K images in the intersection of MSCOCO valid and VG, # which is used in evaluating LXMERT pretraining performance. # It is saved as the top 5K features in val2014_obj36.tsv # if topk is None: # topk = 5000 img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk)) img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'nominival' in dataset.splits: # nominival = mscoco val - minival img_data.extend( load_obj_tsv('data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'test' in dataset.name: # If dataset contains any test split img_data.extend( load_obj_tsv( '/data/datasets/vqa_mutant/data/mscoco_imgfeat/test2015_obj36.tsv', topk=topk)) # Convert img list to dict self.imgid2img = {} for img_datum in img_data: self.imgid2img[img_datum['img_id']] = img_datum # Only kept the data with loaded image features self.data = [] for datum in tqdm(self.raw_dataset.data, ascii=True, desc="Loading Image features"): if datum['img_id'] in self.imgid2img: self.data.append(datum) self.raw_dataset.data = self.data print("Use %d data in torch dataset" % (len(self.data)), flush=True) print(flush=True) ans_embed = np.load( "/data/datasets/vqa_mutant/data/vqa/mutant_l2a/answer_embs.npy" ) + 1e-8 ans_embed = torch.tensor(ans_embed) self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1) self.nlp = spacy.load('en_core_web_lg')
def __init__(self, dataset: VQADataset): super().__init__() self.raw_dataset = dataset if args.tiny: topk = TINY_IMG_NUM elif args.fast: topk = FAST_IMG_NUM else: topk = None if args.mutant: print("Mutant Data to be loaded",flush=True) # Loading detection features to img_data img_data = [] if 'train' in dataset.splits: print("TRAIN") if args.mutant: img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/train_obj36.tsv', topk=topk)) img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk)) img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk)) img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) if 'valid' in dataset.splits: print("VALID") img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk)) if 'minival' in dataset.splits: print("MINIVAL") # minival is 5K images in the intersection of MSCOCO valid and VG, # which is used in evaluating LXMERT pretraining performance. # It is saved as the top 5K features in val2014_obj36.tsv # if topk is None: # topk = 50000 if args.mutant: img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mutant_imgfeat/valid_obj36.tsv', topk=topk)) img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/train2014_obj36.tsv', topk=topk)) img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/val2014_obj36.tsv', topk=topk)) # img_data.extend(load_obj_tsv('/scratch/tgokhale/mutant_notcrowd/valid_obj36.tsv', topk=100)) if 'nominival' in dataset.splits: print("NOMINIVAL") if 'test' in dataset.name: # If dataset contains any test split print("TEST") img_data.extend(load_obj_tsv('/data/datasets/vqa_mutant/data/mscoco_imgfeat/test2015_obj36.tsv', topk=topk)) # Convert img list to dict self.imgid2img = {} for img_datum in img_data: self.imgid2img[img_datum['img_id']] = img_datum # Only kept the data with loaded image features self.data = [] valid_imgs = [] # for datum in tqdm(self.raw_dataset.data,ascii=True,desc="Loading Image features"): for datum in self.raw_dataset.data: if datum['img_id'] in self.imgid2img: self.data.append(datum) valid_imgs.append(datum['img_id']) # orig and mutant self.raw_dataset.data = self.data # Only keep images with loaded data valid_imgs = set(valid_imgs) all_imgs = set(self.imgid2img) invalid_imgs = all_imgs - valid_imgs for unwanted_key in invalid_imgs: del self.imgid2img[unwanted_key] self.color_qtypes = ['what color is the', 'what color are the', 'what color', 'what color is', 'what is the color of the'] print("Use %d data in torch dataset" % (len(self.data)),flush=True) print(flush=True)