def load_combined_vocab(cls, data_dir='data/CLEVR_v1.0'): data_dir = to_path(data_dir) q_vocab_file = data_dir.joinpath(f'{cls.name}/q_vocab.json') a_vocab_file = data_dir.joinpath(f'{cls.name}/a_vocab.json') if q_vocab_file.exists() and a_vocab_file.exists(): q_vocab = Vocab.build_from_counter_file('q_vocab', data_dir.joinpath(cls.name), q_vocab_file, special_symbols='<padding>') a_vocab = Vocab.build_from_counter_file('a_vocab', data_dir.joinpath(cls.name), a_vocab_file) return q_vocab, a_vocab ann_files = [data_dir.joinpath(f'questions/CLEVR_{split}_questions.json') for split in ('train', 'val')] q_anns = [json.load(a_file.open())['questions'] for a_file in ann_files] if not q_vocab_file.exists(): questions = [ann_item['question'] for q_ann in q_anns for ann_item in q_ann] q_tokens = [] for question in tqdm(questions, desc='Tokenizing questions'): q_tokens.append(cls.q_tokenize(question)) q_vocab = Vocab.build_from_sequences('q_vocab', data_dir.joinpath(f'{cls.name}'), q_tokens, special_symbols='<padding>') q_vocab.dump_to_file(q_vocab_file) else: q_vocab = Vocab.build_from_counter_file('q_vocab', data_dir.joinpath(cls.name), q_vocab_file, special_symbols='<padding>') if not a_vocab_file.exists(): answers = [ann_item['answer'] for ann in q_anns for ann_item in ann] a_tokens = [] for answer in tqdm(answers, desc='Tokenizing answers'): a_tokens.append(cls.a_tokenize(answer)) a_vocab = Vocab.build_from_sequences('a_vocab', data_dir.joinpath(f'{cls.name}'), a_tokens) a_vocab.dump_to_file(a_vocab_file) else: a_vocab = Vocab.build_from_counter_file('a_vocab', data_dir.joinpath(cls.name), a_vocab_file) q_vocab.strict = True a_vocab.strict = True return q_vocab, a_vocab
def __init__(self, work_dir: str, step2log: int = 1, writer_kwargs=None): super().__init__() self.work_dir = to_path(work_dir) self.step2log = step2log if not self.work_dir.exists(): self.work_dir.mkdir(parents=True) self.writer = SummaryWriter(**writer_kwargs)
def auto_config_file(self, params): config_dir = to_path( f'./{params.work_dir}/{params.proj_name}/{params.exp_name}/{params.exp_version}/experiment' ) if not config_dir.exists(): config_dir.mkdir(parents=True) return config_dir.joinpath('params.json')
def load_combined_vocab(cls, data_dir='work_dir/data/vqa2'): data_dir = utils.to_path(data_dir) q_vocab_file = data_dir.joinpath(f'{cls.name}/q_vocab.json') a_vocab_file = data_dir.joinpath(f'{cls.name}/a_vocab.json') if not q_vocab_file.exists(): logger.info(f'Creating question vocab {q_vocab_file.name}') q_dict = json.load( data_dir.joinpath(cls.name, 'train_q_dict.json').open()) id2words = ['<padding>'] + list(q_dict['itow'].values()) word2ids = {word: idx for idx, word in enumerate(id2words)} q_vocab = Vocab.build_from_statistic( id2words, word2ids, 'q_vocab', data_dir.joinpath(f'{cls.name}')) q_vocab.save(q_vocab_file) if not a_vocab_file.exists(): logger.info(f'Creating answer vocab {a_vocab_file.name}') a_dict = json.load( data_dir.joinpath(cls.name, 'train_a_dict.json').open()) id2words = list(a_dict['itow'].values()) + ['<padding>'] word2ids = {word: idx for idx, word in enumerate(id2words)} a_vocab = Vocab.build_from_statistic( id2words, word2ids, 'a_vocab', data_dir.joinpath(f'{cls.name}')) a_vocab.save(a_vocab_file) q_vocab = Vocab.from_file('q_vocab', data_dir.joinpath(cls.name), q_vocab_file) a_vocab = Vocab.from_file('a_vocab', data_dir.joinpath(cls.name), a_vocab_file) return q_vocab, a_vocab
def load_combined_anns(cls, data_dir, data_split): data_dir = to_path(data_dir) ann_file = data_dir.joinpath(f'{cls.name}/{data_split}_ann.pt') if ann_file.exists(): anns_t = torch.load(ann_file) else: sys_logger.info(f'Creating combined annotation files {ann_file.name}') q_vocab, a_vocab = cls.load_combined_vocab(data_dir) if data_split in ('valB_30k', 'valB_120k'): q_anns = json.load(data_dir.joinpath('questions', f'CLEVR_valB_questions.json').open())['questions'] if data_split == 'valB_30k': q_anns = q_anns[:30000] else: q_anns = q_anns[30000:] else: q_anns = json.load(data_dir.joinpath('questions', f'CLEVR_{data_split}_questions.json').open())['questions'] anns = defaultdict(list) for q_item in tqdm(q_anns, desc=f'Creating {ann_file.name}'): anns['q_ids'].append(q_item['question_index']) anns['img_ids'].append(q_item['image_index']) anns['img_names'].append(q_item['image_filename']) anns['q_tokens'].append(cls.q_tokenize(q_item['question'])) anns['q_types'].append(q_item['question_family_index']) anns['a_tokens'].append(q_item['answer']) anns['q_lens'].append(len(anns['q_tokens'][-1])) q_label = [q_vocab[token] for token in anns['q_tokens'][-1]] q_label_t = torch.empty(47).fill_(q_vocab.padding_idx).long() q_label_t[:len(q_label)] = torch.tensor(q_label).long() anns['q_labels'].append(q_label_t) anns['a_labels'].append(a_vocab[q_item['answer']]) anns_t = {name: to_tensor(value) for name, value in anns.items()} torch.save(anns_t, ann_file) return anns_t
def __init__( self, save_dir: str, restore_file: str = None, epoch2save: int = 1, only_best: bool = True, strict_mode: str = False, monitor: str = 'val_loss', ): super().__init__() self.save_dir = to_path(save_dir) if not self.save_dir.exists(): self.save_dir.mkdir(parents=True) self.restore_file = to_path(restore_file) self.epoch2save = epoch2save self.only_best = only_best self.strict_mode = strict_mode self.monitor = monitor self.best_val_acc = 0
def __init__(self, img_dir, req_field_names=None, file2idx_fn=None, img_size=None, transforms=None): self.img_dir = to_path(img_dir) assert self.img_dir.exists() self.file2idx_fn = file2idx_fn or self._file2idx_fn self.transforms = transforms or self.load_transforms(img_size) super().__init__(req_field_names)
def __init__(self, name, logger_dir): self.name = name self.logger_dir = to_path(logger_dir) self.json_file = self.logger_dir.joinpath(f'{name}.json') self.histories = dict() if not self.json_file.exists() else json.load( self.json_file.open()) self.histories = { int(key): value for key, value in self.histories.items() } self.is_first = True
def __init__(self, data_dir: str, split: str, req_field_names: List[str], is_lazy: bool = False): self.data_dir = to_path(data_dir) self.split = split self.loaders = None self._length = None py_data.Dataset.__init__(self) Reader.__init__(self, req_field_names, is_lazy) PtBase.__init__(self)
def __init__( self, data_dir, split, req_field_names=None, is_load=False, ): self.data_dir = to_path(data_dir) h5_file = self.data_dir.joinpath(f'{split}_images.h5') if not h5_file.exists(): self.create_h5_file(self.data_dir, split) super().__init__(h5_file, req_field_names, is_load=is_load)
def __init__( self, data_dir: str, split: str, req_field_names: List[str], ): self.data_dir = to_path(data_dir) self.split = split self.loaders = None py_data.Dataset.__init__(self) Reader.__init__(self, req_field_names) PtBase.__init__(self)
def load_combined_vocab(cls, data_dir='work_dir/data/vqa2'): data_dir = to_path(data_dir) q_vocab_file = data_dir.joinpath(f'{cls.name}/q_vocab.json') a_vocab_file = data_dir.joinpath(f'{cls.name}/a_vocab.json') if not q_vocab_file.exists(): logger.info(f'Creating question vocab {q_vocab_file.name}') q_files = (f'vqacp_v2_{key}_questions.json' for key in ('train', 'test')) q_files = (data_dir.joinpath(f'annotations/{q_file}') for q_file in q_files) questions = functools.reduce(lambda x, y: x + y, (json.load(q_file.open()) for q_file in q_files)) q_tokens = [] for q in tqdm.tqdm(questions): tokens = cls.q_tokenize(q['question']) q_tokens.append(tokens) q_vocab = Vocab.build_from_sequences( 'q_vocab', data_dir.joinpath(f'{cls.name}'), q_tokens, special_symbols='<padding>') q_vocab.dump_to_file(q_vocab_file) if not a_vocab_file.exists(): logger.info(f'Creating answer vocab {a_vocab_file.name}') a_files = (f'vqacp_v2_{split}_annotations.json' for split in ('train', 'test')) a_files = (data_dir.joinpath(f'annotations/{a_file}') for a_file in a_files) answers = functools.reduce(lambda x, y: x + y, (json.load(a_file.open()) for a_file in a_files)) a_tokens = [] for answer in tqdm.tqdm(answers): gtruth = cls.a_tokenize(answer['multiple_choice_answer']) a_tokens.append(gtruth) a_vocab = Vocab.build_from_sequences( 'a_vocab', data_dir.joinpath(f'{cls.name}'), a_tokens, max_word_num=3000, special_symbols='<padding>') a_vocab.dump_to_file(a_vocab_file) q_vocab = Vocab.build_from_counter_file('q_vocab', data_dir.joinpath(cls.name), q_vocab_file, special_symbols='<padding>') a_vocab = Vocab.build_from_counter_file('a_vocab', data_dir.joinpath(cls.name), a_vocab_file, special_symbols='<padding>') return q_vocab, a_vocab
def __init__( self, data_dir, split, req_field_names=None, transforms=None, is_load=False, ): self.data_dir = to_path(data_dir) if split in ('valB_30k', 'valB_120k'): split = 'valB' super().__init__(self.data_dir.parent.joinpath('images', f'{split}'), req_field_names, transforms=transforms)
def load_combined_anns(cls, data_dir: str = 'work_dir/data/gqa', data_split: str = 'train', q_max=30): data_dir = to_path(data_dir).joinpath(cls.name) ann_file = data_dir.joinpath(f'{data_split}_ann.pt') if ann_file.exists(): return torch.load(ann_file) logger.info(f'Creating combined annotation files {ann_file.name}') q_vocab, a_vocab = cls.load_combined_vocab(data_dir.parent) if data_split in ('train', 'val'): q_file = data_dir.parent.joinpath( f'questions/{data_split}_balanced_questions.json') obj_infos = json.load( data_dir.parent.joinpath( 'objects/gqa_objects_info.json').open()) # scene_file = data_dir.joinpath(f'sceneGraphs/{data_split}_sceneGraphs.json') q_anns = json.load(q_file.open()) else: raise NotImplementedError() anns = collections.defaultdict(list) pbar = tqdm.tqdm( range(len(q_anns)), desc=f'Creating combined annotation files {ann_file.name}') for q_id, q_ann in q_anns.items(): anns['splits'].append(data_split) anns['q_ids'].append(q_id) anns['img_ids'].append(q_ann['imageId']) anns['q_tokens'].append(cls.q_tokenize(q_ann['question'])) q_label = [ q_vocab[token] for token in anns['q_tokens'][-1][:q_max] ] anns['q_lens'].append(len(q_label)) q_label_t = torch.empty(q_max).fill_(q_vocab.padding_idx).long() q_label_t[:len(q_label)] = torch.tensor(q_label).long() anns['q_labels'].append(q_label_t) obj_info = obj_infos[q_ann['imageId']] anns['img_shapes'].append( torch.tensor((obj_info['width'], obj_info['height'])).float()) anns['img_obj_nums'].append(torch.tensor(obj_info['objectsNum'])) if 'answer' in q_ann: anns['a_tokens'].append(q_ann['answer']) anns['a_labels'].append(torch.tensor(a_vocab[q_ann['answer']])) pbar.update(1) anns_t = {key: to_tensor(value) for key, value in anns.items()} torch.save(anns_t, ann_file, pickle_module=dill) return anns_t
def __init__( self, name: str, work_dir: str, step2log: int, splits: List[str], ): self.name = name self.work_dir = to_path(work_dir) if not self.work_dir.exists(): self.work_dir.mkdir(parents=True) self.step2log = step2log self.metric = Metric(name, self.work_dir) self.splits = splits super().__init__()
def init_args(cls, params, sub_cls=None): cls.default_init_args(params) try_set_attr(params, f'{cls.prefix_name()}_name', 'logger_group') try_set_attr( params, f'{cls.prefix_name()}_logger_dir', to_path(params.root_dir).joinpath(f'loggers/{params.proj_name}')) logger_cls = cls.load_cls( try_get_attr(params, f'{cls.prefix_name()}_logger_cls', check=False)) if logger_cls is not None: logger_cls.init_args(params) setattr( params, f'{cls.prefix_name()}_logger_kwargs', load_func_kwargs(params, logger_cls.__init__, cls.prefix_name()))
def load_combined_vocab(cls, data_dir='work_dir/data/gqa'): data_dir = to_path(data_dir).joinpath(cls.name) q_vocab_file = data_dir.joinpath('q_vocab.json') a_vocab_file = data_dir.joinpath('a_vocab.json') if not q_vocab_file.exists(): logger.info(f'Creating question vocab {q_vocab_file.name}') q_vocab = Vocab.build_from_txt( 'q_vocab', data_dir.joinpath('vocabulary_gqa.txt')) q_vocab.save(q_vocab_file) else: q_vocab = Vocab.from_file('q_vocab', q_vocab_file) if not a_vocab_file.exists(): logger.info(f'Creating answer vocab {a_vocab_file.name}') a_vocab = Vocab.build_from_txt( 'a_vocab', data_dir.joinpath('answers_gqa.txt')) a_vocab.save(a_vocab_file) else: a_vocab = Vocab.from_file('a_vocab', a_vocab_file) q_vocab.embed_init = np.load( data_dir.joinpath('gloves_gqa_no_pad.npy')) return q_vocab, a_vocab
def load_combined_anns( cls, data_dir: str = 'data/vqa2', split: str = 'train', ): data_dir = utils.to_path(data_dir) ann_file = data_dir.joinpath(f'{cls.name}/{split}_ann.pt') if ann_file.exists(): anns_t = torch.load(ann_file) else: logger.info(f'Creating combined annotation files {ann_file.name}') q_vocab, a_vocab = cls.load_combined_vocab(data_dir) if split in ('train', 'val'): json_anns = json.load( data_dir.joinpath(cls.name, f'vqa_{split}_final_3000.json').open()) ins_anns = json.load( data_dir.joinpath( f'annotations/instances_{split}2014.json').open()) img_anns = {ann['id']: ann for ann in ins_anns['images']} elif split == 'test': json_anns = json.load( data_dir.joinpath(cls.name, f'vqa_test_toked.json').open()) q_anns = json.load( data_dir.joinpath( f'annotations/v2_OpenEnded_mscoco_{split}2015_questions.json' ).open())['questions'] ins_anns = json.load( data_dir.joinpath( f'annotations/image_info_{split}2015.json').open()) img_anns = {ann['id']: ann for ann in ins_anns['images']} elif split == 'train_val': train_anns, val_anns = [ cls.load_combined_anns(data_dir, split) for split in ('train', 'val') ] anns_t = {} for key in train_anns.keys(): if torch.is_tensor(train_anns[key]): anns_t[key] = torch.cat( (train_anns[key], val_anns[key]), dim=0) elif isinstance(train_anns[key], (list, tuple)): anns_t[key] = list(train_anns[key]) + list( val_anns[key]) else: raise NotImplementedError() return anns_t else: raise NotImplementedError() anns = collections.defaultdict(list) for idx, json_ann in enumerate(tqdm(json_anns)): anns['splits'].append(split) anns['q_ids'].append(json_ann['question_id']) anns['img_ids'].append(int(json_ann['image_id'])) anns['q_tokens'].append(json_ann['question_toked']) anns['q_lens'].append(len(anns['q_tokens'][-1])) q_label = [q_vocab[token] for token in anns['q_tokens'][-1]] q_label_t = torch.empty(30).fill_(q_vocab.padding_idx).long() q_label_t[:len(q_label)] = torch.tensor(q_label).long() anns['q_labels'].append(q_label_t) img_ann = img_anns[int(json_ann['image_id'])] anns['img_names'].append(img_ann['file_name']) anns['img_shapes'].append( (img_ann['width'], img_ann['height'])) if split == 'test': continue anns['a_counts'].append([ a_count for a_count in json_ann['answers'] if a_count[0] in a_vocab.word2idx ]) anns['a_tokens'].append(json_ann['answer']) anns['a_scores'].append([ a_score for a_score in json_ann['answers_w_scores'] if a_score[0] in a_vocab.word2idx ]) anns_t = { key: utils.to_tensor(value) for key, value in anns.items() } torch.save(anns_t, ann_file) return anns_t
def load_config(self, config_file): config_file = to_path(config_file) assert config_file.exists() config = json.load(open(config_file)) return config
def init_args(cls, params, sub_cls=None): cls.default_init_args(params) try_set_attr( params, f'{cls.prefix_name()}_work_dir', to_path(params.root_dir).joinpath(f'loggers/{params.proj_name}')) try_set_attr(params, f'{cls.prefix_name()}_env', params.proj_name)
def load_combined_anns( cls, data_dir: str = 'data/vqa2', data_split: str = 'train', ): data_dir = to_path(data_dir) ann_file = data_dir.joinpath(f'{cls.name}/{data_split}_ann.pt') if ann_file.exists(): anns_t = torch.load(ann_file) if 'splits' not in anns_t: anns_t['splits'] = [data_split] * len(anns_t['q_ids']) torch.save(anns_t, ann_file) else: logger.info(f'Creating combined annotation files {ann_file.name}') q_vocab, a_vocab = cls.load_combined_vocab(data_dir) if data_split in ('train', 'val'): origin_anns = json.load( data_dir.joinpath( f'annotations/v2_mscoco_{data_split}2014_annotations.json' ).open())['annotations'] q_anns = json.load( data_dir.joinpath( f'annotations/v2_OpenEnded_mscoco_{data_split}2014_questions.json' ).open())['questions'] ins_anns = json.load( data_dir.joinpath( f'annotations/instances_{data_split}2014.json').open()) img_anns = {ann['id']: ann for ann in ins_anns['images']} elif data_split == 'test': origin_anns = None q_anns = json.load( data_dir.joinpath( f'annotations/v2_OpenEnded_mscoco_{data_split}2015_questions.json' ).open())['questions'] ins_anns = json.load( data_dir.joinpath( f'annotations/image_info_{data_split}2015.json').open( )) img_anns = {ann['id']: ann for ann in ins_anns['images']} elif data_split == 'train_val': train_anns, val_anns = [ cls.load_combined_anns(data_dir, split) for split in ('train', 'val') ] anns_t = {} for key in train_anns.keys(): if torch.is_tensor(train_anns[key]): anns_t[key] = torch.cat( (train_anns[key], val_anns[key]), dim=0) elif isinstance(train_anns[key], (list, tuple)): anns_t[key] = list(train_anns[key]) + list( val_anns[key]) else: raise NotImplementedError() return anns_t else: raise NotImplementedError() anns = collections.defaultdict(list) for idx, q in enumerate(tqdm.tqdm(q_anns)): anns['splits'].append(data_split) anns['q_ids'].append(q['question_id']) anns['img_ids'].append(q['image_id']) anns['q_tokens'].append(cls.q_tokenize(q['question'])) anns['q_lens'].append(len(anns['q_tokens'][-1])) q_label = [q_vocab[token] for token in anns['q_tokens'][-1]] q_label_t = torch.empty(24).fill_(q_vocab.padding_idx).long() q_label_t[:len(q_label)] = torch.tensor(q_label).long() anns['q_labels'].append(q_label_t) img_ann = img_anns[q['image_id']] anns['img_names'].append(img_ann['file_name']) anns['img_shapes'].append( (img_ann['width'], img_ann['height'])) if origin_anns is None: continue assert q['question_id'] == origin_anns[idx]['question_id'] ori_ann = origin_anns[idx] anns['a_tokens'].append( cls.a_tokenize(ori_ann['multiple_choice_answer'])) answers = [] for ans in ori_ann['answers']: answ = cls.a_tokenize(ans['answer']) if answ in a_vocab.words: answers.append(answ) anns['a_counts'].append( collections.Counter(answers).most_common()) # a_label_count = torch.empty(len(a_vocab)).fill_(a_vocab.padding_idx).float() # for a_token, a_count in anns['a_counts'][-1]: # a_label_count[a_vocab[a_token]] = a_count # anns['a_label_counts'].append(a_label_count) accepted_answers = sum( [ans[1] for ans in anns['a_counts'][-1]]) anns['a_scores'].append([(ans[0], ans[1] / accepted_answers) for ans in anns['a_counts'][-1]]) # a_label_score = torch.empty(len(a_vocab)).fill_(a_vocab.padding_idx).float() # for a_token, a_score in anns['a_scores'][-1]: # a_label_score[a_vocab[a_token]] = a_score # anns['a_label_scores'].append(a_label_score) anns_t = {key: to_tensor(value) for key, value in anns.items()} torch.save(anns_t, ann_file) return anns_t
def load_combined_anns( cls, data_dir: str = 'data/vqa2', data_split: str = 'train', ): data_dir = to_path(data_dir) ann_file = data_dir.joinpath(f'{cls.name}/{data_split}_ann.pt') if ann_file.exists(): anns_t = torch.load(ann_file) if 'splits' not in anns_t: anns_t['splits'] = [data_split] * len(anns_t['q_ids']) torch.save(anns_t, ann_file) else: logger.info(f'Creating combined annotation files {ann_file.name}') q_vocab, a_vocab = cls.load_combined_vocab(data_dir) img_ann_dict = {} for split in ('train', 'val'): ins_anns = json.load( data_dir.joinpath( f'annotations/instances_{split}2014.json').open()) img_anns = {ann['id']: ann for ann in ins_anns['images']} img_ann_dict[split] = img_anns if data_split in ('train', 'test'): origin_anns = json.load( data_dir.joinpath( f'annotations/vqacp_v2_{data_split}_annotations.json'). open()) q_anns = json.load( data_dir.joinpath( f'annotations/vqacp_v2_{data_split}_questions.json'). open()) else: raise NotImplementedError() anns = collections.defaultdict(list) for idx, q in enumerate(tqdm.tqdm(q_anns)): anns['splits'].append(q['coco_split'][:-4]) anns['q_ids'].append(q['question_id']) anns['img_ids'].append(q['image_id']) anns['q_tokens'].append(cls.q_tokenize(q['question'])) anns['q_lens'].append(len(anns['q_tokens'][-1])) q_label = [q_vocab[token] for token in anns['q_tokens'][-1]] q_label_t = torch.empty(24).fill_(q_vocab.padding_idx).long() q_label_t[:len(q_label)] = torch.tensor(q_label).long() anns['q_labels'].append(q_label_t) img_anns = img_ann_dict[anns['splits'][-1]] img_ann = img_anns[q['image_id']] anns['img_names'].append(img_ann['file_name']) anns['img_shapes'].append( (img_ann['width'], img_ann['height'])) if origin_anns is None: continue assert q['question_id'] == origin_anns[idx]['question_id'] ori_ann = origin_anns[idx] anns['a_tokens'].append( cls.a_tokenize(ori_ann['multiple_choice_answer'])) answers = [] for ans in ori_ann['answers']: answ = cls.a_tokenize(ans['answer']) if answ in a_vocab.words: answers.append(answ) anns['a_counts'].append( collections.Counter(answers).most_common()) accepted_answers = sum( [ans[1] for ans in anns['a_counts'][-1]]) anns['a_scores'].append([(ans[0], ans[1] / accepted_answers) for ans in anns['a_counts'][-1]]) anns_t = {key: to_tensor(value) for key, value in anns.items()} torch.save(anns_t, ann_file) return anns_t