def __init__(self,
config: Dict[str, Union[int, str]],
dialogs_jsonpath: str,
dense_annotations_jsonpath: Optional[str] = None,
overfit: bool = False,
in_memory: bool = False):
super().__init__()
self.config = config
self.dialogs_reader = DialogsReader(dialogs_jsonpath)
if "val" in self.split and dense_annotations_jsonpath is not None:
self.annotations_reader = DenseAnnotationsReader(dense_annotations_jsonpath)
else:
self.annotations_reader = None
self.vocabulary = Vocabulary(
config["word_counts_json"], min_count=config["vocab_min_count"]
)
# initialize image features reader according to split
image_features_hdfpath = config["image_features_train_h5"]
if "val" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_val_h5"]
elif "test" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_test_h5"]
self.hdf_reader = ImageFeaturesHdfReader(image_features_hdfpath, in_memory)
# keep a list of image_ids as primary keys to access data
self.image_ids = list(self.dialogs_reader.dialogs.keys())
if overfit:
self.image_ids = self.image_ids[:5]
def __init__(
self,
config: Dict[str, Any],
dialogs_jsonpath: str,
dense_annotations_jsonpath: str,
fold_split: Optional[str] = 'train',
fold: int = -1,
return_adjusted_gt_relevance: bool = False,
overfit: bool = False,
in_memory: bool = False,
return_options: bool = True,
add_boundary_toks: bool = False,
proj_to_senq_id: bool = False,
):
super().__init__()
self.config = config
self.return_options = return_options
self.return_adjusted_gt_relevance = return_adjusted_gt_relevance
self.add_boundary_toks = add_boundary_toks
self.proj_to_senq_id = proj_to_senq_id
self.dialogs_reader = DialogsReader(dialogs_jsonpath, use_bert=True)
self.annotations_reader = DenseAnnotationsReader(
dense_annotations_jsonpath)
self.fold_split = fold_split
self.fold = fold
self.n_folds = config['n_folds']
assert (config['word_embedding_type'] == 'bert')
self.vocabulary = BertVocabulary(
token_counts_path=config['bert_counts_json'],
min_count=config['vocab_min_count'])
# Initialize image features reader according to split.
image_features_hdfpath = config["image_features_train_h5"]
if ("val" in self.dialogs_reader.split
and "fake" not in self.dialogs_reader.split):
image_features_hdfpath = config["image_features_val_h5"]
elif "test" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_test_h5"]
self.hdf_reader = ImageFeaturesHdfReader(image_features_hdfpath,
in_memory)
# Keep a list of image_ids as primary keys to access data.
all_image_ids = np.array(list(self.dialogs_reader.dialogs.keys()))
if fold < 0 or fold_split == 'test':
self.image_ids = all_image_ids.tolist()
else:
kf = KFold(n_splits=self.n_folds, shuffle=False, random_state=606)
train_index, val_index = list(kf.split(all_image_ids))[fold]
if fold_split == 'train':
self.image_ids = all_image_ids[train_index].tolist()
elif fold_split == 'val':
self.image_ids = all_image_ids[val_index].tolist()
else:
raise NotImplementedError()
if overfit:
self.image_ids = self.image_ids[:5]
def __init__(
self,
config: Dict[str, Any],
dialogs_jsonpath: str,
dense_annotations_jsonpath: Optional[str] = None,
overfit: bool = False,
in_memory: bool = False,
return_options: bool = True,
add_boundary_toks: bool = False,
sample_flag: bool = False,
):
super().__init__()
self.config = config
self.return_options = return_options
self.add_boundary_toks = add_boundary_toks
self.dialogs_reader = DialogsReader(dialogs_jsonpath,config)
if "val" in self.split and dense_annotations_jsonpath is not None:
self.annotations_reader = DenseAnnotationsReader(
dense_annotations_jsonpath
)
else:
self.annotations_reader = None
self.vocabulary = Vocabulary(
config["word_counts_json"], min_count=config["vocab_min_count"]
)
# Initialize image features reader according to split.
image_features_hdfpath = config["image_features_train_h5"]
if "val" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_val_h5"]
elif "test" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_test_h5"]
self.hdf_reader = ImageFeaturesHdfReader(
image_features_hdfpath, in_memory
)
if sample_flag == False:
self.image_ids = list(self.dialogs_reader.dialogs.keys())
# Keep a list of image_ids as primary keys to access data.
if sample_flag == True:
samplefile = open('data/visdial_1.0_train_dense_sample.json', 'r') ####for answer score sampling (fine-tune)
sample = json.loads(samplefile.read())
samplefile.close()
ndcg_id_list = []
for idx in range(len(sample)):
ndcg_id_list.append(sample[idx]['image_id'])
self.image_ids = ndcg_id_list
if overfit:
self.image_ids = self.image_ids[:5]
def __init__(
self,
config: Dict[str, Any],
dialogs_jsonpath: str,
dense_annotations_jsonpath: Optional[str] = None,
return_adjusted_gt_relevance: bool = False,
overfit: bool = False,
in_memory: bool = False,
return_options: bool = True,
add_boundary_toks: bool = False,
):
super().__init__()
self.config = config
self.return_options = return_options
self.return_adjusted_gt_relevance = return_adjusted_gt_relevance
self.add_boundary_toks = add_boundary_toks
self.dialogs_reader = DialogsReader(dialogs_jsonpath)
if (("val" in self.split or "dense" in self.split)
and dense_annotations_jsonpath is not None):
self.annotations_reader = DenseAnnotationsReader(
dense_annotations_jsonpath)
else:
self.annotations_reader = None
if config['word_embedding_type'] == 'glove':
self.vocabulary = GloveVocabulary(
word_counts_path=config['word_counts_json'],
min_count=config['vocab_min_count'],
glove_weight_path=config['glove_weight_txt'],
vec_size=config['glove_emb_dim'],
glove_vec_num=config['glove_vec_num'])
else:
self.vocabulary = Vocabulary(
word_counts_path=config["word_counts_json"],
min_count=config["vocab_min_count"])
# Initialize image features reader according to split.
image_features_hdfpath = config["image_features_train_h5"]
if ("val" in self.dialogs_reader.split
and "fake" not in self.dialogs_reader.split):
image_features_hdfpath = config["image_features_val_h5"]
elif "test" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_test_h5"]
self.hdf_reader = ImageFeaturesHdfReader(image_features_hdfpath,
in_memory)
# Keep a list of image_ids as primary keys to access data.
self.image_ids = list(self.dialogs_reader.dialogs.keys())
if overfit:
self.image_ids = self.image_ids[:5]
def __init__(
self,
config: Dict[str, Any],
dialogs_jsonpath: str,
dense_annotations_jsonpath: Optional[str] = None,
overfit: bool = False,
in_memory: bool = False,
num_workers: int = 1,
return_options: bool = True,
add_boundary_toks: bool = False,
):
super().__init__()
self.config = config
self.return_options = return_options
self.add_boundary_toks = add_boundary_toks
self.dialogs_reader = DialogsReader(
dialogs_jsonpath,
num_examples=(5 if overfit else None),
num_workers=num_workers)
if "val" in self.split and dense_annotations_jsonpath is not None:
self.annotations_reader = DenseAnnotationsReader(
dense_annotations_jsonpath)
else:
self.annotations_reader = None
self.vocabulary = Vocabulary(config["word_counts_json"],
min_count=config["vocab_min_count"])
# Initialize image features reader according to split.
image_features_hdfpath = config["image_features_train_h5"]
if "val" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_val_h5"]
elif "test" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_test_h5"]
self.hdf_reader = ImageFeaturesHdfReader(image_features_hdfpath,
in_memory)
# Keep a list of image_ids as primary keys to access data.
self.image_ids = list(self.dialogs_reader.dialogs.keys())
if overfit:
self.image_ids = self.image_ids[:5]
def __init__(
self,
config: Dict[str, Any],
dialogs_jsonpath: str,
dense_annotations_jsonpath: Optional[str] = None,
augment_dense_annotations_jsonpath: Optional[str] = None,
use_pretrained_emb: bool = False,
qa_emb_file_path: Optional[str] = None, # SA: todo remove this
hist_emb_file_path: Optional[str] = None, # SA: todo remove this
use_caption: bool = True,
num_hist_turns: int = 10,
finetune: bool = False,
overfit: bool = False,
in_memory: bool = False,
num_workers: int = 1,
return_options: bool = True,
add_boundary_toks: bool = False):
super().__init__()
self.config = config
# SA: embedding reader
self.use_pretrained_emb = use_pretrained_emb
self.return_options = return_options
self.add_boundary_toks = add_boundary_toks
self.dialogs_reader = DialogsReader(
dialogs_jsonpath,
num_examples=(5 if overfit else None),
num_workers=num_workers,
use_pretrained_emb=self.use_pretrained_emb)
self.finetune = finetune
self.use_caption = use_caption
# SA: embedding reader
if self.use_pretrained_emb:
assert qa_emb_file_path, "Did you forget to set emb file path?"
# @todo: for now coming through argparse
self.qa_emb_file_path = qa_emb_file_path
self.hist_emb_file_path = hist_emb_file_path
# hist_emb_file_path = config["hist_emb_file_path"]
# TransformerEmbeddingsHdfReader(embedding_path, in_memory)
# self.embedding_reader = TransformerEmbeddingsHdfReader(hist_emb_file_path,
# in_memory)
self.question_reader = QuesEmbeddingsHdfReader(
qa_emb_file_path, in_memory)
self.ans_reader = AnswerEmbeddingsHdfReader(
qa_emb_file_path, in_memory)
self.caption_reader = CaptionEmbeddingsHdfReader(
qa_emb_file_path, in_memory)
# SA: we dont pass in_memory here because history is too big
# SA: todo this key would change
self.hist_reader = HistEmbeddingsHdfReader(hist_emb_file_path,
hdfs_key="hist")
# SA: if finetuning for train/val otherwise just validation set
if self.finetune or ("val" in self.split
and dense_annotations_jsonpath is not None):
self.annotations_reader = DenseAnnotationsReader(
dense_annotations_jsonpath)
else:
self.annotations_reader = None
if augment_dense_annotations_jsonpath is not None:
self.augmented_annotations_reader = AugmentedDenseAnnotationsReader(
augment_dense_annotations_jsonpath)
self.use_augment_dense = True
else:
self.use_augment_dense = False
self.vocabulary = Vocabulary(config["word_counts_json"],
min_count=config["vocab_min_count"])
# Initialize image features reader according to split.
image_features_hdfpath = config["image_features_train_h5"]
if "val" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_val_h5"]
elif "test" in self.dialogs_reader.split:
image_features_hdfpath = config["image_features_test_h5"]
self.hdf_reader = ImageFeaturesHdfReader(image_features_hdfpath,
in_memory)
# Keep a list of image_ids as primary keys to access data.
# For finetune we use only those image id where we have dense annotations
if self.finetune:
self.image_ids = list(self.annotations_reader.keys)
else:
self.image_ids = list(self.dialogs_reader.dialogs.keys())
if overfit:
self.image_ids = self.image_ids[:5]