def __init__(self,
yaml_file,
tokenizer=None,
add_od_labels=True,
max_img_seq_length=50,
max_seq_length=70,
max_seq_a_length=40,
is_train=True,
mask_prob=0.15,
max_masked_tokens=3,
add_conf=False,
**kwargs):
"""Constructor.
Args:
yaml file with all required data (image feature, caption, labels, etc)
tokenizer: tokenizer for text processing.
add_od_labels: whether to add labels from yaml file to BERT.
max_img_seq_length: max image sequence length.
max_seq_length: max text sequence length.
max_seq_a_length: max caption sequence length.
is_train: train or test mode.
mask_prob: probability to mask a input token.
max_masked_tokens: maximum number of tokens to be masked in one sentence.
kwargs: other arguments.
"""
self.yaml_file = yaml_file
self.cfg = load_from_yaml_file(yaml_file)
self.root = op.dirname(yaml_file)
self.label_file = find_file_path_in_yaml(self.cfg['label'], self.root)
self.feat_file = find_file_path_in_yaml(self.cfg['feature'], self.root)
self.caption_file = find_file_path_in_yaml(self.cfg.get('caption'),
self.root)
assert op.isfile(self.feat_file)
if add_od_labels: assert op.isfile(self.label_file)
if is_train:
assert op.isfile(self.caption_file) and tokenizer is not None
self.label_tsv = None if not self.label_file else TSVFile(
self.label_file)
self.feat_tsv = TSVFile(self.feat_file)
if self.caption_file and op.isfile(self.caption_file):
with open(self.caption_file, 'r') as f:
self.captions = json.load(f)
self.tokenizer = tokenizer
self.tensorizer = CaptionTensorizer(self.tokenizer,
max_img_seq_length,
max_seq_length,
max_seq_a_length,
mask_prob,
max_masked_tokens,
is_train=is_train)
self.add_od_labels = add_od_labels
self.is_train = is_train
self.kwargs = kwargs
self.image_keys = self.prepare_image_keys()
self.key2index = self.prepare_image_key_to_index()
self.key2captions = self.prepare_image_key_to_captions()
self.add_conf = add_conf
def __init__(self,
yaml_file,
args=None,
tokenizer=None,
seq_len=35,
encoding="utf-8",
corpus_lines=None,
on_memory=True,
**kwargs):
self.cfg = load_from_yaml_file(yaml_file)
self.root = os.path.dirname(yaml_file)
self.vocab = tokenizer.vocab
self.tokenizer = tokenizer
self.seq_len = seq_len
self.on_memory = on_memory
self.corpus_lines = corpus_lines # number of non-empty lines in input corpus
self.corpus_tsvfile = TSVFile(
os.path.join(self.root, self.cfg['corpus_file']))
if 'textb_sample_mode' in kwargs:
self.textb_sample_mode = kwargs['textb_sample_mode']
else:
self.textb_sample_mode = args.textb_sample_mode
self.datasets_names = self.cfg['corpus'].split('_')
self.datasets_with_splits = [
'googlecc', 'sbu', 'oi', 'objects365', 'tagoi'
]
self.datasets_with_onesplit = ['coco', 'flickr30k', 'gqa']
logging.info('Datasets: {}'.format(','.join(self.datasets_names)))
self.image_label_path = self.cfg['image_label_path']
for key, val in self.image_label_path.items():
# get the absolute path
if key in self.datasets_names:
self.image_label_path[key] = os.path.join(self.root, val)
self.image_feature_path = self.cfg['image_feature_path']
self.image_file_name = 'features.tsv'
if args.data_dir is not None:
for key, val in self.image_feature_path.items():
# get the absolute path
if key in self.datasets_names:
self.image_feature_path[key] = os.path.join(
args.data_dir, val)
else:
logging.info("Data {} with path {} is not used in the "
"training.".format(key, val))
self.encoding = encoding
self.current_doc = 0 # to avoid random sentence from same doc
self.current_img = '' # to avoid random sentence from same image
self.args = args
# for loading samples directly from file
self.sample_counter = 0 # used to keep track of full epochs on file
self.line_buffer = None # keep second sentence of a pair in memory and use as first sentence in next pair
# for loading samples in memory
self.current_random_doc = 0
self.num_docs = 0
self.sample_to_doc = [] # map sample index to doc and line
self.chunk_list = None
if 0 <= args.chunk_start_id <= args.chunk_end_id and args.chunk_end_id >= 0:
self.chunk_list = [
str(c_i)
for c_i in range(args.chunk_start_id, args.chunk_end_id)
]
logging.info('Chunk list: {}'.format(','.join(self.chunk_list)))
# load image tags and features
t_start = time.time()
self.img_label_file = None
self.img_qa_file = None
self.img_label_offset_map = None
self.img_qa_offset_map = None
self.img_feature_file = None
self.img_feat_offset_map = None
self.load_img_labels()
self.load_img_tsv_features()
t_end = time.time()
logging.info(
'Info: loading img features using {} secs'.format(t_end - t_start))
# load samples into memory
if on_memory:
self.all_docs = []
self.all_qa_docs = []
self.imgid2labels = {}
self.corpus_lines = 0
max_tokens = 0
for line_no in tqdm(range(len(self.corpus_tsvfile))):
doc = []
row = self.corpus_tsvfile.seek(line_no)
img_info = row[0].split('_')
label_info = row[1].split('_')
assert img_info[0] == label_info[
0], "Dataset names for image and label do not match!"
dataset_name = label_info[0]
if dataset_name == 'cc':
dataset_name = 'googlecc'
if dataset_name not in self.datasets_names:
continue
if dataset_name in self.datasets_with_splits:
chunk_id = img_info[-2]
if self.chunk_list is not None and chunk_id not in self.chunk_list:
continue
else:
img_feat_offset_map = self.img_feat_offset_map[
dataset_name][chunk_id]
else:
img_feat_offset_map = self.img_feat_offset_map[
dataset_name]
assert img_info[
-1] in img_feat_offset_map, "{}: Image id {} cannot be found in image feature imageid_to_index file!".format(
row[0], img_info[-1])
# append id info
doc.append('%s|%s' % (row[0], row[1]))
# append text_a info
self.corpus_lines = self.corpus_lines + 1
sample = {"doc_id": len(self.all_docs), "line": len(doc)}
self.sample_to_doc.append(sample)
assert len(row[2]) != 0, "Text_a is empty in {} : {}"\
.format(dataset_name, row[0])
doc.append(row[2])
# append text_b info
self.corpus_lines = self.corpus_lines + 1
label_id = label_info[-1]
if 'qa' in label_info:
assert img_info[-1] == label_info[
-2], "Image ids for image and qa do not match!"
label_line_no = self.img_qa_offset_map[dataset_name][
label_id]
rowb = self.img_qa_file[dataset_name].seek(label_line_no)
else:
assert img_info[-1] == label_info[
-1], "Image ids for image and label do not match!"
label_line_no = self.img_label_offset_map[dataset_name][
label_id]
rowb = self.img_label_file[dataset_name].seek(
label_line_no)
assert label_id == rowb[0]
results = json.loads(rowb[1])
if 'qa' not in label_info: # more intuitively, should be if 'qa' not in label_info:
objects = results['objects']
if row[0] not in self.imgid2labels:
self.imgid2labels[row[0]] = {
"image_h": results["image_h"],
"image_w": results["image_w"],
"boxes": None
}
else:
assert results["image_h"] == self.imgid2labels[row[0]][
"image_h"], "Image_h does not match in image {}!".format(
row[0])
assert results["image_w"] == self.imgid2labels[row[0]][
"image_w"], "Image_w does not match in image {}!".format(
row[0])
if args.use_gtlabels and 'gt_objects' in results:
# use ground-truth tags for text_b
textb = ' '.join([
cur_d['class'] for cur_d in results["gt_objects"]
])
else:
textb = ' '.join([cur_d['class'] for cur_d in objects])
else:
tag_label_line_no = self.img_label_offset_map[
dataset_name][img_info[-1]]
tag_rowb = self.img_label_file[dataset_name].seek(
tag_label_line_no)
tag_results = json.loads(tag_rowb[1])
if row[0] not in self.imgid2labels:
self.imgid2labels[row[0]] = {
"image_h": tag_results["image_h"],
"image_w": tag_results["image_w"],
"boxes": None
}
else:
assert tag_results["image_h"] == self.imgid2labels[row[0]][
"image_h"], "Image_h does not match in image {}!".format(
row[0])
assert tag_results["image_w"] == self.imgid2labels[row[0]][
"image_w"], "Image_w does not match in image {}!".format(
row[0])
textb = ' '.join(results['labels'])
assert len(textb) != 0, "Text_b is empty in {} : {}".format(
dataset_name, row[1])
doc.append(textb)
# add to all_docs
max_tokens = max(
max_tokens,
len(doc[1].split(' ')) + len(doc[2].split(' ')))
if 'qa' in label_info:
self.all_qa_docs.append({
"doc": doc,
"doc_id": len(self.all_docs)
})
self.all_docs.append(doc)
self.num_docs = len(self.all_docs)
logging.info("Max_tokens: {}".format(max_tokens))
# load samples later lazily from disk
else:
raise ValueError("on_memory = False Not supported yet!")
logging.info("Total docs - Corpus_lines: {}-{}".format(
self.num_docs, self.corpus_lines))
logging.info("Total QA docs - Corpus_lines: {}".format(
len(self.all_qa_docs)))