def build_vocabulary(config):
""" Build the vocabulary from the training data and save it to a file. """
coco = COCO(config.train_caption_file)
coco.filter_by_cap_len(config.max_caption_length)
vocabulary = Vocabulary(config.vocabulary_size)
vocabulary.build(coco.all_captions())
vocabulary.save(config.vocabulary_file)
return vocabulary
def build_vocabulary(config, max_ann_num=None):
""" Build the vocabulary from the training data and save it to a file. """
coco = COCO(config.train_caption_file, config.max_train_ann_num)
coco.filter_by_cap_len(config.max_caption_length)
vocabulary = Vocabulary(config.vocabulary_size)
if not config.max_train_ann_num:
vocabulary.build(coco.all_captions())
else:
vocabulary.build((coco.all_captions())[:config.max_train_ann_num])
vocabulary.save(config.vocabulary_file)
return vocabulary
def prepare_train_data(config):
""" Prepare the data for training the model. """
coco = COCO(config.train_caption_file)
coco.filter_by_cap_len(config.max_caption_length)
print("Building the vocabulary...")
vocabulary = Vocabulary(config.vocabulary_size)
if not os.path.exists(config.vocabulary_file):
vocabulary.build(coco.all_captions())
vocabulary.save(config.vocabulary_file)
else:
vocabulary.load(config.vocabulary_file)
print("Vocabulary built.")
print("Number of words = %d" % (vocabulary.size))
coco.filter_by_words(set(vocabulary.words))
print("Processing the captions...")
if not os.path.exists(config.temp_annotation_file):
captions = [coco.anns[ann_id]['caption'] for ann_id in coco.anns]
image_ids = [coco.anns[ann_id]['image_id'] for ann_id in coco.anns]
image_files = [
os.path.join(config.train_image_dir,
coco.imgs[image_id]['file_name'])
for image_id in image_ids
]
annotations = pd.DataFrame({
'image_id': image_ids,
'image_file': image_files,
'caption': captions
})
annotations.to_csv(config.temp_annotation_file)
else:
annotations = pd.read_csv(config.temp_annotation_file)
captions = annotations['caption'].values
image_ids = annotations['image_id'].values
image_files = annotations['image_file'].values
if not os.path.exists(config.temp_data_file):
word_idxs = []
masks = []
for caption in tqdm(captions):
current_word_idxs_ = vocabulary.process_sentence(caption)
current_num_words = len(current_word_idxs_)
current_word_idxs = np.zeros(config.max_caption_length,
dtype=np.int32)
current_masks = np.zeros(config.max_caption_length)
current_word_idxs[:current_num_words] = np.array(
current_word_idxs_)
current_masks[:current_num_words] = 1.0
word_idxs.append(current_word_idxs)
masks.append(current_masks)
word_idxs = np.array(word_idxs)
masks = np.array(masks)
data = {'word_idxs': word_idxs, 'masks': masks}
np.save(config.temp_data_file, data)
else:
data = np.load(config.temp_data_file, encoding="latin1").item()
word_idxs = data['word_idxs']
masks = data['masks']
print("Captions processed.")
print("Number of captions = %d" % (len(captions)))
print("Building the dataset...")
dataset = DataSet(image_ids, image_files, config.batch_size, word_idxs,
masks, True, True)
print("Dataset built.")
return dataset
def prepare_train_data(config):
""" Prepare the data for training the model. """
coco = COCO(config.train_caption_file)
coco.filter_by_cap_len(config.max_caption_length)
print("Building the vocabulary...")
vocabulary = Vocabulary(config.vocabulary_size)
if not os.path.exists(config.vocabulary_file):
vocabulary.build(coco.all_captions())
vocabulary.save(config.vocabulary_file)
else:
vocabulary.load(config.vocabulary_file)
print("Vocabulary built.")
print("Number of words = %d" % (vocabulary.size))
coco.filter_by_words(set(vocabulary.words))
print("Processing the captions...")
if not os.path.exists(config.temp_annotation_file):
captions = [coco.anns[ann_id]['caption'] for ann_id in coco.anns]
image_ids = [coco.anns[ann_id]['image_id'] for ann_id in coco.anns]
image_files = [
os.path.join(config.train_image_dir,
coco.imgs[image_id]['file_name'])
for image_id in image_ids
]
annotations = pd.DataFrame({
'image_id': image_ids,
'image_file': image_files,
'caption': captions
})
annotations.to_csv(config.temp_annotation_file)
else:
annotations = pd.read_csv(config.temp_annotation_file)
captions = annotations['caption'].values
image_ids = annotations['image_id'].values
image_files = annotations['image_file'].values
if not os.path.exists(config.temp_data_file):
word_idxs = []
masks = []
for caption in tqdm(captions):
current_word_idxs_ = vocabulary.process_sentence(caption)
current_num_words = len(current_word_idxs_)
current_word_idxs = np.zeros(config.max_caption_length,
dtype=np.int32)
current_masks = np.zeros(config.max_caption_length)
current_word_idxs[:current_num_words] = np.array(
current_word_idxs_)
current_masks[:current_num_words] = 1.0
word_idxs.append(current_word_idxs)
masks.append(current_masks)
word_idxs = np.array(word_idxs)
masks = np.array(masks)
data = {'word_idxs': word_idxs, 'masks': masks}
np.save(config.temp_data_file, data)
else:
data = np.load(config.temp_data_file, encoding='latin1').item()
word_idxs = data['word_idxs']
masks = data['masks']
print("Captions processed.")
print("Number of captions = %d" % (len(captions)))
print("Building the dataset...")
if (config.train_data_count_limit > 0):
print("-----------------------------------------------")
print("Restricting Sz:\t", config.train_data_count_limit)
print("Batch Sz:\t", config.batch_size)
image_ids = image_ids[0:config.train_data_count_limit]
image_files = image_files[0:config.train_data_count_limit]
word_idxs = word_idxs[0:config.train_data_count_limit]
masks = masks[0:config.train_data_count_limit]
""" Dump the image paths to a file """
filepath = 'train_images.csv'
with open(filepath, 'w') as file_handler:
for i in range(0, config.train_data_count_limit):
file_handler.write("{}\n".format(image_files[i]))
#print(image_files)
print("-----------------------------------------------")
dataset = DataSet(image_ids, image_files, config.batch_size, word_idxs,
masks, True, True)
print("Dataset built.")
return dataset
def prepare_train_data(config):
""" Prepare the data for training the model. """
coco = COCO(config.train_caption_file)
coco.filter_by_cap_len(config.max_caption_length)
if config.distributed:
images = os.listdir(config.train_image_dir)
ids = [int(x[15:27]) for x in images]
print 'Input Path: ' + config.train_image_dir + ' Number of files in input path: ' + str(
int(len(ids)))
print("Building the vocabulary...")
vocabulary = Vocabulary(config.vocabulary_size)
if not os.path.exists(config.vocabulary_file):
vocabulary.build(coco.all_captions())
vocabulary.save(config.vocabulary_file)
else:
vocabulary.load(config.vocabulary_file)
print("Vocabulary built.")
print("Number of words = %d" % (vocabulary.size))
coco.filter_by_words(set(vocabulary.words))
if config.distributed:
print('Filter captions by images')
coco.filter_by_images(ids)
#print(coco.getImgIds(ids))
print("Processing the captions...")
if not os.path.exists(config.temp_annotation_file):
captions = [coco.anns[ann_id]['caption'] for ann_id in coco.anns]
image_ids = [coco.anns[ann_id]['image_id'] for ann_id in coco.anns]
image_files = [
os.path.join(config.train_image_dir,
coco.imgs[image_id]['file_name'])
for image_id in image_ids
]
annotations = pd.DataFrame({
'image_id': image_ids,
'image_file': image_files,
'caption': captions
})
annotations.set_index('image_id', inplace=True)
annotations = annotations.loc[ids]
if not config.distributed:
annotations.to_csv(config.temp_annotation_file)
else:
annotations = pd.read_csv(config.temp_annotation_file)
captions = annotations['caption'].values
image_ids = annotations['image_id'].values
image_files = annotations['image_file'].values
if not os.path.exists(config.temp_data_file):
word_idxs = []
masks = []
for caption in tqdm(captions):
current_word_idxs_ = vocabulary.process_sentence(caption)
current_num_words = len(current_word_idxs_)
current_word_idxs = np.zeros(config.max_caption_length,
dtype=np.int32)
current_masks = np.zeros(config.max_caption_length)
current_word_idxs[:current_num_words] = np.array(
current_word_idxs_)
current_masks[:current_num_words] = 1.0
word_idxs.append(current_word_idxs)
masks.append(current_masks)
word_idxs = np.array(word_idxs)
masks = np.array(masks)
data = {'word_idxs': word_idxs, 'masks': masks}
if not config.distributed:
np.save(config.temp_data_file, data)
else:
data = np.load(config.temp_data_file).item()
word_idxs = data['word_idxs']
masks = data['masks']
print("Captions processed.")
print("Number of captions = %d" % (len(captions)))
print("Building the dataset...")
dataset = DataSet(image_ids, image_files, config.batch_size, word_idxs,
masks, True, True)
print("Dataset built.")
#print "Input Path: " + config.train_image_dir + " Number of files after data preparation: " + str(len(image_files))
#print "Images IDs to be used on this server: " + str(image_ids)
return dataset
