def load_data(self):
# Load dataset
annotations = images = question_dicts = []
if self.taskType == 'all' or self.taskType == 'OpenEnded':
annotations_tmp, images_tmp, question_dicts_tmp = loadData(
cut_data=self.cut_data,
taskType='OpenEnded',
dataSubType=self.subset,
dataDir=self.originalDataDir)
annotations = annotations + annotations_tmp
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
if self.taskType == 'all' or self.taskType == 'MultipleChoice':
annotations_tmp, images_tmp, question_dicts_tmp = loadData(
cut_data=self.cut_data,
taskType='MultipleChoice',
dataSubType=self.subset,
dataDir=self.originalDataDir)
annotations = annotations + annotations_tmp
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
# Extract questions and associated answers
questions = []
answers = []
for i in range(len(question_dicts)):
# Sanity check
assert (annotations[i]["question_id"] == question_dicts[i]
["question_id"])
# Extract question and answer
questions.append(question_dicts[i]["question"])
answers.append(self.majority_voting(annotations[i]["answers"]))
# Load question and answer features
questions, answers = self.load_question_features(questions, answers)
# Get dictionary size
self.dic_size = np.max(questions) + 1
# Extract image features
image_features = []
if self.cnn == 'Baseline':
# Use pre-extracted features
image_features = self.load_image_features(annotations)
elif self.cnn == 'RawImages':
image_features = images #return image names to load in-situ in NN
else:
# Extract features using a CNN
for img in images:
image_features.append(self.cnn.extract_image_features(img))
image_features = np.vstack(image_features)
return [image_features, questions, answers, annotations]
def load_test_set(self, set_name='test2015', taskType='all'):
self.subset = 'test2015'
self.taskType = taskType
images = question_dicts = []
if self.taskType == 'all' or self.taskType == 'OpenEnded':
_, images_tmp, question_dicts_tmp = loadData(cut_data=self.cut_data, taskType='OpenEnded', dataSubType=set_name,
dataDir=self.originalDataDir, answer_type=self.answer_type)
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
if self.taskType == 'all' or self.taskType == 'MultipleChoice':
_, images_tmp, question_dicts_tmp = loadData(cut_data=self.cut_data, taskType='MultipleChoice', dataSubType=set_name,
dataDir=self.originalDataDir, answer_type=self.answer_type)
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
# Extract questions
self._original_questions = question_dicts
questions = []
for i in range(len(question_dicts)):
# Extract question and answer
questions.append(question_dicts[i]["question"])
# Load question features using precomputed dictionary
questions = [q.replace(u"\u2018", "'").replace(u"\u2019", "'") for q in questions]
np.savetxt(self.output_path + '/questions_permuted.txt', questions, fmt="%s")
questionPreprocesser = PreprocessingQuestions(filePath=self.output_path + '/questions_permuted.txt',
pad_length=self.pad_length,
threshold=self.question_threshold,
dictionary=self._question_dict)
questionPreprocesser.preprocess()
if self.questions_sparse:
questions = questionPreprocesser.to_indices()
else:
questions = questionPreprocesser.onehotvectormatrix()
# Get dictionary size
self.dic_size = len(self._question_dict) + 1
# Extract image features
image_features = []
if self.cnn == 'Baseline':
# Use pre-extracted features
image_features = self.load_image_features(question_dicts)
elif self.cnn == 'RawImages':
image_features = images
else:
# Extract features using a CNN
for img in images:
image_features.append(self.cnn.extract_image_features(img))
image_features = np.vstack(image_features)
return [image_features, questions]
def generate_input_file(self):
# Get images
# Load dataset
images = []
if self.taskType == 'all' or self.taskType == 'OpenEnded':
_, images_tmp, _ = loadData(cut_data=self.cut_data, taskType='OpenEnded', dataSubType=self.subset, dataDir=self.originalDataDir)
images = images + images_tmp
if self.taskType == 'all' or self.taskType == 'MultipleChoice':
_, images_tmp, _ = loadData(cut_data=self.cut_data, taskType='MultipleChoice', dataSubType=self.subset, dataDir=self.originalDataDir)
images = images + images_tmp
images = ['../' + i for i in images] # Adjust path
images.append('^C')
images = np.unique(np.array(images))
np.savetxt(self.output_folder + '/' + self.input_fileName, images, fmt='%s')
self.images = images[0:-1]
def load_data(self):
# Load dataset
annotations = images = question_dicts = []
if self.subset == 'trainval' or self.subset == 'train2014':
if self.taskType == 'all' or self.taskType == 'OpenEnded':
annotations_tmp, images_tmp, question_dicts_tmp = loadData(cut_data=self.cut_data, taskType='OpenEnded', dataSubType='train2014',
dataDir=self.originalDataDir, answer_type=self.answer_type)
annotations = annotations + annotations_tmp
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
if self.taskType == 'all' or self.taskType == 'MultipleChoice':
annotations_tmp, images_tmp, question_dicts_tmp = loadData(cut_data=self.cut_data, taskType='MultipleChoice', dataSubType='train2014',
dataDir=self.originalDataDir, answer_type=self.answer_type)
annotations = annotations + annotations_tmp
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
if self.subset == 'trainval' or self.subset == 'val2014':
if self.taskType == 'all' or self.taskType == 'OpenEnded':
annotations_tmp, images_tmp, question_dicts_tmp = loadData(cut_data=self.cut_data, taskType='OpenEnded', dataSubType='val2014',
dataDir=self.originalDataDir, answer_type=self.answer_type)
annotations = annotations + annotations_tmp
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
if self.taskType == 'all' or self.taskType == 'MultipleChoice':
annotations_tmp, images_tmp, question_dicts_tmp = loadData(cut_data=self.cut_data, taskType='MultipleChoice', dataSubType='val2014',
dataDir=self.originalDataDir, answer_type=self.answer_type)
annotations = annotations + annotations_tmp
images = images + images_tmp
question_dicts = question_dicts + question_dicts_tmp
# Extract questions and associated answers
self._original_questions = question_dicts
questions = []
for i in range(len(question_dicts)):
# Sanity check
assert(annotations[i]["question_id"] == question_dicts[i]["question_id"])
# Extract question and answer
questions.append(question_dicts[i]["question"])
# Load question features
questions = self.load_question_features(questions)
# Get dictionary size
self.dic_size = len(self._question_dict) + 1
# Load answers
if self._answer_to_int == None:
self.create_answer_dictionary(annotations)
answers = self.create_answer_matrix(annotations)
# Extract image features
image_features = []
if self.cnn == 'Baseline':
# Use pre-extracted features
image_features = self.load_image_features(annotations)
elif self.cnn == 'RawImages':
image_features = images
else:
# Extract features using a CNN
for img in images:
image_features.append(self.cnn.extract_image_features(img))
image_features = np.vstack(image_features)
return [image_features, questions, answers, annotations]