def predict(self, dataSubType):
"""Predict the answers for a dataSubType. The method loads the data
first.
Parameters
----------
dataSubType : str
"val2014" for example.
"""
# File names for the annotations and questions
self.annFile_test = ann_file(self.dataDir, self.versionType,
self.dataType, dataSubType)
self.quesFile_test = ques_file(self.dataDir, self.versionType,
self.taskType, self.dataType, dataSubType)
# Initialize VQA api for the dataSubType on which to make predictions
# on the answers
self.vqa_test = VQA(self.annFile_test, self.quesFile_test)
question_ids_test = [annotation["question_id"] for annotation
in self.vqa_test.dataset["annotations"]]
# Result list [{"answer": "yes", "question_id": 1}]
res = [{"answer": "yes", "question_id": question_id}
for question_id in question_ids_test]
# Save the results
self.results_file = save_results(res, self.dataDir, self.taskType,
self.dataType, dataSubType,
self.__class__.__name__)
def __init__(self,
dataDir,
versionType="",
taskType="OpenEnded",
dataType="mscoco",
dataSubTypesTrain=["train2014"]):
"""
Parameters
----------
dataDir : str
Data directory path.
versionType : str, optional
"" for v1.0 dataset, "v2_" for v2.0 dataset.
taskType : str, optional
"OpenEnded" only for v2.0. "OpenEnded" or "MultipleChoice" for v1.0.
dataType : str, optional
"mscoco" only for v1.0. "mscoco" for real and "abstract_v002" for
abstract for v1.0.
dataSubTypeTrain : list or str, optional
dataSubTypes to train the method on. Can be a single string
("train2014") or a list (["train2014", "val2014"]).
"""
if type(dataSubTypesTrain) == list:
dataSubTypesTrain = dataSubTypesTrain
else:
dataSubTypesTrain = [dataSubTypesTrain]
self.dataDir = dataDir
self.versionType = versionType
self.taskType = taskType
self.dataType = dataType
# File names for the annotations and the questions
for dataSubType in dataSubTypesTrain:
setattr(self, "annFile_{}".format(dataSubType),
ann_file(dataDir, versionType, dataType, dataSubType))
setattr(
self, "quesFile_{}".format(dataSubType),
ques_file(dataDir, versionType, taskType, dataType,
dataSubType))
# Initialize VQA api for each dataSubType
for dataSubType in dataSubTypesTrain:
print "--> {}".format(dataSubType)
setattr(
self, "vqa_{}".format(dataSubType),
VQA(getattr(self, "annFile_{}".format(dataSubType)),
getattr(self, "quesFile_{}".format(dataSubType))))
# Merge the annotations of the two different dataSubTypesTrain
self.annotations = getattr(self, "vqa_{}".format(dataSubTypesTrain[0]))\
.dataset["annotations"]
if len(dataSubTypesTrain) > 1:
print "--> Merging the annotations of the different dataSubTypesTrain"
for dataSubType in dataSubTypesTrain[1:]:
self.annotations += getattr(self, "vqa_{}".format(dataSubType))\
.dataset["annotations"]
def __init__(self,
dataDir,
versionType="",
taskType="OpenEnded",
dataType="mscoco",
dataSubTypesTrain=["train2014"],
dataSubTypeTest="val2014",
n_train=None,
n_test=None):
self.dataDir = dataDir
self.versionType = versionType
self.taskType = taskType
self.dataType = dataType
if type(dataSubTypesTrain) == list:
self.dataSubTypesTrain = dataSubTypesTrain
else:
self.dataSubTypesTrain = [dataSubTypesTrain]
self.dataSubTypeTest = dataSubTypeTest
# File names for the annotations and the questions
for dataSubType in dataSubTypesTrain:
setattr(self, "annFile_{}".format(dataSubType),
ann_file(dataDir, versionType, dataType, dataSubType))
setattr(
self, "quesFile_{}".format(dataSubType),
ques_file(dataDir, versionType, taskType, dataType,
dataSubType))
# Initialize VQA api for each dataSubType
for dataSubType in dataSubTypesTrain:
print("---> {}".format(dataSubType))
setattr(
self, "vqa_{}".format(dataSubType),
VQA(getattr(self, "annFile_{}".format(dataSubType)),
getattr(self, "quesFile_{}".format(dataSubType))))
# Merge the questions of the two different dataSubTypesTrain
self.questions_train = getattr(self, "vqa_{}".format(
dataSubTypesTrain[0])).questions["questions"]
for dic in self.questions_train:
dic["data_subtype"] = dataSubTypesTrain[0]
if len(dataSubTypesTrain) > 1:
print(
"--> Merging the annotations of the different dataSubTypesTrain"
)
for dataSubType in dataSubTypesTrain[1:]:
questions = getattr(self, "vqa_{}".format(dataSubType))\
.questions["questions"]
for dic in questions:
dic["data_subtype"] = dataSubType
self.questions_train += questions
# Reduce the size of the train set
if n_train:
self.questions_train = self.questions_train[:n_train]
self.n_train = n_train
# File names for the annotations and questions
self.annFile_test = ann_file(self.dataDir, self.versionType,
self.dataType, self.dataSubTypeTest)
self.quesFile_test = ques_file(self.dataDir, self.versionType,
self.taskType, self.dataType,
self.dataSubTypeTest)
# Initialize VQA api for the self.dataSubTypeTest on which to make predictions
# on the answers
print("\n")
print("---> {}".format(dataSubTypeTest))
self.vqa_test = VQA(self.annFile_test, self.quesFile_test)
self.questions_test = self.vqa_test.questions["questions"]
for dic in self.questions_test:
dic["data_subtype"] = self.dataSubTypeTest
# Reduce the size of the test set
self.questions_test = self.questions_test[:n_test]
self.n_test = n_test
# Load the skipthoughts model and initialize the encoder
print("\n")
print("---> Loading the skipthoughts model ...")
self.model = skipthoughts.load_model()
self.encoder = skipthoughts.Encoder(self.model)
# Load vggnet fc7 layer
print("\n")
print("---> Loading the VGG16 model ...")
vgg_model = keras.applications.vgg16.VGG16(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.fc7 = Model(inputs=vgg_model.input,
outputs=vgg_model.get_layer("fc2").output)
def __init__(self,
dataDir,
versionType="",
taskType="OpenEnded",
dataType="mscoco",
dataSubTypesTrain=["train2014"],
dataSubTypeTest="val2014"):
"""Load train and test questions and annotations using the VQA API.
Parameters
----------
dataDir : TYPE
Description
versionType : str, optional
Description
taskType : str, optional
Description
dataType : str, optional
Description
dataSubTypesTrain : list, optional
Description
dataSubTypeTest : str, optional
Description
"""
self.dataDir = dataDir
self.versionType = versionType
self.taskType = taskType
self.dataType = dataType
if type(dataSubTypesTrain) == list:
self.dataSubTypesTrain = dataSubTypesTrain
else:
self.dataSubTypesTrain = [dataSubTypesTrain]
self.dataSubTypeTest = dataSubTypeTest
# File names for the annotations and the questions
for dataSubType in dataSubTypesTrain:
setattr(self, "annFile_{}".format(dataSubType),
ann_file(dataDir, versionType, dataType, dataSubType))
setattr(
self, "quesFile_{}".format(dataSubType),
ques_file(dataDir, versionType, taskType, dataType,
dataSubType))
# Initialize VQA api for each dataSubType
for dataSubType in dataSubTypesTrain:
print "--> {}".format(dataSubType)
setattr(
self, "vqa_{}".format(dataSubType),
VQA(getattr(self, "annFile_{}".format(dataSubType)),
getattr(self, "quesFile_{}".format(dataSubType))))
# Merge the questions of the two different dataSubTypesTrain
# Number of questions for each training set
self.questions_train = getattr(self, "vqa_{}".format(
dataSubTypesTrain[0])).questions["questions"]
for dic in self.questions_train:
dic["data_subtype"] = dataSubTypesTrain[0]
if len(dataSubTypesTrain) > 1:
print "--> Merging the annotations of the different dataSubTypesTrain"
for dataSubType in dataSubTypesTrain[1:]:
questions = getattr(self, "vqa_{}".format(dataSubType))\
.questions["questions"]
for dic in questions:
dic["data_subtype"] = dataSubType
self.questions_train += questions
# Merge the annotations of the two different dataSubTypes
self.annotations_train = getattr(self, "vqa_{}".format(
dataSubTypesTrain[0])).dataset["annotations"]
if len(dataSubTypesTrain) > 1:
print "--> Merging the annotations of the different dataSubTypesTrain"
for dataSubType in dataSubTypesTrain[1:]:
self.annotations_train += getattr(
self,
"vqa_{}".format(dataSubTypeTrain)).dataset["annotations"]
# File names for the annotations and questions
self.annFile_test = ann_file(self.dataDir, self.versionType,
self.dataType, self.dataSubTypeTest)
self.quesFile_test = ques_file(self.dataDir, self.versionType,
self.taskType, self.dataType,
self.dataSubTypeTest)
# Initialize VQA api for the self.dataSubTypeTest on which to make predictions
# on the answers
print("\n")
print("---> {}".format(dataSubTypeTest))
self.vqa_test = VQA(self.annFile_test, self.quesFile_test)
self.questions_test = self.vqa_test.questions["questions"]
for dic in self.questions_test:
dic["data_subtype"] = dataSubTypeTest
# Test annotations
self.annotations_test = self.vqa_test.dataset["annotations"]
# Keep track of the questions ids
self.train_questions_ids = [
dic["question_id"] for dic in self.questions_train
]
self.test_questions_ids = [
dic["question_id"] for dic in self.questions_test
]