def predict(self, dataSubType): """Predict the answers for a dataSubType. The method loads the data first. Parameters ---------- dataSubType : str "val2014" for example. """ # File names for the annotations and questions self.annFile_test = ann_file(self.dataDir, self.versionType, self.dataType, dataSubType) self.quesFile_test = ques_file(self.dataDir, self.versionType, self.taskType, self.dataType, dataSubType) # Initialize VQA api for the dataSubType on which to make predictions # on the answers self.vqa_test = VQA(self.annFile_test, self.quesFile_test) question_ids_test = [annotation["question_id"] for annotation in self.vqa_test.dataset["annotations"]] # Result list [{"answer": "yes", "question_id": 1}] res = [{"answer": "yes", "question_id": question_id} for question_id in question_ids_test] # Save the results self.results_file = save_results(res, self.dataDir, self.taskType, self.dataType, dataSubType, self.__class__.__name__)
def __init__(self, dataDir, versionType="", taskType="OpenEnded", dataType="mscoco", dataSubTypesTrain=["train2014"]): """ Parameters ---------- dataDir : str Data directory path. versionType : str, optional "" for v1.0 dataset, "v2_" for v2.0 dataset. taskType : str, optional "OpenEnded" only for v2.0. "OpenEnded" or "MultipleChoice" for v1.0. dataType : str, optional "mscoco" only for v1.0. "mscoco" for real and "abstract_v002" for abstract for v1.0. dataSubTypeTrain : list or str, optional dataSubTypes to train the method on. Can be a single string ("train2014") or a list (["train2014", "val2014"]). """ if type(dataSubTypesTrain) == list: dataSubTypesTrain = dataSubTypesTrain else: dataSubTypesTrain = [dataSubTypesTrain] self.dataDir = dataDir self.versionType = versionType self.taskType = taskType self.dataType = dataType # File names for the annotations and the questions for dataSubType in dataSubTypesTrain: setattr(self, "annFile_{}".format(dataSubType), ann_file(dataDir, versionType, dataType, dataSubType)) setattr( self, "quesFile_{}".format(dataSubType), ques_file(dataDir, versionType, taskType, dataType, dataSubType)) # Initialize VQA api for each dataSubType for dataSubType in dataSubTypesTrain: print "--> {}".format(dataSubType) setattr( self, "vqa_{}".format(dataSubType), VQA(getattr(self, "annFile_{}".format(dataSubType)), getattr(self, "quesFile_{}".format(dataSubType)))) # Merge the annotations of the two different dataSubTypesTrain self.annotations = getattr(self, "vqa_{}".format(dataSubTypesTrain[0]))\ .dataset["annotations"] if len(dataSubTypesTrain) > 1: print "--> Merging the annotations of the different dataSubTypesTrain" for dataSubType in dataSubTypesTrain[1:]: self.annotations += getattr(self, "vqa_{}".format(dataSubType))\ .dataset["annotations"]
def __init__(self, dataDir, versionType="", taskType="OpenEnded", dataType="mscoco", dataSubTypesTrain=["train2014"], dataSubTypeTest="val2014", n_train=None, n_test=None): self.dataDir = dataDir self.versionType = versionType self.taskType = taskType self.dataType = dataType if type(dataSubTypesTrain) == list: self.dataSubTypesTrain = dataSubTypesTrain else: self.dataSubTypesTrain = [dataSubTypesTrain] self.dataSubTypeTest = dataSubTypeTest # File names for the annotations and the questions for dataSubType in dataSubTypesTrain: setattr(self, "annFile_{}".format(dataSubType), ann_file(dataDir, versionType, dataType, dataSubType)) setattr( self, "quesFile_{}".format(dataSubType), ques_file(dataDir, versionType, taskType, dataType, dataSubType)) # Initialize VQA api for each dataSubType for dataSubType in dataSubTypesTrain: print("---> {}".format(dataSubType)) setattr( self, "vqa_{}".format(dataSubType), VQA(getattr(self, "annFile_{}".format(dataSubType)), getattr(self, "quesFile_{}".format(dataSubType)))) # Merge the questions of the two different dataSubTypesTrain self.questions_train = getattr(self, "vqa_{}".format( dataSubTypesTrain[0])).questions["questions"] for dic in self.questions_train: dic["data_subtype"] = dataSubTypesTrain[0] if len(dataSubTypesTrain) > 1: print( "--> Merging the annotations of the different dataSubTypesTrain" ) for dataSubType in dataSubTypesTrain[1:]: questions = getattr(self, "vqa_{}".format(dataSubType))\ .questions["questions"] for dic in questions: dic["data_subtype"] = dataSubType self.questions_train += questions # Reduce the size of the train set if n_train: self.questions_train = self.questions_train[:n_train] self.n_train = n_train # File names for the annotations and questions self.annFile_test = ann_file(self.dataDir, self.versionType, self.dataType, self.dataSubTypeTest) self.quesFile_test = ques_file(self.dataDir, self.versionType, self.taskType, self.dataType, self.dataSubTypeTest) # Initialize VQA api for the self.dataSubTypeTest on which to make predictions # on the answers print("\n") print("---> {}".format(dataSubTypeTest)) self.vqa_test = VQA(self.annFile_test, self.quesFile_test) self.questions_test = self.vqa_test.questions["questions"] for dic in self.questions_test: dic["data_subtype"] = self.dataSubTypeTest # Reduce the size of the test set self.questions_test = self.questions_test[:n_test] self.n_test = n_test # Load the skipthoughts model and initialize the encoder print("\n") print("---> Loading the skipthoughts model ...") self.model = skipthoughts.load_model() self.encoder = skipthoughts.Encoder(self.model) # Load vggnet fc7 layer print("\n") print("---> Loading the VGG16 model ...") vgg_model = keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000) self.fc7 = Model(inputs=vgg_model.input, outputs=vgg_model.get_layer("fc2").output)
def __init__(self, dataDir, versionType="", taskType="OpenEnded", dataType="mscoco", dataSubTypesTrain=["train2014"], dataSubTypeTest="val2014"): """Load train and test questions and annotations using the VQA API. Parameters ---------- dataDir : TYPE Description versionType : str, optional Description taskType : str, optional Description dataType : str, optional Description dataSubTypesTrain : list, optional Description dataSubTypeTest : str, optional Description """ self.dataDir = dataDir self.versionType = versionType self.taskType = taskType self.dataType = dataType if type(dataSubTypesTrain) == list: self.dataSubTypesTrain = dataSubTypesTrain else: self.dataSubTypesTrain = [dataSubTypesTrain] self.dataSubTypeTest = dataSubTypeTest # File names for the annotations and the questions for dataSubType in dataSubTypesTrain: setattr(self, "annFile_{}".format(dataSubType), ann_file(dataDir, versionType, dataType, dataSubType)) setattr( self, "quesFile_{}".format(dataSubType), ques_file(dataDir, versionType, taskType, dataType, dataSubType)) # Initialize VQA api for each dataSubType for dataSubType in dataSubTypesTrain: print "--> {}".format(dataSubType) setattr( self, "vqa_{}".format(dataSubType), VQA(getattr(self, "annFile_{}".format(dataSubType)), getattr(self, "quesFile_{}".format(dataSubType)))) # Merge the questions of the two different dataSubTypesTrain # Number of questions for each training set self.questions_train = getattr(self, "vqa_{}".format( dataSubTypesTrain[0])).questions["questions"] for dic in self.questions_train: dic["data_subtype"] = dataSubTypesTrain[0] if len(dataSubTypesTrain) > 1: print "--> Merging the annotations of the different dataSubTypesTrain" for dataSubType in dataSubTypesTrain[1:]: questions = getattr(self, "vqa_{}".format(dataSubType))\ .questions["questions"] for dic in questions: dic["data_subtype"] = dataSubType self.questions_train += questions # Merge the annotations of the two different dataSubTypes self.annotations_train = getattr(self, "vqa_{}".format( dataSubTypesTrain[0])).dataset["annotations"] if len(dataSubTypesTrain) > 1: print "--> Merging the annotations of the different dataSubTypesTrain" for dataSubType in dataSubTypesTrain[1:]: self.annotations_train += getattr( self, "vqa_{}".format(dataSubTypeTrain)).dataset["annotations"] # File names for the annotations and questions self.annFile_test = ann_file(self.dataDir, self.versionType, self.dataType, self.dataSubTypeTest) self.quesFile_test = ques_file(self.dataDir, self.versionType, self.taskType, self.dataType, self.dataSubTypeTest) # Initialize VQA api for the self.dataSubTypeTest on which to make predictions # on the answers print("\n") print("---> {}".format(dataSubTypeTest)) self.vqa_test = VQA(self.annFile_test, self.quesFile_test) self.questions_test = self.vqa_test.questions["questions"] for dic in self.questions_test: dic["data_subtype"] = dataSubTypeTest # Test annotations self.annotations_test = self.vqa_test.dataset["annotations"] # Keep track of the questions ids self.train_questions_ids = [ dic["question_id"] for dic in self.questions_train ] self.test_questions_ids = [ dic["question_id"] for dic in self.questions_test ]