def predict_answer_class(self, img_path, question): f = self.fe.extract_image_features(img_path) questions_matrix_shape = self.meta['questions_matrix_shape'] if len(questions_matrix_shape) == 2: max_seq_length = questions_matrix_shape[0] question_matrix = np.zeros(shape=(1, max_seq_length, 300)) words = word_tokenize(question.lower()) for i, word in enumerate(words[0:min(max_seq_length, len(words))]): question_matrix[0, i, :] = self.glove_model.encode_word(word) input_data = [ f.as_in_context(self.model_ctx), nd.array(question_matrix, ctx=self.model_ctx).reshape(1, max_seq_length * 300) ] output = self.model(input_data) return nd.argmax(output, axis=1).astype(np.uint8).asscalar() else: words = word_tokenize(question.lower()) E = np.zeros(shape=(300, len(words))) for j, word in enumerate(words): E[:, j] = self.glove_model.encode_word(word) question_matrix = np.sum(E, axis=1) input_data = [ f.as_in_context(self.model_ctx), nd.array(question_matrix, ctx=self.model_ctx).reshape(1, 300) ] output = self.model(input_data) return nd.argmax(output, axis=1).astype(np.uint8).asscalar()
def get_questions_matrix_sum(data_dir_path, max_lines_retrieved=-1, split='val'): questions = get_questions(data_dir_path, max_lines_retrieved, split) glove_word2emb = glove_word2emb_300(data_dir_path) logging.debug('glove: %d words loaded', len(glove_word2emb)) seq_list = [] for i, question in enumerate(questions): words = word_tokenize(question[0].lower()) E = np.zeros(shape=(300, len(words))) for j, word in enumerate(words): if word in glove_word2emb: emb = glove_word2emb[word] E[:, j] = emb E = np.sum(E, axis=1) if (i + 1) % 10000 == 0: logging.debug('loaded %d questions', i + 1) seq_list.append(E) question_matrix = np.array(seq_list) return question_matrix
def get_questions_matrix_concat(data_dir_path, max_lines_retrieved=-1, split='val', max_sequence_length=-1): questions = get_questions(data_dir_path, max_lines_retrieved, split) glove_word2emb = glove_word2emb_300(data_dir_path) logging.debug('glove: %d words loaded', len(glove_word2emb)) seq_list = [] for i, question in enumerate(questions): words = word_tokenize(question[0].lower()) seq = [] for word in words: emb = np.zeros(shape=300) if word in glove_word2emb: emb = glove_word2emb[word] seq.append(emb) if (i + 1) % 10000 == 0: logging.debug('loaded %d questions', i + 1) seq_list.append(seq) question_matrix = pad_sequences(seq_list, max_sequence_length=max_sequence_length) return question_matrix