def encode(self, texts: Union[List[str], List[List[str]]], pooling: str, max_seq_length: int, is_tokenized: bool = False, **kwargs) -> Optional[np.array]: input_ids, input_masks, segment_ids = [], [], [] for text in tqdm(texts, desc="Converting texts to features"): input_id, input_mask, segment_id = self._model_single_input( text, max_seq_length, is_tokenized) input_ids.append(input_id) input_masks.append(input_mask) segment_ids.append(segment_id) albert_inputs = dict(input_ids=np.array(input_ids), input_mask=np.array(input_masks), segment_ids=np.array(segment_ids)) bert_outputs = self.albert_module(albert_inputs, signature="tokens", as_dict=True) sequence_output = bert_outputs["sequence_output"] token_embeddings = self.sess.run(sequence_output) if not pooling: return token_embeddings else: if pooling not in POOL_FUNC_MAP.keys(): print(f"Pooling method \"{pooling}\" not implemented") return None pooling_func = POOL_FUNC_MAP[pooling] pooled = pooling_func(token_embeddings, axis=1) return pooled
def encode(self, texts: Union[List[str], List[List[str]]], pooling: str, max_seq_length: int, is_tokenized: bool = False, **kwargs) -> Optional[np.array]: tokenized_texts = texts if not is_tokenized: tokenized_texts = [Embeddings.tokenize(text) for text in texts] tokenized_text_words = [[ self.word2idx.get(w, self.word2idx['unk']) for w in text ] for text in tokenized_texts] embeddings = [] for x in tokenized_text_words: x = np.reshape(x, (1, len(x))) embeddings.append(self.ulmfit_model.predict(x)[1][0]) if not pooling: return embeddings else: if pooling not in POOL_FUNC_MAP.keys(): raise NotImplementedError( f"Pooling method \"{pooling}\" not implemented") pooling_func = POOL_FUNC_MAP[pooling] pooled = pooling_func(embeddings, axis=1) return pooled
def encode(self, texts: Union[List[str], List[List[str]]], pooling: str, is_tokenized: bool = False, **kwargs ) -> Optional[np.array]: text_tokens = texts if not is_tokenized: text_tokens = [Embeddings.tokenize(text) for text in texts] text_tokens = [Embeddings.padded_tokens(tokens, self.max_seq_length) for tokens in text_tokens] seq_length = [self.max_seq_length] * len(texts) elmo_inputs = { self.tokens: np.array(text_tokens), self.sequence_len: np.array(seq_length) } token_embeddings = self.sess.run(self.elmo_outputs, feed_dict=elmo_inputs)["elmo"] if not pooling: return token_embeddings else: if pooling not in POOL_FUNC_MAP.keys(): print(f"Pooling method \"{pooling}\" not implemented") return None pooling_func = POOL_FUNC_MAP[pooling] pooled = pooling_func(token_embeddings, axis=1) return pooled
def encode(self, texts: Union[List[str], List[List[str]]], pooling: str, max_seq_length: int, is_tokenized: bool = False, **kwargs ) -> Optional[np.array]: text_tokens = texts if not is_tokenized: text_tokens = [Embeddings.tokenize(text) for text in texts] if max_seq_length: text_tokens = [Embeddings.padded_tokens(tokens, max_seq_length) for tokens in text_tokens] seq_length = [max_seq_length] * len(texts) else: seq_length = [len(tokens) for tokens in text_tokens] sequence_output = self.elmo_module(inputs={"tokens": text_tokens, "sequence_len": seq_length}, signature="tokens", as_dict=True)["elmo"] token_embeddings = self.sess.run(sequence_output) if not pooling: return token_embeddings else: if pooling not in POOL_FUNC_MAP.keys(): print(f"Pooling method \"{pooling}\" not implemented") return None pooling_func = POOL_FUNC_MAP[pooling] pooled = pooling_func(token_embeddings, axis=1) return pooled
def encode(self, texts: Union[List[str], List[List[str]]], pooling: str, is_tokenized: bool = False, **kwargs ) -> Optional[np.array]: oov_vector = np.zeros(Embeddings.EMBEDDING_MODELS[self.model_name].dimensions, dtype="float32") token_embeddings = np.array([self._single_encode_text(text, oov_vector, is_tokenized) for text in texts]) if not pooling: return token_embeddings else: if pooling not in POOL_FUNC_MAP.keys(): raise NotImplementedError(f"Pooling method \"{pooling}\" not implemented") pooling_func = POOL_FUNC_MAP[pooling] pooled = pooling_func(token_embeddings, axis=1) return pooled
def encode(self, texts: Union[List[str], List[List[str]]], pooling: str, max_seq_length: int, is_tokenized: bool = False, **kwargs) -> Optional[np.array]: input_ids, input_masks, segment_ids = [], [], [] for text in tqdm(texts, desc="Converting texts to features"): input_id, input_mask, segment_id = self._model_single_input( text, max_seq_length, is_tokenized) input_ids.append(input_id) input_masks.append(input_mask) segment_ids.append(segment_id) # Construct an XLNet model xlnet_model = xlnet.XLNetModel(xlnet_config=self.xlnet_config, run_config=self.run_config, input_ids=np.array(input_ids, dtype=np.int32), seg_ids=np.array(segment_ids, dtype=np.int32), input_mask=np.array(input_masks, dtype=np.float32)) self.sess.run(tf.initializers.global_variables()) # Get a sequence output sequence_output = xlnet_model.get_sequence_output() token_embeddings = self.sess.run(sequence_output) if not pooling: return token_embeddings else: if pooling not in POOL_FUNC_MAP.keys(): raise NotImplementedError( f"Pooling method \"{pooling}\" not implemented") pooling_func = POOL_FUNC_MAP[pooling] pooled = pooling_func(token_embeddings, axis=1) return pooled