def _compute_sequence_encodings(
self,
batch_examples: List[Message],
module: Any,
attribute: Text = TEXT) -> Tuple[np.ndarray, List[int]]:
list_of_tokens = [
example.get(TOKENS_NAMES[attribute]) for example in batch_examples
]
number_of_tokens_in_sentence = [
len(sent_tokens) for sent_tokens in list_of_tokens
]
# join the tokens to get a clean text to ensure the sequence length of
# the returned embeddings from ConveRT matches the length of the tokens
# (including sub-tokens)
tokenized_texts = self._tokens_to_text(list_of_tokens)
token_features = self._sequence_encoding_of_text(
tokenized_texts, module)
# ConveRT might split up tokens into sub-tokens
# take the mean of the sub-token vectors and use that as the token vector
token_features = train_utils.align_token_features(
list_of_tokens, token_features)
return token_features, number_of_tokens_in_sentence
def test_align_token_features_convert():
tokens = [
Token("This", 0, data={NUMBER_OF_SUB_TOKENS: 1}),
Token("is", 5, data={NUMBER_OF_SUB_TOKENS: 1}),
Token("a", 8, data={NUMBER_OF_SUB_TOKENS: 1}),
Token("sentence", 10, data={NUMBER_OF_SUB_TOKENS: 2}),
Token("embedding", 19, data={NUMBER_OF_SUB_TOKENS: 4}),
]
seq_dim = sum(t.get(NUMBER_OF_SUB_TOKENS) for t in tokens)
token_features = np.random.rand(1, seq_dim, 64)
actual_features = train_utils.align_token_features([tokens], token_features)
assert np.all(actual_features[0][0] == token_features[0][0])
assert np.all(actual_features[0][1] == token_features[0][1])
assert np.all(actual_features[0][2] == token_features[0][2])
# sentence is split into 2 sub-tokens
assert np.all(actual_features[0][3] == np.mean(token_features[0][3:5], axis=0))
# embedding is split into 4 sub-tokens
assert np.all(actual_features[0][4] == np.mean(token_features[0][5:10], axis=0))
def _get_model_features_for_batch(
self,
batch_token_ids: List[List[int]],
batch_tokens: List[List[Token]],
batch_examples: List[Message],
attribute: Text,
inference_mode: bool = False,
) -> Tuple[np.ndarray, np.ndarray]:
"""Computes dense features of each example in the batch.
We first add the special tokens corresponding to each language model. Next, we
add appropriate padding and compute a mask for that padding so that it doesn't
affect the feature computation. The padded batch is next fed to the language
model and token level embeddings are computed. Using the pre-computed mask,
embeddings for non-padding tokens are extracted and subsequently sentence
level embeddings are computed.
Args:
batch_token_ids: List of token ids of each example in the batch.
batch_tokens: List of token objects for each example in the batch.
batch_examples: List of examples in the batch.
attribute: attribute of the Message object to be processed.
inference_mode: Whether the call is during training or during inference.
Returns:
Sentence and token level dense representations.
"""
# Let's first add tokenizer specific special tokens to all examples
batch_token_ids_augmented = self._add_lm_specific_special_tokens(
batch_token_ids)
# Compute sequence lengths for all examples
(
actual_sequence_lengths,
max_input_sequence_length,
) = self._extract_sequence_lengths(batch_token_ids_augmented)
# Validate that all sequences can be processed based on their sequence
# lengths and the maximum sequence length the model can handle
self._validate_sequence_lengths(actual_sequence_lengths,
batch_examples, attribute,
inference_mode)
# Add padding so that whole batch can be fed to the model
padded_token_ids = self._add_padding_to_batch(
batch_token_ids_augmented, max_input_sequence_length)
# Compute attention mask based on actual_sequence_length
batch_attention_mask = self._compute_attention_mask(
actual_sequence_lengths, max_input_sequence_length)
# Get token level features from the model
sequence_hidden_states = self._compute_batch_sequence_features(
batch_attention_mask, padded_token_ids)
# Extract features for only non-padding tokens
sequence_nonpadded_embeddings = self._extract_nonpadded_embeddings(
sequence_hidden_states, actual_sequence_lengths)
# Extract sentence level and post-processed features
(
sentence_embeddings,
sequence_embeddings,
) = self._post_process_sequence_embeddings(
sequence_nonpadded_embeddings)
# Pad zeros for examples which were truncated in inference mode.
# This is intentionally done after sentence embeddings have been
# extracted so that they are not affected
sequence_embeddings = self._add_extra_padding(sequence_embeddings,
actual_sequence_lengths)
# shape of matrix for all sequence embeddings
batch_dim = len(sequence_embeddings)
seq_dim = max(e.shape[0] for e in sequence_embeddings)
feature_dim = sequence_embeddings[0].shape[1]
shape = (batch_dim, seq_dim, feature_dim)
# align features with tokens so that we have just one vector per token
# (don't include sub-tokens)
sequence_embeddings = train_utils.align_token_features(
batch_tokens, sequence_embeddings, shape)
# sequence_embeddings is a padded numpy array
# remove the padding, keep just the non-zero vectors
sequence_final_embeddings = []
for embeddings, tokens in zip(sequence_embeddings, batch_tokens):
sequence_final_embeddings.append(embeddings[:len(tokens)])
sequence_final_embeddings = np.array(sequence_final_embeddings)
return sentence_embeddings, sequence_final_embeddings
def _get_model_features_for_batch(
self, batch_token_ids: List[List[int]],
batch_tokens: List[List[Token]]) -> Tuple[np.ndarray, np.ndarray]:
"""Compute dense features of each example in the batch.
We first add the special tokens corresponding to each language model. Next, we
add appropriate padding and compute a mask for that padding so that it doesn't
affect the feature computation. The padded batch is next fed to the language
model and token level embeddings are computed. Using the pre-computed mask,
embeddings for non-padding tokens are extracted and subsequently sentence
level embeddings are computed.
Args:
batch_token_ids: List of token ids of each example in the batch.
Returns:
Sentence and token level dense representations.
"""
# Let's first add tokenizer specific special tokens to all examples
batch_token_ids_augmented = self._add_lm_specific_special_tokens(
batch_token_ids)
# Let's first add padding so that whole batch can be fed to the model
actual_sequence_lengths, padded_token_ids = self._add_padding_to_batch(
batch_token_ids_augmented)
# Compute attention mask based on actual_sequence_length
batch_attention_mask = self._compute_attention_mask(
actual_sequence_lengths)
# Get token level features from the model
sequence_hidden_states = self._compute_batch_sequence_features(
batch_attention_mask, padded_token_ids)
# Extract features for only non-padding tokens
sequence_nonpadded_embeddings = self._extract_nonpadded_embeddings(
sequence_hidden_states, actual_sequence_lengths)
# Extract sentence level and post-processed features
(
sentence_embeddings,
sequence_embeddings,
) = self._post_process_sequence_embeddings(
sequence_nonpadded_embeddings)
# shape of matrix for all sequence embeddings
batch_dim = len(sequence_embeddings)
seq_dim = max(e.shape[0] for e in sequence_embeddings)
feature_dim = sequence_embeddings[0].shape[1]
shape = (batch_dim, seq_dim, feature_dim)
# align features with tokens so that we have just one vector per token
# (don't include sub-tokens)
sequence_embeddings = train_utils.align_token_features(
batch_tokens, sequence_embeddings, shape)
# sequence_embeddings is a padded numpy array
# remove the padding, keep just the non-zero vectors
sequence_final_embeddings = []
for embeddings, tokens in zip(sequence_embeddings, batch_tokens):
sequence_final_embeddings.append(embeddings[:len(tokens)])
sequence_final_embeddings = np.array(sequence_final_embeddings)
return sentence_embeddings, sequence_final_embeddings
