def _create_label_data(
self,
training_data: TrainingData,
label_id_dict: Dict[Text, int],
attribute: Text,
) -> RasaModelData:
"""Create matrix with label_ids encoded in rows as bag of words.
Find a training example for each label and get the encoded features
from the corresponding Message object.
If the features are already computed, fetch them from the message object
else compute a one hot encoding for the label as the feature vector.
"""
# Collect one example for each label
labels_idx_examples = []
for label_name, idx in label_id_dict.items():
label_example = self._find_example_for_label(
label_name, training_data.intent_examples, attribute)
labels_idx_examples.append((idx, label_example))
# Sort the list of tuples based on label_idx
labels_idx_examples = sorted(labels_idx_examples, key=lambda x: x[0])
labels_example = [example for (_, example) in labels_idx_examples]
# Collect features, precomputed if they exist, else compute on the fly
if self._check_labels_features_exist(labels_example, attribute):
features = self._extract_labels_precomputed_features(
labels_example, attribute)
else:
features = self._compute_default_label_features(labels_example)
label_data = RasaModelData()
label_data.add_features(LABEL_FEATURES, features)
label_ids = np.array([idx for (idx, _) in labels_idx_examples])
# explicitly add last dimension to label_ids
# to track correctly dynamic sequences
label_data.add_features(LABEL_IDS, [np.expand_dims(label_ids, -1)])
label_data.add_mask(LABEL_MASK, LABEL_FEATURES)
return label_data
def _create_model_data(
self,
training_data: List[Message],
label_id_dict: Optional[Dict[Text, int]] = None,
tag_id_dict: Optional[Dict[Text, int]] = None,
label_attribute: Optional[Text] = None,
) -> RasaModelData:
"""Prepare data for training and create a RasaModelData object"""
X_sparse = []
X_dense = []
Y_sparse = []
Y_dense = []
label_ids = []
tag_ids = []
for e in training_data:
if label_attribute is None or e.get(label_attribute):
_sparse, _dense = self._extract_features(e, TEXT)
if _sparse is not None:
X_sparse.append(_sparse)
if _dense is not None:
X_dense.append(_dense)
if e.get(label_attribute):
_sparse, _dense = self._extract_features(e, label_attribute)
if _sparse is not None:
Y_sparse.append(_sparse)
if _dense is not None:
Y_dense.append(_dense)
if label_id_dict:
label_ids.append(label_id_dict[e.get(label_attribute)])
if self.component_config.get(ENTITY_RECOGNITION) and tag_id_dict:
if self.component_config[BILOU_FLAG]:
_tags = bilou_utils.tags_to_ids(e, tag_id_dict)
else:
_tags = []
for t in e.get(TOKENS_NAMES[TEXT]):
_tag = determine_token_labels(t, e.get(ENTITIES), None)
_tags.append(tag_id_dict[_tag])
# transpose to have seq_len x 1
tag_ids.append(np.array([_tags]).T)
X_sparse = np.array(X_sparse)
X_dense = np.array(X_dense)
Y_sparse = np.array(Y_sparse)
Y_dense = np.array(Y_dense)
label_ids = np.array(label_ids)
tag_ids = np.array(tag_ids)
model_data = RasaModelData(label_key=self.label_key)
model_data.add_features(TEXT_FEATURES, [X_sparse, X_dense])
model_data.add_features(LABEL_FEATURES, [Y_sparse, Y_dense])
if label_attribute and model_data.feature_not_exist(LABEL_FEATURES):
# no label features are present, get default features from _label_data
model_data.add_features(
LABEL_FEATURES, self._use_default_label_features(label_ids))
# explicitly add last dimension to label_ids
# to track correctly dynamic sequences
model_data.add_features(LABEL_IDS, [np.expand_dims(label_ids, -1)])
model_data.add_features(TAG_IDS, [tag_ids])
model_data.add_mask(TEXT_MASK, TEXT_FEATURES)
model_data.add_mask(LABEL_MASK, LABEL_FEATURES)
return model_data
