def test_load_multi_file_training_data(default_domain):
# the stories file in `data/test_multifile_stories` is the same as in
# `data/test_stories/stories.md`, but split across multiple files
featurizer = MaxHistoryTrackerFeaturizer(BinarySingleStateFeaturizer(),
max_history=2)
trackers = training.load_data("data/test_stories/stories.md",
default_domain,
augmentation_factor=0)
(tr_as_sts, tr_as_acts) = featurizer.training_states_and_actions(
trackers, default_domain)
hashed = []
for sts, acts in zip(tr_as_sts, tr_as_acts):
hashed.append(json.dumps(sts + acts, sort_keys=True))
hashed = sorted(hashed, reverse=True)
data = featurizer.featurize_trackers(trackers, default_domain)
featurizer_mul = MaxHistoryTrackerFeaturizer(BinarySingleStateFeaturizer(),
max_history=2)
trackers_mul = training.load_data("data/test_multifile_stories",
default_domain,
augmentation_factor=0)
(tr_as_sts_mul, tr_as_acts_mul) = featurizer.training_states_and_actions(
trackers_mul, default_domain)
hashed_mul = []
for sts_mul, acts_mul in zip(tr_as_sts_mul, tr_as_acts_mul):
hashed_mul.append(json.dumps(sts_mul + acts_mul, sort_keys=True))
hashed_mul = sorted(hashed_mul, reverse=True)
data_mul = featurizer_mul.featurize_trackers(trackers_mul, default_domain)
assert hashed == hashed_mul
assert np.all(data.X.sort(axis=0) == data_mul.X.sort(axis=0))
assert np.all(data.y.sort(axis=0) == data_mul.y.sort(axis=0))
def test_create_train_data_no_history(default_domain):
featurizer = MaxHistoryTrackerFeaturizer(max_history=1)
training_trackers = training.load_data(
DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0
)
assert len(training_trackers) == 3
(decoded, _) = featurizer.training_states_and_actions(
training_trackers, default_domain)
# decoded needs to be sorted
hashed = []
for states in decoded:
hashed.append(json.dumps(states, sort_keys=True))
hashed = sorted(hashed, reverse=True)
assert hashed == [
'[{}]',
'[{"intent_greet": 1.0, "prev_utter_greet": 1.0}]',
'[{"intent_greet": 1.0, "prev_action_listen": 1.0}]',
'[{"intent_goodbye": 1.0, "prev_utter_goodbye": 1.0}]',
'[{"intent_goodbye": 1.0, "prev_action_listen": 1.0}]',
'[{"intent_default": 1.0, "prev_utter_default": 1.0}]',
'[{"intent_default": 1.0, "prev_utter_default": 1.0, '
'"slot_name_0": 1.0}]',
'[{"intent_default": 1.0, "prev_action_listen": 1.0}]',
'[{"intent_default": 1.0, "prev_action_listen": 1.0, '
'"slot_name_0": 1.0}]',
'[{"entity_name": 1.0, "intent_greet": 1.0, '
'"prev_utter_greet": 1.0, "slot_name_0": 1.0}]',
'[{"entity_name": 1.0, "intent_greet": 1.0, '
'"prev_action_listen": 1.0, "slot_name_0": 1.0}]'
]
async def test_create_train_data_no_history(default_domain):
featurizer = MaxHistoryTrackerFeaturizer(max_history=1)
training_trackers = await training.load_data(DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0)
assert len(training_trackers) == 3
(decoded,
_) = featurizer.training_states_and_actions(training_trackers,
default_domain)
# decoded needs to be sorted
hashed = []
for states in decoded:
hashed.append(json.dumps(states, sort_keys=True))
hashed = sorted(hashed, reverse=True)
assert hashed == [
'[{}]', '[{"intent_greet": 1.0, "prev_utter_greet": 1.0}]',
'[{"intent_greet": 1.0, "prev_action_listen": 1.0}]',
'[{"intent_goodbye": 1.0, "prev_utter_goodbye": 1.0}]',
'[{"intent_goodbye": 1.0, "prev_action_listen": 1.0}]',
'[{"intent_default": 1.0, "prev_utter_default": 1.0}]',
'[{"intent_default": 1.0, "prev_utter_default": 1.0, '
'"slot_name_0": 1.0}]',
'[{"intent_default": 1.0, "prev_action_listen": 1.0}]',
'[{"intent_default": 1.0, "prev_action_listen": 1.0, '
'"slot_name_0": 1.0}]', '[{"entity_name": 1.0, "intent_greet": 1.0, '
'"prev_utter_greet": 1.0, "slot_name_0": 1.0}]',
'[{"entity_name": 1.0, "intent_greet": 1.0, '
'"prev_action_listen": 1.0, "slot_name_0": 1.0}]'
]
def test_load_multi_file_training_data(default_domain):
# the stories file in `data/test_multifile_stories` is the same as in
# `data/test_stories/stories.md`, but split across multiple files
featurizer = MaxHistoryTrackerFeaturizer(
BinarySingleStateFeaturizer(), max_history=2)
trackers = training.load_data(
"data/test_stories/stories.md",
default_domain,
augmentation_factor=0
)
(tr_as_sts, tr_as_acts) = featurizer.training_states_and_actions(
trackers, default_domain)
hashed = []
for sts, acts in zip(tr_as_sts, tr_as_acts):
hashed.append(json.dumps(sts + acts, sort_keys=True))
hashed = sorted(hashed, reverse=True)
data = featurizer.featurize_trackers(trackers,
default_domain)
featurizer_mul = MaxHistoryTrackerFeaturizer(
BinarySingleStateFeaturizer(), max_history=2)
trackers_mul = training.load_data(
"data/test_multifile_stories",
default_domain,
augmentation_factor=0
)
(tr_as_sts_mul, tr_as_acts_mul) = featurizer.training_states_and_actions(
trackers_mul, default_domain)
hashed_mul = []
for sts_mul, acts_mul in zip(tr_as_sts_mul, tr_as_acts_mul):
hashed_mul.append(json.dumps(sts_mul + acts_mul, sort_keys=True))
hashed_mul = sorted(hashed_mul, reverse=True)
data_mul = featurizer_mul.featurize_trackers(trackers_mul,
default_domain)
assert hashed == hashed_mul
assert np.all(data.X.sort(axis=0) == data_mul.X.sort(axis=0))
assert np.all(data.y.sort(axis=0) == data_mul.y.sort(axis=0))