def test_mapping_wins_over_form(events: List[Event]):
domain = """
forms:
- test-form
"""
domain = Domain.from_yaml(domain)
tracker = DialogueStateTracker.from_events("test", events, [])
ensemble = SimplePolicyEnsemble(
[
MappingPolicy(),
ConstantPolicy(priority=1, predict_index=0),
FormPolicy(),
FallbackPolicy(),
]
)
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain, RegexInterpreter()
)
max_confidence_index = result.index(max(result))
next_action = domain.action_for_index(max_confidence_index, None)
index_of_mapping_policy = 0
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
def test_rule_based_data_warnings_no_rule_policy():
trackers = [DialogueStateTracker("some-id", slots=[], is_rule_tracker=True)]
policies = [FallbackPolicy()]
ensemble = SimplePolicyEnsemble(policies)
with pytest.warns(UserWarning) as record:
ensemble.train(trackers, Domain.empty(), RegexInterpreter())
assert (
"Found rule-based training data but no policy supporting rule-based data."
) in record[0].message.args[0]
def test_fallback_wins_over_mapping():
domain = Domain.load("data/test_domains/default.yml")
events = [
ActionExecuted(ACTION_LISTEN_NAME),
# Low confidence should trigger fallback
utilities.user_uttered(USER_INTENT_RESTART, 0.0001),
]
tracker = DialogueStateTracker.from_events("test", events, [])
ensemble = SimplePolicyEnsemble([FallbackPolicy(), MappingPolicy()])
result, best_policy = ensemble.probabilities_using_best_policy(
tracker, domain)
max_confidence_index = result.index(max(result))
index_of_fallback_policy = 0
next_action = domain.action_for_index(max_confidence_index, None)
assert best_policy == f"policy_{index_of_fallback_policy}_{FallbackPolicy.__name__}"
assert next_action.name() == ACTION_DEFAULT_FALLBACK_NAME
def train_dialogue(domain_file='domain.yml',
model_path='./models/dialogue',
training_data_file='./data/stories.md'):
fallback = FallbackPolicy(fallback_action_name="utter_unclear",
core_threshold=0.3,
nlu_threshold=0.75)
agent = Agent(domain_file,
policies=[
MemoizationPolicy(max_history=7),
KerasPolicy(current_epoch=100, max_history=7), fallback
])
data = asyncio.run(agent.load_data(training_data_file))
agent.train(data)
# agent.train(
# data,
# epochs=500,
# batch_size=50,
# validation_split=0.2)
agent.persist(model_path)
return agent
async def test_infer_fallback():
policy = FallbackPolicy.load("{}/models/fallback".format(prj_dir))
# "top_confidence, all_confidences, last_action_name, should_nlu_fallback",
all_data = [
(0.1, [0.1], "some_action", False),
(0.1, [0.1], "action_listen", True),
(0.9, [0.9, 0.1], "some_action", False),
(0.9, [0.9, 0.1], "action_listen", False),
(0.4, [0.4, 0.35], "some_action", False),
(0.4, [0.4, 0.35], "action_listen", True),
(0.9, [0.9, 0.85], "action_listen", True),
]
for data in all_data:
nlu_data = {
"intent": {
"confidence": data[0]
},
"intent_ranking": [{
"confidence": confidence
} for confidence in data[1]],
}
should_nlu_fallback = policy.should_nlu_fallback(nlu_data, data[2])
assert should_nlu_fallback == data[3]
from rasa.core.policies.keras_policy import KerasPolicy
from rasa.core.policies.memoization import MemoizationPolicy
from rasa.core.policies.fallback import FallbackPolicy
from rasa.core.policies.mapping_policy import MappingPolicy
logger = logging.getLogger(__name__)
async def train_core(domain_file, training_data_file, model_directory):
agent = Agent(domain_file, policies=[MemoizationPolicy(max_history=3), MappingPolicy(), KerasPolicy(epochs=500)])
training_data = await agent.load_data(training_data_file, augmentation_factor=10)
agent.train(training_data)
# Attention: agent.persist stores the model and all meta data into a folder.
# The folder itself is not zipped.
model_path = os.path.join(model_directory, "core")
agent.persist(model_path)
logger.info(f"Model trained. Stored in '{model_path}'.")
return model_path
if __name__ == "__main__":
rasa.utils.io.configure_colored_logging(loglevel="INFO")
training_data_file = "./data/stories.md"
model_path = "./models/dialogue"
fallback = FallbackPolicy(fallback_action_name="action_default_fallback", core_threshold=0.25, nlu_threshold=0.75)
loop = asyncio.get_event_loop()
loop.run_until_complete(train_core(domain_file='domain.yml', training_data_file=training_data_file, model_directory=model_path))
max_confidence_index = result.index(max(result))
next_action = domain.action_for_index(max_confidence_index, None)
index_of_mapping_policy = 0
assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}"
assert next_action.name() == ACTION_RESTART_NAME
@pytest.mark.parametrize(
"ensemble",
[
SimplePolicyEnsemble(
[
FormPolicy(),
ConstantPolicy(FORM_POLICY_PRIORITY - 1, 0),
FallbackPolicy(),
]
),
SimplePolicyEnsemble([FormPolicy(), MappingPolicy()]),
],
)
def test_form_wins_over_everything_else(ensemble: SimplePolicyEnsemble):
form_name = "test-form"
domain = f"""
forms:
- {form_name}
"""
domain = Domain.from_yaml(domain)
events = [
Form("test-form"),
def create_policy(self, featurizer, priority):
p = FallbackPolicy(priority=priority)
return p
def create_policy(
self, featurizer: Optional[TrackerFeaturizer], priority: int
) -> Policy:
return FallbackPolicy(priority=priority)
)
def test_supported_data(policy: Type[Policy], supported_data: SupportedData):
assert policy.supported_data() == supported_data
class RuleAndMLPolicy(Policy):
"""Test policy that supports both rule-based and ML-based training data."""
@staticmethod
def supported_data() -> SupportedData:
return SupportedData.ML_AND_RULE_DATA
@pytest.mark.parametrize(
"policy,n_rule_trackers,n_ml_trackers",
[
(FallbackPolicy(), 0, 3),
(RulePolicy(), 2, 0),
(RuleAndMLPolicy, 2, 3), # policy can be passed as a `type` as well
],
)
def test_get_training_trackers_for_policy(policy: Policy, n_rule_trackers: int,
n_ml_trackers):
# create five trackers (two rule-based and three ML trackers)
trackers = [
DialogueStateTracker("id1", slots=[], is_rule_tracker=True),
DialogueStateTracker("id2", slots=[], is_rule_tracker=False),
DialogueStateTracker("id3", slots=[], is_rule_tracker=False),
DialogueStateTracker("id4", slots=[], is_rule_tracker=True),
DialogueStateTracker("id5", slots=[], is_rule_tracker=False),
]
async def test_train_fallback_policy():
policy = FallbackPolicy(priority=1)
policy.persist("{}/models/fallback".format(prj_dir))
