def test_mapping_wins_over_form(events: List[Event]): domain = """ forms: - test-form """ domain = Domain.from_yaml(domain) tracker = DialogueStateTracker.from_events("test", events, []) ensemble = SimplePolicyEnsemble( [ MappingPolicy(), ConstantPolicy(priority=1, predict_index=0), FormPolicy(), FallbackPolicy(), ] ) result, best_policy = ensemble.probabilities_using_best_policy( tracker, domain, RegexInterpreter() ) max_confidence_index = result.index(max(result)) next_action = domain.action_for_index(max_confidence_index, None) index_of_mapping_policy = 0 assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}" assert next_action.name() == ACTION_RESTART_NAME
def test_rule_based_data_warnings_no_rule_policy(): trackers = [DialogueStateTracker("some-id", slots=[], is_rule_tracker=True)] policies = [FallbackPolicy()] ensemble = SimplePolicyEnsemble(policies) with pytest.warns(UserWarning) as record: ensemble.train(trackers, Domain.empty(), RegexInterpreter()) assert ( "Found rule-based training data but no policy supporting rule-based data." ) in record[0].message.args[0]
def test_fallback_wins_over_mapping(): domain = Domain.load("data/test_domains/default.yml") events = [ ActionExecuted(ACTION_LISTEN_NAME), # Low confidence should trigger fallback utilities.user_uttered(USER_INTENT_RESTART, 0.0001), ] tracker = DialogueStateTracker.from_events("test", events, []) ensemble = SimplePolicyEnsemble([FallbackPolicy(), MappingPolicy()]) result, best_policy = ensemble.probabilities_using_best_policy( tracker, domain) max_confidence_index = result.index(max(result)) index_of_fallback_policy = 0 next_action = domain.action_for_index(max_confidence_index, None) assert best_policy == f"policy_{index_of_fallback_policy}_{FallbackPolicy.__name__}" assert next_action.name() == ACTION_DEFAULT_FALLBACK_NAME
def train_dialogue(domain_file='domain.yml', model_path='./models/dialogue', training_data_file='./data/stories.md'): fallback = FallbackPolicy(fallback_action_name="utter_unclear", core_threshold=0.3, nlu_threshold=0.75) agent = Agent(domain_file, policies=[ MemoizationPolicy(max_history=7), KerasPolicy(current_epoch=100, max_history=7), fallback ]) data = asyncio.run(agent.load_data(training_data_file)) agent.train(data) # agent.train( # data, # epochs=500, # batch_size=50, # validation_split=0.2) agent.persist(model_path) return agent
async def test_infer_fallback(): policy = FallbackPolicy.load("{}/models/fallback".format(prj_dir)) # "top_confidence, all_confidences, last_action_name, should_nlu_fallback", all_data = [ (0.1, [0.1], "some_action", False), (0.1, [0.1], "action_listen", True), (0.9, [0.9, 0.1], "some_action", False), (0.9, [0.9, 0.1], "action_listen", False), (0.4, [0.4, 0.35], "some_action", False), (0.4, [0.4, 0.35], "action_listen", True), (0.9, [0.9, 0.85], "action_listen", True), ] for data in all_data: nlu_data = { "intent": { "confidence": data[0] }, "intent_ranking": [{ "confidence": confidence } for confidence in data[1]], } should_nlu_fallback = policy.should_nlu_fallback(nlu_data, data[2]) assert should_nlu_fallback == data[3]
from rasa.core.policies.keras_policy import KerasPolicy from rasa.core.policies.memoization import MemoizationPolicy from rasa.core.policies.fallback import FallbackPolicy from rasa.core.policies.mapping_policy import MappingPolicy logger = logging.getLogger(__name__) async def train_core(domain_file, training_data_file, model_directory): agent = Agent(domain_file, policies=[MemoizationPolicy(max_history=3), MappingPolicy(), KerasPolicy(epochs=500)]) training_data = await agent.load_data(training_data_file, augmentation_factor=10) agent.train(training_data) # Attention: agent.persist stores the model and all meta data into a folder. # The folder itself is not zipped. model_path = os.path.join(model_directory, "core") agent.persist(model_path) logger.info(f"Model trained. Stored in '{model_path}'.") return model_path if __name__ == "__main__": rasa.utils.io.configure_colored_logging(loglevel="INFO") training_data_file = "./data/stories.md" model_path = "./models/dialogue" fallback = FallbackPolicy(fallback_action_name="action_default_fallback", core_threshold=0.25, nlu_threshold=0.75) loop = asyncio.get_event_loop() loop.run_until_complete(train_core(domain_file='domain.yml', training_data_file=training_data_file, model_directory=model_path))
max_confidence_index = result.index(max(result)) next_action = domain.action_for_index(max_confidence_index, None) index_of_mapping_policy = 0 assert best_policy == f"policy_{index_of_mapping_policy}_{MappingPolicy.__name__}" assert next_action.name() == ACTION_RESTART_NAME @pytest.mark.parametrize( "ensemble", [ SimplePolicyEnsemble( [ FormPolicy(), ConstantPolicy(FORM_POLICY_PRIORITY - 1, 0), FallbackPolicy(), ] ), SimplePolicyEnsemble([FormPolicy(), MappingPolicy()]), ], ) def test_form_wins_over_everything_else(ensemble: SimplePolicyEnsemble): form_name = "test-form" domain = f""" forms: - {form_name} """ domain = Domain.from_yaml(domain) events = [ Form("test-form"),
def create_policy(self, featurizer, priority): p = FallbackPolicy(priority=priority) return p
def create_policy( self, featurizer: Optional[TrackerFeaturizer], priority: int ) -> Policy: return FallbackPolicy(priority=priority)
) def test_supported_data(policy: Type[Policy], supported_data: SupportedData): assert policy.supported_data() == supported_data class RuleAndMLPolicy(Policy): """Test policy that supports both rule-based and ML-based training data.""" @staticmethod def supported_data() -> SupportedData: return SupportedData.ML_AND_RULE_DATA @pytest.mark.parametrize( "policy,n_rule_trackers,n_ml_trackers", [ (FallbackPolicy(), 0, 3), (RulePolicy(), 2, 0), (RuleAndMLPolicy, 2, 3), # policy can be passed as a `type` as well ], ) def test_get_training_trackers_for_policy(policy: Policy, n_rule_trackers: int, n_ml_trackers): # create five trackers (two rule-based and three ML trackers) trackers = [ DialogueStateTracker("id1", slots=[], is_rule_tracker=True), DialogueStateTracker("id2", slots=[], is_rule_tracker=False), DialogueStateTracker("id3", slots=[], is_rule_tracker=False), DialogueStateTracker("id4", slots=[], is_rule_tracker=True), DialogueStateTracker("id5", slots=[], is_rule_tracker=False), ]
async def test_train_fallback_policy(): policy = FallbackPolicy(priority=1) policy.persist("{}/models/fallback".format(prj_dir))