def test_dialogue_from_parameters():
domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS_AND_NO_ACTIONS)
filename = "data/test_dialogues/default.json"
tracker = tracker_from_dialogue_file(filename, domain)
serialised_dialogue = InMemoryTrackerStore.serialise_tracker(tracker)
deserialised_dialogue = Dialogue.from_parameters(
json.loads(serialised_dialogue))
assert tracker.as_dialogue().as_dict() == deserialised_dialogue.as_dict()
def test_custom_slot_type(tmpdir):
domain_path = utilities.write_text_to_file(
tmpdir,
"domain.yml",
"""
slots:
custom:
type: tests.core.conftest.CustomSlot
templates:
utter_greet:
- text: hey there!
actions:
- utter_greet """,
)
Domain.load(domain_path)
def test_inmemory_tracker_store(pair):
filename, domainpath = pair
domain = Domain.load(domainpath)
tracker = tracker_from_dialogue_file(filename, domain)
tracker_store = InMemoryTrackerStore(domain)
tracker_store.save(tracker)
restored = tracker_store.retrieve(tracker.sender_id)
assert restored == tracker
def test_dialogue_from_parameters():
domain = Domain.load("examples/restaurantbot/domain.yml")
filename = "data/test_dialogues/restaurantbot.json"
tracker = tracker_from_dialogue_file(filename, domain)
serialised_dialogue = InMemoryTrackerStore.serialise_tracker(tracker)
deserialised_dialogue = Dialogue.from_parameters(
json.loads(serialised_dialogue))
assert tracker.as_dialogue().as_dict() == deserialised_dialogue.as_dict()
def test_custom_slot_type(tmpdir: Path):
domain_path = str(tmpdir / "domain.yml")
io_utils.write_text_file(
"""
slots:
custom:
type: tests.core.conftest.CustomSlot
responses:
utter_greet:
- text: hey there!
actions:
- utter_greet """,
domain_path,
)
Domain.load(domain_path)
async def train_core_async(
domain: Union[Domain, Text],
config: Text,
stories: Text,
output: Text,
train_path: Optional[Text] = None,
fixed_model_name: Optional[Text] = None,
uncompress: bool = False,
kwargs: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Core model.
Args:
domain: Path to the domain file.
config: Path to the config file for Core.
stories: Path to the Core training data.
output: Output path.
train_path: If `None` the model will be trained in a temporary
directory, otherwise in the provided directory.
fixed_model_name: Name of model to be stored.
uncompress: If `True` the model will not be compressed.
kwargs: Additional training parameters.
Returns:
If `train_path` is given it returns the path to the model archive,
otherwise the path to the directory with the trained model files.
"""
config = _get_valid_config(config, CONFIG_MANDATORY_KEYS_CORE)
skill_imports = SkillSelector.load(config)
if isinstance(domain, str):
try:
domain = Domain.load(domain, skill_imports)
except InvalidDomain as e:
print_error(
"Could not load domain due to: '{}'. To specify a valid domain path "
"use the '--domain' argument.".format(e))
return None
story_directory = data.get_core_directory(stories, skill_imports)
if not os.listdir(story_directory):
print_error("No stories given. Please provide stories in order to "
"train a Rasa Core model using the '--stories' argument.")
return
return await _train_core_with_validated_data(
domain=domain,
config=config,
story_directory=story_directory,
output=output,
train_path=train_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
kwargs=kwargs,
)
async def train_async(
domain: Union[Domain, Text],
config: Text,
training_files: Optional[Union[Text, List[Text]]],
output_path: Text = DEFAULT_MODELS_PATH,
force_training: bool = False,
fixed_model_name: Optional[Text] = None,
kwargs: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU).
Args:
domain: Path to the domain file.
config: Path to the config for Core and NLU.
training_files: Paths to the training data for Core and NLU.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
skill_imports = SkillSelector.load(config, training_files)
try:
domain = Domain.load(domain, skill_imports)
domain.check_missing_templates()
except InvalidDomain:
domain = None
story_directory, nlu_data_directory = data.get_core_nlu_directories(
training_files, skill_imports)
with ExitStack() as stack:
train_path = stack.enter_context(TempDirectoryPath(tempfile.mkdtemp()))
nlu_data = stack.enter_context(TempDirectoryPath(nlu_data_directory))
story = stack.enter_context(TempDirectoryPath(story_directory))
if domain is None:
return handle_domain_if_not_exists(config, nlu_data_directory,
output_path, fixed_model_name)
return await _train_async_internal(
domain,
config,
train_path,
nlu_data,
story,
output_path,
force_training,
fixed_model_name,
kwargs,
)
if domain is None:
return handle_domain_if_not_exists(config, nlu_data_directory,
output_path, fixed_model_name)
async def test_train_keras_policy():
default_domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS)
trackers = await training.load_data(DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0,
debug_plots=False)
policy = KerasPolicy(featurizer=featurizer(), priority=1)
policy.train(trackers, default_domain)
policy.persist("{}/models/keras".format(prj_dir))
def load(
cls,
model_path: Text,
interpreter: Optional[NaturalLanguageInterpreter] = None,
generator: Union[EndpointConfig, NaturalLanguageGenerator] = None,
tracker_store: Optional[TrackerStore] = None,
lock_store: Optional[LockStore] = None,
action_endpoint: Optional[EndpointConfig] = None,
model_server: Optional[EndpointConfig] = None,
remote_storage: Optional[Text] = None,
path_to_model_archive: Optional[Text] = None,
) -> "Agent":
"""Load a persisted model from the passed path."""
try:
if not model_path:
raise ModelNotFound("No path specified.")
elif not os.path.exists(model_path):
raise ModelNotFound(f"No file or directory at '{model_path}'.")
elif os.path.isfile(model_path):
model_path = get_model(model_path)
except ModelNotFound:
raise ValueError(
"You are trying to load a MODEL from '{}', which is not possible. \n"
"The model path should be a 'tar.gz' file or a directory "
"containing the various model files in the sub-directories 'core' "
"and 'nlu'. \n\nIf you want to load training data instead of "
"a model, use `agent.load_data(...)` instead.".format(
model_path))
core_model, nlu_model = get_model_subdirectories(model_path)
if not interpreter and nlu_model:
interpreter = NaturalLanguageInterpreter.create(nlu_model)
domain = None
ensemble = None
if core_model:
domain = Domain.load(os.path.join(core_model, DEFAULT_DOMAIN_PATH))
ensemble = PolicyEnsemble.load(core_model) if core_model else None
# ensures the domain hasn't changed between test and train
domain.compare_with_specification(core_model)
return cls(
domain=domain,
policies=ensemble,
interpreter=interpreter,
generator=generator,
tracker_store=tracker_store,
lock_store=lock_store,
action_endpoint=action_endpoint,
model_directory=model_path,
model_server=model_server,
remote_storage=remote_storage,
path_to_model_archive=path_to_model_archive,
)
def __init__(self, domain_path='agents/prototypes/concertbot/domain.yml'):
self.executor = ActionExecutor()
action_package_name = 'actions.procs'
self.executor.register_package(action_package_name)
endpoint = EndpointConfig("http://localhost:5000")
self.interpreter = RasaNLUHttpInterpreter(model_name="nlu",
project_name='chinese',
endpoint=endpoint)
self.domain = Domain.load(domain_path)
def tracker_from_dialogue_file(filename: Text, domain: Domain = None):
dialogue = read_dialogue_file(filename)
if not domain:
domain = Domain.load(DEFAULT_DOMAIN_PATH)
tracker = DialogueStateTracker(dialogue.name, domain.slots)
tracker.recreate_from_dialogue(dialogue)
return tracker
async def test_memorise(self, trained_policy):
domain = Domain.load("data/test_domains/form.yml")
trackers = await training.load_data("data/test_stories/stories_form.md", domain)
trained_policy.train(trackers, domain)
(
all_states,
all_actions,
) = trained_policy.featurizer.training_states_and_actions(trackers, domain)
for tracker, states, actions in zip(trackers, all_states, all_actions):
for state in states:
if state is not None:
# check that 'form: inform' was ignored
assert "intent_inform" not in state.keys()
recalled = trained_policy.recall(states, tracker, domain)
active_form = trained_policy._get_active_form_name(states[-1])
if states[0] is not None and states[-1] is not None:
# explicitly set intents and actions before listen after
# which FormPolicy should not predict a form action and
# should add FormValidation(False) event
# @formatter:off
is_no_validation = (
(
"prev_some_form" in states[0].keys()
and "intent_default" in states[-1].keys()
)
or (
"prev_some_form" in states[0].keys()
and "intent_stop" in states[-1].keys()
)
or (
"prev_utter_ask_continue" in states[0].keys()
and "intent_affirm" in states[-1].keys()
)
or (
"prev_utter_ask_continue" in states[0].keys()
and "intent_deny" in states[-1].keys()
)
)
# @formatter:on
else:
is_no_validation = False
if "intent_start_form" in states[-1]:
# explicitly check that intent that starts the form
# is not memorized as non validation intent
assert recalled is None
elif is_no_validation:
assert recalled == active_form
else:
assert recalled is None
nums = np.random.randn(domain.num_states)
random_states = [{f: num for f, num in zip(domain.input_states, nums)}]
assert trained_policy.recall(random_states, None, domain) is None
async def test_memorise(self, trained_policy: FormPolicy,
default_domain: Domain):
domain = Domain.load("data/test_domains/form.yml")
trackers = await training.load_data(
"data/test_stories/stories_form.md", domain)
trained_policy.train(trackers, domain, RegexInterpreter())
(
all_states,
all_actions,
) = trained_policy.featurizer.training_states_and_actions(
trackers, domain)
for tracker, states, actions in zip(trackers, all_states, all_actions):
for state in states:
if state is not None:
# check that 'form: inform' was ignored
if state.get(USER):
assert not state.get(USER).get(INTENT) == "inform"
recalled = trained_policy.recall(states, tracker, domain)
active_form = trained_policy._get_active_form_name(states[-1])
if states[0] is not None and states[-1] is not None:
# explicitly set intents and actions before listen after
# which FormPolicy should not predict a form action and
# should add FormValidation(False) event
# @formatter:off
is_no_validation = (
self._test_for_previous_action_and_intent(
states, "default", "some_form")
or self._test_for_previous_action_and_intent(
states, "stop", "some_form")
or self._test_for_previous_action_and_intent(
states, "affirm", "utter_ask_continue")
or self._test_for_previous_action_and_intent(
states, "deny", "utter_ask_continue")
# comes from the fact that intent_inform after utter_ask_continue
# is not read from stories
or self._test_for_previous_action_and_intent(
states, "stop", "utter_ask_continue"))
# @formatter:on
else:
is_no_validation = False
if "intent_start_form" in states[-1]:
# explicitly check that intent that starts the form
# is not memorized as non validation intent
assert recalled is None
elif is_no_validation:
assert recalled == active_form
else:
assert recalled is None
nums = np.random.randn(domain.num_states)
random_states = [{f: num for f, num in zip(domain.input_states, nums)}]
assert trained_policy.recall(random_states, None, domain) is None
def test_load_on_invalid_domain():
with pytest.raises(InvalidDomain):
Domain.load("data/test_domains/duplicate_intents.yml")
with pytest.raises(InvalidDomain):
Domain.load("data/test_domains/duplicate_actions.yml")
with pytest.raises(InvalidDomain):
Domain.load("data/test_domains/duplicate_templates.yml")
with pytest.raises(InvalidDomain):
Domain.load("data/test_domains/duplicate_entities.yml")
def test_clean_domain_for_file():
domain_path = "data/test_domains/default_unfeaturized_entities.yml"
cleaned = Domain.load(domain_path).cleaned_domain()
expected = {
"intents": [
{
"greet": {
USE_ENTITIES_KEY: ["name"]
}
},
{
"default": {
IGNORE_ENTITIES_KEY: ["unrelated_recognized_entity"]
}
},
{
"goodbye": {
USE_ENTITIES_KEY: []
}
},
{
"thank": {
USE_ENTITIES_KEY: []
}
},
"ask",
{
"why": {
USE_ENTITIES_KEY: []
}
},
"pure_intent",
],
"entities": ["name", "other", "unrelated_recognized_entity"],
"responses": {
"utter_greet": [{
"text": "hey there!"
}],
"utter_goodbye": [{
"text": "goodbye :("
}],
"utter_default": [{
"text": "default message"
}],
},
"actions": ["utter_default", "utter_goodbye", "utter_greet"],
"session_config": {
"carry_over_slots_to_new_session": True,
"session_expiration_time":
DEFAULT_SESSION_EXPIRATION_TIME_IN_MINUTES,
},
}
assert cleaned == expected
async def get_domain(self) -> Domain:
domain = Domain.empty()
try:
domain = Domain.load(self._domain_path)
domain.check_missing_templates()
except InvalidDomain:
logger.debug(
"Loading domain from '{}' failed. Using empty domain.".format(
self._domain_path))
return domain
async def test_MaxHistoryTrackerFeaturizer():
# viz_domain(default_domain)
default_domain = Domain.load("{}/domain_with_slots.yml".format(prj_dir))
stories_file = "{}/data/stories.md".format(prj_dir)
trackers = await training.load_data(
stories_file, default_domain, augmentation_factor=0, debug_plots=True
)
viz_trackers(trackers)
featurizer = MaxHistoryTrackerFeaturizer(max_history=5)
(decoded, actions) = featurizer.training_states_and_actions(trackers, default_domain)
def tracker_from_dialogue_file(
filename: Text,
domain: Optional[Domain] = None) -> DialogueStateTracker:
dialogue = read_dialogue_file(filename)
if not domain:
domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS)
tracker = DialogueStateTracker(dialogue.name, domain.slots)
tracker.recreate_from_dialogue(dialogue)
return tracker
async def get_domain(self) -> Domain:
domain = Domain.empty()
try:
domain = Domain.load(self._domain_path)
domain.check_missing_templates()
except InvalidDomain as e:
raise_warning(
f"Loading domain from '{self._domain_path}' failed. Using "
f"empty domain. Error: '{e.message}'")
return domain
def test_utter_templates():
domain_file = "examples/moodbot/domain.yml"
domain = Domain.load(domain_file)
expected_template = {
"text": "Hey! How are you?",
"buttons": [
{"title": "great", "payload": "/mood_great"},
{"title": "super sad", "payload": "/mood_unhappy"},
],
}
assert domain.random_template_for("utter_greet") == expected_template
def _create_domain(domain: Union[None, Domain, Text]) -> Domain:
if isinstance(domain, str):
return Domain.load(domain)
elif isinstance(domain, Domain):
return domain
elif domain is not None:
raise ValueError(
"Invalid param `domain`. Expected a path to a domain "
"specification or a domain instance. But got "
"type '{}' with value '{}'".format(type(domain), domain))
async def get_domain(self) -> Domain:
domain = Domain.empty()
try:
domain = Domain.load(self._domain_path)
domain.check_missing_templates()
except InvalidDomain as e:
logger.warning(
"Loading domain from '{}' failed. Using empty domain. Error: '{}'"
.format(self._domain_path, e.message))
return domain
async def on_trynow(self, request):
res_data = await request.json()
print("----------- Inside Try now --from SID {}--------------".format(res_data['sessionId']))
result = await ExportProject.main(res_data['sessionId'], res_data['projectObjectId'], 'SESSION')
print(result)
if result is not None:
return web.json_response({"status": "Error", "message": result})
import rasa.model as model
from rasa.core.agent import Agent
from rasa.core.tracker_store import MongoTrackerStore
from rasa.core.domain import Domain
from rasa.train import train_async
from rasa.utils.endpoints import EndpointConfig
base_path = CONFIG.get('api_gateway', 'SESSION_MODEL_PATH')
config = "config.yml"
training_files = "data/"
domain = "domain.yml"
output = "models/"
endpoints = EndpointConfig(url="http://action_server:5055/webhook")
base_path = base_path + res_data['sessionId'] + "/"
config = base_path + config
training_files = base_path + training_files
domain = base_path + domain
output = base_path + output
start_time = time.time()
try:
model_path = await train_async(domain, config, [training_files], output, additional_arguments={"augmentation_factor": 10})
end_time = time.time()
print("it took this long to run: {}".format(end_time - start_time))
unpacked = model.get_model(model_path)
domain = Domain.load(domain)
_tracker_store = MongoTrackerStore(domain=domain,
host=CONFIG.get('api_gateway', 'MONGODB_URL'),
db=CONFIG.get('api_gateway', 'MONGODB_NAME'),
username=None,
password=None,
auth_source="admin",
collection="conversations",
event_broker=None)
print("*************** Actions Endpoint as per data ********** {}".format(endpoints.url))
self.agent = Agent.load(unpacked, tracker_store=_tracker_store, action_endpoint=endpoints)
return web.json_response({"status": "Success", "message": "Ready to chat"})
#await sio.emit('chatResponse', {"status": "Success", "message": "Ready to chat"}, namespace='/trynow', room=sid)
except Exception as e:
print("Exception while try Now --- "+str(e))
#await sio.emit('chatResponse', {"status": "Error", "message": repr(e)}, namespace='/trynow', room=sid)
return web.json_response({"status": "Error", "message": repr(e)})
def test_transform_intents_for_file_with_mapping():
domain_path = "data/test_domains/default_with_mapping.yml"
domain = Domain.load(domain_path)
transformed = domain._transform_intents_for_file()
expected = [
{"greet": {"triggers": "utter_greet", USE_ENTITIES_KEY: True}},
{"default": {"triggers": "utter_default", USE_ENTITIES_KEY: True}},
{"goodbye": {USE_ENTITIES_KEY: True}},
]
assert transformed == expected
def test_clean_domain():
domain_path = "data/test_domains/default_unfeaturized_entities.yml"
cleaned = Domain.load(domain_path).cleaned_domain()
expected = {
"intents": [
{
"greet": {
"use_entities": ["name"]
}
},
{
"default": {
"ignore_entities": ["unrelated_recognized_entity"]
}
},
{
"goodbye": {
"use_entities": []
}
},
{
"thank": {
"use_entities": []
}
},
"ask",
{
"why": {
"use_entities": []
}
},
"pure_intent",
],
"entities": ["name", "other", "unrelated_recognized_entity"],
"templates": {
"utter_greet": [{
"text": "hey there!"
}],
"utter_goodbye": [{
"text": "goodbye :("
}],
"utter_default": [{
"text": "default message"
}],
},
"actions": ["utter_default", "utter_goodbye", "utter_greet"],
}
expected = Domain.from_dict(expected)
actual = Domain.from_dict(cleaned)
assert hash(actual) == hash(expected)
def test_clean_domain_deprecated_templates():
domain_path = "data/test_domains/default_deprecated_templates.yml"
cleaned = Domain.load(domain_path).cleaned_domain()
expected = {
"intents": [
{
"greet": {
USE_ENTITIES_KEY: ["name"]
}
},
{
"default": {
IGNORE_ENTITIES_KEY: ["unrelated_recognized_entity"]
}
},
{
"goodbye": {
USE_ENTITIES_KEY: []
}
},
{
"thank": {
USE_ENTITIES_KEY: []
}
},
"ask",
{
"why": {
USE_ENTITIES_KEY: []
}
},
"pure_intent",
],
"entities": ["name", "unrelated_recognized_entity", "other"],
"responses": {
"utter_greet": [{
"text": "hey there!"
}],
"utter_goodbye": [{
"text": "goodbye :("
}],
"utter_default": [{
"text": "default message"
}],
},
"actions": ["utter_default", "utter_greet", "utter_goodbye"],
}
expected = Domain.from_dict(expected)
actual = Domain.from_dict(cleaned)
assert actual.as_dict() == expected.as_dict()
async def test_infer():
default_domain = Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS)
print("action names: {}".format(default_domain.action_names))
trackers = await training.load_data(DEFAULT_STORIES_FILE,
default_domain,
augmentation_factor=0,
debug_plots=True)
policy = TEDPolicy.load("{}/models/ted".format(prj_dir))
for tracker in trackers:
y_pred = policy.predict_action_probabilities(tracker, default_domain)
index = y_pred.index(max(y_pred))
print(default_domain.action_names[index])
async def test_FullDialogueTrackerFeaturizer():
# viz_domain(default_domain)
default_domain = Domain.load("{}/data/domain_with_slots.yml".format(prj_dir))
stories_file = "{}/data/stories.md".format(prj_dir)
trackers = await training.load_data(
stories_file, default_domain, augmentation_factor=0, debug_plots=False
)
# viz_trackers(trackers)
featurizer = FullDialogueTrackerFeaturizer(state_featurizer=BinarySingleStateFeaturizer())
# print_title("START TRAINING STATES")
(trackers_as_states, trackers_as_actions) = featurizer.training_states_and_actions(trackers, default_domain)
print_data_training(trackers_as_states, trackers_as_actions)
async def test_dispatcher_utter_buttons_from_domain_templ(default_tracker):
domain_file = "examples/moodbot/domain.yml"
domain = Domain.load(domain_file)
bot = CollectingOutputChannel()
nlg = TemplatedNaturalLanguageGenerator(domain.templates)
dispatcher = Dispatcher("my-sender", bot, nlg)
await dispatcher.utter_template("utter_greet", default_tracker)
assert len(bot.messages) == 1
assert bot.messages[0]["text"] == "Hey! How are you?"
assert bot.messages[0]["buttons"] == [
{"payload": "great", "title": "great"},
{"payload": "super sad", "title": "super sad"},
]
def load(
cls,
unpacked_model_path: Text,
interpreter: Optional[NaturalLanguageInterpreter] = None,
generator: Union[EndpointConfig, NaturalLanguageGenerator] = None,
tracker_store: Optional[TrackerStore] = None,
action_endpoint: Optional[EndpointConfig] = None,
model_server: Optional[EndpointConfig] = None,
remote_storage: Optional[Text] = None,
) -> "Agent":
"""Load a persisted model from the passed path."""
if not os.path.exists(unpacked_model_path) or not os.path.isdir(
unpacked_model_path
):
raise ValueError(
"You are trying to load a MODEL from "
"('{}'), which is not possible. \n"
"The persisted path should be a directory "
"containing the various model files in the "
"sub-directories 'core' and 'nlu'. \n\n"
"If you want to load training data instead of "
"a model, use `agent.load_data(...)` "
"instead.".format(unpacked_model_path)
)
core_model, nlu_model = get_model_subdirectories(unpacked_model_path)
if not interpreter and os.path.exists(nlu_model):
interpreter = NaturalLanguageInterpreter.create(nlu_model)
domain = None
ensemble = None
if os.path.exists(core_model):
domain = Domain.load(os.path.join(core_model, DEFAULT_DOMAIN_PATH))
ensemble = PolicyEnsemble.load(core_model) if core_model else None
# ensures the domain hasn't changed between test and train
domain.compare_with_specification(core_model)
return cls(
domain=domain,
policies=ensemble,
interpreter=interpreter,
generator=generator,
tracker_store=tracker_store,
action_endpoint=action_endpoint,
model_directory=unpacked_model_path,
model_server=model_server,
remote_storage=remote_storage,
)
