async def post_event(request, conversation_id, user):
_event = request.json
try:
_ = await _stack_service(
request, RASA_PRODUCTION_ENVIRONMENT).append_events_to_tracker(
conversation_id, _event)
if tracker_utils.is_user_event(_event):
# add annotated user messages to training data
data_service = DataService(request[REQUEST_DB_SESSION_KEY])
data_service.save_user_event_as_example(
user, config.project_name, _event)
telemetry.track_message_annotated(
telemetry.MESSAGE_ANNOTATED_INTERACTIVE_LEARNING)
return response.json(_event)
except ClientError as e:
logger.warning(
f"Creating event for sender '{conversation_id}' failed. Error: {e}"
)
return rasa_x_utils.error(
HTTPStatus.BAD_REQUEST,
"EventCreationError",
f"Error when creating event for sender '{conversation_id}'.",
details=e,
)
except ValueError as e:
return rasa_x_utils.error(HTTPStatus.NOT_FOUND,
"ServiceNotFound",
details=e)
async def get_domain_warnings(
self,
project_id: Text = config.project_name
) -> Optional[Tuple[Dict[Text, Dict[Text, List[Text]]], int]]:
"""Get domain warnings.
Args:
project_id: The project id of the domain.
Returns:
Dict of domain warnings and the total count of elements.
"""
domain = self._get_domain(project_id)
if domain:
from rasax.community.services.data_service import DataService
from rasax.community.services.nlg_service import NlgService
from rasax.community.services.story_service import StoryService
domain_object = RasaDomain.from_dict(domain.as_dict())
training_data = DataService(
self.session).get_nlu_training_data_object(
project_id=project_id)
# actions are response names and story bot actions
actions = NlgService(self.session).fetch_all_response_names()
# intents are training data intents and story intents
intents = training_data.intents
# entities are training data entities without `extractor` attribute
entity_examples = training_data.entity_examples
entities = self._get_entities_from_training_data(entity_examples)
# slots are simply story slots
slots = set()
story_events = await StoryService(
self.session).fetch_domain_items_from_stories()
if story_events:
actions.update(story_events[0])
intents.update(story_events[1])
slots.update(story_events[2])
entities.update(story_events[3])
# exclude unfeaturized slots from warnings
slots = self._remove_unfeaturized_slots(slots, domain_object)
domain_warnings = self._domain_warnings_as_list(
domain_object, intents, entities, actions, slots)
return domain_warnings, self._count_total_warnings(domain_warnings)
return None
async def inject_files_from_disk(
project_path: Union[Path, Text],
data_path: Text,
session: orm.Session,
username: Optional[Text] = constants.COMMUNITY_USERNAME,
config_path: Optional[Text] = config.default_config_path,
) -> Tuple[Dict[Text, Any], List[Dict[Text, Any]], "NluTrainingData"]:
"""Injects local files into database.
Args:
project_path: Path to the project of which the data should be injected.
data_path: Path to the data within this project.
session: Database session.
username: The username which is used to inject the data.
config_path: Path to the config file within the project
Returns:
Tuple of domain, stories, and NLU training data.
"""
import rasa.data
from rasax.community.local import LOCAL_DOMAIN_PATH
from rasax.community.services.data_service import DataService
from rasax.community.services.settings_service import SettingsService
utils.set_project_directory(project_path)
domain_service = DomainService(session)
domain = inject_domain(
os.path.join(project_path, LOCAL_DOMAIN_PATH),
domain_service,
constants.COMMUNITY_PROJECT_NAME,
username,
)
settings_service = SettingsService(session)
inject_config(os.path.join(project_path, config_path), settings_service)
story_files, nlu_files = rasa.data.get_core_nlu_files([data_path])
story_service = StoryService(session)
story_blocks = await inject_stories(story_files, story_service, username,
constants.COMMUNITY_TEAM_NAME)
data_service = DataService(session)
nlu_data = inject_nlu_data(nlu_files, constants.COMMUNITY_PROJECT_NAME,
username, data_service)
return domain, story_blocks, nlu_data
def stack_services(
self,
project_id: Text = config.project_name
) -> Dict[Text, StackService]:
"""Create StackServices for all Stack servers."""
from rasax.community.services.stack_service import RasaCredentials
environments_config = self.get_environments_config(project_id).get(
"environments", {})
_stack_services = {}
for k, v in environments_config.get("rasa", {}).items():
credentials = RasaCredentials(url=v["url"], token=v.get("token"))
_stack_services[k] = StackService(
credentials,
DataService(self.session),
StoryService(self.session),
DomainService(self.session),
self,
)
return _stack_services
def __init__(self, session: Session):
self.domain_service = DomainService(session)
self.data_service = DataService(session)
super().__init__(session)
class IntentService(DbService):
def __init__(self, session: Session):
self.domain_service = DomainService(session)
self.data_service = DataService(session)
super().__init__(session)
def add_temporary_intent(self, intent: Dict[str, str],
project_id: Text) -> None:
"""Creates a new temporary intent.
Args:
intent: The intent object which should be added.
project_id: The project id which the intent should belong to.
"""
mapped_to = intent.get(INTENT_MAPPED_TO_KEY)
permanent_intents = self.get_permanent_intents(project_id)
if mapped_to and mapped_to not in permanent_intents:
raise ValueError(
"Intent cannot be mapped to '{}' since this intent"
" does not exist.".format(mapped_to))
intent_name = intent[INTENT_NAME_KEY]
existing = self._get_temporary_intent(intent_name, project_id)
if existing is not None:
examples = existing.temporary_examples
if INTENT_MAPPED_TO_KEY in intent.keys() and mapped_to is None:
for e in examples:
self.data_service.delete_example_by_hash(
project_id, e.example_hash)
else:
self.data_service.update_intent(
mapped_to or intent_name,
[e.example_hash for e in examples],
project_id,
)
temporary = Intent(
name=intent_name,
mapped_to=mapped_to,
project_id=project_id,
is_temporary=True,
)
if existing:
temporary.id = existing.id
self.merge(temporary)
else:
self.add(temporary)
def get_temporary_intents(
self,
project_id: Text,
include_example_hashes: bool = True) -> List[Dict[str, str]]:
"""Returns all temporary intents.
Args:
project_id: Project id of the temporary intents.
include_example_hashes: If `True` include the hashes of the examples for
each intent.
Returns:
List of intent objects.
"""
temporary_intents = (self.query(Intent).filter(
and_(Intent.project_id == project_id, Intent.is_temporary)).all())
return [t.as_dict(include_example_hashes) for t in temporary_intents]
def get_temporary_intent(self, intent_id: Text,
project_id: Text) -> Optional[Dict[Text, Text]]:
"""Returns a single temporary intent matching the name.
Args:
intent_id: Name of the temporary intent which is searched for.
project_id: Project id which the temporary intent belongs to.
Returns:
The intent object if the temporary intent was found, otherwise
`None`.
"""
intent = self._get_temporary_intent(intent_id, project_id)
if intent:
return intent.as_dict()
else:
return None
def _get_temporary_intent(self, intent_id: Text,
project_id: Text) -> Intent:
return (self.query(Intent).filter(
and_(
Intent.project_id == project_id,
Intent.is_temporary,
Intent.name == intent_id,
)).first())
def update_temporary_intent(self, intent_id: Text, intent: Dict,
project_id: Text):
"""Update an existing temporary intent.
Args:
intent_id: Temporary intent which should be updated.
intent: The intent object which is used to update the
existing intent.
project_id: Project id of the temporary intent.
"""
if intent.get(INTENT_NAME_KEY) != intent_id:
raise ValueError("Intent name '{}' cannot be changed to '{}'."
"".format(intent_id, intent.get(INTENT_NAME_KEY)))
self.add_temporary_intent(intent, project_id)
def delete_temporary_intent(self, intent_id: Text, project_id: Text):
"""Deletes a temporary intent and related training data.
Args:
intent_id: Name of the temporary intent which should be deleted.
project_id: Project id of the intent.
"""
existing = self._get_temporary_intent(intent_id, project_id)
if existing:
for e in existing.temporary_examples:
self.data_service.delete_example_by_hash(
project_id, e.example_hash)
self.delete(existing)
def get_permanent_intents(self, project_id: Text) -> Set[Text]:
"""Returns a list of all permanent (not temporary) intents
Args:
project_id: Project id which these intents belong to.
Returns:
Set of unique permanent intents.
"""
intents = self.domain_service.get_intents_from_domain(project_id)
intents |= {
i[INTENT_NAME_KEY]
for i in self.data_service.get_intents(project_id)
}
return intents # pytype: disable=bad-return-type
def get_intents(
self,
project_id: Text,
include_temporary_intents: bool = True,
fields_query: List[Tuple[Text, bool]] = False,
) -> List[Dict[str, Union[str, List[str]]]]:
"""Returns all intents including related data.
Args:
project_id: Project id which these intents belong to.
include_temporary_intents: If `False` exclude temporary intents.
fields_query: Query to select which fields should be included/excluded.
Returns:
List of intent objects including lists of `suggestions`,
related `training_data`, and `user_goal`.
"""
fields_query = fields_query or []
include_examples = (INTENT_EXAMPLES_KEY, False) not in fields_query
intents = self.data_service.get_intents(project_id, include_examples)
domain_intents = self.domain_service.get_intents_from_domain(
project_id)
flat_intents = [i[INTENT_NAME_KEY] for i in intents]
for intent_name in domain_intents:
if intent_name not in flat_intents:
intents.append({INTENT_NAME_KEY: intent_name})
include_suggestions = (INTENT_SUGGESTIONS_KEY,
False) not in fields_query
if include_suggestions:
self._add_suggestion_to_intents(intents)
if include_temporary_intents:
intents += self.get_temporary_intents(project_id, include_examples)
include_user_goals = (INTENT_USER_GOAL_KEY, False) not in fields_query
if include_user_goals:
self._add_user_goals_to_intents(intents, project_id)
return intents
def _add_suggestion_to_intents(
self, intents: List[Dict[str, Union[str, List[str]]]]) -> None:
logs_service = LogsService(self.session)
for i in intents:
suggestions, _ = logs_service.get_suggestions(
intent_query=i[INTENT_NAME_KEY], fields_query=[("hash", True)])
i[INTENT_SUGGESTIONS_KEY] = [s.get("hash") for s in suggestions]
def _add_user_goals_to_intents(self, intents: List[Dict[str,
Union[str,
List[str]]]],
project_id: Text) -> None:
# merge user goals
user_goal_service = UserGoalService(self.session)
user_goals = user_goal_service.get_user_goals(project_id)
for g in user_goals:
for i in intents:
if i[INTENT_NAME_KEY] in g[GOAL_INTENTS_KEY]:
i[INTENT_USER_GOAL_KEY] = g[GOAL_NAME_KEY]
def add_example_to_temporary_intent(self, intent: Text, example_hash: Text,
project_id: Text) -> None:
"""Adds a training example to a temporary intent.
Args:
intent: Name of the intent which the example should be added to.
example_hash: Text hash of the training example.
project_id: Project id of the temporary intent.
"""
intent = self._get_temporary_intent(intent, project_id)
if intent:
example = TemporaryIntentExample(intent_id=intent.id,
example_hash=example_hash)
self.add(example)
else:
logger.warning(
"Intent '{}' was not found. Hence, the example with "
"hash '{}' could not be deleted."
"".format(intent, example_hash))
def remove_example_from_temporary_intent(self, intent: Text,
example_hash: Text,
project_id: Text) -> None:
"""Remove a training example to a temporary intent.
Args:
intent: Name of the intent which the example should be removed
from.
example_hash: Text hash of the training example.
project_id: Project id of the temporary intent.
"""
temporary_intent = self._get_temporary_intent(intent, project_id)
example = (self.query(TemporaryIntentExample).filter(
and_(
TemporaryIntentExample.intent_id == temporary_intent.id,
TemporaryIntentExample.example_hash == example_hash,
)).first())
if example:
self.delete(example)
async def _get_project_status_event(
session: Session,
project_id: Text = config.project_name) -> Dict[Text, Any]:
"""Collect data used in `status` event.
Args:
session: Database session.
project_id: The project ID.
Returns:
A dictionary containing statistics describing the current project's status.
"""
from rasax.community.services.event_service import EventService
from rasax.community.services.domain_service import DomainService
from rasax.community.services.model_service import ModelService
from rasax.community.services.data_service import DataService
from rasax.community.services.story_service import StoryService
from rasax.community.services.settings_service import SettingsService
import rasax.community.services.test_service as test_service
from rasax.community.services import stack_service
event_service = EventService(session)
domain_service = DomainService(session)
model_service = ModelService(config.rasa_model_dir, session)
data_service = DataService(session)
story_service = StoryService(session)
settings_service = SettingsService(session)
domain = domain_service.get_domain(project_id) or {}
nlu_data = data_service.get_nlu_training_data_object(project_id=project_id)
stories = story_service.fetch_stories()
num_conversations = event_service.get_conversation_metadata_for_all_clients(
).count
num_events = event_service.get_events_count()
num_models = model_service.get_model_count()
lookup_tables = data_service.get_lookup_tables(project_id,
include_filenames=True)
num_lookup_table_files = len(
{table["filename"]
for table in lookup_tables})
num_lookup_table_entries = sum(
table.get("number_of_elements", 0) for table in lookup_tables)
synonyms = data_service.get_entity_synonyms(project_id)
num_synonyms = sum(len(entry["synonyms"]) for entry in synonyms)
num_regexes = data_service.get_regex_features(project_id).count
rasa_services = settings_service.stack_services(project_id)
version_responses = await stack_service.collect_version_calls(
rasa_services, timeout_in_seconds=ENVIRONMENT_LIVE_TIMEOUT)
environment_names = _environment_names(rasa_services)
tags = event_service.get_all_conversation_tags()
conversations_with_tags = set()
for tag in tags:
conversations_with_tags.update(tag["conversations"])
e2e_tests = test_service.get_tests_from_file()
return {
# Use the SHA256 of the project ID in case its value contains
# information about the user's use of Rasa X. On the analytics side,
# having the original value or the hash makes no difference. This
# reasoning is also applied on other values sent in this module.
"project":
hashlib.sha256(project_id.encode("utf-8")).hexdigest(),
"local_mode":
config.LOCAL_MODE,
"rasa_x":
__version__,
"rasa_open_source":
_rasa_version(version_responses),
"num_intent_examples":
len(nlu_data.intent_examples),
"num_entity_examples":
len(nlu_data.entity_examples),
"num_actions":
len(domain.get("actions", [])),
"num_templates":
len(
domain.get("responses", [])
), # Old nomenclature from when 'responses' were still called 'templates' in the domain
"num_slots":
len(domain.get("slots", [])),
"num_forms":
len(domain.get("forms", [])),
"num_intents":
len(domain.get("intents", [])),
"num_entities":
len(domain.get("entities", [])),
"num_stories":
len(stories),
"num_conversations":
num_conversations,
"num_events":
num_events,
"num_models":
num_models,
"num_lookup_table_files":
num_lookup_table_files,
"num_lookup_table_entries":
num_lookup_table_entries,
"num_synonyms":
num_synonyms,
"num_regexes":
num_regexes,
"num_environments":
len(environment_names),
"environment_names":
environment_names,
"num_live_environments":
_number_of_live_rasa_environments(version_responses),
"uptime_seconds":
utils.get_uptime(),
"num_tags":
len(tags),
"num_conversations_with_tags":
len(conversations_with_tags),
"num_e2e_tests":
len(e2e_tests),
}
def _dump_lookup_tables(self, lookup_table_changes: Set[int]) -> None:
from rasax.community.services.data_service import DataService
data_service = DataService(self.session)
data_service.dump_lookup_tables(lookup_table_changes)
def _dump_nlu_files(self, nlu_changes: Set[Text]) -> None:
from rasax.community.services.data_service import DataService
data_service = DataService(self.session)
data_service.dump_nlu_data(files=nlu_changes)