async def _inject_domain_from_model(self, project: Text,
domain_path: Text) -> None:
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
# do not inject if domain_service already contains a domain
if not domain_service.has_empty_or_no_domain(project):
return
from rasax.community.services.nlg_service import NlgService
from rasa.utils.io import read_yaml_file
data = read_yaml_file(domain_path)
# store responses if no responses found in NLG service
_, number_of_responses = NlgService(self.session).fetch_responses()
should_store_responses = number_of_responses == 0
domain_service.store_domain(
data,
project,
path=None,
store_responses=should_store_responses,
# responses injected by a model were already included in a training
have_responses_been_edited=False,
username=config.SYSTEM_USER,
)
async def save_stories(
self,
story_string: Text,
team: Text,
filename: Optional[Text] = None,
username: Optional[Text] = None,
dump_stories: bool = True,
add_story_items_to_domain: bool = True,
) -> List[Dict[Text, Any]]:
"""Saves stories from story string as individual stories."""
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
domain = domain_service.get_domain()
if not filename:
filename = self.assign_filename(team)
# split up data into blocks (a new StoryStep begins with #)
# a split on `#` covers stories beginning with either `##` or `#`
split_story_string = re.split("(\n|^)##?", story_string)
# blocks are the non-empty entries of `split_story_string`
blocks = [s for s in split_story_string if s not in ("", "\n")]
inserted = []
for b in blocks:
# we need to add the initial hashes again (##) to
# mark the beginning of a story block
story = "".join(("##", b)).strip()
# Here, we get a list of StorySteps. A StoryStep is a
# single story block that may or may not contain
# checkpoints. A story file contains one or more StorySteps
steps = await self.get_story_steps(story, domain)
if steps and steps[0].block_name:
new_story = Story(
name=steps[0].block_name,
story=story,
annotated_at=time.time(),
user=username,
filename=filename,
)
self.add(new_story)
self.flush() # flush to get inserted story id
if add_story_items_to_domain:
await self.add_domain_items_for_story(new_story.id)
inserted.append(new_story.as_dict())
if inserted:
if dump_stories:
background_dump_service.add_story_change(filename)
return inserted
else:
return []
def _generate_chat_token(session) -> None:
domain_service = DomainService(session)
existing_token = domain_service.get_token()
if not existing_token:
generated_token = domain_service.generate_and_save_token()
logger.debug(
"Generated chat token '{}' with expiry date {}"
"".format(generated_token.token, generated_token.expires)
)
async def create_new_action(request: Request,
project_id: Text) -> HTTPResponse:
domain_service = DomainService(request[REQUEST_DB_SESSION_KEY])
try:
created = domain_service.add_new_action(request.json, project_id)
return response.json(created, status=201)
except ValueError as e:
return rasa_x_utils.error(400,
"ActionCreationError",
"Action already exists.",
details=e)
async def delete_action(request: Request, action_id: int,
project_id: Text) -> HTTPResponse:
domain_service = DomainService(request[REQUEST_DB_SESSION_KEY])
try:
domain_service.delete_action(action_id)
return response.text("", 204)
except ValueError as e:
return rasa_x_utils.error(
404,
"ActionNotFound",
f"Action with id '{action_id}' was not found.",
details=e,
)
async def update_action(request: Request, action_id,
project_id: Text) -> HTTPResponse:
domain_service = DomainService(request[REQUEST_DB_SESSION_KEY])
try:
updated = domain_service.update_action(action_id, request.json)
return response.json(updated)
except ValueError as e:
return rasa_x_utils.error(
404,
"ActionNotFound",
f"Action with id '{action_id}' was not found.",
details=e,
)
def initialise_services(_session):
return (
UserService(_session),
SettingsService(_session),
DomainService(_session),
RoleService(_session),
)
async def _add_story_items_to_domain(
self,
project_id: Text,
story_events: Tuple[Set[Text], Set[Text], Set[Text], Set[Text]],
):
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
domain_service.add_items_to_domain(
actions=story_events[0],
intents=story_events[1],
slots=story_events[2],
entities=story_events[3],
project_id=project_id,
dump_data=False,
origin="stories",
)
async def get_domain_warnings(request: Request) -> HTTPResponse:
domain_service = DomainService(request[REQUEST_DB_SESSION_KEY])
domain_warnings = await domain_service.get_domain_warnings()
if domain_warnings is None:
return rasa_x_utils.error(400, "DomainNotFound",
"Could not find domain.")
return response.json(domain_warnings[0],
headers={"X-Total-Count": domain_warnings[1]})
async def _fetch_domain_items_from_story(
self, story_id: Text
) -> Optional[Tuple[Set[Text], Set[Text], Set[Text], Set[Text]]]:
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
domain = domain_service.get_domain()
story = self.fetch_story(story_id)
if not story:
return None
steps = await self.get_story_steps(story["story"], domain)
story_actions = set()
story_intents = set()
story_entities = set()
story_slots = set()
for step in steps:
for e in step.events:
if (isinstance(e, ActionExecuted)
# exclude default actions and utter actions
and e.action_name not in default_action_names() and
not e.action_name.startswith(UTTER_PREFIX)):
story_actions.add(e.action_name)
elif isinstance(e, UserUttered):
intent = e.intent
entities = e.entities
if intent:
story_intents.add(intent.get("name"))
if entities:
entity_names = [e["entity"] for e in entities]
story_entities.update(entity_names)
elif isinstance(e, SlotSet):
slot = e.key
if slot:
story_slots.add(slot)
return story_actions, story_intents, story_slots, story_entities
def _initialize_with_local_data(
project_path: Text,
data_path: Text,
session: Session,
rasa_port: Union[int, Text],
config_path: Text,
) -> Tuple[Dict[Text, Any], List[Dict[Text, Any]], TrainingData]:
settings_service = SettingsService(session)
default_env = default_environments_config_local(rasa_port)
settings_service.save_environments_config(COMMUNITY_PROJECT_NAME,
default_env.get("environments"))
loop = asyncio.get_event_loop()
# inject data
domain, story_blocks, nlu_data = loop.run_until_complete(
rasax.community.initialise.inject_files_from_disk(
project_path, data_path, session, config_path=config_path))
# dump domain once
domain_service = DomainService(session)
domain_service.dump_domain()
return domain, story_blocks, nlu_data
async def get_domain(request: Request) -> HTTPResponse:
domain_service = DomainService(request[REQUEST_DB_SESSION_KEY])
domain = domain_service.get_domain()
if domain is None:
return rasa_x_utils.error(400, "DomainNotFound",
"Could not find domain.")
domain_service.remove_domain_edited_states(domain)
return response.text(domain_service.dump_cleaned_domain_yaml(domain))
async def get_domain_actions(request: Request,
project_id: Text) -> HTTPResponse:
domain_actions = DomainService(
request[REQUEST_DB_SESSION_KEY]).get_actions_from_domain(
project_id)
if domain_actions is None:
return rasa_x_utils.error(400, "DomainNotFound",
"Could not find domain.")
# convert to list for json serialisation
domain_actions = list(domain_actions)
return response.json(domain_actions,
headers={"X-Total-Count": len(domain_actions)})
async def update_story(self, _id: Text, story_string: Text,
user: Dict[Text, Any]) -> Optional[Dict[Text, Any]]:
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
domain = domain_service.get_domain()
story_steps = await self.get_story_steps(story_string, domain)
if story_steps:
story = self.query(Story).filter(Story.id == _id).first()
if story:
story.user = user["username"]
story.annotated_at = time.time()
story.name = story_steps[0].block_name
story.story = story_string.strip()
background_dump_service.add_story_change(story.filename)
await self.add_domain_items_for_story(_id)
return story.as_dict()
return None
async def inject_files_from_disk(
project_path: Union[Path, Text],
data_path: Text,
session: orm.Session,
username: Optional[Text] = constants.COMMUNITY_USERNAME,
config_path: Optional[Text] = config.default_config_path,
) -> Tuple[Dict[Text, Any], List[Dict[Text, Any]], "NluTrainingData"]:
"""Injects local files into database.
Args:
project_path: Path to the project of which the data should be injected.
data_path: Path to the data within this project.
session: Database session.
username: The username which is used to inject the data.
config_path: Path to the config file within the project
Returns:
Tuple of domain, stories, and NLU training data.
"""
import rasa.data
from rasax.community.local import LOCAL_DOMAIN_PATH
from rasax.community.services.data_service import DataService
from rasax.community.services.settings_service import SettingsService
utils.set_project_directory(project_path)
domain_service = DomainService(session)
domain = inject_domain(
os.path.join(project_path, LOCAL_DOMAIN_PATH),
domain_service,
constants.COMMUNITY_PROJECT_NAME,
username,
)
settings_service = SettingsService(session)
inject_config(os.path.join(project_path, config_path), settings_service)
story_files, nlu_files = rasa.data.get_core_nlu_files([data_path])
story_service = StoryService(session)
story_blocks = await inject_stories(story_files, story_service, username,
constants.COMMUNITY_TEAM_NAME)
data_service = DataService(session)
nlu_data = inject_nlu_data(nlu_files, constants.COMMUNITY_PROJECT_NAME,
username, data_service)
return domain, story_blocks, nlu_data
async def update_domain(request: Request, user: Dict) -> HTTPResponse:
rasa_x_utils.handle_deprecated_request_parameters(
request, "store_templates", "store_responses")
store_responses = utils.bool_arg(request, "store_responses", False)
domain_yaml = rasa_core_utils.convert_bytes_to_string(request.body)
try:
updated_domain = DomainService(request[REQUEST_DB_SESSION_KEY]
).validate_and_store_domain_yaml(
domain_yaml,
store_responses=store_responses,
username=user["username"])
except InvalidDomain as e:
return rasa_x_utils.error(400, "InvalidDomainError",
"Could not update domain.", e)
return response.text(updated_domain)
def stack_services(
self,
project_id: Text = config.project_name
) -> Dict[Text, StackService]:
"""Create StackServices for all Stack servers."""
from rasax.community.services.stack_service import RasaCredentials
environments_config = self.get_environments_config(project_id).get(
"environments", {})
_stack_services = {}
for k, v in environments_config.get("rasa", {}).items():
credentials = RasaCredentials(url=v["url"], token=v.get("token"))
_stack_services[k] = StackService(
credentials,
DataService(self.session),
StoryService(self.session),
DomainService(self.session),
self,
)
return _stack_services
def __init__(self, session: Session):
self.domain_service = DomainService(session)
self.data_service = DataService(session)
super().__init__(session)
class IntentService(DbService):
def __init__(self, session: Session):
self.domain_service = DomainService(session)
self.data_service = DataService(session)
super().__init__(session)
def add_temporary_intent(self, intent: Dict[str, str],
project_id: Text) -> None:
"""Creates a new temporary intent.
Args:
intent: The intent object which should be added.
project_id: The project id which the intent should belong to.
"""
mapped_to = intent.get(INTENT_MAPPED_TO_KEY)
permanent_intents = self.get_permanent_intents(project_id)
if mapped_to and mapped_to not in permanent_intents:
raise ValueError(
"Intent cannot be mapped to '{}' since this intent"
" does not exist.".format(mapped_to))
intent_name = intent[INTENT_NAME_KEY]
existing = self._get_temporary_intent(intent_name, project_id)
if existing is not None:
examples = existing.temporary_examples
if INTENT_MAPPED_TO_KEY in intent.keys() and mapped_to is None:
for e in examples:
self.data_service.delete_example_by_hash(
project_id, e.example_hash)
else:
self.data_service.update_intent(
mapped_to or intent_name,
[e.example_hash for e in examples],
project_id,
)
temporary = Intent(
name=intent_name,
mapped_to=mapped_to,
project_id=project_id,
is_temporary=True,
)
if existing:
temporary.id = existing.id
self.merge(temporary)
else:
self.add(temporary)
def get_temporary_intents(
self,
project_id: Text,
include_example_hashes: bool = True) -> List[Dict[str, str]]:
"""Returns all temporary intents.
Args:
project_id: Project id of the temporary intents.
include_example_hashes: If `True` include the hashes of the examples for
each intent.
Returns:
List of intent objects.
"""
temporary_intents = (self.query(Intent).filter(
and_(Intent.project_id == project_id, Intent.is_temporary)).all())
return [t.as_dict(include_example_hashes) for t in temporary_intents]
def get_temporary_intent(self, intent_id: Text,
project_id: Text) -> Optional[Dict[Text, Text]]:
"""Returns a single temporary intent matching the name.
Args:
intent_id: Name of the temporary intent which is searched for.
project_id: Project id which the temporary intent belongs to.
Returns:
The intent object if the temporary intent was found, otherwise
`None`.
"""
intent = self._get_temporary_intent(intent_id, project_id)
if intent:
return intent.as_dict()
else:
return None
def _get_temporary_intent(self, intent_id: Text,
project_id: Text) -> Intent:
return (self.query(Intent).filter(
and_(
Intent.project_id == project_id,
Intent.is_temporary,
Intent.name == intent_id,
)).first())
def update_temporary_intent(self, intent_id: Text, intent: Dict,
project_id: Text):
"""Update an existing temporary intent.
Args:
intent_id: Temporary intent which should be updated.
intent: The intent object which is used to update the
existing intent.
project_id: Project id of the temporary intent.
"""
if intent.get(INTENT_NAME_KEY) != intent_id:
raise ValueError("Intent name '{}' cannot be changed to '{}'."
"".format(intent_id, intent.get(INTENT_NAME_KEY)))
self.add_temporary_intent(intent, project_id)
def delete_temporary_intent(self, intent_id: Text, project_id: Text):
"""Deletes a temporary intent and related training data.
Args:
intent_id: Name of the temporary intent which should be deleted.
project_id: Project id of the intent.
"""
existing = self._get_temporary_intent(intent_id, project_id)
if existing:
for e in existing.temporary_examples:
self.data_service.delete_example_by_hash(
project_id, e.example_hash)
self.delete(existing)
def get_permanent_intents(self, project_id: Text) -> Set[Text]:
"""Returns a list of all permanent (not temporary) intents
Args:
project_id: Project id which these intents belong to.
Returns:
Set of unique permanent intents.
"""
intents = self.domain_service.get_intents_from_domain(project_id)
intents |= {
i[INTENT_NAME_KEY]
for i in self.data_service.get_intents(project_id)
}
return intents # pytype: disable=bad-return-type
def get_intents(
self,
project_id: Text,
include_temporary_intents: bool = True,
fields_query: List[Tuple[Text, bool]] = False,
) -> List[Dict[str, Union[str, List[str]]]]:
"""Returns all intents including related data.
Args:
project_id: Project id which these intents belong to.
include_temporary_intents: If `False` exclude temporary intents.
fields_query: Query to select which fields should be included/excluded.
Returns:
List of intent objects including lists of `suggestions`,
related `training_data`, and `user_goal`.
"""
fields_query = fields_query or []
include_examples = (INTENT_EXAMPLES_KEY, False) not in fields_query
intents = self.data_service.get_intents(project_id, include_examples)
domain_intents = self.domain_service.get_intents_from_domain(
project_id)
flat_intents = [i[INTENT_NAME_KEY] for i in intents]
for intent_name in domain_intents:
if intent_name not in flat_intents:
intents.append({INTENT_NAME_KEY: intent_name})
include_suggestions = (INTENT_SUGGESTIONS_KEY,
False) not in fields_query
if include_suggestions:
self._add_suggestion_to_intents(intents)
if include_temporary_intents:
intents += self.get_temporary_intents(project_id, include_examples)
include_user_goals = (INTENT_USER_GOAL_KEY, False) not in fields_query
if include_user_goals:
self._add_user_goals_to_intents(intents, project_id)
return intents
def _add_suggestion_to_intents(
self, intents: List[Dict[str, Union[str, List[str]]]]) -> None:
logs_service = LogsService(self.session)
for i in intents:
suggestions, _ = logs_service.get_suggestions(
intent_query=i[INTENT_NAME_KEY], fields_query=[("hash", True)])
i[INTENT_SUGGESTIONS_KEY] = [s.get("hash") for s in suggestions]
def _add_user_goals_to_intents(self, intents: List[Dict[str,
Union[str,
List[str]]]],
project_id: Text) -> None:
# merge user goals
user_goal_service = UserGoalService(self.session)
user_goals = user_goal_service.get_user_goals(project_id)
for g in user_goals:
for i in intents:
if i[INTENT_NAME_KEY] in g[GOAL_INTENTS_KEY]:
i[INTENT_USER_GOAL_KEY] = g[GOAL_NAME_KEY]
def add_example_to_temporary_intent(self, intent: Text, example_hash: Text,
project_id: Text) -> None:
"""Adds a training example to a temporary intent.
Args:
intent: Name of the intent which the example should be added to.
example_hash: Text hash of the training example.
project_id: Project id of the temporary intent.
"""
intent = self._get_temporary_intent(intent, project_id)
if intent:
example = TemporaryIntentExample(intent_id=intent.id,
example_hash=example_hash)
self.add(example)
else:
logger.warning(
"Intent '{}' was not found. Hence, the example with "
"hash '{}' could not be deleted."
"".format(intent, example_hash))
def remove_example_from_temporary_intent(self, intent: Text,
example_hash: Text,
project_id: Text) -> None:
"""Remove a training example to a temporary intent.
Args:
intent: Name of the intent which the example should be removed
from.
example_hash: Text hash of the training example.
project_id: Project id of the temporary intent.
"""
temporary_intent = self._get_temporary_intent(intent, project_id)
example = (self.query(TemporaryIntentExample).filter(
and_(
TemporaryIntentExample.intent_id == temporary_intent.id,
TemporaryIntentExample.example_hash == example_hash,
)).first())
if example:
self.delete(example)
async def _get_project_status_event(
session: Session,
project_id: Text = config.project_name) -> Dict[Text, Any]:
"""Collect data used in `status` event.
Args:
session: Database session.
project_id: The project ID.
Returns:
A dictionary containing statistics describing the current project's status.
"""
from rasax.community.services.event_service import EventService
from rasax.community.services.domain_service import DomainService
from rasax.community.services.model_service import ModelService
from rasax.community.services.data_service import DataService
from rasax.community.services.story_service import StoryService
from rasax.community.services.settings_service import SettingsService
import rasax.community.services.test_service as test_service
from rasax.community.services import stack_service
event_service = EventService(session)
domain_service = DomainService(session)
model_service = ModelService(config.rasa_model_dir, session)
data_service = DataService(session)
story_service = StoryService(session)
settings_service = SettingsService(session)
domain = domain_service.get_domain(project_id) or {}
nlu_data = data_service.get_nlu_training_data_object(project_id=project_id)
stories = story_service.fetch_stories()
num_conversations = event_service.get_conversation_metadata_for_all_clients(
).count
num_events = event_service.get_events_count()
num_models = model_service.get_model_count()
lookup_tables = data_service.get_lookup_tables(project_id,
include_filenames=True)
num_lookup_table_files = len(
{table["filename"]
for table in lookup_tables})
num_lookup_table_entries = sum(
table.get("number_of_elements", 0) for table in lookup_tables)
synonyms = data_service.get_entity_synonyms(project_id)
num_synonyms = sum(len(entry["synonyms"]) for entry in synonyms)
num_regexes = data_service.get_regex_features(project_id).count
rasa_services = settings_service.stack_services(project_id)
version_responses = await stack_service.collect_version_calls(
rasa_services, timeout_in_seconds=ENVIRONMENT_LIVE_TIMEOUT)
environment_names = _environment_names(rasa_services)
tags = event_service.get_all_conversation_tags()
conversations_with_tags = set()
for tag in tags:
conversations_with_tags.update(tag["conversations"])
e2e_tests = test_service.get_tests_from_file()
return {
# Use the SHA256 of the project ID in case its value contains
# information about the user's use of Rasa X. On the analytics side,
# having the original value or the hash makes no difference. This
# reasoning is also applied on other values sent in this module.
"project":
hashlib.sha256(project_id.encode("utf-8")).hexdigest(),
"local_mode":
config.LOCAL_MODE,
"rasa_x":
__version__,
"rasa_open_source":
_rasa_version(version_responses),
"num_intent_examples":
len(nlu_data.intent_examples),
"num_entity_examples":
len(nlu_data.entity_examples),
"num_actions":
len(domain.get("actions", [])),
"num_templates":
len(
domain.get("responses", [])
), # Old nomenclature from when 'responses' were still called 'templates' in the domain
"num_slots":
len(domain.get("slots", [])),
"num_forms":
len(domain.get("forms", [])),
"num_intents":
len(domain.get("intents", [])),
"num_entities":
len(domain.get("entities", [])),
"num_stories":
len(stories),
"num_conversations":
num_conversations,
"num_events":
num_events,
"num_models":
num_models,
"num_lookup_table_files":
num_lookup_table_files,
"num_lookup_table_entries":
num_lookup_table_entries,
"num_synonyms":
num_synonyms,
"num_regexes":
num_regexes,
"num_environments":
len(environment_names),
"environment_names":
environment_names,
"num_live_environments":
_number_of_live_rasa_environments(version_responses),
"uptime_seconds":
utils.get_uptime(),
"num_tags":
len(tags),
"num_conversations_with_tags":
len(conversations_with_tags),
"num_e2e_tests":
len(e2e_tests),
}
def _domain_service(request: Request) -> DomainService:
return DomainService(request[REQUEST_DB_SESSION_KEY])
def _dump_domain(self) -> None:
from rasax.community.services.domain_service import DomainService
domain_service = DomainService(self.session)
domain_service.dump_domain()
