async def send_confirmation_link(mail: str): """ Sends a link to the user's mail id for account verification :param mail: the mail id of the user :return: mail id, mail subject and mail body """ if AccountProcessor.EMAIL_ENABLED: if isinstance(mail_check(mail), ValidationFailure): raise AppException("Please enter valid email id") Utility.is_exist(UserEmailConfirmation, exp_message="Email already confirmed!", email__iexact=mail.strip()) if not Utility.is_exist( User, email__iexact=mail.strip(), raise_error=False): raise AppException( "Error! There is no user with the following mail id") token = Utility.generate_token(mail) link = Utility.email_conf["app"]["url"] + '/verify/' + token body = Utility.email_conf['email']['templates'][ 'confirmation_body'] + link subject = Utility.email_conf['email']['templates'][ 'confirmation_subject'] return mail, subject, body else: raise AppException("Error! Email verification is not enabled")
def verify_intents(self, raise_exception: bool = True): """ Validated intents in nlu.yml. @param raise_exception: Set this flag to false to prevent raising exceptions. @return: """ intents_mismatch_summary = [] nlu_data_intents = {e.data["intent"] for e in self.intents.intent_examples} self.component_count['intents'] = len(nlu_data_intents) for intent in self.domain.intents: if intent not in nlu_data_intents and intent not in DEFAULT_INTENTS: msg = f"The intent '{intent}' is listed in the domain file, but " \ f"is not found in the NLU training data." if raise_exception: raise AppException(msg) intents_mismatch_summary.append(msg) for intent in nlu_data_intents: if intent not in self.domain.intents and intent not in DEFAULT_INTENTS: msg = f"There is a message in the training data labeled with intent '{intent}'." \ f" This intent is not listed in your domain." if raise_exception: raise AppException(msg) intents_mismatch_summary.append(msg) self.summary['intents'] = intents_mismatch_summary
async def send_confirmation_link(mail: str): """ Sends a link to the user's mail id for account verification :param mail: the mail id of the user :return: mail id, mail subject and mail body """ email_enabled = Utility.email_conf["email"]["enable"] if email_enabled: if isinstance(mail_check(mail), ValidationFailure): raise AppException("Please enter valid email id") Utility.is_exist(UserEmailConfirmation, exp_message="Email already confirmed!", email__iexact=mail.strip()) if not Utility.is_exist(User, email__iexact=mail.strip(), status=True, raise_error=False): raise AppException( "Error! There is no user with the following mail id") user = AccountProcessor.get_user(mail) token = Utility.generate_token(mail) link = Utility.email_conf["app"]["url"] + '/verify/' + token return mail, user['first_name'], link else: raise AppException("Error! Email verification is not enabled")
def __update_role(bot: Text, accessor_email: Text, user: Text, role: ACCESS_ROLES = ACCESS_ROLES.TESTER.value, status: ACTIVITY_STATUS = ACTIVITY_STATUS.ACTIVE.value, validate_ownership_modification: bool = True): AccountProcessor.get_user(accessor_email) try: bot_access = BotAccess.objects( accessor_email=accessor_email, bot=bot, status__ne=ACTIVITY_STATUS.DELETED.value).get() if Utility.email_conf["email"][ "enable"] and bot_access.status == ACTIVITY_STATUS.INVITE_NOT_ACCEPTED.value: raise AppException('User is yet to accept the invite') if validate_ownership_modification and ACCESS_ROLES.OWNER.value in { role, bot_access.role }: raise AppException('Ownership modification denied') bot_access.role = role bot_access.user = user bot_access.status = status bot_access.timestamp = datetime.utcnow() bot_access.save() except DoesNotExist: raise AppException('User not yet invited to collaborate')
def verify_intents_in_stories(self, raise_exception: bool = True): """ Validates intents in stories. @param raise_exception: Set this flag to false to prevent raising exceptions. @return: """ intents_mismatched = [] self.verify_intents(raise_exception) stories_intents = { event.intent["name"] for story in self.story_graph.story_steps for event in story.events if isinstance(event, UserUttered) } for story_intent in stories_intents: if story_intent not in self.domain.intents and story_intent not in DEFAULT_INTENTS: msg = f"The intent '{story_intent}' is used in your stories, but it is not listed in " \ f"the domain file. You should add it to your domain file!" if raise_exception: raise AppException(msg) intents_mismatched.append(msg) for intent in self.domain.intents: if intent not in stories_intents and intent not in DEFAULT_INTENTS: msg = f"The intent '{intent}' is not used in any story." if raise_exception: raise AppException(msg) intents_mismatched.append(msg) if not self.summary.get('intents'): self.summary['intents'] = [] self.summary['intents'] = self.summary['intents'] + intents_mismatched
def verify_utterances(self, raise_exception: bool = True): """ Validated utterances in domain. @param raise_exception: Set this flag to false to prevent raising exceptions. @return: """ utterance_mismatch_summary = [] actions = self.domain.action_names_or_texts utterance_templates = set(self.domain.responses) self.component_count['utterances'] = len(self.domain.responses) for utterance in utterance_templates: if utterance not in actions: msg = f"The utterance '{utterance}' is not listed under 'actions' in the domain file." \ f" It can only be used as a template." if raise_exception: raise AppException(msg) utterance_mismatch_summary.append(msg) for action in actions: if action.startswith(UTTER_PREFIX): if action not in utterance_templates: msg = f"There is no template for the utterance action '{action}'. " \ f"The action is listed in your domains action list, but there is no " \ f"template defined with this name. You should add a template with this key." if raise_exception: raise AppException(msg) utterance_mismatch_summary.append(msg) self.summary['utterances'] = utterance_mismatch_summary
async def from_training_files(cls, training_data_paths: str, domain_path: str, config_path: str, root_dir): """ Create validator from training files. @param training_data_paths: nlu.yml file path. @param domain_path: domain.yml file path. @param config_path: config.yml file path. @param root_dir: training data root directory. @return: """ if not (os.path.exists(training_data_paths) and os.path.exists(domain_path) and os.path.exists(config_path)): raise AppException("Some training files are absent!") try: file_importer = RasaFileImporter( domain_path=domain_path, training_data_paths=training_data_paths, config_file=config_path, ) cls.actions = Utility.read_yaml(os.path.join(root_dir, 'actions.yml')) return await TrainingDataValidator.from_importer(file_importer) except YamlValidationException as e: exc = Utility.replace_file_name(str(e), root_dir) raise AppException(exc) except YamlSyntaxException as e: exc = Utility.replace_file_name(str(e), root_dir) raise AppException(exc) except Exception as e: raise AppException(e)
def update_bot(name: Text, bot: Text): if Utility.check_empty_string(name): raise AppException('Name cannot be empty') try: bot_info = Bot.objects(id=bot, status=True).get() bot_info.name = name bot_info.save() except DoesNotExist: raise AppException('Bot not found')
def validate_history_id(doc_id): try: history = TrainingDataGenerator.objects().get(id=doc_id) if not history.response: logger.info("response field is empty.") raise AppException("No Training Data Generated") except DoesNotExist as e: logger.error(str(e)) raise AppException("Matching history_id not found!")
def verify_utterances_in_stories(self, raise_exception: bool = True): """ Validates utterances in stories. @param raise_exception: Set this flag to false to prevent raising exceptions. @return: """ utterance_mismatch_summary = [] story_utterance_not_found_in_domain = [] self.verify_utterances(raise_exception) utterance_actions = self.validator._gather_utterance_actions() fallback_action = DataUtility.parse_fallback_action(self.config) system_triggered_actions = DEFAULT_ACTIONS.union(SYSTEM_TRIGGERED_UTTERANCES) stories_utterances = set() for story in self.story_graph.story_steps: for event in story.events: if not isinstance(event, ActionExecuted): continue if not event.action_name.startswith(UTTER_PREFIX): # we are only interested in utter actions continue if event.action_name in stories_utterances: # we already processed this one before, we only want to warn once continue if event.action_name not in utterance_actions and event.action_name not in system_triggered_actions: msg = f"The action '{event.action_name}' is used in the stories, " \ f"but is not a valid utterance action. Please make sure " \ f"the action is listed in your domain and there is a " \ f"template defined with its name." if raise_exception: raise AppException(msg) story_utterance_not_found_in_domain.append(msg) stories_utterances.add(event.action_name) for utterance in utterance_actions: if utterance not in stories_utterances and utterance not in system_triggered_actions.union(fallback_action): msg = f"The utterance '{utterance}' is not used in any story." if raise_exception: raise AppException(msg) utterance_mismatch_summary.append(msg) if not self.summary.get('utterances'): self.summary['utterances'] = [] self.summary['utterances'] = self.summary['utterances'] + utterance_mismatch_summary if not self.summary.get('stories'): self.summary['stories'] = [] self.summary['stories'] = self.summary['stories'] + story_utterance_not_found_in_domain
def run_tests_on_model(bot: str, run_e2e: bool = False): """ Runs tests on a trained model. Args: bot: bot id for which test is run. run_e2e: if True, test is initiated on test stories and nlu data. Returns: dictionary with evaluation results """ from kairon import Utility from rasa.utils.common import run_in_loop bot_home = os.path.join('testing_data', bot) logger.info(f"model test data path: {bot_home}") try: model_path = Utility.get_latest_model(bot) nlu_path, stories_path = TestDataGenerator.create(bot, run_e2e) stories_results = run_in_loop(ModelTester.run_test_on_stories(stories_path, model_path, run_e2e)) nlu_results = ModelTester.run_test_on_nlu(nlu_path, model_path) return nlu_results, stories_results except Exception as e: raise AppException(f'Model testing failed: {e}') finally: if os.path.exists(bot_home): Utility.delete_directory(bot_home)
def wrapped(current_user: User, **kwargs): account = Account.objects().get(id=current_user.account) count = Intents.objects(bot=current_user.get_bot()).count() limit = account.license[ 'intents'] if "intents" in account.license else 10 if count >= limit: raise AppException("Intent limit exhausted!")
def wrapped(current_user: User, **kwargs): account = Account.objects().get(id=current_user.account) limit = account.license[ 'augmentation'] if "augmentation" in account.license else 5 count = ModelTraining.objects(bot=current_user.get_bot()).count() if count >= limit: raise AppException("Daily augmentation limit exhausted!")
def wrapped(current_user: User, **kwargs): account = Account.objects().get(id=current_user.account) limit = account.license[ 'examples'] if "examples" in account.license else 50 count = TrainingExamples.objects(bot=current_user.get_bot()).count() if count >= limit: raise AppException("Training example limit exhausted!")
def add_bot_for_user(bot: Text, email: Text): try: user = User.objects().get(email=email, status=True) user.bot.append(bot) user.save() except DoesNotExist: raise AppException('User not found')
def add_account(name: str, user: str): """ adds a new account :param name: account name :param user: user id :return: account id """ if Utility.check_empty_string(name): raise AppException("Account Name cannot be empty or blank spaces") Utility.is_exist( Account, exp_message="Account name already exists!", name__iexact=name, status=True, ) license = { "bots": 2, "intents": 10, "examples": 50, "training": 3, "augmentation": 5 } return Account(name=name.strip(), user=user, license=license).save().to_mongo().to_dict()
def fetch_user_history(bot: Text, sender_id: Text, month: int = 1): """ loads list of conversation events from chat history :param month: default is current month and max is last 6 months :param bot: bot id :param sender_id: user id :param latest_history: whether to fetch latest history or complete history, default is latest :return: list of conversation events """ client, db_name, collection, message = ChatHistory.get_mongo_connection(bot) with client as client: try: db = client.get_database(db_name) conversations = db.get_collection(collection) values = list(conversations .aggregate([{"$match": {"sender_id": sender_id, "events.timestamp": {"$gte": Utility.get_timestamp_previous_month(month)}}}, {"$unwind": "$events"}, {"$match": {"events.event": {"$in": ["user", "bot", "action"]}}}, {"$group": {"_id": None, "events": {"$push": "$events"}}}, {"$project": {"_id": 0, "events": 1}}]) ) if values: return ( values[0]['events'], message ) return [], message except Exception as e: raise AppException(e)
def verify_story_structure(self, raise_exception: bool = True, max_history: Optional[int] = None): """ Validates whether the bot behaviour in stories is deterministic. @param raise_exception: Set this flag to false to prevent raising exceptions. @param max_history: @return: """ self.component_count['stories'] = 0 self.component_count['rules'] = 0 for steps in self.story_graph.story_steps: if isinstance(steps, RuleStep): self.component_count['rules'] += 1 elif isinstance(steps, StoryStep): self.component_count['stories'] += 1 trackers = TrainingDataGenerator( self.story_graph, domain=self.domain, remove_duplicates=False, augmentation_factor=0, ).generate_story_trackers() conflicts = find_story_conflicts( trackers, self.domain, max_history ) if conflicts: conflict_msg = [] for conflict in conflicts: conflict_msg.append(str(conflict)) if raise_exception: raise AppException(str(conflict)) self.summary['stories'] = conflict_msg
def create(bot: str, use_test_stories: bool = False): from kairon import Utility from itertools import chain from rasa.shared.nlu.training_data.training_data import TrainingData bot_home = os.path.join('testing_data', bot) Utility.make_dirs(bot_home) processor = MongoProcessor() intents_and_training_examples = processor.get_intents_and_training_examples(bot) aug_training_examples = map(lambda training_data: TestDataGenerator.__prepare_nlu(training_data[0], training_data[1]), intents_and_training_examples.items()) messages = list(chain.from_iterable(aug_training_examples)) nlu_data = TrainingData(training_examples=messages) stories = processor.load_stories(bot) rules = processor.get_rules_for_training(bot) stories = stories.merge(rules) if stories.is_empty() or nlu_data.is_empty(): raise AppException('Not enough training data exists. Please add some training data.') nlu_as_str = nlu_data.nlu_as_yaml().encode() nlu_path = os.path.join(bot_home, "nlu.yml") Utility.write_to_file(nlu_path, nlu_as_str) if use_test_stories: stories_path = os.path.join(bot_home, "test_stories.yml") else: stories_path = os.path.join(bot_home, "stories.yml") YAMLStoryWriter().dump(stories_path, stories.story_steps, is_test_story=use_test_stories) return nlu_path, stories_path
def fetch_chat_users(bot: Text, month: int = 1): """ fetches user list who has conversation with the agent :param month: default is current month and max is last 6 months :param bot: bot id :return: list of user id """ client, db_name, collection, message = ChatHistory.get_mongo_connection( bot) db = client.get_database(db_name) conversations = db.get_collection(collection) users = [] try: values = conversations.find( { "events.timestamp": { "$gte": Utility.get_timestamp_previous_month(month) } }, { "_id": 0, "sender_id": 1 }) users = [sender["sender_id"] for sender in values] except Exception as e: raise AppException(e) finally: client.close() return users, message
def add_integration(name: Text, bot: Text, user: Text, role: ACCESS_ROLES, iat: datetime = datetime.utcnow(), expiry: datetime = None, access_list: list = None): integration_limit = Utility.environment['security'].get( 'integrations_per_user') or 2 current_integrations_count = Integration.objects( bot=bot, status__ne=INTEGRATION_STATUS.DELETED.value).count() if current_integrations_count >= integration_limit: raise AppException('Integrations limit reached!') Utility.is_exist( Integration, 'Integration token with this name has already been initiated', name=name, bot=bot, status__ne=INTEGRATION_STATUS.DELETED.value) Integration(name=name, bot=bot, user=user, role=role, iat=iat, expiry=expiry, access_list=access_list, status=INTEGRATION_STATUS.ACTIVE.value).save()
def train_model_for_bot(bot: str): """ loads bot data from mongo into individual files for training :param bot: bot id :return: model path """ processor = MongoProcessor() nlu = processor.load_nlu(bot) if not nlu.training_examples: raise AppException("Training data does not exists!") domain = processor.load_domain(bot) stories = processor.load_stories(bot) config = processor.load_config(bot) rules = processor.get_rules_for_training(bot) directory = Utility.write_training_data(nlu, domain, config, stories, rules) output = os.path.join(DEFAULT_MODELS_PATH, bot) model = train( domain=os.path.join(directory, DEFAULT_DOMAIN_PATH), config=os.path.join(directory, DEFAULT_CONFIG_PATH), training_files=os.path.join(directory, DEFAULT_DATA_PATH), output=output, ) Utility.delete_directory(directory) del processor del nlu del domain del stories del config return model
def wrapped(current_user: User, **kwargs): today = datetime.today() today_start = today.replace(hour=0, minute=0, second=0) account = Account.objects().get(id=current_user.account) limit = account.license['training'] if "training" in account.license else Utility.environment['model']['train'][ "limit_per_day"] count = ModelTraining.objects(bot=current_user.get_bot(), start_timestamp__gte=today_start).count() if count >= limit: raise AppException("Training limit exhausted!")
def is_password_reset_request_limit_exceeded(email: Text): reset_password_request_limit = Utility.environment['user'][ 'reset_password_request_limit'] reset_password_request_count = UserActivityLog.objects( user=email, type=UserActivityType.reset_password_request.value, timestamp__gte=datetime.utcnow().date()).count() if reset_password_request_count >= reset_password_request_limit: raise AppException('Password reset limit exhausted for today.')
def fetch_role_for_user(email: Text, bot: Text): try: return BotAccess.objects(accessor_email=email, bot=bot, status=ACTIVITY_STATUS.ACTIVE.value).get( ).to_mongo().to_dict() except DoesNotExist as e: logging.error(e) raise AppException('Access to bot is denied')
def get_client(sso_type: str): """ Fetches user details using code received in the request. :param sso_type: one of supported types - google/facebook/linkedin. """ if not LoginSSOFactory.sso_clients.get(sso_type): raise AppException(f'{sso_type} login is not supported') return LoginSSOFactory.sso_clients[sso_type]()
async def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData: """ loads training examples """ training_data = self.processor.load_nlu(self.bot) if not training_data.training_examples: raise AppException("Training data does not exists!") return training_data
def validate_config(self, raise_exception: bool = True): """ Validates config.yml. @param raise_exception: Set this flag to false to prevent raising exceptions. @return: """ config_errors = TrainingDataValidator.validate_rasa_config(self.config) self.summary['config'] = config_errors if config_errors and raise_exception: raise AppException("Invalid config.yml. Check logs!")
def add_bot(name: str, account: int, user: str, is_new_account: bool = False): """ add a bot to account :param name: bot name :param account: account id :param user: user id :param is_new_account: True if it is a new account :return: bot id """ from kairon.shared.data.processor import MongoProcessor from kairon.shared.data.data_objects import BotSettings if Utility.check_empty_string(name): raise AppException("Bot Name cannot be empty or blank spaces") if Utility.check_empty_string(user): raise AppException("user cannot be empty or blank spaces") Utility.is_exist( Bot, exp_message="Bot already exists!", name__iexact=name, account=account, status=True, ) bot = Bot(name=name, account=account, user=user).save().to_mongo().to_dict() bot_id = bot['_id'].__str__() if not is_new_account: AccountProcessor.__allow_access_to_bot( bot_id, user, user, account, ACCESS_ROLES.OWNER.value, ACTIVITY_STATUS.ACTIVE.value) BotSettings(bot=bot_id, user=user).save() processor = MongoProcessor() config = processor.load_config(bot_id) processor.add_or_overwrite_config(config, bot_id, user) processor.add_default_fallback_data(bot_id, user, True, True) processor.add_system_required_slots(bot_id, user) return bot
def validate_pattern(cls, f, values, **kwargs): from kairon.shared.utils import Utility import re if Utility.check_empty_string(f): raise ValueError("Regex pattern cannot be empty or a blank space") try: re.compile(f) except Exception: raise AppException("invalid regular expression") return f