def get_integration_user(bot: str, account: int): """ creates integration user if it does not exist :param bot: bot id :param account: account id :return: dict """ if not Utility.is_exist(User, raise_error=False, bot=bot, is_integration_user=True, status=True): email = bot + "@integration.com" password = Utility.generate_password() return AccountProcessor.add_user( email=email, password=password, first_name=bot, last_name=bot, account=account, bot=bot, user="******", is_integration_user=True, ) else: return (User.objects(bot=bot).get( is_integration_user=True).to_mongo().to_dict())
def train_model_for_bot(bot: str): """ loads bot data from mongo into individual files for training :param bot: bot id :return: model path """ processor = MongoProcessor() nlu = processor.load_nlu(bot) if not nlu.training_examples: raise AppException("Training data does not exists!") domain = processor.load_domain(bot) stories = processor.load_stories(bot) config = processor.load_config(bot) rules = processor.get_rules_for_training(bot) directory = Utility.write_training_data(nlu, domain, config, stories, rules) output = os.path.join(DEFAULT_MODELS_PATH, bot) model = train( domain=os.path.join(directory, DEFAULT_DOMAIN_PATH), config=os.path.join(directory, DEFAULT_CONFIG_PATH), training_files=os.path.join(directory, DEFAULT_DATA_PATH), output=output, ) Utility.delete_directory(directory) del processor del nlu del domain del stories del config return model
async def send_confirmation_link(mail: str): """ Sends a link to the user's mail id for account verification :param mail: the mail id of the user :return: mail id, mail subject and mail body """ if AccountProcessor.EMAIL_ENABLED: if isinstance(mail_check(mail), ValidationFailure): raise AppException("Please enter valid email id") Utility.is_exist(UserEmailConfirmation, exp_message="Email already confirmed!", email__iexact=mail.strip()) if not Utility.is_exist( User, email__iexact=mail.strip(), raise_error=False): raise AppException( "Error! There is no user with the following mail id") token = Utility.generate_token(mail) link = Utility.email_conf["app"]["url"] + '/verify/' + token body = Utility.email_conf['email']['templates'][ 'confirmation_body'] + link subject = Utility.email_conf['email']['templates'][ 'confirmation_subject'] return mail, subject, body else: raise AppException("Error! Email verification is not enabled")
async def from_training_files(cls, training_data_paths: str, domain_path: str, config_path: str, root_dir): """ Create validator from training files. @param training_data_paths: nlu.yml file path. @param domain_path: domain.yml file path. @param config_path: config.yml file path. @param root_dir: training data root directory. @return: """ if not (os.path.exists(training_data_paths) and os.path.exists(domain_path) and os.path.exists(config_path)): raise AppException("Some training files are absent!") try: file_importer = RasaFileImporter( domain_path=domain_path, training_data_paths=training_data_paths, config_file=config_path, ) cls.http_actions = Utility.read_yaml(os.path.join(root_dir, 'http_action.yml')) return await TrainingDataValidator.from_importer(file_importer) except YamlValidationException as e: exc = Utility.replace_file_name(str(e), root_dir) raise AppException(exc) except YamlSyntaxException as e: exc = Utility.replace_file_name(str(e), root_dir) raise AppException(exc) except Exception as e: raise AppException(e)
def setup(self): os.environ["system_file"] = "./tests/testing_data/system.yaml" Utility.load_evironment() db_url = Utility.environment['database']["url"] pytest.db_url = db_url connect(host=db_url)
def init_connection(self): os.environ["system_file"] = "./tests/testing_data/system.yaml" Utility.load_evironment() connect(host=Utility.environment["database"]['url']) pytest.bot = 'test' yield None shutil.rmtree(os.path.join('training_data', pytest.bot))
def validate(self, clean=True): Utility.validate_document_list(self.events) if Utility.check_empty_string(self.block_name): raise ValidationError( "Story path name cannot be empty or blank spaces") elif not self.events: raise ValidationError("Stories cannot be empty")
def add_account(name: str, user: str): """ adds a new account :param name: account name :param user: user id :return: account id """ if Utility.check_empty_string(name): raise AppException("Account Name cannot be empty or blank spaces") Utility.is_exist( Account, exp_message="Account name already exists!", name__iexact=name, status=True, ) license = { "bots": 2, "intents": 10, "examples": 50, "training": 3, "augmentation": 5 } return Account(name=name.strip(), user=user, license=license).save().to_mongo().to_dict()
def main(): parser = create_arg_parser() arguments = parser.parse_args() Utility.load_evironment() connect(host=Utility.environment['database']['url']) logger.info(arguments.bot) logger.info(arguments.user) start_training(arguments.bot, arguments.user, arguments.token)
def validate(self, clean=True): if not self.title or not self.payload: raise ValidationError("title and payload must be present!") elif Utility.check_empty_string( self.title) or Utility.check_empty_string( self.payload.strip()): raise ValidationError( "Response title and payload cannot be empty or blank spaces")
def validate_password(cls, v, values, **kwargs): from kairon.utils import Utility try: Utility.valid_password(v.get_secret_value()) except AppException as e: raise ValueError(str(e)) return v
async def train( background_tasks: BackgroundTasks, current_user: User = Depends(auth.get_current_user_and_bot), ): """ Trains the chatbot """ Utility.train_model(background_tasks, current_user.get_bot(), current_user.get_user(), current_user.email, 'train') return {"message": "Model training started."}
def validate(self, clean=True): if (Utility.check_empty_string(self.email) or Utility.check_empty_string(self.first_name) or Utility.check_empty_string(self.last_name) or Utility.check_empty_string(self.password)): raise ValidationError( "Email, FirstName, LastName and password cannot be empty or blank space" ) elif isinstance(email(self.email), ValidationFailure): raise ValidationError("Please enter valid email address")
def user_retention(bot: Text, month: int = 1): """ Computes the user retention percentage of the bot :param bot: bot id :param month: default is current month and max is last 6 months :return: user retention percentage """ client, database, collection, message = ChatHistory.get_mongo_connection(bot) with client as client: db = client.get_database(database) conversations = db.get_collection(collection) total = [] repeating_users = [] try: total = list( conversations.aggregate([{"$match": {"latest_event_time": { "$gte": Utility.get_timestamp_previous_month(month)}}}, {"$group": {"_id": None, "count": {"$sum": 1}}}, {"$project": {"_id": 0, "count": 1}} ])) except Exception as e: message = str(e) try: repeating_users = list( conversations.aggregate([{"$unwind": {"path": "$events", "includeArrayIndex": "arrayIndex"}}, {"$match": {"events.name": {"$regex": ".*session_start*.", "$options": "$i"}}}, {"$group": {"_id": '$sender_id', "count": {"$sum": 1}, "latest_event_time": {"$first": "$latest_event_time"}}}, {"$match": {"count": {"$gte": 2}}}, {"$match": {"latest_event_time": { "$gte": Utility.get_timestamp_previous_month(month)}}}, {"$group": {"_id": None, "count": {"$sum": 1}}}, {"$project": {"_id": 0, "count": 1}} ])) except Exception as e: message = str(e) if not total: total_count = 1 else: total_count = total[0]['count'] if total[0]['count'] else 1 if not repeating_users: repeat_count = 0 else: repeat_count = repeating_users[0]['count'] if repeating_users[0]['count'] else 0 return ( {"user_retention": 100*(repeat_count/total_count)}, message )
def validate(self, clean=True): if Utility.check_empty_string(self.name) or Utility.check_empty_string( self.pattern): raise ValidationError( "Regex name and pattern cannot be empty or blank spaces") else: try: re.compile(self.pattern) except AppException as e: raise AppException("invalid regular expression " + self.pattern)
def test_get_action_url(self, monkeypatch): actual = Utility.get_action_url({}) assert actual.url == "http://localhost:5055/webhook" actual = Utility.get_action_url( {"action_endpoint": { "url": "http://action-server:5055/webhook" }}) assert actual.url == "http://action-server:5055/webhook" monkeypatch.setitem(Utility.environment['action'], "url", None) actual = Utility.get_action_url({}) assert actual is None
def check(cls, values): from kairon.utils import Utility if Utility.check_empty_string(values.get('key')): raise ValueError("key cannot be empty") if values.get('parameter_type' ) == ParameterChoice.slot and Utility.check_empty_string( values.get('value')): raise ValueError("Provide name of the slot as value") return values
def successful_conversations(bot: Text, month: int = 1): """ Counts the number of successful conversations of the bot :param bot: bot id :param month: default is current month and max is last 6 months :return: number of successful conversations """ fallback_action, nlu_fallback_action = Utility.load_fallback_actions(bot) client, database, collection, message = ChatHistory.get_mongo_connection(bot) with client as client: db = client.get_database(database) conversations = db.get_collection(collection) total = [] fallback_count = [] try: total = list( conversations.aggregate([{"$match": {"latest_event_time": { "$gte": Utility.get_timestamp_previous_month(month)}}}, {"$group": {"_id": None, "count": {"$sum": 1}}}, {"$project": {"_id": 0, "count": 1}} ])) except Exception as e: message = str(e) try: fallback_count = list( conversations.aggregate([ {"$unwind": {"path": "$events", "includeArrayIndex": "arrayIndex"}}, {"$match": {"events.timestamp": {"$gte": Utility.get_timestamp_previous_month(month)}}}, {"$match": {'$or': [{"events.name": fallback_action}, {"events.name": nlu_fallback_action}]}}, {"$group": {"_id": "$sender_id"}}, {"$group": {"_id": None, "count": {"$sum": 1}}}, {"$project": {"_id": 0, "count": 1}} ])) except Exception as e: message = str(e) if not total: total_count = 0 else: total_count = total[0]['count'] if total[0]['count'] else 0 if not fallback_count: fallbacks_count = 0 else: fallbacks_count = fallback_count[0]['count'] if fallback_count[0]['count'] else 0 return ( {"successful_conversations": total_count-fallbacks_count}, message )
def main(): parser = create_arg_parser() arguments = parser.parse_args() Utility.load_evironment() connect(host=Utility.environment['database']['url']) logger.info(arguments.bot) logger.info(arguments.user) logger.info(arguments.token) logger.debug("-t: " + arguments.train) if arguments.train.lower() == '--train' or arguments.train.lower() == '-t': start_training(arguments.bot, arguments.user, arguments.token)
def flatten_conversations(bot: Text, month: int = 3): """ Retrieves the flattened conversation data of the bot :param bot: bot id :param month: default is 3 months :return: dictionary of the bot users and their conversation data """ client, database, collection, message = ChatHistory.get_mongo_connection(bot) with client as client: db = client.get_database(database) conversations = db.get_collection(collection) user_data = [] try: user_data = list( conversations.aggregate( [{"$match": {"latest_event_time": {"$gte": Utility.get_timestamp_previous_month(month)}}}, {"$unwind": {"path": "$events", "includeArrayIndex": "arrayIndex"}}, {"$match": {"$or": [{"events.event": {"$in": ['bot', 'user']}}, {"$and": [{"events.event": "action"}, {"events.name": {"$nin": ['action_listen', 'action_session_start']}}]}]}}, {"$match": {"events.timestamp": {"$gte": Utility.get_timestamp_previous_month(month)}}}, {"$group": {"_id": "$sender_id", "events": {"$push": "$events"}, "allevents": {"$push": "$events"}}}, {"$unwind": "$events"}, {"$match": {"events.event": 'user'}}, {"$group": {"_id": "$_id", "events": {"$push": "$events"}, "user_array": {"$push": "$events"}, "all_events": {"$first": "$allevents"}}}, {"$unwind": "$events"}, {"$project": {"user_input": "$events.text", "intent": "$events.parse_data.intent.name", "message_id": "$events.message_id", "timestamp": "$events.timestamp", "confidence": "$events.parse_data.intent.confidence", "action_bot_array": { "$cond": [{"$gte": [{"$indexOfArray": ["$all_events", {"$arrayElemAt": ["$user_array", {"$add": [{"$indexOfArray": ["$user_array","$events"]}, 1]}]}]}, {"$indexOfArray": ["$all_events", "$events"]}]}, {"$slice": ["$all_events", {"$add": [{"$indexOfArray":["$all_events", "$events"]}, 1]}, {"$subtract": [{"$subtract": [{"$indexOfArray": ["$all_events", {"$arrayElemAt": ["$user_array", {"$add": [{"$indexOfArray": ["$user_array", "$events"]}, 1]}]}]}, {"$indexOfArray": ["$all_events", "$events"]}]}, 1]}]}, {"$slice": ["$all_events", {"$add": [{"$indexOfArray": ["$all_events", "$events"]}, 1]}, 100]}]}}}, {"$project": {"user_input": 1, "intent": 1, "confidence": 1, "action": "$action_bot_array.name", "message_id": 1, "timestamp": 1, "bot_response": "$action_bot_array.text"}} ])) except Exception as e: message = str(e) return ( {"conversation_data": user_data}, message )
def validate(self, clean=True): if (Utility.check_empty_string(self.type) or Utility.check_empty_string(self.url) or Utility.check_empty_string(self.db)): raise ValidationError( "Type, Url and DB cannot be blank or empty spaces") else: if self.type == "mongo": try: parse_uri(self.url) except InvalidURI: raise AppException("Invalid tracker url!")
def fallback_count_range(bot: Text, month: int = 6): """ Computes the trend for fallback counts :param bot: bot id :param month: default is 6 months :return: dictionary of fallback counts for the previous months """ fallback_action, nlu_fallback_action = Utility.load_fallback_actions(bot) client, database, collection, message = ChatHistory.get_mongo_connection(bot) with client as client: db = client.get_database(database) conversations = db.get_collection(collection) fallback_counts = [] try: fallback_counts = list( conversations.aggregate([{"$unwind": {"path": "$events"}}, {"$match": {"events.event": "action", "events.timestamp": { "$gte": Utility.get_timestamp_previous_month( month)}}}, {"$match": {'$or': [{"events.name": fallback_action}, {"events.name": nlu_fallback_action}]}}, {"$addFields": {"month": { "$month": {"$toDate": {"$multiply": ["$events.timestamp", 1000]}}}}}, {"$group": {"_id": "$month", "count": {"$sum": 1}}}, {"$project": {"_id": 1, "count": 1}} ])) action_counts = list( conversations.aggregate([{"$unwind": {"path": "$events"}}, {"$match": {"$and": [{"events.event": "action"}, {"events.name": {"$nin": ['action_listen', 'action_session_start']}}]}}, {"$match": {"events.timestamp": { "$gte": Utility.get_timestamp_previous_month(month)}}}, {"$addFields": {"month": { "$month": {"$toDate": {"$multiply": ["$events.timestamp", 1000]}}}}}, {"$group": {"_id": "$month", "total_count": {"$sum": 1}}}, {"$project": {"_id": 1, "total_count": 1}} ])) except Exception as e: message = str(e) action_count = {d['_id']: d['total_count'] for d in action_counts} fallback_count = {d['_id']: d['count'] for d in fallback_counts} final_trend = {k: [fallback_count.get(k), action_count.get(k)] for k in list(fallback_count.keys())} return ( {"fallback_counts": final_trend}, message )
def start_training(bot: str, user: str, token: str = None, reload=True): """ prevents training of the bot, if the training session is in progress otherwise start training :param reload: whether to reload model in the cache :param bot: bot id :param token: JWT token for remote model reload :param user: user id :return: model path """ exception = None model_file = None training_status = None if Utility.environment.get('model') and Utility.environment['model'][ 'train'].get('event_url'): Utility.train_model_event(bot, user, token) else: try: apm_client = Utility.initiate_apm_client() if apm_client: elasticapm.instrument() apm_client.begin_transaction(transaction_type="script") model_file = train_model_for_bot(bot) training_status = MODEL_TRAINING_STATUS.DONE.value agent_url = Utility.environment['model']['train'].get('agent_url') if agent_url: if token: Utility.http_request( 'get', urljoin(agent_url, "/api/bot/model/reload"), token, user) else: if reload: AgentProcessor.reload(bot) except Exception as e: logging.exception(e) training_status = MODEL_TRAINING_STATUS.FAIL.value exception = str(e) finally: if apm_client: apm_client.end_transaction(name=__name__, result="success") ModelProcessor.set_training_status( bot=bot, user=user, status=training_status, model_path=model_file, exception=exception, ) return model_file
def validate_rasa_config(config: Dict): """ validates bot config.yml content for invalid entries :param config: configuration :return: None """ config_errors = [] from rasa.nlu.registry import registered_components as nlu_components if config.get('pipeline'): for item in config['pipeline']: component_cfg = item['name'] if not (component_cfg in nlu_components or component_cfg in ["custom.ner.SpacyPatternNER", "custom.fallback.FallbackIntentFilter"]): config_errors.append("Invalid component " + component_cfg) else: config_errors.append("You didn't define any pipeline") if config.get('policies'): core_policies = Utility.get_rasa_core_policies() for policy in config['policies']: if policy['name'] not in core_policies: config_errors.append("Invalid policy " + policy['name']) else: config_errors.append("You didn't define any policies") return config_errors
def validate(self, clean=True): if Utility.check_empty_string(self.name): raise ValidationError( "Action name cannot be empty or blank spaces") if self.name.startswith('utter_'): raise ValidationError("Action name cannot start with utter_")
def test_overwrite_password_with_valid_entries(self, monkeypatch): monkeypatch.setattr(Utility, 'trigger_smtp', self.mock_smtp) token = Utility.generate_token('*****@*****.**') loop = asyncio.new_event_loop() loop.run_until_complete( AccountProcessor.overwrite_password(token, "Welcome@3")) assert True
def __authenticate_user(self, username: str, password: str): user = AccountProcessor.get_user_details(username) if not user: return False if not Utility.verify_password(password, user["password"]): return False return user
def fetch_chat_users(bot: Text, month: int = 1): """ fetches user list who has conversation with the agent :param month: default is current month and max is last 6 months :param bot: bot id :return: list of user id """ client, db_name, collection, message = ChatHistory.get_mongo_connection( bot) db = client.get_database(db_name) conversations = db.get_collection(collection) users = [] try: values = conversations.find( { "events.timestamp": { "$gte": Utility.get_timestamp_previous_month(month) } }, { "_id": 0, "sender_id": 1 }) users = [sender["sender_id"] for sender in values] except Exception as e: raise AppException(e) finally: client.close() return users, message
def validate(self, clean=True): if self.entities: for ent in self.entities: ent.validate() extracted_ent = self.text[ent.start:ent.end] if extracted_ent != ent.value: raise ValidationError( "Invalid entity: " + ent.entity + ", value: " + ent.value + " does not match with the position in the text " + extracted_ent) elif Utility.check_empty_string( self.text) or Utility.check_empty_string(self.intent): raise ValidationError( "Training Example name and text cannot be empty or blank spaces" )
def validate(self, clean=True): if Utility.check_empty_string(self.name): raise ValidationError("Form name cannot be empty or blank spaces") try: _validate_slot_mappings({self.name: self.mapping}) except Exception as e: raise ValidationError(e)