def __check_response_existence(self,
response: Dict,
bot: Text,
exp_message: Text = None,
raise_error=True):
saved_responses = list(
Responses.objects(bot=bot, status=True).aggregate([{
"$group": {
"_id": "$name",
"texts": {
"$push": "$text"
},
"customs": {
"$push": "$custom"
},
}
}]))
saved_items = list(
itertools.chain.from_iterable([
items["texts"] + items["customs"] for items in saved_responses
]))
if response in saved_items:
if raise_error:
if Utility.check_empty_string(exp_message):
raise AppException("Exception message cannot be empty")
raise AppException(exp_message)
else:
return True
else:
if not raise_error:
return False
def __check_event_existence(self,
events: List[Dict],
bot: Text,
exp_message: Text = None,
raise_error=True):
saved_events = list(
Stories.objects(bot=bot, status=True).aggregate([{
"$group": {
"_id": "$name",
"events": {
"$push": "$events"
}
}
}]))
saved_items = list(
itertools.chain.from_iterable(
[items["events"] for items in saved_events]))
if events in saved_items:
if raise_error:
if Utility.check_empty_string(exp_message):
raise AppException("Exception message cannot be empty")
raise AppException(exp_message)
else:
return True
else:
if not raise_error:
return False
async def save_from_path(self,
path: Text,
bot: Text,
overwrite: bool = True,
user="******"):
try:
story_files, nlu_files = get_core_nlu_files(
os.path.join(path, DEFAULT_DATA_PATH))
nlu = utils.training_data_from_paths(nlu_files, "en")
domain = Domain.from_file(os.path.join(path, DEFAULT_DOMAIN_PATH))
domain.check_missing_templates()
story_steps = await StoryFileReader.read_from_files(
story_files, domain)
config = read_config_file(os.path.join(path, DEFAULT_CONFIG_PATH))
self.save_domain(domain, bot, user)
self.save_stories(story_steps, bot, user)
self.save_nlu(nlu, bot, user)
self.save_config(config, bot, user)
except InvalidDomain as e:
logging.info(e)
raise AppException("""Failed to validate yaml file.
Please make sure the file is initial and all mandatory parameters are specified"""
)
except Exception as e:
logging.info(e)
raise AppException(e)
def deploy_model(endpoint: Dict, bot: Text):
if not endpoint or not endpoint.get("bot_endpoint"):
raise AppException(
"Please configure the bot endpoint for deployment!")
headers = {"Content-type": "application/json", "Accept": "text/plain"}
url = endpoint["bot_endpoint"].get("url")
if endpoint["bot_endpoint"].get(
"token_type") and endpoint["bot_endpoint"].get("token"):
headers["Authorization"] = (
endpoint["bot_endpoint"].get("token_type") + " " +
endpoint["bot_endpoint"].get("token"))
try:
response = requests.put(
url + "/model",
json={
"model_file":
Utility.get_latest_file(
os.path.join(DEFAULT_MODELS_PATH, bot))
},
headers=headers,
)
json_response = response.json()
if "message" in json_response:
result = json_response["message"]
elif "reason" in json_response:
result = json_response["reason"]
else:
result = json_response
except requests.exceptions.ConnectionError as e:
raise AppException("Host is not reachable")
return result
def remove_document(self, document: Document, id: Text, bot: Text,
user: Text):
try:
doc = document.objects(bot=bot).get(id=id)
doc.update(status=False, user=user)
except DoesNotExist as e:
logging.info(e)
raise AppException("Unable to remove document")
except Exception as e:
logging.info(e)
raise AppException("Unable to remove document")
def is_exist(
document: Document, query: Dict, exp_message: Text = None, raise_error=True,
):
doc = document.objects(status=True, __raw__=query)
if doc.__len__():
if raise_error:
if Utility.check_empty_string(exp_message):
raise AppException("Exception message cannot be empty")
raise AppException(exp_message)
else:
return True
else:
if not raise_error:
return False
def start_training(bot: str, user: str):
""" Prevents training of the bot if the training session is in progress otherwise start training """
exception = None
model_file = None
training_status = None
ModelProcessor.set_training_status(
bot=bot,
user=user,
status=MODEL_TRAINING_STATUS.INPROGRESS.value,
)
try:
model_file = train_model_for_bot(bot)
training_status = MODEL_TRAINING_STATUS.DONE.value
except Exception as e:
logging.exception(e)
training_status = MODEL_TRAINING_STATUS.FAIL.value
exception = str(e)
raise AppException(exception)
finally:
ModelProcessor.set_training_status(
bot=bot,
user=user,
status=training_status,
model_path=model_file,
exception=exception,
)
AgentProcessor.reload(bot)
return model_file
def train_model_for_bot(bot: str):
""" Trains the rasa model, using the data that is loaded onto
Mongo, through the bot files """
processor = MongoProcessor()
nlu = processor.load_nlu(bot)
if not nlu.training_examples:
raise AppException("Training data does not exists!")
domain = processor.load_domain(bot)
stories = processor.load_stories(bot)
config = processor.load_config(bot)
directory = Utility.save_files(
nlu.nlu_as_markdown().encode(),
domain.as_yaml().encode(),
stories.as_story_string().encode(),
yaml.dump(config).encode(),
)
output = os.path.join(DEFAULT_MODELS_PATH, bot)
model = train(domain=os.path.join(directory,DEFAULT_DOMAIN_PATH),
config=os.path.join(directory,DEFAULT_CONFIG_PATH),
training_files=os.path.join(directory,DEFAULT_DATA_PATH),
output=output)
Utility.delete_directory(directory)
return model
async def get_nlu_data(self,
language: Optional[Text] = "en") -> TrainingData:
""" Loads the data from the nlu file of the bot """
training_data = self.processor.load_nlu(self.bot)
if not training_data.training_examples:
raise AppException("Training data does not exists!")
return training_data
def is_exist(document: Document,
exp_message: Text = None,
raise_error=True,
*args,
**kwargs):
doc = document.objects(**kwargs)
if doc.__len__():
if raise_error:
if Utility.check_empty_string(exp_message):
raise AppException("Exception message cannot be empty")
raise AppException(exp_message)
else:
return True
else:
if not raise_error:
return False
def is_training_inprogress(bot: Text, raise_exception=True):
if ModelTraining.objects(
bot=bot,
status=MODEL_TRAINING_STATUS.INPROGRESS.value).count():
if raise_exception:
raise AppException("Previous model training in progress.")
else:
return True
else:
return False
def validate(self, clean=True):
if Utility.check_empty_string(self.name) or Utility.check_empty_string(
self.pattern):
raise ValidationError(
"Regex name and pattern cannot be empty or blank spaces")
else:
try:
re.compile(self.pattern)
except AppException as e:
raise AppException("invalid regular expression " +
self.pattern)
def __save_session_config(self, session_config: SessionConfig, bot: Text,
user: Text):
try:
if session_config:
try:
session = SessionConfigs.objects().get(bot=bot)
session.session_expiration_time = (
session_config.session_expiration_time)
session.carryOverSlots = True
session.user = user
except DoesNotExist:
session = self.__extract_session_config(
session_config, bot, user)
session.save()
except NotUniqueError as e:
logging.info(e)
raise AppException("Session Config already exists for the bot")
except Exception as e:
logging.info(e)
raise AppException("Internal Server Error")
def validate(self, clean=True):
if (Utility.check_empty_string(self.type)
or Utility.check_empty_string(self.url)
or Utility.check_empty_string(self.db)):
raise ValidationError(
"Type, Url and DB cannot be blank or empty spaces")
else:
if self.type == "mongo":
try:
parse_uri(self.url)
except InvalidURI:
raise AppException("Invalid tracker url!")
def reload(bot: Text):
try:
endpoint = AgentProcessor.mongo_processor.get_endpoints(
bot, raise_exception=False)
action_endpoint = (EndpointConfig(
url=endpoint["action_endpoint"]["url"]) if endpoint
and endpoint.get("action_endpoint") else None)
model_path = AgentProcessor.get_latest_model(bot)
agent = Agent.load(model_path, action_endpoint=action_endpoint)
InMemoryAgentCache.set(bot, agent)
except Exception as e:
logging.info(e)
raise AppException("Please train the bot first")
def get_endpoints(self, bot: Text, raise_exception=True):
try:
endpoint = Endpoints.objects().get(bot=bot).to_mongo().to_dict()
endpoint.pop("bot")
endpoint.pop("user")
endpoint.pop("timestamp")
endpoint["_id"] = endpoint["_id"].__str__()
return endpoint
except DoesNotExist as e:
logging.info(e)
if raise_exception:
raise AppException("Endpoint Configuration does not exists!")
else:
return {}
async def download_model(
background_tasks: BackgroundTasks,
current_user: User = Depends(auth.get_current_user),
):
"""Download latest trained model file"""
try:
model_path = AgentProcessor.get_latest_model(current_user.get_bot())
response = FileResponse(model_path,
filename=os.path.basename(model_path),
background=background_tasks)
response.headers[
"Content-Disposition"] = "attachment; filename=" + os.path.basename(
model_path)
return response
except Exception as e:
raise AppException(str(e))
def is_daily_training_limit_exceeded(bot: Text, raise_exception=True):
today = datetime.today()
today_start = today.replace(hour=0, minute=0, second=0)
today_end = today.replace(hour=23, minute=59, second=59)
doc_count = ModelTraining.objects(
bot=bot,
start_timestamp__gte=today_start,
start_timestamp__lte=today_end,
end_timestamp__gte=today_start,
end_timestamp__lte=today_end,
).count()
if doc_count >= Utility.environment["MODEL_TRAINING_LIMIT_PER_DAY"]:
if raise_exception:
raise AppException("Daily model training limit exceeded.")
else:
return True
else:
return False
def valid_password(password: Text):
"""Used to validate password against the password policy"""
results = Utility.password_policy.test(password)
if results:
response = []
for result in results:
if isinstance(result, Length):
response.append("Password length must be " +
str(result.length))
elif isinstance(result, Special):
response.append("Missing " + str(result.count) +
" special letter")
elif isinstance(result, Uppercase):
response.append("Missing " + str(result.count) +
" uppercase letter")
elif isinstance(result, Numbers):
response.append("Missing " + str(result.count) + "number")
if response:
raise AppException("\n".join(response))
def add_endpoints(self, endpoint_config: Dict, bot: Text, user: Text):
try:
endpoint = Endpoints.objects().get(bot=bot)
except DoesNotExist:
if Endpoints.objects(bot=bot):
raise AppException("Endpoint Configuration already exists!")
endpoint = Endpoints()
endpoint.bot_endpoint = (EndPointBot(
**endpoint_config.get("bot_endpoint")) if
endpoint_config.get("bot_endpoint") else None)
endpoint.action_endpoint = (
EndPointAction(**endpoint_config.get("action_endpoint"))
if endpoint_config.get("action_endpoint") else None)
endpoint.tracker_endpoint = (
EndPointTracker(**endpoint_config.get("tracker_endpoint"))
if endpoint_config.get("tracker_endpoint") else None)
endpoint.bot = bot
endpoint.user = user
return endpoint.save().to_mongo().to_dict()["_id"].__str__()
def add_session_config(
self,
bot: Text,
user: Text,
id: Text = None,
sesssionExpirationTime: int = 60,
carryOverSlots: bool = True,
):
if not Utility.check_empty_string(id):
session_config = SessionConfigs.objects().get(id=id)
session_config.sesssionExpirationTime = sesssionExpirationTime
session_config.carryOverSlots = carryOverSlots
else:
if SessionConfigs.objects(bot=bot):
raise AppException("Session config already exists!")
session_config = SessionConfigs(
sesssionExpirationTime=sesssionExpirationTime,
carryOverSlots=carryOverSlots,
bot=bot,
user=user,
)
return session_config.save().to_mongo().to_dict()["_id"].__str__()
def get_latest_file(folder):
if not os.path.exists(folder):
raise AppException("Folder does not exists!")
return max(glob.iglob(folder + "/*"), key=os.path.getctime)
def validate(self, clean=True):
if isinstance(url(self.url), ValidationFailure):
raise AppException("Invalid Bot server url")
def _inprogress_execption_response(*args, **kwargs):
raise AppException("Previous model training in progress.")