def create(bot: str, use_test_stories: bool = False):
from kairon import Utility
from itertools import chain
from rasa.shared.nlu.training_data.training_data import TrainingData
bot_home = os.path.join('testing_data', bot)
Utility.make_dirs(bot_home)
processor = MongoProcessor()
intents_and_training_examples = processor.get_intents_and_training_examples(bot)
aug_training_examples = map(lambda training_data: TestDataGenerator.__prepare_nlu(training_data[0], training_data[1]), intents_and_training_examples.items())
messages = list(chain.from_iterable(aug_training_examples))
nlu_data = TrainingData(training_examples=messages)
stories = processor.load_stories(bot)
rules = processor.get_rules_for_training(bot)
stories = stories.merge(rules)
if stories.is_empty() or nlu_data.is_empty():
raise AppException('Not enough training data exists. Please add some training data.')
nlu_as_str = nlu_data.nlu_as_yaml().encode()
nlu_path = os.path.join(bot_home, "nlu.yml")
Utility.write_to_file(nlu_path, nlu_as_str)
if use_test_stories:
stories_path = os.path.join(bot_home, "test_stories.yml")
else:
stories_path = os.path.join(bot_home, "stories.yml")
YAMLStoryWriter().dump(stories_path, stories.story_steps, is_test_story=use_test_stories)
return nlu_path, stories_path
def push_notification(sender, document, **kwargs):
from kairon import Utility
from kairon.shared.data.data_objects import ModelTraining, ModelDeployment, TrainingDataGenerator
from kairon.shared.importer.data_objects import ValidationLogs
from kairon.shared.test.data_objects import ModelTestingLogs
is_enabled = Utility.environment['notifications']['enable']
message_type_events = [ModelTraining, ModelTestingLogs, ModelDeployment, TrainingDataGenerator, ValidationLogs]
message_type_events = {event.__name__ for event in message_type_events}
if is_enabled:
try:
metadata = document.to_mongo().to_dict()
for key in metadata:
if isinstance(metadata[key], ObjectId):
metadata[key] = metadata[key].__str__()
elif isinstance(metadata[key], datetime.datetime):
metadata[key] = metadata[key].__str__()
if sender.__name__ in message_type_events:
event_type = 'message'
elif kwargs.get('created'):
event_type = 'create'
else:
event_type = 'update'
if metadata.get('status') is False:
event_type = 'delete'
Utility.push_notification(document.bot, event_type, sender.__name__, metadata)
except Exception as e:
logger.exception(e)
def run_tests_on_model(bot: str, run_e2e: bool = False):
"""
Runs tests on a trained model.
Args:
bot: bot id for which test is run.
run_e2e: if True, test is initiated on test stories and nlu data.
Returns: dictionary with evaluation results
"""
from kairon import Utility
from rasa.utils.common import run_in_loop
bot_home = os.path.join('testing_data', bot)
logger.info(f"model test data path: {bot_home}")
try:
model_path = Utility.get_latest_model(bot)
nlu_path, stories_path = TestDataGenerator.create(bot, run_e2e)
stories_results = run_in_loop(ModelTester.run_test_on_stories(stories_path, model_path, run_e2e))
nlu_results = ModelTester.run_test_on_nlu(nlu_path, model_path)
return nlu_results, stories_results
except Exception as e:
raise AppException(f'Model testing failed: {e}')
finally:
if os.path.exists(bot_home):
Utility.delete_directory(bot_home)
def init_connection(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_environment()
connect(**Utility.mongoengine_connection(
Utility.environment['database']["url"]))
tmp_dir = tempfile.mkdtemp()
pytest.tmp_dir = tmp_dir
from rasa import train
# model without entities
train_result = train(
domain='tests/testing_data/model_tester/domain.yml',
config='tests/testing_data/model_tester/config.yml',
training_files=[
'tests/testing_data/model_tester/nlu_with_entities/nlu.yml',
'tests/testing_data/model_tester/training_stories_success/stories.yml'
],
output='tests/testing_data/model_tester/models',
core_additional_arguments={"augmentation_factor": 100},
force_training=True)
pytest.model_path = train_result.model
responses.add(
'POST',
Utility.environment["augmentation"]["paraphrase_url"],
json={'data': {
'paraphrases': ['common training example']
}})
responses.start()
yield None
responses.stop()
shutil.rmtree(pytest.tmp_dir)
shutil.rmtree('tests/testing_data/model_tester/models')
async def test_trigger_data_importer_validate_existing_data(self, monkeypatch, get_training_data):
bot = 'test_trigger_data_importer_domain_only'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(uuid.uuid4()))
Utility.make_dirs(test_data_path)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot, user)
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 2
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert [action.get('data') for action in logs[0].get('actions') if action.get('type') == 'http_actions'] == [[]]
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
mongo_processor = MongoProcessor()
assert len(mongo_processor.fetch_stories(bot)) == 2
assert len(list(mongo_processor.fetch_training_examples(bot))) == 7
assert len(list(mongo_processor.fetch_responses(bot))) == 3
assert len(mongo_processor.fetch_actions(bot)) == 2
assert len(mongo_processor.fetch_rule_block_names(bot)) == 3
def init(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_evironment()
connect(host=Utility.environment["database"]['url'])
tmp_dir = tempfile.mkdtemp()
pytest.tmp_dir = tmp_dir
yield None
shutil.rmtree(tmp_dir)
def init(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_environment()
connect(**Utility.mongoengine_connection(Utility.environment['database']["url"]))
tmp_dir = tempfile.mkdtemp()
pytest.tmp_dir = tmp_dir
yield None
shutil.rmtree(tmp_dir)
def init_connection(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_evironment()
bot = 'agent_testing_user'
pytest.bot = bot
model_path = os.path.join('models', bot)
os.mkdir(model_path)
shutil.copy('tests/testing_data/model/20210512-172208.tar.gz',
model_path)
yield None
shutil.rmtree(model_path)
def __init__(self):
"""Initializes sso client if enabled else throws exception."""
Utility.check_is_enabled(SSO_TYPES.FACEBOOK.value)
self.sso_client = FacebookSSO(
Utility.environment["sso"][SSO_TYPES.FACEBOOK.value]["client_id"],
Utility.environment["sso"][
SSO_TYPES.FACEBOOK.value]["client_secret"],
urljoin(Utility.environment["sso"]["redirect_url"],
SSO_TYPES.FACEBOOK.value),
allow_insecure_http=False,
use_state=True)
async def test_trigger_data_importer_rules_only(self, monkeypatch,
get_training_data):
bot = 'test_trigger_data_importer_rules_only'
user = '******'
test_data_path = os.path.join(pytest.tmp_dir, str(datetime.utcnow()))
data_path = os.path.join(test_data_path, 'data')
Utility.make_dirs(data_path)
shutil.copy2('tests/testing_data/validator/valid/data/rules.yml',
data_path)
nlu, story_graph, domain, config, http_actions = await get_training_data(
'tests/testing_data/validator/valid')
mongo_processor = MongoProcessor()
mongo_processor.save_domain(domain, bot, user)
mongo_processor.save_nlu(nlu, bot, user)
config["bot"] = bot
config["user"] = user
config_obj = Configs._from_son(config)
config_obj.save()
mongo_processor.save_stories(story_graph.story_steps, bot, user)
mongo_processor.save_http_action(http_actions, bot, user)
def _path(*args, **kwargs):
return test_data_path
monkeypatch.setattr(Utility, "get_latest_file", _path)
DataImporterLogProcessor.add_log(bot, user, files_received=["rules"])
await EventsTrigger.trigger_data_importer(bot, user, True, False)
logs = list(DataImporterLogProcessor.get_logs(bot))
assert len(logs) == 1
assert not logs[0].get('intents').get('data')
assert not logs[0].get('stories').get('data')
assert not logs[0].get('utterances').get('data')
assert not logs[0].get('http_actions').get('data')
assert not logs[0].get('training_examples').get('data')
assert not logs[0].get('domain').get('data')
assert not logs[0].get('config').get('data')
assert not logs[0].get('exception')
assert logs[0]['is_data_uploaded']
assert logs[0]['start_timestamp']
assert logs[0]['end_timestamp']
assert logs[0]['status'] == 'Success'
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
assert len(mongo_processor.fetch_stories(bot)) == 2
assert len(list(mongo_processor.fetch_training_examples(bot))) == 7
assert len(list(mongo_processor.fetch_responses(bot))) == 2
assert len(mongo_processor.fetch_actions(bot)) == 2
assert len(mongo_processor.fetch_rule_block_names(bot)) == 3
async def test_trigger_model_testing_event_run_tests_on_model(self, load_data, create_model, monkeypatch):
import rasa.utils.common
bot = 'test_events_bot'
user = '******'
config_path = 'tests/testing_data/model_tester/config.yml'
domain_path = 'tests/testing_data/model_tester/domain.yml'
nlu_path = 'tests/testing_data/model_tester/nlu_success/nlu.yml'
stories_path = 'tests/testing_data/model_tester/training_stories_success/stories.yml'
await load_data(config_path, domain_path, nlu_path, stories_path, bot, user)
create_model(pytest.model_path, bot)
def _mock_stories_output(*args, **kwargs):
return {
"precision": 0.91,
"f1": 0.98,
"accuracy": 0.99,
"failed_stories": [],
}
monkeypatch.setattr(rasa.utils.common, 'run_in_loop', _mock_stories_output)
responses.add('POST',
Utility.environment["augmentation"]["paraphrase_url"],
json={'data': {'paraphrases': ['common training example']}})
responses.start()
EventsTrigger.trigger_model_testing(bot, user, False)
logs = list(ModelTestingLogProcessor.get_logs(bot))
assert len(logs) == 2
assert not logs[0].get('exception')
assert logs[0]['start_timestamp']
assert logs[0].get('data')
assert logs[0].get('end_timestamp')
assert not Utility.check_empty_string(logs[0].get('status'))
assert logs[0]['event_status'] == EVENT_STATUS.COMPLETED.value
assert not os.path.exists(os.path.join('./testing_data', bot))
def push_bulk_update_notification(sender, documents, **kwargs):
from kairon import Utility
is_enabled = Utility.environment['notifications']['enable']
if is_enabled:
try:
if isinstance(documents, QuerySet):
documents = list(documents)
if documents:
channel = kwargs['bot']
event_type = kwargs['event_type']
metadata = []
for doc in documents:
doc = doc.to_mongo().to_dict()
metadata.append({'_id': doc['_id'].__str__()})
Utility.push_notification(channel, event_type, sender.__name__, metadata)
except Exception as e:
logger.exception(e)
def init_connection(self):
from rasa import train
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_environment()
bot = 'agent_testing_user'
pytest.bot = bot
model_path = os.path.join('models', bot)
os.mkdir(model_path)
train(
domain='tests/testing_data/model_tester/domain.yml',
config='tests/testing_data/model_tester/config.yml',
training_files=[
'tests/testing_data/model_tester/nlu_with_entities/nlu.yml',
'tests/testing_data/model_tester/training_stories_success/stories.yml'
],
output=model_path,
core_additional_arguments={"augmentation_factor": 100},
force_training=True)
yield None
shutil.rmtree(model_path)
def init(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_environment()
connect(**Utility.mongoengine_connection(Utility.environment['database']["url"]))
tmp_dir = tempfile.mkdtemp()
pytest.tmp_dir = tmp_dir
from rasa import train
# model without entities
train_result = train(
domain='tests/testing_data/model_tester/domain.yml',
config='tests/testing_data/model_tester/config.yml',
training_files=['tests/testing_data/model_tester/nlu_with_entities/nlu.yml',
'tests/testing_data/model_tester/training_stories_success/stories.yml'],
output='tests/testing_data/model_tester/models',
core_additional_arguments={"augmentation_factor": 100},
force_training=True
)
pytest.model_path = train_result.model
yield None
shutil.rmtree(tmp_dir)
shutil.rmtree('models/test_events_bot')
async def _read_and_get_data(path: str):
domain_path = os.path.join(path, DEFAULT_DOMAIN_PATH)
training_data_path = os.path.join(path, DEFAULT_DATA_PATH)
config_path = os.path.join(path, DEFAULT_CONFIG_PATH)
http_actions_path = os.path.join(path, 'actions.yml')
importer = RasaFileImporter.load_from_config(config_path=config_path,
domain_path=domain_path,
training_data_paths=training_data_path)
domain = await importer.get_domain()
story_graph = await importer.get_stories()
config = await importer.get_config()
nlu = await importer.get_nlu_data(config.get('language'))
http_actions = Utility.read_yaml(http_actions_path)
return nlu, story_graph, domain, config, http_actions
def get_logs(bot: str,
log_type: str = None,
reference_id: str = None,
start_idx: int = 0,
page_size: int = 10):
"""
Get all logs for data importer event.
@param reference_id: test reference_id
@param bot: bot id.
@param log_type: log data type: 'stories', 'nlu'
@param start_idx: start index in list field
@param page_size: number of rows from start index
@return: list of logs.
"""
from kairon import Utility
if not (Utility.check_empty_string(log_type)
and Utility.check_empty_string(reference_id)):
logs = ModelTestingLogProcessor.get_by_id_and_type(
reference_id, bot, log_type, start_idx, page_size)
else:
logs = ModelTestingLogProcessor.get_all(bot)
return logs
def init(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_environment()
connect(**Utility.mongoengine_connection(
Utility.environment['database']["url"]))
def test_config_validation_invalid_config(self):
config = Utility.load_yaml(
"./tests/testing_data/yml_training_files/config.yml")
config.get('policies').append({'name': "XYZ"})
error = TrainingDataValidator.validate_rasa_config(config)
assert error[0] == "Invalid policy XYZ"
def test_config_validation(self):
config = Utility.load_yaml(
"./tests/testing_data/yml_training_files/config.yml")
TrainingDataValidator.validate_rasa_config(config)
async def sso_enabled_login_list():
"""
List social media logins enabled.
"""
return Response(data=Utility.get_enabled_sso())
def init(self):
os.environ["system_file"] = "./tests/testing_data/system.yaml"
Utility.load_evironment()
connect(host=Utility.environment["database"]['url'])
def test_get_agent_not_cached(self, mock_agent_properties):
AgentProcessor.cache_provider = Utility.create_cache()
assert AgentProcessor.get_agent(pytest.bot)
def run_test_on_nlu(nlu_path: str, model_path: str):
"""
Run tests on stories.
Args:
nlu_path: path where nlu test data is present as YAML.
model_path: Model path where model on which test has to be run is present.
Returns: dictionary with evaluation results
"""
from rasa.model import get_model
import rasa.shared.nlu.training_data.loading
from rasa.nlu.model import Interpreter
from rasa.nlu.test import (
remove_pretrained_extractors,
get_eval_data,
evaluate_intents,
evaluate_response_selections,
get_entity_extractors,
)
from kairon import Utility
unpacked_model = get_model(model_path)
nlu_model = os.path.join(unpacked_model, "nlu")
interpreter = Interpreter.load(nlu_model)
interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline)
test_data = rasa.shared.nlu.training_data.loading.load_data(
nlu_path, interpreter.model_metadata.language
)
result: Dict[Text, Optional[Dict]] = {
"intent_evaluation": None,
"entity_evaluation": None,
"response_selection_evaluation": None,
}
(intent_results, response_selection_results, entity_results) = get_eval_data(
interpreter, test_data
)
if intent_results:
successes = []
errors = []
result["intent_evaluation"] = evaluate_intents(intent_results, None, False, False, True)
if result["intent_evaluation"].get('predictions'):
del result["intent_evaluation"]['predictions']
del result["intent_evaluation"]['report']
for r in intent_results:
if r.intent_target == r.intent_prediction:
pass
# successes.append({
# "text": r.message,
# "intent": r.intent_target,
# "intent_prediction": {
# 'name': r.intent_prediction,
# "confidence": r.confidence,
# },
# })
else:
errors.append({
"text": r.message,
"intent": r.intent_target,
"intent_prediction": {
'name': r.intent_prediction,
"confidence": r.confidence,
},
})
result["intent_evaluation"]['total_count'] = len(successes) + len(errors)
result["intent_evaluation"]['success_count'] = len(successes)
result["intent_evaluation"]['failure_count'] = len(errors)
result["intent_evaluation"]['successes'] = successes
result["intent_evaluation"]['errors'] = errors
if response_selection_results:
successes = []
errors = []
result["response_selection_evaluation"] = evaluate_response_selections(
response_selection_results,
None,
False,
False,
True
)
if result["response_selection_evaluation"].get('predictions'):
del result["response_selection_evaluation"]['predictions']
del result["response_selection_evaluation"]['report']
for r in response_selection_results:
if r.intent_response_key_prediction == r.intent_response_key_target:
pass
# successes.append({
# "text": r.message,
# "intent_response_key_target": r.intent_response_key_target,
# "intent_response_key_prediction": {
# "name": r.intent_response_key_prediction,
# "confidence": r.confidence,
# },
# })
else:
if not Utility.check_empty_string(r.intent_response_key_target):
errors.append(
{
"text": r.message,
"intent_response_key_target": r.intent_response_key_target,
"intent_response_key_prediction": {
"name": r.intent_response_key_prediction,
"confidence": r.confidence,
},
}
)
result["response_selection_evaluation"]['total_count'] = len(successes) + len(errors)
result["response_selection_evaluation"]['success_count'] = len(successes)
result["response_selection_evaluation"]['failure_count'] = len(errors)
result["response_selection_evaluation"]['successes'] = successes
result["response_selection_evaluation"]['errors'] = errors
if any(entity_results):
extractors = get_entity_extractors(interpreter)
result["entity_evaluation"] = ModelTester.__evaluate_entities(entity_results, extractors)
return result
referrer=referrer,
permissions=permissions_value,
cache=cache_value,
content=content)
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
expose_headers=["content-disposition"],
)
app.add_middleware(GZipMiddleware)
Utility.load_environment()
apm_client = Utility.initiate_fastapi_apm_client()
if apm_client:
app.add_middleware(ElasticAPM, client=apm_client)
@app.middleware("http")
async def add_secure_headers(request: Request, call_next):
"""Add security headers."""
response = await call_next(request)
secure_headers.framework.fastapi(response)
return response
@app.middleware("http")
async def log_requests(request: Request, call_next):
