def __init__(self):
self.intent = None
self.user_id = "local_testing_1"
self.chat_id = "1"
self.message_id = 0
self.device_channel = "MP_SBOL_IOS"
self.device_channel_version = "9.1"
self.device_client_type = "RETAIL"
self.device_platform_name = "iPhone"
self.device_platform_version = "11.1"
self.csa_profile_id = 123
self.message_name = "MESSAGE_TO_SKILL"
self.token = "test_token"
self.project_name = "test_project_name"
self.user_channel = None
self.__message = {
"original_text": ""
}
self.new_session = False
from smart_kit.configs import get_app_config
self.config = get_app_config()
def create_app():
os.environ.setdefault("SMART_KIT_APP_CONFIG", "app_config")
app_config = get_app_config()
app_name = app_config.APP_NAME
configs_path, secret_path, settings_cls, references_path = \
app_config.CONFIGS_PATH, app_config.SECRET_PATH, app_config.SETTINGS, app_config.REFERENCES_PATH
resources_cls, model_cls, dialogue_manager_cls, main_loop_cls, user_cls, parametrizer_cls = \
app_config.RESOURCES, app_config.MODEL, app_config.DIALOGUE_MANAGER, app_config.MAIN_LOOP, \
app_config.USER, app_config.PARAMETRIZER
log("START SETTINGS CREATE", params={log_const.KEY_NAME: "timings"})
settings = settings_cls(config_path=configs_path,
secret_path=secret_path,
references_path=references_path)
log("FINISHED SETTINGS CREATE", params={log_const.KEY_NAME: "timings"})
source = settings.get_source()
log("START RESOURCES CREATE", params={log_const.KEY_NAME: "timings"})
resource = resources_cls(source, references_path, settings)
log("FINISHED RESOURCES CREATE", params={log_const.KEY_NAME: "timings"})
log("START MODEL CREATE", params={log_const.KEY_NAME: "timings"})
model = model_cls(resource,
dialogue_manager_cls,
settings,
app_name=app_name)
log("FINISHED MODEL CREATE", params={log_const.KEY_NAME: "timings"})
log("START MAIN_LOOP CREATE", params={log_const.KEY_NAME: "timings"})
loop = main_loop_cls(app_name, model, user_cls, parametrizer_cls, settings)
log("FINISHED MAIN_LOOP CREATE", params={log_const.KEY_NAME: "timings"})
return loop
def __load_everything(self):
nltk.download('punkt')
from smart_kit.configs import get_app_config
app_config = get_app_config()
text_normalizer_params = FileRepository(
f"{app_config.STATIC_PATH}/.text_normalizer_resources/static_workdata.json",
loader=ordered_json,
)
text_normalizer_params.load()
synonyms = FileRepository(
f"{app_config.STATIC_PATH}/.text_normalizer_resources/dict_synonyms.json",
loader=reverse_json_dict,
)
synonyms.load()
text_normalizer_params.data["synonyms"] = synonyms.data
text2num_dict = FileRepository(
f"{app_config.STATIC_PATH}/.text_normalizer_resources/text2num_dict.json",
loader=ordered_json,
)
text2num_dict.load()
text_normalizer_params.data["text2num_dict"] = text2num_dict.data
text_normalizer_params = text_normalizer_params.data or {}
self.convert_plan = text_normalizer_params["convert_plan"]
self.processor_pipeline = text_normalizer_params["processor_pipeline"]
word_false_stoppings = text_normalizer_params.get("word_false_stoppings", [])
words_without_splitting_point = text_normalizer_params.get("word_no_splitting_point", [])
synonyms = text_normalizer_params.get("synonyms", {})
text2num = text_normalizer_params.get("text2num_dict", {})
sberbank_phones = text_normalizer_params.get("sberbank_phones", [])
unicode_symbols = text_normalizer_params.get("unicode_symbols", {})
self.sentence_tokenizer = ru_sent_tokenize
self.word_tokenizer = NLTKWordTokenizer(word_false_stoppings, words_without_splitting_point)
skip_func = lambda x: x
self.converter_pipeline = {
'Объединение цифр после stt': NumbersUnionAfterSTT(text2num),
'Конверсия юникодовых символов': UnicodeSymbolsConverter(unicode_symbols),
'Цифры и буквы отдельно': unmerge_numbers_and_letters,
'Номера телефонов': NormalizePhoneNumbers(),
'Номера телефонов из голоса': NormalizePhoneNumbersVoice(),
'Номера карт': MergeCardNumbers(),
'Номера карт из голоса': MergeCardNumbersVoice(),
'Объединение сумм': merge_numbers,
'Символы валют': replace_currencies_symbols,
"Претокенизация математических операций": AdditionalMathSplitter()
}
self.tokens_processor = {
"Synonyms": ReplaceSynonyms(synonyms) if synonyms else skip_func,
"Text2Num": Text2Num(text2num, sberbank_phones) if text2num else skip_func,
"Currency": CurrencyTokensOneIterationMerger(),
"Grammemes": self.morph,
}
self.__ready_to_use = True
def __init__(self):
from smart_kit.configs import get_app_config
self.__message = {"original_text": ""}
self.character_appeal = "official"
self.character_gender = "male"
self.character_id = "sber"
self.character_name = "Сбер"
self.chat_id = "1"
self.config = get_app_config()
self.device_capabilities_misc = True
self.device_capabilities_screen = True
self.device_capabilities_speak = True
self.device_channel = "MP_SBOL_IOS"
self.device_channel_version = "9.1"
self.device_client_type = "RETAIL"
self.device_platform_name = "iPhone"
self.device_platform_type = "IOS"
self.device_platform_version = "11.1"
self.device_surface = "SBOL"
self.device_surface_version = "testSurfaceVersion"
self.intent = None
self.message_id = 0
self.message_name = "MESSAGE_TO_SKILL"
self.meta_time_timestamp = 1432233446145000
self.meta_time_timezone_id = "Europe/Moscow"
self.meta_time_timezone_offset_sec = 10800
self.new_session = False
self.project_name = "test_project_name"
self.user_channel = None
self.user_id = "local_testing_1"
def process_message(self, raw_message: str, headers: tuple = ()) -> typing.Tuple[typing.Any, list]:
from smart_kit.configs import get_app_config
masking_fields = self.settings["template_settings"].get("masking_fields")
message = SmartAppFromMessage(raw_message, headers=headers, masking_fields=masking_fields,
validators=get_app_config().FROM_MSG_VALIDATORS)
user = self.__user_cls(self.environment.user_id, message, self.user_data, self.settings,
self.app_model.scenario_descriptions,
self.__parametrizer_cls, load_error=False)
answers = self.app_model.answer(message, user)
return user, answers or []
def tokenized_elements_list_pymorphy(self):
if self._tokenized_elements_list_pymorphy is None:
from smart_kit.configs import get_app_config
app_config = get_app_config()
self._tokenized_elements_list_pymorphy = app_config.NORMALIZER(
self.original_text)["tokenized_elements_list"]
return self._tokenized_elements_list_pymorphy
def __init__(self,
items: Dict[str, Any],
id: Optional[str] = None) -> None:
super(EnvironmentRequirement, self).__init__(items, id)
self.values = items.get("values", [])
# Из конфига получаем среду исполнения
from smart_kit.configs import get_app_config
app_config = get_app_config()
self.environment = app_config.ENVIRONMENT
# Если среда исполнения задана, то проверям, что среда в списке возможных значений для сценария, иначе - False
self.check_result = self.environment in self.values if self.environment else False
def log(message,
user=None,
params=None,
level="INFO",
exc_info=None,
log_store_for=0):
try:
level_name = logging.getLevelName(level)
current_frame = inspect.currentframe()
previous_frame = current_frame.f_back
module_name = previous_frame.f_globals["__name__"]
logger = logging.getLogger(module_name)
if not logger.isEnabledFor(level_name):
return
instance = previous_frame.f_locals.get('self', None)
from smart_kit.configs import get_app_config
try:
message_maker = get_app_config().LOGGER_MESSAGE_CREATOR
if isinstance(message_maker, Mock):
raise AttributeError
except AttributeError:
message_maker = LoggerMessageCreator
log_store_for_map = getattr(logging, "log_store_for_map", None)
if log_store_for_map is not None and params is not None:
log_store_for = log_store_for_map.get(
params.get(log_const.KEY_NAME), log_store_for)
if instance is not None:
params = message_maker.make_message(user, params,
instance.__class__.__name__,
log_store_for)
else:
params = message_maker.make_message(user,
params,
log_store_for=log_store_for)
# эскейпим сишное форматирование логгера,
# см. tests.core_tests.test_utils.test_logger.TestLogger.test_escaping
message = message_maker.escape(message)
logger.log(level_name, message, params, exc_info=exc_info)
except timeout_decorator.TimeoutError:
raise
except:
default_logger.log(logging.getLevelName("ERROR"),
"Failed to write a log. Exception occurred",
params,
exc_info=True,
stack_info=True)
def execute_from_command_line(argv):
from smart_kit.configs import get_app_config
app_config = get_app_config()
manager = AppManager()
manager.register_command("local_testing", LocalTestingCommand, app_config)
manager.register_command("run_app", RunAppCommand, app_config)
manager.register_command("tests", TestsCommand, app_config)
manager.register_command("cache", CreateCacheCommand, app_config)
manager.register_command("help", HelpCommand, manager.commands)
manager.register_command("get_bundles", GetBundleCommand, app_config)
activate_plugins(app_config, manager)
return manager.execute_from_command_line(argv)
def normalized_text_pymorphy(self):
if self._normalized_text_pymorphy is None:
from smart_kit.configs import get_app_config
app_config = get_app_config()
normalized_words = [
app_config.NORMALIZER.morph.pymorphy_analyzer.parse(
tokenized_word)[0].normalized.normal_form
for tokenized_word in nltk.tokenize.word_tokenize(
self.original_text)
]
self._normalized_text_pymorphy = " ".join(normalized_words)
return self._normalized_text_pymorphy
def __init__(self,
items: Dict[str, Any],
id: Optional[str] = None) -> None:
super(NormalizedInputWordsRequirement, self).__init__(items, id)
# Получаем используемый нормализатор из конфига аппа
from smart_kit.configs import get_app_config
app_config = get_app_config()
self.normalizer = app_config.NORMALIZER
# Нормализуем входные слова из условия
self.input_words = set(items["input_words"])
self.normalized_input_words = set([
norm_res["normalized_text"].replace(".", "").strip()
for norm_res in self.normalizer.normalize_sequence(
list(self.input_words))
])
def __init__(self,
items: Optional[Dict[str, Any]],
id: Optional[str] = None) -> None:
super(IntersectionFieldFiller, self).__init__(items, id)
self.cases = items.get("cases") or {}
self.default = items.get("default")
from smart_kit.configs import get_app_config
app_config = get_app_config()
self.normalized_cases = []
for key, val in self.cases.items():
tokens_list = []
for message in app_config.NORMALIZER.normalize_sequence(val):
case = set()
for norm in message["tokenized_elements_list"]:
if norm.get("token_type") != "SENTENCE_ENDPOINT_TOKEN":
case.add(norm.get("lemma"))
tokens_list.append(case)
def text(self, value):
from smart_kit.configs import get_app_config
config = get_app_config()
try:
recognizer = config.NORMALIZER
norm = recognizer(value)
tpr = {'original_text': norm['original_text'],
'normalized_text': norm['normalized_text'],
'tokenized_elements_list': norm['tokenized_elements_list']}
except (requests.exceptions.RequestException, NormalizationError) as exc:
tpr = {"original_text": value}
logging.getLogger(__file__).warning(
f"\nError due connection to Text Normalizer Server at {self.config.NORMALIZER_ADDRESS}.\n"
f"Result replaced with empty dict.\n"
f"You see this warning because you have enabled debug mode at app_config.py.\n"
f"Error: {exc}"
)
self.__message = tpr
def combine_commands(commands: typing.List[Command], user: User,
**kwargs) -> typing.List[Command]:
user_answers = []
for command in commands:
if command.name == ANSWER_TO_USER:
user_answers.append(command)
for user_answer in user_answers:
commands.remove(user_answer)
if user_answers:
answer_to_user = combine_answer_to_user(user_answers)
last_scenario_name = user.last_scenarios.last_scenario_name
if field.INTENT not in answer_to_user.payload:
if last_scenario_name is not None:
answer_to_user.payload[field.INTENT] = last_scenario_name
debug_info = answer_to_user.payload.setdefault(field.DEBUG_INFO, {})
debug_info[field.INTENT] = last_scenario_name
debug_info[field.DEBUG_INFO_APP_KEY] = user.message.app_info.project_id
if field.FINISHED not in answer_to_user.payload:
finished = last_scenario_name is None
answer_to_user.payload[field.FINISHED] = finished
from smart_kit.configs import get_app_config
config = get_app_config()
if answer_to_user.payload.get(field.AUTO_LISTENING) is None:
answer_to_user.payload[
field.AUTO_LISTENING] = config.AUTO_LISTENING
events = user.history.get_events()
if events and user.history.enabled:
history_data = dict()
history_data[field.EVENTS] = events
answer_to_user.payload[field.HISTORY_DATA] = history_data
commands.append(answer_to_user) # Order isnt important
return commands
def combine_answer_to_user(commands: typing.List[Command]) -> Command:
from smart_kit.configs import get_app_config
config = get_app_config()
answer = Command(name=ANSWER_TO_USER,
request_data=commands[0].request_data,
request_type=commands[0].request_type)
summary_pronounce_text = []
auto_listening = None
for command in commands:
if command.request_data != answer.request_data:
raise ValueError(
f"Cant combine {ANSWER_TO_USER} commands, request_data is different"
)
if command.request_type != answer.request_type:
raise ValueError(
f"Cant combine {ANSWER_TO_USER} commands, request_type is different"
)
payload = command.payload
pronounce_text = payload.pop(field.PRONOUNCE_TEXT, None)
items = payload.pop(field.ITEMS, None)
if auto_listening is None:
auto_listening = payload.get(field.AUTO_LISTENING, None)
if pronounce_text:
summary_pronounce_text.append(pronounce_text)
if items is not None:
answer.payload.setdefault(field.ITEMS, []).extend(items)
answer.payload.update(payload)
if summary_pronounce_text:
answer.payload[field.PRONOUNCE_TEXT] = " ".join(summary_pronounce_text)
answer.payload[field.AUTO_LISTENING] = auto_listening
return answer
def __init__(self,
items: Optional[Dict[str, Any]],
id: Optional[str] = None) -> None:
super(ApproveFiller, self).__init__(items, id)
from smart_kit.configs import get_app_config
app_config = get_app_config()
self.yes_words = items.get("yes_words")
self.no_words = items.get("no_words")
self.set_yes_words: Set = set(self.yes_words or [])
self.set_no_words: Set = set(self.no_words or [])
self.yes_words_normalized: Set = {
TextPreprocessingResult(result).tokenized_string
for result in app_config.NORMALIZER.normalize_sequence(
self.set_yes_words)
}
self.no_words_normalized: Set = {
TextPreprocessingResult(result).tokenized_string
for result in app_config.NORMALIZER.normalize_sequence(
self.set_no_words)
}
