def __init__(self, *args, **kwargs):
super(Settings, self).__init__()
self.configs_path = kwargs.get("config_path")
self.references_path = kwargs.get("references_path")
self.secret_path = kwargs.get("secret_path")
self.app_name = kwargs.get("app_name")
self.adapters = {
Settings.CephAdapterKey: CephAdapter,
self.OSAdapterKey: OSAdapter
}
self.repositories = [
UpdatableFileRepository(self.subfolder_path("template_config.yml"),
loader=yaml.safe_load,
key="template_settings"),
FileRepository(self.subfolder_secret_path("kafka_config.yml"),
loader=yaml.safe_load,
key="kafka"),
FileRepository(self.subfolder_path("ceph_config.yml"),
loader=yaml.safe_load,
key=self.CephAdapterKey),
FileRepository(self.subfolder_path("aiohttp.yml"),
loader=yaml.safe_load,
key="aiohttp"),
]
self.repositories = self.override_repositories(self.repositories)
self.init()
def __init__(self, source, references_path, settings):
super(SmartAppResources, self).__init__(source=source)
self.references_path = references_path
self.repositories = [
FolderRepository(self.subfolder_path("forms"), loader=ordered_json, source=source,
key="forms"),
FolderRepository(self.subfolder_path("scenarios"), loader=ordered_json, source=source,
key="scenarios"),
FileRepository(self.subfolder_path("preprocessing_messages_for_scenarios_settings.json"),
loader=ordered_json,
source=source, key="preprocessing_messages_for_scenarios"),
FileRepository(self.subfolder_path("last_scenarios_descriptions.json"), loader=ordered_json,
source=source, key="last_scenarios"),
FileRepository(self.subfolder_path("history.json"), loader=ordered_json, source=source,
key="history"),
FolderRepository(self.subfolder_path("behaviors"), loader=ordered_json, source=source,
key="behaviors"),
FolderRepository(self.subfolder_path("actions"), loader=ordered_json, source=source,
key="external_actions"),
FolderRepository(self.subfolder_path("requirements"), loader=ordered_json, source=source,
key="external_requirements"),
FolderRepository(self.subfolder_path("field_fillers"), loader=ordered_json, source=source,
key="external_field_fillers"),
FileRepository(self.subfolder_path("responses.json"), loader=ordered_json, source=source,
key="responses"),
FileRepository(self.subfolder_path("last_action_ids.json"), loader=ordered_json,
source=source, key="last_action_ids"),
FolderRepository(self.subfolder_path("bundles"), loader=ordered_json, source=source,
key="bundles")
]
self.repositories = self.override_repositories(self.repositories)
self.init()
def items_for_normalized(self):
references_path = self.app_config.REFERENCES_PATH
items_for_normalized = []
for r, d, f in os.walk(references_path):
for file in f:
if self._ext in file:
filename = os.path.join(r, file)
rep = FileRepository(filename=filename, loader=json.loads)
rep.load()
data = rep.data
item = self._split_items(
self._collect_items_for_normalized(data))
items_for_normalized.extend(item)
return items_for_normalized
def __init__(self, config_path):
self.config_path = config_path
super(LoggerConfig, self).__init__()
self.repositories = [
FileRepository(self.subfolder_path("logging_config.yml"),
loader=yaml.safe_load, key=LOGGING_CONFIG)
]
def __load_everything(self):
nltk.download('punkt')
from smart_kit.configs import get_app_config
app_config = get_app_config()
text_normalizer_params = FileRepository(
f"{app_config.STATIC_PATH}/.text_normalizer_resources/static_workdata.json",
loader=ordered_json,
)
text_normalizer_params.load()
synonyms = FileRepository(
f"{app_config.STATIC_PATH}/.text_normalizer_resources/dict_synonyms.json",
loader=reverse_json_dict,
)
synonyms.load()
text_normalizer_params.data["synonyms"] = synonyms.data
text2num_dict = FileRepository(
f"{app_config.STATIC_PATH}/.text_normalizer_resources/text2num_dict.json",
loader=ordered_json,
)
text2num_dict.load()
text_normalizer_params.data["text2num_dict"] = text2num_dict.data
text_normalizer_params = text_normalizer_params.data or {}
self.convert_plan = text_normalizer_params["convert_plan"]
self.processor_pipeline = text_normalizer_params["processor_pipeline"]
word_false_stoppings = text_normalizer_params.get(
"word_false_stoppings", [])
words_without_splitting_point = text_normalizer_params.get(
"word_no_splitting_point", [])
synonyms = text_normalizer_params.get("synonyms", {})
text2num = text_normalizer_params.get("text2num_dict", {})
sberbank_phones = text_normalizer_params.get("sberbank_phones", [])
unicode_symbols = text_normalizer_params.get("unicode_symbols", {})
self.sentence_tokenizer = ru_sent_tokenize
self.word_tokenizer = NLTKWordTokenizer(word_false_stoppings,
words_without_splitting_point)
self.morph = RNNMorphWrapper()
skip_func = lambda x: x
self.converter_pipeline = {
'Объединение цифр после stt':
NumbersUnionAfterSTT(text2num),
'Конверсия юникодовых символов':
UnicodeSymbolsConverter(unicode_symbols),
'Цифры и буквы отдельно':
unmerge_numbers_and_letters,
'Ё на Е':
replace_yo_to_e,
'Номера телефонов':
NormalizePhoneNumbers(),
'Номера телефонов из голоса':
NormalizePhoneNumbersVoice(),
'Номера карт':
MergeCardNumbers(),
'Номера карт из голоса':
MergeCardNumbersVoice(),
'Объединение сумм':
merge_numbers,
'Символы валют':
replace_currencies_symbols,
"Претокенизация математических операций":
AdditionalMathSplitter()
}
self.tokens_processor = {
"Synonyms": ReplaceSynonyms(synonyms) if synonyms else skip_func,
"Text2Num":
Text2Num(text2num, sberbank_phones) if text2num else skip_func,
"Currency": CurrencyTokensOneIterationMerger(),
"Grammemes": self.morph,
}
self.__ready_to_use = True