def __init__(self, *args, **kwargs): super(Settings, self).__init__() self.configs_path = kwargs.get("config_path") self.references_path = kwargs.get("references_path") self.secret_path = kwargs.get("secret_path") self.app_name = kwargs.get("app_name") self.adapters = { Settings.CephAdapterKey: CephAdapter, self.OSAdapterKey: OSAdapter } self.repositories = [ UpdatableFileRepository(self.subfolder_path("template_config.yml"), loader=yaml.safe_load, key="template_settings"), FileRepository(self.subfolder_secret_path("kafka_config.yml"), loader=yaml.safe_load, key="kafka"), FileRepository(self.subfolder_path("ceph_config.yml"), loader=yaml.safe_load, key=self.CephAdapterKey), FileRepository(self.subfolder_path("aiohttp.yml"), loader=yaml.safe_load, key="aiohttp"), ] self.repositories = self.override_repositories(self.repositories) self.init()
def __init__(self, source, references_path, settings): super(SmartAppResources, self).__init__(source=source) self.references_path = references_path self.repositories = [ FolderRepository(self.subfolder_path("forms"), loader=ordered_json, source=source, key="forms"), FolderRepository(self.subfolder_path("scenarios"), loader=ordered_json, source=source, key="scenarios"), FileRepository(self.subfolder_path("preprocessing_messages_for_scenarios_settings.json"), loader=ordered_json, source=source, key="preprocessing_messages_for_scenarios"), FileRepository(self.subfolder_path("last_scenarios_descriptions.json"), loader=ordered_json, source=source, key="last_scenarios"), FileRepository(self.subfolder_path("history.json"), loader=ordered_json, source=source, key="history"), FolderRepository(self.subfolder_path("behaviors"), loader=ordered_json, source=source, key="behaviors"), FolderRepository(self.subfolder_path("actions"), loader=ordered_json, source=source, key="external_actions"), FolderRepository(self.subfolder_path("requirements"), loader=ordered_json, source=source, key="external_requirements"), FolderRepository(self.subfolder_path("field_fillers"), loader=ordered_json, source=source, key="external_field_fillers"), FileRepository(self.subfolder_path("responses.json"), loader=ordered_json, source=source, key="responses"), FileRepository(self.subfolder_path("last_action_ids.json"), loader=ordered_json, source=source, key="last_action_ids"), FolderRepository(self.subfolder_path("bundles"), loader=ordered_json, source=source, key="bundles") ] self.repositories = self.override_repositories(self.repositories) self.init()
def items_for_normalized(self): references_path = self.app_config.REFERENCES_PATH items_for_normalized = [] for r, d, f in os.walk(references_path): for file in f: if self._ext in file: filename = os.path.join(r, file) rep = FileRepository(filename=filename, loader=json.loads) rep.load() data = rep.data item = self._split_items( self._collect_items_for_normalized(data)) items_for_normalized.extend(item) return items_for_normalized
def __init__(self, config_path): self.config_path = config_path super(LoggerConfig, self).__init__() self.repositories = [ FileRepository(self.subfolder_path("logging_config.yml"), loader=yaml.safe_load, key=LOGGING_CONFIG) ]
def __load_everything(self): nltk.download('punkt') from smart_kit.configs import get_app_config app_config = get_app_config() text_normalizer_params = FileRepository( f"{app_config.STATIC_PATH}/.text_normalizer_resources/static_workdata.json", loader=ordered_json, ) text_normalizer_params.load() synonyms = FileRepository( f"{app_config.STATIC_PATH}/.text_normalizer_resources/dict_synonyms.json", loader=reverse_json_dict, ) synonyms.load() text_normalizer_params.data["synonyms"] = synonyms.data text2num_dict = FileRepository( f"{app_config.STATIC_PATH}/.text_normalizer_resources/text2num_dict.json", loader=ordered_json, ) text2num_dict.load() text_normalizer_params.data["text2num_dict"] = text2num_dict.data text_normalizer_params = text_normalizer_params.data or {} self.convert_plan = text_normalizer_params["convert_plan"] self.processor_pipeline = text_normalizer_params["processor_pipeline"] word_false_stoppings = text_normalizer_params.get( "word_false_stoppings", []) words_without_splitting_point = text_normalizer_params.get( "word_no_splitting_point", []) synonyms = text_normalizer_params.get("synonyms", {}) text2num = text_normalizer_params.get("text2num_dict", {}) sberbank_phones = text_normalizer_params.get("sberbank_phones", []) unicode_symbols = text_normalizer_params.get("unicode_symbols", {}) self.sentence_tokenizer = ru_sent_tokenize self.word_tokenizer = NLTKWordTokenizer(word_false_stoppings, words_without_splitting_point) self.morph = RNNMorphWrapper() skip_func = lambda x: x self.converter_pipeline = { 'Объединение цифр после stt': NumbersUnionAfterSTT(text2num), 'Конверсия юникодовых символов': UnicodeSymbolsConverter(unicode_symbols), 'Цифры и буквы отдельно': unmerge_numbers_and_letters, 'Ё на Е': replace_yo_to_e, 'Номера телефонов': NormalizePhoneNumbers(), 'Номера телефонов из голоса': NormalizePhoneNumbersVoice(), 'Номера карт': MergeCardNumbers(), 'Номера карт из голоса': MergeCardNumbersVoice(), 'Объединение сумм': merge_numbers, 'Символы валют': replace_currencies_symbols, "Претокенизация математических операций": AdditionalMathSplitter() } self.tokens_processor = { "Synonyms": ReplaceSynonyms(synonyms) if synonyms else skip_func, "Text2Num": Text2Num(text2num, sberbank_phones) if text2num else skip_func, "Currency": CurrencyTokensOneIterationMerger(), "Grammemes": self.morph, } self.__ready_to_use = True