def __init__(self, app_name, dialogue_manager): super(HandlerText, self).__init__(app_name) log(f"{self.__class__.__name__}.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE}) self.dialogue_manager = dialogue_manager log(f"{self.__class__.__name__}.__init__ finished.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE})
def run( self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None ) -> Optional[List[Command]]: callback_id = user.message.callback_id log("%(class_name)s.run: got callback_id %(callback_id)s.", params={ log_const.KEY_NAME: "process_behavior_action", "class_name": self.__class__.__name__, "callback_id": callback_id }, user=user) if not user.behaviors.has_callback(callback_id): log("%(class_name)s.run: user.behaviors has no callback %(callback_id)s.", params={ log_const.KEY_NAME: "process_behavior_action_warning", "class_name": self.__class__.__name__, "callback_id": callback_id }, level="WARNING", user=user) return None if user.message.payload: return user.behaviors.success(callback_id) return user.behaviors.fail(callback_id)
def __init__(self, resources: SmartAppResources, dialogue_manager_cls, custom_settings, **kwargs): log(f"{self.__class__.__name__}.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE}) self.resources = resources self.template_settings = custom_settings["template_settings"] self.app_name = custom_settings.app_name self.dialogue_manager = dialogue_manager_cls( scenario_descriptions=self.scenario_descriptions, app_name=self.app_name) handler_text = HandlerText(self.app_name, dialogue_manager=self.dialogue_manager) self._handlers = { MESSAGE_TO_SKILL: handler_text, RUN_APP: handler_text, LOCAL_TIMEOUT: HandlerTimeout(self.app_name), SERVER_ACTION: HandlerServerAction(self.app_name), CLOSE_APP: HandlerCloseApp(self.app_name) } self._handlers.update({ message_name: HandlerRespond(self.app_name, action_name=action_name) for message_name, action_name in self.resources.get( "responses", {}).items() }) log(f"{self.__class__.__name__}.__init__ finished.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE})
def save_behavior_timeouts(self, user, mq_message, kafka_key): for i, (expire_time_us, callback_id) in enumerate( user.behaviors.get_behavior_timeouts()): # two behaviors can be created in one query, so we need add some salt to make theirs key unique unique_key = expire_time_us + i * 1e-5 log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout on %(unique_key)s", params={ log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, "unique_key": unique_key }) self.behaviors_timeouts.push( unique_key, self.behaviors_timeouts_value_cls._make( (user.message.db_uid, callback_id, mq_message, kafka_key))) for callback_id in user.behaviors.get_returned_callbacks(): log("%(class_name)s: removing local_timeout on callback %(callback_id)s", params={ log_const.KEY_NAME: "removing_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id }) self.behaviors_timeouts.remove(callback_id)
def _process_message(self, msg: KafkaMessage): err = msg.error() if err: if err.code() == KafkaError._PARTITION_EOF: return None else: monitoring.got_counter("kafka_consumer_exception") params = { "code": err.code(), "pid": os.getpid(), "topic": msg.topic(), "partition": msg.partition(), "offset": msg.offset(), log_const.KEY_NAME: log_const.EXCEPTION_VALUE } log( "KafkaConsumer Error %(code)s at pid %(pid)s: topic=%(topic)s partition=[%(partition)s] " "reached end at offset %(offset)s\n", params=params, level="WARNING") raise KafkaException(err) if msg.value(): if msg.headers() is None: msg.set_headers([]) return msg
def run(self, payload, user): callback_id = user.message.callback_id if user.behaviors.has_callback(callback_id): params = {log_const.KEY_NAME: "handling_respond"} log("HandlerRespond started", user, params) action_params = user.behaviors.get_callback_action_params( callback_id) if action_params: app_info = None for original_message_name in [ MESSAGE_TO_SKILL, SERVER_ACTION, RUN_APP ]: if original_message_name in action_params: app_info = AppInfo( action_params[original_message_name].get( APP_INFO, {})) break smart_kit_metrics.counter_incoming(self.app_name, user.message.message_name, self.__class__.__name__, user, app_info=app_info) text_preprocessing_result = TextPreprocessingResult( payload.get("message", {})) params = { log_const.KEY_NAME: log_const.NORMALIZED_TEXT_VALUE, "normalized_text": str(text_preprocessing_result.raw), } log("text preprocessing result: '%(normalized_text)s'", user, params) action_name = self.get_action_name(payload, user) action = user.descriptions["external_actions"][action_name] action_params = self.get_action_params(payload) return action.run(user, text_preprocessing_result, action_params)
def _set_value(self, value): self._value = value message = "BasicField: %(description_id)s filled by value: %(field_value)s" params = {log_const.KEY_NAME: log_const.FILLER_RESULT_VALUE, "description_id": self.description.id, "field_value": str(value)} log(message, None, params)
def expire(self): callback_id_for_delete = [] for callback_id, (behavior_id, expiration_time, *_) in self._callbacks.items(): if expiration_time <= time(): callback_id_for_delete.append(callback_id) for callback_id in callback_id_for_delete: callback_action_params = self.get_callback_action_params( callback_id) to_message_name = callback_action_params.get(TO_MESSAGE_NAME) app_info = callback_action_params.get(APP_INFO, {}) smart_kit_metrics.counter_behavior_expire( self._user.settings.app_name, to_message_name) log_params = { log_const.KEY_NAME: "behavior_expire", log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id, log_const.BEHAVIOR_DATA_VALUE: str(self._callbacks[callback_id]), "to_message_name": to_message_name } log_params.update(app_info) log(f"behavior.expire: if you see this - something went wrong(should be timeout in normal case) callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s, with to_message_name %(to_message_name)s", params=log_params, level="WARNING", user=self._user) self._delete(callback_id)
def get_reply(self, user, text_preprocessing_result, reply_actions, question_field, form): if question_field: question_field.set_available() actions = question_field.description.questions params = { log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE, "field": question_field.description.id } message = "Ask question on field: %(field)s" log(message, user, params) action_messages = self.get_action_results( user, text_preprocessing_result, actions) else: actions = reply_actions params = { log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE, "id": self.id } message = "Finished scenario: %(id)s" log(message, user, params) user.preprocessing_messages_for_scenarios.clear() action_messages = self.get_action_results( user, text_preprocessing_result, actions) user.last_scenarios.delete(self.id) return action_messages
def _extract(self, form, text_normalization_result, user, params): result = {} for field_key, field_descr in form.description.fields.items(): field = form.fields[field_key] if field.available: check = field_descr.requirement.check( text_normalization_result, user, params) log_params = self._log_params() log_params[ "requirement"] = field_descr.requirement.__class__.__name__, log_params["requirement_check_result"] = check log_params["field_key"] = str(field_key) message = "FormFillingScenario.extract: field %(field_key)s requirement %(requirement)s return value: %(requirement_check_result)s" log(message, user, log_params) if check: result[field_key] = field_descr.filler.run( user, text_normalization_result, params) event = Event( type=HistoryConstants.types.FIELD_EVENT, scenario=self.root_id, content={ HistoryConstants.content_fields.FIELD: field_key }, results=HistoryConstants.event_results.FILLED) user.history.add_event(event) return result
def on_run_error(self, text_preprocessing_result, user): log("exc_handler: Action failed to run. Return None. MESSAGE: {}.". format(user.message.masked_value), user, {log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, level="ERROR", exc_info=True) return None
def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessing_result_raw=None, action_params=None): text_preprocessing_result_raw = text_preprocessing_result_raw or {} # behavior will be removed after timeout + EXPIRATION_DELAY expiration_time = int(time()) + self.descriptions[behavior_id].timeout(self._user) + self.EXPIRATION_DELAY callback = self.Callback(behavior_id=behavior_id, expire_time=expiration_time, scenario_id=scenario_id, text_preprocessing_result=text_preprocessing_result_raw, action_params=action_params) self._callbacks[callback_id] = callback log( f"behaviors.add: adding behavior %({log_const.BEHAVIOUR_ID_VALUE})s with scenario_id" f" %({log_const.CHOSEN_SCENARIO_VALUE})s for callback %({log_const.BEHAVIOUR_CALLBACK_ID_VALUE})s" f" expiration_time: %(expiration_time)s.", user=self._user, params={log_const.KEY_NAME: log_const.BEHAVIOUR_ADD_VALUE, log_const.BEHAVIOUR_CALLBACK_ID_VALUE: callback_id, log_const.BEHAVIOUR_ID_VALUE: behavior_id, log_const.CHOSEN_SCENARIO_VALUE: scenario_id, "expiration_time": expiration_time}) behavior_description = self.descriptions[behavior_id] expire_time_us = behavior_description.get_expire_time_from_now(self._user) self._add_behaviour_timeout(expire_time_us, callback_id)
def run(self, payload, user): super().run(payload, user) callback_id = user.message.callback_id if user.behaviors.has_callback(callback_id): params = {log_const.KEY_NAME: "handling_timeout"} log("TimeoutHandler started", user, params) action_params = user.behaviors.get_callback_action_params( callback_id) if action_params: app_info = None for original_message_name in [SERVER_ACTION, RUN_APP]: if original_message_name in action_params: app_info = AppInfo( action_params[original_message_name].get( APP_INFO, {})) break smart_kit_metrics.counter_incoming(self.app_name, user.message.message_name, self.__class__.__name__, user, app_info=app_info) callback_id = user.message.callback_id result = user.behaviors.timeout(callback_id) return result
def run(self): aiohttp_config = self.settings["aiohttp"] if not aiohttp_config: log("aiohttp.yml is empty or missing. Server will be started with default parameters", level="WARN") asyncio.get_event_loop().run_until_complete(self.async_init()) aiohttp.web.run_app(app=self.app, **aiohttp_config)
def iterate(self, environ, start_response): stats = "" with StatsTimer() as poll_timer: try: content_length = int(environ.get('CONTENT_LENGTH', '0')) request = environ["wsgi.input"].read(content_length).decode() headers = self._get_headers(environ) except KeyError: log("Error in request data", level="ERROR") raise Exception("Error in request data") stats += "Polling time: {} msecs\n".format(poll_timer.msecs) message = SmartAppFromMessage(request, headers=headers, headers_required=False) if not message.validate(): start_response("400 BAD REQUEST", self._get_outgoing_headers(headers)) return [b'{"message": "invalid message"}'] answer, stats = self.handle_message(message, stats) with StatsTimer() as publish_timer: if not answer: start_response("204 NO CONTENT", self._get_outgoing_headers(headers)) return [b'{"message": "no answer"}'] start_response("200 OK", self._get_outgoing_headers(headers, answer)) answer = SmartAppToMessage(answer, message, request=None) stats += "Publishing time: {} msecs".format(publish_timer.msecs) log(stats, params={log_const.KEY_NAME: "timings"}) return [answer.value.encode()]
def check_misstate(self, callback_id: str): log(f"behavior.check_misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={ log_const.KEY_NAME: log_const.BEHAVIOR_CHECK_MISSTATE_VALUE, log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id }) callback = self._callbacks.get(callback_id) if callback: callback_scenario_id = callback.scenario_id if callback_scenario_id is not None: last_scenario_equal_callback_scenario = self._user.last_scenarios.last_scenario_name != callback_scenario_id if not last_scenario_equal_callback_scenario: log(f"behavior.check_misstate: GOT MISSTATE for callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s: user_scenario: %(user_scenario)s, callback_scenario: %(callback_scenario)s.", self._user, params={ log_const.KEY_NAME: log_const.BEHAVIOR_CHECK_MISSTATE_VALUE, log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id, "user_scenario": self._user.last_scenarios.last_scenario_name, "callback_scenario": callback_scenario_id }, level="WARNING") return last_scenario_equal_callback_scenario
def _initialize(self): db_version = self.variables.get(self.USER_DB_VERSION, default=0) self.variables.set(self.USER_DB_VERSION, db_version + 1) log("%(class_name)s.__init__ USER %(uid)s LOAD db_version = %(db_version)s.", self, {"db_version": str(db_version), "uid": str(self.id)}) self.behaviors.initialize()
def _error_callback(self, err): params = { "error": str(err), log_const.KEY_NAME: log_const.EXCEPTION_VALUE } log("KafkaConsumer: Error: %(error)s", params=params, level="WARNING") monitoring.got_counter("kafka_consumer_exception")
def save_user(self, db_uid, user, message): no_collisions = True if user.do_not_save: log("User %(uid)s will not saved", user=user, params={ "uid": user.id, log_const.KEY_NAME: "user_will_not_saved" }) else: no_collisions = True try: str_data = user.raw_str if user.initial_db_data and self.user_save_check_for_collisions: no_collisions = self.db_adapter.replace_if_equals( db_uid, sample=user.initial_db_data, data=str_data) else: self.db_adapter.save(db_uid, str_data) except (DBAdapterException, ValueError): log("Failed to set user data", params={ log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value) }, level="ERROR") smart_kit_metrics.counter_save_error(self.app_name) if not no_collisions: smart_kit_metrics.counter_save_collision(self.app_name) return no_collisions
def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> Optional[bool]: original_text = ' '.join( text_preprocessing_result.original_text.split()).lower().rstrip( '!.)') if original_text in self.set_yes_words: params = self._log_params() params["original_text"] = original_text params["yes_words"] = self.set_yes_words message = "Filler: %(filler)s, original_text: %(original_text)s, self.yes_words: %(yes_words)s" log(message, user, params) response = True elif text_preprocessing_result.words_tokenized_set.intersection( self.no_words_normalized): params = self._log_params() params[ "words_tokenized_set"] = text_preprocessing_result.words_tokenized_set params["no_words"] = self.set_no_words message = "Filler: %(filler)s, words_normalized_set: %(words_tokenized_set)s, self.no_words: %(no_words)s" log(message, user, params) response = False else: response = None return response
def check_message_key(self, from_message, message_key, user): sub = from_message.sub channel = from_message.channel uid = from_message.uid message_key = message_key or b"" try: params = [channel, sub, uid] valid_key = "" for value in params: if value: valid_key = "{}{}{}".format( valid_key, "_", value) if valid_key else "{}".format(value) key_str = message_key.decode() message_key_is_valid = key_str == valid_key if not message_key_is_valid: log(f"Failed to check Kafka message key {message_key} != {valid_key}", params={ log_const.KEY_NAME: "check_kafka_key_validation", MESSAGE_ID_STR: from_message.incremental_id, UID_STR: uid }, user=user, level="WARNING") except: log(f"Exception to check Kafka message key {message_key}", params={ log_const.KEY_NAME: "check_kafka_key_error", MESSAGE_ID_STR: from_message.incremental_id, UID_STR: uid }, user=user, level="ERROR")
def __init__(self, id, message, db_data, settings, descriptions, parametrizer_cls, load_error=False): self.settings = settings try: user_values = json.loads(db_data) if db_data else None except ValueError: user_values = None smart_kit_metrics.counter_load_error(settings.app_name) log(f"%(class_name)s.__init__ User load ValueError %(uid)s with user data {db_data}", params={ log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, "uid": str(id) }, level="ERROR") load_error = True super(User, self).__init__(id, message, user_values, descriptions, load_error) self.__parametrizer_cls = parametrizer_cls self.do_not_save = False self.initial_db_data = db_data
def get_reply(self, user, text_preprocessing_result, reply_actions, field, form): action_params = {} if field: field.set_available() actions = field.description.requests params = { log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE, "field": field.description.id } message = "Ask question on field: %(field)s" log(message, user, params) action_params[REQUEST_FIELD] = { "type": field.description.type, "id": field.description.id } action_messages = self.get_action_results( user, text_preprocessing_result, actions, action_params) else: actions = reply_actions params = { log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE, "id": self.id } message = "Finished scenario: %(id)s" log(message, user, params) user.preprocessing_messages_for_scenarios.clear() action_messages = self.get_action_results( user, text_preprocessing_result, actions, action_params) return action_messages
def on_answer_error(self, message, user): user.do_not_save = True smart_kit_metrics.counter_exception(self.app_name) params = { log_const.KEY_NAME: log_const.DIALOG_ERROR_VALUE, "message_id": user.message.incremental_id } log("exc_handler: Failed to process message. Exception occurred. Fail in MESSAGE: {}" .format(user.message.masked_value), user, params, level="ERROR", exc_info=True) callback_action_params = get_callback_action_params(user) exc_type, exc_value, exc_traceback = sys.exc_info() error = '\n'.join( traceback.format_exception(exc_type, exc_value, exc_traceback)) if user.settings["template_settings"].get("debug_info"): set_debug_info(self.app_name, callback_action_params, error) exception_action = user.descriptions["external_actions"][ "exception_action"] commands = exception_action.run(user=user, text_preprocessing_result=None, params=callback_action_params) return commands
def success(self, callback_id: str): log(f"behavior.success started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={ log_const.KEY_NAME: log_const.BEHAVIOR_SUCCESS_VALUE, log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id }) callback = self._get_callback(callback_id) result = None if callback: self._check_hostname(callback_id, callback) self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params self._log_callback( callback_id, "behavior_success", smart_kit_metrics.counter_behavior_success, "success", callback_action_params, ) text_preprocessing_result = TextPreprocessingResult( callback.text_preprocessing_result) result = behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result
def on_extract_error(self, text_preprocessing_result, user, params=None): log("exc_handler: ElseFiller failed to extract. Return None. MESSAGE: {}." .format(user.message.masked_value), user, {log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, level="ERROR", exc_info=True) return None
def __init__(self, id, message, db_data, settings, descriptions, parametrizer_cls, load_error=False): self.settings = settings try: user_values = json.loads(db_data) if db_data else None except ValueError: user_values = None load_error = True super(User, self).__init__(id, message, user_values, descriptions, load_error) self.__parametrizer_cls = parametrizer_cls self.do_not_save = False self.initial_db_data = db_data db_version = self.variables.get(self.USER_DB_VERSION, default=0) self.variables.set(self.USER_DB_VERSION, db_version + 1) log( "%(class_name)s.__init__ USER %(uid)s LOAD db_version = %(db_version)s.", self, { "db_version": str(db_version), "uid": str(self.id) })
def process_message(self, message: SmartAppFromMessage, *args, **kwargs): stats = "" log("INCOMING DATA: %(masked_message)s", params={ log_const.KEY_NAME: "incoming_policy_message", "masked_message": message.masked_value, "message_id": message.incremental_id, }) db_uid = message.db_uid with StatsTimer() as load_timer: user = self.load_user(db_uid, message) stats += "Loading time: {} msecs\n".format(load_timer.msecs) with StatsTimer() as script_timer: commands = self.model.answer(message, user) if commands: answer = self._generate_answers(user, commands, message) else: answer = None stats += "Script time: {} msecs\n".format(script_timer.msecs) with StatsTimer() as save_timer: self.save_user(db_uid, user, message) stats += "Saving time: {} msecs\n".format(save_timer.msecs) log(stats, user=user, params={log_const.KEY_NAME: "timings"}) self.postprocessor.postprocess(user, message) return answer, stats
def on_check_error(self, text_preprocessing_result, user): log("exc_handler: Requirement failed to check. Return False. MESSAGE: {}." .format(user.message.masked_value), user, {log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, level="ERROR", exc_info=True) return False
def run(self): self._server = make_server('0.0.0.0', 8000, self.iterate) log(''' Application start via "python manage.py run_app" recommended only for local testing. For production it is recommended to start using "gunicorn --config wsgi_config.py 'wsgi:create_app()' ''', level="WARNING") self._server.serve_forever()