def __init__(self, app_name, dialogue_manager):
super(HandlerText, self).__init__(app_name)
log(f"{self.__class__.__name__}.__init__ started.",
params={log_const.KEY_NAME: log_const.STARTUP_VALUE})
self.dialogue_manager = dialogue_manager
log(f"{self.__class__.__name__}.__init__ finished.",
params={log_const.KEY_NAME: log_const.STARTUP_VALUE})
def run(
self,
user: User,
text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None
) -> Optional[List[Command]]:
callback_id = user.message.callback_id
log("%(class_name)s.run: got callback_id %(callback_id)s.",
params={
log_const.KEY_NAME: "process_behavior_action",
"class_name": self.__class__.__name__,
"callback_id": callback_id
},
user=user)
if not user.behaviors.has_callback(callback_id):
log("%(class_name)s.run: user.behaviors has no callback %(callback_id)s.",
params={
log_const.KEY_NAME: "process_behavior_action_warning",
"class_name": self.__class__.__name__,
"callback_id": callback_id
},
level="WARNING",
user=user)
return None
if user.message.payload:
return user.behaviors.success(callback_id)
return user.behaviors.fail(callback_id)
def __init__(self, resources: SmartAppResources, dialogue_manager_cls,
custom_settings, **kwargs):
log(f"{self.__class__.__name__}.__init__ started.",
params={log_const.KEY_NAME: log_const.STARTUP_VALUE})
self.resources = resources
self.template_settings = custom_settings["template_settings"]
self.app_name = custom_settings.app_name
self.dialogue_manager = dialogue_manager_cls(
scenario_descriptions=self.scenario_descriptions,
app_name=self.app_name)
handler_text = HandlerText(self.app_name,
dialogue_manager=self.dialogue_manager)
self._handlers = {
MESSAGE_TO_SKILL: handler_text,
RUN_APP: handler_text,
LOCAL_TIMEOUT: HandlerTimeout(self.app_name),
SERVER_ACTION: HandlerServerAction(self.app_name),
CLOSE_APP: HandlerCloseApp(self.app_name)
}
self._handlers.update({
message_name: HandlerRespond(self.app_name,
action_name=action_name)
for message_name, action_name in self.resources.get(
"responses", {}).items()
})
log(f"{self.__class__.__name__}.__init__ finished.",
params={log_const.KEY_NAME: log_const.STARTUP_VALUE})
def save_behavior_timeouts(self, user, mq_message, kafka_key):
for i, (expire_time_us, callback_id) in enumerate(
user.behaviors.get_behavior_timeouts()):
# two behaviors can be created in one query, so we need add some salt to make theirs key unique
unique_key = expire_time_us + i * 1e-5
log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout on %(unique_key)s",
params={
log_const.KEY_NAME: "adding_local_timeout",
"class_name": self.__class__.__name__,
"callback_id": callback_id,
"unique_key": unique_key
})
self.behaviors_timeouts.push(
unique_key,
self.behaviors_timeouts_value_cls._make(
(user.message.db_uid, callback_id, mq_message, kafka_key)))
for callback_id in user.behaviors.get_returned_callbacks():
log("%(class_name)s: removing local_timeout on callback %(callback_id)s",
params={
log_const.KEY_NAME: "removing_local_timeout",
"class_name": self.__class__.__name__,
"callback_id": callback_id
})
self.behaviors_timeouts.remove(callback_id)
def _process_message(self, msg: KafkaMessage):
err = msg.error()
if err:
if err.code() == KafkaError._PARTITION_EOF:
return None
else:
monitoring.got_counter("kafka_consumer_exception")
params = {
"code": err.code(),
"pid": os.getpid(),
"topic": msg.topic(),
"partition": msg.partition(),
"offset": msg.offset(),
log_const.KEY_NAME: log_const.EXCEPTION_VALUE
}
log(
"KafkaConsumer Error %(code)s at pid %(pid)s: topic=%(topic)s partition=[%(partition)s] "
"reached end at offset %(offset)s\n",
params=params,
level="WARNING")
raise KafkaException(err)
if msg.value():
if msg.headers() is None:
msg.set_headers([])
return msg
def run(self, payload, user):
callback_id = user.message.callback_id
if user.behaviors.has_callback(callback_id):
params = {log_const.KEY_NAME: "handling_respond"}
log("HandlerRespond started", user, params)
action_params = user.behaviors.get_callback_action_params(
callback_id)
if action_params:
app_info = None
for original_message_name in [
MESSAGE_TO_SKILL, SERVER_ACTION, RUN_APP
]:
if original_message_name in action_params:
app_info = AppInfo(
action_params[original_message_name].get(
APP_INFO, {}))
break
smart_kit_metrics.counter_incoming(self.app_name,
user.message.message_name,
self.__class__.__name__,
user,
app_info=app_info)
text_preprocessing_result = TextPreprocessingResult(
payload.get("message", {}))
params = {
log_const.KEY_NAME: log_const.NORMALIZED_TEXT_VALUE,
"normalized_text": str(text_preprocessing_result.raw),
}
log("text preprocessing result: '%(normalized_text)s'", user, params)
action_name = self.get_action_name(payload, user)
action = user.descriptions["external_actions"][action_name]
action_params = self.get_action_params(payload)
return action.run(user, text_preprocessing_result, action_params)
def _set_value(self, value):
self._value = value
message = "BasicField: %(description_id)s filled by value: %(field_value)s"
params = {log_const.KEY_NAME: log_const.FILLER_RESULT_VALUE,
"description_id": self.description.id,
"field_value": str(value)}
log(message, None, params)
def expire(self):
callback_id_for_delete = []
for callback_id, (behavior_id, expiration_time,
*_) in self._callbacks.items():
if expiration_time <= time():
callback_id_for_delete.append(callback_id)
for callback_id in callback_id_for_delete:
callback_action_params = self.get_callback_action_params(
callback_id)
to_message_name = callback_action_params.get(TO_MESSAGE_NAME)
app_info = callback_action_params.get(APP_INFO, {})
smart_kit_metrics.counter_behavior_expire(
self._user.settings.app_name, to_message_name)
log_params = {
log_const.KEY_NAME: "behavior_expire",
log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id,
log_const.BEHAVIOR_DATA_VALUE:
str(self._callbacks[callback_id]),
"to_message_name": to_message_name
}
log_params.update(app_info)
log(f"behavior.expire: if you see this - something went wrong(should be timeout in normal case) callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s, with to_message_name %(to_message_name)s",
params=log_params,
level="WARNING",
user=self._user)
self._delete(callback_id)
def get_reply(self, user, text_preprocessing_result, reply_actions,
question_field, form):
if question_field:
question_field.set_available()
actions = question_field.description.questions
params = {
log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE,
"field": question_field.description.id
}
message = "Ask question on field: %(field)s"
log(message, user, params)
action_messages = self.get_action_results(
user, text_preprocessing_result, actions)
else:
actions = reply_actions
params = {
log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE,
"id": self.id
}
message = "Finished scenario: %(id)s"
log(message, user, params)
user.preprocessing_messages_for_scenarios.clear()
action_messages = self.get_action_results(
user, text_preprocessing_result, actions)
user.last_scenarios.delete(self.id)
return action_messages
def _extract(self, form, text_normalization_result, user, params):
result = {}
for field_key, field_descr in form.description.fields.items():
field = form.fields[field_key]
if field.available:
check = field_descr.requirement.check(
text_normalization_result, user, params)
log_params = self._log_params()
log_params[
"requirement"] = field_descr.requirement.__class__.__name__,
log_params["requirement_check_result"] = check
log_params["field_key"] = str(field_key)
message = "FormFillingScenario.extract: field %(field_key)s requirement %(requirement)s return value: %(requirement_check_result)s"
log(message, user, log_params)
if check:
result[field_key] = field_descr.filler.run(
user, text_normalization_result, params)
event = Event(
type=HistoryConstants.types.FIELD_EVENT,
scenario=self.root_id,
content={
HistoryConstants.content_fields.FIELD: field_key
},
results=HistoryConstants.event_results.FILLED)
user.history.add_event(event)
return result
def on_run_error(self, text_preprocessing_result, user):
log("exc_handler: Action failed to run. Return None. MESSAGE: {}.".
format(user.message.masked_value),
user, {log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE},
level="ERROR",
exc_info=True)
return None
def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessing_result_raw=None,
action_params=None):
text_preprocessing_result_raw = text_preprocessing_result_raw or {}
# behavior will be removed after timeout + EXPIRATION_DELAY
expiration_time = int(time()) + self.descriptions[behavior_id].timeout(self._user) + self.EXPIRATION_DELAY
callback = self.Callback(behavior_id=behavior_id,
expire_time=expiration_time,
scenario_id=scenario_id,
text_preprocessing_result=text_preprocessing_result_raw,
action_params=action_params)
self._callbacks[callback_id] = callback
log(
f"behaviors.add: adding behavior %({log_const.BEHAVIOUR_ID_VALUE})s with scenario_id"
f" %({log_const.CHOSEN_SCENARIO_VALUE})s for callback %({log_const.BEHAVIOUR_CALLBACK_ID_VALUE})s"
f" expiration_time: %(expiration_time)s.",
user=self._user,
params={log_const.KEY_NAME: log_const.BEHAVIOUR_ADD_VALUE,
log_const.BEHAVIOUR_CALLBACK_ID_VALUE: callback_id,
log_const.BEHAVIOUR_ID_VALUE: behavior_id,
log_const.CHOSEN_SCENARIO_VALUE: scenario_id,
"expiration_time": expiration_time})
behavior_description = self.descriptions[behavior_id]
expire_time_us = behavior_description.get_expire_time_from_now(self._user)
self._add_behaviour_timeout(expire_time_us, callback_id)
def run(self, payload, user):
super().run(payload, user)
callback_id = user.message.callback_id
if user.behaviors.has_callback(callback_id):
params = {log_const.KEY_NAME: "handling_timeout"}
log("TimeoutHandler started", user, params)
action_params = user.behaviors.get_callback_action_params(
callback_id)
if action_params:
app_info = None
for original_message_name in [SERVER_ACTION, RUN_APP]:
if original_message_name in action_params:
app_info = AppInfo(
action_params[original_message_name].get(
APP_INFO, {}))
break
smart_kit_metrics.counter_incoming(self.app_name,
user.message.message_name,
self.__class__.__name__,
user,
app_info=app_info)
callback_id = user.message.callback_id
result = user.behaviors.timeout(callback_id)
return result
def run(self):
aiohttp_config = self.settings["aiohttp"]
if not aiohttp_config:
log("aiohttp.yml is empty or missing. Server will be started with default parameters",
level="WARN")
asyncio.get_event_loop().run_until_complete(self.async_init())
aiohttp.web.run_app(app=self.app, **aiohttp_config)
def iterate(self, environ, start_response):
stats = ""
with StatsTimer() as poll_timer:
try:
content_length = int(environ.get('CONTENT_LENGTH', '0'))
request = environ["wsgi.input"].read(content_length).decode()
headers = self._get_headers(environ)
except KeyError:
log("Error in request data", level="ERROR")
raise Exception("Error in request data")
stats += "Polling time: {} msecs\n".format(poll_timer.msecs)
message = SmartAppFromMessage(request,
headers=headers,
headers_required=False)
if not message.validate():
start_response("400 BAD REQUEST",
self._get_outgoing_headers(headers))
return [b'{"message": "invalid message"}']
answer, stats = self.handle_message(message, stats)
with StatsTimer() as publish_timer:
if not answer:
start_response("204 NO CONTENT",
self._get_outgoing_headers(headers))
return [b'{"message": "no answer"}']
start_response("200 OK",
self._get_outgoing_headers(headers, answer))
answer = SmartAppToMessage(answer, message, request=None)
stats += "Publishing time: {} msecs".format(publish_timer.msecs)
log(stats, params={log_const.KEY_NAME: "timings"})
return [answer.value.encode()]
def check_misstate(self, callback_id: str):
log(f"behavior.check_misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.",
self._user,
params={
log_const.KEY_NAME: log_const.BEHAVIOR_CHECK_MISSTATE_VALUE,
log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id
})
callback = self._callbacks.get(callback_id)
if callback:
callback_scenario_id = callback.scenario_id
if callback_scenario_id is not None:
last_scenario_equal_callback_scenario = self._user.last_scenarios.last_scenario_name != callback_scenario_id
if not last_scenario_equal_callback_scenario:
log(f"behavior.check_misstate: GOT MISSTATE for callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s: user_scenario: %(user_scenario)s, callback_scenario: %(callback_scenario)s.",
self._user,
params={
log_const.KEY_NAME:
log_const.BEHAVIOR_CHECK_MISSTATE_VALUE,
log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id,
"user_scenario":
self._user.last_scenarios.last_scenario_name,
"callback_scenario": callback_scenario_id
},
level="WARNING")
return last_scenario_equal_callback_scenario
def _initialize(self):
db_version = self.variables.get(self.USER_DB_VERSION, default=0)
self.variables.set(self.USER_DB_VERSION, db_version + 1)
log("%(class_name)s.__init__ USER %(uid)s LOAD db_version = %(db_version)s.", self,
{"db_version": str(db_version),
"uid": str(self.id)})
self.behaviors.initialize()
def _error_callback(self, err):
params = {
"error": str(err),
log_const.KEY_NAME: log_const.EXCEPTION_VALUE
}
log("KafkaConsumer: Error: %(error)s", params=params, level="WARNING")
monitoring.got_counter("kafka_consumer_exception")
def save_user(self, db_uid, user, message):
no_collisions = True
if user.do_not_save:
log("User %(uid)s will not saved",
user=user,
params={
"uid": user.id,
log_const.KEY_NAME: "user_will_not_saved"
})
else:
no_collisions = True
try:
str_data = user.raw_str
if user.initial_db_data and self.user_save_check_for_collisions:
no_collisions = self.db_adapter.replace_if_equals(
db_uid, sample=user.initial_db_data, data=str_data)
else:
self.db_adapter.save(db_uid, str_data)
except (DBAdapterException, ValueError):
log("Failed to set user data",
params={
log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION,
log_const.REQUEST_VALUE: str(message.value)
},
level="ERROR")
smart_kit_metrics.counter_save_error(self.app_name)
if not no_collisions:
smart_kit_metrics.counter_save_collision(self.app_name)
return no_collisions
def extract(self,
text_preprocessing_result: TextPreprocessingResult,
user: User,
params: Dict[str, Any] = None) -> Optional[bool]:
original_text = ' '.join(
text_preprocessing_result.original_text.split()).lower().rstrip(
'!.)')
if original_text in self.set_yes_words:
params = self._log_params()
params["original_text"] = original_text
params["yes_words"] = self.set_yes_words
message = "Filler: %(filler)s, original_text: %(original_text)s, self.yes_words: %(yes_words)s"
log(message, user, params)
response = True
elif text_preprocessing_result.words_tokenized_set.intersection(
self.no_words_normalized):
params = self._log_params()
params[
"words_tokenized_set"] = text_preprocessing_result.words_tokenized_set
params["no_words"] = self.set_no_words
message = "Filler: %(filler)s, words_normalized_set: %(words_tokenized_set)s, self.no_words: %(no_words)s"
log(message, user, params)
response = False
else:
response = None
return response
def check_message_key(self, from_message, message_key, user):
sub = from_message.sub
channel = from_message.channel
uid = from_message.uid
message_key = message_key or b""
try:
params = [channel, sub, uid]
valid_key = ""
for value in params:
if value:
valid_key = "{}{}{}".format(
valid_key, "_",
value) if valid_key else "{}".format(value)
key_str = message_key.decode()
message_key_is_valid = key_str == valid_key
if not message_key_is_valid:
log(f"Failed to check Kafka message key {message_key} != {valid_key}",
params={
log_const.KEY_NAME: "check_kafka_key_validation",
MESSAGE_ID_STR: from_message.incremental_id,
UID_STR: uid
},
user=user,
level="WARNING")
except:
log(f"Exception to check Kafka message key {message_key}",
params={
log_const.KEY_NAME: "check_kafka_key_error",
MESSAGE_ID_STR: from_message.incremental_id,
UID_STR: uid
},
user=user,
level="ERROR")
def __init__(self,
id,
message,
db_data,
settings,
descriptions,
parametrizer_cls,
load_error=False):
self.settings = settings
try:
user_values = json.loads(db_data) if db_data else None
except ValueError:
user_values = None
smart_kit_metrics.counter_load_error(settings.app_name)
log(f"%(class_name)s.__init__ User load ValueError %(uid)s with user data {db_data}",
params={
log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION,
"uid": str(id)
},
level="ERROR")
load_error = True
super(User, self).__init__(id, message, user_values, descriptions,
load_error)
self.__parametrizer_cls = parametrizer_cls
self.do_not_save = False
self.initial_db_data = db_data
def get_reply(self, user, text_preprocessing_result, reply_actions, field,
form):
action_params = {}
if field:
field.set_available()
actions = field.description.requests
params = {
log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE,
"field": field.description.id
}
message = "Ask question on field: %(field)s"
log(message, user, params)
action_params[REQUEST_FIELD] = {
"type": field.description.type,
"id": field.description.id
}
action_messages = self.get_action_results(
user, text_preprocessing_result, actions, action_params)
else:
actions = reply_actions
params = {
log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE,
"id": self.id
}
message = "Finished scenario: %(id)s"
log(message, user, params)
user.preprocessing_messages_for_scenarios.clear()
action_messages = self.get_action_results(
user, text_preprocessing_result, actions, action_params)
return action_messages
def on_answer_error(self, message, user):
user.do_not_save = True
smart_kit_metrics.counter_exception(self.app_name)
params = {
log_const.KEY_NAME: log_const.DIALOG_ERROR_VALUE,
"message_id": user.message.incremental_id
}
log("exc_handler: Failed to process message. Exception occurred. Fail in MESSAGE: {}"
.format(user.message.masked_value),
user,
params,
level="ERROR",
exc_info=True)
callback_action_params = get_callback_action_params(user)
exc_type, exc_value, exc_traceback = sys.exc_info()
error = '\n'.join(
traceback.format_exception(exc_type, exc_value, exc_traceback))
if user.settings["template_settings"].get("debug_info"):
set_debug_info(self.app_name, callback_action_params, error)
exception_action = user.descriptions["external_actions"][
"exception_action"]
commands = exception_action.run(user=user,
text_preprocessing_result=None,
params=callback_action_params)
return commands
def success(self, callback_id: str):
log(f"behavior.success started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.",
self._user,
params={
log_const.KEY_NAME: log_const.BEHAVIOR_SUCCESS_VALUE,
log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id
})
callback = self._get_callback(callback_id)
result = None
if callback:
self._check_hostname(callback_id, callback)
self._add_returned_callback(callback_id)
behavior = self.descriptions[callback.behavior_id]
callback_action_params = callback.action_params
self._log_callback(
callback_id,
"behavior_success",
smart_kit_metrics.counter_behavior_success,
"success",
callback_action_params,
)
text_preprocessing_result = TextPreprocessingResult(
callback.text_preprocessing_result)
result = behavior.success_action.run(self._user,
text_preprocessing_result,
callback_action_params)
self._delete(callback_id)
return result
def on_extract_error(self, text_preprocessing_result, user, params=None):
log("exc_handler: ElseFiller failed to extract. Return None. MESSAGE: {}."
.format(user.message.masked_value),
user, {log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE},
level="ERROR",
exc_info=True)
return None
def __init__(self,
id,
message,
db_data,
settings,
descriptions,
parametrizer_cls,
load_error=False):
self.settings = settings
try:
user_values = json.loads(db_data) if db_data else None
except ValueError:
user_values = None
load_error = True
super(User, self).__init__(id, message, user_values, descriptions,
load_error)
self.__parametrizer_cls = parametrizer_cls
self.do_not_save = False
self.initial_db_data = db_data
db_version = self.variables.get(self.USER_DB_VERSION, default=0)
self.variables.set(self.USER_DB_VERSION, db_version + 1)
log(
"%(class_name)s.__init__ USER %(uid)s LOAD db_version = %(db_version)s.",
self, {
"db_version": str(db_version),
"uid": str(self.id)
})
def process_message(self, message: SmartAppFromMessage, *args, **kwargs):
stats = ""
log("INCOMING DATA: %(masked_message)s",
params={
log_const.KEY_NAME: "incoming_policy_message",
"masked_message": message.masked_value,
"message_id": message.incremental_id,
})
db_uid = message.db_uid
with StatsTimer() as load_timer:
user = self.load_user(db_uid, message)
stats += "Loading time: {} msecs\n".format(load_timer.msecs)
with StatsTimer() as script_timer:
commands = self.model.answer(message, user)
if commands:
answer = self._generate_answers(user, commands, message)
else:
answer = None
stats += "Script time: {} msecs\n".format(script_timer.msecs)
with StatsTimer() as save_timer:
self.save_user(db_uid, user, message)
stats += "Saving time: {} msecs\n".format(save_timer.msecs)
log(stats, user=user, params={log_const.KEY_NAME: "timings"})
self.postprocessor.postprocess(user, message)
return answer, stats
def on_check_error(self, text_preprocessing_result, user):
log("exc_handler: Requirement failed to check. Return False. MESSAGE: {}."
.format(user.message.masked_value),
user, {log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE},
level="ERROR",
exc_info=True)
return False
def run(self):
self._server = make_server('0.0.0.0', 8000, self.iterate)
log('''
Application start via "python manage.py run_app" recommended only for local testing.
For production it is recommended to start using "gunicorn --config wsgi_config.py 'wsgi:create_app()'
''',
level="WARNING")
self._server.serve_forever()
