def run(
self,
user: BaseUser,
text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None
) -> List[Command]:
params = user.parametrizer.collect(
text_preprocessing_result, filter_params={"command": self.command})
answer_params = dict()
result = []
nodes = self.nodes.items() if self.nodes else []
for key, template in nodes:
if template:
choice_index = random.randint(0, len(template) - 1)
rendered = self._get_rendered_tree(template[choice_index],
params, self.no_empty_nodes)
if rendered != "" or not self.no_empty_nodes:
answer_params[key] = rendered
if answer_params:
result = [
Command(self.command,
answer_params,
self.id,
request_type=self.request_type,
request_data=self.request_data)
]
return result
def run(
self,
user: User,
text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None
) -> List[Command]:
params = params or {}
command_params = {
"surface":
self.surface,
"content":
self._generate_command_context(user, text_preprocessing_result,
params),
"project_id":
user.settings["template_settings"]["project_id"]
}
requests_data = self._render_request_data(params)
commands = [
Command(self.command,
command_params,
self.id,
request_type=self.request_type,
request_data=requests_data)
]
return commands
def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]:
result = []
params = user.parametrizer.collect(text_preprocessing_result, filter_params={self.COMMAND: self.command})
rendered = self._get_rendered_tree(self.nodes[self.STATIC], params, self.no_empty_nodes)
if self._nodes[self.RANDOM_CHOICE]:
random_node = random.choice(self.nodes[self.RANDOM_CHOICE])
rendered_random = self._get_rendered_tree(random_node, params, self.no_empty_nodes)
rendered.update(rendered_random)
out = {}
for item in self.items:
if item.requirement.check(text_preprocessing_result, user):
out.setdefault(self.ITEMS, []).append(item.render(rendered))
if self._suggests_template is not None:
out[self.SUGGESTIONS] = self._get_rendered_tree(self.nodes[self.SUGGESTIONS_TEMPLATE], params, self.no_empty_nodes)
else:
for suggest in self.suggests:
if suggest.requirement.check(text_preprocessing_result, user):
data_dict = out.setdefault(self.SUGGESTIONS, {self.BUTTONS: []})
buttons = data_dict[self.BUTTONS]
rendered_text = suggest.render(rendered)
buttons.append(rendered_text)
for part in self.root:
if part.requirement.check(text_preprocessing_result, user):
out.update(part.render(rendered))
if rendered or not self.no_empty_nodes:
result = [Command(self.command, out, self.id)]
return result
def test_1(self):
expected = {"message_name": "my_name", "payload": {}}
command = Command("my_name")
result = command.raw
self.assertDictEqual(expected, result)
def test_string_action(self):
expected = [
Command("cmd_id", {
"item": "template",
"params": "params"
})
]
user = MagicMock()
template = Mock()
template.get_template = Mock(
return_value=["nlpp.payload.personInfo.identityCard"])
user.descriptions = {"render_templates": template}
params = {"params": "params"}
user.parametrizer = MockSimpleParametrizer(user, {"data": params})
items = {
"command": "cmd_id",
"nodes": {
"item": "template",
"params": "{{params}}"
}
}
action = StringAction(items)
result = action.run(user, None)
self.assertEqual(expected[0].name, result[0].name)
self.assertEqual(expected[0].payload, result[0].payload)
def test_2(self):
expected = {"messageName": "my_name", "payload": {"id": 5}}
command = Command("my_name", {"id": 5})
result = command.raw
self.assertDictEqual(expected, result)
def run(
self,
user: BaseUser,
text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None
) -> List[Command]:
# Example: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}})
command_params = dict()
params = params or {}
collected = user.parametrizer.collect(
text_preprocessing_result, filter_params={"command": self.command})
params.update(collected)
for key, value in self.nodes.items():
rendered = self._get_rendered_tree(value, params,
self.no_empty_nodes)
if rendered != "" or not self.no_empty_nodes:
command_params[key] = rendered
commands = [
Command(self.command,
command_params,
self.id,
request_type=self.request_type,
request_data=self.request_data)
]
return commands
def combine_answer_to_user(commands: typing.List[Command]) -> Command:
answer = Command(name=ANSWER_TO_USER, request_data=commands[0].request_data, request_type=commands[0].request_type)
summary_pronounce_text = []
auto_listening = None
for command in commands:
if command.request_data != answer.request_data:
raise ValueError(f"Cant combine {ANSWER_TO_USER} commands, request_data is different")
if command.request_type != answer.request_type:
raise ValueError(f"Cant combine {ANSWER_TO_USER} commands, request_type is different")
payload = command.payload
pronounce_text = payload.pop(field.PRONOUNCE_TEXT, None)
items = payload.pop(field.ITEMS, None)
if auto_listening is None:
auto_listening = payload.get(field.AUTO_LISTENING, None)
if pronounce_text:
summary_pronounce_text.append(pronounce_text)
if items is not None:
answer.payload.setdefault(field.ITEMS, []).extend(items)
answer.payload.update(payload)
if summary_pronounce_text:
answer.payload[field.PRONOUNCE_TEXT] = " ".join(summary_pronounce_text)
answer.payload[field.AUTO_LISTENING] = auto_listening
return answer
def _run(self, user, text_preprocessing_result, params=None):
action_params = copy.copy(params or {})
command_params = dict()
collected = user.parametrizer.collect(text_preprocessing_result, filter_params={"command": self.command})
action_params.update(collected)
scenario = None
if self._check_scenario:
scenario = user.last_scenarios.last_scenario_name
for key, value in self.nodes.items():
rendered = self._get_rendered_tree(value, action_params, self.no_empty_nodes)
if rendered != "" or not self.no_empty_nodes:
command_params[key] = rendered
callback_id = user.message.generate_new_callback_id()
request_data = copy.copy(self.request_data or {})
request_data.update(self._get_extra_request_data(user, params, callback_id))
save_params = self._get_save_params(user, action_params, command_params)
self._save_behavior(callback_id, user, scenario, text_preprocessing_result, save_params)
commands = [Command(self.command, command_params, self.id, request_type=self.request_type,
request_data=request_data)]
return commands
def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]:
result = []
params = user.parametrizer.collect(text_preprocessing_result, filter_params={"command": self.command})
rendered = self._get_rendered_tree(self.nodes, params, self.no_empty_nodes)
for j in self.RANDOM_PATH:
self.random_by_path(rendered, j)
if rendered or not self.no_empty_nodes:
result = [Command(self.command, rendered, self.id)]
return result
def run(
self,
user: BaseUser,
text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None
) -> List[Command]:
commands = [
Command(self.command,
self.nodes,
self.id,
request_type=self.request_type,
request_data=self.request_data)
]
return commands
def run(
self,
user: BaseUser,
text_preprocessing_result: BaseTextPreprocessingResult,
params: Optional[Dict[str, Union[str, float, int]]] = None
) -> List[Command]:
# Example: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}})
params = params or {}
command_params = self._generate_command_context(
user, text_preprocessing_result, params)
commands = [
Command(self.command,
command_params,
self.id,
request_type=self.request_type,
request_data=self.request_data)
]
return commands
class BaseHttpMainLoop(BaseMainLoop):
HEADER_START_WITH = "HTTP_SMART_APP_"
BAD_REQUEST_COMMAND = Command(message_names.ERROR, {
"code": -1,
"description": "Invalid Request Message"
})
NO_ANSWER_COMMAND = Command(message_names.NOTHING_FOUND)
BAD_ANSWER_COMMAND = Command(message_names.ERROR, {
"code": -1,
"description": "Invalid Answer Message"
})
def run(self):
raise NotImplementedError
def stop(self, signum, frame):
raise NotImplementedError
def handle_message(
self, message: SmartAppFromMessage
) -> typing.Tuple[int, str, SmartAppToMessage]:
if not message.validate():
result = 400, "BAD REQUEST", SmartAppToMessage(
self.BAD_REQUEST_COMMAND,
message=message,
request=None,
)
try:
result[2].as_dict
except (json.JSONDecodeError, KeyError):
result = 400, "BAD REQUEST", SmartAppToMessage(
self.BAD_REQUEST_COMMAND,
message=basic_error_message,
request=None,
)
finally:
return result
answer, stats = self.process_message(message)
if not answer:
return 204, "NO CONTENT", SmartAppToMessage(self.NO_ANSWER_COMMAND,
message=message,
request=None)
answer_message = SmartAppToMessage(answer,
message,
request=None,
validators=self.to_msg_validators)
if answer_message.validate():
return 200, "OK", answer_message
else:
return 500, "BAD ANSWER", SmartAppToMessage(
self.BAD_ANSWER_COMMAND, message=message, request=None)
def process_message(self, message: SmartAppFromMessage, *args, **kwargs):
stats = ""
log("INCOMING DATA: %(masked_message)s",
params={
log_const.KEY_NAME: "incoming_policy_message",
"masked_message": message.masked_value,
"message_id": message.incremental_id,
})
db_uid = message.db_uid
with StatsTimer() as load_timer:
user = self.load_user(db_uid, message)
stats += "Loading time: {} msecs\n".format(load_timer.msecs)
with StatsTimer() as script_timer:
commands = self.model.answer(message, user)
if commands:
answer = self._generate_answers(user, commands, message)
else:
answer = None
stats += "Script time: {} msecs\n".format(script_timer.msecs)
with StatsTimer() as save_timer:
self.save_user(db_uid, user, message)
stats += "Saving time: {} msecs\n".format(save_timer.msecs)
log(stats, user=user, params={log_const.KEY_NAME: "timings"})
self.postprocessor.postprocess(user, message)
return answer, stats
def _get_headers(self, environ):
return [(key, value) for key, value in environ.items()
if key.startswith(self.HEADER_START_WITH)]
# noinspection PyMethodMayBeStatic
def _get_outgoing_headers(self, incoming_headers, command=None):
headers = {"CONTENT_TYPE": "application/json"}
headers.update(incoming_headers)
if command:
callback_id = command.request_data.get(CALLBACK_ID_HEADER)
if callback_id:
headers[CALLBACK_ID_HEADER] = callback_id
return list(headers.items())
def _generate_answers(self, user, commands, message, **kwargs):
commands = combine_commands(commands, user)
if len(commands) > 1:
raise ValueError
answer = commands.pop() if commands else None
return answer
class MainLoop(BaseMainLoop):
MAX_LOG_TIME = 20
BAD_ANSWER_COMMAND = Command(message_names.ERROR, {
"code": -1,
"description": "Invalid Answer Message"
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
log("%(class_name)s.__init__ started.",
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"class_name": self.__class__.__name__
})
try:
kafka_config = _enrich_config_from_secret(
self.settings["kafka"]["template-engine"],
self.settings.get("secret_kafka", {}))
consumers = {}
publishers = {}
log("%(class_name)s START CONSUMERS/PUBLISHERS CREATE",
params={"class_name": self.__class__.__name__},
level="WARNING")
for key, config in kafka_config.items():
if config.get("consumer"):
consumers.update({key: KafkaConsumer(kafka_config[key])})
if config.get("publisher"):
publishers.update({key: KafkaPublisher(kafka_config[key])})
log("%(class_name)s FINISHED CONSUMERS/PUBLISHERS CREATE",
params={"class_name": self.__class__.__name__},
level="WARNING")
self.app_name = self.settings.app_name
self.consumers = consumers
for key in self.consumers:
self.consumers[key].subscribe()
self.publishers = publishers
self.behaviors_timeouts_value_cls = namedtuple(
'behaviors_timeouts_value',
'db_uid, callback_id, mq_message, kafka_key')
self.behaviors_timeouts = HeapqKV(
value_to_key_func=lambda val: val.callback_id)
log("%(class_name)s.__init__ completed.",
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"class_name": self.__class__.__name__
})
except:
log("%(class_name)s.__init__ exception.",
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"class_name": self.__class__.__name__
},
level="ERROR",
exc_info=True)
raise
def pre_handle(self):
self.iterate_behavior_timeouts()
def run(self):
log("%(class_name)s.run started",
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"class_name": self.__class__.__name__
})
while self.is_work:
self.pre_handle()
for kafka_key in self.consumers:
self.iterate(kafka_key)
if self.health_check_server:
with StatsTimer() as health_check_server_timer:
self.health_check_server.iterate()
if health_check_server_timer.msecs >= self.MAX_LOG_TIME:
log("Health check iterate time: {} msecs\n".format(
health_check_server_timer.msecs),
params={
log_const.KEY_NAME: "slow_health_check",
"time_msecs": health_check_server_timer.msecs
},
level="WARNING")
log("Stopping Kafka handler", level="WARNING")
for kafka_key in self.consumers:
self.consumers[kafka_key].close()
log("Kafka consumer connection is closed", level="WARNING")
self.publishers[kafka_key].close()
log("Kafka publisher connection is closed", level="WARNING")
log("Kafka handler is stopped", level="WARNING")
def _generate_answers(self, user, commands, message, **kwargs):
topic_key = kwargs["topic_key"]
kafka_key = kwargs["kafka_key"]
answers = []
commands = commands or []
commands = combine_commands(commands, user)
for command in commands:
request = SmartKitKafkaRequest(id=None, items=command.request_data)
request.update_empty_items({
"topic_key": topic_key,
"kafka_key": kafka_key
})
to_message = get_to_message(command.name)
answer = to_message(command=command,
message=message,
request=request,
masking_fields=self.masking_fields,
validators=self.to_msg_validators)
if answer.validate():
answers.append(answer)
else:
answers.append(
SmartAppToMessage(self.BAD_ANSWER_COMMAND,
message=message,
request=request))
smart_kit_metrics.counter_outgoing(self.app_name, command.name,
answer, user)
return answers
def _get_timeout_from_message(self, orig_message_raw, callback_id,
headers):
orig_message_raw = json.dumps(orig_message_raw)
timeout_from_message = SmartAppFromMessage(
orig_message_raw,
headers=headers,
masking_fields=self.masking_fields,
validators=self.from_msg_validators)
timeout_from_message.callback_id = callback_id
return timeout_from_message
def iterate_behavior_timeouts(self):
now = time.time()
while now > (self.behaviors_timeouts.get_head_key() or float("inf")):
_, behavior_timeout_value = self.behaviors_timeouts.pop()
db_uid, callback_id, mq_message, kafka_key = behavior_timeout_value
try:
save_tries = 0
user_save_no_collisions = False
user = None
while save_tries < self.user_save_collisions_tries and not user_save_no_collisions:
save_tries += 1
orig_message_raw = json.loads(mq_message.value())
orig_message_raw[
SmartAppFromMessage.
MESSAGE_NAME] = message_names.LOCAL_TIMEOUT
timeout_from_message = self._get_timeout_from_message(
orig_message_raw,
callback_id,
headers=mq_message.headers())
user = self.load_user(db_uid, timeout_from_message)
commands = self.model.answer(timeout_from_message, user)
topic_key = self._get_topic_key(mq_message, kafka_key)
answers = self._generate_answers(
user=user,
commands=commands,
message=timeout_from_message,
topic_key=topic_key,
kafka_key=kafka_key)
user_save_no_collisions = self.save_user(
db_uid, user, mq_message)
if user and not user_save_no_collisions:
log("MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME:
"ignite_collision",
"db_uid":
db_uid,
"message_key":
mq_message.key(),
"kafka_key":
kafka_key,
"uid":
user.id,
"db_version":
str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
continue
if not user_save_no_collisions:
log("MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME:
"ignite_collision",
"db_uid":
db_uid,
"message_key":
mq_message.key(),
"message_partition":
mq_message.partition(),
"kafka_key":
kafka_key,
"uid":
user.id,
"db_version":
str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
smart_kit_metrics.counter_save_collision_tries_left(
self.app_name)
self.save_behavior_timeouts(user, mq_message, kafka_key)
for answer in answers:
self._send_request(user, answer, mq_message)
except:
log("%(class_name)s error.",
params={
log_const.KEY_NAME: "error_handling_timeout",
"class_name": self.__class__.__name__,
log_const.REQUEST_VALUE: str(mq_message.value())
},
level="ERROR",
exc_info=True)
def _get_topic_key(self, mq_message, kafka_key):
topic_names_2_key = self._topic_names_2_key(kafka_key)
return self.default_topic_key(kafka_key) or topic_names_2_key[
mq_message.topic()]
def process_message(self, mq_message, consumer, kafka_key, stats):
topic_key = self._get_topic_key(mq_message, kafka_key)
save_tries = 0
user_save_no_collisions = False
user = None
db_uid = None
message = None
while save_tries < self.user_save_collisions_tries and not user_save_no_collisions:
save_tries += 1
message_value = mq_message.value()
message = SmartAppFromMessage(
message_value,
headers=mq_message.headers(),
masking_fields=self.masking_fields,
creation_time=consumer.get_msg_create_time(mq_message))
# TODO вернуть проверку ключа!!!
if message.validate():
waiting_message_time = 0
if message.creation_time:
waiting_message_time = time.time(
) * 1000 - message.creation_time
stats += "Waiting message: {} msecs\n".format(
waiting_message_time)
stats += "Mid: {}\n".format(message.incremental_id)
smart_kit_metrics.sampling_mq_waiting_time(
self.app_name, waiting_message_time / 1000)
self.check_message_key(message, mq_message.key(), user)
log("INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s",
params={
log_const.KEY_NAME: "incoming_message",
"topic": mq_message.topic(),
"message_partition": mq_message.partition(),
"message_key": mq_message.key(),
"message_id": message.incremental_id,
"kafka_key": kafka_key,
"incoming_data": str(message.masked_value),
"length": len(message.value),
"headers": str(mq_message.headers()),
"waiting_message": waiting_message_time,
"surface": message.device.surface,
MESSAGE_ID_STR: message.incremental_id
},
user=user)
db_uid = message.db_uid
with StatsTimer() as load_timer:
user = self.load_user(db_uid, message)
smart_kit_metrics.sampling_load_time(self.app_name,
load_timer.secs)
stats += "Loading time: {} msecs\n".format(load_timer.msecs)
with StatsTimer() as script_timer:
commands = self.model.answer(message, user)
answers = self._generate_answers(user=user,
commands=commands,
message=message,
topic_key=topic_key,
kafka_key=kafka_key)
smart_kit_metrics.sampling_script_time(self.app_name,
script_timer.secs)
stats += "Script time: {} msecs\n".format(script_timer.msecs)
with StatsTimer() as save_timer:
user_save_no_collisions = self.save_user(
db_uid, user, message)
smart_kit_metrics.sampling_save_time(self.app_name,
save_timer.secs)
stats += "Saving time: {} msecs\n".format(save_timer.msecs)
if not user_save_no_collisions:
log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME:
"ignite_collision",
"db_uid":
db_uid,
"message_key":
mq_message.key(),
"message_partition":
mq_message.partition(),
"kafka_key":
kafka_key,
"uid":
user.id,
"db_version":
str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
continue
self.save_behavior_timeouts(user, mq_message, kafka_key)
if mq_message.headers() is None:
mq_message.set_headers([])
if answers:
for answer in answers:
with StatsTimer() as publish_timer:
self._send_request(user, answer, mq_message)
stats += "Publishing time: {} msecs".format(
publish_timer.msecs)
log(stats, user=user)
else:
try:
data = message.masked_value
except:
data = "<DATA FORMAT ERROR>"
log(f"Message validation failed, skip message handling.",
params={
log_const.KEY_NAME: "invalid_message",
"data": data
},
level="ERROR")
smart_kit_metrics.counter_invalid_message(self.app_name)
if user and not user_save_no_collisions:
log("MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME: "ignite_collision",
"db_uid": db_uid,
"message_key": mq_message.key(),
"message_partition": mq_message.partition(),
"kafka_key": kafka_key,
"uid": user.id,
"db_version": str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
self.postprocessor.postprocess(user, message)
smart_kit_metrics.counter_save_collision_tries_left(self.app_name)
consumer.commit_offset(mq_message)
def iterate(self, kafka_key):
consumer = self.consumers[kafka_key]
mq_message = None
message_value = None
try:
mq_message = None
message_value = None
with StatsTimer() as poll_timer:
mq_message = consumer.poll()
if mq_message:
stats = "Polling time: {} msecs\n".format(poll_timer.msecs)
message_value = mq_message.value() # DRY!
self.process_message(mq_message, consumer, kafka_key, stats)
except KafkaException as kafka_exp:
log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(
message_value),
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"kafka_exp": str(kafka_exp),
log_const.REQUEST_VALUE: str(message_value)
},
level="ERROR",
exc_info=True)
except Exception:
try:
log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}."
.format(message_value),
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"kafka_key": kafka_key
},
level="ERROR",
exc_info=True)
consumer.commit_offset(mq_message)
except Exception:
log("Error handling worker fail exception.",
level="ERROR",
exc_info=True)
def check_message_key(self, from_message, message_key, user):
sub = from_message.sub
channel = from_message.channel
uid = from_message.uid
message_key = message_key or b""
try:
params = [channel, sub, uid]
valid_key = ""
for value in params:
if value:
valid_key = "{}{}{}".format(
valid_key, "_",
value) if valid_key else "{}".format(value)
key_str = message_key.decode()
message_key_is_valid = key_str == valid_key
if not message_key_is_valid:
log(f"Failed to check Kafka message key {message_key} != {valid_key}",
params={
log_const.KEY_NAME: "check_kafka_key_validation",
MESSAGE_ID_STR: from_message.incremental_id,
UID_STR: uid
},
user=user,
level="WARNING")
except:
log(f"Exception to check Kafka message key {message_key}",
params={
log_const.KEY_NAME: "check_kafka_key_error",
MESSAGE_ID_STR: from_message.incremental_id,
UID_STR: uid
},
user=user,
level="ERROR")
def _send_request(self, user, answer, mq_message):
kafka_broker_settings = self.settings["template_settings"].get(
"route_kafka_broker") or []
request = answer.request
for kb_setting in kafka_broker_settings:
if (kb_setting["from_channel"] == answer.incoming_message.channel
and kb_setting["to_topic"] == request.topic_key):
request.kafka_key = kb_setting["route_to_broker"]
request_params = dict()
request_params["publishers"] = self.publishers
request_params["mq_message"] = mq_message
request_params["payload"] = answer.value
request_params["masked_value"] = answer.masked_value
request.run(answer.value, request_params)
self._log_request(user, request, answer, mq_message)
def _log_request(self, user, request, answer, original_mq_message):
log("OUTGOING TO TOPIC_KEY: %(topic_key)s DATA: %(data)s",
params={
log_const.KEY_NAME: "outgoing_message",
"topic_key": request.topic_key,
"headers": str(request._get_new_headers(original_mq_message)),
"data": answer.masked_value,
"length": len(answer.value)
},
user=user)
@lru_cache()
def _topic_names_2_key(self, kafka_key):
topics = self.settings["kafka"]["template-engine"][kafka_key][
"consumer"]["topics"]
return {name: key for key, name in topics.items()}
def default_topic_key(self, kafka_key):
return self.settings["kafka"]["template-engine"][kafka_key].get(
"default_topic_key")
@lazy
def masking_fields(self):
return self.settings["template_settings"].get("masking_fields")
def save_behavior_timeouts(self, user, mq_message, kafka_key):
for i, (expire_time_us, callback_id) in enumerate(
user.behaviors.get_behavior_timeouts()):
# two behaviors can be created in one query, so we need add some salt to make theirs key unique
unique_key = expire_time_us + i * 1e-5
log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout on %(unique_key)s",
params={
log_const.KEY_NAME: "adding_local_timeout",
"class_name": self.__class__.__name__,
"callback_id": callback_id,
"unique_key": unique_key
})
self.behaviors_timeouts.push(
unique_key,
self.behaviors_timeouts_value_cls._make(
(user.message.db_uid, callback_id, mq_message, kafka_key)))
for callback_id in user.behaviors.get_returned_callbacks():
log("%(class_name)s: removing local_timeout on callback %(callback_id)s",
params={
log_const.KEY_NAME: "removing_local_timeout",
"class_name": self.__class__.__name__,
"callback_id": callback_id
})
self.behaviors_timeouts.remove(callback_id)
def stop(self, signum, frame):
self.is_work = False
def run(self, user, text_preprocessing_result, params):
self.called = True
if self.command_name:
return [Command(self.command_name)]