def iterate(self, environ, start_response):
stats = ""
with StatsTimer() as poll_timer:
try:
content_length = int(environ.get('CONTENT_LENGTH', '0'))
request = environ["wsgi.input"].read(content_length).decode()
headers = self._get_headers(environ)
except KeyError:
log("Error in request data", level="ERROR")
raise Exception("Error in request data")
stats += "Polling time: {} msecs\n".format(poll_timer.msecs)
message = SmartAppFromMessage(request,
headers=headers,
headers_required=False)
if not message.validate():
start_response("400 BAD REQUEST",
self._get_outgoing_headers(headers))
return [b'{"message": "invalid message"}']
answer, stats = self.handle_message(message, stats)
with StatsTimer() as publish_timer:
if not answer:
start_response("204 NO CONTENT",
self._get_outgoing_headers(headers))
return [b'{"message": "no answer"}']
start_response("200 OK",
self._get_outgoing_headers(headers, answer))
answer = SmartAppToMessage(answer, message, request=None)
stats += "Publishing time: {} msecs".format(publish_timer.msecs)
log(stats, params={log_const.KEY_NAME: "timings"})
return [answer.value.encode()]
def _get_timeout_from_message(self, orig_message_raw, callback_id,
headers):
timeout_from_message = SmartAppFromMessage(
orig_message_raw,
headers=headers,
masking_fields=self.masking_fields)
timeout_from_message.callback_id = callback_id
return timeout_from_message
def test_valid_false(self):
input_msg = {
"uuid": {"userChannel": "web", "userId": 99, "chatId": 80},
"payload": "some payload"
}
headers = [('test_header', 'result')]
json_input_msg = json.dumps(input_msg, ensure_ascii=False)
message = SmartAppFromMessage(json_input_msg, headers=headers)
self.assertFalse(message.validate())
def _get_timeout_from_message(self, orig_message_raw, callback_id,
headers):
orig_message_raw = json.dumps(orig_message_raw)
timeout_from_message = SmartAppFromMessage(
orig_message_raw,
headers=headers,
masking_fields=self.masking_fields,
validators=self.from_msg_validators)
timeout_from_message.callback_id = callback_id
return timeout_from_message
def test_valid_true(self):
input_msg = {
"messageId": 2,
"sessionId": 234,
"uuid": {"userChannel": "web", "userId": 99, "chatId": 80},
"payload": {"key": "some payload"},
"messageName": "some_type"
}
json_input_msg = json.dumps(input_msg, ensure_ascii=False)
headers = [('test_header', 'result')]
message = SmartAppFromMessage(json_input_msg, headers=headers)
self.assertTrue(message.validate())
def handle_message(
self, message: SmartAppFromMessage
) -> typing.Tuple[int, str, SmartAppToMessage]:
if not message.validate():
result = 400, "BAD REQUEST", SmartAppToMessage(
self.BAD_REQUEST_COMMAND,
message=message,
request=None,
)
try:
result[2].as_dict
except (json.JSONDecodeError, KeyError):
result = 400, "BAD REQUEST", SmartAppToMessage(
self.BAD_REQUEST_COMMAND,
message=basic_error_message,
request=None,
)
finally:
return result
answer, stats = self.process_message(message)
if not answer:
return 204, "NO CONTENT", SmartAppToMessage(self.NO_ANSWER_COMMAND,
message=message,
request=None)
answer_message = SmartAppToMessage(answer,
message,
request=None,
validators=self.to_msg_validators)
if answer_message.validate():
return 200, "OK", answer_message
else:
return 500, "BAD ANSWER", SmartAppToMessage(
self.BAD_ANSWER_COMMAND, message=message, request=None)
def process_message(self, raw_message: str, headers: tuple = ()) -> typing.Tuple[typing.Any, list]:
masking_fields = self.settings["template_settings"].get("masking_fields")
message = SmartAppFromMessage(raw_message, headers=headers, masking_fields=masking_fields)
user = self.__user_cls(self.environment.user_id, message, self.user_data, self.settings,
self.app_model.scenario_descriptions,
self.__parametrizer_cls, load_error=False)
answers = self.app_model.answer(message, user)
return user, answers or []
def process_message(self, raw_message: str, headers: tuple = ()) -> typing.Tuple[typing.Any, list]:
from smart_kit.configs import get_app_config
masking_fields = self.settings["template_settings"].get("masking_fields")
message = SmartAppFromMessage(raw_message, headers=headers, masking_fields=masking_fields,
validators=get_app_config().FROM_MSG_VALIDATORS)
user = self.__user_cls(self.environment.user_id, message, self.user_data, self.settings,
self.app_model.scenario_descriptions,
self.__parametrizer_cls, load_error=False)
answers = self.app_model.answer(message, user)
return user, answers or []
def process_message(self, mq_message, consumer, kafka_key, stats):
# ну тут чутка копипасты
mutex = None
try:
message_value = mq_message.value()
message = SmartAppFromMessage(message_value,
headers=mq_message.headers(),
masking_fields=self.masking_fields)
if message.validate():
mutex = self._locks.setdefault(message.db_uid,
threading.Lock())
mutex.acquire()
super().process_message(mq_message, consumer, kafka_key, stats)
except Exception:
if mutex and mutex.locked():
mutex.release()
raise
if mutex and mutex.locked():
mutex.release()
async def iterate(self, request: aiohttp.web.Request):
headers = self._get_headers(request.headers)
body = await request.text()
message = SmartAppFromMessage(body, headers=headers, headers_required=False,
validators=self.from_msg_validators)
status, reason, answer = await self.handle_message(message)
return aiohttp.web.json_response(
status=status, reason=reason, data=answer.as_dict,
headers=self._get_outgoing_headers(headers, answer.command)
)
def test_validation(self):
input_msg = {
"uuid": {
"userChannel": "web",
"userId": 99,
"chatId": 80
},
"messageName": "some_name",
"messageId": "random_id",
"sessionId": "random_id",
"payload": {
"pi": 3.14159265358979,
}
}
headers = [('test_header', 'result')]
message = SmartAppFromMessage(json.dumps(input_msg,
ensure_ascii=False),
headers=headers,
validators=(PieMessageValidator(), ))
self.assertTrue(message.validate())
input_msg["payload"]["pi"] = 2.7182818284
message = SmartAppFromMessage(json.dumps(input_msg,
ensure_ascii=False),
headers=headers,
validators=(PieMessageValidator(), ))
self.assertFalse(message.validate())
def handle_message(
self, message: SmartAppFromMessage
) -> typing.Tuple[int, str, SmartAppToMessage]:
if not message.validate():
return 400, "BAD REQUEST", SmartAppToMessage(
self.BAD_REQUEST_COMMAND, message=message, request=None)
answer, stats = self.process_message(message)
if not answer:
return 204, "NO CONTENT", SmartAppToMessage(self.NO_ANSWER_COMMAND,
message=message,
request=None)
return 200, "OK", SmartAppToMessage(answer, message, request=None)
def test_1(self):
input_msg = {
"messageId": 2,
"uuid": {"userChannel": "B2C", "userId": "userId", "sub": "sub"},
"payload": {
"message": {
"original_text": "сверни приложение"
},
"device": {
"platformType": "android",
"platformVersion": "9",
"surface": "STARGATE",
"surfaceVersion": "1.56.20200828144304",
"features": {"appTypes": ["APK", "WEB_APP", "DIALOG"]},
"capabilities": {"mic": {"available": True},
"screen": {"available": True},
"speak": {"available": True}},
"deviceId": "34534545345345",
"deviceManufacturer": "SberDevices",
"deviceModel": "stargate",
"additionalInfo": {}
}
},
"messageName": "MESSAGE_TO_SKILL"
}
json_input_msg = json.dumps(input_msg, ensure_ascii=False)
topic = "test"
headers = []
current_time = current_time_ms()
message = SmartAppFromMessage(value=json_input_msg, topic_key=topic, headers=headers, creation_time=current_time)
self.assertAlmostEqual(message.creation_time, current_time)
self.assertEqual(2, message.incremental_id)
self.assertEqual(input_msg["uuid"]["userChannel"], message.channel)
self.assertEqual(input_msg["messageName"], message.type)
self.assertEqual(input_msg["uuid"]["userId"], message.uid)
self.assertEqual(json_input_msg, message.value)
self.assertEqual("sub_userId_B2C", message.db_uid)
self.assertDictEqual(input_msg["uuid"], message.uuid)
self.assertDictEqual(input_msg["payload"], message.payload)
device = input_msg["payload"]["device"]
self.assertEqual(device["platformType"], message.device.platform_type)
self.assertEqual(device["platformVersion"], message.device.platform_version)
self.assertEqual(device["surface"], message.device.surface)
self.assertEqual(device["surfaceVersion"], message.device.surface_version)
self.assertEqual(device["features"], message.device.features)
self.assertEqual(device["capabilities"], message.device.capabilities)
self.assertEqual(device["additionalInfo"], message.device.additional_info)
self.assertEqual(topic, message.topic_key)
async def handle_message(self, message: SmartAppFromMessage) -> typing.Tuple[int, str, SmartAppToMessage]:
if not message.validate():
return 400, "BAD REQUEST", SmartAppToMessage(self.BAD_REQUEST_COMMAND, message=message, request=None)
answer, stats = await self.process_message(message)
if not answer:
return 204, "NO CONTENT", SmartAppToMessage(self.NO_ANSWER_COMMAND, message=message, request=None)
answer_message = SmartAppToMessage(
answer, message, request=None,
validators=self.to_msg_validators)
if answer_message.validate():
return 200, "OK", answer_message
else:
return 500, "BAD ANSWER", SmartAppToMessage(self.BAD_ANSWER_COMMAND, message=message, request=None)
def iterate(self, environ, start_response):
try:
content_length = int(environ.get('CONTENT_LENGTH', '0'))
body = environ["wsgi.input"].read(content_length).decode()
headers = self._get_headers(environ)
except KeyError:
log("Error in request data", level="ERROR")
raise Exception("Error in request data")
message = SmartAppFromMessage(body,
headers=headers,
headers_required=False)
status, reason, answer = self.handle_message(message)
start_response(f"{status} {reason}",
self._get_outgoing_headers(headers, answer.command))
return [answer.value.encode()]
def create_message(self, data, headers=None):
defaults = Environment().as_dict
predefined_fields = data.get("predefined_fields")
is_payload_field = data.get("payload")
if predefined_fields:
predefined_fields_data = self.storaged_predefined_fields[
predefined_fields]
if not is_payload_field and not predefined_fields_data.get(
"payload"):
predefined_fields_data = {"payload": predefined_fields_data}
if is_payload_field and predefined_fields_data.get("payload"):
raise Exception(
"Payload field is in test case and in predefined_fields object, check it!"
)
defaults.update(predefined_fields_data)
del data["predefined_fields"]
message = data.get("message")
if message:
defaults["payload"].update({"message": message})
defaults.update(data)
return SmartAppFromMessage(json.dumps(defaults), headers=headers)
def iterate(self, kafka_key):
consumer = self.consumers[kafka_key]
mq_message = None
message_value = None
try:
mq_message = None
message_value = None
stats = ""
with StatsTimer() as poll_timer:
mq_message = consumer.poll()
if mq_message:
stats += "Polling time: {} msecs\n".format(poll_timer.msecs)
topic_key = self._get_topic_key(mq_message)
save_tries = 0
user_save_no_collisions = False
user = None
db_uid = None
while save_tries < self.user_save_collisions_tries and not user_save_no_collisions:
save_tries += 1
message_value = mq_message.value()
message = SmartAppFromMessage(
message_value,
headers=mq_message.headers(),
masking_fields=self.masking_fields)
# TODO вернуть проверку ключа!!!
if message.validate(
): # and self.check_message_key(message, mq_message.key()):
log("INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s",
params={
log_const.KEY_NAME: "incoming_message",
"topic": mq_message.topic(),
"message_partition": mq_message.partition(),
"message_key": mq_message.key(),
"kafka_key": kafka_key,
"incoming_data": str(message.masked_value),
"headers": str(mq_message.headers()),
MESSAGE_ID_STR: message.incremental_id
})
db_uid = message.db_uid
span = jaeger_utils.get_incoming_spam(
self.tracer, message, mq_message)
with self.tracer.scope_manager.activate(span,
True) as scope:
with StatsTimer() as load_timer:
user = self.load_user(db_uid, message)
with self.tracer.scope_manager.activate(span,
True) as scope:
with self.tracer.start_span(
'Loading time',
child_of=scope.span) as span:
smart_kit_metrics.sampling_load_time(
self.app_name, load_timer.secs)
stats += "Loading time: {} msecs\n".format(
load_timer.msecs)
with StatsTimer() as script_timer:
commands = self.model.answer(message, user)
with self.tracer.start_span(
'Script time',
child_of=scope.span) as span:
answers = self._generate_answers(
user=user,
commands=commands,
message=message,
topic_key=topic_key,
kafka_key=kafka_key)
smart_kit_metrics.sampling_script_time(
self.app_name, script_timer.secs)
stats += "Script time: {} msecs\n".format(
script_timer.msecs)
with self.tracer.start_span(
'Saving time',
child_of=scope.span) as span:
with StatsTimer() as save_timer:
user_save_no_collisions = self.save_user(
db_uid, user, message)
smart_kit_metrics.sampling_save_time(
self.app_name, save_timer.secs)
stats += "Saving time: {} msecs\n".format(
save_timer.msecs)
if not user_save_no_collisions:
log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME:
"ignite_collision",
"db_uid":
db_uid,
"message_key":
mq_message.key(),
"message_partition":
mq_message.partition(),
"kafka_key":
kafka_key,
"uid":
user.id,
"db_version":
str(
user.variables.get(
user.USER_DB_VERSION))
},
level="WARNING")
continue
self.save_behavior_timeouts(
user, mq_message, kafka_key)
if mq_message.headers() is None:
mq_message.set_headers([])
self.tracer.inject(span_context=span.context,
format=jaeger_kafka_codec.KAFKA_MAP,
carrier=mq_message.headers())
if answers:
for answer in answers:
with StatsTimer() as publish_timer:
self._send_request(user, answer,
mq_message)
stats += "Publishing time: {} msecs".format(
publish_timer.msecs)
log(stats)
else:
try:
data = message.masked_value
except:
data = "<DATA FORMAT ERROR>"
log(f"Message validation failed, skip message handling.",
params={
log_const.KEY_NAME: "invalid_message",
"data": data
},
level="ERROR")
smart_kit_metrics.counter_invalid_message(
self.app_name)
if user and not user_save_no_collisions:
log("MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME:
"ignite_collision",
"db_uid":
db_uid,
"message_key":
mq_message.key(),
"message_partition":
mq_message.partition(),
"kafka_key":
kafka_key,
"uid":
user.id,
"db_version":
str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
smart_kit_metrics.counter_save_collision_tries_left(
self.app_name)
consumer.commit_offset(mq_message)
except KafkaException as kafka_exp:
log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(
message_value),
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"kafka_exp": str(kafka_exp),
log_const.REQUEST_VALUE: str(message_value)
},
level="ERROR",
exc_info=True)
except Exception:
try:
log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}."
.format(message_value),
params={
log_const.KEY_NAME: log_const.STARTUP_VALUE,
"kafka_key": kafka_key
},
level="ERROR",
exc_info=True)
consumer.commit_offset(mq_message)
except Exception:
log("Error handling worker fail exception.",
level="ERROR",
exc_info=True)
def process_message(self, mq_message, consumer, kafka_key, stats):
topic_key = self._get_topic_key(mq_message, kafka_key)
save_tries = 0
user_save_no_collisions = False
user = None
db_uid = None
message = None
while save_tries < self.user_save_collisions_tries and not user_save_no_collisions:
save_tries += 1
message_value = mq_message.value()
message = SmartAppFromMessage(
message_value,
headers=mq_message.headers(),
masking_fields=self.masking_fields,
creation_time=consumer.get_msg_create_time(mq_message))
# TODO вернуть проверку ключа!!!
if message.validate():
waiting_message_time = 0
if message.creation_time:
waiting_message_time = time.time(
) * 1000 - message.creation_time
stats += "Waiting message: {} msecs\n".format(
waiting_message_time)
stats += "Mid: {}\n".format(message.incremental_id)
smart_kit_metrics.sampling_mq_waiting_time(
self.app_name, waiting_message_time / 1000)
self.check_message_key(message, mq_message.key(), user)
log("INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s",
params={
log_const.KEY_NAME: "incoming_message",
"topic": mq_message.topic(),
"message_partition": mq_message.partition(),
"message_key": mq_message.key(),
"message_id": message.incremental_id,
"kafka_key": kafka_key,
"incoming_data": str(message.masked_value),
"length": len(message.value),
"headers": str(mq_message.headers()),
"waiting_message": waiting_message_time,
"surface": message.device.surface,
MESSAGE_ID_STR: message.incremental_id
},
user=user)
db_uid = message.db_uid
with StatsTimer() as load_timer:
user = self.load_user(db_uid, message)
smart_kit_metrics.sampling_load_time(self.app_name,
load_timer.secs)
stats += "Loading time: {} msecs\n".format(load_timer.msecs)
with StatsTimer() as script_timer:
commands = self.model.answer(message, user)
answers = self._generate_answers(user=user,
commands=commands,
message=message,
topic_key=topic_key,
kafka_key=kafka_key)
smart_kit_metrics.sampling_script_time(self.app_name,
script_timer.secs)
stats += "Script time: {} msecs\n".format(script_timer.msecs)
with StatsTimer() as save_timer:
user_save_no_collisions = self.save_user(
db_uid, user, message)
smart_kit_metrics.sampling_save_time(self.app_name,
save_timer.secs)
stats += "Saving time: {} msecs\n".format(save_timer.msecs)
if not user_save_no_collisions:
log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME:
"ignite_collision",
"db_uid":
db_uid,
"message_key":
mq_message.key(),
"message_partition":
mq_message.partition(),
"kafka_key":
kafka_key,
"uid":
user.id,
"db_version":
str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
continue
self.save_behavior_timeouts(user, mq_message, kafka_key)
if mq_message.headers() is None:
mq_message.set_headers([])
if answers:
for answer in answers:
with StatsTimer() as publish_timer:
self._send_request(user, answer, mq_message)
stats += "Publishing time: {} msecs".format(
publish_timer.msecs)
log(stats, user=user)
else:
try:
data = message.masked_value
except:
data = "<DATA FORMAT ERROR>"
log(f"Message validation failed, skip message handling.",
params={
log_const.KEY_NAME: "invalid_message",
"data": data
},
level="ERROR")
smart_kit_metrics.counter_invalid_message(self.app_name)
if user and not user_save_no_collisions:
log("MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.",
user=user,
params={
log_const.KEY_NAME: "ignite_collision",
"db_uid": db_uid,
"message_key": mq_message.key(),
"message_partition": mq_message.partition(),
"kafka_key": kafka_key,
"uid": user.id,
"db_version": str(user.variables.get(user.USER_DB_VERSION))
},
level="WARNING")
self.postprocessor.postprocess(user, message)
smart_kit_metrics.counter_save_collision_tries_left(self.app_name)
consumer.commit_offset(mq_message)
def create_message(data, headers=None):
defaults = Environment().as_dict
defaults.update(data)
return SmartAppFromMessage(json.dumps(defaults), headers=headers)