def handle_version_id(req, res, result, version_id):
if version_id in _VERSIONS_TPL_DICT:
result['elements'].append(_parse_version(version_id, req))
res.body = rest_utils.as_json(result, sort_keys=True)
res.status = falcon.HTTP_200
else:
error_body = {'message': '%s is not valid version' % version_id}
res.body = rest_utils.as_json(error_body)
res.status = falcon.HTTP_400
def handle_version_id(req, res, result, version_id):
if version_id in _VERSIONS_TPL_DICT:
result['elements'].append(_parse_version(version_id, req))
res.body = rest_utils.as_json(result, sort_keys=True)
res.status = falcon.HTTP_200
else:
error_body = {'message': '%s is not valid version' % version_id}
res.body = rest_utils.as_json(error_body)
res.status = falcon.HTTP_400
def transform(metrics, tenant_id, region):
transformed_metric = {'metric': {},
'meta': {'tenantId': tenant_id, 'region': region},
'creation_time': timeutils.utcnow_ts()}
if isinstance(metrics, list):
transformed_metrics = []
for metric in metrics:
transformed_metric['metric'] = metric
transformed_metrics.append(rest_utils.as_json(transformed_metric))
return transformed_metrics
else:
transformed_metric['metric'] = metrics
return [rest_utils.as_json(transformed_metric)]
def _truncate(self, envelope):
"""Truncates the message if needed.
Each message send to kafka is verified.
Method checks if message serialized to json
exceeds maximum allowed size that can be posted to kafka
queue. If so, method truncates message property of the log
by difference between message and allowed size.
:param Envelope envelope: envelope to check
:return: truncated message if size is exceeded, otherwise message
is left unmodified
"""
msg_str = rest_utils.as_json(envelope)
max_size = CONF.log_publisher.max_message_size
envelope_size = ((len(bytearray(msg_str)) +
_TIMESTAMP_KEY_SIZE +
_KAFKA_META_DATA_SIZE)
if msg_str is not None else -1)
size_diff = (envelope_size - max_size) + _TRUNCATION_SAFE_OFFSET
LOG.debug('_truncate(max_message_size=%d, message_size=%d, diff=%d)',
max_size, envelope_size, size_diff)
if size_diff > 1:
truncate_by = size_diff + _TRUNCATED_PROPERTY_SIZE
LOG.warn(('Detected message that exceeds %d bytes,'
'message will be truncated by %d bytes'),
max_size,
truncate_by)
log_msg = envelope['log']['message']
truncated_log_msg = log_msg[:-truncate_by]
envelope['log']['truncated'] = True
envelope['log']['message'] = truncated_log_msg
# will just transform message once again without truncation
return rest_utils.as_json(envelope)
return msg_str
def _truncate(self, envelope):
"""Truncates the message if needed.
Each message send to kafka is verified.
Method checks if message serialized to json
exceeds maximum allowed size that can be posted to kafka
queue. If so, method truncates message property of the log
by difference between message and allowed size.
:param Envelope envelope: original envelope
:return: serialized message
:rtype: str
"""
msg_str = rest_utils.as_json(envelope)
envelope_size = ((len(bytearray(msg_str)) +
_TIMESTAMP_KEY_SIZE +
_KAFKA_META_DATA_SIZE)
if msg_str is not None else -1)
diff_size = ((envelope_size - self.max_message_size) +
_TRUNCATION_SAFE_OFFSET)
if diff_size > 1:
truncated_by = diff_size + _TRUNCATED_PROPERTY_SIZE
LOG.warn(('Detected message that exceeds %d bytes,'
'message will be truncated by %d bytes'),
self.max_message_size,
truncated_by)
log_msg = envelope['log']['message']
truncated_log_msg = log_msg[:-truncated_by]
envelope['log']['truncated'] = True
envelope['log']['message'] = truncated_log_msg
self._logs_truncated_gauge.send(name=None, value=truncated_by)
msg_str = rest_utils.as_json(envelope)
else:
self._logs_truncated_gauge.send(name=None, value=0)
return msg_str
def serialize_envelope(envelope):
"""Returns json representation of an envelope.
:return: json object of envelope
:rtype: json or a bytestring `encoding` encoded
representation of it.
"""
json = rest_utils.as_json(envelope, ensure_ascii=False)
return encodeutils.safe_decode(json, 'utf-8')
def _transform_message_to_json(self, message):
"""Transforms message into JSON.
Method transforms message to JSON and
encode to utf8
:param str message: instance of message
:return: serialized message
:rtype: str
"""
msg_json = rest_utils.as_json(message)
return msg_json.encode('utf-8')
def _truncate(self, envelope):
"""Truncates the message if needed.
Each message send to kafka is verified.
Method checks if message serialized to json
exceeds maximum allowed size that can be posted to kafka
queue. If so, method truncates message property of the log
by difference between message and allowed size.
:param Envelope envelope: original envelope
:return: serialized message
:rtype: str
"""
msg_str = rest_utils.as_json(envelope)
envelope_size = ((len(bytearray(msg_str)) + _TIMESTAMP_KEY_SIZE +
_KAFKA_META_DATA_SIZE)
if msg_str is not None else -1)
diff_size = ((envelope_size - self.max_message_size) +
_TRUNCATION_SAFE_OFFSET)
if diff_size > 1:
truncated_by = diff_size + _TRUNCATED_PROPERTY_SIZE
LOG.warn(('Detected message that exceeds %d bytes,'
'message will be truncated by %d bytes'),
self.max_message_size, truncated_by)
log_msg = envelope['log']['message']
truncated_log_msg = log_msg[:-truncated_by]
envelope['log']['truncated'] = True
envelope['log']['message'] = truncated_log_msg
self._logs_truncated_gauge.send(name=None, value=truncated_by)
msg_str = rest_utils.as_json(envelope)
else:
self._logs_truncated_gauge.send(name=None, value=0)
return msg_str
def to_json(data):
"""Converts data to JSON string.
:param dict data: data to be transformed to JSON
:return: JSON string
:rtype: str
:raises: Exception
"""
try:
# NOTE(trebskit) ensure_ascii => UTF-8
return rest_utils.as_json(data, ensure_ascii=False)
except Exception as ex:
LOG.exception(ex)
raise
def serialize_envelope(envelope):
"""Returns json representation of an envelope.
:return: json object of envelope
:rtype: six.text_type
"""
json = rest_utils.as_json(envelope, ensure_ascii=False)
if six.PY2:
raw = unicode(json.replace(r'\\', r'\\\\'),
encoding='utf-8',
errors='replace')
else:
raw = json
return raw
def on_get(self, req, res):
# at this point we know API is alive, so
# keep up good work and verify kafka status
kafka_result = self._kafka_check.healthcheck()
# in case it'd be unhealthy,
# message will contain error string
status_data = {'kafka': kafka_result.message}
# Really simple approach, ideally that should be
# part of monasca-common with some sort of registration of
# healthchecks concept
res.status = (self.HEALTHY_CODE_GET
if kafka_result.healthy else self.NOT_HEALTHY_CODE)
res.cache_control = self.CACHE_CONTROL
res.body = rest_utils.as_json(status_data)
def _create_log_envelope(self, tenant_id, cross_tenant_id, region='',
dimensions=None, log_message=None):
"""Create a log envelope and return it as a json string."""
if log_message is None:
log_message = {}
if dimensions is None:
dimensions = {}
log_message['dimensions'] = dimensions
envelope = {
'creation_time': timeutils.utcnow_ts(),
'meta': {
'tenantId': tenant_id if tenant_id else cross_tenant_id,
'region': region
},
'log': log_message
}
return rest_utils.as_json(envelope)
def on_get(self, req, res):
# at this point we know API is alive, so
# keep up good work and verify kafka status
kafka_result = self._kafka_check.healthcheck()
# in case it'd be unhealthy,
# message will contain error string
status_data = {
'kafka': kafka_result.message
}
# Really simple approach, ideally that should be
# part of monasca-common with some sort of registration of
# healthchecks concept
res.status = (self.HEALTHY_CODE_GET
if kafka_result.healthy else self.NOT_HEALTHY_CODE)
res.cache_control = self.CACHE_CONTROL
res.body = rest_utils.as_json(status_data)
def send_message(self, message):
"""Sends message to each configured topic.
Prior to sending a message, unique key is being
calculated for the given message using:
* tenant id
* application type
* dimensions
Note:
Falsy messages (i.e. empty) are not shipped to kafka
See :py:meth:`monasca_log_api.v2.common.service.LogCreator`
`.new_log_envelope`
:param dict message: instance of log envelope
"""
if not message:
return
if not self._is_message_valid(message):
raise InvalidMessageException()
key = self._build_key(message['meta']['tenantId'], message['log'])
msg = rest_utils.as_json(message).encode('utf8')
service.Validations.validate_envelope_size(msg)
LOG.debug('Build key [%s] for message', key)
LOG.debug('Sending message {topics=%s,key=%s,message=%s}',
self._topics, key, msg)
try:
for topic in self._topics:
self._publisher().publish(topic, msg, key)
except Exception as ex:
LOG.error(ex.message)
raise ex
def _transform_message(self, event_element, event_project_id):
"""Transform the message
:param dict event_element: original event element
:param str event_project_id: project id
:return: message payload
"""
try:
msg_json = rest_utils.as_json(event_element)
msg_json = encodeutils.safe_encode(msg_json, 'utf-8')
event_envelope = envelope.Envelope.new_envelope(
event=msg_json,
project_id=event_project_id,
)
msg_payload = (super(EventsBulkProcessor,
self)._transform_message(event_envelope))
return msg_payload
except Exception as ex:
LOG.error("Event transformation failed, rejecting event")
LOG.exception(ex)
return None
def handle_none_version_id(req, res, result):
for version in _VERSIONS_TPL_DICT:
selected_version = _parse_version(version, req)
result['elements'].append(selected_version)
res.body = rest_utils.as_json(result, sort_keys=True)
res.status = falcon.HTTP_200
def test_as_json_success(self):
data = mock.Mock()
dumped_json = utils.as_json(data)
self.assertEqual(dumped_json, self.mock_json.dumps.return_value)
def test_as_json_success(self):
data = mock.Mock()
dumped_json = utils.as_json(data)
self.assertEqual(dumped_json, self.mock_json.dumps.return_value)
def test_from_json(self):
body_json = {'test_body': 'test'}
req = request.Request(
testing.create_environ(body=rest_utils.as_json(body_json), ))
response = helpers.from_json(req)
self.assertEqual(body_json, response)
def handle_none_version_id(req, res, result):
for version in _VERSIONS_TPL_DICT:
selected_version = _parse_version(version, req)
result['elements'].append(selected_version)
res.body = rest_utils.as_json(result, sort_keys=True)
res.status = falcon.HTTP_200
