def handle_version_id(req, res, result, version_id): if version_id in _VERSIONS_TPL_DICT: result['elements'].append(_parse_version(version_id, req)) res.body = rest_utils.as_json(result, sort_keys=True) res.status = falcon.HTTP_200 else: error_body = {'message': '%s is not valid version' % version_id} res.body = rest_utils.as_json(error_body) res.status = falcon.HTTP_400
def handle_version_id(req, res, result, version_id): if version_id in _VERSIONS_TPL_DICT: result['elements'].append(_parse_version(version_id, req)) res.body = rest_utils.as_json(result, sort_keys=True) res.status = falcon.HTTP_200 else: error_body = {'message': '%s is not valid version' % version_id} res.body = rest_utils.as_json(error_body) res.status = falcon.HTTP_400
def transform(metrics, tenant_id, region): transformed_metric = {'metric': {}, 'meta': {'tenantId': tenant_id, 'region': region}, 'creation_time': timeutils.utcnow_ts()} if isinstance(metrics, list): transformed_metrics = [] for metric in metrics: transformed_metric['metric'] = metric transformed_metrics.append(rest_utils.as_json(transformed_metric)) return transformed_metrics else: transformed_metric['metric'] = metrics return [rest_utils.as_json(transformed_metric)]
def _truncate(self, envelope): """Truncates the message if needed. Each message send to kafka is verified. Method checks if message serialized to json exceeds maximum allowed size that can be posted to kafka queue. If so, method truncates message property of the log by difference between message and allowed size. :param Envelope envelope: envelope to check :return: truncated message if size is exceeded, otherwise message is left unmodified """ msg_str = rest_utils.as_json(envelope) max_size = CONF.log_publisher.max_message_size envelope_size = ((len(bytearray(msg_str)) + _TIMESTAMP_KEY_SIZE + _KAFKA_META_DATA_SIZE) if msg_str is not None else -1) size_diff = (envelope_size - max_size) + _TRUNCATION_SAFE_OFFSET LOG.debug('_truncate(max_message_size=%d, message_size=%d, diff=%d)', max_size, envelope_size, size_diff) if size_diff > 1: truncate_by = size_diff + _TRUNCATED_PROPERTY_SIZE LOG.warn(('Detected message that exceeds %d bytes,' 'message will be truncated by %d bytes'), max_size, truncate_by) log_msg = envelope['log']['message'] truncated_log_msg = log_msg[:-truncate_by] envelope['log']['truncated'] = True envelope['log']['message'] = truncated_log_msg # will just transform message once again without truncation return rest_utils.as_json(envelope) return msg_str
def _truncate(self, envelope): """Truncates the message if needed. Each message send to kafka is verified. Method checks if message serialized to json exceeds maximum allowed size that can be posted to kafka queue. If so, method truncates message property of the log by difference between message and allowed size. :param Envelope envelope: original envelope :return: serialized message :rtype: str """ msg_str = rest_utils.as_json(envelope) envelope_size = ((len(bytearray(msg_str)) + _TIMESTAMP_KEY_SIZE + _KAFKA_META_DATA_SIZE) if msg_str is not None else -1) diff_size = ((envelope_size - self.max_message_size) + _TRUNCATION_SAFE_OFFSET) if diff_size > 1: truncated_by = diff_size + _TRUNCATED_PROPERTY_SIZE LOG.warn(('Detected message that exceeds %d bytes,' 'message will be truncated by %d bytes'), self.max_message_size, truncated_by) log_msg = envelope['log']['message'] truncated_log_msg = log_msg[:-truncated_by] envelope['log']['truncated'] = True envelope['log']['message'] = truncated_log_msg self._logs_truncated_gauge.send(name=None, value=truncated_by) msg_str = rest_utils.as_json(envelope) else: self._logs_truncated_gauge.send(name=None, value=0) return msg_str
def serialize_envelope(envelope): """Returns json representation of an envelope. :return: json object of envelope :rtype: json or a bytestring `encoding` encoded representation of it. """ json = rest_utils.as_json(envelope, ensure_ascii=False) return encodeutils.safe_decode(json, 'utf-8')
def _transform_message_to_json(self, message): """Transforms message into JSON. Method transforms message to JSON and encode to utf8 :param str message: instance of message :return: serialized message :rtype: str """ msg_json = rest_utils.as_json(message) return msg_json.encode('utf-8')
def _truncate(self, envelope): """Truncates the message if needed. Each message send to kafka is verified. Method checks if message serialized to json exceeds maximum allowed size that can be posted to kafka queue. If so, method truncates message property of the log by difference between message and allowed size. :param Envelope envelope: original envelope :return: serialized message :rtype: str """ msg_str = rest_utils.as_json(envelope) envelope_size = ((len(bytearray(msg_str)) + _TIMESTAMP_KEY_SIZE + _KAFKA_META_DATA_SIZE) if msg_str is not None else -1) diff_size = ((envelope_size - self.max_message_size) + _TRUNCATION_SAFE_OFFSET) if diff_size > 1: truncated_by = diff_size + _TRUNCATED_PROPERTY_SIZE LOG.warn(('Detected message that exceeds %d bytes,' 'message will be truncated by %d bytes'), self.max_message_size, truncated_by) log_msg = envelope['log']['message'] truncated_log_msg = log_msg[:-truncated_by] envelope['log']['truncated'] = True envelope['log']['message'] = truncated_log_msg self._logs_truncated_gauge.send(name=None, value=truncated_by) msg_str = rest_utils.as_json(envelope) else: self._logs_truncated_gauge.send(name=None, value=0) return msg_str
def to_json(data): """Converts data to JSON string. :param dict data: data to be transformed to JSON :return: JSON string :rtype: str :raises: Exception """ try: # NOTE(trebskit) ensure_ascii => UTF-8 return rest_utils.as_json(data, ensure_ascii=False) except Exception as ex: LOG.exception(ex) raise
def serialize_envelope(envelope): """Returns json representation of an envelope. :return: json object of envelope :rtype: six.text_type """ json = rest_utils.as_json(envelope, ensure_ascii=False) if six.PY2: raw = unicode(json.replace(r'\\', r'\\\\'), encoding='utf-8', errors='replace') else: raw = json return raw
def on_get(self, req, res): # at this point we know API is alive, so # keep up good work and verify kafka status kafka_result = self._kafka_check.healthcheck() # in case it'd be unhealthy, # message will contain error string status_data = {'kafka': kafka_result.message} # Really simple approach, ideally that should be # part of monasca-common with some sort of registration of # healthchecks concept res.status = (self.HEALTHY_CODE_GET if kafka_result.healthy else self.NOT_HEALTHY_CODE) res.cache_control = self.CACHE_CONTROL res.body = rest_utils.as_json(status_data)
def _create_log_envelope(self, tenant_id, cross_tenant_id, region='', dimensions=None, log_message=None): """Create a log envelope and return it as a json string.""" if log_message is None: log_message = {} if dimensions is None: dimensions = {} log_message['dimensions'] = dimensions envelope = { 'creation_time': timeutils.utcnow_ts(), 'meta': { 'tenantId': tenant_id if tenant_id else cross_tenant_id, 'region': region }, 'log': log_message } return rest_utils.as_json(envelope)
def on_get(self, req, res): # at this point we know API is alive, so # keep up good work and verify kafka status kafka_result = self._kafka_check.healthcheck() # in case it'd be unhealthy, # message will contain error string status_data = { 'kafka': kafka_result.message } # Really simple approach, ideally that should be # part of monasca-common with some sort of registration of # healthchecks concept res.status = (self.HEALTHY_CODE_GET if kafka_result.healthy else self.NOT_HEALTHY_CODE) res.cache_control = self.CACHE_CONTROL res.body = rest_utils.as_json(status_data)
def send_message(self, message): """Sends message to each configured topic. Prior to sending a message, unique key is being calculated for the given message using: * tenant id * application type * dimensions Note: Falsy messages (i.e. empty) are not shipped to kafka See :py:meth:`monasca_log_api.v2.common.service.LogCreator` `.new_log_envelope` :param dict message: instance of log envelope """ if not message: return if not self._is_message_valid(message): raise InvalidMessageException() key = self._build_key(message['meta']['tenantId'], message['log']) msg = rest_utils.as_json(message).encode('utf8') service.Validations.validate_envelope_size(msg) LOG.debug('Build key [%s] for message', key) LOG.debug('Sending message {topics=%s,key=%s,message=%s}', self._topics, key, msg) try: for topic in self._topics: self._publisher().publish(topic, msg, key) except Exception as ex: LOG.error(ex.message) raise ex
def _transform_message(self, event_element, event_project_id): """Transform the message :param dict event_element: original event element :param str event_project_id: project id :return: message payload """ try: msg_json = rest_utils.as_json(event_element) msg_json = encodeutils.safe_encode(msg_json, 'utf-8') event_envelope = envelope.Envelope.new_envelope( event=msg_json, project_id=event_project_id, ) msg_payload = (super(EventsBulkProcessor, self)._transform_message(event_envelope)) return msg_payload except Exception as ex: LOG.error("Event transformation failed, rejecting event") LOG.exception(ex) return None
def handle_none_version_id(req, res, result): for version in _VERSIONS_TPL_DICT: selected_version = _parse_version(version, req) result['elements'].append(selected_version) res.body = rest_utils.as_json(result, sort_keys=True) res.status = falcon.HTTP_200
def test_as_json_success(self): data = mock.Mock() dumped_json = utils.as_json(data) self.assertEqual(dumped_json, self.mock_json.dumps.return_value)
def test_as_json_success(self): data = mock.Mock() dumped_json = utils.as_json(data) self.assertEqual(dumped_json, self.mock_json.dumps.return_value)
def test_from_json(self): body_json = {'test_body': 'test'} req = request.Request( testing.create_environ(body=rest_utils.as_json(body_json), )) response = helpers.from_json(req) self.assertEqual(body_json, response)
def handle_none_version_id(req, res, result): for version in _VERSIONS_TPL_DICT: selected_version = _parse_version(version, req) result['elements'].append(selected_version) res.body = rest_utils.as_json(result, sort_keys=True) res.status = falcon.HTTP_200