def start_udp(self):
address_family = socket.AF_INET
if netutils.is_valid_ipv6(cfg.CONF.collector.udp_address):
address_family = socket.AF_INET6
udp = socket.socket(address_family, socket.SOCK_DGRAM)
udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
udp.bind((cfg.CONF.collector.udp_address,
cfg.CONF.collector.udp_port))
self.udp_run = True
while self.udp_run:
# NOTE(jd) Arbitrary limit of 64K because that ought to be
# enough for anybody.
data, source = udp.recvfrom(64 * units.Ki)
try:
sample = msgpack.loads(data, encoding='utf-8')
except Exception:
LOG.warning(_("UDP: Cannot decode data sent by %s"), source)
else:
if publisher_utils.verify_signature(
sample, cfg.CONF.publisher.telemetry_secret):
try:
LOG.debug("UDP: Storing %s", sample)
self.meter_manager.map_method(
'record_metering_data', sample)
except Exception:
LOG.exception(_("UDP: Unable to store meter"))
else:
LOG.warning(_LW('sample signature invalid, '
'discarding: %s'), sample)
def record_metering_data(self, data):
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(
"metering data %(counter_name)s " "for %(resource_id)s @ %(timestamp)s: %(counter_volume)s",
{
"counter_name": meter["counter_name"],
"resource_id": meter["resource_id"],
"timestamp": meter.get("timestamp", "NO TIMESTAMP"),
"counter_volume": meter["counter_volume"],
},
)
if publisher_utils.verify_signature(meter, self.conf.publisher.telemetry_secret):
try:
# Convert the timestamp to a datetime instance.
# Storage engines are responsible for converting
# that value to something they can store.
if meter.get("timestamp"):
ts = timeutils.parse_isotime(meter["timestamp"])
meter["timestamp"] = timeutils.normalize_time(ts)
self.meter_conn.record_metering_data(meter)
except Exception as err:
LOG.exception(_LE("Failed to record metering data: %s"), err)
# raise the exception to propagate it up in the chain.
raise
else:
LOG.warning(_LW("message signature invalid, discarding message: %r"), meter)
def record_metering_data(self, data):
if self.target == "":
# if the target was not set, do not do anything
LOG.error(
_(
"Dispatcher target was not set, no meter will "
"be posted. Set the target in the ceilometer.conf "
"file"
)
)
return
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(
"metering data %(counter_name)s " "for %(resource_id)s @ %(timestamp)s: %(counter_volume)s",
{
"counter_name": meter["counter_name"],
"resource_id": meter["resource_id"],
"timestamp": meter.get("timestamp", "NO TIMESTAMP"),
"counter_volume": meter["counter_volume"],
},
)
if publisher_utils.verify_signature(meter, self.conf.publisher.telemetry_secret):
try:
# Every meter should be posted to the target
res = requests.post(self.target, data=json.dumps(meter), headers=self.headers, timeout=self.timeout)
LOG.debug("Message posting finished with status code " "%d.", res.status_code)
except Exception as err:
LOG.exception(_("Failed to record metering data: %s"), err)
else:
LOG.warning(_("message signature invalid, discarding message: %r"), meter)
def record_metering_data(self, data):
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(_(
'metering data %(counter_name)s '
'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s')
% ({'counter_name': meter['counter_name'],
'resource_id': meter['resource_id'],
'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
'counter_volume': meter['counter_volume']}))
if publisher_utils.verify_signature(
meter, self.conf.publisher.telemetry_secret):
try:
# Convert the timestamp to a datetime instance.
# Storage engines are responsible for converting
# that value to something they can store.
if meter.get('timestamp'):
ts = timeutils.parse_isotime(meter['timestamp'])
meter['timestamp'] = timeutils.normalize_time(ts)
self.meter_conn.record_metering_data(meter)
except Exception as err:
LOG.exception(_('Failed to record metering data: %s'),
err)
# raise the exception to propagate it up in the chain.
raise
else:
LOG.warning(_(
'message signature invalid, discarding message: %r'),
meter)
def record_metering_data(self, data):
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(
_('metering data %(counter_name)s '
'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s') %
({
'counter_name': meter['counter_name'],
'resource_id': meter['resource_id'],
'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
'counter_volume': meter['counter_volume']
}))
if publisher_utils.verify_signature(
meter, self.conf.publisher.metering_secret):
try:
# Convert the timestamp to a datetime instance.
# Storage engines are responsible for converting
# that value to something they can store.
if meter.get('timestamp'):
ts = timeutils.parse_isotime(meter['timestamp'])
meter['timestamp'] = timeutils.normalize_time(ts)
self.storage_conn.record_metering_data(meter)
except Exception as err:
LOG.exception(_('Failed to record metering data: %s'), err)
else:
LOG.warning(
_('message signature invalid, discarding message: %r'),
meter)
def test_udp_receive_valid_encoding(self):
self._setup_messaging(False)
mock_dispatcher = self._setup_fake_dispatcher()
self.data_sent = []
with mock.patch("socket.socket", return_value=self._make_fake_socket(self.utf8_msg)):
self.srv.start()
self.assertTrue(utils.verify_signature(mock_dispatcher.method_calls[0][1][0], "not-so-secret"))
def start_udp(self):
address_family = socket.AF_INET
if netutils.is_valid_ipv6(cfg.CONF.collector.udp_address):
address_family = socket.AF_INET6
udp = socket.socket(address_family, socket.SOCK_DGRAM)
udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
udp.bind((cfg.CONF.collector.udp_address, cfg.CONF.collector.udp_port))
self.udp_run = True
while self.udp_run:
# NOTE(jd) Arbitrary limit of 64K because that ought to be
# enough for anybody.
data, source = udp.recvfrom(64 * units.Ki)
try:
sample = msgpack.loads(data, encoding='utf-8')
except Exception:
LOG.warning(_("UDP: Cannot decode data sent by %s"), source)
else:
if publisher_utils.verify_signature(
sample, cfg.CONF.publisher.telemetry_secret):
try:
LOG.debug("UDP: Storing %s", sample)
self.meter_manager.map_method('record_metering_data',
sample)
except Exception:
LOG.exception(_("UDP: Unable to store meter"))
else:
LOG.warning(
_LW('sample signature invalid, '
'discarding: %s'), sample)
def record_events(self, events):
if not isinstance(events, list):
events = [events]
event_list = []
for ev in events:
if publisher_utils.verify_signature(
ev, self.conf.publisher.telemetry_secret):
try:
event_list.append(
models.Event(
message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[models.Trait(
name, dtype,
models.Trait.convert_value(dtype, value))
for name, dtype, value in ev['traits']],
raw=ev.get('raw', {}))
)
except Exception:
LOG.exception(_LE("Error processing event and it will be "
"dropped: %s"), ev)
else:
LOG.warning(_LW(
'event signature invalid, discarding event: %s'), ev)
self.event_conn.record_events(event_list)
def sample(self, messages):
events = chain.from_iterable(m["payload"] for m in messages)
events = [
models.Event(message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[
models.Trait(
name, dtype,
models.Trait.convert_value(dtype, value))
for name, dtype, value in ev['traits']
],
raw=ev.get('raw', {})) for ev in events
if publisher_utils.verify_signature(
ev, self.conf.publisher.telemetry_secret)
]
try:
with self.publish_context as p:
p(events)
except Exception:
if not self.conf.notification.ack_on_event_error:
return oslo_messaging.NotificationResult.REQUEUE
raise
return oslo_messaging.NotificationResult.HANDLED
def test_verify_unicode_symbols(self):
data = {u'a\xe9\u0437': 'A',
'b': u'B\xe9\u0437'
}
data['message_signature'] = utils.compute_signature(
data,
'not-so-secret')
jsondata = json.loads(json.dumps(data))
self.assertTrue(utils.verify_signature(jsondata, 'not-so-secret'))
def test_verify_unicode_symbols(self):
data = {u'a\xe9\u0437': 'A',
'b': u'B\xe9\u0437'
}
data['message_signature'] = utils.compute_signature(
data,
'not-so-secret')
jsondata = jsonutils.loads(jsonutils.dumps(data))
self.assertTrue(utils.verify_signature(jsondata, 'not-so-secret'))
def test_udp_receive_valid_encoding(self):
self._setup_messaging(False)
mock_dispatcher = self._setup_fake_dispatcher()
self.data_sent = []
with mock.patch('socket.socket',
return_value=self._make_fake_socket(self.utf8_msg)):
self.srv.start()
self.assertTrue(
utils.verify_signature(mock_dispatcher.method_calls[0][1][0],
"not-so-secret"))
def verify_and_record_events(self, events):
"""Verify event signature and record them."""
goods = []
for event in events:
if utils.verify_signature(
event, self.conf.publisher.telemetry_secret):
goods.append(event)
else:
LOG.warning(_LW(
'event signature invalid, discarding event: %s'), event)
return self.record_events(goods)
def test_verify_signature_nested(self):
data = {'a': 'A',
'b': 'B',
'nested': {'a': 'A',
'b': 'B',
},
}
data['message_signature'] = utils.compute_signature(
data,
'not-so-secret')
self.assertTrue(utils.verify_signature(data, 'not-so-secret'))
def test_udp_receive_valid_encoding(self):
self.data_sent = []
with mock.patch('socket.socket',
return_value=self._make_fake_socket(self.utf8_msg)):
self.srv.rpc_server = mock.MagicMock()
mock_dispatcher = mock.MagicMock()
self.srv.dispatcher_manager = \
self._make_test_manager(mock_dispatcher)
self.srv.start_udp()
self.assertTrue(utils.verify_signature(
mock_dispatcher.method_calls[0][1][0],
"not-so-secret"))
def test_udp_receive_valid_encoding(self):
self.data_sent = []
with mock.patch('socket.socket',
return_value=self._make_fake_socket(self.utf8_msg)):
self.srv.rpc_server = mock.MagicMock()
mock_dispatcher = mock.MagicMock()
self.srv.dispatcher_manager = \
self._make_test_manager(mock_dispatcher)
self.srv.start_udp()
self.assertTrue(
utils.verify_signature(mock_dispatcher.method_calls[0][1][0],
"not-so-secret"))
def test_udp_receive_valid_encoding(self):
self._setup_messaging(False)
mock_dispatcher = self._setup_fake_dispatcher()
self.data_sent = []
with mock.patch('socket.socket',
return_value=self._make_fake_socket(self.utf8_msg)):
self.srv.run()
self.addCleanup(self.srv.terminate)
self.srv.udp_thread.join(5)
self.assertFalse(self.srv.udp_thread.is_alive())
self.assertTrue(utils.verify_signature(
mock_dispatcher.method_calls[0][1][0],
"not-so-secret"))
def test_udp_receive_valid_encoding(self):
self._setup_messaging(False)
self.data_sent = []
sock = self._make_fake_socket(self.utf8_msg)
with mock.patch('select.select', return_value=([sock], [], [])):
with mock.patch('socket.socket', return_value=sock):
self.srv.run()
self.addCleanup(self.srv.terminate)
self.srv.udp_thread.join(5)
self.assertFalse(self.srv.udp_thread.is_alive())
self.assertTrue(utils.verify_signature(
self.mock_dispatcher.method_calls[0][1][0],
"not-so-secret"))
def record_metering_data(self, data):
if self.target == '':
# if the target was not set, do not do anything
LOG.error(
_('Dispatcher target was not set, no meter will '
'be posted. Set the target in the ceilometer.conf '
'file'))
return
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(
_('metering data %(counter_name)s '
'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s') %
({
'counter_name': meter['counter_name'],
'resource_id': meter['resource_id'],
'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
'counter_volume': meter['counter_volume']
}))
if publisher_utils.verify_signature(
meter, self.conf.publisher.telemetry_secret):
try:
if self.cadf_only:
# Only cadf messages are being wanted.
req_data = meter.get('resource_metadata',
{}).get('request')
if req_data and 'CADF_EVENT' in req_data:
data = req_data['CADF_EVENT']
else:
continue
else:
# Every meter should be posted to the target
data = meter
res = requests.post(self.target,
data=json.dumps(data),
headers=self.headers,
timeout=self.timeout)
LOG.debug(
_('Message posting finished with status code '
'%d.') % res.status_code)
except Exception as err:
LOG.exception(_('Failed to record metering data: %s'), err)
else:
LOG.warning(
_('message signature invalid, discarding message: %r'),
meter)
def test_verify_signature_nested_json(self):
data = {'a': 'A',
'b': 'B',
'nested': {'a': 'A',
'b': 'B',
'c': ('c',),
'd': ['d']
},
}
data['message_signature'] = utils.compute_signature(
data,
'not-so-secret')
jsondata = jsonutils.loads(jsonutils.dumps(data))
self.assertTrue(utils.verify_signature(jsondata, 'not-so-secret'))
def verify_and_record_metering_data(self, datapoints):
"""Verify metering data's signature and record valid ones."""
if not isinstance(datapoints, list):
datapoints = [datapoints]
valid_datapoints = []
for datapoint in datapoints:
if utils.verify_signature(datapoint,
self.conf.publisher.telemetry_secret):
valid_datapoints.append(datapoint)
else:
LOG.warning(_LW('Message signature is invalid, discarding '
'it: <%r>.'), datapoint)
return self.record_metering_data(valid_datapoints)
def record_metering_data(self, data):
if self.target == '':
# if the target was not set, do not do anything
LOG.error(_('Dispatcher target was not set, no meter will '
'be posted. Set the target in the ceilometer.conf '
'file'))
return
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(_(
'metering data %(counter_name)s '
'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s')
% ({'counter_name': meter['counter_name'],
'resource_id': meter['resource_id'],
'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
'counter_volume': meter['counter_volume']}))
if publisher_utils.verify_signature(
meter,
self.conf.publisher.metering_secret):
try:
if self.cadf_only:
# Only cadf messages are being wanted.
req_data = meter.get('resource_metadata',
{}).get('request')
if req_data and 'CADF_EVENT' in req_data:
data = req_data['CADF_EVENT']
else:
continue
else:
# Every meter should be posted to the target
data = meter
res = requests.post(self.target,
data=json.dumps(data),
headers=self.headers,
timeout=self.timeout)
LOG.debug(_('Message posting finished with status code '
'%d.') % res.status_code)
except Exception as err:
LOG.exception(_('Failed to record metering data: %s'),
err)
else:
LOG.warning(_(
'message signature invalid, discarding message: %r'),
meter)
def sample(self, ctxt, publisher_id, event_type, payload, metadata):
events = [
models.Event(
message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[models.Trait(name, dtype,
models.Trait.convert_value(dtype, value))
for name, dtype, value in ev['traits']],
raw=ev.get('raw', {}))
for ev in payload if publisher_utils.verify_signature(
ev, cfg.CONF.publisher.telemetry_secret)
]
with self.publish_context as p:
p(events)
def sample(self, ctxt, publisher_id, event_type, payload, metadata):
events = [
models.Event(
message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[models.Trait(name, dtype,
models.Trait.convert_value(dtype, value))
for name, dtype, value in ev['traits']],
raw=ev.get('raw', {}))
for ev in payload if publisher_utils.verify_signature(
ev, cfg.CONF.publisher.telemetry_secret)
]
with self.publish_context as p:
p(events)
def sample(self, ctxt, publisher_id, event_type, payload, metadata):
samples = [
sample_util.Sample(name=s['counter_name'],
type=s['counter_type'],
unit=s['counter_unit'],
volume=s['counter_volume'],
user_id=s['user_id'],
project_id=s['project_id'],
resource_id=s['resource_id'],
timestamp=s['timestamp'],
resource_metadata=s['resource_metadata'],
source=s.get('source')) for s in payload
if publisher_utils.verify_signature(
s, cfg.CONF.publisher.telemetry_secret)
]
with self.publish_context as p:
p(samples)
def sample(self, ctxt, publisher_id, event_type, payload, metadata):
samples = [
sample_util.Sample(name=s['counter_name'],
type=s['counter_type'],
unit=s['counter_unit'],
volume=s['counter_volume'],
user_id=s['user_id'],
project_id=s['project_id'],
resource_id=s['resource_id'],
timestamp=s['timestamp'],
resource_metadata=s['resource_metadata'],
source=s.get('source'))
for s in payload if publisher_utils.verify_signature(
s, cfg.CONF.publisher.telemetry_secret)
]
with self.publish_context as p:
p(samples)
def sample(self, messages):
samples = chain.from_iterable(m["payload"] for m in messages)
samples = [
sample_util.Sample(name=s['counter_name'],
type=s['counter_type'],
unit=s['counter_unit'],
volume=s['counter_volume'],
user_id=s['user_id'],
project_id=s['project_id'],
resource_id=s['resource_id'],
timestamp=s['timestamp'],
resource_metadata=s['resource_metadata'],
source=s.get('source')) for s in samples
if publisher_utils.verify_signature(
s, self.conf.publisher.telemetry_secret)
]
with self.publish_context as p:
p(sorted(samples, key=methodcaller('get_iso_timestamp')))
def sample(self, messages):
samples = chain.from_iterable(m["payload"] for m in messages)
samples = [
sample_util.Sample(name=s['counter_name'],
type=s['counter_type'],
unit=s['counter_unit'],
volume=s['counter_volume'],
user_id=s['user_id'],
project_id=s['project_id'],
resource_id=s['resource_id'],
timestamp=s['timestamp'],
resource_metadata=s['resource_metadata'],
source=s.get('source'))
for s in samples if publisher_utils.verify_signature(
s, self.conf.publisher.telemetry_secret)
]
with self.publish_context as p:
p(sorted(samples, key=methodcaller('get_iso_timestamp')))
def record_events(self, events):
if not isinstance(events, list):
events = [events]
for event in events:
if publisher_utils.verify_signature(event, self.conf.publisher.telemetry_secret):
res = None
try:
res = requests.post(self.event_target, data=event, headers=self.headers, timeout=self.timeout)
res.raise_for_status()
except Exception:
error_code = res.status_code if res else "unknown"
LOG.exception(
_LE("Status Code: %{code}s. Failed to" "dispatch event: %{event}s"),
{"code": error_code, "event": event},
)
else:
LOG.warning(_LW("event signature invalid, discarding event: %s"), event)
def sample(self, messages):
"""RPC endpoint for notification messages
When another service sends a notification over the message
bus, this method receives it.
"""
goods = []
for sample in chain.from_iterable(m["payload"] for m in messages):
if publisher_utils.verify_signature(sample, self.secret):
goods.append(sample)
else:
LOG.warning(_LW('notification signature invalid, '
'discarding: %s'), sample)
try:
self.dispatcher_manager.map_method(self.method, goods)
except Exception:
LOG.exception(_LE("Dispatcher failed to handle the notification, "
"re-queuing it."))
return oslo_messaging.NotificationResult.REQUEUE
def start_udp(self):
address_family = socket.AF_INET
if netutils.is_valid_ipv6(self.conf.collector.udp_address):
address_family = socket.AF_INET6
udp = socket.socket(address_family, socket.SOCK_DGRAM)
udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
# NOTE(zhengwei): linux kernel >= 3.9
udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
LOG.warning(
_LW("System does not support socket.SO_REUSEPORT "
"option. Only one worker will be able to process "
"incoming data."))
udp.bind(
(self.conf.collector.udp_address, self.conf.collector.udp_port))
self.udp_run = True
while self.udp_run:
# NOTE(sileht): return every 10 seconds to allow
# clear shutdown
if not select.select([udp], [], [], 10.0)[0]:
continue
# NOTE(jd) Arbitrary limit of 64K because that ought to be
# enough for anybody.
data, source = udp.recvfrom(64 * units.Ki)
try:
sample = msgpack.loads(data, encoding='utf-8')
except Exception:
LOG.warning(_("UDP: Cannot decode data sent by %s"), source)
else:
if publisher_utils.verify_signature(
sample, self.conf.publisher.telemetry_secret):
try:
LOG.debug("UDP: Storing %s", sample)
self.meter_manager.map_method('record_metering_data',
sample)
except Exception:
LOG.exception(_("UDP: Unable to store meter"))
else:
LOG.warning(
_LW('sample signature invalid, '
'discarding: %s'), sample)
def sample(self, messages):
samples = chain.from_iterable(m["payload"] for m in messages)
samples = [
sample_util.Sample(
name=s["counter_name"],
type=s["counter_type"],
unit=s["counter_unit"],
volume=s["counter_volume"],
user_id=s["user_id"],
project_id=s["project_id"],
resource_id=s["resource_id"],
timestamp=s["timestamp"],
resource_metadata=s["resource_metadata"],
source=s.get("source"),
)
for s in samples
if publisher_utils.verify_signature(s, cfg.CONF.publisher.telemetry_secret)
]
with self.publish_context as p:
p(samples)
def sample(self, messages):
"""RPC endpoint for notification messages
When another service sends a notification over the message
bus, this method receives it.
"""
goods = []
for sample in chain.from_iterable(m["payload"] for m in messages):
if publisher_utils.verify_signature(sample, self.secret):
goods.append(sample)
else:
LOG.warning(
'notification signature invalid, '
'discarding: %s', sample)
try:
self.dispatcher_manager.map_method(self.method, goods)
except Exception:
LOG.exception("Dispatcher failed to handle the notification, "
"re-queuing it.")
return oslo_messaging.NotificationResult.REQUEUE
def start_udp(self):
address_family = socket.AF_INET
if netutils.is_valid_ipv6(self.conf.collector.udp_address):
address_family = socket.AF_INET6
udp = socket.socket(address_family, socket.SOCK_DGRAM)
udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
# NOTE(zhengwei): linux kernel >= 3.9
udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
LOG.warning(_LW("System does not support socket.SO_REUSEPORT "
"option. Only one worker will be able to process "
"incoming data."))
udp.bind((self.conf.collector.udp_address,
self.conf.collector.udp_port))
self.udp_run = True
while self.udp_run:
# NOTE(sileht): return every 10 seconds to allow
# clear shutdown
if not select.select([udp], [], [], 10.0)[0]:
continue
# NOTE(jd) Arbitrary limit of 64K because that ought to be
# enough for anybody.
data, source = udp.recvfrom(64 * units.Ki)
try:
sample = msgpack.loads(data, encoding='utf-8')
except Exception:
LOG.warning(_("UDP: Cannot decode data sent by %s"), source)
else:
if publisher_utils.verify_signature(
sample, self.conf.publisher.telemetry_secret):
try:
LOG.debug("UDP: Storing %s", sample)
self.meter_manager.map_method(
'record_metering_data', sample)
except Exception:
LOG.exception(_("UDP: Unable to store meter"))
else:
LOG.warning(_LW('sample signature invalid, '
'discarding: %s'), sample)
def record_events(self, events):
if not isinstance(events, list):
events = [events]
for event in events:
if publisher_utils.verify_signature(
event, self.conf.publisher.telemetry_secret):
res = None
try:
res = requests.post(self.event_target, data=event,
headers=self.headers,
timeout=self.timeout)
res.raise_for_status()
except Exception:
error_code = res.status_code if res else 'unknown'
LOG.exception(_LE('Status Code: %{code}s. Failed to'
'dispatch event: %{event}s'),
{'code': error_code, 'event': event})
else:
LOG.warning(_LW(
'event signature invalid, discarding event: %s'), event)
def process_notifications(self, priority, notifications):
samples = chain.from_iterable(m["payload"] for m in notifications)
samples = [
sample_util.Sample(name=s['counter_name'],
type=s['counter_type'],
unit=s['counter_unit'],
volume=s['counter_volume'],
user_id=s['user_id'],
project_id=s['project_id'],
resource_id=s['resource_id'],
timestamp=s['timestamp'],
resource_metadata=s['resource_metadata'],
source=s.get('source'),
# NOTE(sileht): May come from an older node,
# Put None in this case.
monotonic_time=s.get('monotonic_time'))
for s in samples if publisher_utils.verify_signature(
s, self.conf.publisher.telemetry_secret)
]
with self.publisher as p:
p(sorted(samples, key=methodcaller('get_iso_timestamp')))
def sample(self, ctxt, publisher_id, event_type, payload, metadata):
events = [
models.Event(
message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[models.Trait(name, dtype,
models.Trait.convert_value(dtype, value))
for name, dtype, value in ev['traits']],
raw=ev.get('raw', {}))
for ev in payload if publisher_utils.verify_signature(
ev, cfg.CONF.publisher.telemetry_secret)
]
try:
with self.publish_context as p:
p(events)
except Exception:
if not cfg.CONF.notification.ack_on_event_error:
return oslo_messaging.NotificationResult.REQUEUE
raise
return oslo_messaging.NotificationResult.HANDLED
def record_metering_data(self, data):
if self.target == '':
# if the target was not set, do not do anything
LOG.error(_('Dispatcher target was not set, no meter will '
'be posted. Set the target in the ceilometer.conf '
'file'))
return
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(meter)
if publisher_utils.verify_signature(
meter, self.conf.publisher.telemetry_secret):
try:
if self.cadf_only:
# Only cadf messages are being wanted.
data = meter.get('resource_metadata')
if not data:
continue
else:
# Every meter should be posted to the target
data = meter
res = requests.post(self.target,
data=json.dumps(data),
headers=self.headers,
timeout=self.timeout)
LOG.debug('Message posting finished with status code '
'%d.', res.status_code)
except Exception as err:
LOG.exception(_('Failed to record metering data: %s'),
err)
else:
LOG.warning(_(
'message signature invalid, discarding message: %r'),
meter)
def sample(self, messages):
events = chain.from_iterable(m["payload"] for m in messages)
events = [
models.Event(
message_id=ev["message_id"],
event_type=ev["event_type"],
generated=timeutils.normalize_time(timeutils.parse_isotime(ev["generated"])),
traits=[
models.Trait(name, dtype, models.Trait.convert_value(dtype, value))
for name, dtype, value in ev["traits"]
],
raw=ev.get("raw", {}),
)
for ev in events
if publisher_utils.verify_signature(ev, cfg.CONF.publisher.telemetry_secret)
]
try:
with self.publish_context as p:
p(events)
except Exception:
if not cfg.CONF.notification.ack_on_event_error:
return oslo_messaging.NotificationResult.REQUEUE
raise
return oslo_messaging.NotificationResult.HANDLED
def test_verify_no_secret(self):
data = {'a': 'A', 'b': 'B'}
self.assertTrue(utils.verify_signature(data, ''))
def test_verify_signature_unsigned(self):
data = {'a': 'A', 'b': 'B'}
self.assertFalse(utils.verify_signature(data, 'not-so-secret'))
def test_verify_signature_unicode(self):
data = {'a': 'A', 'b': 'B',
'message_signature': u''}
self.assertFalse(utils.verify_signature(data, 'not-so-secret'))
def test_verify_no_secret(self):
data = {"a": "A", "b": "B"}
self.assertTrue(utils.verify_signature(data, ""))
def test_verify_signature_incorrect(self):
data = {'a': 'A', 'b': 'B',
'message_signature': 'Not the same'}
self.assertFalse(utils.verify_signature(data, 'not-so-secret'))
def record_metering_data(self, data):
# We may have receive only one counter on the wire
if not isinstance(data, list):
data = [data]
for meter in data:
LOG.debug(
_('metering data %(counter_name)s '
'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s') %
({
'counter_name': meter['counter_name'],
'resource_id': meter['resource_id'],
'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
'counter_volume': meter['counter_volume']
}))
if publisher_utils.verify_signature(
meter, self.conf.publisher.metering_secret):
try:
# Convert the timestamp to a datetime instance.
# Storage engines are responsible for converting
# that value to something they can store.
if meter.get('timestamp'):
ts = timeutils.parse_isotime(meter['timestamp'])
#meter['timestamp'] = timeutils.normalize_time(ts)
if meter.get('resource_metadata'):
resource_metadata = meter['resource_metadata']
if resource_metadata.get('instance_id'):
instance_uuid = resource_metadata['instance_id']
meter_name = meter['counter_name']
resource_id = meter['resource_id']
meter_map = self.instance_map.get(instance_uuid)
if meter_map is None:
meter_map = {}
self.instance_map[instance_uuid] = meter_map
resource_set = meter_map.get(meter_name)
if resource_set is None:
resource_set = set()
meter_map[meter_name] = resource_set
resource_set.add(resource_id)
meter_map_json = json.dumps(meter_map,
cls=SetJSONEncoder)
print meter_map_json
self.redis_conn.set("metermeta-" + instance_uuid,
meter_map_json)
counter_map = {}
counter_map['counter_name'] = meter['counter_name']
counter_map['counter_unit'] = meter['counter_unit']
counter_map['counter_type'] = meter['counter_type']
counter_map['counter_volume'] = meter[
'counter_volume']
counter_map['timestamp'] = meter['timestamp']
counter_map_json = json.dumps(counter_map)
print counter_map_json
self.redis_conn.set(meter_name + '-' + resource_id,
counter_map_json)
print "%s instance: %s, meter_name: %s, volume: %s" %\
(meter['timestamp'], resource_metadata['instance_id'],
meter['counter_name'], meter['counter_volume'])
elif resource_metadata.get('hostname'):
hostname = resource_metadata['hostname']
meter_name = meter['counter_name']
resource_id = meter['resource_id']
meter_map = self.host_map.get(hostname)
if meter_map is None:
meter_map = {}
self.host_map[hostname] = meter_map
resource_set = meter_map.get(meter_name)
if resource_set is None:
resource_set = set()
meter_map[meter_name] = resource_set
resource_set.add(resource_id)
meter_map_json = json.dumps(meter_map,
cls=SetJSONEncoder)
print meter_map_json
self.redis_conn.set("metermeta-" + hostname,
meter_map_json)
counter_map = {}
counter_map['counter_name'] = meter['counter_name']
counter_map['counter_unit'] = meter['counter_unit']
counter_map['counter_type'] = meter['counter_type']
counter_map['counter_volume'] = meter[
'counter_volume']
counter_map['timestamp'] = meter['timestamp']
counter_map_json = json.dumps(counter_map)
print counter_map_json
self.redis_conn.set(meter_name + '-' + resource_id,
counter_map_json)
print "%s hostname: %s, meter_name: %s, volume: %s" % \
(meter['timestamp'], resource_metadata['hostname'],
meter['counter_name'], meter['counter_volume'])
except Exception as err:
LOG.exception(_('Failed to record metering data: %s'), err)
else:
LOG.warning(
_('message signature invalid, discarding message: %r'),
meter)
def test_verify_signature_invalid_encoding(self):
data = {'a': 'A', 'b': 'B',
'message_signature': ''}
self.assertFalse(utils.verify_signature(data, 'not-so-secret'))
def test_verify_signature_signed(self):
data = {'a': 'A', 'b': 'B'}
sig1 = utils.compute_signature(data, 'not-so-secret')
data['message_signature'] = sig1
self.assertTrue(utils.verify_signature(data, 'not-so-secret'))