def test_save_events_no_traits(self):
now = datetime.datetime.utcnow()
m = [models.Event("Foo", now, None), models.Event("Zoo", now, [])]
self.conn.record_events(m)
for model in m:
self.assertTrue(model.id >= 0)
self.assertNotEqual(m[0].id, m[1].id)
def test_bad_event(self):
now = datetime.datetime.utcnow()
m = [models.Event("1", "Foo", now, []),
models.Event("2", "Zoo", now, [])]
with mock.patch.object(self.conn, "_record_event") as mock_save:
mock_save.side_effect = MyException("Boom")
problem_events = self.conn.record_events(m)
self.assertEqual(2, len(problem_events))
for bad, event in problem_events:
self.assertEqual(bad, models.Event.UNKNOWN_PROBLEM)
def get_events(self, event_filter):
"""Return an iter of models.Event objects.
:param event_filter: storage.EventFilter object, consists of filters
for events that are stored in database.
"""
q, start, stop = hbase_utils.make_events_query_from_filter(
event_filter)
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
gen = events_table.scan(filter=q, row_start=start, row_stop=stop)
for event_id, data in gen:
traits = []
events_dict = hbase_utils.deserialize_entry(data)[0]
for key, value in events_dict.items():
if (not key.startswith('event_type')
and not key.startswith('timestamp')):
trait_name, trait_dtype = key.rsplit('+', 1)
traits.append(
models.Trait(name=trait_name,
dtype=int(trait_dtype),
value=value))
ts, mess = event_id.split('_', 1)
yield models.Event(message_id=mess,
event_type=events_dict['event_type'],
generated=events_dict['timestamp'],
traits=sorted(traits,
key=operator.attrgetter('dtype')))
def _generate_models(self):
event_models = []
base = 0
self.trait_time = datetime.datetime(2013, 12, 31, 5, 0)
for event_type in ['Foo', 'Bar', 'Zoo']:
trait_models = [models.Trait(name, type, value)
for name, type, value in [
('trait_A', models.Trait.TEXT_TYPE,
"my_%s_text" % event_type),
('trait_B', models.Trait.INT_TYPE,
base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D', models.Trait.DATETIME_TYPE,
self.trait_time)]]
# Message ID for test will be 'base'. So, message ID for the first
# event will be '0', the second '100', and so on.
event_models.append(
models.Event(message_id=str(base),
event_type=event_type,
generated=self.trait_time,
traits=trait_models))
base += 100
self.conn.record_events(event_models)
def get_events(self, event_filter):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start)
end = utils.dt_to_decimal(event_filter.end)
session = sqlalchemy_session.get_session()
with session.begin():
sub_query = session.query(Event.id)\
.join(Trait, Trait.event_id == Event.id)\
.filter(Event.generated >= start, Event.generated <= end)
if event_filter.event_name:
event_name = self._get_unique(session, event_filter.event_name)
sub_query = sub_query.filter(Event.unique_name == event_name)
if event_filter.traits:
for key, value in event_filter.traits.iteritems():
if key == 'key':
key = self._get_unique(session, value)
sub_query = sub_query.filter(Trait.name == key)
elif key == 't_string':
sub_query = sub_query.filter(Trait.t_string == value)
elif key == 't_int':
sub_query = sub_query.filter(Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
sub_query = sub_query.filter(Trait.t_datetime == dt)
elif key == 't_float':
sub_query = sub_query.filter(Trait.t_datetime == value)
sub_query = sub_query.subquery()
all_data = session.query(Trait)\
.join(sub_query, Trait.event_id == sub_query.c.id)
# Now convert the sqlalchemy objects back into Models ...
event_models_dict = {}
for trait in all_data.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(trait.event.unique_name.key,
generated, [])
event_models_dict[trait.event_id] = event
value = trait.get_value()
trait_model = api_models.Trait(trait.name.key, trait.t_type,
value)
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))
def to_event(self, notification_body):
event_type = notification_body['event_type']
message_id = notification_body['message_id']
when = self._extract_when(notification_body)
traits = (self.traits[t].to_trait(notification_body)
for t in self.traits)
# Only accept non-None value traits ...
traits = [trait for trait in traits if trait is not None]
event = models.Event(message_id, event_type, when, traits)
return event
def test_save_events_traits(self):
event_models = []
for event_name in ['Foo', 'Bar', 'Zoo']:
now = datetime.datetime.utcnow()
trait_models = \
[models.Trait(name, dtype, value)
for name, dtype, value in [
('trait_A', models.Trait.TEXT_TYPE, "my_text"),
('trait_B', models.Trait.INT_TYPE, 199),
('trait_C', models.Trait.FLOAT_TYPE, 1.23456),
('trait_D', models.Trait.DATETIME_TYPE, now)]]
event_models.append(models.Event(event_name, now, trait_models))
self.conn.record_events(event_models)
for model in event_models:
for trait in model.traits:
self.assertTrue(trait.id >= 0)
def _message_to_event(self, body):
"""Convert message to Ceilometer Event.
NOTE: this is currently based on the Nova notification format.
We will need to make this driver-based to support other formats.
NOTE: the rpc layer currently rips out the notification
delivery_info, which is critical to determining the
source of the notification. This will have to get added back later.
"""
event_name = body['event_type']
when = self._extract_when(body)
LOG.debug('Saving event "%s"', event_name)
message_id = body.get('message_id')
# TODO(sandy) - check we have not already saved this notification.
# (possible on retries) Use message_id to spot dups.
publisher = body.get('publisher_id')
request_id = body.get('_context_request_id')
tenant_id = body.get('_context_tenant')
text = models.Trait.TEXT_TYPE
all_traits = [
models.Trait('message_id', text, message_id),
models.Trait('service', text, publisher),
models.Trait('request_id', text, request_id),
models.Trait('tenant_id', text, tenant_id),
]
# Only store non-None value traits ...
traits = [trait for trait in all_traits if trait.value is not None]
event = models.Event(event_name, when, traits)
exc_info = None
for dispatcher in self.dispatcher_manager:
try:
dispatcher.obj.record_events(event)
except Exception:
LOG.exception('Error while saving events with dispatcher %s',
dispatcher)
exc_info = sys.exc_info()
# Don't ack the message if any of the dispatchers fail
if exc_info:
raise exc_info[1], None, exc_info[2]
def get_events(self, event_filter):
"""Return an iter of models.Event objects.
:param event_filter: storage.EventFilter object, consists of filters
for events that are stored in database.
"""
q = pymongo_utils.make_events_query_from_filter(event_filter)
for event in self.db.event.find(q):
traits = []
for trait in event['traits']:
traits.append(
models.Trait(name=trait['trait_name'],
dtype=int(trait['trait_type']),
value=trait['trait_value']))
yield models.Event(message_id=event['_id'],
event_type=event['event_type'],
generated=event['timestamp'],
traits=traits)
def _message_to_event(self, body):
"""Convert message to Ceilometer Event.
NOTE: this is currently based on the Nova notification format.
We will need to make this driver-based to support other formats.
NOTE: the rpc layer currently rips out the notification
delivery_info, which is critical to determining the
source of the notification. This will have to get added back later.
"""
event_name = body['event_type']
when = self._extract_when(body)
LOG.debug('Saving event "%s"', event_name)
message_id = body.get('message_id')
# TODO(sandy) - check we have not already saved this notification.
# (possible on retries) Use message_id to spot dups.
publisher = body.get('publisher_id')
request_id = body.get('_context_request_id')
tenant_id = body.get('_context_tenant')
text = models.Trait.TEXT_TYPE
all_traits = [
models.Trait('message_id', text, message_id),
models.Trait('service', text, publisher),
models.Trait('request_id', text, request_id),
models.Trait('tenant_id', text, tenant_id),
]
# Only store non-None value traits ...
traits = [trait for trait in all_traits if trait.value is not None]
event = models.Event(event_name, when, traits)
try:
self.storage_conn.record_events([
event,
])
except Exception as err:
LOG.exception(_("Unable to store events: %s"), err)
# By re-raising we avoid ack()'ing the message.
raise
def prepare_data(self):
event_models = []
base = 0
self.start = datetime.datetime(2013, 12, 31, 5, 0)
now = self.start
for event_name in ['Foo', 'Bar', 'Zoo']:
trait_models = \
[models.Trait(name, dtype, value)
for name, dtype, value in [
('trait_A', models.Trait.TEXT_TYPE,
"my_%s_text" % event_name),
('trait_B', models.Trait.INT_TYPE,
base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D', models.Trait.DATETIME_TYPE, now)]]
event_models.append(models.Event(event_name, now, trait_models))
base += 100
now = now + datetime.timedelta(hours=1)
self.end = now
self.conn.record_events(event_models)
def _message_to_event(self, body):
"""Convert message to Ceilometer Event.
NOTE: this is currently based on the Nova notification format.
We will need to make this driver-based to support other formats.
NOTE: the rpc layer currently rips out the notification
delivery_info, which is critical to determining the
source of the notification. This will have to get added back later.
"""
message_id = body.get('message_id')
event_name = body['event_type']
when = self._extract_when(body)
LOG.debug('Saving event "%s"', event_name)
publisher = body.get('publisher_id')
request_id = body.get('_context_request_id')
tenant_id = body.get('_context_tenant')
text = models.Trait.TEXT_TYPE
all_traits = [
models.Trait('service', text, publisher),
models.Trait('request_id', text, request_id),
models.Trait('tenant_id', text, tenant_id),
]
# Only store non-None value traits ...
traits = [trait for trait in all_traits if trait.value is not None]
event = models.Event(message_id, event_name, when, traits)
problem_events = []
for dispatcher in self.dispatcher_manager:
problem_events.extend(dispatcher.obj.record_events(event))
if models.Event.UNKNOWN_PROBLEM in [x[0] for x in problem_events]:
# Don't ack the message, raise to requeue it
# if ack_on_error = False
raise UnableToSaveEventException()
def test_event_repr_no_traits(self):
x = models.Event("1", "name", "now", None)
self.assertEqual("<Event: 1, name, now, >", repr(x))
def get_events(self, event_filter):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
"""
start = event_filter.start_time
end = event_filter.end_time
session = self._engine_facade.get_session()
LOG.debug(_("Getting events that match filter: %s") % event_filter)
with session.begin():
event_query = session.query(models.Event)
# Build up the join conditions
event_join_conditions = [
models.EventType.id == models.Event.event_type_id
]
if event_filter.event_type:
event_join_conditions.append(
models.EventType.desc == event_filter.event_type)
event_query = event_query.join(models.EventType,
and_(*event_join_conditions))
# Build up the where conditions
event_filter_conditions = []
if event_filter.message_id:
event_filter_conditions.append(
models.Event.message_id == event_filter.message_id)
if start:
event_filter_conditions.append(models.Event.generated >= start)
if end:
event_filter_conditions.append(models.Event.generated <= end)
if event_filter_conditions:
event_query = (event_query.filter(
and_(*event_filter_conditions)))
event_models_dict = {}
if event_filter.traits_filter:
for trait_filter in event_filter.traits_filter:
# Build a sub query that joins Trait to TraitType
# where the trait name matches
trait_name = trait_filter.pop('key')
op = trait_filter.pop('op', 'eq')
conditions = [
models.Trait.trait_type_id == models.TraitType.id,
models.TraitType.desc == trait_name
]
for key, value in six.iteritems(trait_filter):
sql_utils.trait_op_condition(conditions, key, value,
op)
trait_query = (session.query(models.Trait.event_id).join(
models.TraitType, and_(*conditions)).subquery())
event_query = (event_query.join(
trait_query,
models.Event.id == trait_query.c.event_id))
else:
# If there are no trait filters, grab the events from the db
query = (session.query(models.Event.id, models.Event.generated,
models.Event.message_id,
models.EventType.desc).join(
models.EventType,
and_(*event_join_conditions)))
if event_filter_conditions:
query = query.filter(and_(*event_filter_conditions))
for (id_, generated, message_id, desc_) in query.all():
event_models_dict[id_] = api_models.Event(
message_id, desc_, generated, [])
# Build event models for the events
event_query = event_query.subquery()
query = (session.query(models.Trait).join(
models.TraitType,
models.Trait.trait_type_id == models.TraitType.id).join(
event_query, models.Trait.event_id == event_query.c.id))
# Now convert the sqlalchemy objects back into Models ...
for trait in query.all():
event = event_models_dict.get(trait.event_id)
if not event:
event = api_models.Event(trait.event.message_id,
trait.event.event_type.desc,
trait.event.generated, [])
event_models_dict[trait.event_id] = event
trait_model = api_models.Trait(trait.trait_type.desc,
trait.trait_type.data_type,
trait.get_value())
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))