def _generate_models(self): event_models = [] base = 0 self.trait_time = datetime.datetime(2013, 12, 31, 5, 0) for event_type in ['Foo', 'Bar', 'Zoo']: trait_models = [models.Trait(name, type, value) for name, type, value in [ ('trait_A', models.Trait.TEXT_TYPE, "my_%s_text" % event_type), ('trait_B', models.Trait.INT_TYPE, base + 1), ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, self.trait_time)]] # Message ID for test will be 'base'. So, message ID for the first # event will be '0', the second '100', and so on. event_models.append( models.Event(message_id=str(base), event_type=event_type, generated=self.trait_time, traits=trait_models)) base += 100 self.conn.record_events(event_models)
def get_traits(self, event_type, trait_type=None): """Return all trait instances associated with an event_type. If trait_type is specified, only return instances of that trait type. :param event_type: the type of the Event to filter by :param trait_type: the name of the Trait to filter by """ q = hbase_utils.make_query(event_type=event_type, trait_type=trait_type) traits = [] with self.conn_pool.connection() as conn: events_table = conn.table(self.EVENT_TABLE) gen = events_table.scan(filter=q) for event_id, data in gen: events_dict = hbase_utils.deserialize_entry(data)[0] for key, value in events_dict.items(): if (not key.startswith('event_type') and not key.startswith('timestamp')): name, tt_number = key.rsplit('+', 1) traits.append( models.Trait(name=name, dtype=int(tt_number), value=value)) for trait in sorted(traits, key=operator.attrgetter('dtype')): yield trait
def get_traits(self, event_type, trait_name=None): """Return all trait instances associated with an event_type. If trait_type is specified, only return instances of that trait type. :param event_type: the type of the Event to filter by :param trait_name: the name of the Trait to filter by """ if not trait_name: events = self.db.event.find({'event_type': event_type}) else: # We choose events that simultaneously have event_type and certain # trait_name, and retrieve events contains only mentioned traits. events = self.db.event.find( { '$and': [{ 'event_type': event_type }, { 'traits.trait_name': trait_name }] }, {'traits': { '$elemMatch': { 'trait_name': trait_name } }}) traits = [] for event in events: for trait in event['traits']: traits.append( models.Trait(name=trait['trait_name'], dtype=trait['trait_type'], value=trait['trait_value'])) for trait in sorted(traits, key=operator.attrgetter('dtype')): yield trait
def get_traits(self, event_type, trait_type=None): """Return all trait instances associated with an event_type. If trait_type is specified, only return instances of that trait type. :param event_type: the type of the Event to filter by :param trait_type: the name of the Trait to filter by """ session = self._engine_facade.get_session() with session.begin(): trait_type_filters = [ models.TraitType.id == models.Trait.trait_type_id ] if trait_type: trait_type_filters.append(models.TraitType.desc == trait_type) query = (session.query(models.Trait).join( models.TraitType, and_(*trait_type_filters)).join( models.Event, models.Event.id == models.Trait.event_id).join( models.EventType, and_(models.EventType.id == models.Event.event_type_id, models.EventType.desc == event_type))) for trait in query.all(): type = trait.trait_type yield api_models.Trait(name=type.desc, dtype=type.data_type, value=trait.get_value())
def get_events(self, event_filter): """Return an iter of models.Event objects. :param event_filter: storage.EventFilter object, consists of filters for events that are stored in database. """ q, start, stop = hbase_utils.make_events_query_from_filter( event_filter) with self.conn_pool.connection() as conn: events_table = conn.table(self.EVENT_TABLE) gen = events_table.scan(filter=q, row_start=start, row_stop=stop) for event_id, data in gen: traits = [] events_dict = hbase_utils.deserialize_entry(data)[0] for key, value in events_dict.items(): if (not key.startswith('event_type') and not key.startswith('timestamp')): trait_name, trait_dtype = key.rsplit('+', 1) traits.append( models.Trait(name=trait_name, dtype=int(trait_dtype), value=value)) ts, mess = event_id.split('_', 1) yield models.Event(message_id=mess, event_type=events_dict['event_type'], generated=events_dict['timestamp'], traits=sorted(traits, key=operator.attrgetter('dtype')))
def _message_to_event(self, body): """Convert message to Ceilometer Event. NOTE: this is currently based on the Nova notification format. We will need to make this driver-based to support other formats. NOTE: the rpc layer currently rips out the notification delivery_info, which is critical to determining the source of the notification. This will have to get added back later. """ event_name = body['event_type'] when = self._extract_when(body) LOG.debug('Saving event "%s"', event_name) message_id = body.get('message_id') # TODO(sandy) - check we have not already saved this notification. # (possible on retries) Use message_id to spot dups. publisher = body.get('publisher_id') request_id = body.get('_context_request_id') tenant_id = body.get('_context_tenant') text = models.Trait.TEXT_TYPE all_traits = [ models.Trait('message_id', text, message_id), models.Trait('service', text, publisher), models.Trait('request_id', text, request_id), models.Trait('tenant_id', text, tenant_id), ] # Only store non-None value traits ... traits = [trait for trait in all_traits if trait.value is not None] event = models.Event(event_name, when, traits) exc_info = None for dispatcher in self.dispatcher_manager: try: dispatcher.obj.record_events(event) except Exception: LOG.exception('Error while saving events with dispatcher %s', dispatcher) exc_info = sys.exc_info() # Don't ack the message if any of the dispatchers fail if exc_info: raise exc_info[1], None, exc_info[2]
def test_int_traits(self): model = models.Trait("Foo", models.Trait.INT_TYPE, 100) trait = self.conn._make_trait(model, None) self.assertEqual(trait.trait_type.data_type, models.Trait.INT_TYPE) self.assertIsNone(trait.t_float) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_datetime) self.assertEqual(trait.t_int, 100) self.assertIsNotNone(trait.trait_type.desc)
def test_string_traits(self): model = models.Trait("Foo", models.Trait.TEXT_TYPE, "my_text") trait = self.conn._make_trait(model, None) self.assertEqual(models.Trait.TEXT_TYPE, trait.trait_type.data_type) self.assertIsNone(trait.t_float) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_datetime) self.assertEqual("my_text", trait.t_string) self.assertIsNotNone(trait.trait_type.desc)
def test_string_traits(self): model = models.Trait("Foo", models.Trait.TEXT_TYPE, "my_text") trait = self.conn._make_trait(model, None) self.assertEquals(trait.t_type, models.Trait.TEXT_TYPE) self.assertIsNone(trait.t_float) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_datetime) self.assertEquals(trait.t_string, "my_text") self.assertIsNotNone(trait.name)
def test_float_traits(self): model = models.Trait("Foo", models.Trait.FLOAT_TYPE, 123.456) trait = self.conn._make_trait(model, None) self.assertEquals(trait.t_type, models.Trait.FLOAT_TYPE) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_datetime) self.assertEquals(trait.t_float, 123.456) self.assertIsNotNone(trait.name)
def test_float_traits(self): model = models.Trait("Foo", models.Trait.FLOAT_TYPE, 123.456) trait = self.conn._make_trait(model, None) self.assertEqual(models.Trait.FLOAT_TYPE, trait.trait_type.data_type) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_datetime) self.assertEqual(123.456, trait.t_float) self.assertIsNotNone(trait.trait_type.desc)
def test_datetime_traits(self): now = datetime.datetime.utcnow() model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) trait = self.conn._make_trait(model, None) self.assertEquals(trait.t_type, models.Trait.DATETIME_TYPE) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_float) self.assertEquals(trait.t_datetime, utils.dt_to_decimal(now)) self.assertIsNotNone(trait.name)
def _message_to_event(self, body): """Convert message to Ceilometer Event. NOTE: this is currently based on the Nova notification format. We will need to make this driver-based to support other formats. NOTE: the rpc layer currently rips out the notification delivery_info, which is critical to determining the source of the notification. This will have to get added back later. """ event_name = body['event_type'] when = self._extract_when(body) LOG.debug('Saving event "%s"', event_name) message_id = body.get('message_id') # TODO(sandy) - check we have not already saved this notification. # (possible on retries) Use message_id to spot dups. publisher = body.get('publisher_id') request_id = body.get('_context_request_id') tenant_id = body.get('_context_tenant') text = models.Trait.TEXT_TYPE all_traits = [ models.Trait('message_id', text, message_id), models.Trait('service', text, publisher), models.Trait('request_id', text, request_id), models.Trait('tenant_id', text, tenant_id), ] # Only store non-None value traits ... traits = [trait for trait in all_traits if trait.value is not None] event = models.Event(event_name, when, traits) try: self.storage_conn.record_events([ event, ]) except Exception as err: LOG.exception(_("Unable to store events: %s"), err) # By re-raising we avoid ack()'ing the message. raise
def test_datetime_traits(self): now = datetime.datetime.utcnow() model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) trait = self.conn._make_trait(model, None) self.assertEqual(models.Trait.DATETIME_TYPE, trait.trait_type.data_type) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_float) self.assertEqual(now, trait.t_datetime) self.assertIsNotNone(trait.trait_type.desc)
def get_events(self, event_filter): """Return an iterable of model.Event objects. :param event_filter: EventFilter instance """ start = utils.dt_to_decimal(event_filter.start) end = utils.dt_to_decimal(event_filter.end) session = sqlalchemy_session.get_session() with session.begin(): sub_query = session.query(Event.id)\ .join(Trait, Trait.event_id == Event.id)\ .filter(Event.generated >= start, Event.generated <= end) if event_filter.event_name: event_name = self._get_unique(session, event_filter.event_name) sub_query = sub_query.filter(Event.unique_name == event_name) if event_filter.traits: for key, value in event_filter.traits.iteritems(): if key == 'key': key = self._get_unique(session, value) sub_query = sub_query.filter(Trait.name == key) elif key == 't_string': sub_query = sub_query.filter(Trait.t_string == value) elif key == 't_int': sub_query = sub_query.filter(Trait.t_int == value) elif key == 't_datetime': dt = utils.dt_to_decimal(value) sub_query = sub_query.filter(Trait.t_datetime == dt) elif key == 't_float': sub_query = sub_query.filter(Trait.t_datetime == value) sub_query = sub_query.subquery() all_data = session.query(Trait)\ .join(sub_query, Trait.event_id == sub_query.c.id) # Now convert the sqlalchemy objects back into Models ... event_models_dict = {} for trait in all_data.all(): event = event_models_dict.get(trait.event_id) if not event: generated = utils.decimal_to_dt(trait.event.generated) event = api_models.Event(trait.event.unique_name.key, generated, []) event_models_dict[trait.event_id] = event value = trait.get_value() trait_model = api_models.Trait(trait.name.key, trait.t_type, value) event.append_trait(trait_model) event_models = event_models_dict.values() return sorted(event_models, key=operator.attrgetter('generated'))
def _message_to_event(self, body): """Convert message to Ceilometer Event. NOTE: this is currently based on the Nova notification format. We will need to make this driver-based to support other formats. NOTE: the rpc layer currently rips out the notification delivery_info, which is critical to determining the source of the notification. This will have to get added back later. """ message_id = body.get('message_id') event_name = body['event_type'] when = self._extract_when(body) LOG.debug('Saving event "%s"', event_name) publisher = body.get('publisher_id') request_id = body.get('_context_request_id') tenant_id = body.get('_context_tenant') text = models.Trait.TEXT_TYPE all_traits = [ models.Trait('service', text, publisher), models.Trait('request_id', text, request_id), models.Trait('tenant_id', text, tenant_id), ] # Only store non-None value traits ... traits = [trait for trait in all_traits if trait.value is not None] event = models.Event(message_id, event_name, when, traits) problem_events = [] for dispatcher in self.dispatcher_manager: problem_events.extend(dispatcher.obj.record_events(event)) if models.Event.UNKNOWN_PROBLEM in [x[0] for x in problem_events]: # Don't ack the message, raise to requeue it # if ack_on_error = False raise UnableToSaveEventException()
def test_save_events_traits(self): event_models = [] for event_name in ['Foo', 'Bar', 'Zoo']: now = datetime.datetime.utcnow() trait_models = \ [models.Trait(name, dtype, value) for name, dtype, value in [ ('trait_A', models.Trait.TEXT_TYPE, "my_text"), ('trait_B', models.Trait.INT_TYPE, 199), ('trait_C', models.Trait.FLOAT_TYPE, 1.23456), ('trait_D', models.Trait.DATETIME_TYPE, now)]] event_models.append(models.Event(event_name, now, trait_models)) self.conn.record_events(event_models) for model in event_models: for trait in model.traits: self.assertTrue(trait.id >= 0)
def get_events(self, event_filter): """Return an iter of models.Event objects. :param event_filter: storage.EventFilter object, consists of filters for events that are stored in database. """ q = pymongo_utils.make_events_query_from_filter(event_filter) for event in self.db.event.find(q): traits = [] for trait in event['traits']: traits.append( models.Trait(name=trait['trait_name'], dtype=int(trait['trait_type']), value=trait['trait_value'])) yield models.Event(message_id=event['_id'], event_type=event['event_type'], generated=event['timestamp'], traits=traits)
def to_trait(self, notification_body): values = [match for match in self.fields.find(notification_body) if match.value is not None] if self.plugin is not None: value_map = [('.'.join(self._get_path(match)), match.value) for match in values] value = self.plugin.trait_value(value_map) else: value = values[0].value if values else None if value is None: return None # NOTE(mdragon): some openstack projects (mostly Nova) emit '' # for null fields for things like dates. if self.trait_type != models.Trait.TEXT_TYPE and value == '': return None value = models.Trait.convert_value(self.trait_type, value) return models.Trait(self.name, self.trait_type, value)
def prepare_data(self): event_models = [] base = 0 self.start = datetime.datetime(2013, 12, 31, 5, 0) now = self.start for event_name in ['Foo', 'Bar', 'Zoo']: trait_models = \ [models.Trait(name, dtype, value) for name, dtype, value in [ ('trait_A', models.Trait.TEXT_TYPE, "my_%s_text" % event_name), ('trait_B', models.Trait.INT_TYPE, base + 1), ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, now)]] event_models.append(models.Event(event_name, now, trait_models)) base += 100 now = now + datetime.timedelta(hours=1) self.end = now self.conn.record_events(event_models)
def get_events(self, event_filter): """Return an iterable of model.Event objects. :param event_filter: EventFilter instance """ start = event_filter.start_time end = event_filter.end_time session = self._engine_facade.get_session() LOG.debug(_("Getting events that match filter: %s") % event_filter) with session.begin(): event_query = session.query(models.Event) # Build up the join conditions event_join_conditions = [ models.EventType.id == models.Event.event_type_id ] if event_filter.event_type: event_join_conditions.append( models.EventType.desc == event_filter.event_type) event_query = event_query.join(models.EventType, and_(*event_join_conditions)) # Build up the where conditions event_filter_conditions = [] if event_filter.message_id: event_filter_conditions.append( models.Event.message_id == event_filter.message_id) if start: event_filter_conditions.append(models.Event.generated >= start) if end: event_filter_conditions.append(models.Event.generated <= end) if event_filter_conditions: event_query = (event_query.filter( and_(*event_filter_conditions))) event_models_dict = {} if event_filter.traits_filter: for trait_filter in event_filter.traits_filter: # Build a sub query that joins Trait to TraitType # where the trait name matches trait_name = trait_filter.pop('key') op = trait_filter.pop('op', 'eq') conditions = [ models.Trait.trait_type_id == models.TraitType.id, models.TraitType.desc == trait_name ] for key, value in six.iteritems(trait_filter): sql_utils.trait_op_condition(conditions, key, value, op) trait_query = (session.query(models.Trait.event_id).join( models.TraitType, and_(*conditions)).subquery()) event_query = (event_query.join( trait_query, models.Event.id == trait_query.c.event_id)) else: # If there are no trait filters, grab the events from the db query = (session.query(models.Event.id, models.Event.generated, models.Event.message_id, models.EventType.desc).join( models.EventType, and_(*event_join_conditions))) if event_filter_conditions: query = query.filter(and_(*event_filter_conditions)) for (id_, generated, message_id, desc_) in query.all(): event_models_dict[id_] = api_models.Event( message_id, desc_, generated, []) # Build event models for the events event_query = event_query.subquery() query = (session.query(models.Trait).join( models.TraitType, models.Trait.trait_type_id == models.TraitType.id).join( event_query, models.Trait.event_id == event_query.c.id)) # Now convert the sqlalchemy objects back into Models ... for trait in query.all(): event = event_models_dict.get(trait.event_id) if not event: event = api_models.Event(trait.event.message_id, trait.event.event_type.desc, trait.event.generated, []) event_models_dict[trait.event_id] = event trait_model = api_models.Trait(trait.trait_type.desc, trait.trait_type.data_type, trait.get_value()) event.append_trait(trait_model) event_models = event_models_dict.values() return sorted(event_models, key=operator.attrgetter('generated'))