def _generate_models(self): event_models = [] self.s_time = datetime.datetime(2013, 12, 31, 5, 0) event_models.append( models.Event(message_id='1', event_type='empty_ev', generated=self.s_time, traits=[ models.Trait('random', models.Trait.TEXT_TYPE, 'blah') ], raw={})) event_models.append( models.Event(message_id='2', event_type='admin_ev', generated=self.s_time, traits=[ models.Trait('project_id', models.Trait.TEXT_TYPE, self.admin_proj_id), models.Trait('user_id', models.Trait.TEXT_TYPE, self.admin_user_id) ], raw={})) event_models.append( models.Event(message_id='3', event_type='user_ev', generated=self.s_time, traits=[ models.Trait('project_id', models.Trait.TEXT_TYPE, self.proj_id), models.Trait('user_id', models.Trait.TEXT_TYPE, self.user_id) ], raw={})) self.conn.record_events(event_models)
def test_duplicate_message_id(self): now = datetime.datetime.utcnow() m = [ models.Event("1", "Foo", now, None, {}), models.Event("1", "Zoo", now, [], {}) ] with mock.patch('%s.LOG' % self.conn.record_events.__module__) as log: self.conn.record_events(m) self.assertEqual(1, log.info.call_count)
def record_events(self, events): if not isinstance(events, list): events = [events] event_list = [] for ev in events: try: event_list.append( models.Event(message_id=ev['message_id'], event_type=ev['event_type'], generated=timeutils.normalize_time( timeutils.parse_isotime(ev['generated'])), traits=[ models.Trait( name, dtype, models.Trait.convert_value( dtype, value)) for name, dtype, value in ev['traits'] ], raw=ev.get('raw', {}))) except Exception: LOG.exception( "Error processing event and it will be " "dropped: %s", ev) self.conn.record_events(event_list)
def _generate_models(self): event_models = [] base = 0 self.s_time = datetime.datetime(2013, 12, 31, 5, 0) self.trait_time = datetime.datetime(2013, 12, 31, 5, 0) for event_type in ['Foo', 'Bar', 'Zoo']: trait_models = [ models.Trait(name, type, value) for name, type, value in [( 'trait_A', models.Trait.TEXT_TYPE, "my_%s_text" % event_type), ('trait_B', models.Trait.INT_TYPE, base + 1), ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, self.trait_time)] ] # Message ID for test will be 'base'. So, message ID for the first # event will be '0', the second '100', and so on. # trait_time in first event will be equal to self.trait_time # (datetime.datetime(2013, 12, 31, 5, 0)), next will add 1 day, so # second will be (datetime.datetime(2014, 01, 01, 5, 0)) and so on. event_models.append( models.Event(message_id=str(base), event_type=event_type, generated=self.trait_time, traits=trait_models, raw={'status': { 'nested': 'started' }})) base += 100 self.trait_time += datetime.timedelta(days=1) self.conn.record_events(event_models)
def _generate_models(self): event_models = [] base = 0 self.s_time = datetime.datetime(2013, 12, 31, 5, 0) self.trait_time = datetime.datetime(2013, 12, 31, 5, 0) for i in range(20): trait_models = [ models.Trait(name, type, value) for name, type, value in [( 'trait_A', models.Trait.TEXT_TYPE, "my_text"), ('trait_B', models.Trait.INT_TYPE, base + 1), ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, self.trait_time)] ] event_models.append( models.Event(message_id=uuidutils.generate_uuid(), event_type='foo.bar', generated=self.trait_time, traits=trait_models, raw={'status': { 'nested': 'started' }})) self.trait_time += datetime.timedelta(seconds=1) self.conn.record_events(event_models)
def test_trait_bigint(self): big = 99999999999999 new_events = [ models.Event("id_testid", "MessageIDTest", self.start, [models.Trait('int', models.Trait.INT_TYPE, big)], {}) ] self.conn.record_events(new_events)
def make_test_data(conn, start, end, interval, event_types): # Compute start and end timestamps for the new data. if isinstance(start, datetime.datetime): timestamp = start else: timestamp = timeutils.parse_strtime(start) if not isinstance(end, datetime.datetime): end = timeutils.parse_strtime(end) increment = datetime.timedelta(minutes=interval) print('Adding new events') n = 0 while timestamp <= end: data = [] for i in range(event_types): traits = [ models.Trait('id1_%d' % i, 1, uuidutils.generate_uuid()), models.Trait('id2_%d' % i, 2, random.randint(1, 10)), models.Trait('id3_%d' % i, 3, random.random()), models.Trait('id4_%d' % i, 4, timestamp) ] data.append( models.Event(uuidutils.generate_uuid(), 'event_type%d' % i, timestamp, traits, {})) n += 1 conn.record_events(data) timestamp = timestamp + increment print('Added %d new events' % n)
def get_events(self, event_filter, pagination=None): """Return an iter of models.Event objects. :param event_filter: storage.EventFilter object, consists of filters for events that are stored in database. :param pagination: Pagination parameters. """ limit = None if pagination: if pagination.get('sort'): LOG.warning(_LW('Driver does not support sort functionality')) limit = pagination.get('limit') if limit == 0: return q = pymongo_utils.make_events_query_from_filter(event_filter) if limit is not None: results = self.db.event.find(q, limit=limit) else: results = self.db.event.find(q) for event in results: traits = [] for trait in event['traits']: traits.append( models.Trait(name=trait['trait_name'], dtype=int(trait['trait_type']), value=trait['trait_value'])) yield models.Event(message_id=event['_id'], event_type=event['event_type'], generated=event['timestamp'], traits=traits, raw=event.get('raw'))
def test_conn(self): event = models.Event(uuidutils.generate_uuid(), 'test', datetime.datetime(2012, 7, 2, 13, 53, 40), [], {}).serialize() with mock.patch.object(self.dispatcher.conn, 'record_events') as record_events: self.dispatcher.record_events(event) self.assertEqual(1, len(record_events.call_args_list[0][0][0]))
def test_bad_event(self): now = datetime.datetime.utcnow() broken_event = models.Event("1", "Foo", now, None, {}) del (broken_event.__dict__['raw']) m = [broken_event, broken_event] with mock.patch('%s.LOG' % self.conn.record_events.__module__) as log: self.assertRaises(AttributeError, self.conn.record_events, m) # ensure that record_events does not break on first error but # delays exception and tries to record each event. self.assertEqual(2, log.exception.call_count)
def setUp(self): super(TestBaseApiEventRBAC, self).setUp() traits = [ models.Trait('project_id', 1, 'project-good'), models.Trait('user_id', 1, 'user-good') ] self.message_id = uuidutils.generate_uuid() ev = models.Event(self.message_id, 'event_type', datetime.datetime.now(), traits, {}) self.conn.record_events([ev])
def test_get_by_message_id(self): new_events = [ models.Event("id_testid", "MessageIDTest", self.start, [], {}) ] self.conn.record_events(new_events) event_filter = storage.EventFilter(message_id="id_testid") events = [event for event in self.conn.get_events(event_filter)] self.assertEqual(1, len(events)) event = events[0] self.assertEqual("id_testid", event.message_id)
def test_simple_get_event_no_traits(self): new_events = [ models.Event("id_notraits", "NoTraits", self.start, [], {}) ] self.conn.record_events(new_events) event_filter = storage.EventFilter(self.start, self.end, "NoTraits") events = [event for event in self.conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("id_notraits", events[0].message_id) self.assertEqual("NoTraits", events[0].event_type) self.assertEqual(0, len(events[0].traits))
def _verify_data(self, trait, trait_table): now = datetime.datetime.utcnow() ev = models.Event('1', 'name', now, [trait], {}) self.conn.record_events([ev]) session = self.conn._engine_facade.get_session() t_tables = [ sql_models.TraitText, sql_models.TraitFloat, sql_models.TraitInt, sql_models.TraitDatetime ] for table in t_tables: if table == trait_table: self.assertEqual(1, session.query(table).count()) else: self.assertEqual(0, session.query(table).count())
def test_trait_type_enforced_on_none(self): new_events = [ models.Event("id_testid", "MessageIDTest", self.start, [ models.Trait('text', models.Trait.TEXT_TYPE, ''), models.Trait('int', models.Trait.INT_TYPE, 0), models.Trait('float', models.Trait.FLOAT_TYPE, 0.0) ], {}) ] self.conn.record_events(new_events) event_filter = storage.EventFilter(message_id="id_testid") events = [event for event in self.conn.get_events(event_filter)] options = [(models.Trait.TEXT_TYPE, ''), (models.Trait.INT_TYPE, 0.0), (models.Trait.FLOAT_TYPE, 0.0)] for trait in events[0].traits: options.remove((trait.dtype, trait.value))
def get_events(self, event_filter, pagination=None): limit = None if pagination: if pagination.get('sort'): LOG.warning('Driver does not support sort functionality') limit = pagination.get('limit') if limit == 0: return iclient = es.client.IndicesClient(self.conn) indices = iclient.get_mapping('%s_*' % self.index_name).keys() if indices: filter_args = self._make_dsl_from_filter(indices, event_filter) if limit is not None: filter_args['size'] = limit results = self.conn.search( fields=['_id', 'timestamp', '_type', '_source'], sort='timestamp:asc', **filter_args) trait_mappings = {} for record in results['hits']['hits']: trait_list = [] if not record['_type'] in trait_mappings: trait_mappings[record['_type']] = list( self.get_trait_types(record['_type'])) for key in record['_source']['traits'].keys(): value = record['_source']['traits'][key] for t_map in trait_mappings[record['_type']]: if t_map['name'] == key: dtype = t_map['data_type'] break else: dtype = models.Trait.TEXT_TYPE trait_list.append( models.Trait(name=key, dtype=dtype, value=models.Trait.convert_value( dtype, value))) gen_ts = timeutils.normalize_time( timeutils.parse_isotime(record['_source']['timestamp'])) yield models.Event(message_id=record['_id'], event_type=record['_type'], generated=gen_ts, traits=sorted( trait_list, key=operator.attrgetter('dtype')), raw=record['_source']['raw'])
def start_fixture(self): """Create some events.""" super(EventDataFixture, self).start_fixture() events = [] name_list = ['chocolate.chip', 'peanut.butter', 'sugar'] for ix, name in enumerate(name_list): timestamp = datetime.datetime.utcnow() message_id = 'fea1b15a-1d47-4175-85a5-a4bb2c72924{}'.format(ix) traits = [ models.Trait('type', 1, name), models.Trait('ate', 2, ix) ] event = models.Event(message_id, 'cookies_{}'.format(name), timestamp, traits, {'nested': { 'inside': 'value' }}) events.append(event) self.conn.record_events(events)
def get_events(self, event_filter, pagination=None): """Return an iter of models.Event objects. :param event_filter: storage.EventFilter object, consists of filters for events that are stored in database. :param pagination: Pagination parameters. """ limit = None if pagination: if pagination.get('sort'): LOG.warning('Driver does not support sort functionality') limit = pagination.get('limit') if limit == 0: return q, start, stop = hbase_utils.make_events_query_from_filter( event_filter) with self.conn_pool.connection() as conn: events_table = conn.table(self.EVENT_TABLE) gen = events_table.scan(filter=q, row_start=start, row_stop=stop, limit=limit) for event_id, data in gen: traits = [] events_dict = hbase_utils.deserialize_entry(data)[0] for key, value in events_dict.items(): if isinstance(key, tuple): trait_name, trait_dtype = key traits.append( models.Trait(name=trait_name, dtype=int(trait_dtype), value=value)) ts, mess = event_id.split(':') yield models.Event(message_id=hbase_utils.unquote(mess), event_type=events_dict['event_type'], generated=events_dict['timestamp'], traits=sorted(traits, key=operator.attrgetter('dtype')), raw=events_dict['raw'])
def prepare_data(self): self.models = [] base = 0 self.start = datetime.datetime(2013, 12, 31, 5, 0) now = self.start for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']: trait_models = [ models.Trait(name, dtype, value) for name, dtype, value in [('trait_A', models.Trait.TEXT_TYPE, "my_%s_text" % event_type), ('trait_B', models.Trait.INT_TYPE, base + 1), ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, now)] ] self.models.append( models.Event("id_%s_%d" % (event_type, base), event_type, now, trait_models, {'status': { 'nested': 'started' }})) base += 100 now = now + datetime.timedelta(hours=1) self.end = now self.conn.record_events(self.models)
def get_events(self, event_filter, pagination=None): """Return an iterable of model.Event objects. :param event_filter: EventFilter instance :param pagination: Pagination parameters. """ pagination = pagination or {} session = self._engine_facade.get_session() with session.begin(): # Build up the join conditions event_join_conditions = [ models.EventType.id == models.Event.event_type_id ] if event_filter.event_type: event_join_conditions.append( models.EventType.desc == event_filter.event_type) # Build up the where conditions event_filter_conditions = [] if event_filter.message_id: event_filter_conditions.append( models.Event.message_id == event_filter.message_id) if event_filter.start_timestamp: event_filter_conditions.append( models.Event.generated >= event_filter.start_timestamp) if event_filter.end_timestamp: event_filter_conditions.append( models.Event.generated <= event_filter.end_timestamp) trait_subq = None # Build trait filter if event_filter.traits_filter: filters = list(event_filter.traits_filter) trait_filter = filters.pop() key = trait_filter.pop('key') op = trait_filter.pop('op', 'eq') trait_type, value = list(trait_filter.items())[0] trait_model, conditions = _get_model_and_conditions( trait_type, key, value, op) trait_subq = (session.query( trait_model.event_id.label('ev_id')).filter(*conditions)) first_model = trait_model for label_num, trait_filter in enumerate(filters): key = trait_filter.pop('key') op = trait_filter.pop('op', 'eq') trait_type, value = list(trait_filter.items())[0] trait_model, conditions = _get_model_and_conditions( trait_type, key, value, op) trait_subq = (trait_subq.add_columns( trait_model.event_id.label('l%d' % label_num)).filter( first_model.event_id == trait_model.event_id, *conditions)) trait_subq = trait_subq.subquery() query = (session.query(models.Event.id).join( models.EventType, sa.and_(*event_join_conditions))) if trait_subq is not None: query = query.join(trait_subq, trait_subq.c.ev_id == models.Event.id) if event_filter.admin_proj: no_proj_q = session.query(models.TraitText.event_id).filter( models.TraitText.key == 'project_id') admin_q = (session.query( models.TraitText.event_id).filter(~sa.exists().where( models.TraitText.event_id == no_proj_q.subquery().c.event_id)).union( session.query(models.TraitText.event_id).filter( sa.and_( models.TraitText.key == 'project_id', models.TraitText.value == event_filter.admin_proj, models.Event.id == models.TraitText.event_id)))) query = query.filter(sa.exists().where( models.Event.id == admin_q.subquery().c.trait_text_event_id)) if event_filter_conditions: query = query.filter(sa.and_(*event_filter_conditions)) query = self._get_pagination_query(query, pagination, api_models.Event, models.Event) event_list = collections.OrderedDict() # get a list of all events that match filters for (id_, generated, message_id, desc, raw) in query.add_columns( models.Event.generated, models.Event.message_id, models.EventType.desc, models.Event.raw).all(): event_list[id_] = api_models.Event(message_id, desc, generated, [], raw) # Query all traits related to events. # NOTE (gordc): cast is done because pgsql defaults to TEXT when # handling unknown values such as null. trait_q = (session.query( models.TraitDatetime.event_id, models.TraitDatetime.key, models.TraitDatetime.value, sa.cast(sa.null(), sa.Integer), sa.cast(sa.null(), sa.Float(53)), sa.cast(sa.null(), sa.String(255))).filter(sa.exists().where( models.TraitDatetime.event_id == query.subquery().c.id)) ).union_all( session.query(models.TraitInt.event_id, models.TraitInt.key, sa.null(), models.TraitInt.value, sa.null(), sa.null()).filter(sa.exists().where( models.TraitInt.event_id == query.subquery().c.id)), session.query(models.TraitFloat.event_id, models.TraitFloat.key, sa.null(), sa.null(), models.TraitFloat.value, sa.null()).filter(sa.exists().where( models.TraitFloat.event_id == query.subquery().c.id)), session.query(models.TraitText.event_id, models.TraitText.key, sa.null(), sa.null(), sa.null(), models.TraitText.value).filter( sa.exists().where( models.TraitText.event_id == query.subquery().c.id))) for id_, key, t_date, t_int, t_float, t_text in (trait_q.order_by( models.TraitDatetime.key)).all(): if t_int is not None: dtype = api_models.Trait.INT_TYPE val = t_int elif t_float is not None: dtype = api_models.Trait.FLOAT_TYPE val = t_float elif t_date is not None: dtype = api_models.Trait.DATETIME_TYPE val = t_date else: dtype = api_models.Trait.TEXT_TYPE val = t_text try: trait_model = api_models.Trait(key, dtype, val) event_list[id_].append_trait(trait_model) except KeyError: # NOTE(gordc): this is expected as we do not set REPEATABLE # READ (bug 1506717). if query is run while recording new # event data, trait query may return more data than event # query. they can be safely discarded. pass return event_list.values()