def _generate_models(self):
event_models = []
self.s_time = datetime.datetime(2013, 12, 31, 5, 0)
event_models.append(
models.Event(message_id='1',
event_type='empty_ev',
generated=self.s_time,
traits=[
models.Trait('random', models.Trait.TEXT_TYPE,
'blah')
],
raw={}))
event_models.append(
models.Event(message_id='2',
event_type='admin_ev',
generated=self.s_time,
traits=[
models.Trait('project_id', models.Trait.TEXT_TYPE,
self.admin_proj_id),
models.Trait('user_id', models.Trait.TEXT_TYPE,
self.admin_user_id)
],
raw={}))
event_models.append(
models.Event(message_id='3',
event_type='user_ev',
generated=self.s_time,
traits=[
models.Trait('project_id', models.Trait.TEXT_TYPE,
self.proj_id),
models.Trait('user_id', models.Trait.TEXT_TYPE,
self.user_id)
],
raw={}))
self.event_conn.record_events(event_models)
def test_duplicate_message_id(self):
now = datetime.datetime.utcnow()
m = [event_models.Event("1", "Foo", now, None, {}),
event_models.Event("1", "Zoo", now, [], {})]
with mock.patch('%s.LOG' %
self.event_conn.record_events.__module__) as log:
self.event_conn.record_events(m)
self.assertEqual(1, log.info.call_count)
def prepare_data(self):
self.event_models = []
base = 0
self.start = datetime.datetime(2013, 12, 31, 5, 0)
now = self.start
for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']:
trait_models = [event_models.Trait(name, dtype, value)
for name, dtype, value in [
('trait_A', event_models.Trait.TEXT_TYPE,
"my_%s_text" % event_type),
('trait_B', event_models.Trait.INT_TYPE,
base + 1),
('trait_C', event_models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D', event_models.Trait.DATETIME_TYPE,
now)]]
self.event_models.append(
event_models.Event("id_%s_%d" % (event_type, base),
event_type, now, trait_models,
{'status': {'nested': 'started'}}))
base += 100
now = now + datetime.timedelta(hours=1)
self.end = now
self.event_conn.record_events(self.event_models)
def make_test_data(conn, start, end, interval, event_types):
# Compute start and end timestamps for the new data.
if isinstance(start, datetime.datetime):
timestamp = start
else:
timestamp = timeutils.parse_strtime(start)
if not isinstance(end, datetime.datetime):
end = timeutils.parse_strtime(end)
increment = datetime.timedelta(minutes=interval)
print('Adding new events')
n = 0
while timestamp <= end:
data = []
for i in range(event_types):
traits = [models.Trait('id1_%d' % i, 1, str(uuid.uuid4())),
models.Trait('id2_%d' % i, 2, random.randint(1, 10)),
models.Trait('id3_%d' % i, 3, random.random()),
models.Trait('id4_%d' % i, 4, timestamp)]
data.append(models.Event(str(uuid.uuid4()),
'event_type%d' % i,
timestamp,
traits,
{}))
n += 1
conn.record_events(data)
timestamp = timestamp + increment
print('Added %d new events' % n)
def _generate_models(self):
event_models = []
base = 0
self.s_time = datetime.datetime(2013, 12, 31, 5, 0)
self.trait_time = datetime.datetime(2013, 12, 31, 5, 0)
for i in range(20):
trait_models = [
models.Trait(name, type, value)
for name, type, value in [(
'trait_A', models.Trait.TEXT_TYPE,
"my_text"), ('trait_B', models.Trait.INT_TYPE, base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D',
models.Trait.DATETIME_TYPE,
self.trait_time)]
]
event_models.append(
models.Event(message_id=str(uuid.uuid4()),
event_type='foo.bar',
generated=self.trait_time,
traits=trait_models,
raw={'status': {
'nested': 'started'
}}))
self.trait_time += datetime.timedelta(seconds=1)
self.event_conn.record_events(event_models)
def _generate_models(self):
event_models = []
base = 0
self.s_time = datetime.datetime(2013, 12, 31, 5, 0)
self.trait_time = datetime.datetime(2013, 12, 31, 5, 0)
for event_type in ['Foo', 'Bar', 'Zoo']:
trait_models = [
models.Trait(name, type, value)
for name, type, value in [(
'trait_A', models.Trait.TEXT_TYPE, "my_%s_text" %
event_type), ('trait_B', models.Trait.INT_TYPE, base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D',
models.Trait.DATETIME_TYPE,
self.trait_time)]
]
# Message ID for test will be 'base'. So, message ID for the first
# event will be '0', the second '100', and so on.
# trait_time in first event will be equal to self.trait_time
# (datetime.datetime(2013, 12, 31, 5, 0)), next will add 1 day, so
# second will be (datetime.datetime(2014, 01, 01, 5, 0)) and so on.
event_models.append(
models.Event(message_id=str(base),
event_type=event_type,
generated=self.trait_time,
traits=trait_models,
raw={'status': {
'nested': 'started'
}}))
base += 100
self.trait_time += datetime.timedelta(days=1)
self.event_conn.record_events(event_models)
def test_event_conn(self):
event = event_models.Event(uuid.uuid4(), 'test',
datetime.datetime(2012, 7, 2, 13, 53, 40),
[], {}).serialize()
with mock.patch.object(self.dispatcher.event_conn,
'record_events') as record_events:
self.dispatcher.record_events(event)
self.assertEqual(1, len(record_events.call_args_list[0][0][0]))
def setUp(self):
super(TestBaseApiEventRBAC, self).setUp()
traits = [ev_model.Trait('project_id', 1, 'project-good'),
ev_model.Trait('user_id', 1, 'user-good')]
self.message_id = str(uuid.uuid4())
ev = ev_model.Event(self.message_id, 'event_type',
datetime.datetime.now(), traits, {})
self.event_conn.record_events([ev])
def test_simple_get_event_no_traits(self):
new_events = [event_models.Event("id_notraits", "NoTraits",
self.start, [], {})]
self.event_conn.record_events(new_events)
event_filter = event_storage.EventFilter(
self.start, self.end, "NoTraits")
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("id_notraits", events[0].message_id)
self.assertEqual("NoTraits", events[0].event_type)
self.assertEqual(0, len(events[0].traits))
def test_bad_event(self):
now = datetime.datetime.utcnow()
broken_event = event_models.Event("1", "Foo", now, None, {})
del(broken_event.__dict__['raw'])
m = [broken_event, broken_event]
with mock.patch('%s.LOG' %
self.event_conn.record_events.__module__) as log:
self.assertRaises(AttributeError, self.event_conn.record_events, m)
# ensure that record_events does not break on first error but
# delays exception and tries to record each event.
self.assertEqual(2, log.exception.call_count)
def test_get_by_message_id(self):
new_events = [event_models.Event("id_testid",
"MessageIDTest",
self.start,
[], {})]
self.event_conn.record_events(new_events)
event_filter = event_storage.EventFilter(message_id="id_testid")
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
event = events[0]
self.assertEqual("id_testid", event.message_id)
def _verify_data(self, trait, trait_table):
now = datetime.datetime.utcnow()
ev = models.Event('1', 'name', now, [trait], {})
self.event_conn.record_events([ev])
session = self.event_conn._engine_facade.get_session()
t_tables = [
sql_models.TraitText, sql_models.TraitFloat, sql_models.TraitInt,
sql_models.TraitDatetime
]
for table in t_tables:
if table == trait_table:
self.assertEqual(1, session.query(table).count())
else:
self.assertEqual(0, session.query(table).count())
def test_trait_type_enforced_on_none(self):
new_events = [event_models.Event(
"id_testid", "MessageIDTest", self.start,
[event_models.Trait('text', event_models.Trait.TEXT_TYPE, ''),
event_models.Trait('int', event_models.Trait.INT_TYPE, 0),
event_models.Trait('float', event_models.Trait.FLOAT_TYPE, 0.0)],
{})]
self.event_conn.record_events(new_events)
event_filter = event_storage.EventFilter(message_id="id_testid")
events = [event for event in self.event_conn.get_events(event_filter)]
options = [(event_models.Trait.TEXT_TYPE, ''),
(event_models.Trait.INT_TYPE, 0.0),
(event_models.Trait.FLOAT_TYPE, 0.0)]
for trait in events[0].traits:
options.remove((trait.dtype, trait.value))
def start_fixture(self):
"""Create some events."""
super(EventDataFixture, self).start_fixture()
events = []
name_list = ['chocolate.chip', 'peanut.butter', 'sugar']
for ix, name in enumerate(name_list):
timestamp = datetime.datetime.utcnow()
message_id = 'fea1b15a-1d47-4175-85a5-a4bb2c72924{}'.format(ix)
traits = [models.Trait('type', 1, name),
models.Trait('ate', 2, ix)]
event = models.Event(message_id,
'cookies_{}'.format(name),
timestamp,
traits, {'nested': {'inside': 'value'}})
events.append(event)
self.conn.record_events(events)
def get_events(self, event_filter, pagination=None):
limit = None
if pagination:
if pagination.get('sort'):
LOG.warning(_LW('Driver does not support sort functionality'))
limit = pagination.get('limit')
if limit == 0:
return
iclient = es.client.IndicesClient(self.conn)
indices = iclient.get_mapping('%s_*' % self.index_name).keys()
if indices:
filter_args = self._make_dsl_from_filter(indices, event_filter)
if limit is not None:
filter_args['size'] = limit
results = self.conn.search(
fields=['_id', 'timestamp', '_type', '_source'],
sort='timestamp:asc',
**filter_args)
trait_mappings = {}
for record in results['hits']['hits']:
trait_list = []
if not record['_type'] in trait_mappings:
trait_mappings[record['_type']] = list(
self.get_trait_types(record['_type']))
for key in record['_source']['traits'].keys():
value = record['_source']['traits'][key]
for t_map in trait_mappings[record['_type']]:
if t_map['name'] == key:
dtype = t_map['data_type']
break
else:
dtype = models.Trait.TEXT_TYPE
trait_list.append(
models.Trait(name=key,
dtype=dtype,
value=models.Trait.convert_value(
dtype, value)))
gen_ts = timeutils.normalize_time(
timeutils.parse_isotime(record['_source']['timestamp']))
yield models.Event(message_id=record['_id'],
event_type=record['_type'],
generated=gen_ts,
traits=sorted(
trait_list,
key=operator.attrgetter('dtype')),
raw=record['_source']['raw'])
def get_events(self, event_filter, pagination=None):
"""Return an iter of models.Event objects.
:param event_filter: storage.EventFilter object, consists of filters
for events that are stored in database.
:param pagination: Pagination parameters.
"""
limit = None
if pagination:
if pagination.get('sort'):
LOG.warning(_LW('Driver does not support sort functionality'))
limit = pagination.get('limit')
if limit == 0:
return
q, start, stop = hbase_utils.make_events_query_from_filter(
event_filter)
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
gen = events_table.scan(filter=q,
row_start=start,
row_stop=stop,
limit=limit)
for event_id, data in gen:
traits = []
events_dict = hbase_utils.deserialize_entry(data)[0]
for key, value in events_dict.items():
if isinstance(key, tuple):
trait_name, trait_dtype = key
traits.append(
models.Trait(name=trait_name,
dtype=int(trait_dtype),
value=value))
ts, mess = event_id.split(':')
yield models.Event(message_id=hbase_utils.unquote(mess),
event_type=events_dict['event_type'],
generated=events_dict['timestamp'],
traits=sorted(traits,
key=operator.attrgetter('dtype')),
raw=events_dict['raw'])
def record_events(self, events):
if not isinstance(events, list):
events = [events]
event_list = []
for ev in events:
try:
event_list.append(
models.Event(
message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[models.Trait(
name, dtype,
models.Trait.convert_value(dtype, value))
for name, dtype, value in ev['traits']],
raw=ev.get('raw', {}))
)
except Exception:
LOG.exception(_LE("Error processing event and it will be "
"dropped: %s"), ev)
self.event_conn.record_events(event_list)
def get_events(self, event_filter, pagination=None):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
:param pagination: Pagination parameters.
"""
pagination = pagination or {}
session = self._engine_facade.get_session()
with session.begin():
# Build up the join conditions
event_join_conditions = [
models.EventType.id == models.Event.event_type_id
]
if event_filter.event_type:
event_join_conditions.append(
models.EventType.desc == event_filter.event_type)
# Build up the where conditions
event_filter_conditions = []
if event_filter.message_id:
event_filter_conditions.append(
models.Event.message_id == event_filter.message_id)
if event_filter.start_timestamp:
event_filter_conditions.append(
models.Event.generated >= event_filter.start_timestamp)
if event_filter.end_timestamp:
event_filter_conditions.append(
models.Event.generated <= event_filter.end_timestamp)
trait_subq = None
# Build trait filter
if event_filter.traits_filter:
filters = list(event_filter.traits_filter)
trait_filter = filters.pop()
key = trait_filter.pop('key')
op = trait_filter.pop('op', 'eq')
trait_type, value = list(trait_filter.items())[0]
trait_subq = _build_trait_query(session, trait_type, key,
value, op)
for trait_filter in filters:
key = trait_filter.pop('key')
op = trait_filter.pop('op', 'eq')
trait_type, value = list(trait_filter.items())[0]
q = _build_trait_query(session, trait_type, key, value, op)
trait_subq = trait_subq.filter(
trait_subq.subquery().c.ev_id == q.subquery().c.ev_id)
trait_subq = trait_subq.subquery()
query = (session.query(models.Event.id).join(
models.EventType, sa.and_(*event_join_conditions)))
if trait_subq is not None:
query = query.join(trait_subq,
trait_subq.c.ev_id == models.Event.id)
if event_filter.admin_proj:
no_proj_q = session.query(models.TraitText.event_id).filter(
models.TraitText.key == 'project_id')
admin_q = (session.query(
models.TraitText.event_id).filter(~sa.exists().where(
models.TraitText.event_id ==
no_proj_q.subquery().c.event_id)).union(
session.query(models.TraitText.event_id).filter(
sa.and_(
models.TraitText.key == 'project_id',
models.TraitText.value ==
event_filter.admin_proj, models.Event.id ==
models.TraitText.event_id))))
query = query.filter(sa.exists().where(
models.Event.id ==
admin_q.subquery().c.trait_text_event_id))
if event_filter_conditions:
query = query.filter(sa.and_(*event_filter_conditions))
query = self._get_pagination_query(query, pagination,
api_models.Event, models.Event)
event_list = collections.OrderedDict()
# get a list of all events that match filters
for (id_, generated, message_id, desc, raw) in query.add_columns(
models.Event.generated, models.Event.message_id,
models.EventType.desc, models.Event.raw).all():
event_list[id_] = api_models.Event(message_id, desc, generated,
[], raw)
# Query all traits related to events.
# NOTE (gordc): cast is done because pgsql defaults to TEXT when
# handling unknown values such as null.
trait_q = (session.query(
models.TraitDatetime.event_id, models.TraitDatetime.key,
models.TraitDatetime.value, sa.cast(sa.null(), sa.Integer),
sa.cast(sa.null(), sa.Float(53)),
sa.cast(sa.null(), sa.String(255))).filter(sa.exists().where(
models.TraitDatetime.event_id == query.subquery().c.id))
).union_all(
session.query(models.TraitInt.event_id,
models.TraitInt.key, sa.null(),
models.TraitInt.value, sa.null(),
sa.null()).filter(sa.exists().where(
models.TraitInt.event_id ==
query.subquery().c.id)),
session.query(models.TraitFloat.event_id,
models.TraitFloat.key, sa.null(),
sa.null(), models.TraitFloat.value,
sa.null()).filter(sa.exists().where(
models.TraitFloat.event_id ==
query.subquery().c.id)),
session.query(models.TraitText.event_id,
models.TraitText.key, sa.null(),
sa.null(), sa.null(),
models.TraitText.value).filter(
sa.exists().where(
models.TraitText.event_id ==
query.subquery().c.id)))
for id_, key, t_date, t_int, t_float, t_text in (trait_q.order_by(
models.TraitDatetime.key)).all():
if t_int is not None:
dtype = api_models.Trait.INT_TYPE
val = t_int
elif t_float is not None:
dtype = api_models.Trait.FLOAT_TYPE
val = t_float
elif t_date is not None:
dtype = api_models.Trait.DATETIME_TYPE
val = t_date
else:
dtype = api_models.Trait.TEXT_TYPE
val = t_text
try:
trait_model = api_models.Trait(key, dtype, val)
event_list[id_].append_trait(trait_model)
except KeyError:
# NOTE(gordc): this is expected as we do not set REPEATABLE
# READ (bug 1506717). if query is run while recording new
# event data, trait query may return more data than event
# query. they can be safely discarded.
pass
return event_list.values()