def make_test_data(conn, start, end, interval, event_types):
# Compute start and end timestamps for the new data.
if isinstance(start, datetime.datetime):
timestamp = start
else:
timestamp = timeutils.parse_strtime(start)
if not isinstance(end, datetime.datetime):
end = timeutils.parse_strtime(end)
increment = datetime.timedelta(minutes=interval)
print('Adding new events')
n = 0
while timestamp <= end:
data = []
for i in range(event_types):
traits = [
models.Trait('id1_%d' % i, 1, uuidutils.generate_uuid()),
models.Trait('id2_%d' % i, 2, random.randint(1, 10)),
models.Trait('id3_%d' % i, 3, random.random()),
models.Trait('id4_%d' % i, 4, timestamp)
]
data.append(
models.Event(uuidutils.generate_uuid(), 'event_type%d' % i,
timestamp, traits, {}))
n += 1
conn.record_events(data)
timestamp = timestamp + increment
print('Added %d new events' % n)
def _generate_models(self):
event_models = []
self.s_time = datetime.datetime(2013, 12, 31, 5, 0)
event_models.append(
models.Event(message_id='1',
event_type='empty_ev',
generated=self.s_time,
traits=[
models.Trait('random', models.Trait.TEXT_TYPE,
'blah')
],
raw={}))
event_models.append(
models.Event(message_id='2',
event_type='admin_ev',
generated=self.s_time,
traits=[
models.Trait('project_id', models.Trait.TEXT_TYPE,
self.admin_proj_id),
models.Trait('user_id', models.Trait.TEXT_TYPE,
self.admin_user_id)
],
raw={}))
event_models.append(
models.Event(message_id='3',
event_type='user_ev',
generated=self.s_time,
traits=[
models.Trait('project_id', models.Trait.TEXT_TYPE,
self.proj_id),
models.Trait('user_id', models.Trait.TEXT_TYPE,
self.user_id)
],
raw={}))
self.conn.record_events(event_models)
def setUp(self):
super(TestBaseApiEventRBAC, self).setUp()
traits = [
models.Trait('project_id', 1, 'project-good'),
models.Trait('user_id', 1, 'user-good')
]
self.message_id = uuidutils.generate_uuid()
ev = models.Event(self.message_id, 'event_type',
datetime.datetime.now(), traits, {})
self.conn.record_events([ev])
def test_trait_type_enforced_on_none(self):
new_events = [
models.Event("id_testid", "MessageIDTest", self.start, [
models.Trait('text', models.Trait.TEXT_TYPE, ''),
models.Trait('int', models.Trait.INT_TYPE, 0),
models.Trait('float', models.Trait.FLOAT_TYPE, 0.0)
], {})
]
self.conn.record_events(new_events)
event_filter = storage.EventFilter(message_id="id_testid")
events = [event for event in self.conn.get_events(event_filter)]
options = [(models.Trait.TEXT_TYPE, ''), (models.Trait.INT_TYPE, 0.0),
(models.Trait.FLOAT_TYPE, 0.0)]
for trait in events[0].traits:
options.remove((trait.dtype, trait.value))
def _generate_models(self):
event_models = []
base = 0
self.s_time = datetime.datetime(2013, 12, 31, 5, 0)
self.trait_time = datetime.datetime(2013, 12, 31, 5, 0)
for event_type in ['Foo', 'Bar', 'Zoo']:
trait_models = [
models.Trait(name, type, value)
for name, type, value in [(
'trait_A', models.Trait.TEXT_TYPE, "my_%s_text" %
event_type), ('trait_B', models.Trait.INT_TYPE, base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D',
models.Trait.DATETIME_TYPE,
self.trait_time)]
]
# Message ID for test will be 'base'. So, message ID for the first
# event will be '0', the second '100', and so on.
# trait_time in first event will be equal to self.trait_time
# (datetime.datetime(2013, 12, 31, 5, 0)), next will add 1 day, so
# second will be (datetime.datetime(2014, 01, 01, 5, 0)) and so on.
event_models.append(
models.Event(message_id=str(base),
event_type=event_type,
generated=self.trait_time,
traits=trait_models,
raw={'status': {
'nested': 'started'
}}))
base += 100
self.trait_time += datetime.timedelta(days=1)
self.conn.record_events(event_models)
def get_traits(self, event_type, trait_type=None):
"""Return all trait instances associated with an event_type.
If trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_type: the name of the Trait to filter by
"""
session = self._engine_facade.get_session()
with session.begin():
for trait_model in [
models.TraitText, models.TraitInt, models.TraitFloat,
models.TraitDatetime
]:
query = (session.query(
trait_model.key, trait_model.value).join(
models.Event,
models.Event.id == trait_model.event_id).join(
models.EventType,
sa.and_(
models.EventType.id ==
models.Event.event_type_id,
models.EventType.desc == event_type)).order_by(
trait_model.key))
if trait_type:
query = query.filter(trait_model.key == trait_type)
dtype = TRAIT_MODEL_TO_ID.get(trait_model)
for k, v in query.all():
yield api_models.Trait(name=k, dtype=dtype, value=v)
def _generate_models(self):
event_models = []
base = 0
self.s_time = datetime.datetime(2013, 12, 31, 5, 0)
self.trait_time = datetime.datetime(2013, 12, 31, 5, 0)
for i in range(20):
trait_models = [
models.Trait(name, type, value)
for name, type, value in [(
'trait_A', models.Trait.TEXT_TYPE,
"my_text"), ('trait_B', models.Trait.INT_TYPE, base + 1),
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D',
models.Trait.DATETIME_TYPE,
self.trait_time)]
]
event_models.append(
models.Event(message_id=uuidutils.generate_uuid(),
event_type='foo.bar',
generated=self.trait_time,
traits=trait_models,
raw={'status': {
'nested': 'started'
}}))
self.trait_time += datetime.timedelta(seconds=1)
self.conn.record_events(event_models)
def record_events(self, events):
if not isinstance(events, list):
events = [events]
event_list = []
for ev in events:
try:
event_list.append(
models.Event(message_id=ev['message_id'],
event_type=ev['event_type'],
generated=timeutils.normalize_time(
timeutils.parse_isotime(ev['generated'])),
traits=[
models.Trait(
name, dtype,
models.Trait.convert_value(
dtype, value))
for name, dtype, value in ev['traits']
],
raw=ev.get('raw', {})))
except Exception:
LOG.exception(
"Error processing event and it will be "
"dropped: %s", ev)
self.conn.record_events(event_list)
def get_traits(self, event_type, trait_name=None):
"""Return all trait instances associated with an event_type.
If trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_name: the name of the Trait to filter by
"""
if not trait_name:
events = self.db.event.find({'event_type': event_type})
else:
# We choose events that simultaneously have event_type and certain
# trait_name, and retrieve events contains only mentioned traits.
events = self.db.event.find(
{
'$and': [{
'event_type': event_type
}, {
'traits.trait_name': trait_name
}]
}, {'traits': {
'$elemMatch': {
'trait_name': trait_name
}
}})
for event in events:
for trait in event['traits']:
yield models.Trait(name=trait['trait_name'],
dtype=trait['trait_type'],
value=trait['trait_value'])
def test_trait_bigint(self):
big = 99999999999999
new_events = [
models.Event("id_testid", "MessageIDTest", self.start,
[models.Trait('int', models.Trait.INT_TYPE, big)], {})
]
self.conn.record_events(new_events)
def get_events(self, event_filter, pagination=None):
"""Return an iter of models.Event objects.
:param event_filter: storage.EventFilter object, consists of filters
for events that are stored in database.
:param pagination: Pagination parameters.
"""
limit = None
if pagination:
if pagination.get('sort'):
LOG.warning(_LW('Driver does not support sort functionality'))
limit = pagination.get('limit')
if limit == 0:
return
q = pymongo_utils.make_events_query_from_filter(event_filter)
if limit is not None:
results = self.db.event.find(q, limit=limit)
else:
results = self.db.event.find(q)
for event in results:
traits = []
for trait in event['traits']:
traits.append(
models.Trait(name=trait['trait_name'],
dtype=int(trait['trait_type']),
value=trait['trait_value']))
yield models.Event(message_id=event['_id'],
event_type=event['event_type'],
generated=event['timestamp'],
traits=traits,
raw=event.get('raw'))
def start_fixture(self):
"""Create some events."""
super(EventDataFixture, self).start_fixture()
events = []
name_list = ['chocolate.chip', 'peanut.butter', 'sugar']
for ix, name in enumerate(name_list):
timestamp = datetime.datetime.utcnow()
message_id = 'fea1b15a-1d47-4175-85a5-a4bb2c72924{}'.format(ix)
traits = [
models.Trait('type', 1, name),
models.Trait('ate', 2, ix)
]
event = models.Event(message_id, 'cookies_{}'.format(name),
timestamp, traits,
{'nested': {
'inside': 'value'
}})
events.append(event)
self.conn.record_events(events)
def get_traits(self, event_type, trait_type=None):
t_types = dict((res['name'], res['data_type'])
for res in self.get_trait_types(event_type))
if not t_types or (trait_type and trait_type not in t_types.keys()):
return
result = self.conn.search('%s_*' % self.index_name, event_type)
for ev in result['hits']['hits']:
if trait_type and ev['_source']['traits'].get(trait_type):
yield models.Trait(name=trait_type,
dtype=t_types[trait_type],
value=models.Trait.convert_value(
t_types[trait_type],
ev['_source']['traits'][trait_type]))
else:
for trait in ev['_source']['traits'].keys():
yield models.Trait(name=trait,
dtype=t_types[trait],
value=models.Trait.convert_value(
t_types[trait],
ev['_source']['traits'][trait]))
def get_events(self, event_filter, pagination=None):
limit = None
if pagination:
if pagination.get('sort'):
LOG.warning('Driver does not support sort functionality')
limit = pagination.get('limit')
if limit == 0:
return
iclient = es.client.IndicesClient(self.conn)
indices = iclient.get_mapping('%s_*' % self.index_name).keys()
if indices:
filter_args = self._make_dsl_from_filter(indices, event_filter)
if limit is not None:
filter_args['size'] = limit
results = self.conn.search(
fields=['_id', 'timestamp', '_type', '_source'],
sort='timestamp:asc',
**filter_args)
trait_mappings = {}
for record in results['hits']['hits']:
trait_list = []
if not record['_type'] in trait_mappings:
trait_mappings[record['_type']] = list(
self.get_trait_types(record['_type']))
for key in record['_source']['traits'].keys():
value = record['_source']['traits'][key]
for t_map in trait_mappings[record['_type']]:
if t_map['name'] == key:
dtype = t_map['data_type']
break
else:
dtype = models.Trait.TEXT_TYPE
trait_list.append(
models.Trait(name=key,
dtype=dtype,
value=models.Trait.convert_value(
dtype, value)))
gen_ts = timeutils.normalize_time(
timeutils.parse_isotime(record['_source']['timestamp']))
yield models.Event(message_id=record['_id'],
event_type=record['_type'],
generated=gen_ts,
traits=sorted(
trait_list,
key=operator.attrgetter('dtype')),
raw=record['_source']['raw'])
def get_traits(self, event_type, trait_type=None):
"""Return all trait instances associated with an event_type.
If trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_type: the name of the Trait to filter by
"""
q = hbase_utils.make_query(event_type=event_type,
trait_type=trait_type)
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
gen = events_table.scan(filter=q)
for event_id, data in gen:
events_dict = hbase_utils.deserialize_entry(data)[0]
for key, value in events_dict.items():
if isinstance(key, tuple):
trait_name, trait_type = key
yield models.Trait(name=trait_name,
dtype=int(trait_type), value=value)
def get_events(self, event_filter, pagination=None):
"""Return an iter of models.Event objects.
:param event_filter: storage.EventFilter object, consists of filters
for events that are stored in database.
:param pagination: Pagination parameters.
"""
limit = None
if pagination:
if pagination.get('sort'):
LOG.warning('Driver does not support sort functionality')
limit = pagination.get('limit')
if limit == 0:
return
q, start, stop = hbase_utils.make_events_query_from_filter(
event_filter)
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
gen = events_table.scan(filter=q,
row_start=start,
row_stop=stop,
limit=limit)
for event_id, data in gen:
traits = []
events_dict = hbase_utils.deserialize_entry(data)[0]
for key, value in events_dict.items():
if isinstance(key, tuple):
trait_name, trait_dtype = key
traits.append(
models.Trait(name=trait_name,
dtype=int(trait_dtype),
value=value))
ts, mess = event_id.split(':')
yield models.Event(message_id=hbase_utils.unquote(mess),
event_type=events_dict['event_type'],
generated=events_dict['timestamp'],
traits=sorted(traits,
key=operator.attrgetter('dtype')),
raw=events_dict['raw'])
def prepare_data(self):
self.models = []
base = 0
self.start = datetime.datetime(2013, 12, 31, 5, 0)
now = self.start
for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']:
trait_models = [
models.Trait(name, dtype, value) for name, dtype, value in
[('trait_A', models.Trait.TEXT_TYPE, "my_%s_text" %
event_type), ('trait_B', models.Trait.INT_TYPE, base + 1),
('trait_C', models.Trait.FLOAT_TYPE, float(base) +
0.123456), ('trait_D', models.Trait.DATETIME_TYPE, now)]
]
self.models.append(
models.Event("id_%s_%d" % (event_type, base), event_type, now,
trait_models, {'status': {
'nested': 'started'
}}))
base += 100
now = now + datetime.timedelta(hours=1)
self.end = now
self.conn.record_events(self.models)
def get_events(self, event_filter, pagination=None):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
:param pagination: Pagination parameters.
"""
pagination = pagination or {}
session = self._engine_facade.get_session()
with session.begin():
# Build up the join conditions
event_join_conditions = [
models.EventType.id == models.Event.event_type_id
]
if event_filter.event_type:
event_join_conditions.append(
models.EventType.desc == event_filter.event_type)
# Build up the where conditions
event_filter_conditions = []
if event_filter.message_id:
event_filter_conditions.append(
models.Event.message_id == event_filter.message_id)
if event_filter.start_timestamp:
event_filter_conditions.append(
models.Event.generated >= event_filter.start_timestamp)
if event_filter.end_timestamp:
event_filter_conditions.append(
models.Event.generated <= event_filter.end_timestamp)
trait_subq = None
# Build trait filter
if event_filter.traits_filter:
filters = list(event_filter.traits_filter)
trait_filter = filters.pop()
key = trait_filter.pop('key')
op = trait_filter.pop('op', 'eq')
trait_type, value = list(trait_filter.items())[0]
trait_model, conditions = _get_model_and_conditions(
trait_type, key, value, op)
trait_subq = (session.query(
trait_model.event_id.label('ev_id')).filter(*conditions))
first_model = trait_model
for label_num, trait_filter in enumerate(filters):
key = trait_filter.pop('key')
op = trait_filter.pop('op', 'eq')
trait_type, value = list(trait_filter.items())[0]
trait_model, conditions = _get_model_and_conditions(
trait_type, key, value, op)
trait_subq = (trait_subq.add_columns(
trait_model.event_id.label('l%d' % label_num)).filter(
first_model.event_id == trait_model.event_id,
*conditions))
trait_subq = trait_subq.subquery()
query = (session.query(models.Event.id).join(
models.EventType, sa.and_(*event_join_conditions)))
if trait_subq is not None:
query = query.join(trait_subq,
trait_subq.c.ev_id == models.Event.id)
if event_filter.admin_proj:
no_proj_q = session.query(models.TraitText.event_id).filter(
models.TraitText.key == 'project_id')
admin_q = (session.query(
models.TraitText.event_id).filter(~sa.exists().where(
models.TraitText.event_id ==
no_proj_q.subquery().c.event_id)).union(
session.query(models.TraitText.event_id).filter(
sa.and_(
models.TraitText.key == 'project_id',
models.TraitText.value ==
event_filter.admin_proj, models.Event.id ==
models.TraitText.event_id))))
query = query.filter(sa.exists().where(
models.Event.id ==
admin_q.subquery().c.trait_text_event_id))
if event_filter_conditions:
query = query.filter(sa.and_(*event_filter_conditions))
query = self._get_pagination_query(query, pagination,
api_models.Event, models.Event)
event_list = collections.OrderedDict()
# get a list of all events that match filters
for (id_, generated, message_id, desc, raw) in query.add_columns(
models.Event.generated, models.Event.message_id,
models.EventType.desc, models.Event.raw).all():
event_list[id_] = api_models.Event(message_id, desc, generated,
[], raw)
# Query all traits related to events.
# NOTE (gordc): cast is done because pgsql defaults to TEXT when
# handling unknown values such as null.
trait_q = (session.query(
models.TraitDatetime.event_id, models.TraitDatetime.key,
models.TraitDatetime.value, sa.cast(sa.null(), sa.Integer),
sa.cast(sa.null(), sa.Float(53)),
sa.cast(sa.null(), sa.String(255))).filter(sa.exists().where(
models.TraitDatetime.event_id == query.subquery().c.id))
).union_all(
session.query(models.TraitInt.event_id,
models.TraitInt.key, sa.null(),
models.TraitInt.value, sa.null(),
sa.null()).filter(sa.exists().where(
models.TraitInt.event_id ==
query.subquery().c.id)),
session.query(models.TraitFloat.event_id,
models.TraitFloat.key, sa.null(),
sa.null(), models.TraitFloat.value,
sa.null()).filter(sa.exists().where(
models.TraitFloat.event_id ==
query.subquery().c.id)),
session.query(models.TraitText.event_id,
models.TraitText.key, sa.null(),
sa.null(), sa.null(),
models.TraitText.value).filter(
sa.exists().where(
models.TraitText.event_id ==
query.subquery().c.id)))
for id_, key, t_date, t_int, t_float, t_text in (trait_q.order_by(
models.TraitDatetime.key)).all():
if t_int is not None:
dtype = api_models.Trait.INT_TYPE
val = t_int
elif t_float is not None:
dtype = api_models.Trait.FLOAT_TYPE
val = t_float
elif t_date is not None:
dtype = api_models.Trait.DATETIME_TYPE
val = t_date
else:
dtype = api_models.Trait.TEXT_TYPE
val = t_text
try:
trait_model = api_models.Trait(key, dtype, val)
event_list[id_].append_trait(trait_model)
except KeyError:
# NOTE(gordc): this is expected as we do not set REPEATABLE
# READ (bug 1506717). if query is run while recording new
# event data, trait query may return more data than event
# query. they can be safely discarded.
pass
return event_list.values()
def test_float_traits(self):
model = models.Trait("Foo", models.Trait.FLOAT_TYPE, 123.456)
self._verify_data(model, sql_models.TraitFloat)
def test_int_traits(self):
model = models.Trait("Foo", models.Trait.INT_TYPE, 100)
self._verify_data(model, sql_models.TraitInt)
def test_string_traits(self):
model = models.Trait("Foo", models.Trait.TEXT_TYPE, "my_text")
self._verify_data(model, sql_models.TraitText)
def test_datetime_traits(self):
now = datetime.datetime.utcnow()
model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now)
self._verify_data(model, sql_models.TraitDatetime)
