def test_decimal_to_datetime(self):
expected = 1356093296.12
dexpected = decimal.Decimal(str(expected)) # Python 2.6 wants str()
expected_datetime = datetime.datetime.utcfromtimestamp(expected)
actual_datetime = utils.decimal_to_dt(dexpected)
# Python 3 have rounding issue on this, so use float
self.assertAlmostEqual(utils.dt_to_decimal(expected_datetime),
utils.dt_to_decimal(actual_datetime),
places=5)
def test_decimal_to_datetime(self):
expected = 1356093296.12
dexpected = decimal.Decimal(str(expected)) # Python 2.6 wants str()
expected_datetime = datetime.datetime.utcfromtimestamp(expected)
actual_datetime = utils.decimal_to_dt(dexpected)
# Python 3 have rounding issue on this, so use float
self.assertAlmostEqual(utils.dt_to_decimal(expected_datetime),
utils.dt_to_decimal(actual_datetime),
places=5)
def get_events(self, event_filter):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start)
end = utils.dt_to_decimal(event_filter.end)
session = sqlalchemy_session.get_session()
with session.begin():
sub_query = session.query(Event.id)\
.join(Trait, Trait.event_id == Event.id)\
.filter(Event.generated >= start, Event.generated <= end)
if event_filter.event_name:
event_name = self._get_unique(session, event_filter.event_name)
sub_query = sub_query.filter(Event.unique_name == event_name)
if event_filter.traits:
for key, value in event_filter.traits.iteritems():
if key == 'key':
key = self._get_unique(session, value)
sub_query = sub_query.filter(Trait.name == key)
elif key == 't_string':
sub_query = sub_query.filter(Trait.t_string == value)
elif key == 't_int':
sub_query = sub_query.filter(Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
sub_query = sub_query.filter(Trait.t_datetime == dt)
elif key == 't_float':
sub_query = sub_query.filter(Trait.t_datetime == value)
sub_query = sub_query.subquery()
all_data = session.query(Trait)\
.join(sub_query, Trait.event_id == sub_query.c.id)
# Now convert the sqlalchemy objects back into Models ...
event_models_dict = {}
for trait in all_data.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(trait.event.message_id,
trait.event.unique_name.key,
generated, [])
event_models_dict[trait.event_id] = event
value = trait.get_value()
trait_model = api_models.Trait(trait.name.key, trait.t_type,
value)
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))
def get_events(self, event_filter):
"""Return an iterable of model.Event objects.
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start)
end = utils.dt_to_decimal(event_filter.end)
session = sqlalchemy_session.get_session()
with session.begin():
sub_query = session.query(Event.id)\
.join(Trait, Trait.event_id == Event.id)\
.filter(Event.generated >= start, Event.generated <= end)
if event_filter.event_name:
event_name = self._get_unique(session, event_filter.event_name)
sub_query = sub_query.filter(Event.unique_name == event_name)
if event_filter.traits:
for key, value in event_filter.traits.iteritems():
if key == 'key':
key = self._get_unique(session, value)
sub_query = sub_query.filter(Trait.name == key)
elif key == 't_string':
sub_query = sub_query.filter(Trait.t_string == value)
elif key == 't_int':
sub_query = sub_query.filter(Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
sub_query = sub_query.filter(Trait.t_datetime == dt)
elif key == 't_float':
sub_query = sub_query.filter(Trait.t_datetime == value)
sub_query = sub_query.subquery()
all_data = session.query(Trait)\
.join(sub_query, Trait.event_id == sub_query.c.id)
# Now convert the sqlalchemy objects back into Models ...
event_models_dict = {}
for trait in all_data.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(trait.event.unique_name.key,
generated, [])
event_models_dict[trait.event_id] = event
value = trait.get_value()
trait_model = api_models.Trait(trait.name.key, trait.t_type,
value)
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))
def test_datetime_traits(self):
now = datetime.datetime.utcnow()
model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now)
trait = self.conn._make_trait(model, None)
self.assertEquals(trait.t_type, models.Trait.DATETIME_TYPE)
self.assertIsNone(trait.t_int)
self.assertIsNone(trait.t_string)
self.assertIsNone(trait.t_float)
self.assertEquals(trait.t_datetime, utils.dt_to_decimal(now))
self.assertIsNotNone(trait.name)
def test_datetime_traits(self):
now = datetime.datetime.utcnow()
model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now)
trait = self.conn._make_trait(model, None)
self.assertEqual(trait.t_type, models.Trait.DATETIME_TYPE)
self.assertIsNone(trait.t_int)
self.assertIsNone(trait.t_string)
self.assertIsNone(trait.t_float)
self.assertEqual(trait.t_datetime, utils.dt_to_decimal(now))
self.assertIsNotNone(trait.name)
def test_datetime_traits(self):
now = datetime.datetime.utcnow()
model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now)
event = MagicMock()
event.id = 1
trait = self.conn._make_trait(model, event)
self.assertEqual(trait['trait_type'], models.Trait.DATETIME_TYPE)
self.assertIsNone(trait['t_float'])
self.assertIsNone(trait['t_string'])
self.assertIsNone(trait['t_int'])
self.assertEqual(trait['t_datetime'], utils.dt_to_decimal(now))
self.assertIsNotNone(trait['trait_type_id'])
def _make_trait(self, trait_model, event, session=None):
"""Make a new Trait from a Trait model.
Doesn't flush or add to session.
"""
name = self._get_or_create_unique_name(trait_model.name, session=session)
value_map = Trait._value_map
values = {"t_string": None, "t_float": None, "t_int": None, "t_datetime": None}
value = trait_model.value
if trait_model.dtype == api_models.Trait.DATETIME_TYPE:
value = utils.dt_to_decimal(value)
values[value_map[trait_model.dtype]] = value
return Trait(name, event, trait_model.dtype, **values)
def _make_trait(self, trait_model, event):
"""Make a new Trait from a Trait model.
Doesn't flush or add to session.
"""
name = self._get_or_create_unique_name(trait_model.name)
value_map = Trait._value_map
values = {'t_string': None, 't_float': None,
't_int': None, 't_datetime': None}
value = trait_model.value
if trait_model.dtype == api_models.Trait.DATETIME_TYPE:
value = utils.dt_to_decimal(value)
values[value_map[trait_model.dtype]] = value
return Trait(name, event, trait_model.dtype, **values)
def _make_trait(cls, trait_model, event, session=None):
"""Make a new Trait from a Trait model.
Doesn't flush or add to session.
"""
trait_type = cls._get_or_create_trait_type(trait_model.name,
trait_model.dtype,
session)
value_map = models.Trait._value_map
values = {'t_string': None, 't_float': None,
't_int': None, 't_datetime': None}
value = trait_model.value
if trait_model.dtype == api_models.Trait.DATETIME_TYPE:
value = utils.dt_to_decimal(value)
values[value_map[trait_model.dtype]] = value
return models.Trait(trait_type, event, **values)
def _record_event(self, event_model):
"""Store a single Event, including related Traits.
"""
unique = self._get_or_create_unique_name(event_model.event_name)
generated = utils.dt_to_decimal(event_model.generated)
event = Event(unique, generated)
self.session.add(event)
new_traits = []
if event_model.traits:
for trait in event_model.traits:
t = self._make_trait(trait, event)
self.session.add(t)
new_traits.append(t)
# Note: we don't flush here, explicitly (unless a new uniquename
# does it). Otherwise, just wait until all the Events are staged.
return (event, new_traits)
def _make_trait(self, trait_model, event, session=None):
"""Make a new Trait from a Trait model.
Doesn't flush or add to session.
"""
trait_type = self._get_or_create_trait_type(trait_model.name,
trait_model.dtype,
session)
value_map = models.Trait._value_map
fields = {'t_string': None, 't_float': None,
't_int': None, 't_datetime': None}
value = trait_model.value
if trait_model.dtype == api_models.Trait.DATETIME_TYPE:
value = utils.dt_to_decimal(value)
fields[value_map[trait_model.dtype]] = value
fields['trait_type_id'] = trait_type.id
fields['event_id'] = event.id
fields['trait_type'] = trait_model.dtype
return fields
def _record_event(self, session, event_model):
"""Store a single Event, including related Traits.
"""
with session.begin(subtransactions=True):
event_type = self._get_or_create_event_type(event_model.event_type,
session=session)
generated = utils.dt_to_decimal(event_model.generated)
event = models.Event(event_model.message_id, event_type, generated)
session.add(event)
new_traits = []
if event_model.traits:
for trait in event_model.traits:
t = self._make_trait(trait, event, session=session)
session.add(t)
new_traits.append(t)
# Note: we don't flush here, explicitly (unless a new event_type
# does it). Otherwise, just wait until all the Events are staged.
return (event, new_traits)
def _record_event(self, session, event_model):
"""Store a single Event, including related Traits.
"""
with session.begin(subtransactions=True):
event_type = self._get_or_create_event_type(event_model.event_type,
session=session)
generated = utils.dt_to_decimal(event_model.generated)
event = models.Event(event_model.message_id, event_type, generated)
session.add(event)
# NOTE(apmelton) We need to flush to get the event's ID so that we
# can bulk insert the traits.
session.flush()
new_traits = []
if event_model.traits:
for trait in event_model.traits:
t = self._make_trait(trait, event, session=session)
new_traits.append(t)
session.execute(models.Trait.__table__.insert(), new_traits)
# Note: we don't flush here, explicitly (unless a new trait or event
# does it). Otherwise, just wait until all the Events are staged.
return (event, new_traits)
def test_datetime_to_decimal(self):
expected = 1356093296.12
utc_datetime = datetime.datetime.utcfromtimestamp(expected)
actual = utils.dt_to_decimal(utc_datetime)
self.assertEqual(float(actual), expected)
def process_bind_param(value, dialect):
if value is None:
return value
elif dialect.name == 'mysql':
return utils.dt_to_decimal(value)
return value
def test_process_result_value_datetime_mysql(self):
dec_value = utils.dt_to_decimal(self._date)
result = self._type.process_result_value(dec_value,
self._mysql_dialect)
self.assertEqual(self._date, result)
def test_process_bind_param_store_decimal_mysql(self):
expected = utils.dt_to_decimal(self._date)
result = self._type.process_bind_param(self._date, self._mysql_dialect)
self.assertEqual(expected, result)
def test_process_result_value_datetime_mysql(self):
dec_value = utils.dt_to_decimal(self._date)
result = self._type.process_result_value(dec_value,
self._mysql_dialect)
self.assertEqual(self._date, result)
def test_process_bind_param_store_decimal_mysql(self):
expected = utils.dt_to_decimal(self._date)
result = self._type.process_bind_param(self._date, self._mysql_dialect)
self.assertEqual(expected, result)
def process_bind_param(self, value, dialect):
if value is None:
return value
elif dialect.name == 'mysql':
return utils.dt_to_decimal(value)
return value
def get_events(self, event_filter):
"""Return an iterable of model.Event objects. The event model objects
have their Trait model objects available -- filtered by any traits
in the event_filter.
:param event_filter: EventFilter instance
"""
start = utils.dt_to_decimal(event_filter.start)
end = utils.dt_to_decimal(event_filter.end)
session = sqlalchemy_session.get_session()
with session.begin():
sub_query = session.query(models.Event.id)\
.join(models.EventType,
models.Event.event_type_id == models.EventType.id)\
.join(models.Trait,
models.Trait.event_id == models.Event.id)\
.filter(models.Event.generated >= start,
models.Event.generated <= end)
if event_filter.event_type:
event_type = event_filter.event_type
sub_query = sub_query\
.filter(models.EventType.desc == event_type)
event_models_dict = {}
if event_filter.traits:
for key, value in event_filter.traits.iteritems():
if key == 'key':
key = self._get_unique(session, value)
sub_query = sub_query.filter(models.Trait.name == key)
elif key == 't_string':
sub_query = sub_query.filter(
models.Trait.t_string == value)
elif key == 't_int':
sub_query = sub_query.filter(
models.Trait.t_int == value)
elif key == 't_datetime':
dt = utils.dt_to_decimal(value)
sub_query = sub_query.filter(
models.Trait.t_datetime == dt)
elif key == 't_float':
sub_query = sub_query.filter(
models.Trait.t_datetime == value)
else:
# Pre-populate event_models_dict to cover Events without traits
events = session.query(models.Event)\
.filter(models.Event.generated >= start)\
.filter(models.Event.generated <= end)
if event_filter.event_type:
events = events\
.join(models.EventType,
models.EventType.id ==
models.Event.event_type_id)\
.filter(models.EventType.desc ==
event_filter.event_type)
for db_event in events.all():
generated = utils.decimal_to_dt(db_event.generated)
api_event = api_models.Event(db_event.message_id,
db_event.event_type.desc,
generated, [])
event_models_dict[db_event.id] = api_event
sub_query = sub_query.subquery()
all_data = session.query(models.Trait)\
.join(sub_query, models.Trait.event_id == sub_query.c.id)
# Now convert the sqlalchemy objects back into Models ...
for trait in all_data.all():
event = event_models_dict.get(trait.event_id)
if not event:
generated = utils.decimal_to_dt(trait.event.generated)
event = api_models.Event(trait.event.message_id,
trait.event.event_type.desc,
generated, [])
event_models_dict[trait.event_id] = event
value = trait.get_value()
trait_model = api_models.Trait(trait.name.key, trait.t_type,
value)
event.append_trait(trait_model)
event_models = event_models_dict.values()
return sorted(event_models, key=operator.attrgetter('generated'))
def test_datetime_to_decimal(self):
expected = 1356093296.12
utc_datetime = datetime.datetime.utcfromtimestamp(expected)
actual = utils.dt_to_decimal(utc_datetime)
self.assertEqual(float(actual), expected)
