def test_decimal_to_datetime(self): expected = 1356093296.12 dexpected = decimal.Decimal(str(expected)) # Python 2.6 wants str() expected_datetime = datetime.datetime.utcfromtimestamp(expected) actual_datetime = utils.decimal_to_dt(dexpected) # Python 3 have rounding issue on this, so use float self.assertAlmostEqual(utils.dt_to_decimal(expected_datetime), utils.dt_to_decimal(actual_datetime), places=5)
def get_events(self, event_filter): """Return an iterable of model.Event objects. :param event_filter: EventFilter instance """ start = utils.dt_to_decimal(event_filter.start) end = utils.dt_to_decimal(event_filter.end) session = sqlalchemy_session.get_session() with session.begin(): sub_query = session.query(Event.id)\ .join(Trait, Trait.event_id == Event.id)\ .filter(Event.generated >= start, Event.generated <= end) if event_filter.event_name: event_name = self._get_unique(session, event_filter.event_name) sub_query = sub_query.filter(Event.unique_name == event_name) if event_filter.traits: for key, value in event_filter.traits.iteritems(): if key == 'key': key = self._get_unique(session, value) sub_query = sub_query.filter(Trait.name == key) elif key == 't_string': sub_query = sub_query.filter(Trait.t_string == value) elif key == 't_int': sub_query = sub_query.filter(Trait.t_int == value) elif key == 't_datetime': dt = utils.dt_to_decimal(value) sub_query = sub_query.filter(Trait.t_datetime == dt) elif key == 't_float': sub_query = sub_query.filter(Trait.t_datetime == value) sub_query = sub_query.subquery() all_data = session.query(Trait)\ .join(sub_query, Trait.event_id == sub_query.c.id) # Now convert the sqlalchemy objects back into Models ... event_models_dict = {} for trait in all_data.all(): event = event_models_dict.get(trait.event_id) if not event: generated = utils.decimal_to_dt(trait.event.generated) event = api_models.Event(trait.event.message_id, trait.event.unique_name.key, generated, []) event_models_dict[trait.event_id] = event value = trait.get_value() trait_model = api_models.Trait(trait.name.key, trait.t_type, value) event.append_trait(trait_model) event_models = event_models_dict.values() return sorted(event_models, key=operator.attrgetter('generated'))
def get_events(self, event_filter): """Return an iterable of model.Event objects. :param event_filter: EventFilter instance """ start = utils.dt_to_decimal(event_filter.start) end = utils.dt_to_decimal(event_filter.end) session = sqlalchemy_session.get_session() with session.begin(): sub_query = session.query(Event.id)\ .join(Trait, Trait.event_id == Event.id)\ .filter(Event.generated >= start, Event.generated <= end) if event_filter.event_name: event_name = self._get_unique(session, event_filter.event_name) sub_query = sub_query.filter(Event.unique_name == event_name) if event_filter.traits: for key, value in event_filter.traits.iteritems(): if key == 'key': key = self._get_unique(session, value) sub_query = sub_query.filter(Trait.name == key) elif key == 't_string': sub_query = sub_query.filter(Trait.t_string == value) elif key == 't_int': sub_query = sub_query.filter(Trait.t_int == value) elif key == 't_datetime': dt = utils.dt_to_decimal(value) sub_query = sub_query.filter(Trait.t_datetime == dt) elif key == 't_float': sub_query = sub_query.filter(Trait.t_datetime == value) sub_query = sub_query.subquery() all_data = session.query(Trait)\ .join(sub_query, Trait.event_id == sub_query.c.id) # Now convert the sqlalchemy objects back into Models ... event_models_dict = {} for trait in all_data.all(): event = event_models_dict.get(trait.event_id) if not event: generated = utils.decimal_to_dt(trait.event.generated) event = api_models.Event(trait.event.unique_name.key, generated, []) event_models_dict[trait.event_id] = event value = trait.get_value() trait_model = api_models.Trait(trait.name.key, trait.t_type, value) event.append_trait(trait_model) event_models = event_models_dict.values() return sorted(event_models, key=operator.attrgetter('generated'))
def test_datetime_traits(self): now = datetime.datetime.utcnow() model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) trait = self.conn._make_trait(model, None) self.assertEquals(trait.t_type, models.Trait.DATETIME_TYPE) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_float) self.assertEquals(trait.t_datetime, utils.dt_to_decimal(now)) self.assertIsNotNone(trait.name)
def test_datetime_traits(self): now = datetime.datetime.utcnow() model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) trait = self.conn._make_trait(model, None) self.assertEqual(trait.t_type, models.Trait.DATETIME_TYPE) self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_float) self.assertEqual(trait.t_datetime, utils.dt_to_decimal(now)) self.assertIsNotNone(trait.name)
def test_datetime_traits(self): now = datetime.datetime.utcnow() model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) event = MagicMock() event.id = 1 trait = self.conn._make_trait(model, event) self.assertEqual(trait['trait_type'], models.Trait.DATETIME_TYPE) self.assertIsNone(trait['t_float']) self.assertIsNone(trait['t_string']) self.assertIsNone(trait['t_int']) self.assertEqual(trait['t_datetime'], utils.dt_to_decimal(now)) self.assertIsNotNone(trait['trait_type_id'])
def _make_trait(self, trait_model, event, session=None): """Make a new Trait from a Trait model. Doesn't flush or add to session. """ name = self._get_or_create_unique_name(trait_model.name, session=session) value_map = Trait._value_map values = {"t_string": None, "t_float": None, "t_int": None, "t_datetime": None} value = trait_model.value if trait_model.dtype == api_models.Trait.DATETIME_TYPE: value = utils.dt_to_decimal(value) values[value_map[trait_model.dtype]] = value return Trait(name, event, trait_model.dtype, **values)
def _make_trait(self, trait_model, event): """Make a new Trait from a Trait model. Doesn't flush or add to session. """ name = self._get_or_create_unique_name(trait_model.name) value_map = Trait._value_map values = {'t_string': None, 't_float': None, 't_int': None, 't_datetime': None} value = trait_model.value if trait_model.dtype == api_models.Trait.DATETIME_TYPE: value = utils.dt_to_decimal(value) values[value_map[trait_model.dtype]] = value return Trait(name, event, trait_model.dtype, **values)
def _make_trait(cls, trait_model, event, session=None): """Make a new Trait from a Trait model. Doesn't flush or add to session. """ trait_type = cls._get_or_create_trait_type(trait_model.name, trait_model.dtype, session) value_map = models.Trait._value_map values = {'t_string': None, 't_float': None, 't_int': None, 't_datetime': None} value = trait_model.value if trait_model.dtype == api_models.Trait.DATETIME_TYPE: value = utils.dt_to_decimal(value) values[value_map[trait_model.dtype]] = value return models.Trait(trait_type, event, **values)
def _record_event(self, event_model): """Store a single Event, including related Traits. """ unique = self._get_or_create_unique_name(event_model.event_name) generated = utils.dt_to_decimal(event_model.generated) event = Event(unique, generated) self.session.add(event) new_traits = [] if event_model.traits: for trait in event_model.traits: t = self._make_trait(trait, event) self.session.add(t) new_traits.append(t) # Note: we don't flush here, explicitly (unless a new uniquename # does it). Otherwise, just wait until all the Events are staged. return (event, new_traits)
def _make_trait(self, trait_model, event, session=None): """Make a new Trait from a Trait model. Doesn't flush or add to session. """ trait_type = self._get_or_create_trait_type(trait_model.name, trait_model.dtype, session) value_map = models.Trait._value_map fields = {'t_string': None, 't_float': None, 't_int': None, 't_datetime': None} value = trait_model.value if trait_model.dtype == api_models.Trait.DATETIME_TYPE: value = utils.dt_to_decimal(value) fields[value_map[trait_model.dtype]] = value fields['trait_type_id'] = trait_type.id fields['event_id'] = event.id fields['trait_type'] = trait_model.dtype return fields
def _record_event(self, session, event_model): """Store a single Event, including related Traits. """ with session.begin(subtransactions=True): event_type = self._get_or_create_event_type(event_model.event_type, session=session) generated = utils.dt_to_decimal(event_model.generated) event = models.Event(event_model.message_id, event_type, generated) session.add(event) new_traits = [] if event_model.traits: for trait in event_model.traits: t = self._make_trait(trait, event, session=session) session.add(t) new_traits.append(t) # Note: we don't flush here, explicitly (unless a new event_type # does it). Otherwise, just wait until all the Events are staged. return (event, new_traits)
def _record_event(self, session, event_model): """Store a single Event, including related Traits. """ with session.begin(subtransactions=True): event_type = self._get_or_create_event_type(event_model.event_type, session=session) generated = utils.dt_to_decimal(event_model.generated) event = models.Event(event_model.message_id, event_type, generated) session.add(event) # NOTE(apmelton) We need to flush to get the event's ID so that we # can bulk insert the traits. session.flush() new_traits = [] if event_model.traits: for trait in event_model.traits: t = self._make_trait(trait, event, session=session) new_traits.append(t) session.execute(models.Trait.__table__.insert(), new_traits) # Note: we don't flush here, explicitly (unless a new trait or event # does it). Otherwise, just wait until all the Events are staged. return (event, new_traits)
def test_datetime_to_decimal(self): expected = 1356093296.12 utc_datetime = datetime.datetime.utcfromtimestamp(expected) actual = utils.dt_to_decimal(utc_datetime) self.assertEqual(float(actual), expected)
def process_bind_param(value, dialect): if value is None: return value elif dialect.name == 'mysql': return utils.dt_to_decimal(value) return value
def test_process_result_value_datetime_mysql(self): dec_value = utils.dt_to_decimal(self._date) result = self._type.process_result_value(dec_value, self._mysql_dialect) self.assertEqual(self._date, result)
def test_process_bind_param_store_decimal_mysql(self): expected = utils.dt_to_decimal(self._date) result = self._type.process_bind_param(self._date, self._mysql_dialect) self.assertEqual(expected, result)
def process_bind_param(self, value, dialect): if value is None: return value elif dialect.name == 'mysql': return utils.dt_to_decimal(value) return value
def get_events(self, event_filter): """Return an iterable of model.Event objects. The event model objects have their Trait model objects available -- filtered by any traits in the event_filter. :param event_filter: EventFilter instance """ start = utils.dt_to_decimal(event_filter.start) end = utils.dt_to_decimal(event_filter.end) session = sqlalchemy_session.get_session() with session.begin(): sub_query = session.query(models.Event.id)\ .join(models.EventType, models.Event.event_type_id == models.EventType.id)\ .join(models.Trait, models.Trait.event_id == models.Event.id)\ .filter(models.Event.generated >= start, models.Event.generated <= end) if event_filter.event_type: event_type = event_filter.event_type sub_query = sub_query\ .filter(models.EventType.desc == event_type) event_models_dict = {} if event_filter.traits: for key, value in event_filter.traits.iteritems(): if key == 'key': key = self._get_unique(session, value) sub_query = sub_query.filter(models.Trait.name == key) elif key == 't_string': sub_query = sub_query.filter( models.Trait.t_string == value) elif key == 't_int': sub_query = sub_query.filter( models.Trait.t_int == value) elif key == 't_datetime': dt = utils.dt_to_decimal(value) sub_query = sub_query.filter( models.Trait.t_datetime == dt) elif key == 't_float': sub_query = sub_query.filter( models.Trait.t_datetime == value) else: # Pre-populate event_models_dict to cover Events without traits events = session.query(models.Event)\ .filter(models.Event.generated >= start)\ .filter(models.Event.generated <= end) if event_filter.event_type: events = events\ .join(models.EventType, models.EventType.id == models.Event.event_type_id)\ .filter(models.EventType.desc == event_filter.event_type) for db_event in events.all(): generated = utils.decimal_to_dt(db_event.generated) api_event = api_models.Event(db_event.message_id, db_event.event_type.desc, generated, []) event_models_dict[db_event.id] = api_event sub_query = sub_query.subquery() all_data = session.query(models.Trait)\ .join(sub_query, models.Trait.event_id == sub_query.c.id) # Now convert the sqlalchemy objects back into Models ... for trait in all_data.all(): event = event_models_dict.get(trait.event_id) if not event: generated = utils.decimal_to_dt(trait.event.generated) event = api_models.Event(trait.event.message_id, trait.event.event_type.desc, generated, []) event_models_dict[trait.event_id] = event value = trait.get_value() trait_model = api_models.Trait(trait.name.key, trait.t_type, value) event.append_trait(trait_model) event_models = event_models_dict.values() return sorted(event_models, key=operator.attrgetter('generated'))