def create_process_event(self, source, event): self._last_instance_id += 1 return common.ProcessEvent(event_id=common.EventId( server=self._server_id, instance=self._last_instance_id), source=source, event_type=event.event_type, source_timestamp=event.source_timestamp, payload=event.payload)
async def test_get_last_event_id(server_id): conf = {'module': 'hat.event.server.backends.dummy'} backend = await hat.event.server.backends.dummy.create(conf) event_id = await backend.get_last_event_id(server_id) assert event_id == common.EventId(server_id, 0) await backend.async_close()
async def get_last_event_id(self, server_id): """See :meth:`common.Backend.get_last_event_id`""" if server_id not in self._last_instance_ids: result = await self._conn.execute( "SELECT max(instance_id) FROM events " "WHERE server_id = :server_id", {'server_id': server_id}) instance_id = result[0][0] if result[0][0] is not None else 0 self._last_instance_ids[server_id] = instance_id return common.EventId(server=server_id, instance=self._last_instance_ids[server_id])
def create_process_event(self, source: common.Source, event: common.RegisterEvent ) -> common.ProcessEvent: """Create process event""" self._last_instance_id += 1 return common.ProcessEvent( event_id=common.EventId( server=self._server_id, instance=self._last_instance_id), source=source, event_type=event.event_type, source_timestamp=event.source_timestamp, payload=event.payload)
async def test_get_last_event_id(backend_module_name, create_backend_module): conf = {'server_id': 123, 'backend': {'module': backend_module_name}} event_id = common.EventId(conf['server_id'], 321) def get_last(server_id): assert server_id == conf['server_id'] return event_id with create_backend_module(get_last_event_id_cb=get_last): engine = await hat.event.server.backend_engine.create(conf) result = await engine.get_last_event_id() assert result == event_id await engine.async_close()
async def test_register(backend_module_name, create_backend_module): conf = {'server_id': 123, 'backend': {'module': backend_module_name}} event_ids = [common.EventId(conf['server_id'], i) for i in range(10)] process_events = [ common.ProcessEvent(event_id=event_id, source=common.Source(common.SourceType.MODULE, 'abc', 1), event_type=[], source_timestamp=common.now(), payload=None) for event_id in event_ids ] def register(events): return events with create_backend_module(register_cb=register): engine = await hat.event.server.backend_engine.create(conf) events = await engine.register(process_events) assert [i.event_id for i in events] == event_ids await engine.async_close()
async def test_query(backend_module_name, create_backend_module): conf = {'server_id': 123, 'backend': {'module': backend_module_name}} event_ids = [common.EventId(conf['server_id'], i) for i in range(10)] events = [ common.Event(event_id=event_id, event_type=[], timestamp=common.now(), source_timestamp=None, payload=None) for event_id in event_ids ] query_data = common.QueryData() def query(data): assert query_data == data return events with create_backend_module(query_cb=query): engine = await hat.event.server.backend_engine.create(conf) result = await engine.query(query_data) assert result == events await engine.async_close()
async def test_query(): query_data = common.QueryData() events = [ common.Event(event_id=common.EventId(1, i), event_type=(), timestamp=common.now(), source_timestamp=None, payload=None) for i in range(10) ] def on_query(data): assert data == query_data return events conf = {'modules': []} backend = BackendEngine(query_cb=on_query) engine = await hat.event.server.module_engine.create(conf, backend) result = await engine.query(query_data) assert result == events await engine.async_close() await backend.async_close()
async def get_last_event_id(self): return common.EventId(self._server_id, 0)
async def query(self, data): """See :meth:`common.Backend.query`""" conditions = [] event_type_id_condition = None params = {} if data.max_results is not None and data.max_results <= 0: return [] if data.event_ids is not None: if not data.event_ids: return [] subconditions = [] for i, event_id in enumerate(data.event_ids): server_label = f'server_id_{i}' instance_label = f'instance_id_{i}' params[server_label] = event_id.server params[instance_label] = event_id.instance subconditions.append(f"(server_id = :{server_label} AND " f"instance_id = :{instance_label})") conditions.append(f"({' OR '.join(subconditions)})") if data.event_types is not None: event_type_ids = self._event_type_registry.query_identifiers( data.event_types) if not event_type_ids: return [] labels = [] for i, event_type_id in enumerate(event_type_ids): label = f'event_type_id_{i}' params[label] = event_type_id labels.append(label) event_type_id_condition = ("event_type_id IN (" + ', '.join(f":{label}" for label in labels) + ")") if data.t_from is not None: conditions.append("timestamp >= :t_from") params['t_from'] = common.timestamp_to_bytes(data.t_from) if data.t_to is not None: conditions.append("timestamp <= :t_to") params['t_to'] = common.timestamp_to_bytes(data.t_to) if data.source_t_from is not None: conditions.append("source_timestamp >= :source_t_from") params['source_t_from'] = common.timestamp_to_bytes( data.source_t_from) if data.source_t_to is not None: conditions.append("source_timestamp <= :source_t_to") params['source_t_to'] = common.timestamp_to_bytes(data.source_t_to) if data.payload is not None: conditions.append("payload = :payload") params['payload'] = self._encode_payload(data.payload) order_cols = { common.OrderBy.TIMESTAMP: ('timestamp', 'source_timestamp'), common.OrderBy.SOURCE_TIMESTAMP: ('source_timestamp', 'timestamp') }[data.order_by] order_by = { common.Order.ASCENDING: " ASC NULLS LAST", common.Order.DESCENDING: " DESC NULLS LAST" } ordering_term = (f"{order_cols[0]} {order_by[data.order]}, " f"{order_cols[1]} DESC NULLS LAST") if data.unique_type: key_columns = ['server_id', 'instance_id'] # uglier, but much, much faster than PARTITION BY if # len(mappings) << len(events) because with this query # proper indexes are used sql = ("SELECT e.* FROM events AS e JOIN (SELECT " + ", ".join( f"(SELECT {column} FROM events " + " WHERE event_type_id = m.event_type_id" + (" AND " + " AND ".join(conditions) if conditions else "") + f" ORDER BY {ordering_term} LIMIT 1) AS t_{column}" for column in key_columns) + " FROM mappings m" + (f" WHERE {event_type_id_condition}" if event_type_id_condition else "") + ") ON " + " AND ".join(f"e.{column} = t_{column}" for column in key_columns) + f" ORDER BY {ordering_term}" + (f" LIMIT {data.max_results}" if data.max_results else "")) else: if event_type_id_condition: conditions.append(event_type_id_condition) sql = ( "SELECT * FROM events" + (" WHERE " + " AND ".join(conditions) if conditions else "") + f" ORDER BY {ordering_term}" + (f" LIMIT {data.max_results}" if data.max_results else "")) async with self._query_pool.acquire() as conn: result = await conn.execute(sql, params) return [ common.Event( event_id=common.EventId(server=row[0], instance=row[1]), event_type=self._event_type_registry.get_event_type(row[2]), timestamp=common.timestamp_from_bytes(row[3]), source_timestamp=(common.timestamp_from_bytes(row[4]) if row[4] is not None else None), payload=(self._decode_payload(row[5]) if row[5] is not None else None)) for row in result ]
async def get_last_event_id(self, server_id): """See :meth:`common.Backend.get_last_event_id`""" return await self._group.spawn(aio.call, lambda: common.EventId(server_id, 0))
async def get_last_event_id(self, server_id: int) -> common.EventId: result = common.EventId(server_id, 0) return await self._async_group.spawn(aio.call, lambda: result)
previous_dt = None for _ in range(10): now_dt = common.timestamp_to_datetime(common.now()) delta = datetime.datetime.now(datetime.timezone.utc) - now_dt assert delta < datetime.timedelta(seconds=1) if previous_dt is not None: assert now_dt >= previous_dt previous_dt = now_dt @pytest.mark.parametrize("event", [ common.Event( event_id=common.EventId(0, 0), event_type=('a',), timestamp=common.now(), source_timestamp=None, payload=None), common.Event( event_id=common.EventId(0, 0), event_type=('a',), timestamp=common.now(), source_timestamp=common.now(), payload=common.EventPayload( type=common.EventPayloadType.BINARY, data=b'123')), common.Event(
async def get_last_event_id(self, server_id: int) -> common.EventId: event_ids = (e.event_id for e in self._events if e.server == server_id) key = lambda event_id: event_id.instance # NOQA default = common.EventId(server=server_id, instance=0) return max(event_ids, key=key, default=default)