def create_process_event(self, source, event):
self._last_instance_id += 1
return common.ProcessEvent(event_id=common.EventId(
server=self._server_id, instance=self._last_instance_id),
source=source,
event_type=event.event_type,
source_timestamp=event.source_timestamp,
payload=event.payload)
async def test_get_last_event_id(server_id):
conf = {'module': 'hat.event.server.backends.dummy'}
backend = await hat.event.server.backends.dummy.create(conf)
event_id = await backend.get_last_event_id(server_id)
assert event_id == common.EventId(server_id, 0)
await backend.async_close()
async def get_last_event_id(self, server_id):
"""See :meth:`common.Backend.get_last_event_id`"""
if server_id not in self._last_instance_ids:
result = await self._conn.execute(
"SELECT max(instance_id) FROM events "
"WHERE server_id = :server_id", {'server_id': server_id})
instance_id = result[0][0] if result[0][0] is not None else 0
self._last_instance_ids[server_id] = instance_id
return common.EventId(server=server_id,
instance=self._last_instance_ids[server_id])
def create_process_event(self,
source: common.Source,
event: common.RegisterEvent
) -> common.ProcessEvent:
"""Create process event"""
self._last_instance_id += 1
return common.ProcessEvent(
event_id=common.EventId(
server=self._server_id,
instance=self._last_instance_id),
source=source,
event_type=event.event_type,
source_timestamp=event.source_timestamp,
payload=event.payload)
async def test_get_last_event_id(backend_module_name, create_backend_module):
conf = {'server_id': 123, 'backend': {'module': backend_module_name}}
event_id = common.EventId(conf['server_id'], 321)
def get_last(server_id):
assert server_id == conf['server_id']
return event_id
with create_backend_module(get_last_event_id_cb=get_last):
engine = await hat.event.server.backend_engine.create(conf)
result = await engine.get_last_event_id()
assert result == event_id
await engine.async_close()
async def test_register(backend_module_name, create_backend_module):
conf = {'server_id': 123, 'backend': {'module': backend_module_name}}
event_ids = [common.EventId(conf['server_id'], i) for i in range(10)]
process_events = [
common.ProcessEvent(event_id=event_id,
source=common.Source(common.SourceType.MODULE,
'abc', 1),
event_type=[],
source_timestamp=common.now(),
payload=None) for event_id in event_ids
]
def register(events):
return events
with create_backend_module(register_cb=register):
engine = await hat.event.server.backend_engine.create(conf)
events = await engine.register(process_events)
assert [i.event_id for i in events] == event_ids
await engine.async_close()
async def test_query(backend_module_name, create_backend_module):
conf = {'server_id': 123, 'backend': {'module': backend_module_name}}
event_ids = [common.EventId(conf['server_id'], i) for i in range(10)]
events = [
common.Event(event_id=event_id,
event_type=[],
timestamp=common.now(),
source_timestamp=None,
payload=None) for event_id in event_ids
]
query_data = common.QueryData()
def query(data):
assert query_data == data
return events
with create_backend_module(query_cb=query):
engine = await hat.event.server.backend_engine.create(conf)
result = await engine.query(query_data)
assert result == events
await engine.async_close()
async def test_query():
query_data = common.QueryData()
events = [
common.Event(event_id=common.EventId(1, i),
event_type=(),
timestamp=common.now(),
source_timestamp=None,
payload=None) for i in range(10)
]
def on_query(data):
assert data == query_data
return events
conf = {'modules': []}
backend = BackendEngine(query_cb=on_query)
engine = await hat.event.server.module_engine.create(conf, backend)
result = await engine.query(query_data)
assert result == events
await engine.async_close()
await backend.async_close()
async def get_last_event_id(self):
return common.EventId(self._server_id, 0)
async def query(self, data):
"""See :meth:`common.Backend.query`"""
conditions = []
event_type_id_condition = None
params = {}
if data.max_results is not None and data.max_results <= 0:
return []
if data.event_ids is not None:
if not data.event_ids:
return []
subconditions = []
for i, event_id in enumerate(data.event_ids):
server_label = f'server_id_{i}'
instance_label = f'instance_id_{i}'
params[server_label] = event_id.server
params[instance_label] = event_id.instance
subconditions.append(f"(server_id = :{server_label} AND "
f"instance_id = :{instance_label})")
conditions.append(f"({' OR '.join(subconditions)})")
if data.event_types is not None:
event_type_ids = self._event_type_registry.query_identifiers(
data.event_types)
if not event_type_ids:
return []
labels = []
for i, event_type_id in enumerate(event_type_ids):
label = f'event_type_id_{i}'
params[label] = event_type_id
labels.append(label)
event_type_id_condition = ("event_type_id IN (" +
', '.join(f":{label}"
for label in labels) + ")")
if data.t_from is not None:
conditions.append("timestamp >= :t_from")
params['t_from'] = common.timestamp_to_bytes(data.t_from)
if data.t_to is not None:
conditions.append("timestamp <= :t_to")
params['t_to'] = common.timestamp_to_bytes(data.t_to)
if data.source_t_from is not None:
conditions.append("source_timestamp >= :source_t_from")
params['source_t_from'] = common.timestamp_to_bytes(
data.source_t_from)
if data.source_t_to is not None:
conditions.append("source_timestamp <= :source_t_to")
params['source_t_to'] = common.timestamp_to_bytes(data.source_t_to)
if data.payload is not None:
conditions.append("payload = :payload")
params['payload'] = self._encode_payload(data.payload)
order_cols = {
common.OrderBy.TIMESTAMP: ('timestamp', 'source_timestamp'),
common.OrderBy.SOURCE_TIMESTAMP: ('source_timestamp', 'timestamp')
}[data.order_by]
order_by = {
common.Order.ASCENDING: " ASC NULLS LAST",
common.Order.DESCENDING: " DESC NULLS LAST"
}
ordering_term = (f"{order_cols[0]} {order_by[data.order]}, "
f"{order_cols[1]} DESC NULLS LAST")
if data.unique_type:
key_columns = ['server_id', 'instance_id']
# uglier, but much, much faster than PARTITION BY if
# len(mappings) << len(events) because with this query
# proper indexes are used
sql = ("SELECT e.* FROM events AS e JOIN (SELECT " + ", ".join(
f"(SELECT {column} FROM events " +
" WHERE event_type_id = m.event_type_id" +
(" AND " + " AND ".join(conditions) if conditions else "") +
f" ORDER BY {ordering_term} LIMIT 1) AS t_{column}"
for column in key_columns) + " FROM mappings m" +
(f" WHERE {event_type_id_condition}"
if event_type_id_condition else "") + ") ON " +
" AND ".join(f"e.{column} = t_{column}"
for column in key_columns) +
f" ORDER BY {ordering_term}" +
(f" LIMIT {data.max_results}" if data.max_results else ""))
else:
if event_type_id_condition:
conditions.append(event_type_id_condition)
sql = (
"SELECT * FROM events" +
(" WHERE " + " AND ".join(conditions) if conditions else "") +
f" ORDER BY {ordering_term}" +
(f" LIMIT {data.max_results}" if data.max_results else ""))
async with self._query_pool.acquire() as conn:
result = await conn.execute(sql, params)
return [
common.Event(
event_id=common.EventId(server=row[0], instance=row[1]),
event_type=self._event_type_registry.get_event_type(row[2]),
timestamp=common.timestamp_from_bytes(row[3]),
source_timestamp=(common.timestamp_from_bytes(row[4])
if row[4] is not None else None),
payload=(self._decode_payload(row[5])
if row[5] is not None else None)) for row in result
]
async def get_last_event_id(self, server_id):
"""See :meth:`common.Backend.get_last_event_id`"""
return await self._group.spawn(aio.call,
lambda: common.EventId(server_id, 0))
async def get_last_event_id(self, server_id: int) -> common.EventId:
result = common.EventId(server_id, 0)
return await self._async_group.spawn(aio.call, lambda: result)
previous_dt = None
for _ in range(10):
now_dt = common.timestamp_to_datetime(common.now())
delta = datetime.datetime.now(datetime.timezone.utc) - now_dt
assert delta < datetime.timedelta(seconds=1)
if previous_dt is not None:
assert now_dt >= previous_dt
previous_dt = now_dt
@pytest.mark.parametrize("event", [
common.Event(
event_id=common.EventId(0, 0),
event_type=('a',),
timestamp=common.now(),
source_timestamp=None,
payload=None),
common.Event(
event_id=common.EventId(0, 0),
event_type=('a',),
timestamp=common.now(),
source_timestamp=common.now(),
payload=common.EventPayload(
type=common.EventPayloadType.BINARY,
data=b'123')),
common.Event(
async def get_last_event_id(self, server_id: int) -> common.EventId:
event_ids = (e.event_id for e in self._events if e.server == server_id)
key = lambda event_id: event_id.instance # NOQA
default = common.EventId(server=server_id, instance=0)
return max(event_ids, key=key, default=default)
