async def query(self, data):
"""Query events
Args:
data (common.QueryData): query data
Returns:
List[common.Event]
"""
backend_data = common.BackendQueryData(
event_ids=data.event_ids,
event_type_ids=(self._query_types_to_ids(data.event_types)
if data.event_types is not None else None),
t_from=data.t_from,
t_to=data.t_to,
source_t_from=data.source_t_from,
source_t_to=data.source_t_to,
payload=data.payload,
order=data.order,
order_by=data.order_by,
unique_type=data.unique_type,
max_results=data.max_results)
events = await self._backend.query(backend_data)
return [common.Event(
event_id=event.event_id,
event_type=self._event_type_from_id(event.event_type_id),
timestamp=event.timestamp,
source_timestamp=event.source_timestamp,
payload=event.payload
) for event in events]
async def register(self, events):
"""Register events
Args:
events (List[common.ProcessEvent]): process events
Returns:
List[Optional[common.Event]]
"""
now = common.now()
backend_events = [common.BackendEvent(
event_id=event.event_id,
event_type_id=self._get_event_type_id(event.event_type),
timestamp=now,
source_timestamp=event.source_timestamp,
payload=event.payload
) for event in events]
if self._new_mappings:
await self._backend.add_event_type_id_mappings(self._new_mappings)
self._new_mappings = {}
await self._backend.register(backend_events)
return [common.Event(
event_id=event.event_id,
event_type=event.event_type,
timestamp=now,
source_timestamp=event.source_timestamp,
payload=event.payload
) for event in events]
def register_cb(source, events):
register_queue.put_nowait(events)
process_events = [
engine.create_process_event(source, event) for event in events
]
return [
common.Event(event_id=i.event_id,
event_type=i.event_type,
timestamp=common.now(),
source_timestamp=i.source_timestamp,
payload=i.payload) for i in process_events
]
async def register(self, process_events):
if not self._register_cb:
now = common.now()
return [
common.Event(event_id=process_event.event_id,
event_type=process_event.event_type,
timestamp=now,
source_timestamp=process_event.source_timestamp,
payload=process_event.payload)
for process_event in process_events
]
return self._register_cb(process_events)
async def register(
self, process_events: typing.List[common.ProcessEvent]
) -> typing.List[typing.Optional[common.Event]]:
"""Register events"""
now = common.now()
events = [
common.Event(event_id=process_event.event_id,
event_type=process_event.event_type,
timestamp=now,
source_timestamp=process_event.source_timestamp,
payload=process_event.payload)
for process_event in process_events
]
return await self._backend.register(events)
async def test_query(backend_module_name, create_backend_module):
conf = {'server_id': 123, 'backend': {'module': backend_module_name}}
event_ids = [common.EventId(conf['server_id'], i) for i in range(10)]
events = [
common.Event(event_id=event_id,
event_type=[],
timestamp=common.now(),
source_timestamp=None,
payload=None) for event_id in event_ids
]
query_data = common.QueryData()
def query(data):
assert query_data == data
return events
with create_backend_module(query_cb=query):
engine = await hat.event.server.backend_engine.create(conf)
result = await engine.query(query_data)
assert result == events
await engine.async_close()
async def test_query():
query_data = common.QueryData()
events = [
common.Event(event_id=common.EventId(1, i),
event_type=(),
timestamp=common.now(),
source_timestamp=None,
payload=None) for i in range(10)
]
def on_query(data):
assert data == query_data
return events
conf = {'modules': []}
backend = BackendEngine(query_cb=on_query)
engine = await hat.event.server.module_engine.create(conf, backend)
result = await engine.query(query_data)
assert result == events
await engine.async_close()
await backend.async_close()
async def query(self, data):
"""See :meth:`common.Backend.query`"""
conditions = []
event_type_id_condition = None
params = {}
if data.max_results is not None and data.max_results <= 0:
return []
if data.event_ids is not None:
if not data.event_ids:
return []
subconditions = []
for i, event_id in enumerate(data.event_ids):
server_label = f'server_id_{i}'
instance_label = f'instance_id_{i}'
params[server_label] = event_id.server
params[instance_label] = event_id.instance
subconditions.append(f"(server_id = :{server_label} AND "
f"instance_id = :{instance_label})")
conditions.append(f"({' OR '.join(subconditions)})")
if data.event_types is not None:
event_type_ids = self._event_type_registry.query_identifiers(
data.event_types)
if not event_type_ids:
return []
labels = []
for i, event_type_id in enumerate(event_type_ids):
label = f'event_type_id_{i}'
params[label] = event_type_id
labels.append(label)
event_type_id_condition = ("event_type_id IN (" +
', '.join(f":{label}"
for label in labels) + ")")
if data.t_from is not None:
conditions.append("timestamp >= :t_from")
params['t_from'] = common.timestamp_to_bytes(data.t_from)
if data.t_to is not None:
conditions.append("timestamp <= :t_to")
params['t_to'] = common.timestamp_to_bytes(data.t_to)
if data.source_t_from is not None:
conditions.append("source_timestamp >= :source_t_from")
params['source_t_from'] = common.timestamp_to_bytes(
data.source_t_from)
if data.source_t_to is not None:
conditions.append("source_timestamp <= :source_t_to")
params['source_t_to'] = common.timestamp_to_bytes(data.source_t_to)
if data.payload is not None:
conditions.append("payload = :payload")
params['payload'] = self._encode_payload(data.payload)
order_cols = {
common.OrderBy.TIMESTAMP: ('timestamp', 'source_timestamp'),
common.OrderBy.SOURCE_TIMESTAMP: ('source_timestamp', 'timestamp')
}[data.order_by]
order_by = {
common.Order.ASCENDING: " ASC NULLS LAST",
common.Order.DESCENDING: " DESC NULLS LAST"
}
ordering_term = (f"{order_cols[0]} {order_by[data.order]}, "
f"{order_cols[1]} DESC NULLS LAST")
if data.unique_type:
key_columns = ['server_id', 'instance_id']
# uglier, but much, much faster than PARTITION BY if
# len(mappings) << len(events) because with this query
# proper indexes are used
sql = ("SELECT e.* FROM events AS e JOIN (SELECT " + ", ".join(
f"(SELECT {column} FROM events " +
" WHERE event_type_id = m.event_type_id" +
(" AND " + " AND ".join(conditions) if conditions else "") +
f" ORDER BY {ordering_term} LIMIT 1) AS t_{column}"
for column in key_columns) + " FROM mappings m" +
(f" WHERE {event_type_id_condition}"
if event_type_id_condition else "") + ") ON " +
" AND ".join(f"e.{column} = t_{column}"
for column in key_columns) +
f" ORDER BY {ordering_term}" +
(f" LIMIT {data.max_results}" if data.max_results else ""))
else:
if event_type_id_condition:
conditions.append(event_type_id_condition)
sql = (
"SELECT * FROM events" +
(" WHERE " + " AND ".join(conditions) if conditions else "") +
f" ORDER BY {ordering_term}" +
(f" LIMIT {data.max_results}" if data.max_results else ""))
async with self._query_pool.acquire() as conn:
result = await conn.execute(sql, params)
return [
common.Event(
event_id=common.EventId(server=row[0], instance=row[1]),
event_type=self._event_type_registry.get_event_type(row[2]),
timestamp=common.timestamp_from_bytes(row[3]),
source_timestamp=(common.timestamp_from_bytes(row[4])
if row[4] is not None else None),
payload=(self._decode_payload(row[5])
if row[5] is not None else None)) for row in result
]
for _ in range(10):
now_dt = common.timestamp_to_datetime(common.now())
delta = datetime.datetime.now(datetime.timezone.utc) - now_dt
assert delta < datetime.timedelta(seconds=1)
if previous_dt is not None:
assert now_dt >= previous_dt
previous_dt = now_dt
@pytest.mark.parametrize("event", [
common.Event(
event_id=common.EventId(0, 0),
event_type=('a',),
timestamp=common.now(),
source_timestamp=None,
payload=None),
common.Event(
event_id=common.EventId(0, 0),
event_type=('a',),
timestamp=common.now(),
source_timestamp=common.now(),
payload=common.EventPayload(
type=common.EventPayloadType.BINARY,
data=b'123')),
common.Event(
event_id=common.EventId(123, 456),
event_type=('a', 'b', 'c'),