async def test_query_subscription(executor, env, flush, query, create_event,
order_by):
name = 'name'
subscription = common.Subscription([('*', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=None)
event1 = create_event(('a', ), True)
event2 = create_event(('b', ), True)
db.add(event1)
db.add(event2)
result = await query(db)
assert result == [event2, event1]
subscription = common.Subscription([('a', )])
result = await query(db, subscription=subscription)
assert result == [event1]
subscription = common.Subscription([('b', )])
result = await query(db, subscription=subscription)
assert result == [event2]
async def test_query_payload(executor, env, flush, query, create_event,
order_by):
name = 'name'
subscription = common.Subscription([('*', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=None)
event1 = create_event(('a', ), True)
event2 = create_event(('b', ), True)._replace(
payload=common.EventPayload(common.EventPayloadType.JSON, data=123))
db.add(event1)
db.add(event2)
result = await query(db)
assert result == [event2, event1]
result = await query(db,
payload=common.EventPayload(
common.EventPayloadType.JSON, data=123))
assert result == [event2]
result = await query(db,
payload=common.EventPayload(
common.EventPayloadType.JSON, data=321))
assert result == []
async def test_add(executor, env, flush, query, create_event, order_by):
name = 'name'
subscription = common.Subscription([('a', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=None)
event1 = create_event(('a', ), True)
event2 = create_event(('b', ), True)
event3 = create_event(('a', ), False)
event4 = create_event(('a', ), True)
db.add(event1)
db.add(event2)
result = await query(db)
assert result == [event1]
await flush(db)
db.add(event3)
db.add(event4)
result = await query(db)
if order_by == common.OrderBy.TIMESTAMP:
assert result == [event4, event3, event1]
elif order_by == common.OrderBy.SOURCE_TIMESTAMP:
assert result == [event4, event1]
async def test_query_max_results(executor, env, flush, query, create_event,
order_by, order):
name = 'name'
subscription = common.Subscription([('a', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=None)
event1 = create_event(('a', ), True)
event2 = create_event(('a', ), True)
event3 = create_event(('a', ), True)
result = await query(db, max_results=2, order=order)
assert result == []
db.add(event1)
result = await query(db, max_results=2, order=order)
assert result == [event1]
await flush(db)
result = await query(db, max_results=2, order=order)
assert result == [event1]
db.add(event2)
result = await query(db, max_results=2, order=order)
if order == common.Order.DESCENDING:
assert result == [event2, event1]
elif order == common.Order.ASCENDING:
assert result == [event1, event2]
db.add(event3)
result = await query(db, max_results=2, order=order)
if order == common.Order.DESCENDING:
assert result == [event3, event2]
elif order == common.Order.ASCENDING:
assert result == [event1, event2]
await flush(db)
result = await query(db, max_results=2, order=order)
if order == common.Order.DESCENDING:
assert result == [event3, event2]
elif order == common.Order.ASCENDING:
assert result == [event1, event2]
async def test_create_empty(executor, env):
name = 'name'
subscription = common.Subscription([])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions)
assert db.subscription == subscription
result = list(db.query(None))
assert result == []
async def create(conf: json.Data) -> 'LmdbBackend':
backend = LmdbBackend()
backend._sync_period = conf['sync_period']
backend._executor = aio.create_executor(1)
backend._conditions = Conditions(conf['conditions'])
backend._env = await backend._executor(_ext_create_env,
Path(conf['db_path']),
conf['max_db_size'],
2 + 2 * len(conf['ordered']))
backend._sys_db = await hat.event.server.backends.lmdb.systemdb.create(
backend._executor, backend._env, 'system', conf['server_id'])
subscription = common.Subscription(
tuple(i) for i in conf['latest']['subscriptions'])
backend._latest_db = await hat.event.server.backends.lmdb.latestdb.create(
backend._executor, backend._env, 'latest', subscription,
backend._conditions)
backend._ordered_dbs = collections.deque()
for i, i_conf in enumerate(conf['ordered']):
order_by = common.OrderBy[i_conf['order_by']]
subscription = common.Subscription(
tuple(et) for et in i_conf['subscriptions'])
limit = i_conf.get('limit')
name = f'ordered_{i}'
ordered_dbs = await hat.event.server.backends.lmdb.ordereddb.create(
backend._executor, backend._env, name, subscription,
backend._conditions, order_by, limit)
backend._ordered_dbs.append(ordered_dbs)
backend._async_group = aio.Group()
backend._async_group.spawn(backend._write_loop)
return backend
async def test_subscription_change(executor, env, flush, create_event):
name = 'name'
subscription1 = common.Subscription([('a',)])
subscription2 = common.Subscription([('b',)])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription1,
conditions=conditions)
event = create_event(('a',), None)
db.add(event)
await flush(db)
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription1,
conditions=conditions)
result = set(db.query(None))
assert result == {event}
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription2,
conditions=conditions)
result = set(db.query(None))
assert result == set()
async def test_create_empty(executor, env, query, order_by):
name = 'name'
subscription = common.Subscription([])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=None)
assert db.subscription == subscription
assert db.order_by == order_by
result = await query(db)
assert result == []
async def query(self, data: common.QueryData) -> typing.List[common.Event]:
if (data.event_ids is None and data.t_to is None
and data.source_t_from is None and data.source_t_to is None
and data.payload is None
and data.order == common.Order.DESCENDING
and data.order_by == common.OrderBy.TIMESTAMP
and data.unique_type):
events = self._latest_db.query(event_types=data.event_types)
if data.t_from is not None:
events = (event for event in events
if data.t_from <= event.timestamp)
events = sorted(events,
key=lambda i: (i.timestamp, i.event_id),
reverse=True)
if data.max_results is not None:
events = events[:data.max_results]
return events
subscription = (common.Subscription(data.event_types)
if data.event_types is not None else None)
for db in self._ordered_dbs:
if db.order_by != data.order_by:
continue
if subscription and subscription.isdisjoint(db.subscription):
continue
events = await db.query(subscription=subscription,
event_ids=data.event_ids,
t_from=data.t_from,
t_to=data.t_to,
source_t_from=data.source_t_from,
source_t_to=data.source_t_to,
payload=data.payload,
order=data.order,
unique_type=data.unique_type,
max_results=data.max_results)
return list(events)
return []
async def test_add(executor, env, flush, create_event):
name = 'name'
subscription = common.Subscription([('*',)])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions)
result = list(db.query(None))
assert result == []
event1 = create_event(('a',), None)
db.add(event1)
result = set(db.query(None))
assert result == {event1}
event2 = create_event(('b',), None)
db.add(event2)
result = set(db.query(None))
assert result == {event1, event2}
event3 = create_event(('a',), None)
db.add(event3)
result = set(db.query(None))
assert result == {event2, event3}
await flush(db)
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions)
result = set(db.query(None))
assert result == {event2, event3}
async def test_limit_duration(executor, env, flush, query, create_event,
order_by):
name = 'name'
subscription = common.Subscription([('*', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
limit = {'duration': 1}
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=limit)
t1 = common.now()
t2 = t1._replace(s=t1.s + 2 * limit['duration'])
t3 = t2._replace(s=t2.s + 2 * limit['duration'])
event1 = create_event(('a', ), True)._replace(timestamp=t1,
source_timestamp=t1)
event2 = create_event(('a', ), True)._replace(timestamp=t2,
source_timestamp=t2)
db.add(event1)
db.add(event2)
result = await query(db)
assert result == [event2, event1]
await flush(db, t2)
result = await query(db)
assert result == [event2]
await flush(db, t3)
result = await query(db)
assert result == []
async def test_conditions_change(executor, env, flush, create_event):
name = 'name'
subscription = common.Subscription([('*',)])
conditions1 = hat.event.server.backends.lmdb.conditions.Conditions([])
conditions2 = hat.event.server.backends.lmdb.conditions.Conditions([{
'subscriptions': [['*']],
'condition': {'type': 'json'}}])
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions1)
event = create_event(('a',), None)
db.add(event)
await flush(db)
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions1)
result = set(db.query(None))
assert result == {event}
db = await hat.event.server.backends.lmdb.latestdb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions2)
result = set(db.query(None))
assert result == set()
async def test_limit_min_entries(executor, env, flush, query, create_event,
order_by):
name = 'name'
subscription = common.Subscription([('*', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
limit = {'min_entries': 5, 'max_entries': 3}
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=limit)
for i in range(limit['min_entries'] * 2):
db.add(create_event(('a', ), True))
await flush(db)
expected_len = (i + 1
if i < limit['min_entries'] else limit['min_entries'])
result = await query(db)
assert expected_len == len(result)
def __init__(self, conf: json.Data):
self._conditions = [(common.Subscription(i['subscriptions']),
_create_condition(i['condition'])) for i in conf]
async def test_query_timestamps(executor, env, flush, query, create_event,
order_by, order):
name = 'name'
subscription = common.Subscription([('a', )])
conditions = hat.event.server.backends.lmdb.conditions.Conditions([])
db = await hat.event.server.backends.lmdb.ordereddb.create(
executor=executor,
env=env,
name=name,
subscription=subscription,
conditions=conditions,
order_by=order_by,
limit=None)
event1 = create_event(('a', ), True)
await asyncio.sleep(0.001)
event2 = create_event(('a', ), True)
await asyncio.sleep(0.001)
event3 = create_event(('a', ), True)
await asyncio.sleep(0.001)
event4 = create_event(('a', ), True)
await asyncio.sleep(0.001)
db.add(event1)
db.add(event2)
db.add(event3)
db.add(event4)
for _ in range(2):
result = await query(db, order=order)
if order == common.Order.DESCENDING:
assert result == [event4, event3, event2, event1]
elif order == common.Order.ASCENDING:
assert result == [event1, event2, event3, event4]
result = await query(db, order=order, t_from=event2.timestamp)
if order == common.Order.DESCENDING:
assert result == [event4, event3, event2]
elif order == common.Order.ASCENDING:
assert result == [event2, event3, event4]
result = await query(db,
order=order,
source_t_from=event2.source_timestamp)
if order == common.Order.DESCENDING:
assert result == [event4, event3, event2]
elif order == common.Order.ASCENDING:
assert result == [event2, event3, event4]
result = await query(db, order=order, t_to=event3.timestamp)
if order == common.Order.DESCENDING:
assert result == [event3, event2, event1]
elif order == common.Order.ASCENDING:
assert result == [event1, event2, event3]
result = await query(db,
order=order,
source_t_to=event3.source_timestamp)
if order == common.Order.DESCENDING:
assert result == [event3, event2, event1]
elif order == common.Order.ASCENDING:
assert result == [event1, event2, event3]
result = await query(db,
order=order,
t_from=event1.timestamp,
t_to=event4.timestamp,
source_t_from=event1.source_timestamp,
source_t_to=event4.source_timestamp)
if order == common.Order.DESCENDING:
assert result == [event4, event3, event2, event1]
elif order == common.Order.ASCENDING:
assert result == [event1, event2, event3, event4]
t = common.now()
result = await query(db,
order=order,
t_from=t,
t_to=t,
source_t_from=t,
source_t_to=t)
assert result == []
await flush(db)