async def test_query_subscription(executor, env, flush, query, create_event, order_by): name = 'name' subscription = common.Subscription([('*', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=None) event1 = create_event(('a', ), True) event2 = create_event(('b', ), True) db.add(event1) db.add(event2) result = await query(db) assert result == [event2, event1] subscription = common.Subscription([('a', )]) result = await query(db, subscription=subscription) assert result == [event1] subscription = common.Subscription([('b', )]) result = await query(db, subscription=subscription) assert result == [event2]
async def test_query_payload(executor, env, flush, query, create_event, order_by): name = 'name' subscription = common.Subscription([('*', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=None) event1 = create_event(('a', ), True) event2 = create_event(('b', ), True)._replace( payload=common.EventPayload(common.EventPayloadType.JSON, data=123)) db.add(event1) db.add(event2) result = await query(db) assert result == [event2, event1] result = await query(db, payload=common.EventPayload( common.EventPayloadType.JSON, data=123)) assert result == [event2] result = await query(db, payload=common.EventPayload( common.EventPayloadType.JSON, data=321)) assert result == []
async def test_add(executor, env, flush, query, create_event, order_by): name = 'name' subscription = common.Subscription([('a', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=None) event1 = create_event(('a', ), True) event2 = create_event(('b', ), True) event3 = create_event(('a', ), False) event4 = create_event(('a', ), True) db.add(event1) db.add(event2) result = await query(db) assert result == [event1] await flush(db) db.add(event3) db.add(event4) result = await query(db) if order_by == common.OrderBy.TIMESTAMP: assert result == [event4, event3, event1] elif order_by == common.OrderBy.SOURCE_TIMESTAMP: assert result == [event4, event1]
async def test_query_max_results(executor, env, flush, query, create_event, order_by, order): name = 'name' subscription = common.Subscription([('a', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=None) event1 = create_event(('a', ), True) event2 = create_event(('a', ), True) event3 = create_event(('a', ), True) result = await query(db, max_results=2, order=order) assert result == [] db.add(event1) result = await query(db, max_results=2, order=order) assert result == [event1] await flush(db) result = await query(db, max_results=2, order=order) assert result == [event1] db.add(event2) result = await query(db, max_results=2, order=order) if order == common.Order.DESCENDING: assert result == [event2, event1] elif order == common.Order.ASCENDING: assert result == [event1, event2] db.add(event3) result = await query(db, max_results=2, order=order) if order == common.Order.DESCENDING: assert result == [event3, event2] elif order == common.Order.ASCENDING: assert result == [event1, event2] await flush(db) result = await query(db, max_results=2, order=order) if order == common.Order.DESCENDING: assert result == [event3, event2] elif order == common.Order.ASCENDING: assert result == [event1, event2]
async def test_create_empty(executor, env): name = 'name' subscription = common.Subscription([]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions) assert db.subscription == subscription result = list(db.query(None)) assert result == []
async def create(conf: json.Data) -> 'LmdbBackend': backend = LmdbBackend() backend._sync_period = conf['sync_period'] backend._executor = aio.create_executor(1) backend._conditions = Conditions(conf['conditions']) backend._env = await backend._executor(_ext_create_env, Path(conf['db_path']), conf['max_db_size'], 2 + 2 * len(conf['ordered'])) backend._sys_db = await hat.event.server.backends.lmdb.systemdb.create( backend._executor, backend._env, 'system', conf['server_id']) subscription = common.Subscription( tuple(i) for i in conf['latest']['subscriptions']) backend._latest_db = await hat.event.server.backends.lmdb.latestdb.create( backend._executor, backend._env, 'latest', subscription, backend._conditions) backend._ordered_dbs = collections.deque() for i, i_conf in enumerate(conf['ordered']): order_by = common.OrderBy[i_conf['order_by']] subscription = common.Subscription( tuple(et) for et in i_conf['subscriptions']) limit = i_conf.get('limit') name = f'ordered_{i}' ordered_dbs = await hat.event.server.backends.lmdb.ordereddb.create( backend._executor, backend._env, name, subscription, backend._conditions, order_by, limit) backend._ordered_dbs.append(ordered_dbs) backend._async_group = aio.Group() backend._async_group.spawn(backend._write_loop) return backend
async def test_subscription_change(executor, env, flush, create_event): name = 'name' subscription1 = common.Subscription([('a',)]) subscription2 = common.Subscription([('b',)]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription1, conditions=conditions) event = create_event(('a',), None) db.add(event) await flush(db) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription1, conditions=conditions) result = set(db.query(None)) assert result == {event} db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription2, conditions=conditions) result = set(db.query(None)) assert result == set()
async def test_create_empty(executor, env, query, order_by): name = 'name' subscription = common.Subscription([]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=None) assert db.subscription == subscription assert db.order_by == order_by result = await query(db) assert result == []
async def query(self, data: common.QueryData) -> typing.List[common.Event]: if (data.event_ids is None and data.t_to is None and data.source_t_from is None and data.source_t_to is None and data.payload is None and data.order == common.Order.DESCENDING and data.order_by == common.OrderBy.TIMESTAMP and data.unique_type): events = self._latest_db.query(event_types=data.event_types) if data.t_from is not None: events = (event for event in events if data.t_from <= event.timestamp) events = sorted(events, key=lambda i: (i.timestamp, i.event_id), reverse=True) if data.max_results is not None: events = events[:data.max_results] return events subscription = (common.Subscription(data.event_types) if data.event_types is not None else None) for db in self._ordered_dbs: if db.order_by != data.order_by: continue if subscription and subscription.isdisjoint(db.subscription): continue events = await db.query(subscription=subscription, event_ids=data.event_ids, t_from=data.t_from, t_to=data.t_to, source_t_from=data.source_t_from, source_t_to=data.source_t_to, payload=data.payload, order=data.order, unique_type=data.unique_type, max_results=data.max_results) return list(events) return []
async def test_add(executor, env, flush, create_event): name = 'name' subscription = common.Subscription([('*',)]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions) result = list(db.query(None)) assert result == [] event1 = create_event(('a',), None) db.add(event1) result = set(db.query(None)) assert result == {event1} event2 = create_event(('b',), None) db.add(event2) result = set(db.query(None)) assert result == {event1, event2} event3 = create_event(('a',), None) db.add(event3) result = set(db.query(None)) assert result == {event2, event3} await flush(db) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions) result = set(db.query(None)) assert result == {event2, event3}
async def test_limit_duration(executor, env, flush, query, create_event, order_by): name = 'name' subscription = common.Subscription([('*', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) limit = {'duration': 1} db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=limit) t1 = common.now() t2 = t1._replace(s=t1.s + 2 * limit['duration']) t3 = t2._replace(s=t2.s + 2 * limit['duration']) event1 = create_event(('a', ), True)._replace(timestamp=t1, source_timestamp=t1) event2 = create_event(('a', ), True)._replace(timestamp=t2, source_timestamp=t2) db.add(event1) db.add(event2) result = await query(db) assert result == [event2, event1] await flush(db, t2) result = await query(db) assert result == [event2] await flush(db, t3) result = await query(db) assert result == []
async def test_conditions_change(executor, env, flush, create_event): name = 'name' subscription = common.Subscription([('*',)]) conditions1 = hat.event.server.backends.lmdb.conditions.Conditions([]) conditions2 = hat.event.server.backends.lmdb.conditions.Conditions([{ 'subscriptions': [['*']], 'condition': {'type': 'json'}}]) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions1) event = create_event(('a',), None) db.add(event) await flush(db) db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions1) result = set(db.query(None)) assert result == {event} db = await hat.event.server.backends.lmdb.latestdb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions2) result = set(db.query(None)) assert result == set()
async def test_limit_min_entries(executor, env, flush, query, create_event, order_by): name = 'name' subscription = common.Subscription([('*', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) limit = {'min_entries': 5, 'max_entries': 3} db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=limit) for i in range(limit['min_entries'] * 2): db.add(create_event(('a', ), True)) await flush(db) expected_len = (i + 1 if i < limit['min_entries'] else limit['min_entries']) result = await query(db) assert expected_len == len(result)
def __init__(self, conf: json.Data): self._conditions = [(common.Subscription(i['subscriptions']), _create_condition(i['condition'])) for i in conf]
async def test_query_timestamps(executor, env, flush, query, create_event, order_by, order): name = 'name' subscription = common.Subscription([('a', )]) conditions = hat.event.server.backends.lmdb.conditions.Conditions([]) db = await hat.event.server.backends.lmdb.ordereddb.create( executor=executor, env=env, name=name, subscription=subscription, conditions=conditions, order_by=order_by, limit=None) event1 = create_event(('a', ), True) await asyncio.sleep(0.001) event2 = create_event(('a', ), True) await asyncio.sleep(0.001) event3 = create_event(('a', ), True) await asyncio.sleep(0.001) event4 = create_event(('a', ), True) await asyncio.sleep(0.001) db.add(event1) db.add(event2) db.add(event3) db.add(event4) for _ in range(2): result = await query(db, order=order) if order == common.Order.DESCENDING: assert result == [event4, event3, event2, event1] elif order == common.Order.ASCENDING: assert result == [event1, event2, event3, event4] result = await query(db, order=order, t_from=event2.timestamp) if order == common.Order.DESCENDING: assert result == [event4, event3, event2] elif order == common.Order.ASCENDING: assert result == [event2, event3, event4] result = await query(db, order=order, source_t_from=event2.source_timestamp) if order == common.Order.DESCENDING: assert result == [event4, event3, event2] elif order == common.Order.ASCENDING: assert result == [event2, event3, event4] result = await query(db, order=order, t_to=event3.timestamp) if order == common.Order.DESCENDING: assert result == [event3, event2, event1] elif order == common.Order.ASCENDING: assert result == [event1, event2, event3] result = await query(db, order=order, source_t_to=event3.source_timestamp) if order == common.Order.DESCENDING: assert result == [event3, event2, event1] elif order == common.Order.ASCENDING: assert result == [event1, event2, event3] result = await query(db, order=order, t_from=event1.timestamp, t_to=event4.timestamp, source_t_from=event1.source_timestamp, source_t_to=event4.source_timestamp) if order == common.Order.DESCENDING: assert result == [event4, event3, event2, event1] elif order == common.Order.ASCENDING: assert result == [event1, event2, event3, event4] t = common.now() result = await query(db, order=order, t_from=t, t_to=t, source_t_from=t, source_t_to=t) assert result == [] await flush(db)