async def create(conf, client):
adapter = MockAdapter()
adapter._async_group = aio.Group()
adapter._conf = conf
adapter._client = client
adapter._sessions = []
return adapter
async def test_register_query_on_close(module_engine_conf, register_events,
source_comm):
comm_register = asyncio.Event()
async def unresponsive_cb(async_event, _):
async_event.set()
while True:
await asyncio.sleep(1)
backend_engine = common.create_backend_engine(
register_cb=functools.partial(unresponsive_cb, comm_register))
module_engine = await hat.event.server.module_engine.create(
module_engine_conf, backend_engine)
assert not module_engine.closed.done()
process_events = [
module_engine.create_process_event(source_comm, i)
for i in register_events
]
async with aio.Group() as group:
register_future = group.spawn(module_engine.register, process_events)
await comm_register.wait()
await backend_engine.async_close()
await module_engine.async_close()
assert module_engine.closed.done()
with pytest.raises(asyncio.CancelledError):
await register_future
with pytest.raises(hat.event.server.module_engine.ModuleEngineClosedError):
await module_engine.register(process_events)
with pytest.raises(hat.event.server.module_engine.ModuleEngineClosedError):
await module_engine.query(hat.event.common.QueryData())
async def create(conf, engine):
"""Create communication
Args:
conf (hat.json.Data): configuration defined by
``hat://event/main.yaml#/definitions/communication``
engine (hat.event.module_engine.ModuleEngine): module engine
Returns:
Communication
"""
comm = Communication()
comm._engine = engine
comm._async_group = aio.Group(exception_cb=lambda e: mlog.error(
'exception in communication: %s', e, exc_info=e))
comm._connection_ids = {}
comm._subs_registry = common.SubscriptionRegistry()
chatter_server = await chatter.listen(
sbs_repo=common.sbs_repo,
address=conf['address'],
on_connection_cb=lambda conn: comm._async_group.spawn(
comm._connection_loop, conn))
comm._async_group.spawn(aio.call_on_cancel, chatter_server.async_close)
comm._async_group.spawn(comm._run_engine)
return comm
async def create(conf, event_client):
adapter = MockAdapter()
adapter._event_client = event_client
adapter._group = aio.Group()
adapter._sessions = []
adapter._group.spawn(adapter._event_loop)
return adapter
async def connect(conf):
"""Connect to local monitor server
Connection is established once chatter communication is established.
Args:
conf (hat.json.Data): configuration as defined by
``hat://monitor/client.yaml#``
Returns:
Client
"""
client = Client()
client._name = conf['name']
client._group = conf['group']
client._address = conf['component_address']
client._components = []
client._info = None
client._ready = None
client._change_cbs = util.CallbackRegistry()
client._async_group = aio.Group()
client._conn = await chatter.connect(common.sbs_repo,
conf['monitor_address'])
client._async_group.spawn(aio.call_on_cancel, client._conn.async_close)
mlog.debug("connected to local monitor server %s", conf['monitor_address'])
client._async_group.spawn(client._receive_loop)
return client
def create_backend_engine(register_cb=lambda _: [], query_cb=lambda _: []):
engine = MockBackendEngine()
engine._async_group = aio.Group()
engine._register_cb = register_cb
engine._query_cb = query_cb
engine._server_id = 0
return engine
async def create(conf):
"""Create local monitor server
Args:
conf (hat.json.Data): configuration defined by
``hat://monitor/main.yaml#/definitions/server``
Returns:
Server
"""
server = Server()
server._default_rank = conf['default_rank']
server._rank_cache = {}
server._master = None
server._mid = 0
server._components = []
server._async_group = aio.Group(server._on_exception)
server._change_cbs = util.CallbackRegistry()
server._master_change_handler = None
server._connections = {}
server._local_components = []
chatter_server = await chatter.listen(
sbs_repo=common.sbs_repo,
address=conf['address'],
on_connection_cb=lambda conn: server._async_group.spawn(
server._connection_loop, conn))
server._async_group.spawn(aio.call_on_cancel, chatter_server.async_close)
mlog.debug('monitor server listens clients on %s', conf['address'])
return server
async def run(conf, monitor):
"""Run
Args:
conf (json.Data): configuration defined by ``hat://event/main.yaml#``
monitor (hat.monitor.client.Client): monitor client
"""
async_group = aio.Group()
backend_engine = None
module_engine = None
communication = None
try:
backend_engine = await hat.event.server.backend_engine.create(
conf['backend_engine'])
async_group.spawn(aio.call_on_cancel, backend_engine.async_close)
module_engine = await hat.event.server.module_engine.create(
conf['module_engine'], backend_engine)
async_group.spawn(aio.call_on_cancel, module_engine.async_close)
communication = await hat.event.server.communication.create(
conf['communication'], module_engine)
async_group.spawn(aio.call_on_cancel, communication.async_close)
wait_futures = [
backend_engine.closed, module_engine.closed, communication.closed
]
await asyncio.wait(wait_futures, return_when=asyncio.FIRST_COMPLETED)
finally:
await aio.uncancellable(async_group.async_close())
await asyncio.sleep(0.1)
def test_group_close_empty_group():
g = aio.Group()
assert not g.closing.done()
assert not g.closed.done()
g.close()
assert g.closing.done()
assert g.closed.done()
async def create(conf):
backend = MemoryBackend()
backend._async_group = aio.Group()
backend._mappings = {}
backend._events = []
backend._query_data_queue = aio.Queue()
return backend
async def test_group_async_close_subgroup_without_tasks():
g1 = aio.Group()
g2 = g1.create_subgroup()
await g1.async_close()
assert g1.closed.done()
assert g2.closed.done()
async def create_monitor_group(server_address,
master_address,
master_parents,
default_algorithm='BLESS_ALL',
group_algorithms={}):
server_conf = {'address': server_address, 'default_rank': 1}
master_conf = {
'address': master_address,
'parents': master_parents,
'default_algorithm': default_algorithm,
'group_algorithms': group_algorithms
}
group = aio.Group()
server = await hat.monitor.server.server.create(server_conf)
async def run():
master_run_future = group.spawn(hat.monitor.server.master.run,
master_conf, server.set_master)
try:
await asyncio.wait([server.closed, master_run_future],
return_when=asyncio.FIRST_COMPLETED)
finally:
group.close()
await aio.uncancellable(server.async_close())
group.spawn(run)
return group
async def factory(conf_adapters):
conf = {'adapters': conf_adapters,
'views': None,
'server': None}
adapters = {}
create_default = test_unit.test_gui.mock.create
async def create_view_manager(conf, repo):
return _Closeable()
async def create_server(conf, ui_path, adapters, views):
return _Closeable()
async def create_patch(conf, client):
adapter = await create_default(conf, client)
adapters[conf['name']] = adapter
all_adapters.append(adapter)
return adapter
with monkeypatch.context() as ctx:
ctx.setattr(hat.gui.view, 'create_view_manager',
create_view_manager)
ctx.setattr(hat.gui.server, 'create', create_server)
ctx.setattr(test_unit.test_gui.mock, 'create', create_patch)
async with aio.Group() as group:
group.spawn(
hat.gui.main.run_with_event, conf, None, None,
await event_client_factory(
[test_unit.test_gui.mock.event_type_prefix + ['*']]))
await asyncio.sleep(0.1)
yield adapters
async def create(conf, backend_engine):
"""Create module engine
Args:
conf (hat.json.Data): configuration defined by
``hat://event/main.yaml#/definitions/module_engine``
backend_engine (hat.event.backend_engine.BackendEngine): backend engine
Returns:
ModuleEngine
"""
engine = ModuleEngine()
engine._backend = backend_engine
engine._async_group = aio.Group()
engine._register_queue = aio.Queue()
engine._register_cbs = util.CallbackRegistry()
last_event_id = await engine._backend.get_last_event_id()
engine._server_id = last_event_id.server
engine._last_instance_id = last_event_id.instance
engine._modules = []
for module_conf in conf['modules']:
py_module = importlib.import_module(module_conf['module'])
module = await py_module.create(module_conf, engine)
engine._async_group.spawn(aio.call_on_cancel, module.async_close)
engine._modules.append(module)
engine._async_group.spawn(engine._register_loop)
return engine
async def test_group_spawn_async_close():
async def task():
group.spawn(group.async_close)
await asyncio.Future()
group = aio.Group()
group.spawn(task)
await group.closed
def create_module_engine(register_cb=lambda _: [], query_cb=lambda _: []):
engine = MockModuleEngine()
engine._async_group = aio.Group()
engine._last_instance_id = 0
engine._register_cb = register_cb
engine._query_cb = query_cb
engine._register_event_cbs = util.CallbackRegistry()
return engine
async def create_client(address):
ui_address = urllib.parse.urlparse(address)
ws_address = 'ws://{}:{}/ws'.format(ui_address.hostname,
ui_address.port)
client = Client()
client._conn = await juggler.connect(ws_address)
client._async_group = aio.Group()
return client
async def create_backend(_):
backend = MemoryBackend()
backend._first_id = None
backend._last_id = None
backend._entries = []
backend._async_group = aio.Group(lambda e: print(f"memory: error in {e}"))
backend._change_cbs = util.CallbackRegistry()
return backend
async def create(conf, engine):
module = Module2()
module._async_group = aio.Group()
module._engine = engine
module._source = hat.event.server.common.Source(
type=hat.event.server.common.SourceType.MODULE, id=2)
module.session_queue = aio.Queue()
return module
def create_master(mid=0, components=[]):
master = MockMaster()
master._async_group = aio.Group()
master._mid = mid
master._components = components
master._components_queue = aio.Queue()
master._rank_queue = aio.Queue()
master._change_cbs = util.CallbackRegistry()
return master
async def run_component(conf, async_run_cb):
"""Run component
This method opens new connection to Monitor server and starts client's
loop which manages blessing/ready states.
When blessing token matches ready token, `async_run_cb` is called. While
`async_run_cb` is running, if blessing token changes, `async_run_cb` is
canceled.
If `async_run_cb` finishes or raises exception, this function closes
connection to monitor server and returns `async_run_cb` result. If
connection to monitor server is closed, this function raises exception.
TODO:
* provide opportunity for user to react to blessing token prior to
setting ready token (additional async_ready_cb)
Args:
conf (hat.json.Data): configuration as defined by
``hat://monitor/client.yaml#``
async_run_cb (Callable[[Client],None]): run callback
Returns:
Any
"""
client = await connect(conf)
try:
while True:
await _wait_until_blessed_and_ready(client)
async_group = aio.Group()
run_future = async_group.spawn(async_run_cb, client)
blessed_and_ready_future = async_group.spawn(
_wait_while_blessed_and_ready, client)
try:
done, _ = await asyncio.wait(
[run_future, blessed_and_ready_future, client.closed],
return_when=asyncio.FIRST_COMPLETED)
if run_future.done():
mlog.debug('async_run_cb finished or raised an exception')
return run_future.result()
if client.closed.done():
raise Exception('connection to monitor server closed!')
finally:
if not client.closed.done():
client.set_ready(None)
await async_group.async_close()
except asyncio.CancelledError:
raise
except Exception as e:
mlog.error('run component exception: %s', e, exc_info=e)
raise
finally:
await client.async_close()
mlog.debug('component closed')
def __init__(self, conf):
self._conf = conf
self._status = Status.DELAYED if conf['delay'] else Status.STOPPED
self._revive = conf['revive']
self._change_cbs = util.CallbackRegistry(
exception_cb=lambda e: mlog.warning(
"change callback exception: %s", e, exc_info=e))
self._started_queue = aio.Queue()
self._async_group = aio.Group()
self._async_group.spawn(self._run_loop)
async def test_view_manager_closing(view_factory, view_manager_factory):
view_manager = await view_manager_factory([view_factory('view', [])])
view = await view_manager.get('view')
assert view.data == {}
async with aio.Group() as group:
group.spawn(view_manager.async_close)
with pytest.raises(asyncio.CancelledError):
await view_manager.get('view')
async def _create_listener(address):
listener = _Listener()
listener._cbs = set()
listener._connection_cbs = util.CallbackRegistry()
listener._async_group = aio.Group(
lambda e: mlog.error("error in listener: %s", e, exc_info=e))
chatter_server = await chatter.listen(common.sbs_repo, address,
listener._on_connection)
listener._async_group.spawn(aio.call_on_cancel, chatter_server.async_close)
return listener
async def test_group_spawn_when_not_open():
g = aio.Group()
g.spawn(asyncio.Future)
g.close()
with pytest.raises(Exception):
g.spawn(asyncio.Future)
await g.async_close()
with pytest.raises(Exception):
g.spawn(asyncio.Future)
with pytest.raises(Exception):
g.wrap(asyncio.Future())
async def create_session(self):
session = Module2Session()
session._async_group = aio.Group()
session._module = self
session.changes_notified_new = []
session.changes_notified_deleted = []
session.changes_result_new = []
session.changes_result_deleted = []
session.events_on_close = None
self.session_queue.put_nowait(session)
return session
def _create_connection(sbs_repo, transport, ping_timeout, queue_maxsize):
conn = Connection()
conn._sbs_repo = sbs_repo
conn._transport = transport
conn._last_id = 0
conn._conv_timeouts = {}
conn._msg_queue = aio.Queue(maxsize=queue_maxsize)
conn._async_group = aio.Group(lambda e: mlog.error(
'connection async group exception: %s', e, exc_info=e))
conn._async_group.spawn(conn._read_loop)
conn._async_group.spawn(conn._ping_loop, ping_timeout)
return conn
def main():
message_count = int(get_last_sys_arg())
backend_type = get_last_sys_arg()
create_fn = {
"memory": memory_backend.create_backend,
"sqlite": sqlite_backend.create_backend
}[backend_type]
create_backend, backend_queue = get_create_backend_queue(create_fn)
group = aio.Group()
group.spawn(until_all_messages, backend_queue, group, message_count)
hat.syslog.server.main.create_backend = create_backend
hat.syslog.server.main.main()
async def event_server(event_server_port):
conf = {
'backend_engine': {
'server_id': 1,
'backend': {
'module': 'test_unit.test_event.backends.memory_backend'}},
'module_engine': {'modules': []},
'communication': {
'address': f'tcp+sbs://127.0.0.1:{event_server_port}'}}
async with aio.Group() as group:
group.spawn(hat.event.server.main.run, conf, None)
await asyncio.sleep(0.01) # Wait for event server to start
yield
async def create(conf):
"""Create DummyBackend
Args:
conf (hat.json.Data): configuration
Returns:
DummyBackend
"""
backend = DummyBackend()
backend._group = aio.Group()
return backend
