async def create(conf, client): adapter = MockAdapter() adapter._async_group = aio.Group() adapter._conf = conf adapter._client = client adapter._sessions = [] return adapter
async def test_register_query_on_close(module_engine_conf, register_events, source_comm): comm_register = asyncio.Event() async def unresponsive_cb(async_event, _): async_event.set() while True: await asyncio.sleep(1) backend_engine = common.create_backend_engine( register_cb=functools.partial(unresponsive_cb, comm_register)) module_engine = await hat.event.server.module_engine.create( module_engine_conf, backend_engine) assert not module_engine.closed.done() process_events = [ module_engine.create_process_event(source_comm, i) for i in register_events ] async with aio.Group() as group: register_future = group.spawn(module_engine.register, process_events) await comm_register.wait() await backend_engine.async_close() await module_engine.async_close() assert module_engine.closed.done() with pytest.raises(asyncio.CancelledError): await register_future with pytest.raises(hat.event.server.module_engine.ModuleEngineClosedError): await module_engine.register(process_events) with pytest.raises(hat.event.server.module_engine.ModuleEngineClosedError): await module_engine.query(hat.event.common.QueryData())
async def create(conf, engine): """Create communication Args: conf (hat.json.Data): configuration defined by ``hat://event/main.yaml#/definitions/communication`` engine (hat.event.module_engine.ModuleEngine): module engine Returns: Communication """ comm = Communication() comm._engine = engine comm._async_group = aio.Group(exception_cb=lambda e: mlog.error( 'exception in communication: %s', e, exc_info=e)) comm._connection_ids = {} comm._subs_registry = common.SubscriptionRegistry() chatter_server = await chatter.listen( sbs_repo=common.sbs_repo, address=conf['address'], on_connection_cb=lambda conn: comm._async_group.spawn( comm._connection_loop, conn)) comm._async_group.spawn(aio.call_on_cancel, chatter_server.async_close) comm._async_group.spawn(comm._run_engine) return comm
async def create(conf, event_client): adapter = MockAdapter() adapter._event_client = event_client adapter._group = aio.Group() adapter._sessions = [] adapter._group.spawn(adapter._event_loop) return adapter
async def connect(conf): """Connect to local monitor server Connection is established once chatter communication is established. Args: conf (hat.json.Data): configuration as defined by ``hat://monitor/client.yaml#`` Returns: Client """ client = Client() client._name = conf['name'] client._group = conf['group'] client._address = conf['component_address'] client._components = [] client._info = None client._ready = None client._change_cbs = util.CallbackRegistry() client._async_group = aio.Group() client._conn = await chatter.connect(common.sbs_repo, conf['monitor_address']) client._async_group.spawn(aio.call_on_cancel, client._conn.async_close) mlog.debug("connected to local monitor server %s", conf['monitor_address']) client._async_group.spawn(client._receive_loop) return client
def create_backend_engine(register_cb=lambda _: [], query_cb=lambda _: []): engine = MockBackendEngine() engine._async_group = aio.Group() engine._register_cb = register_cb engine._query_cb = query_cb engine._server_id = 0 return engine
async def create(conf): """Create local monitor server Args: conf (hat.json.Data): configuration defined by ``hat://monitor/main.yaml#/definitions/server`` Returns: Server """ server = Server() server._default_rank = conf['default_rank'] server._rank_cache = {} server._master = None server._mid = 0 server._components = [] server._async_group = aio.Group(server._on_exception) server._change_cbs = util.CallbackRegistry() server._master_change_handler = None server._connections = {} server._local_components = [] chatter_server = await chatter.listen( sbs_repo=common.sbs_repo, address=conf['address'], on_connection_cb=lambda conn: server._async_group.spawn( server._connection_loop, conn)) server._async_group.spawn(aio.call_on_cancel, chatter_server.async_close) mlog.debug('monitor server listens clients on %s', conf['address']) return server
async def run(conf, monitor): """Run Args: conf (json.Data): configuration defined by ``hat://event/main.yaml#`` monitor (hat.monitor.client.Client): monitor client """ async_group = aio.Group() backend_engine = None module_engine = None communication = None try: backend_engine = await hat.event.server.backend_engine.create( conf['backend_engine']) async_group.spawn(aio.call_on_cancel, backend_engine.async_close) module_engine = await hat.event.server.module_engine.create( conf['module_engine'], backend_engine) async_group.spawn(aio.call_on_cancel, module_engine.async_close) communication = await hat.event.server.communication.create( conf['communication'], module_engine) async_group.spawn(aio.call_on_cancel, communication.async_close) wait_futures = [ backend_engine.closed, module_engine.closed, communication.closed ] await asyncio.wait(wait_futures, return_when=asyncio.FIRST_COMPLETED) finally: await aio.uncancellable(async_group.async_close()) await asyncio.sleep(0.1)
def test_group_close_empty_group(): g = aio.Group() assert not g.closing.done() assert not g.closed.done() g.close() assert g.closing.done() assert g.closed.done()
async def create(conf): backend = MemoryBackend() backend._async_group = aio.Group() backend._mappings = {} backend._events = [] backend._query_data_queue = aio.Queue() return backend
async def test_group_async_close_subgroup_without_tasks(): g1 = aio.Group() g2 = g1.create_subgroup() await g1.async_close() assert g1.closed.done() assert g2.closed.done()
async def create_monitor_group(server_address, master_address, master_parents, default_algorithm='BLESS_ALL', group_algorithms={}): server_conf = {'address': server_address, 'default_rank': 1} master_conf = { 'address': master_address, 'parents': master_parents, 'default_algorithm': default_algorithm, 'group_algorithms': group_algorithms } group = aio.Group() server = await hat.monitor.server.server.create(server_conf) async def run(): master_run_future = group.spawn(hat.monitor.server.master.run, master_conf, server.set_master) try: await asyncio.wait([server.closed, master_run_future], return_when=asyncio.FIRST_COMPLETED) finally: group.close() await aio.uncancellable(server.async_close()) group.spawn(run) return group
async def factory(conf_adapters): conf = {'adapters': conf_adapters, 'views': None, 'server': None} adapters = {} create_default = test_unit.test_gui.mock.create async def create_view_manager(conf, repo): return _Closeable() async def create_server(conf, ui_path, adapters, views): return _Closeable() async def create_patch(conf, client): adapter = await create_default(conf, client) adapters[conf['name']] = adapter all_adapters.append(adapter) return adapter with monkeypatch.context() as ctx: ctx.setattr(hat.gui.view, 'create_view_manager', create_view_manager) ctx.setattr(hat.gui.server, 'create', create_server) ctx.setattr(test_unit.test_gui.mock, 'create', create_patch) async with aio.Group() as group: group.spawn( hat.gui.main.run_with_event, conf, None, None, await event_client_factory( [test_unit.test_gui.mock.event_type_prefix + ['*']])) await asyncio.sleep(0.1) yield adapters
async def create(conf, backend_engine): """Create module engine Args: conf (hat.json.Data): configuration defined by ``hat://event/main.yaml#/definitions/module_engine`` backend_engine (hat.event.backend_engine.BackendEngine): backend engine Returns: ModuleEngine """ engine = ModuleEngine() engine._backend = backend_engine engine._async_group = aio.Group() engine._register_queue = aio.Queue() engine._register_cbs = util.CallbackRegistry() last_event_id = await engine._backend.get_last_event_id() engine._server_id = last_event_id.server engine._last_instance_id = last_event_id.instance engine._modules = [] for module_conf in conf['modules']: py_module = importlib.import_module(module_conf['module']) module = await py_module.create(module_conf, engine) engine._async_group.spawn(aio.call_on_cancel, module.async_close) engine._modules.append(module) engine._async_group.spawn(engine._register_loop) return engine
async def test_group_spawn_async_close(): async def task(): group.spawn(group.async_close) await asyncio.Future() group = aio.Group() group.spawn(task) await group.closed
def create_module_engine(register_cb=lambda _: [], query_cb=lambda _: []): engine = MockModuleEngine() engine._async_group = aio.Group() engine._last_instance_id = 0 engine._register_cb = register_cb engine._query_cb = query_cb engine._register_event_cbs = util.CallbackRegistry() return engine
async def create_client(address): ui_address = urllib.parse.urlparse(address) ws_address = 'ws://{}:{}/ws'.format(ui_address.hostname, ui_address.port) client = Client() client._conn = await juggler.connect(ws_address) client._async_group = aio.Group() return client
async def create_backend(_): backend = MemoryBackend() backend._first_id = None backend._last_id = None backend._entries = [] backend._async_group = aio.Group(lambda e: print(f"memory: error in {e}")) backend._change_cbs = util.CallbackRegistry() return backend
async def create(conf, engine): module = Module2() module._async_group = aio.Group() module._engine = engine module._source = hat.event.server.common.Source( type=hat.event.server.common.SourceType.MODULE, id=2) module.session_queue = aio.Queue() return module
def create_master(mid=0, components=[]): master = MockMaster() master._async_group = aio.Group() master._mid = mid master._components = components master._components_queue = aio.Queue() master._rank_queue = aio.Queue() master._change_cbs = util.CallbackRegistry() return master
async def run_component(conf, async_run_cb): """Run component This method opens new connection to Monitor server and starts client's loop which manages blessing/ready states. When blessing token matches ready token, `async_run_cb` is called. While `async_run_cb` is running, if blessing token changes, `async_run_cb` is canceled. If `async_run_cb` finishes or raises exception, this function closes connection to monitor server and returns `async_run_cb` result. If connection to monitor server is closed, this function raises exception. TODO: * provide opportunity for user to react to blessing token prior to setting ready token (additional async_ready_cb) Args: conf (hat.json.Data): configuration as defined by ``hat://monitor/client.yaml#`` async_run_cb (Callable[[Client],None]): run callback Returns: Any """ client = await connect(conf) try: while True: await _wait_until_blessed_and_ready(client) async_group = aio.Group() run_future = async_group.spawn(async_run_cb, client) blessed_and_ready_future = async_group.spawn( _wait_while_blessed_and_ready, client) try: done, _ = await asyncio.wait( [run_future, blessed_and_ready_future, client.closed], return_when=asyncio.FIRST_COMPLETED) if run_future.done(): mlog.debug('async_run_cb finished or raised an exception') return run_future.result() if client.closed.done(): raise Exception('connection to monitor server closed!') finally: if not client.closed.done(): client.set_ready(None) await async_group.async_close() except asyncio.CancelledError: raise except Exception as e: mlog.error('run component exception: %s', e, exc_info=e) raise finally: await client.async_close() mlog.debug('component closed')
def __init__(self, conf): self._conf = conf self._status = Status.DELAYED if conf['delay'] else Status.STOPPED self._revive = conf['revive'] self._change_cbs = util.CallbackRegistry( exception_cb=lambda e: mlog.warning( "change callback exception: %s", e, exc_info=e)) self._started_queue = aio.Queue() self._async_group = aio.Group() self._async_group.spawn(self._run_loop)
async def test_view_manager_closing(view_factory, view_manager_factory): view_manager = await view_manager_factory([view_factory('view', [])]) view = await view_manager.get('view') assert view.data == {} async with aio.Group() as group: group.spawn(view_manager.async_close) with pytest.raises(asyncio.CancelledError): await view_manager.get('view')
async def _create_listener(address): listener = _Listener() listener._cbs = set() listener._connection_cbs = util.CallbackRegistry() listener._async_group = aio.Group( lambda e: mlog.error("error in listener: %s", e, exc_info=e)) chatter_server = await chatter.listen(common.sbs_repo, address, listener._on_connection) listener._async_group.spawn(aio.call_on_cancel, chatter_server.async_close) return listener
async def test_group_spawn_when_not_open(): g = aio.Group() g.spawn(asyncio.Future) g.close() with pytest.raises(Exception): g.spawn(asyncio.Future) await g.async_close() with pytest.raises(Exception): g.spawn(asyncio.Future) with pytest.raises(Exception): g.wrap(asyncio.Future())
async def create_session(self): session = Module2Session() session._async_group = aio.Group() session._module = self session.changes_notified_new = [] session.changes_notified_deleted = [] session.changes_result_new = [] session.changes_result_deleted = [] session.events_on_close = None self.session_queue.put_nowait(session) return session
def _create_connection(sbs_repo, transport, ping_timeout, queue_maxsize): conn = Connection() conn._sbs_repo = sbs_repo conn._transport = transport conn._last_id = 0 conn._conv_timeouts = {} conn._msg_queue = aio.Queue(maxsize=queue_maxsize) conn._async_group = aio.Group(lambda e: mlog.error( 'connection async group exception: %s', e, exc_info=e)) conn._async_group.spawn(conn._read_loop) conn._async_group.spawn(conn._ping_loop, ping_timeout) return conn
def main(): message_count = int(get_last_sys_arg()) backend_type = get_last_sys_arg() create_fn = { "memory": memory_backend.create_backend, "sqlite": sqlite_backend.create_backend }[backend_type] create_backend, backend_queue = get_create_backend_queue(create_fn) group = aio.Group() group.spawn(until_all_messages, backend_queue, group, message_count) hat.syslog.server.main.create_backend = create_backend hat.syslog.server.main.main()
async def event_server(event_server_port): conf = { 'backend_engine': { 'server_id': 1, 'backend': { 'module': 'test_unit.test_event.backends.memory_backend'}}, 'module_engine': {'modules': []}, 'communication': { 'address': f'tcp+sbs://127.0.0.1:{event_server_port}'}} async with aio.Group() as group: group.spawn(hat.event.server.main.run, conf, None) await asyncio.sleep(0.01) # Wait for event server to start yield
async def create(conf): """Create DummyBackend Args: conf (hat.json.Data): configuration Returns: DummyBackend """ backend = DummyBackend() backend._group = aio.Group() return backend