def create_master(mid=0, components=[]):
master = MockMaster()
master._async_group = aio.Group()
master._mid = mid
master._components = components
master._components_queue = aio.Queue()
master._rank_queue = aio.Queue()
master._change_cbs = util.CallbackRegistry()
return master
async def test_queue_put():
queue = aio.Queue(1)
await queue.put(1)
put_future = asyncio.ensure_future(queue.put(1))
asyncio.get_event_loop().call_soon(queue.close)
with pytest.raises(aio.QueueClosedError):
await put_future
async def test_register(module_engine_conf, register_events, source_comm):
def register_cb(events):
return [common.process_event_to_event(i) for i in events]
event_queue = aio.Queue()
backend_engine = common.create_backend_engine(register_cb=register_cb)
module_engine = await hat.event.server.module_engine.create(
module_engine_conf, backend_engine)
module_engine.register_events_cb(
lambda events: event_queue.put_nowait(events))
process_events = [
module_engine.create_process_event(source_comm, i)
for i in register_events
]
events = await module_engine.register(process_events)
events_notified = await event_queue.get()
assert all(
common.compare_proces_event_vs_event(e1, e2)
for e1, e2 in zip(process_events, events))
assert all(
common.compare_events(e1, e2)
for e1, e2 in zip(events, events_notified))
await backend_engine.async_close()
await module_engine.async_close()
assert module_engine.closed.done()
assert event_queue.empty()
def test_queue_get_nowait_until_empty():
queue = aio.Queue()
queue.put_nowait(1)
queue.put_nowait(2)
queue.put_nowait(3)
result = queue.get_nowait_until_empty()
assert result == 3
async def create(local_addr=None, remote_addr=None, queue_size=0, **kwargs):
"""Create new UDP endpoint
Args:
local_addr (Optional[Address]): local address
remote_addr (Optional[Address]): remote address
queue_size (int): receive queue max size
kwargs: additional arguments passed to
:meth:`asyncio.AbstractEventLoop.create_datagram_endpoint`
Returns:
Endpoint
"""
endpoint = Endpoint()
endpoint._local_addr = local_addr
endpoint._remote_addr = remote_addr
endpoint._closed = asyncio.Future()
endpoint._queue = aio.Queue(queue_size)
class Protocol(asyncio.DatagramProtocol):
def connection_lost(self, exc):
endpoint._queue.close()
endpoint._closed.set_result(True)
def datagram_received(self, data, addr):
endpoint._queue.put_nowait((data, Address(addr[0], addr[1])))
loop = asyncio.get_running_loop()
endpoint._transport, endpoint._protocol = \
await loop.create_datagram_endpoint(Protocol, local_addr, remote_addr,
**kwargs)
return endpoint
async def test_run_client(event_server_factory, client_factory,
short_client_reconnect_delay, server_count):
client_queue = aio.Queue()
async def run_cb(event_client):
client_queue.put_nowait(event_client)
try:
while True:
await asyncio.sleep(1)
except asyncio.CancelledError:
client_queue.put_nowait(None)
raise
client_factory(run_cb)
for i in range(server_count):
server_group, _ = event_server_factory()
event_client = await client_queue.get()
assert event_client is not None
register_events = [
hat.event.common.RegisterEvent(event_type=[str(i)],
source_timestamp=None,
payload=None)
]
events = await event_client.register_with_response(register_events)
assert all(
common.compare_register_event_vs_event(r, e)
for r, e in zip(register_events, events))
await server_group.async_close()
assert (await client_queue.get()) is None
async def test_run_client_register_on_cancel(event_server_factory,
client_factory,
short_client_reconnect_delay):
register_events = [
hat.event.common.RegisterEvent(event_type=['test'],
source_timestamp=None,
payload=None)
]
is_running = asyncio.Event()
events_queue = aio.Queue()
async def run_cb(event_client):
try:
is_running.set()
while True:
await asyncio.sleep(1)
finally:
events = await event_client.register_with_response(register_events)
events_queue.put_nowait(events)
event_server_factory()
client_group, _ = client_factory(run_cb)
await is_running.wait()
await client_group.async_close()
events = await events_queue.get()
assert all(
common.compare_register_event_vs_event(r, e)
for r, e in zip(register_events, events))
async def create(conf):
backend = MemoryBackend()
backend._async_group = aio.Group()
backend._mappings = {}
backend._events = []
backend._query_data_queue = aio.Queue()
return backend
async def create(conf, backend_engine):
"""Create module engine
Args:
conf (hat.json.Data): configuration defined by
``hat://event/main.yaml#/definitions/module_engine``
backend_engine (hat.event.backend_engine.BackendEngine): backend engine
Returns:
ModuleEngine
"""
engine = ModuleEngine()
engine._backend = backend_engine
engine._async_group = aio.Group()
engine._register_queue = aio.Queue()
engine._register_cbs = util.CallbackRegistry()
last_event_id = await engine._backend.get_last_event_id()
engine._server_id = last_event_id.server
engine._last_instance_id = last_event_id.instance
engine._modules = []
for module_conf in conf['modules']:
py_module = importlib.import_module(module_conf['module'])
module = await py_module.create(module_conf, engine)
engine._async_group.spawn(aio.call_on_cancel, module.async_close)
engine._modules.append(module)
engine._async_group.spawn(engine._register_loop)
return engine
async def test_query(comm_conf):
def query_cb(data):
query_queue.put_nowait(data)
return mock_result
mock_query = hat.event.common.QueryData()
mock_result = [
hat.event.common.Event(event_id=hat.event.common.EventId(server=0,
instance=0),
event_type=['mock'],
timestamp=hat.event.common.Timestamp(s=0, us=0),
source_timestamp=None,
payload=None)
]
query_queue = aio.Queue()
engine = common.create_module_engine(query_cb=query_cb)
comm = await hat.event.server.communication.create(comm_conf, engine)
client = await hat.event.client.connect(comm_conf['address'])
events = await client.query(mock_query)
query_data = await query_queue.get()
assert events == mock_result
assert query_data == mock_query
await client.async_close()
await comm.async_close()
await engine.async_close()
async def test_register_with_response(comm_conf, register_events):
def register_cb(events):
register_queue.put_nowait(events)
return [common.process_event_to_event(i) for i in events]
register_queue = aio.Queue()
engine = common.create_module_engine(register_cb=register_cb)
comm = await hat.event.server.communication.create(comm_conf, engine)
client = await hat.event.client.connect(comm_conf['address'],
subscriptions=['*'])
client_events = await client.register_with_response(register_events)
process_events = await register_queue.get()
assert all(
type(event) == hat.event.server.common.ProcessEvent
for event in process_events)
assert all(
common.compare_register_event_vs_event(r, p)
for r, p in zip(register_events, process_events))
events = [common.process_event_to_event(i) for i in process_events]
assert events == client_events
subscription_events = await client.receive()
assert events == subscription_events
await client.async_close()
await comm.async_close()
await engine.async_close()
async def test_backend_to_frontend(server_port, ui_port, tmpdir):
c1 = ComponentInfo(cid=1,
mid=10,
name='c1',
group='g1',
address=None,
rank=1,
blessing=None,
ready=None)
c2 = ComponentInfo(cid=2,
mid=11,
name='c2',
group='g2',
address='1.2.3.4',
rank=2,
blessing=3,
ready=3)
ui_address = f'ws://127.0.0.1:{ui_port}/ws'
ui_conf = {'address': ui_address}
monitor_server = MockServer(mid=0)
ui_backend = await hat.monitor.server.ui.create(ui_conf, tmpdir,
monitor_server)
client = await create_client(ui_address)
client_change_queue = aio.Queue()
client.register_change_cb(lambda: client_change_queue.put_nowait(None))
await client_change_queue.get()
assert client.mid == 0 and client.components == []
monitor_server._set_mid(1)
await client_change_queue.get()
assert client.mid == 1 and client.components == []
monitor_server._set_components([c1, c2])
await client_change_queue.get()
assert client.mid == 1 and client.components == [c1, c2]
monitor_server._set_mid(2)
await client_change_queue.get()
assert client.mid == 2 and client.components == [c1, c2]
monitor_server._set_components([c2])
await client_change_queue.get()
assert client.mid == 2 and client.components == [c2]
monitor_server._set_components([c1])
await client_change_queue.get()
assert client.mid == 2 and client.components == [c1]
monitor_server._set_components([])
await client_change_queue.get()
assert client.mid == 2 and client.components == []
await client.async_close()
assert not ui_backend.closed.done()
await monitor_server.async_close()
await ui_backend.async_close()
assert ui_backend.closed.done()
async def test_queue_put_cancel():
queue = aio.Queue(1)
await queue.put(1)
put_future = asyncio.ensure_future(queue.put(1))
asyncio.get_event_loop().call_soon(put_future.cancel)
with pytest.raises(asyncio.CancelledError):
await put_future
async def create(conf, engine):
module = Module2()
module._async_group = aio.Group()
module._engine = engine
module._source = hat.event.server.common.Source(
type=hat.event.server.common.SourceType.MODULE, id=2)
module.session_queue = aio.Queue()
return module
def get_create_backend_queue(create_fn):
async def create_wrapper(create_fn, *args, **kwargs):
backend = await create_fn(*args, **kwargs)
queue.put_nowait(backend)
return backend
queue = aio.Queue()
create_backend = functools.partial(create_wrapper, create_fn)
return create_backend, queue
async def _wait_until_blessed_and_ready(client):
queue = aio.Queue()
with client.register_change_cb(lambda: queue.put_nowait(None)):
while (client.info is None or client.info.blessing is None
or client.info.blessing != client.info.ready):
await queue.get_until_empty()
if client.info is None:
continue
client.set_ready(client.info.blessing)
async def create_ui_client(address):
client = MockUIClient()
client._connection = await hat.juggler.connect(address)
client._connection.register_change_cb(client._remote_data_change_cb)
client._state_queue = aio.Queue()
return client
def __init__(self, conf):
self._conf = conf
self._status = Status.DELAYED if conf['delay'] else Status.STOPPED
self._revive = conf['revive']
self._change_cbs = util.CallbackRegistry(
exception_cb=lambda e: mlog.warning(
"change callback exception: %s", e, exc_info=e))
self._started_queue = aio.Queue()
self._async_group = aio.Group()
self._async_group.spawn(self._run_loop)
async def until_all_messages(backend_queue, group, message_count):
backend = await backend_queue.get()
entry_queue = aio.Queue()
backend.register_change_cb(lambda i: entry_queue.put_nowait(i))
while True:
entries = await entry_queue.get()
for e in entries:
if e.id == message_count:
await backend.async_close()
group.close()
async def test_queue_with_size():
queue_size = 5
queue = aio.Queue(queue_size)
assert queue.maxsize == queue_size
for _ in range(queue_size):
queue.put_nowait(None)
with pytest.raises(aio.QueueFullError):
queue.put_nowait(None)
async def test_client_server_master(server_conf, client_conf):
c2 = ComponentInfo(cid=2,
mid=1,
name='c2',
group='g2',
address='1.2.3.4',
rank=2,
blessing=3,
ready=3)
client_change_queue = aio.Queue()
server = await hat.monitor.server.server.create(server_conf)
client = await hat.monitor.client.connect(client_conf)
client.register_change_cb(lambda: client_change_queue.put_nowait(None))
master = common.create_master(mid=1, components=[])
client_info = None
while client_info is None:
await client_change_queue.get()
client_info = client.info
server_components = server.components
server.set_master(master)
await client_change_queue.get()
assert client.components == master.components and client.info is None
local_components = await master._components_queue.get()
assert (len(local_components) == 1 and local_components
== [i._replace(mid=master.mid) for i in server_components])
c1 = local_components[0]._replace(mid=1, rank=7)
master._set_components([c1, c2])
assert server.components == master.components
await client_change_queue.get()
assert client.components == master.components and client.info == c1
client.set_ready(12)
local_components = await master._components_queue.get()
assert len(local_components) == 1 and local_components[0].ready == 12
c1 = local_components[0]._replace(mid=1, ready=12, blessing=34)
master._set_components([c1, c2])
assert server.components == master.components
await client_change_queue.get()
assert client.info == c1 and client.components == master.components
master._set_mid(2)
await client_change_queue.get()
assert client.info.mid == 2
server.set_master(None)
await client_change_queue.get()
assert client.info.mid == 0 and client.components == [client.info]
await client.async_close()
assert client.closed.done()
await server.async_close()
assert server.closed.done()
def process_queue(event_loop, monkeypatch):
queue = aio.Queue()
create_subprocess_exec = asyncio.create_subprocess_exec
async def mock(*args, **kwargs):
p = await create_subprocess_exec(*args, **kwargs)
queue.put_nowait(p)
return p
monkeypatch.setattr(asyncio, 'create_subprocess_exec', mock)
return queue
async def _address_loop(monitor_client, address_change, server_group):
nonlocal address
queue = aio.Queue()
with monitor_client.register_change_cb(lambda: queue.put_nowait(None)):
while True:
new_address = _get_server_address(monitor_client.components,
server_group)
if new_address != address:
address = new_address
address_change.set()
await queue.get()
async def create_session(self, client):
session = MockAdapterSession()
session._adapter_client = client
session._event_queue = aio.Queue()
session._group = self._group.create_subgroup()
client.register_change_cb(session._on_change)
session._event_client = self._event_client
session._group.spawn(session._client_loop)
session._group.spawn(session._event_loop)
self._sessions.append(session)
return session
def device_queue(monkeypatch):
async def create_wrapper(*args, **kwargs):
device = await create(*args, **kwargs)
queue.put_nowait(device)
return device
queue = aio.Queue()
create = mock_device.create
monkeypatch.setattr(mock_device, 'create', create_wrapper)
yield queue
def _create_connection(sbs_repo, transport, ping_timeout, queue_maxsize):
conn = Connection()
conn._sbs_repo = sbs_repo
conn._transport = transport
conn._last_id = 0
conn._conv_timeouts = {}
conn._msg_queue = aio.Queue(maxsize=queue_maxsize)
conn._async_group = aio.Group(lambda e: mlog.error(
'connection async group exception: %s', e, exc_info=e))
conn._async_group.spawn(conn._read_loop)
conn._async_group.spawn(conn._ping_loop, ping_timeout)
return conn
async def test_queue():
queue = aio.Queue()
assert not queue.closed.done()
f = asyncio.ensure_future(queue.get())
assert not f.done()
queue.put_nowait(1)
assert 1 == await f
for _ in range(5):
queue.close()
assert queue.closed.done()
with pytest.raises(aio.QueueClosedError):
await queue.get()
async def open(conf):
"""Open serial port
Args:
conf (hat.json.Data): configuration defined by
'hat://drivers/serial.yaml#'
Returns:
Connection
"""
executor = aio.create_executor()
s = await executor(
functools.partial(serial.Serial,
port=conf['port'],
baudrate=conf.get('baudrate', 9600),
bytesize=getattr(serial,
conf.get('bytesize', 'EIGHTBITS')),
parity=getattr(serial,
conf.get('parity', 'PARITY_NONE')),
stopbits=getattr(
serial, conf.get('stopbits', 'STOPBITS_ONE')),
rtscts=conf.get('rtscts', False),
dsrdtr=conf.get('dsrdtr', False),
timeout=1))
read_queue = aio.Queue()
write_queue = aio.Queue()
async_group = aio.Group()
async_group.spawn(_action_loop, async_group, executor, read_queue, 0)
async_group.spawn(_action_loop, async_group, executor, write_queue,
conf.get('silent_interval', 0))
async_group.spawn(aio.call_on_cancel, executor, _ext_close, s)
conn = Connection()
conn._s = s
conn._read_queue = read_queue
conn._write_queue = write_queue
conn._async_group = async_group
return conn
async def test_modules(module_engine_conf, register_events, source_comm):
def register_cb(events):
return [common.process_event_to_event(i) for i in events]
backend_engine = common.create_backend_engine(register_cb=register_cb)
module_engine_conf['modules'] = [{
'module':
'test_unit.test_event.modules.module2'
}, {
'module':
'test_unit.test_event.modules.module1'
}, {
'module':
'test_unit.test_event.modules.transparent'
}]
with contextlib.ExitStack() as stack:
modules = [
stack.enter_context(common.get_return_values(m)) for m in [
test_unit.test_event.modules.module1.create,
test_unit.test_event.modules.module2.create,
test_unit.test_event.modules.transparent.create
]
]
module_engine = await hat.event.server.module_engine.create(
module_engine_conf, backend_engine)
modules = [m for ms in modules for m in ms]
assert len(modules) == 3
assert all(not module.closed.done() for module in modules)
event_queue = aio.Queue()
module_engine.register_events_cb(
lambda events: event_queue.put_nowait(events))
process_events = [
module_engine.create_process_event(source_comm, i)
for i in register_events
]
events_on_register = await module_engine.register(process_events)
sessions = [await module.session_queue.get() for module in modules]
await asyncio.gather(*(session.closed for session in sessions))
assert_notified_changes(sessions, process_events)
events = await event_queue.get()
assert events == events_on_register
assert_events_vs_changes_result(sessions, process_events, events)
assert all(session.events_on_close == events for session in sessions)
await assert_closure(backend_engine, module_engine, modules)
async def test_first():
queue = aio.Queue()
queue.put_nowait(1)
queue.put_nowait(2)
queue.put_nowait(3)
queue.close()
result = await aio.first(queue, lambda i: i == 2)
assert result == 2
assert not queue.empty()
assert queue.get_nowait() == 3
assert queue.empty()
queue = aio.Queue()
queue.put_nowait(1)
queue.put_nowait(2)
queue.put_nowait(3)
queue.close()
result = await aio.first(queue, lambda i: i == 4)
assert result is None
assert queue.empty()