async def test_aio_read():
a0.File.remove("foo")
file = a0.File("foo")
w = a0.Writer(file)
def thread_main():
for i in range(5):
time.sleep(0.1)
w.write("keep going")
w.write("done")
t = threading.Thread(target=thread_main)
t.start()
assert (await
a0.aio_read_one(file,
a0.INIT_MOST_RECENT)).payload == b"keep going"
cnt = 0
async for pkt in a0.aio_read(file, a0.INIT_OLDEST):
cnt += 1
assert pkt.payload in [b"keep going", b"done"]
if pkt.payload == b"done":
break
assert cnt == 6
assert (await a0.aio_read_one(file,
a0.INIT_MOST_RECENT)).payload == b"done"
t.join()
async def test_ls(sandbox):
await sandbox.WaitUntilStartedAsync(timeout=1.0)
async with aiohttp.ClientSession() as session:
async with session.get("http://localhost:24880/api/ls") as resp:
assert resp.status == 200
assert await resp.json() == [
{
"filename": "a0_heartbeat__api",
"protocol": "heartbeat",
"container": "api",
},
]
a0.File("a0_pubsub__aaa__bbb")
a0.File("a0_pubsub__aaa__ccc")
a0.File("a0_rpc__bbb__ddd")
async with session.get("http://localhost:24880/api/ls") as resp:
assert resp.status == 200
assert await resp.json() == [
{
"filename": "a0_heartbeat__api",
"protocol": "heartbeat",
"container": "api",
},
{
"filename": "a0_pubsub__aaa__bbb",
"protocol": "pubsub",
"container": "aaa",
"topic": "bbb",
},
{
"filename": "a0_pubsub__aaa__ccc",
"protocol": "pubsub",
"container": "aaa",
"topic": "ccc",
},
{
"filename": "a0_rpc__bbb__ddd",
"protocol": "rpc",
"container": "bbb",
"topic": "ddd",
},
]
def test_ls(sandbox):
resp = requests.get(f"http://localhost:{os.environ['PORT_STR']}/api/ls")
assert resp.status_code == 200
assert resp.headers["Access-Control-Allow-Origin"] == "*"
assert resp.json() == ["api_ready.pubsub.a0"]
a0.File("aaa/bbb.pubsub.a0")
a0.File("aaa/ccc.pubsub.a0")
a0.File("bbb/ddd.rpc.a0")
resp = requests.get(f"http://localhost:{os.environ['PORT_STR']}/api/ls")
assert resp.status_code == 200
assert resp.json() == [
"aaa/bbb.pubsub.a0",
"aaa/ccc.pubsub.a0",
"api_ready.pubsub.a0",
"bbb/ddd.rpc.a0",
]
def test_file():
a0.File.remove("foo")
file = a0.File("foo")
assert len(file.arena.buf) == 16 * 1024 * 1024
assert file.size == 16 * 1024 * 1024
assert file.path == "/dev/shm/alephzero/foo"
assert file.fd > 3
assert file.stat.st_size == 16 * 1024 * 1024
path = file.path
del file
assert os.path.exists(path)
a0.File.remove("foo")
assert not os.path.exists(path)
fileopts = a0.File.Options.DEFAULT
fileopts.create_options.size = 1024
file = a0.File("foo", fileopts)
assert len(file.arena.buf) == 1024
def test_transport():
a0.File.remove("foo")
file = a0.File("foo")
w = a0.Writer(file)
w.write("aaa")
w.write("bbb")
w.write("ccc")
transport = a0.Transport(file)
tlk = transport.lock()
assert not tlk.empty()
tlk.jump_head()
assert tlk.iter_valid()
assert a0.FlatPacket(tlk.frame()).payload == b"aaa"
tlk.jump_tail()
assert tlk.iter_valid()
assert a0.FlatPacket(tlk.frame()).payload == b"ccc"
tlk.jump_head()
assert tlk.has_next()
tlk.step_next()
assert tlk.iter_valid()
assert a0.FlatPacket(tlk.frame()).payload == b"bbb"
tlk.jump_tail()
assert tlk.has_prev()
tlk.step_prev()
assert tlk.iter_valid()
assert a0.FlatPacket(tlk.frame()).payload == b"bbb"
@contextlib.contextmanager
def thread_sleep_write(pkt, timeout):
def sleep_write(pkt, timeout):
time.sleep(timeout)
w.write(pkt)
t = threading.Thread(target=sleep_write, args=(pkt, timeout))
t.start()
yield
t.join()
tlk.jump_tail()
with thread_sleep_write("ddd", timeout=0.1):
tlk.wait(tlk.has_next)
tlk.step_next()
assert a0.FlatPacket(tlk.frame()).payload == b"ddd"
with thread_sleep_write("eee", timeout=0.1):
tlk.wait(tlk.has_next, timeout=0.2)
tlk.step_next()
assert a0.FlatPacket(tlk.frame()).payload == b"eee"
with thread_sleep_write("fff", timeout=0.2):
with pytest.raises(RuntimeError, match="Connection timed out"):
tlk.wait(tlk.has_next, timeout=0.1)
# Note we need to release the lock here.
# The wait returned, having reacquired the lock.
# The thread needs to acquire the lock to do the write.
tlk = None
def clear(topic):
"""Clear the log history for the given topic."""
t = a0.Transport(a0.File(a0.env.topic_tmpl_log().format(topic=topic)))
tlk = t.lock()
tlk.clear()
def test_reader_writer():
a0.File.remove("foo")
file = a0.File("foo")
w = a0.Writer(file)
rs = a0.ReaderSync(file, a0.INIT_OLDEST)
cv = threading.Condition()
class State:
payloads = []
def callback(pkt):
with cv:
State.payloads.append(pkt.payload)
cv.notify()
r = a0.Reader(file, a0.INIT_OLDEST, callback)
assert not rs.can_read()
w.write("hello")
assert rs.can_read()
pkt = rs.read()
assert pkt.payload == b"hello"
assert pkt.headers == []
assert not rs.can_read()
w.push(a0.add_transport_seq_header())
w2 = w.wrap(a0.add_writer_seq_header())
w.write("aaa")
w2.write("bbb")
pkt = rs.read()
assert pkt.payload == b"aaa"
assert pkt.headers == [("a0_transport_seq", "1")]
pkt = rs.read()
assert pkt.payload == b"bbb"
assert sorted(pkt.headers) == [("a0_transport_seq", "2"),
("a0_writer_seq", "0")]
with cv:
cv.wait_for(lambda: len(State.payloads) == 3)
assert State.payloads == [b"hello", b"aaa", b"bbb"]
def sleep_write(timeout, pkt):
time.sleep(timeout)
w.write(pkt)
t = threading.Thread(target=sleep_write, args=(0.1, b"post_sleep"))
t.start()
assert rs.read_blocking().payload == b"post_sleep"
t.join()
t = threading.Thread(target=sleep_write, args=(0.1, b"post_sleep"))
t.start()
assert rs.read_blocking(timeout=0.2).payload == b"post_sleep"
t.join()
t = threading.Thread(target=sleep_write, args=(0.2, b"post_sleep"))
t.start()
with pytest.raises(RuntimeError, match="Connection timed out"):
rs.read_blocking(timeout=a0.TimeMono.now() + 0.1)
t.join()
def test_read_zero_copy():
a0.File.remove("foo")
file = a0.File("foo")
w = a0.Writer(file)
w.write("aaa")
w.write("bbb")
w.write("ccc")
rszc = a0.ReaderSyncZeroCopy(file, a0.INIT_OLDEST)
class Want:
checked = False
payload = b""
off = 0
seq = 0
def callback(tlk, fpkt):
assert fpkt.payload == Want.payload
assert a0.FlatPacket(tlk.frame()).payload == Want.payload
assert tlk.frame().off == Want.off
assert tlk.frame().seq == Want.seq
Want.checked = True
@contextlib.contextmanager
def want_context(payload, off, seq):
Want.checked = False
Want.payload = payload
Want.off = off
Want.seq = seq
yield
assert Want.checked
with want_context(b"aaa", 144, 1):
rszc.read(callback)
with want_context(b"bbb", 240, 2):
rszc.read(callback)
with want_context(b"ccc", 336, 3):
rszc.read(callback)
with want_context(b"ccc", 336, 3):
a0.read_random_access(file, 336, callback)
@contextlib.contextmanager
def thread_sleep_write(pkt, timeout):
def sleep_write(pkt, timeout):
time.sleep(timeout)
w.write(pkt)
t = threading.Thread(target=sleep_write, args=(pkt, timeout))
t.start()
yield
t.join()
with want_context(b"ddd", 432, 4):
with thread_sleep_write("ddd", timeout=0.1):
rszc.read_blocking(callback)
with want_context(b"eee", 528, 5):
with thread_sleep_write("eee", timeout=0.1):
rszc.read_blocking(callback, timeout=0.2)
with want_context(b"fff", 624, 6):
with thread_sleep_write("fff", timeout=0.1):
rszc.read_blocking(callback, timeout=a0.TimeMono.now() + 0.2)
with thread_sleep_write("ggg", timeout=0.2):
with pytest.raises(RuntimeError, match="Connection timed out"):
rszc.read_blocking(callback, timeout=0.1)
def test_discovery():
try:
a0.File.remove_all("discovery_test")
except Exception as _:
pass
a0.File("discovery_test/unused")
a0.File.remove("discovery_test/unused")
cv = threading.Condition()
class State:
paths = []
def callback(path):
with cv:
State.paths.append(path)
cv.notify()
d = a0.Discovery("/dev/shm/alephzero/discovery_test/**/*.a0", callback)
a0.File("discovery_test/file.a0")
a0.File("discovery_test/a/file.a0")
a0.File("discovery_test/a/b/file.a0")
a0.File("discovery_test/a/b/c/d/file.a0")
a0.File("discovery_test/a/b/c/d/file2.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file2.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file3.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file4.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file5.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file6.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file7.a0")
a0.File("discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file8.a0")
with cv:
cv.wait_for(lambda: len(State.paths) >= 13)
d = None
State.paths.sort()
assert State.paths == [
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file2.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file3.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file4.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file5.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file6.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file7.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/e/f/g/h/i/j/k/l/m/file8.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/file.a0",
"/dev/shm/alephzero/discovery_test/a/b/c/d/file2.a0",
"/dev/shm/alephzero/discovery_test/a/b/file.a0",
"/dev/shm/alephzero/discovery_test/a/file.a0",
"/dev/shm/alephzero/discovery_test/file.a0",
]
def clear(topic):
"""Clear the pubsub topic."""
t = a0.Transport(a0.File(a0.env.topic_tmpl_pubsub().format(topic=topic)))
tlk = t.lock()
tlk.clear()
async def test_read(sandbox):
endpoint = f"ws://localhost:{os.environ['PORT_STR']}/wsapi/read"
sub_data = {
"path": "myread",
"init": "OLDEST",
"iter": "NEXT",
}
w = a0.Writer(a0.File("myread"))
w.write("payload 0")
w.write("payload 1")
async with websockets.connect(endpoint) as ws:
await ws.send(json.dumps(sub_data))
try:
pkt = json.loads(await asyncio.wait_for(ws.recv(), timeout=1.0))
assert pkt["payload"] == "payload 0"
except asyncio.TimeoutError:
assert False
try:
pkt = json.loads(await asyncio.wait_for(ws.recv(), timeout=1.0))
assert pkt["payload"] == "payload 1"
except asyncio.TimeoutError:
assert False
timed_out = False
try:
await asyncio.wait_for(ws.recv(), timeout=3.0)
except asyncio.TimeoutError:
timed_out = True
assert timed_out
w.write("payload 2")
try:
pkt = json.loads(await asyncio.wait_for(ws.recv(), timeout=1.0))
assert pkt["payload"] == "payload 2"
except asyncio.TimeoutError:
assert False
sub_data["response_encoding"] = "base64"
sub_data["scheduler"] = "ON_ACK"
async with websockets.connect(endpoint) as ws:
await ws.send(json.dumps(sub_data))
try:
pkt = json.loads(await asyncio.wait_for(ws.recv(), timeout=1.0))
assert atob(pkt["payload"]) == "payload 0"
except asyncio.TimeoutError:
assert False
timed_out = False
try:
await asyncio.wait_for(ws.recv(), timeout=1.0)
except asyncio.TimeoutError:
timed_out = True
assert timed_out
await ws.send("ACK")
try:
pkt = json.loads(await asyncio.wait_for(ws.recv(), timeout=1.0))
assert atob(pkt["payload"]) == "payload 1"
except asyncio.TimeoutError:
assert False
timed_out = False
try:
await asyncio.wait_for(ws.recv(), timeout=1.0)
except asyncio.TimeoutError:
timed_out = True
assert timed_out
await ws.send("ACK")
try:
pkt = json.loads(await asyncio.wait_for(ws.recv(), timeout=1.0))
assert atob(pkt["payload"]) == "payload 2"
except asyncio.TimeoutError:
assert False
def test_write(sandbox):
endpoint = f"http://localhost:{os.environ['PORT_STR']}/api/write"
pub_data = {
"path": "mypath",
"packet": {
"headers": [
["xyz", "123"],
["zzz", "www"],
],
"payload": "Hello, World!",
},
}
# Normal publish.
resp = requests.post(endpoint, data=json.dumps(pub_data))
assert resp.status_code == 200
assert resp.text == "success"
# Base64 request_encoding.
pub_data["request_encoding"] = "base64"
pub_data["packet"]["payload"] = btoa("Goodbye, World!")
resp = requests.post(endpoint, data=json.dumps(pub_data))
assert resp.status_code == 200
assert resp.text == "success"
pub_data["packet"]["payload"] = "Hello, World!"
pub_data.pop("request_encoding")
# Missing "path".
pub_data.pop("path")
resp = requests.post(endpoint, data=json.dumps(pub_data))
assert resp.status_code == 400
assert resp.text == "Request missing required field: path"
pub_data["path"] = "mypath"
# Not JSON.
resp = requests.post(endpoint, data="not json")
assert resp.status_code == 400
assert resp.text == "Request must be json."
# Not JSON object.
resp = requests.post(endpoint, data=json.dumps(["not object"]))
assert resp.status_code == 400
assert resp.text == "Request must be a json object."
# Standard headers.
pub_data["standard_headers"] = True
resp = requests.post(endpoint, data=json.dumps(pub_data))
assert resp.status_code == 200
assert resp.text == "success"
pub_data.pop("standard_headers")
reader = a0.ReaderSync(a0.File("mypath"), a0.INIT_OLDEST)
hdrs = []
msgs = []
while reader.can_read():
pkt = reader.read()
hdrs.append(list(pkt.headers)) # Inspect copies of headers.
msgs.append(pkt.payload)
assert len(hdrs) == 3
assert len(msgs) == 3
assert msgs == [b"Hello, World!", b"Goodbye, World!", b"Hello, World!"]
for hdr in hdrs[:-1]:
assert sorted([k for k, v in hdr]) == ["xyz", "zzz"]
assert sorted([k for k, v in hdrs[-1]]) == [
"a0_time_mono",
"a0_time_wall",
"a0_transport_seq",
"a0_writer_id",
"a0_writer_seq",
"xyz",
"zzz",
]