async def test_async_comprehension() -> None:
xs = AsyncObservable.from_iterable([1, 2, 3])
async with subscribe(xs) as stream:
result = [x async for x in stream]
assert result == [1, 2, 3]
async def test_pipe_complex_pipe():
xs = AsyncObservable.from_iterable([1, 2, 3])
result = []
def mapper(value):
return value * 10
async def predicate(value):
await asyncio.sleep(0.1)
return value > 1
async def long_running(value):
return AsyncObservable.from_iterable([value])
ys = pipe(xs,
op.filter(predicate),
op.map(mapper),
op.flat_map(long_running)
)
async with subscribe(ys) as stream:
async for value in stream:
result.append(value)
assert result == [20, 30]
async def tweets_ws_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
async with subscribe(tweets.new_tweets_stream) as stream:
async for tweet in stream:
ws.send_json(tweet)
return ws
async def test_async_iteration_aync_with() -> None:
xs = AsyncObservable.from_iterable([1, 2, 3])
result = []
async with subscribe(xs) as stream:
async for x in stream:
result.append(x)
assert result == [1, 2, 3]
async def test_unit_happy_future_resolve():
fut = asyncio.Future()
xs = unit(fut)
obv = AsyncAnonymousObserver()
async with subscribe(xs, obv):
fut.set_result(42)
await obv
assert obv.values == [(0, 42), (0, )]
async def test_async_iteration_aync_with() -> None:
xs = AsyncObservable.from_iterable([1, 2, 3])
result = []
obv = AsyncIteratorObserver()
async with subscribe(xs, obv):
async for x in obv:
result.append(x)
assert result == [1, 2, 3]
async def main():
xs = AsyncObservable.from_iterable([1, 2, 3, 4, 5])
async def mapper(value):
fut = executor.submit(long_running, value)
return AsyncObservable.unit(wrap_future(fut))
ys = xs | op.flat_map(mapper)
async with subscribe(ys) as stream:
async for x in stream:
print(x)
async def test_unit_future_cancel():
fut = asyncio.Future()
xs = unit(fut)
obv = AsyncAnonymousObserver()
async with subscribe(xs, obv):
fut.cancel()
with pytest.raises(asyncio.CancelledError):
await obv
assert obv.values == [(0, )]
async def test_stream_cancel_context():
xs = AsyncSingleStream()
sink = AsyncAnonymousObserver()
async with subscribe(xs, sink):
pass
await xs.asend_later(1, 10)
await xs.asend_later(1, 20)
assert sink.values == []
async def test_async_iteration_inception() -> None:
# iterable to async source to async iterator to async source
ys = await subscribe(AsyncObservable.from_iterable([1, 2, 3]))
xs = AsyncObservable.from_iterable(ys)
result = []
async with subscribe(xs) as stream:
async for x in stream:
result.append(x)
assert result == [1, 2, 3]
async def test_unit_future_exception():
fut = asyncio.Future()
ex = Exception("ex")
xs = unit(fut)
obv = AsyncAnonymousObserver()
async with subscribe(xs, obv):
fut.set_exception(ex)
with pytest.raises(Exception):
await obv
assert obv.values == [(0, ex)]
async def test_stream_cancel_context():
xs = AsyncStream()
obv = AsyncAnonymousObserver()
async with subscribe(xs, obv):
pass
await xs.asend_later(1, 10)
await xs.asend_later(1, 20)
assert obv.values == []
async def test_delay_done():
xs = AsyncStream()
ys = delay(0.5, xs)
obv = AsyncAnonymousObserver()
async with subscribe(ys, obv):
await xs.asend_later(0, 10)
await xs.asend_later(1, 20)
await xs.aclose_later(1)
await obv
assert obv.values == [(0.5, 10), (1.5, 20), (2.5, )]
async def test_stream_cold_close():
xs = AsyncSingleStream()
sink = AsyncAnonymousObserver()
async def aclose():
await xs.aclose()
asyncio.ensure_future(aclose())
await asyncio.sleep(10)
async with subscribe(xs, sink):
await xs.asend_later(1, 20)
assert sink.values == [(10, )]
async def test_delay_cancel_before_done():
xs = AsyncStream()
result = []
async def asend(value):
nonlocal result
result.append(value)
ys = delay(0.3, xs)
async with subscribe(ys, AsyncAnonymousObserver(asend)):
await xs.asend(10)
await asyncio.sleep(1.5)
await xs.asend(20)
await asyncio.sleep(1)
assert result == [10]
async def test_stream_cancel_mapper():
xs = AsyncSingleStream()
sub = None
def mapper(value):
sub.cancel()
return value * 10
ys = map(mapper, xs)
sink = AnonymousAsyncObserver()
async with subscribe(ys, sink) as sub:
await xs.asend_later(1, 10)
await xs.asend_later(1, 20)
assert sink.values == []
async def test_stream_cancel_asend():
xs = AsyncStream()
stream = None
async def asend(value):
stream.cancel()
await asyncio.sleep(0)
def mapper(value):
return value * 10
ys = map(mapper, xs)
sink = AnonymousAsyncObserver(asend)
async with subscribe(ys, sink) as stream:
await xs.asend_later(1, 10)
await xs.asend_later(1, 20)
assert sink.values == [(1, 100)]
async def test_stream_cancel_asend():
xs = AsyncSingleStream()
sub = None
async def asend(value):
await sub.adispose()
await asyncio.sleep(0)
def mapper(value):
return value * 10
ys = map(mapper, xs)
sink = AsyncAnonymousObserver(asend)
async with subscribe(ys, sink) as sub:
await xs.asend_later(1, 10)
await xs.asend_later(1, 20)
assert sink.values == [(1, 100)]
async def test_stream_cancel_asend() -> None:
xs = AsyncStream()
subscription = None
async def asend(value) -> None:
await subscription.adispose()
await asyncio.sleep(0)
def mapper(value) -> int:
return value * 10
ys = map(mapper, xs)
obv = AsyncAnonymousObserver(asend)
async with subscribe(ys, obv) as sub:
subscription = sub
await xs.asend_later(1, 10)
await xs.asend_later(1, 20)
assert obv.values == [(1, 100)]
async def test_stream_cancel_mapper():
xs = AsyncStream()
subscription = None
def mapper(value):
asyncio.ensure_future(subscription.adispose())
return value * 10
ys = map(mapper, xs)
obv = AsyncAnonymousObserver()
async with subscribe(ys, obv) as subscription:
await xs.asend_later(100, 10)
await xs.asend_later(100, 20)
await xs.asend_later(100, 30)
await xs.asend_later(100, 40)
await xs.asend_later(100, 50)
await xs.asend_later(100, 60)
assert obv.values == [(100, 100)]
async def test_map_subscription_cancel():
xs = AsyncStream()
result = []
sub = None
def mapper(value):
return value * 10
ys = xs | _.map(mapper)
async def asend(value):
result.append(value)
await sub.adispose()
await asyncio.sleep(0)
async with subscribe(ys, AsyncAnonymousObserver(asend)) as sub:
await xs.asend(10)
await asyncio.sleep(0)
await xs.asend(20)
assert result == [100]
async def test_withlatestfrom_done():
xs = AsyncStream()
ys = AsyncStream()
result = []
async def asend(value):
log.debug("test_withlatestfrom_done:asend(%s)", value)
nonlocal result
asyncio.sleep(0.1)
result.append(value)
zs = with_latest_from(lambda x, y: x + y, ys, xs)
obv = AsyncAnonymousObserver(asend)
async with subscribe(zs, obv):
await xs.asend(1)
await ys.asend(2)
await xs.asend(3)
await xs.aclose()
await obv
assert result == [5]
async def test_map_subscription_cancel():
xs = AsyncStream()
result = []
sub = None
def mapper(value):
return value * 10
print("-----------------------")
print([xs, op.map])
ys = xs | op.map(mapper)
async def asend(value):
result.append(value)
sub.dispose()
await asyncio.sleep(0)
async with subscribe(ys, AnonymousAsyncObserver(asend)) as sub:
await xs.asend(10)
await asyncio.sleep(0)
await xs.asend(20)
assert result == [100]
async def new_task_watch():
db_host = os.environ.get('DATABASE_HOST')
db_port = os.environ.get('DATABASE_PORT')
conn = await connection(db_host, db_port)
async def new_change(arg):
try:
row, webhooks = arg[0], arg[1]
logger.debug(f"Dispatching row: {row}")
event = row.get('event', 'undefined')
hooks = webhooks.get(event, [])
if len(hooks) > 0:
webhook_future = asyncio.ensure_future(
service.webhook(hooks, row))
try:
handler = dispatch[event]
await handler(conn, row)
except KeyError:
logger.debug(f"No handler for event type {row['event']}")
except:
logger.exception("Unknown exception in task processing")
try:
if len(hooks) > 0:
await webhook_future
except:
logger.exception("Unknown exception in webhook.")
except:
logger.exception("Unknown exception in task watch.")
async def delayed(tasks):
logger.debug(f"Got update in delayed tasks: {tasks}")
if len(tasks) == 0:
await asyncio.sleep(60)
return
task = next(iter(tasks))
delta = datetime.fromtimestamp(
task['at'], timezone.utc) - datetime.now(timezone.utc)
await asyncio.sleep(delta.total_seconds())
await conn.run(new_task(conn, task['event'], task['parameters']))
await asyncio.shield(conn.run(remove_delayed(conn, task)))
ready_tasks = conn.start_with_and_changes(conn.db().table('tasks') \
.filter(lambda row: row['status'] == 'ready')
) | Operators.map(lambda c: c['new_val'])\
| Operators.filter(lambda x: x is not None)
webhooks = conn.changes_accumulate(conn.db().table('webhooks'))\
| Operators.map(group_by("event"))
return await asyncio.gather(
subscribe(
ready_tasks | with_latest_from(webhooks),
AsyncAnonymousObserver(new_change)
),
subscribe(
conn.changes_accumulate(conn.db().table('delayed_tasks'))\
| Operators.map(lambda tasks: sorted(tasks, key=lambda task: task['at'])),
AsyncRepeatedlyCallWithLatest(delayed)
)
)
