async def test_timeout(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(3) | pipe.timeout(5)
await assert_run(xs, [0, 1, 2])
assert event_loop.steps == []
with event_loop.assert_cleanup():
xs = stream.range(3) + stream.never()
ys = xs | pipe.timeout(1)
await assert_run(ys, [0, 1, 2], asyncio.TimeoutError())
assert event_loop.steps == [1]
async def test_timeout(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(3) | pipe.timeout(5)
await assert_run(xs, [0, 1, 2])
assert event_loop.steps == [5, 5, 5, 5]
with event_loop.assert_cleanup():
xs = stream.range(3) + stream.never()
ys = xs | pipe.timeout(1)
await assert_run(ys, [0, 1, 2], asyncio.TimeoutError())
assert event_loop.steps == [1, 1, 1, 1]
async def test_cycle(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.empty() | pipe.cycle() | pipe.timeout(1)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = stream.empty() | add_resource.pipe(
1) | pipe.cycle() | pipe.timeout(1)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = stream.just(1) | add_resource.pipe(1) | pipe.cycle()
await assert_run(xs[:5], [1] * 5)
assert event_loop.steps == [1] * 5
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 1, 1]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5),
task_limit=1)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5),
task_limit=2)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x+2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 5, 5]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x+2, interval=5),
task_limit=1)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x+2, interval=5),
task_limit=2)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 4, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x+4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
async def test_cycle(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.empty() | pipe.cycle() | pipe.timeout(1)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = (
stream.empty()
| add_resource.pipe(1)
| pipe.cycle()
| pipe.timeout(1)
)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = stream.just(1) | add_resource.pipe(1) | pipe.cycle()
await assert_run(xs[:5], [1]*5)
assert event_loop.steps == [1]*5
def _read_device(config: dict[str, Any]) -> AsyncGenerator[Any, None]:
on_read: partial
timeout: float
on_read, timeout = config["on_read"]
if inspect.isasyncgenfunction(on_read.func):
return stream.iterate(on_read()) | pipe.timeout(timeout)
return (stream.repeat(config["on_read"], interval=config["interval"])
| pipe.starmap(lambda func, timeout: stream.just(func()) | pipe
.timeout(timeout))
| pipe.concat(task_limit=1))
def _read_sensor( # pylint: disable=too-many-arguments
self, sid: int, interval: float, unit: str, topic: str, timeout: float
) -> AsyncGenerator[DataEvent, None]:
if self.__uuid is None:
raise SensorNotReady("You must enumerate the sensor before reading.")
return (
stream.repeat(stream.call(self._device.get_by_function_id, sid), interval=interval)
| pipe.concat(task_limit=1)
| pipe.timeout(timeout)
| pipe.map(lambda value: DataEvent(sender=self.__uuid, topic=topic, value=value, sid=sid, unit=str(unit)))
)
async def test_merge(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys, [0, 1, 2, 3, 4])
assert event_loop.steps == [1, 1, 1, 1]
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys[:3], [0, 1, 2])
assert event_loop.steps == [1, 1]
with event_loop.assert_cleanup():
xs = stream.just(1) + stream.never()
ys = xs | pipe.merge(xs) | pipe.timeout(1)
await assert_run(ys, [1, 1], asyncio.TimeoutError())
assert event_loop.steps == [1]
# Reproduce issue #65
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2])
ys = stream.iterate([3, 4])
zs = stream.merge(xs, ys) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = stream.throw(ZeroDivisionError)
zs = stream.merge(xs, ys) | pipe.delay(1) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
# Silencing of a CancelledError
async def agen1():
if False:
yield
try:
await asyncio.sleep(2)
except asyncio.CancelledError:
return
async def agen2():
yield 1
with event_loop.assert_cleanup():
xs = stream.merge(agen1(), agen2()) | pipe.delay(1) | pipe.take(1)
await assert_run(xs, [1])
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 5, 5]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x + 2, interval=5), task_limit=1
)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x + 2, interval=5), task_limit=2
)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 4, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
# An exception might get discarded if the result can be produced before the
# processing of the exception is required
with event_loop.assert_cleanup():
xs = stream.iterate([True, False])
ys = xs | pipe.concatmap(
lambda x: stream.range(0, 3, interval=1)
if x
else stream.throw(ZeroDivisionError)
)
zs = ys | pipe.take(3)
await assert_run(zs, [0, 1, 2])
assert event_loop.steps == [1, 1]
async def test_merge(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys, [0, 1, 2, 3, 4])
assert event_loop.steps == [1, 1, 1, 1]
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys[:3], [0, 1, 2])
assert event_loop.steps == [1, 1]
with event_loop.assert_cleanup():
xs = stream.just(1) + stream.never()
ys = xs | pipe.merge(xs) | pipe.timeout(1)
await assert_run(ys, [1, 1], asyncio.TimeoutError())
assert event_loop.steps == [1]
def _configure_and_stream(self, config: dict[str, Any] | None) -> AsyncGenerator[DataEvent, None]:
if config is None:
return stream.empty()
try:
# Run all config steps in order (concat) and one at a time (task_limit=1). Drop the output. There is
# nothing to compare them to (filter => false), then read all sensors of the bicklet and process them in
# parallel (flatten).
config_stream = stream.chain(
stream.iterate(config["on_connect"])
| pipe.starmap(lambda func, timeout: stream.just(func()) | pipe.timeout(timeout))
| pipe.concat(task_limit=1)
| pipe.filter(lambda result: False),
stream.iterate(config["config"].items())
| pipe.starmap(self._parse_configuration)
| pipe.starmap(self._read_sensor)
| pipe.flatten(),
)
return config_stream
except Exception:
self._logger.exception("This should not happen.")
raise
async def test_map(assert_run, event_loop):
# Synchronous/simple
with event_loop.assert_cleanup():
xs = stream.range(5) | pipe.map(lambda x: x**2)
expected = [x**2 for x in range(5)]
await assert_run(xs, expected)
# Synchronous/multiple
with event_loop.assert_cleanup():
xs = stream.range(5)
ys = xs | pipe.map(lambda x, y: x + y, xs)
expected = [x * 2 for x in range(5)]
await assert_run(ys, expected)
# Asynchronous/simple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 1, 1]
# Asynchronous/simple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=1)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 2, 3]
# Asynchronous/multiple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Asynchronous/multiple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 2, 3]
# As completed
with event_loop.assert_cleanup():
xs = stream.iterate([2, 4, 1, 3, 5])
ys = xs | pipe.map(asyncio.sleep, xs, ordered=False)
await assert_run(ys, [1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1, 1, 1]
# Invalid argument
with pytest.raises(ValueError):
await (stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=0))
# Break
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=10)
await assert_run(ys[:3], [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Stuck
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1) | pipe.timeout(5)
await assert_run(ys, [1, 2, 3, 4], asyncio.TimeoutError())
# Force await
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = xs | pipe.map(async_(lambda x: asyncio.sleep(x, x)))
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Map await_
with event_loop.assert_cleanup():
xs = stream.iterate(map(lambda x: asyncio.sleep(x, x), [1, 2, 3]))
ys = xs | pipe.map(await_)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
async def test_never(assert_run, event_loop):
xs = stream.never() | pipe.timeout(30.)
await assert_run(xs, [], asyncio.TimeoutError())
assert event_loop.steps == [30.]
async def test_map(assert_run, event_loop):
# Synchronous/simple
with event_loop.assert_cleanup():
xs = stream.range(5) | pipe.map(lambda x: x**2)
expected = [x**2 for x in range(5)]
await assert_run(xs, expected)
# Synchronous/multiple
with event_loop.assert_cleanup():
xs = stream.range(5)
ys = xs | pipe.map(lambda x, y: x+y, xs)
expected = [x*2 for x in range(5)]
await assert_run(ys, expected)
# Asynchronous/simple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 1, 1]
# Asynchronous/simple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=1)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 2, 3]
# Asynchronous/multiple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Asynchronous/multiple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 2, 3]
# As completed
with event_loop.assert_cleanup():
xs = stream.iterate([2, 4, 1, 3, 5])
ys = xs | pipe.map(asyncio.sleep, xs, ordered=False)
await assert_run(ys, [1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1, 1, 1]
# Invalid argument
with pytest.raises(ValueError):
await (stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=0))
# Break
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=10)
await assert_run(ys[:3], [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Stuck
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1) | pipe.timeout(5)
await assert_run(ys, [1, 2, 3, 4], asyncio.TimeoutError())
# Force await
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = xs | pipe.map(async_(lambda x: asyncio.sleep(x, x)))
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Map await_
with event_loop.assert_cleanup():
xs = stream.iterate(map(lambda x: asyncio.sleep(x, x), [1, 2, 3]))
ys = xs | pipe.map(await_)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
async def test_never(assert_run, event_loop):
xs = stream.never() | pipe.timeout(30.)
await assert_run(xs, [], asyncio.TimeoutError())
assert event_loop.steps == [30.]
def wrapped(self, *args, **kwargs):
return stream.iterate(func(
self, *args, **kwargs)) | pipe.timeout(KRAKEN_NONNULL_RECV_TIMEOUT)
