async def test_just(assert_run):
value = 3
xs = stream.just(value)
await assert_run(xs, [3])
async def four():
return 4
xs = stream.just(four())
await assert_run(xs, [4])
def stream_data(self,
config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]:
"""
Stream the data from the sensor.
Parameters
----------
config: dict
A dictionary containing the sensor configuration.
Returns
-------
AsyncGenerator
The asynchronous stream.
"""
data_stream = (stream.chain(
stream.just(config),
stream.iterate(
event_bus.subscribe(f"nodes/by_uuid/{self.__uuid}/update")))
|
pipe.action(lambda _: logging.getLogger(__name__).info(
"Got new configuration for: %s", self) if config is
not None else logging.getLogger(__name__).
info("Removed configuration for: %s", self))
| pipe.map(self._parse_config)
| pipe.switchmap(lambda conf: stream.empty(
) if conf is None or not conf["enabled"] else
(self._configure_and_stream(conf))))
return data_stream
async def main():
# Create a counting stream with a 0.2 seconds interval
xs = stream.count(interval=0.2)
# Operators can be piped using '|'
ys = xs | pipe.map(lambda x: x**2)
# Streams can be sliced
zs = ys[1:10:2]
# Use a stream context for proper resource management
async with zs.stream() as streamer:
# Asynchronous iteration
async for z in streamer:
# Print 1, 9, 25, 49 and 81
print("->", z)
# Streams can be awaited and return the last value
print("9² = ", await zs)
# Streams can run several times
print("9² = ", await zs)
# Streams can be concatenated
one_two_three = stream.just(1) + stream.range(2, 4)
# Print [1, 2, 3]
print(await stream.list(one_two_three))
async def main():
# Create a counting stream with a 0.2 seconds interval
xs = stream.count(interval=0.2)
# Operators can be piped using '|'
ys = xs | pipe.map(lambda x: x**2)
# Streams can be sliced
zs = ys[1:10:2]
# Use a stream context for proper resource management
async with zs.stream() as streamer:
# Asynchronous iteration
async for z in streamer:
# Print 1, 9, 25, 49 and 81
print('->', z)
# Streams can be awaited and return the last value
print('9² = ', await zs)
# Streams can run several times
print('9² = ', await zs)
# Streams can be concatenated
one_two_three = stream.just(1) + stream.range(2, 4)
# Print [1, 2, 3]
print(await stream.list(one_two_three))
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x+2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 5, 5]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x+2, interval=5),
task_limit=1)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x+2, interval=5),
task_limit=2)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 4, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x+4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
def _read_sensor( # pylint: disable=too-many-arguments
self, source_uuid: UUID, sid: int, unit: str, topic: str,
callback_config: AdvancedCallbackConfiguration
) -> AsyncGenerator[DataEvent, None]:
monitor_stream = (
stream.repeat(self.device, interval=1)
| pipe.map(
async_(lambda sensor: sensor.get_callback_configuration(sid)))
| pipe.map(lambda current_config: None
if current_config == callback_config else self.device)
| pipe.filter(lambda sensor: sensor is not None)
| pipe.action(lambda sensor: logging.getLogger(__name__).info(
"Resetting callback config for %s", sensor))
| pipe.action(
async_(lambda sensor: sensor.set_callback_configuration(
sid, *callback_config)))
| pipe.filter(lambda x: False))
return stream.merge(
stream.just(monitor_stream),
stream.iterate(self.device.read_events(sids=(sid, )))
| pipe.map(lambda item: DataEvent(sender=source_uuid,
topic=topic,
value=item.payload,
sid=item.sid,
unit=str(unit))),
)
def _configure_and_stream(
self,
config: dict[str, Any] | None) -> AsyncGenerator[DataEvent, None]:
if config is None:
return stream.empty()
try:
# Run all config steps in order (concat) and one at a time (task_limit=1). Drop the output. There is
# nothing to compare them to (filter => false), then read all sensors of the bricklet and process them in
# parallel (flatten).
config_stream = stream.chain(
stream.iterate(config["on_connect"])
| pipe.starmap(lambda func, timeout: stream.just(func()) | pipe
.timeout(timeout))
| pipe.concat(task_limit=1)
| pipe.filter(lambda result: False),
stream.iterate(config["config"].items())
| pipe.starmap(self._parse_callback_configuration)
| pipe.starmap(self._set_callback_configuration)
| pipe.flatten()
|
pipe.map(lambda args: self._read_sensor(config["uuid"], *args))
| pipe.flatten(),
)
return config_stream
except NotConnectedError:
# Do not log it
raise
except Exception:
self._logger.exception("This should not happen.")
raise
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 1, 1]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5),
task_limit=1)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5),
task_limit=2)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
def _read_device(config: dict[str, Any]) -> AsyncGenerator[Any, None]:
on_read: partial
timeout: float
on_read, timeout = config["on_read"]
if inspect.isasyncgenfunction(on_read.func):
return stream.iterate(on_read()) | pipe.timeout(timeout)
return (stream.repeat(config["on_read"], interval=config["interval"])
| pipe.starmap(lambda func, timeout: stream.just(func()) | pipe
.timeout(timeout))
| pipe.concat(task_limit=1))
def help_command() -> Stream:
if not arg:
result = overview()
elif arg and arg in self.all_parts:
result = self.all_parts[arg].rendered_help(ctx)
elif arg and arg in self.aliases:
alias = self.aliases[arg]
explain = f"{arg} is an alias for {alias}\n\n"
result = explain + self.all_parts[alias].rendered_help(ctx)
else:
result = f"No command found with this name: {arg}"
return stream.just(result)
def stream_data(self) -> AsyncGenerator[DataEvent, None]:
"""
Generate the initial configuration of the sensor, configure it, and finally stream the data from the sensor.
If there is a configuration update, reconfigure the sensor and start streaming again.
Returns
-------
AsyncGenerator of DataEvent
The data from the device
"""
# Generates the first configuration
# Query the database and if it does not have a config for the sensor, wait until there is one
data_stream = stream.chain(
stream.just(self),
stream.iterate(
event_bus.subscribe(
f"nodes/tinkerforge/{self.device.uid}/remove"))[:1]
| pipe.map(lambda x: None),
) | pipe.switchmap(
lambda sensor: stream.empty() if sensor is None else
(self._stream_config_updates(sensor)
| pipe.switchmap(lambda config: stream.chain(
stream.just(config),
stream.iterate(
event_bus.subscribe(
f"nodes/by_uuid/{config['uuid']}/remove"))[:1]
| pipe.map(lambda x: None),
))
| pipe.action(lambda config: logging.getLogger(__name__).info(
"Got new configuration for: %s",
sensor.device,
))
| pipe.map(self._create_config)
| pipe.switchmap(lambda config: stream.empty()
if config is None or not config["enabled"] else
(self._configure_and_stream(config)))))
return data_stream
async def arg_stream(ic, user, arg):
if type(arg) in [int, str, float]:
return stream.just({'type': type(arg).__name__, 'val': arg})
elif isinstance(arg, pathlib.Path):
path = misc.normpath(ic, arg)
return read_field(ic, user, path)
elif type(arg) is dict and 'fname' in arg:
return display_fn(ic, arg)
else:
logger.warning(f"{ic.user}@{ic.path}: argument {arg}: unknow type")
return stream.empty()
async def test_cycle(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.empty() | pipe.cycle() | pipe.timeout(1)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = stream.empty() | add_resource.pipe(
1) | pipe.cycle() | pipe.timeout(1)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = stream.just(1) | add_resource.pipe(1) | pipe.cycle()
await assert_run(xs[:5], [1] * 5)
assert event_loop.steps == [1] * 5
async def test_merge(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys, [0, 1, 2, 3, 4])
assert event_loop.steps == [1, 1, 1, 1]
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys[:3], [0, 1, 2])
assert event_loop.steps == [1, 1]
with event_loop.assert_cleanup():
xs = stream.just(1) + stream.never()
ys = xs | pipe.merge(xs) | pipe.timeout(1)
await assert_run(ys, [1, 1], asyncio.TimeoutError())
assert event_loop.steps == [1]
# Reproduce issue #65
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2])
ys = stream.iterate([3, 4])
zs = stream.merge(xs, ys) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = stream.throw(ZeroDivisionError)
zs = stream.merge(xs, ys) | pipe.delay(1) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
# Silencing of a CancelledError
async def agen1():
if False:
yield
try:
await asyncio.sleep(2)
except asyncio.CancelledError:
return
async def agen2():
yield 1
with event_loop.assert_cleanup():
xs = stream.merge(agen1(), agen2()) | pipe.delay(1) | pipe.take(1)
await assert_run(xs, [1])
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 5, 5]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x + 2, interval=5), task_limit=1
)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x + 2, interval=5), task_limit=2
)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 4, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
# An exception might get discarded if the result can be produced before the
# processing of the exception is required
with event_loop.assert_cleanup():
xs = stream.iterate([True, False])
ys = xs | pipe.concatmap(
lambda x: stream.range(0, 3, interval=1)
if x
else stream.throw(ZeroDivisionError)
)
zs = ys | pipe.take(3)
await assert_run(zs, [0, 1, 2])
assert event_loop.steps == [1, 1]
async def test_merge(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys, [0, 1, 2, 3, 4])
assert event_loop.steps == [1, 1, 1, 1]
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys[:3], [0, 1, 2])
assert event_loop.steps == [1, 1]
with event_loop.assert_cleanup():
xs = stream.just(1) + stream.never()
ys = xs | pipe.merge(xs) | pipe.timeout(1)
await assert_run(ys, [1, 1], asyncio.TimeoutError())
assert event_loop.steps == [1]
async def test_cycle(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.empty() | pipe.cycle() | pipe.timeout(1)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = (
stream.empty()
| add_resource.pipe(1)
| pipe.cycle()
| pipe.timeout(1)
)
await assert_run(xs, [], asyncio.TimeoutError())
with event_loop.assert_cleanup():
xs = stream.just(1) | add_resource.pipe(1) | pipe.cycle()
await assert_run(xs[:5], [1]*5)
assert event_loop.steps == [1]*5
def stream_data(self):
"""
Discover all Tinkerforge devices connected via this transport.
Yields
-------
"""
data_stream = (
stream.just(self)
| pipe.action(
lambda transport: logging.getLogger(__name__).info(
"Connecting to Tinkerforge host at %s (%s).", transport.uri, transport.label
)
)
| pipe.switchmap(self._stream_transport)
| retry.pipe((ConnectionError, asyncio.TimeoutError), self.reconnect_interval)
)
return data_stream
def stream_data(self):
"""
Discover all Tinkerforge devices connected via this transport.
Yields
-------
"""
data_stream = (
stream.just(self)
| pipe.action(lambda transport: logging.getLogger(__name__).info(
"Connecting to %s at %s (%s).", transport.name, transport.uri,
transport.label))
| pipe.switchmap(self._stream_data)
| retry.pipe(
(GpibError, asyncio.TimeoutError), self.reconnect_interval))
# We need to catch the TimeoutError here, because most protocols like SCPI have no means of synchronizing
# messages. This means, that we will lose sync after a timeout. In these cases, we reconnect the transport.
# In case of a GpibError, we error out and stop the device.
return data_stream
def help_command() -> Stream:
if not arg:
result = overview()
elif arg == "placeholders":
result = placeholders()
elif arg in self.all_parts:
maybe_aliases = self.reverse_alias_names.get(arg)
result = ""
if maybe_aliases:
result += f'{arg} can also invoked via: {", ".join(maybe_aliases)}\n\n'
result += self.all_parts[arg].rendered_help(ctx)
elif arg in self.alias_names:
alias = self.alias_names[arg]
explain = f"{arg} is an alias for {alias}\n\n"
result = explain + self.all_parts[alias].rendered_help(ctx)
elif arg in self.alias_templates:
result = self.alias_templates[arg].rendered_help(ctx)
else:
result = f"No command found with this name: {arg}"
return stream.just(result)
def stream_data(self,
config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]:
"""
Enumerate the device, then read data from it.
Parameters
----------
config: dict
A dict containing the configuration for the device
Yields
-------
DataEvent
The data from the device
"""
return stream.chain(
stream.just(self)
| pipe.action(async_(lambda sensor: sensor.enumerate()))
| pipe.filter(lambda x: False),
super().stream_data(config),
)
async def handle_message_async(body):
msg = json.loads(body.data.decode('utf-8'))
print(f'Received msg: {msg}')
msg = await create_workdir(msg)
filenames = np.empty(3, dtype=object)
msg['filenames'] = filenames
await asyncio.gather(
process_band(msg, 0, 'i'),
process_band(msg, 1, 'r'),
process_band(msg, 2, 'g')
)
await (stream.just(msg)
| pipe.flatmap(get_object_msgs)
| pipe.action(create_data_cube)
| pipe.action(cutout_fits)
| pipe.action(create_jpeg)
| pipe.action(move_files_to_gcs))
await cleanup(msg)
def _configure_and_stream(
self, config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]:
if config is None:
return stream.empty()
# Run all config steps in order (concat) and one at a time (task_limit=1). Drop the output. There is nothing to
# compare them to (filter => false), then read the device.
config_stream = stream.chain(
stream.iterate(config["on_connect"])
| pipe.starmap(lambda func, timeout: stream.just(func()) | pipe.
timeout(timeout))
| pipe.concat(task_limit=1)
| pipe.filter(lambda result: False),
self._read_device(config)
| pipe.map(lambda item: DataEvent(sender=config["uuid"],
topic=config["topic"],
value=item,
sid=0,
unit=config["unit"]))
| finally_action.pipe(
stream.call(self._clean_up, config["on_disconnect"])),
) | catch.pipe(TypeError, on_exc=self.on_error)
return config_stream
async def _set_callback_configuration(
self, sid: int, unit: str, topic: str,
config: AdvancedCallbackConfiguration):
try:
await self.device.set_callback_configuration(sid, *config)
except AssertionError:
self._logger.error(
"Invalid configuration for %s: sid=%i, config=%s", self.device,
sid, config)
return stream.empty()
remote_callback_config: AdvancedCallbackConfiguration
remote_callback_config = await self.device.get_callback_configuration(
sid)
if remote_callback_config.period == 0:
self._logger.warning(
"Callback configuration configuration for %s: sid=%i, config=%s failed. Source disabled.",
self.device,
sid,
config,
)
return stream.empty()
return stream.just((sid, unit, topic, remote_callback_config))
def stream_data(self):
"""
Discover all Tinkerforge devices connected via this transport.
Yields
-------
"""
data_stream = (
stream.just(self)
| pipe.action(
lambda transport: logging.getLogger(__name__).info(
"Connecting to %s at %s (%s).", transport.name, transport.uri, transport.label
)
)
| pipe.switchmap(self._stream_data)
| retry.pipe((OSError, asyncio.TimeoutError), self.reconnect_interval)
)
# We need to catch OSError, which is the parent of ConnectionError, because a connection to localhost
# might resolve to 2 IPs and then return multiple exception at once if all IPs fail, which is an
# OSError.
# We also need to catch the TimeoutError here, because most protocols like SCPI have no means of synchronizing
# messages. This means, that we will lose sync after a timeout. We therefore need to reconnect in these cases.
return data_stream
async def test_just(assert_run):
value = 3
xs = stream.just(value)
await assert_run(xs, [3])
async def handle_message_async(msg):
await (stream.just(msg)
| pipe.action(retrieve_file)
| pipe.action(move_file_to_gcs))
async def test_just(assert_run):
value = 3
xs = stream.just(value)
await assert_run(xs, [3])
