async def test_just(assert_run): value = 3 xs = stream.just(value) await assert_run(xs, [3]) async def four(): return 4 xs = stream.just(four()) await assert_run(xs, [4])
def stream_data(self, config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]: """ Stream the data from the sensor. Parameters ---------- config: dict A dictionary containing the sensor configuration. Returns ------- AsyncGenerator The asynchronous stream. """ data_stream = (stream.chain( stream.just(config), stream.iterate( event_bus.subscribe(f"nodes/by_uuid/{self.__uuid}/update"))) | pipe.action(lambda _: logging.getLogger(__name__).info( "Got new configuration for: %s", self) if config is not None else logging.getLogger(__name__). info("Removed configuration for: %s", self)) | pipe.map(self._parse_config) | pipe.switchmap(lambda conf: stream.empty( ) if conf is None or not conf["enabled"] else (self._configure_and_stream(conf)))) return data_stream
async def main(): # Create a counting stream with a 0.2 seconds interval xs = stream.count(interval=0.2) # Operators can be piped using '|' ys = xs | pipe.map(lambda x: x**2) # Streams can be sliced zs = ys[1:10:2] # Use a stream context for proper resource management async with zs.stream() as streamer: # Asynchronous iteration async for z in streamer: # Print 1, 9, 25, 49 and 81 print("->", z) # Streams can be awaited and return the last value print("9² = ", await zs) # Streams can run several times print("9² = ", await zs) # Streams can be concatenated one_two_three = stream.just(1) + stream.range(2, 4) # Print [1, 2, 3] print(await stream.list(one_two_three))
async def main(): # Create a counting stream with a 0.2 seconds interval xs = stream.count(interval=0.2) # Operators can be piped using '|' ys = xs | pipe.map(lambda x: x**2) # Streams can be sliced zs = ys[1:10:2] # Use a stream context for proper resource management async with zs.stream() as streamer: # Asynchronous iteration async for z in streamer: # Print 1, 9, 25, 49 and 81 print('->', z) # Streams can be awaited and return the last value print('9² = ', await zs) # Streams can run several times print('9² = ', await zs) # Streams can be concatenated one_two_three = stream.just(1) + stream.range(2, 4) # Print [1, 2, 3] print(await stream.list(one_two_three))
async def test_concatmap(assert_run, event_loop): # Concurrent run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap(lambda x: stream.range(x, x+2, interval=5)) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 1, 3, 5, 5] # Sequential run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap( lambda x: stream.range(x, x+2, interval=5), task_limit=1) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [5, 1, 5, 1, 5] # Limited run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap( lambda x: stream.range(x, x+2, interval=5), task_limit=2) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 4, 1, 4, 5] # Make sure item arrive as soon as possible with event_loop.assert_cleanup(): xs = stream.just(2) ys = xs | pipe.concatmap(lambda x: stream.range(x, x+4, interval=1)) zs = ys | pipe.timeout(2) # Sould NOT raise await assert_run(zs, [2, 3, 4, 5]) assert event_loop.steps == [1, 1, 1]
def _read_sensor( # pylint: disable=too-many-arguments self, source_uuid: UUID, sid: int, unit: str, topic: str, callback_config: AdvancedCallbackConfiguration ) -> AsyncGenerator[DataEvent, None]: monitor_stream = ( stream.repeat(self.device, interval=1) | pipe.map( async_(lambda sensor: sensor.get_callback_configuration(sid))) | pipe.map(lambda current_config: None if current_config == callback_config else self.device) | pipe.filter(lambda sensor: sensor is not None) | pipe.action(lambda sensor: logging.getLogger(__name__).info( "Resetting callback config for %s", sensor)) | pipe.action( async_(lambda sensor: sensor.set_callback_configuration( sid, *callback_config))) | pipe.filter(lambda x: False)) return stream.merge( stream.just(monitor_stream), stream.iterate(self.device.read_events(sids=(sid, ))) | pipe.map(lambda item: DataEvent(sender=source_uuid, topic=topic, value=item.payload, sid=item.sid, unit=str(unit))), )
def _configure_and_stream( self, config: dict[str, Any] | None) -> AsyncGenerator[DataEvent, None]: if config is None: return stream.empty() try: # Run all config steps in order (concat) and one at a time (task_limit=1). Drop the output. There is # nothing to compare them to (filter => false), then read all sensors of the bricklet and process them in # parallel (flatten). config_stream = stream.chain( stream.iterate(config["on_connect"]) | pipe.starmap(lambda func, timeout: stream.just(func()) | pipe .timeout(timeout)) | pipe.concat(task_limit=1) | pipe.filter(lambda result: False), stream.iterate(config["config"].items()) | pipe.starmap(self._parse_callback_configuration) | pipe.starmap(self._set_callback_configuration) | pipe.flatten() | pipe.map(lambda args: self._read_sensor(config["uuid"], *args)) | pipe.flatten(), ) return config_stream except NotConnectedError: # Do not log it raise except Exception: self._logger.exception("This should not happen.") raise
async def test_concatmap(assert_run, event_loop): # Concurrent run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5)) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 1, 3, 1, 1] # Sequential run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5), task_limit=1) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [5, 1, 5, 1, 5] # Limited run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5), task_limit=2) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 4, 1, 5] # Make sure item arrive as soon as possible with event_loop.assert_cleanup(): xs = stream.just(2) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1)) zs = ys | pipe.timeout(2) # Sould NOT raise await assert_run(zs, [2, 3, 4, 5]) assert event_loop.steps == [1, 1, 1]
def _read_device(config: dict[str, Any]) -> AsyncGenerator[Any, None]: on_read: partial timeout: float on_read, timeout = config["on_read"] if inspect.isasyncgenfunction(on_read.func): return stream.iterate(on_read()) | pipe.timeout(timeout) return (stream.repeat(config["on_read"], interval=config["interval"]) | pipe.starmap(lambda func, timeout: stream.just(func()) | pipe .timeout(timeout)) | pipe.concat(task_limit=1))
def help_command() -> Stream: if not arg: result = overview() elif arg and arg in self.all_parts: result = self.all_parts[arg].rendered_help(ctx) elif arg and arg in self.aliases: alias = self.aliases[arg] explain = f"{arg} is an alias for {alias}\n\n" result = explain + self.all_parts[alias].rendered_help(ctx) else: result = f"No command found with this name: {arg}" return stream.just(result)
def stream_data(self) -> AsyncGenerator[DataEvent, None]: """ Generate the initial configuration of the sensor, configure it, and finally stream the data from the sensor. If there is a configuration update, reconfigure the sensor and start streaming again. Returns ------- AsyncGenerator of DataEvent The data from the device """ # Generates the first configuration # Query the database and if it does not have a config for the sensor, wait until there is one data_stream = stream.chain( stream.just(self), stream.iterate( event_bus.subscribe( f"nodes/tinkerforge/{self.device.uid}/remove"))[:1] | pipe.map(lambda x: None), ) | pipe.switchmap( lambda sensor: stream.empty() if sensor is None else (self._stream_config_updates(sensor) | pipe.switchmap(lambda config: stream.chain( stream.just(config), stream.iterate( event_bus.subscribe( f"nodes/by_uuid/{config['uuid']}/remove"))[:1] | pipe.map(lambda x: None), )) | pipe.action(lambda config: logging.getLogger(__name__).info( "Got new configuration for: %s", sensor.device, )) | pipe.map(self._create_config) | pipe.switchmap(lambda config: stream.empty() if config is None or not config["enabled"] else (self._configure_and_stream(config))))) return data_stream
async def arg_stream(ic, user, arg): if type(arg) in [int, str, float]: return stream.just({'type': type(arg).__name__, 'val': arg}) elif isinstance(arg, pathlib.Path): path = misc.normpath(ic, arg) return read_field(ic, user, path) elif type(arg) is dict and 'fname' in arg: return display_fn(ic, arg) else: logger.warning(f"{ic.user}@{ic.path}: argument {arg}: unknow type") return stream.empty()
async def test_cycle(assert_run, event_loop): with event_loop.assert_cleanup(): xs = stream.empty() | pipe.cycle() | pipe.timeout(1) await assert_run(xs, [], asyncio.TimeoutError()) with event_loop.assert_cleanup(): xs = stream.empty() | add_resource.pipe( 1) | pipe.cycle() | pipe.timeout(1) await assert_run(xs, [], asyncio.TimeoutError()) with event_loop.assert_cleanup(): xs = stream.just(1) | add_resource.pipe(1) | pipe.cycle() await assert_run(xs[:5], [1] * 5) assert event_loop.steps == [1] * 5
async def test_merge(assert_run, event_loop): with event_loop.assert_cleanup(): xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1) ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs) await assert_run(ys, [0, 1, 2, 3, 4]) assert event_loop.steps == [1, 1, 1, 1] with event_loop.assert_cleanup(): xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1) ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs) await assert_run(ys[:3], [0, 1, 2]) assert event_loop.steps == [1, 1] with event_loop.assert_cleanup(): xs = stream.just(1) + stream.never() ys = xs | pipe.merge(xs) | pipe.timeout(1) await assert_run(ys, [1, 1], asyncio.TimeoutError()) assert event_loop.steps == [1] # Reproduce issue #65 with event_loop.assert_cleanup(): xs = stream.iterate([1, 2]) ys = stream.iterate([3, 4]) zs = stream.merge(xs, ys) | pipe.take(3) await assert_run(zs, [1, 2, 3]) with event_loop.assert_cleanup(): xs = stream.iterate([1, 2, 3]) ys = stream.throw(ZeroDivisionError) zs = stream.merge(xs, ys) | pipe.delay(1) | pipe.take(3) await assert_run(zs, [1, 2, 3]) # Silencing of a CancelledError async def agen1(): if False: yield try: await asyncio.sleep(2) except asyncio.CancelledError: return async def agen2(): yield 1 with event_loop.assert_cleanup(): xs = stream.merge(agen1(), agen2()) | pipe.delay(1) | pipe.take(1) await assert_run(xs, [1])
async def test_concatmap(assert_run, event_loop): # Concurrent run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5)) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 1, 3, 5, 5] # Sequential run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap( lambda x: stream.range(x, x + 2, interval=5), task_limit=1 ) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [5, 1, 5, 1, 5] # Limited run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap( lambda x: stream.range(x, x + 2, interval=5), task_limit=2 ) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 4, 1, 4, 5] # Make sure item arrive as soon as possible with event_loop.assert_cleanup(): xs = stream.just(2) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1)) zs = ys | pipe.timeout(2) # Sould NOT raise await assert_run(zs, [2, 3, 4, 5]) assert event_loop.steps == [1, 1, 1] # An exception might get discarded if the result can be produced before the # processing of the exception is required with event_loop.assert_cleanup(): xs = stream.iterate([True, False]) ys = xs | pipe.concatmap( lambda x: stream.range(0, 3, interval=1) if x else stream.throw(ZeroDivisionError) ) zs = ys | pipe.take(3) await assert_run(zs, [0, 1, 2]) assert event_loop.steps == [1, 1]
async def test_merge(assert_run, event_loop): with event_loop.assert_cleanup(): xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1) ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs) await assert_run(ys, [0, 1, 2, 3, 4]) assert event_loop.steps == [1, 1, 1, 1] with event_loop.assert_cleanup(): xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1) ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs) await assert_run(ys[:3], [0, 1, 2]) assert event_loop.steps == [1, 1] with event_loop.assert_cleanup(): xs = stream.just(1) + stream.never() ys = xs | pipe.merge(xs) | pipe.timeout(1) await assert_run(ys, [1, 1], asyncio.TimeoutError()) assert event_loop.steps == [1]
async def test_cycle(assert_run, event_loop): with event_loop.assert_cleanup(): xs = stream.empty() | pipe.cycle() | pipe.timeout(1) await assert_run(xs, [], asyncio.TimeoutError()) with event_loop.assert_cleanup(): xs = ( stream.empty() | add_resource.pipe(1) | pipe.cycle() | pipe.timeout(1) ) await assert_run(xs, [], asyncio.TimeoutError()) with event_loop.assert_cleanup(): xs = stream.just(1) | add_resource.pipe(1) | pipe.cycle() await assert_run(xs[:5], [1]*5) assert event_loop.steps == [1]*5
def stream_data(self): """ Discover all Tinkerforge devices connected via this transport. Yields ------- """ data_stream = ( stream.just(self) | pipe.action( lambda transport: logging.getLogger(__name__).info( "Connecting to Tinkerforge host at %s (%s).", transport.uri, transport.label ) ) | pipe.switchmap(self._stream_transport) | retry.pipe((ConnectionError, asyncio.TimeoutError), self.reconnect_interval) ) return data_stream
def stream_data(self): """ Discover all Tinkerforge devices connected via this transport. Yields ------- """ data_stream = ( stream.just(self) | pipe.action(lambda transport: logging.getLogger(__name__).info( "Connecting to %s at %s (%s).", transport.name, transport.uri, transport.label)) | pipe.switchmap(self._stream_data) | retry.pipe( (GpibError, asyncio.TimeoutError), self.reconnect_interval)) # We need to catch the TimeoutError here, because most protocols like SCPI have no means of synchronizing # messages. This means, that we will lose sync after a timeout. In these cases, we reconnect the transport. # In case of a GpibError, we error out and stop the device. return data_stream
def help_command() -> Stream: if not arg: result = overview() elif arg == "placeholders": result = placeholders() elif arg in self.all_parts: maybe_aliases = self.reverse_alias_names.get(arg) result = "" if maybe_aliases: result += f'{arg} can also invoked via: {", ".join(maybe_aliases)}\n\n' result += self.all_parts[arg].rendered_help(ctx) elif arg in self.alias_names: alias = self.alias_names[arg] explain = f"{arg} is an alias for {alias}\n\n" result = explain + self.all_parts[alias].rendered_help(ctx) elif arg in self.alias_templates: result = self.alias_templates[arg].rendered_help(ctx) else: result = f"No command found with this name: {arg}" return stream.just(result)
def stream_data(self, config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]: """ Enumerate the device, then read data from it. Parameters ---------- config: dict A dict containing the configuration for the device Yields ------- DataEvent The data from the device """ return stream.chain( stream.just(self) | pipe.action(async_(lambda sensor: sensor.enumerate())) | pipe.filter(lambda x: False), super().stream_data(config), )
async def handle_message_async(body): msg = json.loads(body.data.decode('utf-8')) print(f'Received msg: {msg}') msg = await create_workdir(msg) filenames = np.empty(3, dtype=object) msg['filenames'] = filenames await asyncio.gather( process_band(msg, 0, 'i'), process_band(msg, 1, 'r'), process_band(msg, 2, 'g') ) await (stream.just(msg) | pipe.flatmap(get_object_msgs) | pipe.action(create_data_cube) | pipe.action(cutout_fits) | pipe.action(create_jpeg) | pipe.action(move_files_to_gcs)) await cleanup(msg)
def _configure_and_stream( self, config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]: if config is None: return stream.empty() # Run all config steps in order (concat) and one at a time (task_limit=1). Drop the output. There is nothing to # compare them to (filter => false), then read the device. config_stream = stream.chain( stream.iterate(config["on_connect"]) | pipe.starmap(lambda func, timeout: stream.just(func()) | pipe. timeout(timeout)) | pipe.concat(task_limit=1) | pipe.filter(lambda result: False), self._read_device(config) | pipe.map(lambda item: DataEvent(sender=config["uuid"], topic=config["topic"], value=item, sid=0, unit=config["unit"])) | finally_action.pipe( stream.call(self._clean_up, config["on_disconnect"])), ) | catch.pipe(TypeError, on_exc=self.on_error) return config_stream
async def _set_callback_configuration( self, sid: int, unit: str, topic: str, config: AdvancedCallbackConfiguration): try: await self.device.set_callback_configuration(sid, *config) except AssertionError: self._logger.error( "Invalid configuration for %s: sid=%i, config=%s", self.device, sid, config) return stream.empty() remote_callback_config: AdvancedCallbackConfiguration remote_callback_config = await self.device.get_callback_configuration( sid) if remote_callback_config.period == 0: self._logger.warning( "Callback configuration configuration for %s: sid=%i, config=%s failed. Source disabled.", self.device, sid, config, ) return stream.empty() return stream.just((sid, unit, topic, remote_callback_config))
def stream_data(self): """ Discover all Tinkerforge devices connected via this transport. Yields ------- """ data_stream = ( stream.just(self) | pipe.action( lambda transport: logging.getLogger(__name__).info( "Connecting to %s at %s (%s).", transport.name, transport.uri, transport.label ) ) | pipe.switchmap(self._stream_data) | retry.pipe((OSError, asyncio.TimeoutError), self.reconnect_interval) ) # We need to catch OSError, which is the parent of ConnectionError, because a connection to localhost # might resolve to 2 IPs and then return multiple exception at once if all IPs fail, which is an # OSError. # We also need to catch the TimeoutError here, because most protocols like SCPI have no means of synchronizing # messages. This means, that we will lose sync after a timeout. We therefore need to reconnect in these cases. return data_stream
async def test_just(assert_run): value = 3 xs = stream.just(value) await assert_run(xs, [3])
async def handle_message_async(msg): await (stream.just(msg) | pipe.action(retrieve_file) | pipe.action(move_file_to_gcs))