def _read_sensor( # pylint: disable=too-many-arguments
self, source_uuid: UUID, sid: int, unit: str, topic: str,
callback_config: AdvancedCallbackConfiguration
) -> AsyncGenerator[DataEvent, None]:
monitor_stream = (
stream.repeat(self.device, interval=1)
| pipe.map(
async_(lambda sensor: sensor.get_callback_configuration(sid)))
| pipe.map(lambda current_config: None
if current_config == callback_config else self.device)
| pipe.filter(lambda sensor: sensor is not None)
| pipe.action(lambda sensor: logging.getLogger(__name__).info(
"Resetting callback config for %s", sensor))
| pipe.action(
async_(lambda sensor: sensor.set_callback_configuration(
sid, *callback_config)))
| pipe.filter(lambda x: False))
return stream.merge(
stream.just(monitor_stream),
stream.iterate(self.device.read_events(sids=(sid, )))
| pipe.map(lambda item: DataEvent(sender=source_uuid,
topic=topic,
value=item.payload,
sid=item.sid,
unit=str(unit))),
)
def _stream_data(self, transport):
config_stream = (
with_context(
transport,
on_exit=lambda: logging.getLogger(__name__).info(
"Disconnected from APQ Labnode at %s (%s).", transport.uri, transport.label
),
)
| pipe.action(
lambda _: logging.getLogger(__name__).info(
"Connected to APQ Labnode at %s (%s).", transport.uri, transport.label
)
)
| pipe.map(LabnodeSensor)
| pipe.action(async_(lambda sensor: sensor.enumerate()))
| pipe.switchmap(lambda sensor: sensor.stream_data())
)
return config_stream
def stream_data(self,
config: dict[str, Any]) -> AsyncGenerator[DataEvent, None]:
"""
Enumerate the device, then read data from it.
Parameters
----------
config: dict
A dict containing the configuration for the device
Yields
-------
DataEvent
The data from the device
"""
return stream.chain(
stream.just(self)
| pipe.action(async_(lambda sensor: sensor.enumerate()))
| pipe.filter(lambda x: False),
super().stream_data(config),
)
async def test_map(assert_run, event_loop):
# Synchronous/simple
with event_loop.assert_cleanup():
xs = stream.range(5) | pipe.map(lambda x: x**2)
expected = [x**2 for x in range(5)]
await assert_run(xs, expected)
# Synchronous/multiple
with event_loop.assert_cleanup():
xs = stream.range(5)
ys = xs | pipe.map(lambda x, y: x + y, xs)
expected = [x * 2 for x in range(5)]
await assert_run(ys, expected)
# Asynchronous/simple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 1, 1]
# Asynchronous/simple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=1)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 2, 3]
# Asynchronous/multiple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Asynchronous/multiple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 2, 3]
# As completed
with event_loop.assert_cleanup():
xs = stream.iterate([2, 4, 1, 3, 5])
ys = xs | pipe.map(asyncio.sleep, xs, ordered=False)
await assert_run(ys, [1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1, 1, 1]
# Invalid argument
with pytest.raises(ValueError):
await (stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=0))
# Break
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=10)
await assert_run(ys[:3], [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Stuck
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1) | pipe.timeout(5)
await assert_run(ys, [1, 2, 3, 4], asyncio.TimeoutError())
# Force await
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = xs | pipe.map(async_(lambda x: asyncio.sleep(x, x)))
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Map await_
with event_loop.assert_cleanup():
xs = stream.iterate(map(lambda x: asyncio.sleep(x, x), [1, 2, 3]))
ys = xs | pipe.map(await_)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
async def data_stream_async(url,
files,
columns=None,
map_func=None,
reduce_func=None,
initializer=None,
producer_num=2,
data_handler_num=2,
executor_type='process'):
data_type = files[0].split('.')[-1]
columns_mapping = get_parquet_mapping() if data_type == 'parquet' else None
if columns:
c = [k for k, v in columns_mapping.items() if v in columns]
c = c + list(
set(columns).difference(set(list(columns_mapping.values()))))
else:
c = columns
global pbar_handler
pbar_handler = PbarHandler(len(files))
global executor
if executor_type == 'process':
executor = ProcessPoolExecutor(max_workers=data_handler_num)
elif executor_type == 'thread':
executor = ThreadPoolExecutor(max_workers=data_handler_num)
if map_func:
map_task = partial(data_load, func=map_func)
else:
map_task = data_load
pbar_handler.register(mapper, len(files))
if reduce_func:
reduce_task = partial(reducer, func=reduce_func)
else:
reduce_task = partial(reducer, func=concat)
initializer = pd.DataFrame()
file_streams = [
stream.preserve(file_stream(files[i::producer_num], url))
for i in range(producer_num)
]
file_list = []
aws = (stream.merge(*file_streams)
| pipe.map(
async_(lambda x: mapper(x, map_task, c, columns_mapping)),
task_limit=data_handler_num)
| pipe.map(async_(lambda x: file_list.append(x[0]) or x[1]),
task_limit=data_handler_num))
if reduce_func:
pbar_handler.register(reducer, len(files) - 1)
rs = stream.reduce(aws, async_(reduce_task), initializer)
reduced = await stream.takelast(rs, 1)
return reduced
else:
data_list = await asyncio.gather(stream.list(aws))
data_list = data_list[0]
tmp_list = list(zip(file_list, data_list))
tmp_list = sorted(tmp_list, key=lambda pair: files.index(pair[0]))
if map_func:
return tmp_list
else:
return pd.concat(list(map(lambda pair: pair[1], tmp_list)), axis=0)
async def test_map(assert_run, event_loop):
# Synchronous/simple
with event_loop.assert_cleanup():
xs = stream.range(5) | pipe.map(lambda x: x**2)
expected = [x**2 for x in range(5)]
await assert_run(xs, expected)
# Synchronous/multiple
with event_loop.assert_cleanup():
xs = stream.range(5)
ys = xs | pipe.map(lambda x, y: x+y, xs)
expected = [x*2 for x in range(5)]
await assert_run(ys, expected)
# Asynchronous/simple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 1, 1]
# Asynchronous/simple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=1)
expected = [None] * 3
await assert_run(xs, expected)
assert event_loop.steps == [1, 2, 3]
# Asynchronous/multiple/concurrent
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Asynchronous/multiple/sequential
with event_loop.assert_cleanup():
xs = stream.range(1, 4)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 2, 3]
# As completed
with event_loop.assert_cleanup():
xs = stream.iterate([2, 4, 1, 3, 5])
ys = xs | pipe.map(asyncio.sleep, xs, ordered=False)
await assert_run(ys, [1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1, 1, 1]
# Invalid argument
with pytest.raises(ValueError):
await (stream.range(1, 4) | pipe.map(asyncio.sleep, task_limit=0))
# Break
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=10)
await assert_run(ys[:3], [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Stuck
with event_loop.assert_cleanup():
xs = stream.count(1)
ys = xs | pipe.map(asyncio.sleep, xs, task_limit=1) | pipe.timeout(5)
await assert_run(ys, [1, 2, 3, 4], asyncio.TimeoutError())
# Force await
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = xs | pipe.map(async_(lambda x: asyncio.sleep(x, x)))
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]
# Map await_
with event_loop.assert_cleanup():
xs = stream.iterate(map(lambda x: asyncio.sleep(x, x), [1, 2, 3]))
ys = xs | pipe.map(await_)
await assert_run(ys, [1, 2, 3])
assert event_loop.steps == [1, 1, 1]