def test_concat_basic(nums):
nums_py = list(map(lambda x: x + 1, nums))
nums_py1 = list(map(lambda x: x**2, nums_py))
nums_py2 = list(map(lambda x: -x, nums_py))
nums_py = nums_py1 + nums_py2
nums_pl = aio.map(lambda x: x + 1, nums)
nums_pl1 = aio.map(lambda x: x**2, nums_pl)
nums_pl2 = aio.map(lambda x: -x, nums_pl)
nums_pl = aio.concat([nums_pl1, nums_pl2])
assert sorted(nums_pl) == sorted(nums_py)
def fetch_all(urls):
stage = aio.map(_fetch,urls,
workers = MAX_WORKERS,
on_start = lambda: ClientSession(connector=TCPConnector(limit=None)),
on_done = lambda _status, session: session.close())
return stage
def test_map_square_event_end(nums):
namespace = aio._get_namespace()
namespace.x = 0
namespace.done = False
namespace.active_workers = -1
def set_1():
namespace.x = 1
def set_2(stage_status):
namespace.x = 2
namespace.active_workers = stage_status.active_workers
namespace.done = stage_status.done
nums_pl = aio.map(lambda x: x**2,
nums,
workers=3,
on_start=set_1,
on_done=set_2)
nums_pl = list(nums_pl)
assert namespace.x == 2
assert namespace.done == True
assert namespace.active_workers == 0
def test_map_id(nums):
nums_py = nums
nums_pl = aio.map(lambda x: x, nums)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
def test_map_square_workers(nums):
nums_py = map(lambda x: x**2, nums)
nums_py = list(nums_py)
nums_pl = aio.map(lambda x: x**2, nums, workers=2)
nums_pl = list(nums_pl)
assert sorted(nums_pl) == sorted(nums_py)
def test_map_square(nums):
nums_py = map(lambda x: x**2, nums)
nums_py = list(nums_py)
nums_pl = aio.map(lambda x: x**2, nums)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
def test_concat_multiple(nums):
nums_py = [x + 1 for x in nums]
nums_py1 = nums_py + nums_py
nums_py2 = nums_py1 + nums_py
nums_pl = aio.map(lambda x: x + 1, nums)
nums_pl1 = aio.concat([nums_pl, nums_pl])
nums_pl2 = aio.concat([nums_pl1, nums_pl])
assert sorted(nums_py1) == sorted(list(nums_pl1))
assert sorted(nums_py2) == sorted(list(nums_pl2))
def test_from_to_iterable(nums):
nums_pl = nums
nums_pl = aio.from_iterable(nums_pl)
nums_pl = cz.partition_all(10, nums_pl)
nums_pl = aio.map(sum, nums_pl)
nums_pl = list(nums_pl)
nums_py = nums
nums_py = cz.partition_all(10, nums_py)
nums_py = map(sum, nums_py)
nums_py = list(nums_py)
assert nums_py == nums_pl
def test_flat_map_square_workers(nums):
def _generator(x):
yield x
yield x + 1
yield x + 2
nums_py = map(lambda x: x**2, nums)
nums_py = cz.mapcat(_generator, nums_py)
nums_py = list(nums_py)
nums_pl = aio.map(lambda x: x**2, nums)
nums_pl = aio.flat_map(_generator, nums_pl, workers=3)
nums_pl = list(nums_pl)
assert sorted(nums_pl) == sorted(nums_py)
def test_flat_map_square(nums):
def _generator(x):
yield x
yield x + 1
yield x + 2
nums_py = map(lambda x: x**2, nums)
nums_py = cz.mapcat(_generator, nums_py)
nums_py = list(nums_py)
nums_pl = aio.map(lambda x: x**2, nums)
nums_pl = aio.flat_map(_generator, nums_pl)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
def test_map_square_event_start(nums):
nums_py = map(lambda x: x**2, nums)
nums_py = list(nums_py)
namespace = aio._get_namespace()
namespace.x = 0
def set_1():
namespace.x = 1
nums_pl = aio.map(lambda x: x**2, nums, on_start=set_1)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
assert namespace.x == 1
def test_error_handling():
error = None
def raise_error(x):
raise MyError()
stage = aio.map(raise_error, range(10))
try:
list(stage)
except MyError as e:
error = e
assert isinstance(error, MyError)
def test_flat_map_square_filter_workers_pipe(nums):
def _generator(x):
yield x
yield x + 1
yield x + 2
nums_py = map(lambda x: x**2, nums)
nums_py = cz.mapcat(_generator, nums_py)
nums_py = cz.filter(lambda x: x > 1, nums_py)
nums_py = list(nums_py)
nums_pl = (nums
| aio.map(lambda x: x**2)
| aio.flat_map(_generator, workers=3)
| aio.filter(lambda x: x > 1)
| list)
assert sorted(nums_pl) == sorted(nums_py)
from pypeln import asyncio_task as aio
def batch(x, list_acc, n):
if len(list_acc) == n:
list_out = list(list_acc)
list_acc.clear()
yield list_out
else:
list_acc.append(x)
print(
range(100)
| aio.map(lambda x: x)
| aio.flat_map(lambda x, list_acc: batch(x, list_acc, 10),
on_start=lambda: [])
| aio.map(sum)
| list)
###################
# from_to_iterable
###################
@hp.given(nums=st.lists(st.integers()))
@hp.settings(max_examples=MAX_EXAMPLES)
def test_from_to_iterable(nums):
nums_pl = nums
nums_pl = aio.from_iterable(nums_pl)
nums_pl = cz.partition_all(10, nums_pl)
nums_pl = aio.map(sum, nums_pl)
nums_pl = list(nums_pl)
nums_py = nums
nums_py = cz.partition_all(10, nums_py)
nums_py = map(sum, nums_py)
nums_py = list(nums_py)
assert nums_py == nums_pl
if __name__ == '__main__':
error = None
def raise_error(x):
raise MyError()
stage = aio.map(raise_error, range(10))
list(stage)
from pypeln import asyncio_task as aio
list_acc = []
def batch(x, n):
if len(list_acc) == n:
list_out = list(list_acc)
list_acc.clear()
yield list_out
else:
list_acc.append(x)
print(
range(100)
| aio.from_iterable()
| aio.flat_map(lambda x: batch(x, 10))
| aio.map(sum)
| list)
def test_map_id_pipe(nums):
nums_pl = (nums | aio.map(lambda x: x) | list)
assert nums_pl == nums
