def test_concat_basic(nums):
nums_py = list(map(lambda x: x + 1, nums))
nums_py1 = list(map(lambda x: x ** 2, nums_py))
nums_py2 = list(map(lambda x: -x, nums_py))
nums_py = nums_py1 + nums_py2
nums_pl = pr.map(lambda x: x + 1, nums)
nums_pl1 = pr.map(lambda x: x ** 2, nums_pl)
nums_pl2 = pr.map(lambda x: -x, nums_pl)
nums_pl = pr.concat([nums_pl1, nums_pl2])
assert sorted(nums_pl) == sorted(nums_py)
def test_map_id(nums):
nums_py = nums
nums_pl = pr.map(lambda x: x, nums)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
def test_map_square_workers(nums):
nums_py = map(lambda x: x ** 2, nums)
nums_py = list(nums_py)
nums_pl = pr.map(lambda x: x ** 2, nums, workers=2)
nums_pl = list(nums_pl)
assert sorted(nums_pl) == sorted(nums_py)
def test_map_id_pipe(nums):
nums_pl = (
nums
| pr.map(lambda x: x)
| list
)
assert nums_pl == nums
def test_map_square(nums):
nums_py = map(lambda x: x ** 2, nums)
nums_py = list(nums_py)
nums_pl = pr.map(lambda x: x ** 2, nums)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
def test_concat_multiple(nums):
nums_py = [ x + 1 for x in nums ]
nums_py1 = nums_py + nums_py
nums_py2 = nums_py1 + nums_py
nums_pl = pr.map(lambda x: x + 1, nums)
nums_pl1 = pr.concat([nums_pl, nums_pl])
nums_pl2 = pr.concat([nums_pl1, nums_pl])
assert sorted(nums_py1) == sorted(list(nums_pl1))
assert sorted(nums_py2) == sorted(list(nums_pl2))
def test_flat_map_square(nums):
def _generator(x):
yield x
yield x + 1
yield x + 2
nums_py = map(lambda x: x ** 2, nums)
nums_py = cz.mapcat(_generator, nums_py)
nums_py = list(nums_py)
nums_pl = pr.map(lambda x: x ** 2, nums)
nums_pl = pr.flat_map(_generator, nums_pl)
nums_pl = list(nums_pl)
assert nums_pl == nums_py
def test_flat_map_square_workers(nums):
def _generator(x):
yield x
yield x + 1
yield x + 2
nums_py = map(lambda x: x ** 2, nums)
nums_py = cz.mapcat(_generator, nums_py)
nums_py = list(nums_py)
nums_pl = pr.map(lambda x: x ** 2, nums)
nums_pl = pr.flat_map(_generator, nums_pl, workers=3)
nums_pl = list(nums_pl)
assert sorted(nums_pl) == sorted(nums_py)
def test_flat_map_square_filter_workers_pipe(nums):
def _generator(x):
yield x
yield x + 1
yield x + 2
nums_py = map(lambda x: x ** 2, nums)
nums_py = cz.mapcat(_generator, nums_py)
nums_py = cz.filter(lambda x: x > 1, nums_py)
nums_py = list(nums_py)
nums_pl = (
nums
| pr.map(lambda x: x ** 2)
| pr.flat_map(_generator, workers=3)
| pr.filter(lambda x: x > 1)
| list
)
assert sorted(nums_pl) == sorted(nums_py)
from pypeln import pr
import time
from random import random
def slow_add1(x):
time.sleep(random()) # <= some slow computation
return x + 1
def slow_gt3(x):
time.sleep(random()) # <= some slow computation
return x > 3
data = range(10) # [0, 1, 2, ..., 9]
stage = pr.map(slow_add1, data, workers = 3, maxsize = 4)
stage = pr.filter(slow_gt3, stage, workers = 2)
data = list(stage) # e.g. [5, 6, 9, 4, 8, 10, 7]