def test_flowly_kv_transform__map(executor):
actual = executor(
kv_transform(map(lambda i: 10 * i)),
[(i % 2, i) for i in range(20)],
npartitions=10,
)
assert sorted(actual) == sorted([(i % 2, 10 * i) for i in range(20)])
def test_kv_transform__aggregate():
actual = pipe(
[(0, {
'v': 1
}), (1, {
'v': 2
}), (0, {
'v': 3
}), (1, {
'v': 4
}), (0, {
'v': 5
})],
kv_transform(aggregate({'v': [min, max, sum]})),
)
expected = [
(0, {
('v', 'max'): 5,
('v', 'min'): 1,
('v', 'sum'): 9
}),
(1, {
('v', 'max'): 4,
('v', 'min'): 2,
('v', 'sum'): 6
}),
]
assert actual == expected
def test_flowly_kv_transform__mapcat(executor):
actual = executor(
kv_transform(mapcat(lambda i: [10 * i, 20 * i])),
[(i % 2, i) for i in range(20)],
npartitions=10,
)
assert sorted(actual) == sorted(
it.chain([(i % 2, 10 * i) for i in range(20)],
[(i % 2, 20 * i) for i in range(20)]))
def test_flowly_kv_transform__concat(executor):
actual = executor(
kv_transform(concat),
[(i % 2, [1 * i, 2 * i, 3 * i]) for i in [1, 2, 3, 4, 5, 6, 7]],
npartitions=3,
)
assert sorted(actual) == sorted([(i % 2, factor * i)
for i in [1, 2, 3, 4, 5, 6, 7]
for factor in [1, 2, 3]])
def test_flowly_kv_transform__reduction(executor):
actual = executor(
kv_transform(reduction(None, sum)),
[(i % 2, i) for i in [1, 2, 3, 4, 5, 6, 7]],
npartitions=3,
)
assert sorted(actual) == sorted([
(1, sum([1, 3, 5, 7])),
(0, sum([2, 4, 6])),
])
def test_kv_transform__aggregate():
actual = pipe(
[(0, {'v': 1}), (1, {'v': 2}), (0, {'v': 3}), (1, {'v': 4}), (0, {'v': 5})],
kv_transform(aggregate({'v': [min, max, sum]})),
)
expected = [
(0, {('v', 'max'): 5, ('v', 'min'): 1, ('v', 'sum'): 9}),
(1, {('v', 'max'): 4, ('v', 'min'): 2, ('v', 'sum'): 6}),
]
assert actual == expected
def test_flowly_kv_transform__unknown_function():
with pytest.raises(ValueError):
kv_transform(lambda foo: foo)