def test_flowly_kv_transform__map(executor): actual = executor( kv_transform(map(lambda i: 10 * i)), [(i % 2, i) for i in range(20)], npartitions=10, ) assert sorted(actual) == sorted([(i % 2, 10 * i) for i in range(20)])
def test_kv_transform__aggregate(): actual = pipe( [(0, { 'v': 1 }), (1, { 'v': 2 }), (0, { 'v': 3 }), (1, { 'v': 4 }), (0, { 'v': 5 })], kv_transform(aggregate({'v': [min, max, sum]})), ) expected = [ (0, { ('v', 'max'): 5, ('v', 'min'): 1, ('v', 'sum'): 9 }), (1, { ('v', 'max'): 4, ('v', 'min'): 2, ('v', 'sum'): 6 }), ] assert actual == expected
def test_flowly_kv_transform__mapcat(executor): actual = executor( kv_transform(mapcat(lambda i: [10 * i, 20 * i])), [(i % 2, i) for i in range(20)], npartitions=10, ) assert sorted(actual) == sorted( it.chain([(i % 2, 10 * i) for i in range(20)], [(i % 2, 20 * i) for i in range(20)]))
def test_flowly_kv_transform__concat(executor): actual = executor( kv_transform(concat), [(i % 2, [1 * i, 2 * i, 3 * i]) for i in [1, 2, 3, 4, 5, 6, 7]], npartitions=3, ) assert sorted(actual) == sorted([(i % 2, factor * i) for i in [1, 2, 3, 4, 5, 6, 7] for factor in [1, 2, 3]])
def test_flowly_kv_transform__reduction(executor): actual = executor( kv_transform(reduction(None, sum)), [(i % 2, i) for i in [1, 2, 3, 4, 5, 6, 7]], npartitions=3, ) assert sorted(actual) == sorted([ (1, sum([1, 3, 5, 7])), (0, sum([2, 4, 6])), ])
def test_kv_transform__aggregate(): actual = pipe( [(0, {'v': 1}), (1, {'v': 2}), (0, {'v': 3}), (1, {'v': 4}), (0, {'v': 5})], kv_transform(aggregate({'v': [min, max, sum]})), ) expected = [ (0, {('v', 'max'): 5, ('v', 'min'): 1, ('v', 'sum'): 9}), (1, {('v', 'max'): 4, ('v', 'min'): 2, ('v', 'sum'): 6}), ] assert actual == expected
def test_flowly_kv_transform__unknown_function(): with pytest.raises(ValueError): kv_transform(lambda foo: foo)