def test_add_method_star_str(self):
'''
Tests if the star wrapper is properly applied to a function if the
star_wrap argument to add_method is an string that specifies the
argument key in kargs.
'''
data_1 = (1, 2), (3, 4), (5, 6)
data_2 = 4, 2, 8, -5
data_3 = 5, 6, 3, 6, 2
Pipe.add_method(
gener = min,
is_valve = True,
star_wrap = 'key',
)
self.assertEqual(
Pipe(data_1).min(key=lambda a, b: 1 / a),
(5, 6)
)
self.assertEqual(
Pipe(data_2).min(),
-5
)
# test function that shouldn't be starred
self.assertEqual(
Pipe(data_3).min(key=lambda a: 1 / a),
max(data_3)
)
def test_init_iter_call_next(self):
'''
test __inti__, __iter__, __call__, __next__.
No extra methods added.
'''
data_1 = 1, 2, 3, 4
data_2 = 5, 6, 7, 8
data_3 = ()
pipe_1 = Pipe()
pipe_1(data_1)
self.assertEqual(tuple(pipe_1), data_1)
self.assertEqual(tuple(pipe_1(data_2)), data_2)
pipe_1(data_1)
for p, d in zip(pipe_1, data_1):
self.assertTrue(p is d)
for p, d in zip(pipe_1(data_2), data_2):
self.assertTrue(p is d)
# preloaded iterable
pipe3 = Pipe(data_1)
self.assertEqual(tuple(pipe3), data_1)
with self.assertRaises(StopIteration):
next(pipe3)
self.assertEqual(tuple(pipe3(data_2)), data_2)
def test_limit_size(self):
data_1 = 1, 2, 3
self.assertEqual(tuple(Pipe(data_1).limit_size(3)), data_1)
with self.assertRaises(ValueError):
tuple(Pipe(data_1).limit_size(2))
def test_wrong_closing_bypass(self):
'''
A bypass that is closed with the wrong closer should raise an TypeError.
'''
Pipe().carry_key.re_key
with self.assertRaises(TypeError):
Pipe().carry_key.re_value
def test_grab_with_tuple(self):
data = (1, 2), (3, 4), (5, 6)
ref = 2, 4, 6
self.assertEqual(Pipe(data).grab[1].tuple(), ref)
pipe_reuse = Pipe().grab[1].tuple()
self.assertEqual(pipe_reuse(data), ref)
self.assertEqual(pipe_reuse(data), ref) # reload the pipe
def test_flatten(self):
data = (1, 2), (3, 4, 5)
ref = 1, 2, 3, 4, 5
self.assertEqual(Pipe(data).flatten().tuple(), ref)
pipe_empty = Pipe().flatten().tuple()
self.assertEqual(pipe_empty(data), ref)
self.assertEqual(pipe_empty(data), ref) # reload the pipe
def test_mul(self):
data_1 = 1, 2, 3, 4
self.assertEqual(tuple(Pipe(data_1).mul(5)),
tuple(5 * val for val in data_1))
pipe_1 = Pipe().mul(5)
self.assertEqual(tuple(pipe_1(data_1)),
tuple(5 * val for val in data_1))
def test_drop_key(self):
data = (1, 2), (3, 4), (5, 6)
ref = 2, 4, 6
self.assertEqual(Pipe(data).drop_key.tuple(), ref)
pipe_reuse = Pipe().drop_key.tuple()
self.assertEqual(pipe_reuse(data), ref)
self.assertEqual(pipe_reuse(data), ref) # reload the pipe
def test_add(self):
data_1 = 1, 2, 3, 4
self.assertEqual(tuple(Pipe(data_1).add(5)),
tuple(5 + val for val in data_1))
pipe_1 = Pipe().add(5)
self.assertEqual(tuple(pipe_1(data_1)),
tuple(5 + val for val in data_1))
def test_keyed(self):
data = 1, 2, 3, 4
ref = tuple((val, 2 * val) for val in data)
self.assertEqual(
Pipe(data).keyed.map(lambda val: 2 * val).tuple(), ref)
pipe_def = Pipe().keyed.map(lambda val: 2 * val).tuple()
self.assertEqual(pipe_def(data), ref)
self.assertEqual(pipe_def(data), ref) # not a reference
def test_zip_to_dict(self):
data_1 = ('a', (1, 2)), ('b', (3, 4))
result_1 = {'a': 1, 'b': 3}, {'a': 2, 'b': 4}
self.assertEqual(tuple(Pipe(data_1).zip_to_dict()), result_1)
self.assertEqual(Pipe(data_1).zip_to_dict().tuple(), result_1)
pipe_1 = Pipe().zip_to_dict().limit_size(2).tuple()
self.assertEqual(pipe_1(data_1), result_1)
def test_carry_key_no_size_change(self):
data_1 = (1, 2), (3, 4), (5, 6)
result_1 = (1, 4), (3, 8), (5, 12)
self.assertEqual(
tuple(Pipe(data_1).carry_key.map(lambda b: 2 * b).re_key),
result_1)
pipe_1 = Pipe().carry_key.map(lambda b: 2 * b).re_key
self.assertEqual(tuple(pipe_1(data_1)), result_1)
self.assertEqual(tuple(pipe_1(data_1)), result_1) # not a repeat
def test_fromiter(self):
data_1 = 1, 2, 3
self.assertTrue(np.array_equal(
Pipe(data_1).fromiter(float),
np.array(data_1)
))
pipe_1 = Pipe().fromiter(float)
self.assertTrue(np.array_equal(
pipe_1(data_1),
np.array(data_1)
))
def test_zip_internal(self):
data_1 = (1, 2, 3), (4, 5, 6), (7, 8, 9)
data_1_zipped = tuple(zip(*data_1))
data_2 = ()
self.assertEqual(tuple(Pipe(data_1).zip_internal()), data_1_zipped)
pipe_1 = Pipe().zip_internal()
self.assertEqual(tuple(pipe_1(data_1)), data_1_zipped)
self.assertEqual(tuple(pipe_1(data_1)),
data_1_zipped) # reload the pipe
self.assertEqual(tuple(Pipe(data_2).zip_internal()), ())
def test_map_map_kargs(self):
data_1 = 1, 2, 3, 4
func_1 = lambda val: val > 2
data_2 = (1, 2), (3, 4), (5, 6), (7, 8)
func_2 = lambda a, b: 2 * a > b and b < 8
data_3 = tuple(dict(a=v1, b=v2) for v1, v2 in data_2)
func_3 = func_2
self.assertEqual(tuple(Pipe(data_1).map(func_1)),
tuple(map(func_1, data_1)))
self.assertEqual(tuple(Pipe(data_2).map(func_2)),
tuple(map(lambda pair: func_2(*pair), data_2)))
self.assertEqual(tuple(Pipe(data_3).map_kargs(func_3)),
tuple(map(lambda kargs: func_3(**kargs), data_3)))
def test_grab_with_dict(self):
data = (1, 2), (3, 4), (5, 6)
data = (dict(a=a, b=b) for a, b in data)
ref = 2, 4, 6
print()
print(Pipe().grab)
print()
return
self.assertEqual(Pipe(data).grab['b'].tuple(), ref)
pipe_reuse = Pipe().grab['b'].tuple()
self.assertEqual(pipe_reuse(data), ref)
self.assertEqual(pipe_reuse(data), ref) # reload the pipe
def test_carry_key_shrink(self):
data_1 = (1, 2), (3, 4), (5, 6)
filter_1 = lambda val: val != 4
result_1 = (1, 2), (5, 6)
self.assertEqual(tuple(Pipe(data_1).carry_key.filter(filter_1).re_key),
result_1)
def test_pass_key(self):
data = dict(a=1, b=2, c=3), dict(a=4, b=5, c=6), dict(a=7, b=8, c=9)
self.assertEqual(
Pipe(data).carry_dict['b'].map(
lambda val: 2 * val).return_dict.tuple(),
(dict(a=1, b=4, c=3), dict(a=4, b=10, c=6), dict(a=7, b=16, c=9)))
def test_pipe_valve_iterable(self):
data_1 = [(1, 2), (3, 4), (5, 6)]
Pipe.add_method(gener=list, is_valve=True)
self.assertEqual(
Pipe(data_1).list(),
list(data_1)
)
pipe_1 = Pipe().list()
self.assertEqual(
pipe_1(data_1),
list(data_1)
)
def test_groupby_no_key(self):
data = 1, 2, 3, 1, 2, 3, 4
to_tuple = lambda groups: tuple((key, tuple(group)) for key, group in groups)
self.assertEqual(
to_tuple(Pipe(data).groupby()),
to_tuple(it.groupby(data))
)
def test_add_method_as_property(self):
data = 1, 2, 3, 4
Pipe.add_method(enumerate, as_property=True)
self.assertEqual(
tuple(Pipe(data).enumerate),
tuple(enumerate(data))
)
def test_pipe_valve_non_iterable(self):
data_1 = 4, 2, 8, -5
data_1_min = min(data_1)
data_2 = ()
# test method that directly passes the data through
Pipe.add_map_method(lambda val: val, 'pass_through')
# min
Pipe.add_method(min, is_valve=True, empty_error=ValueError)
'''
The Pipe has been preloaded and a valve has been added to the pipe.
When the preloaded and a valve is added the reservoir should be drained
by the pipe into the valve function and the result should be returned.
'''
self.assertEqual(
Pipe(data_1).min(),
data_1_min
)
self.assertEqual(
Pipe(data_1).pass_through().min(),
data_1_min
)
'''
Pipe has not been preloaded and a Pipe should be returned.
'''
pipe_1 = Pipe().min()
self.assertEqual(pipe_1(data_1), data_1_min)
self.assertEqual(pipe_1(data_1), data_1_min) # not a repeat
pipe_2 = Pipe().pass_through().min()
self.assertEqual(pipe_2(data_1), data_1_min)
self.assertEqual(pipe_2(data_1), data_1_min) # not a repeat
pip_3 = Pipe().min().pass_through()
self.assertEqual(next(pip_3(data_1)), data_1_min)
pip_4 = Pipe().min().min()
self.assertEqual(pipe_1(data_1), data_1_min)
pipe_5 = pipe_1.pass_through()
self.assertEqual(
next(pipe_5(data_1)),
next(pip_3(data_1))
)
pipe_6 = pipe_1.min()
self.assertEqual(
pipe_6(data_1),
pip_4(data_1)
)
# empty preloaded iterator
with self.assertRaises(ValueError):
print(Pipe(data_2).min())
def test_groupby_with_key(self):
data = 1, 2, 3, 1, 2, 3, 4
key = lambda val: val % 2
to_tuple = lambda groups: tuple((key, tuple(group)) for key, group in groups)
ref = to_tuple(it.groupby(data, key))
self.assertEqual(
to_tuple(Pipe(data).groupby(key)),
ref
)
pipe_reuse = Pipe().groupby(key)
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
)
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
) # reload pipe
def test_groupby_key(self):
data = (1, 2), (3, 4), (1, 5)
key = lambda key_val: key_val[0]
to_tuple = lambda groups: tuple((key, tuple(group)) for key, group in groups)
ref = to_tuple(it.groupby(data, key))
self.assertEqual(
to_tuple(Pipe(data).groupby_key()),
ref
)
pipe_reuse = Pipe().groupby_key()
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
)
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
) # reload pipe
def test_add_method_double_star_str(self):
data_1 = tuple(dict(a=a, b=b) for a, b in ((1, 2), (3, 4), (5, 6)))
def min_key_arg(iterable, key):
return min(iterable, key=key)
Pipe.add_method(
gener = min_key_arg,
is_valve = True,
double_star_wrap = 'key',
)
self.assertTrue(hasattr(Pipe, 'min_key_arg'))
self.assertEqual(
Pipe(data_1).min_key_arg(key=lambda a, b: 1 / a),
dict(a=5, b=6)
)
def test_carry_value(self):
# no_size_change
data_1 = (1, 2), (3, 4), (5, 6)
result_1 = (2, 2), (6, 4), (10, 6)
self.assertTrue(hasattr(Pipe, 'carry_key'))
self.assertTrue(hasattr(Pipe, 'carry_value'))
self.assertEqual(
tuple(Pipe(data_1).carry_value.map(lambda a: 2 * a).re_value),
result_1)
# shrink
data_2 = (1, 2), (3, 4), (5, 6)
filter_2 = lambda val: val != 3
result_2 = (1, 2), (5, 6)
self.assertEqual(
tuple(Pipe(data_2).carry_value.filter(filter_2).re_value),
result_2)
# expand
data_3 = (1, 2), (3, 4)
result_3 = (0, 2), (1, 2), (0, 4), (1, 4)
self.assertEqual(tuple(Pipe(data_3).carry_value.Expand().re_value),
result_3)
def test_add_map_method(self):
data_1 = 1, 2, 7, 9
data_2 = (1, 2), (3, 4)
data_3 = dict(a=1, b=2), dict(a=3, b=4)
Pipe.add_map_method(lambda a: a**2, 'square')
self.assertEqual(
tuple(Pipe(data_1).square()),
tuple(d**2 for d in data_1)
)
# method call includes a value
Pipe.add_map_method(min)
self.assertEqual(
tuple(Pipe(data_1).min(5, key=lambda a: 1 / a)),
(5, 5, 7, 9)
)
self.assertEqual(
tuple(Pipe(data_2).min()),
(1, 3)
)
self.assertEqual(
tuple(Pipe(data_2).min(key=lambda a: 1 / a)),
(2, 4)
)
# single star
Pipe.add_map_method(lambda a, b: a + b, 'add', star_wrap=True)
self.assertEqual(
tuple(Pipe(data_2).add()),
(3, 7)
)
# double star
Pipe.add_map_method(lambda a, b: a + b, 'add', double_star_wrap=True, no_over_write=False)
self.assertEqual(
tuple(Pipe(data_3).add()),
(3, 7)
)
def test_init_next_iter(self):
Pipe.add_map_method(lambda a: a**2, 'square')
data_1 = (1, 2), (3, 4), (5, 6)
drip_1 = Drip()
res_1 = Reservoir(data_1)
pipe_1 = Pipe(reservoir=drip_1).square()
result_1 = tuple((a, b**2) for a, b in data_1)
def carry_key(key_val):
return key_val[0], key_val[1]
def re_key(key, bypass_val):
return key, bypass_val
bpp = Bypass(
bypass=pipe_1,
iterable=res_1,
drip_handle=drip_1,
split=carry_key,
merge=re_key,
)
self.assertEqual(tuple(bpp), result_1)
def test_add_method(self):
# enumerate
Pipe.add_method(enumerate)
self.assertTrue(hasattr(Pipe, 'enumerate'))
self.assertTrue(isinstance(Pipe.enumerate, types.FunctionType))
self.assertTrue(isinstance(Pipe().enumerate, types.MethodType))
with self.assertRaises(AttributeError):
Pipe.add_method(enumerate)
with self.assertRaises(AttributeError):
Pipe.add_method(map, name='enumerate')
# no_over_write is False
Pipe.add_method(enumerate, no_over_write=False)
self.assertTrue(hasattr(Pipe, 'enumerate'))
self.assertTrue(isinstance(Pipe.enumerate, types.FunctionType))
self.assertTrue(isinstance(Pipe().enumerate, types.MethodType))
Pipe.add_method(enumerate, 'enumerate', no_over_write=False)
self.assertTrue(hasattr(Pipe, 'enumerate'))
self.assertTrue(isinstance(Pipe.enumerate, types.FunctionType))
self.assertTrue(isinstance(Pipe().enumerate, types.MethodType))
# test added method
data = 1, 2, 3, 4
self.assertEqual(
tuple(Pipe(data).enumerate()),
tuple(enumerate(data))
)
self.assertEqual(
tuple(Pipe(data).enumerate(2)),
tuple(enumerate(data, 2))
)
self.assertEqual(
tuple(Pipe(data).enumerate(start=2)),
tuple(enumerate(data, start=2))
)
from functional_pipes import Pipe
Pipe.load('built_in_functions', 'operator_pipes')
'''
Very simple example of a map reduce.
This apples the absolute value to each of the numbers and then reduces to the
maximum value.
'''
data = 1, -2, -3
absolute_max = Pipe(data).abs().max()
print(absolute_max)
'''
Reusable Piping
A pipe can be created without any data preloaded and be run multiple
times without being rebuilt each time. This is useful because it means you can put
the prebuilt pipe into a loop without the overhead of rebuilding it for every
iteration.
One system that this is good for is outline below:
1.) generates data
2.) runs the data through the pipe
3.) makes changes to the data based on the result from the pipe
4.) rerun the changed data through the pipe
5.) repeat steps
The example below shows an implimentation of this system:
'''
data = (20, 2), (50, 9), (100, 7), (2, 5)
reusable_pipe = Pipe().map(
lambda a, b: (20 * a, a * b) # notice how the tuple is split automatically