def test_add_method_star_str(self):
'''
Tests if the star wrapper is properly applied to a function if the
star_wrap argument to add_method is an string that specifies the
argument key in kargs.
'''
data_1 = (1, 2), (3, 4), (5, 6)
data_2 = 4, 2, 8, -5
data_3 = 5, 6, 3, 6, 2
Pipe.add_method(
gener = min,
is_valve = True,
star_wrap = 'key',
)
self.assertEqual(
Pipe(data_1).min(key=lambda a, b: 1 / a),
(5, 6)
)
self.assertEqual(
Pipe(data_2).min(),
-5
)
# test function that shouldn't be starred
self.assertEqual(
Pipe(data_3).min(key=lambda a: 1 / a),
max(data_3)
)
def test_init_iter_call_next(self):
'''
test __inti__, __iter__, __call__, __next__.
No extra methods added.
'''
data_1 = 1, 2, 3, 4
data_2 = 5, 6, 7, 8
data_3 = ()
pipe_1 = Pipe()
pipe_1(data_1)
self.assertEqual(tuple(pipe_1), data_1)
self.assertEqual(tuple(pipe_1(data_2)), data_2)
pipe_1(data_1)
for p, d in zip(pipe_1, data_1):
self.assertTrue(p is d)
for p, d in zip(pipe_1(data_2), data_2):
self.assertTrue(p is d)
# preloaded iterable
pipe3 = Pipe(data_1)
self.assertEqual(tuple(pipe3), data_1)
with self.assertRaises(StopIteration):
next(pipe3)
self.assertEqual(tuple(pipe3(data_2)), data_2)
def test_limit_size(self):
data_1 = 1, 2, 3
self.assertEqual(tuple(Pipe(data_1).limit_size(3)), data_1)
with self.assertRaises(ValueError):
tuple(Pipe(data_1).limit_size(2))
def test_wrong_closing_bypass(self):
'''
A bypass that is closed with the wrong closer should raise an TypeError.
'''
Pipe().carry_key.re_key
with self.assertRaises(TypeError):
Pipe().carry_key.re_value
def test_drop_key(self):
data = (1, 2), (3, 4), (5, 6)
ref = 2, 4, 6
self.assertEqual(Pipe(data).drop_key.tuple(), ref)
pipe_reuse = Pipe().drop_key.tuple()
self.assertEqual(pipe_reuse(data), ref)
self.assertEqual(pipe_reuse(data), ref) # reload the pipe
def test_mul(self):
data_1 = 1, 2, 3, 4
self.assertEqual(tuple(Pipe(data_1).mul(5)),
tuple(5 * val for val in data_1))
pipe_1 = Pipe().mul(5)
self.assertEqual(tuple(pipe_1(data_1)),
tuple(5 * val for val in data_1))
def test_grab_with_tuple(self):
data = (1, 2), (3, 4), (5, 6)
ref = 2, 4, 6
self.assertEqual(Pipe(data).grab[1].tuple(), ref)
pipe_reuse = Pipe().grab[1].tuple()
self.assertEqual(pipe_reuse(data), ref)
self.assertEqual(pipe_reuse(data), ref) # reload the pipe
def test_add(self):
data_1 = 1, 2, 3, 4
self.assertEqual(tuple(Pipe(data_1).add(5)),
tuple(5 + val for val in data_1))
pipe_1 = Pipe().add(5)
self.assertEqual(tuple(pipe_1(data_1)),
tuple(5 + val for val in data_1))
def test_flatten(self):
data = (1, 2), (3, 4, 5)
ref = 1, 2, 3, 4, 5
self.assertEqual(Pipe(data).flatten().tuple(), ref)
pipe_empty = Pipe().flatten().tuple()
self.assertEqual(pipe_empty(data), ref)
self.assertEqual(pipe_empty(data), ref) # reload the pipe
def test_add_method_as_property(self):
data = 1, 2, 3, 4
Pipe.add_method(enumerate, as_property=True)
self.assertEqual(
tuple(Pipe(data).enumerate),
tuple(enumerate(data))
)
def test_keyed(self):
data = 1, 2, 3, 4
ref = tuple((val, 2 * val) for val in data)
self.assertEqual(
Pipe(data).keyed.map(lambda val: 2 * val).tuple(), ref)
pipe_def = Pipe().keyed.map(lambda val: 2 * val).tuple()
self.assertEqual(pipe_def(data), ref)
self.assertEqual(pipe_def(data), ref) # not a reference
def test_zip_to_dict(self):
data_1 = ('a', (1, 2)), ('b', (3, 4))
result_1 = {'a': 1, 'b': 3}, {'a': 2, 'b': 4}
self.assertEqual(tuple(Pipe(data_1).zip_to_dict()), result_1)
self.assertEqual(Pipe(data_1).zip_to_dict().tuple(), result_1)
pipe_1 = Pipe().zip_to_dict().limit_size(2).tuple()
self.assertEqual(pipe_1(data_1), result_1)
def test_carry_key_no_size_change(self):
data_1 = (1, 2), (3, 4), (5, 6)
result_1 = (1, 4), (3, 8), (5, 12)
self.assertEqual(
tuple(Pipe(data_1).carry_key.map(lambda b: 2 * b).re_key),
result_1)
pipe_1 = Pipe().carry_key.map(lambda b: 2 * b).re_key
self.assertEqual(tuple(pipe_1(data_1)), result_1)
self.assertEqual(tuple(pipe_1(data_1)), result_1) # not a repeat
def test_zip_internal(self):
data_1 = (1, 2, 3), (4, 5, 6), (7, 8, 9)
data_1_zipped = tuple(zip(*data_1))
data_2 = ()
self.assertEqual(tuple(Pipe(data_1).zip_internal()), data_1_zipped)
pipe_1 = Pipe().zip_internal()
self.assertEqual(tuple(pipe_1(data_1)), data_1_zipped)
self.assertEqual(tuple(pipe_1(data_1)),
data_1_zipped) # reload the pipe
self.assertEqual(tuple(Pipe(data_2).zip_internal()), ())
def test_fromiter(self):
data_1 = 1, 2, 3
self.assertTrue(np.array_equal(
Pipe(data_1).fromiter(float),
np.array(data_1)
))
pipe_1 = Pipe().fromiter(float)
self.assertTrue(np.array_equal(
pipe_1(data_1),
np.array(data_1)
))
def test_map_map_kargs(self):
data_1 = 1, 2, 3, 4
func_1 = lambda val: val > 2
data_2 = (1, 2), (3, 4), (5, 6), (7, 8)
func_2 = lambda a, b: 2 * a > b and b < 8
data_3 = tuple(dict(a=v1, b=v2) for v1, v2 in data_2)
func_3 = func_2
self.assertEqual(tuple(Pipe(data_1).map(func_1)),
tuple(map(func_1, data_1)))
self.assertEqual(tuple(Pipe(data_2).map(func_2)),
tuple(map(lambda pair: func_2(*pair), data_2)))
self.assertEqual(tuple(Pipe(data_3).map_kargs(func_3)),
tuple(map(lambda kargs: func_3(**kargs), data_3)))
def test_grab_with_dict(self):
data = (1, 2), (3, 4), (5, 6)
data = (dict(a=a, b=b) for a, b in data)
ref = 2, 4, 6
print()
print(Pipe().grab)
print()
return
self.assertEqual(Pipe(data).grab['b'].tuple(), ref)
pipe_reuse = Pipe().grab['b'].tuple()
self.assertEqual(pipe_reuse(data), ref)
self.assertEqual(pipe_reuse(data), ref) # reload the pipe
def test_add_method_double_star_str(self):
data_1 = tuple(dict(a=a, b=b) for a, b in ((1, 2), (3, 4), (5, 6)))
def min_key_arg(iterable, key):
return min(iterable, key=key)
Pipe.add_method(
gener = min_key_arg,
is_valve = True,
double_star_wrap = 'key',
)
self.assertTrue(hasattr(Pipe, 'min_key_arg'))
self.assertEqual(
Pipe(data_1).min_key_arg(key=lambda a, b: 1 / a),
dict(a=5, b=6)
)
def test_pass_key(self):
data = dict(a=1, b=2, c=3), dict(a=4, b=5, c=6), dict(a=7, b=8, c=9)
self.assertEqual(
Pipe(data).carry_dict['b'].map(
lambda val: 2 * val).return_dict.tuple(),
(dict(a=1, b=4, c=3), dict(a=4, b=10, c=6), dict(a=7, b=16, c=9)))
def test_carry_key_shrink(self):
data_1 = (1, 2), (3, 4), (5, 6)
filter_1 = lambda val: val != 4
result_1 = (1, 2), (5, 6)
self.assertEqual(tuple(Pipe(data_1).carry_key.filter(filter_1).re_key),
result_1)
def test_pipe_valve_iterable(self):
data_1 = [(1, 2), (3, 4), (5, 6)]
Pipe.add_method(gener=list, is_valve=True)
self.assertEqual(
Pipe(data_1).list(),
list(data_1)
)
pipe_1 = Pipe().list()
self.assertEqual(
pipe_1(data_1),
list(data_1)
)
def test_groupby_no_key(self):
data = 1, 2, 3, 1, 2, 3, 4
to_tuple = lambda groups: tuple((key, tuple(group)) for key, group in groups)
self.assertEqual(
to_tuple(Pipe(data).groupby()),
to_tuple(it.groupby(data))
)
def test_init_next_iter(self):
Pipe.add_map_method(lambda a: a**2, 'square')
data_1 = (1, 2), (3, 4), (5, 6)
drip_1 = Drip()
res_1 = Reservoir(data_1)
pipe_1 = Pipe(reservoir=drip_1).square()
result_1 = tuple((a, b**2) for a, b in data_1)
def carry_key(key_val):
return key_val[0], key_val[1]
def re_key(key, bypass_val):
return key, bypass_val
bpp = Bypass(
bypass=pipe_1,
iterable=res_1,
drip_handle=drip_1,
split=carry_key,
merge=re_key,
)
self.assertEqual(tuple(bpp), result_1)
def test_groupby_key(self):
data = (1, 2), (3, 4), (1, 5)
key = lambda key_val: key_val[0]
to_tuple = lambda groups: tuple((key, tuple(group)) for key, group in groups)
ref = to_tuple(it.groupby(data, key))
self.assertEqual(
to_tuple(Pipe(data).groupby_key()),
ref
)
pipe_reuse = Pipe().groupby_key()
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
)
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
) # reload pipe
def test_carry_value(self):
# no_size_change
data_1 = (1, 2), (3, 4), (5, 6)
result_1 = (2, 2), (6, 4), (10, 6)
self.assertTrue(hasattr(Pipe, 'carry_key'))
self.assertTrue(hasattr(Pipe, 'carry_value'))
self.assertEqual(
tuple(Pipe(data_1).carry_value.map(lambda a: 2 * a).re_value),
result_1)
# shrink
data_2 = (1, 2), (3, 4), (5, 6)
filter_2 = lambda val: val != 3
result_2 = (1, 2), (5, 6)
self.assertEqual(
tuple(Pipe(data_2).carry_value.filter(filter_2).re_value),
result_2)
# expand
data_3 = (1, 2), (3, 4)
result_3 = (0, 2), (1, 2), (0, 4), (1, 4)
self.assertEqual(tuple(Pipe(data_3).carry_value.Expand().re_value),
result_3)
def test_groupby_with_key(self):
data = 1, 2, 3, 1, 2, 3, 4
key = lambda val: val % 2
to_tuple = lambda groups: tuple((key, tuple(group)) for key, group in groups)
ref = to_tuple(it.groupby(data, key))
self.assertEqual(
to_tuple(Pipe(data).groupby(key)),
ref
)
pipe_reuse = Pipe().groupby(key)
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
)
self.assertEqual(
to_tuple(pipe_reuse(data)),
ref
) # reload pipe
def test_load_unload(self):
Pipe.load('built_in_functions', 'itertools_pipes')
def get_name(method):
'''
returns the name that the method has been added to Pipe as
'''
if isinstance(method, dict):
if 'name' in method:
name = method['name']
elif 'gener' in method:
name = method['gener'].__name__
elif 'func' in method:
name = method['func'].__name__
else:
raise ValueError('method name cannot be found.')
else:
name = method.__name__
return name
methods = tuple(
get_name(method)
for add_in in (built_in_functions, itertools_pipes)
for collection in ('methods_to_add', 'map_methods_to_add')
for method in getattr(add_in, collection))
self.assertTrue(all(hasattr(Pipe, name) for name in methods))
# check unloading
with self.assertRaises(KeyError):
Pipe.unload('not an add-in')
Pipe.unload('built_in_functions', 'itertools_pipes')
self.assertFalse(any(hasattr(Pipe, name) for name in methods))
self.assertFalse(hasattr(Pipe, 'dict'))
def setUpClass(self):
Pipe.load('built_in_functions')
from functional_pipes import Pipe
Pipe.load('built_in_functions', 'operator_pipes')
'''
Very simple example of a map reduce.
This apples the absolute value to each of the numbers and then reduces to the
maximum value.
'''
data = 1, -2, -3
absolute_max = Pipe(data).abs().max()
print(absolute_max)
'''
Reusable Piping
A pipe can be created without any data preloaded and be run multiple
times without being rebuilt each time. This is useful because it means you can put
the prebuilt pipe into a loop without the overhead of rebuilding it for every
iteration.
One system that this is good for is outline below:
1.) generates data
2.) runs the data through the pipe
3.) makes changes to the data based on the result from the pipe
4.) rerun the changed data through the pipe
5.) repeat steps
The example below shows an implimentation of this system:
'''
data = (20, 2), (50, 9), (100, 7), (2, 5)
reusable_pipe = Pipe().map(
lambda a, b: (20 * a, a * b) # notice how the tuple is split automatically
def tearDownClass(self):
Pipe.unload('built_in_functions')