def test_cannot_store_twice(self):
@fpn.pipe_node()
def pn():
return 12
pni = fpn.PipeNodeInput()
with self.assertRaises(KeyError):
(pn | fpn.store("a") | fpn.store("a"))[pni]
def approach_pipe_4b():
a = (PN4.name_count_per_year(lambda n: n.lower().startswith('lesl'))
| PN4.percent | fpn.store('lesl'))
b = (PN4.name_count_per_year(lambda n: n.lower().startswith('dana'))
| PN4.percent | fpn.store('dana'))
f = (PN4.merge_gender_data(lesl=a, dana=b)
| PN4.year_range(1920, 2000)
| fpn.store('merged') * 100
| PN4.plot('gender.png')
| PN4.open_plot)
pni = PN4.PNI('/tmp')
f[pni]
xlsx_fp = os.path.join(pni.output_dir, 'output.xlsx')
xlsx = pd.ExcelWriter(xlsx_fp)
for k, df in pni.store_items():
df.to_excel(xlsx, k)
xlsx.save()
os.system('libreoffice --calc ' + xlsx_fp)
def approach_pipe_2():
f = (PN2.load_data_dict(FP_ZIP) | PN2.gender_count_per_year
| PN2.percent * 100 | PN2.year_range(1900, 2000)
| PN2.plot | PN2.open_plot)
# with store and recall to do multiple operations
f = (PN2.load_data_dict(FP_ZIP) | PN2.gender_count_per_year
| PN2.percent * 100
| fpn.store('gpcent')
| PN2.year_range(1900, 2000) | PN2.plot | PN2.open_plot
| fpn.recall('gpcent')
| PN2.year_range(2001, 2015) | PN2.plot | PN2.open_plot)
#fpn.run(f) # this implicitly passes a PipeNodeInput
f(pn_input=fpn.PipeNodeInput())
def pn():
# if we are willing to adjust our functions about, we can do even more with PipeNodes. PipeNodes have a protocal; they have to be called in a certain way, and have certain expectations regarding arguments. PipeNodes are created through decorators. Decorated functions receive PipeNode qwargs
a = fpn.pipe_node(lambda **kwargs: kwargs[fpn.PREDECESSOR_RETURN] + 'a')
# another way of doing the same thing
@fpn.pipe_node
@fpn.pipe_kwarg_bind(fpn.PREDECESSOR_RETURN)
def a(s):
return s + 'a'
# capturing the initial input is not automatic
init = fpn.pipe_node(lambda **kwargs: kwargs[fpn.PN_INPUT])
@fpn.pipe_node_factory
def cat(chars, **kwargs):
return kwargs[fpn.PREDECESSOR_RETURN] + chars
f = init | a | cat('b') | cat('c')
print(f(pn_input='*'))
assert f(pn_input='*') == '*abc'
# can also be done as the following
print(f(**{fpn.PN_INPUT: '*'}))
assert f(**{fpn.PN_INPUT: '+'}) == '+abc'
print(f['*'])
assert f['*'] == '*abc'
# with pipenodes, all nodes have access to the PN_INPUT through the common kwargs
@fpn.pipe_node_factory
def replace_init(chars, **kwargs):
return kwargs[fpn.PREDECESSOR_RETURN].replace(kwargs[fpn.PN_INPUT],
chars)
f = init | a | cat('b') | cat('c') * 2 | replace_init('+')
print(f['*'])
assert f['*'] == '+abc+abc'
# as already shown pipe-nodes can take arguments; those arguments can be pipe nodes themselves
@fpn.pipe_node_factory
def interleave(chars, **kwargs):
pred = kwargs[fpn.PREDECESSOR_RETURN]
post = []
for i, c in enumerate(pred):
post.append(c)
post.append(chars[i % len(chars)])
return ''.join(post)
h = init | cat('@@') | cat('__') * 2
f = init | a | cat('b') | cat('c') * 3 | replace_init('+') | interleave(h)
print(f['*'])
assert f['*'] == '+*a@b@c_+_a*b@c@+_a_b*c@'
# if we want to use a PN expression more than once, we can store it and recall it later. To do so, we need to use a PipeNodeInput or itse subclass
class Input(fpn.PipeNodeInput):
def __init__(self, chars):
super().__init__()
self.chars = chars
# we nee dot change our init function to read the chars attr
input_init = fpn.pipe_node(lambda **kwargs: kwargs[fpn.PN_INPUT].chars)
p = input_init | cat('www') | fpn.store('p')
q = input_init | cat('@@') | cat('__') * 2 | fpn.store('q')
r = input_init | a | cat(fpn.recall('p')) | cat('c') * 3 | interleave(
fpn.recall('q'))
f = fpn.call(p, q, r)
pni = Input('x')
print(f[pni])
pni = Input('x') # must create a new one
assert f[pni] == 'xxa@x@w_w_wxc@x@a_x_wxw@w@c_x_axx@w@w_w_cx'
def test_complex_pipeline(self):
@fpn.pipe_node_factory(fpn.PREDECESSOR_RETURN)
def multiply(prev, val):
return prev * val
@fpn.pipe_node(fpn.PN_INPUT, fpn.PREDECESSOR_RETURN)
def add_pni(pni, prev):
return prev + pni.value
@fpn.pipe_node
def init(**kwargs):
return 12
@fpn.pipe_node_factory
def func(arg1, arg2, **kwargs):
return (kwargs[fpn.PN_INPUT].value *
arg1) + (kwargs[fpn.PREDECESSOR_RETURN] / arg2)
class Operations:
DELTA = 0.23
def __init__(self, factor):
self.factor = factor
@fpn.classmethod_pipe_node
def add_delta(cls, **kwargs):
return cls.DELTA + kwargs[fpn.PREDECESSOR_RETURN]
@fpn.staticmethod_pipe_node_factory(fpn.PREDECESSOR_RETURN)
def bound_prev(prev, lower, upper):
return max(min(prev, lower), upper)
@fpn.pipe_node(fpn.PN_INPUT, fpn.PREDECESSOR_RETURN)
def adj_by_factor(self, pni, prev):
return self.factor * (pni.value - prev)
op = Operations(23.717)
expr = (init
| add_pni
| fpn.store("A")
| multiply(-8)
| func(13, 7)
| fpn.store("B")
| op.add_delta
| Operations.add_delta
| op.bound_prev(0, 100)
| fpn.store("C")
| multiply(130.2)
| Operations.bound_prev(100, 200)
| op.adj_by_factor
| fpn.store("D")
| fpn.recall("A")
| op.adj_by_factor
| fpn.recall("B")
| op.adj_by_factor
| fpn.recall("C")
| op.adj_by_factor
| add_pni)
class PNI(fpn.PipeNodeInput):
def __init__(self, value):
super().__init__()
self.value = value
pni = PNI(-89)
post = expr[pni]
# Manually calculate
expected = 12
expected += pni.value
A = expected
expected *= -8
expected = (pni.value * 13) + (expected / 7)
B = expected
expected += op.DELTA
expected += Operations.DELTA
expected = max(min(expected, 0), 100)
C = expected
expected *= 130.2
expected = max(min(expected, 100), 200)
expected = op.factor * (pni.value - expected)
D = expected
expected = op.factor * (pni.value - C)
expected += pni.value
self.assertEqual(-4571.513, expected)
self.assertEqual(-4571.513, post)
self.assertEqual(A, pni._store["A"])
self.assertEqual(B, pni._store["B"])
self.assertEqual(C, pni._store["C"])
self.assertEqual(D, pni._store["D"])
self.assertEqual(pni.store_items(), dict(A=A, B=B, C=C, D=D).items())