def parametrized_coroutine_test():
add_5 = add.params(5)
add_3 = add.params(3)
# Parametrized coroutine provides readable representation
tools.eq_(repr(add_5), 'add.params(5)')
tools.eq_(repr(add_3), 'add.params(3)')
result_5 = []
result_3 = []
pipeline(add_5, collect.params(result_5)).feed([1, 2, 3])
pipeline(add_3, collect.params(result_3)).feed([1, 2, 3])
tools.eq_(result_5, [6, 7, 8])
tools.eq_(result_3, [4, 5, 6])
def parametrized_coroutine_test():
add_5 = add.params(5)
add_3 = add.params(3)
# Parametrized coroutine provides readable representation
tools.eq_(repr(add_5), 'add.params(5)')
tools.eq_(repr(add_3), 'add.params(3)')
result_5 = []
result_3 = []
pipeline(add_5, collect.params(result_5)).feed([1, 2, 3])
pipeline(add_3, collect.params(result_3)).feed([1, 2, 3])
tools.eq_(result_5, [6, 7, 8])
tools.eq_(result_3, [4, 5, 6])
def pipeline_representation_test():
p = pipeline(add.params(1))
with p.fork(split, 2) as (even, odd):
even.connect(
multiply.params(2),
add.params(5),
)
odd.connect(
multiply.params(5),
add.params(2),
)
p.connect(add.params(2))
tools.eq_(
repr(p).strip(),
dedent("""
add.params(1)
split:
-->
multiply.params(2)
add.params(5)
-->
multiply.params(5)
add.params(2)
add.params(2)
""").strip())
def pipeline_representation_test():
p = pipeline(
add.params(1)
)
with p.fork(split, 2) as (even, odd):
even.connect(
multiply.params(2),
add.params(5),
)
odd.connect(
multiply.params(5),
add.params(2),
)
p.connect(
add.params(2)
)
tools.eq_(repr(p).strip(), dedent("""
add.params(1)
split:
-->
multiply.params(2)
add.params(5)
-->
multiply.params(5)
add.params(2)
add.params(2)
""").strip())
def complex_pipeline_test():
odds = []
evens = []
result = []
p = pipeline(
add.params(1)
)
with p.fork(split, 2) as (even, odd):
even.connect(
collect.params(evens),
multiply.params(2),
add.params(5),
)
odd.connect(
collect.params(odds),
multiply.params(5),
add.params(2),
)
p.connect(
collect.params(result)
)
p.feed([1, 2, 3, 4])
tools.eq_(evens, [2, 4])
tools.eq_(odds, [3, 5])
tools.eq_(result, [9, 17, 13, 27])
def replication_pipeline_test():
result = []
p = pipeline(
replicate.params(2),
collect.params(result),
)
p.feed([1,2,3,4])
tools.eq_(result, [1,1,2,2,3,3,4,4])
def forked_pipeline_test():
evens = []
odds = []
p = pipeline()
with p.fork(split, 2) as (even, odd):
even.connect(multiply.params(10), collect.params(evens))
odd.connect(add.params(10), collect.params(odds))
p.feed([1, 2, 3, 4])
tools.eq_(evens, [20, 40])
tools.eq_(odds, [11, 13])
def straight_forward_pipeline_test():
result = []
p = pipeline(
multiply.params(10),
add.params(5),
)
p.connect(
add.params(1),
collect.params(result),
)
p.feed([1, 2, 3, 4])
tools.eq_(result, [16, 26, 36, 46])
def straight_forward_pipeline_test():
result = []
p = pipeline(
multiply.params(10),
add.params(5),
)
p.connect(
add.params(1),
collect.params(result),
)
p.feed([1, 2, 3, 4])
tools.eq_(result, [16, 26, 36, 46])
def plugged_pipeline_test():
result = []
null = []
p = pipeline(
multiply.params(10),
collect.params(result),
)
p.plug()
p.connect(
add.params(1),
collect.params(null),
)
p.feed([1, 2, 3, 4])
tools.eq_(result, [10, 20, 30, 40])
tools.eq_(null, [])
def plugged_pipeline_test():
result = []
null = []
p = pipeline(
multiply.params(10),
collect.params(result),
)
p.plug()
p.connect(
add.params(1),
collect.params(null),
)
p.feed([1, 2, 3, 4])
tools.eq_(result, [10, 20, 30, 40])
tools.eq_(null, [])
def forked_pipeline_test():
evens = []
odds = []
p = pipeline()
with p.fork(split, 2) as (even, odd):
even.connect(
multiply.params(10),
collect.params(evens)
)
odd.connect(
add.params(10),
collect.params(odds)
)
p.feed([1, 2, 3, 4])
tools.eq_(evens, [20, 40])
tools.eq_(odds, [11, 13])
def complex_pipeline_test():
odds = []
evens = []
result = []
p = pipeline(add.params(1))
with p.fork(split, 2) as (even, odd):
even.connect(
collect.params(evens),
multiply.params(2),
add.params(5),
)
odd.connect(
collect.params(odds),
multiply.params(5),
add.params(2),
)
p.connect(collect.params(result))
p.feed([1, 2, 3, 4])
tools.eq_(evens, [2, 4])
tools.eq_(odds, [3, 5])
tools.eq_(result, [9, 17, 13, 27])
def save(file, next=null):
template = u('{0.level:7.7} {0.module:6.6} {0.message}\n')
while True:
record = yield
file.write(template.format(record))
next.send(record)
if __name__ == '__main__':
error_log = StringIO()
first_log = StringIO()
second_log = StringIO()
third_log = StringIO()
p = pipeline(
parse,
filter.params(lambda r: r.level != 'DEBUG'),
)
with p.fork(broadcast, 2) as (modules, errors):
module_names = ()
with modules.fork(split.params(lambda r: r.module),
'first', 'second', 'third') as (first, second, third):
first.connect(save.params(first_log))
second.connect(save.params(second_log))
third.connect(save.params(third_log))
errors.connect(
filter.params(lambda r: r.level in ('ERROR', 'WARNING')),
unique,
save.params(error_log)
)
from copipes import coroutine, pipeline, null
from copipes.macros.pipe import pipe
@pipe
def putStrLn():
"""doc"""
[x]
print x
send(x)
@pipe
def replicate(n):
[x]
for i in xrange(n):
send(x)
if __name__ == '__main__':
pipeline(
putStrLn,
replicate.params(3),
putStrLn,
).feed(range(10))
def save(file, next=null):
template = u('{0.level:7.7} {0.module:6.6} {0.message}\n')
while True:
record = yield
file.write(template.format(record))
next.send(record)
if __name__ == '__main__':
error_log = StringIO()
first_log = StringIO()
second_log = StringIO()
third_log = StringIO()
p = pipeline(
parse,
filter.params(lambda r: r.level != 'DEBUG'),
)
with p.fork(broadcast, 2) as (modules, errors):
module_names = ()
with modules.fork(split.params(lambda r: r.module), 'first', 'second',
'third') as (first, second, third):
first.connect(save.params(first_log))
second.connect(save.params(second_log))
third.connect(save.params(third_log))
errors.connect(
filter.params(lambda r: r.level in ('ERROR', 'WARNING')), unique,
save.params(error_log))
p.feed(line for line in log)
