def test_join_pipe_mix(self):
self.counts = Counter()
def count(arg):
_, string = arg
self.counts[string] += 1
if len(self.counts) == 3:
self.assertEqual(self.counts['string'], 1)
self.assertEqual(self.counts['suring'], 1)
self.assertEqual(self.counts['turing'], 1)
def manipulate(arg):
stage, string = arg
l = list(string)
l[stage] = chr(ord(l[stage]) + 1)
return (stage + 1, ''.join(l))
def dont_manipulate(arg):
stage, string = arg
return (stage + 1, string)
pl = Pipeline()
pl.add(ReplicatingFork(3))
pl.add(Processor(dont_manipulate), Processor(dont_manipulate),
Processor(manipulate))
pl.add(Pipe(), Join(2))
pl.add(Processor(dont_manipulate), Processor(manipulate))
pl.add(Join(2))
pl.add(Processor(count))
pl.run([(0, 'string')])
def test_reopening_pipeline(self):
def add(arg):
return arg + 1
def sub(arg):
return arg - 1
with Pipeline() as pl:
pl.add(BalancingFork(2))
pl.add(Processor(add), Processor(sub))
pl.add(Join(2))
pl.run([3])
self.assertTrue(pl.opened, 'Pipeline should be open')
self.assertFalse(pl.closed, 'Pipeline should be open')
pl.run([4])
pl.run([5])
self.assertTrue(pl.opened, 'Pipeline should be open')
self.assertFalse(pl.closed, 'Pipeline should be open')
pl.run([10])
self.assertTrue(pl.opened, 'Pipeline should be open')
self.assertFalse(pl.closed, 'Pipeline should be open')
self.assertFalse(pl.opened, 'Pipeline should be closed')
self.assertTrue(pl.closed, 'Pipeline should be closed')
def test_open_close(self):
from math import sqrt
def square_root(arg):
return sqrt(arg)
def cube(arg):
return arg ** 3
pl = Pipeline()
pl.add(ReplicatingFork(2))
pl.add(Processor(square_root), Processor(cube))
pl.add(Processor(square_root), Pipe())
pl.add(Pipe())
pl.add(Join(2))
pl.add(Processor(print))
self.assertTrue(pl.closed, 'Pipeline should be closed')
with pl as pipeline:
self.assertTrue(pipeline.opened, 'Pipeline should be opened')
pipeline.run([1, 2])
self.assertTrue(pipeline.opened, 'Pipeline should be opened')
self.assertFalse(pipeline.closed, 'Pipeline should be opened')
pipeline.run([1, 2])
self.assertTrue(pipeline.opened, 'Pipeline should be opened')
self.assertTrue(pl.closed)
def test_no_add_after_run(self):
def add(arg):
return arg + 1
def sub(arg):
return arg - 1
def printer(arg):
print(arg)
with self.assertRaises(Exception) as e:
with Pipeline() as pl:
pl.add(BalancingFork(2))
pl.add(Processor(add), Processor(sub))
pl.add(Join(2))
pl.add(Processor(print))
pl.run([3])
pl.add(Processor(printer))
pl.run([10])
self.assertEqual(str(e.exception),
'Pipelines cannot be modified after being run!')
def test_pipeline_ambiguity(self):
def job1(arg):
return 'job1'
def job2(arg):
return 'job2'
pl = Pipeline()
pl.add(Processor(job1))
pl.add(ReplicatingFork(2))
pl.add(Processor(job1), Processor(job2))
pl.add(ReplicatingFork(4), ReplicatingFork(2))
pl.add(Processor(job2), Processor(job1))
with self.assertRaises(Exception) as e:
pl.add(Join(2), Join(4))
self.assertEqual(str(e.exception),
'Ambiguity Error: Partially joining forks')
def test_fork_join_mix(self):
pl = Pipeline()
pl.add(ReplicatingFork(3))
with self.assertRaises(Exception) as e:
pl.add(Join(2), BalancingFork(2))
self.assertEqual(
str(e.exception),
'Invalid types! All non Pipe objects in stage must be in same subclass'
)
def test_pipeline_valid_start(self):
def finalize(arg):
return True
pl = Pipeline()
# Join (should not be allowed at pipeline head)
with self.assertRaises(Exception) as e:
pl.add(Join(2))
self.assertEqual(
str(e.exception),
'A pipeline cannot start with a Join (nothing to join to!)')
def test_processor_join_mix(self):
def job(arg):
return 'job'
pl = Pipeline()
pl.add(ReplicatingFork(3))
with self.assertRaises(Exception) as e:
pl.add(Processor(job), Join(2))
self.assertEqual(
str(e.exception),
'Invalid types! All non Pipe objects in stage must be in same subclass'
)
def test_automatic_open_close(self):
from math import sqrt
def square_root(arg):
return sqrt(arg)
def cube(arg):
return arg ** 3
pl = Pipeline()
pl.add(ReplicatingFork(2))
pl.add(Processor(square_root), Processor(cube))
pl.add(Processor(square_root), Pipe())
pl.add(Pipe())
pl.add(Join(2))
pl.add(Processor(print))
self.assertTrue(pl.closed, 'Pipeline should be closed')
pl.run([2, 7, 9])
self.assertTrue(pl.closed, 'Pipeline should be closed')
def test_balancing_forks(self):
self.counts = Counter()
def job1(arg):
return 'job1'
def job2(arg):
return 'job2'
def finalize(arg):
self.counts[arg] += 1
pl = Pipeline()
pl.add(BalancingFork(2))
pl.add(Processor(job1), Processor(job2))
pl.add(Join(2))
pl.add(Processor(finalize))
pl.run([False, False])
self.assertEqual(self.counts['job1'], self.counts['job2'])
def test_open_close_no_with(self):
from math import sqrt
def square_root(arg):
return sqrt(arg)
def cube(arg):
return arg ** 3
pl = Pipeline()
pl.add(ReplicatingFork(2))
pl.add(Processor(square_root), Processor(cube))
pl.add(Processor(square_root), Pipe())
pl.add(Join(2))
pl.add(Processor(print))
self.assertTrue(pl.closed, 'Pipeline should be closed')
pl.open()
pl.run([16, 3, 81])
self.assertTrue(pl.opened, 'Pipeline should be open')
pl.close()
self.assertTrue(pl.closed, 'Pipeline should be closed')
pl.open() # Leave it open -- daemon children should be cleaned up
self.assertTrue(pl.opened, 'Pipeline should be open')