def test_open_close(self):
from math import sqrt
def square_root(arg):
return sqrt(arg)
def cube(arg):
return arg ** 3
pl = Pipeline()
pl.add(ReplicatingFork(2))
pl.add(Processor(square_root), Processor(cube))
pl.add(Processor(square_root), Pipe())
pl.add(Pipe())
pl.add(Join(2))
pl.add(Processor(print))
self.assertTrue(pl.closed, 'Pipeline should be closed')
with pl as pipeline:
self.assertTrue(pipeline.opened, 'Pipeline should be opened')
pipeline.run([1, 2])
self.assertTrue(pipeline.opened, 'Pipeline should be opened')
self.assertFalse(pipeline.closed, 'Pipeline should be opened')
pipeline.run([1, 2])
self.assertTrue(pipeline.opened, 'Pipeline should be opened')
self.assertTrue(pl.closed)
def test_fork_and_join1(self):
self.counts = Counter()
def count(arg):
_, string = arg
self.counts[string] += 1
if len(self.counts) == 4:
self.assertEqual(self.counts['ttring'], 1)
self.assertEqual(self.counts['turing'], 1)
self.assertEqual(self.counts['suring'], 1)
self.assertEqual(self.counts['string'], 1)
def dont_manipulate(arg):
stage, string = arg
return (stage + 1, string)
def manipulate(arg):
stage, string = arg
l = list(string)
l[stage] = chr(ord(l[stage]) + 1)
return (stage + 1, ''.join(l))
pl = Pipeline()
pl.add(BalancingFork(2))
pl.add(Pipe())
pl.add(Processor(manipulate),
Processor(dont_manipulate))
pl.add(ReplicatingFork(2))
pl.add(Processor(manipulate), Processor(dont_manipulate),
Processor(manipulate), Processor(dont_manipulate))
pl.add(Join(4))
pl.add(Processor(count))
pl.run([(0, 'string'), (0, 'string')])
def test_pipeline_ambiguity(self):
def job1(arg):
return 'job1'
def job2(arg):
return 'job2'
pl = Pipeline()
pl.add(Processor(job1))
pl.add(ReplicatingFork(2))
pl.add(Processor(job1), Processor(job2))
pl.add(ReplicatingFork(4), ReplicatingFork(2))
pl.add(Processor(job2), Processor(job1))
with self.assertRaises(Exception) as e:
pl.add(Join(2), Join(4))
self.assertEqual(str(e.exception),
'Ambiguity Error: Partially joining forks')
def test_fork_join_mix(self):
pl = Pipeline()
pl.add(ReplicatingFork(3))
with self.assertRaises(Exception) as e:
pl.add(Join(2), BalancingFork(2))
self.assertEqual(
str(e.exception),
'Invalid types! All non Pipe objects in stage must be in same subclass'
)
def main():
pl = Pipeline()
pl.add(Processor(generate_ngrams))
pl.add(ReplicatingFork(3))
pl.add(Processor(train_decision_tree), Processor(train_random_forest),
Processor(train_k_neighbors))
with open("sms_data.txt", "r", encoding='latin-1') as file:
text = file.read().split('\n')
pl.run([text])
def test_too_many_processors(self):
pl = Pipeline()
pl.add(ReplicatingFork(2))
with self.assertRaises(Exception) as e:
pl.add(Processor(dummy_return_arg), Processor(dummy_return_arg),
Processor(dummy_return_arg))
self.assertEqual(
str(e.exception),
'Ambiguity Error: Jobs cannot be divided among fanout of previous stage'
)
def test_processor_join_mix(self):
def job(arg):
return 'job'
pl = Pipeline()
pl.add(ReplicatingFork(3))
with self.assertRaises(Exception) as e:
pl.add(Processor(job), Join(2))
self.assertEqual(
str(e.exception),
'Invalid types! All non Pipe objects in stage must be in same subclass'
)
def test_automatic_open_close(self):
from math import sqrt
def square_root(arg):
return sqrt(arg)
def cube(arg):
return arg ** 3
pl = Pipeline()
pl.add(ReplicatingFork(2))
pl.add(Processor(square_root), Processor(cube))
pl.add(Processor(square_root), Pipe())
pl.add(Pipe())
pl.add(Join(2))
pl.add(Processor(print))
self.assertTrue(pl.closed, 'Pipeline should be closed')
pl.run([2, 7, 9])
self.assertTrue(pl.closed, 'Pipeline should be closed')
def test_processor_pipe_mix(self):
self.counts = Counter()
def count(arg):
_, string = arg
self.counts[string] += 1
if len(self.counts) == 3:
self.assertEqual(self.counts['ttring'], 1)
self.assertEqual(self.counts['string'], 2)
def manipulate(arg):
stage, string = arg
l = list(string)
l[stage] = chr(ord(l[stage]) + 1)
return (stage + 1, ''.join(l))
pl = Pipeline()
pl.add(ReplicatingFork(3))
pl.add(Processor(manipulate), Pipe(), Pipe())
pl.add(Join(3))
pl.add(Processor(count))
pl.run([(0, 'string')])
def test_open_close_no_with(self):
from math import sqrt
def square_root(arg):
return sqrt(arg)
def cube(arg):
return arg ** 3
pl = Pipeline()
pl.add(ReplicatingFork(2))
pl.add(Processor(square_root), Processor(cube))
pl.add(Processor(square_root), Pipe())
pl.add(Join(2))
pl.add(Processor(print))
self.assertTrue(pl.closed, 'Pipeline should be closed')
pl.open()
pl.run([16, 3, 81])
self.assertTrue(pl.opened, 'Pipeline should be open')
pl.close()
self.assertTrue(pl.closed, 'Pipeline should be closed')
pl.open() # Leave it open -- daemon children should be cleaned up
self.assertTrue(pl.opened, 'Pipeline should be open')