def test_run_pipeline_for_rewrapped_data(self):
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B())
post_data = ppl.run({'name': ' foo '}, rewrap=True)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_run_pipeline_for_rewrapped_data(self):
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B())
post_data = ppl.run({'name': ' foo '}, rewrap=True)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_run_pipeline_prefetching_data(self):
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B())
post_data = ppl.run([{'name': ' foo '}], prefetch=5)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_run_pipeline_prefetching_data(self):
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B())
post_data = ppl.run([{'name': ' foo '}], prefetch=5)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_pipe_function_runs(self):
from plumber import Pipeline, pipe
@pipe
def do_something(data):
return data.lower()
ppl = Pipeline(do_something)
post_data = ppl.run(['FOO'])
self.assertEqual(next(post_data), 'foo')
def test_pipe_function_precondition(self):
from plumber import Pipeline, pipe, precondition
@pipe
@precondition(lambda x: isinstance(x, str))
def do_something(data):
return data.lower()
ppl = Pipeline(do_something)
post_data = ppl.run(['FOO'])
self.assertEqual(next(post_data), 'foo')
def test_pipes_are_run_in_right_order(self):
from plumber import Pipeline, Pipe
parent = mock.Mock()
parent.a = mock.MagicMock(spec=Pipe)
parent.a.__iter__.return_value = ['foo']
parent.b = mock.MagicMock(spec=Pipe)
parent.b.__iter__.return_value = ['foo']
ppl = Pipeline(parent.a, parent.b)
post_data = ppl.run([{'name': None}]) # placebo input value
for pd in post_data:
pass # do nothing
parent.mock_calls[0] == mock.call.a.feed([{'name': None}])
parent.mock_calls[1] == mock.call.b.feed(mock.ANY)
def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero(self):
raw_data = [{'name': ' foo '}]
pos_data = [{'name': 'FOO'}]
pf_callable = mock.MagicMock(return_value=pos_data)
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B(), prefetch_callable=pf_callable)
post_data = ppl.run(raw_data, prefetch=5)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
pf_callable.assert_called_with(mock.ANY, 5)
def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero(self):
raw_data = [{'name': ' foo '}]
pos_data = [{'name': 'FOO'}]
pf_callable = self.mocker.mock()
pf_callable(mocker.ANY, 5)
self.mocker.result(pos_data)
self.mocker.replay()
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B(), prefetch_callable=pf_callable)
post_data = ppl.run(raw_data, prefetch=5)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero(
self):
raw_data = [{'name': ' foo '}]
pos_data = [{'name': 'FOO'}]
pf_callable = self.mocker.mock()
pf_callable(mocker.ANY, 5)
self.mocker.result(pos_data)
self.mocker.replay()
from plumber import Pipeline
A = self._makeOneA()
B = self._makeOneB()
ppl = Pipeline(A(), B(), prefetch_callable=pf_callable)
post_data = ppl.run(raw_data, prefetch=5)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_pipes_are_run_in_right_order(self):
from plumber import Pipeline
a = self.mocker.mock()
b = self.mocker.mock()
with self.mocker.order():
a.feed(mocker.ANY)
self.mocker.result(a)
b.feed(mocker.ANY)
self.mocker.result(b)
iter(b)
self.mocker.result(b)
self.mocker.replay()
ppl = Pipeline(a, b)
post_data = ppl.run([{'name': None}]) # placebo input value
for pd in post_data:
self.assertNone(pd)
def test_pipes_are_run_in_right_order(self):
from plumber import Pipeline
a = self.mocker.mock()
b = self.mocker.mock()
with self.mocker.order():
a.feed(mocker.ANY)
self.mocker.result(a)
b.feed(mocker.ANY)
self.mocker.result(b)
iter(b)
self.mocker.result(b)
self.mocker.replay()
ppl = Pipeline(a, b)
post_data = ppl.run([{'name': None}]) # placebo input value
for pd in post_data:
self.assertNone(pd)
def test_prefetching_generators(self):
from plumber import Pipeline
import time
def makeA():
from plumber import Pipe
class A(Pipe):
def transform(self, data):
data['name'] = data['name'].strip()
time.sleep(0.3)
return data
return A
def makeB():
from plumber import Pipe
class B(Pipe):
def transform(self, data):
data['name'] = data['name'].upper()
return data
return B
raw_data = ({'name': ' foo '} for i in range(10))
A = makeA()
B = makeB()
ppl = Pipeline(A(), B())
self.assertTrue(hasattr(raw_data, 'next'))
post_data = ppl.run(raw_data, prefetch=2)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_processing_custom_objects(self):
class Foo(object):
def __init__(self):
self.name = u'Foo name'
raw_data = Foo()
pos_data = [{'name': 'FOO NAME'}]
from plumber import Pipeline, Pipe
class A(Pipe):
def transform(self, data):
return {'name': data.name}
class B(Pipe):
def transform(self, data):
data = {k:v.upper() for k, v in data.items()}
return data
ppl = Pipeline(A(), B())
post_data = ppl.run(raw_data, rewrap=True)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO NAME'})
def test_prefetching_generators(self):
from plumber import Pipeline
import time
def makeA():
from plumber import Pipe
class A(Pipe):
def transform(self, data):
data['name'] = data['name'].strip()
time.sleep(0.3)
return data
return A
def makeB():
from plumber import Pipe
class B(Pipe):
def transform(self, data):
data['name'] = data['name'].upper()
return data
return B
raw_data = ({'name': ' foo '} for i in range(10))
A = makeA()
B = makeB()
ppl = Pipeline(A(), B())
self.assertTrue(hasattr(raw_data, 'next'))
post_data = ppl.run(raw_data, prefetch=2)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO'})
def test_processing_custom_objects(self):
class Foo(object):
def __init__(self):
self.name = u'Foo name'
raw_data = Foo()
pos_data = [{'name': 'FOO NAME'}]
from plumber import Pipeline, Pipe
class A(Pipe):
def transform(self, data):
return {'name': data.name}
class B(Pipe):
def transform(self, data):
data = {k: v.upper() for k, v in data.items()}
return data
ppl = Pipeline(A(), B())
post_data = ppl.run(raw_data, rewrap=True)
for pd in post_data:
self.assertEqual(pd, {'name': 'FOO NAME'})