def test_run_pipeline_for_rewrapped_data(self): from plumber import Pipeline A = self._makeOneA() B = self._makeOneB() ppl = Pipeline(A(), B()) post_data = ppl.run({'name': ' foo '}, rewrap=True) for pd in post_data: self.assertEqual(pd, {'name': 'FOO'})
def test_run_pipeline_prefetching_data(self): from plumber import Pipeline A = self._makeOneA() B = self._makeOneB() ppl = Pipeline(A(), B()) post_data = ppl.run([{'name': ' foo '}], prefetch=5) for pd in post_data: self.assertEqual(pd, {'name': 'FOO'})
def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero( self): raw_data = [{'name': ' foo '}] pos_data = [{'name': 'FOO'}] pf_callable = self.mocker.mock() pf_callable(mocker.ANY, 5) self.mocker.result(pos_data) self.mocker.replay() from plumber import Pipeline A = self._makeOneA() B = self._makeOneB() ppl = Pipeline(A(), B(), prefetch_callable=pf_callable) post_data = ppl.run(raw_data, prefetch=5) for pd in post_data: self.assertEqual(pd, {'name': 'FOO'})
def test_pipes_are_run_in_right_order(self): from plumber import Pipeline a = self.mocker.mock() b = self.mocker.mock() with self.mocker.order(): a.feed(mocker.ANY) self.mocker.result(a) b.feed(mocker.ANY) self.mocker.result(b) iter(b) self.mocker.result(b) self.mocker.replay() ppl = Pipeline(a, b) post_data = ppl.run([{'name': None}]) # placebo input value for pd in post_data: self.assertNone(pd)
def test_prefetching_generators(self): from plumber import Pipeline import time def makeA(): from plumber import Pipe class A(Pipe): def transform(self, data): data['name'] = data['name'].strip() time.sleep(0.3) return data return A def makeB(): from plumber import Pipe class B(Pipe): def transform(self, data): data['name'] = data['name'].upper() return data return B raw_data = ({'name': ' foo '} for i in range(10)) A = makeA() B = makeB() ppl = Pipeline(A(), B()) self.assertTrue(hasattr(raw_data, 'next')) post_data = ppl.run(raw_data, prefetch=2) for pd in post_data: self.assertEqual(pd, {'name': 'FOO'})
def test_processing_custom_objects(self): class Foo(object): def __init__(self): self.name = u'Foo name' raw_data = Foo() pos_data = [{'name': 'FOO NAME'}] from plumber import Pipeline, Pipe class A(Pipe): def transform(self, data): return {'name': data.name} class B(Pipe): def transform(self, data): data = {k: v.upper() for k, v in data.items()} return data ppl = Pipeline(A(), B()) post_data = ppl.run(raw_data, rewrap=True) for pd in post_data: self.assertEqual(pd, {'name': 'FOO NAME'})