Exemple #1
0
    def test_run_pipeline_for_rewrapped_data(self):
        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B())
        post_data = ppl.run({'name': '  foo    '}, rewrap=True)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #2
0
    def test_run_pipeline_for_rewrapped_data(self):
        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B())
        post_data = ppl.run({'name': '  foo    '}, rewrap=True)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #3
0
    def test_run_pipeline_prefetching_data(self):
        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B())
        post_data = ppl.run([{'name': '  foo    '}], prefetch=5)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #4
0
    def test_run_pipeline_prefetching_data(self):
        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B())
        post_data = ppl.run([{'name': '  foo    '}], prefetch=5)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #5
0
    def test_pipe_function_runs(self):
        from plumber import Pipeline, pipe
        @pipe
        def do_something(data):
            return data.lower()

        ppl = Pipeline(do_something)
        post_data = ppl.run(['FOO'])

        self.assertEqual(next(post_data), 'foo')
Exemple #6
0
    def test_pipe_function_precondition(self):
        from plumber import Pipeline, pipe, precondition
        @pipe
        @precondition(lambda x: isinstance(x, str))
        def do_something(data):
            return data.lower()

        ppl = Pipeline(do_something)
        post_data = ppl.run(['FOO'])

        self.assertEqual(next(post_data), 'foo')
Exemple #7
0
    def test_pipes_are_run_in_right_order(self):
        from plumber import Pipeline, Pipe
        parent = mock.Mock()
        parent.a = mock.MagicMock(spec=Pipe)
        parent.a.__iter__.return_value = ['foo']
        parent.b = mock.MagicMock(spec=Pipe)
        parent.b.__iter__.return_value = ['foo']

        ppl = Pipeline(parent.a, parent.b)
        post_data = ppl.run([{'name': None}])  # placebo input value

        for pd in post_data:
            pass  # do nothing

        parent.mock_calls[0] == mock.call.a.feed([{'name': None}])
        parent.mock_calls[1] == mock.call.b.feed(mock.ANY)
Exemple #8
0
    def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero(self):
        raw_data = [{'name': '  foo    '}]
        pos_data = [{'name': 'FOO'}]

        pf_callable = mock.MagicMock(return_value=pos_data)

        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B(), prefetch_callable=pf_callable)
        post_data = ppl.run(raw_data, prefetch=5)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})

        pf_callable.assert_called_with(mock.ANY, 5)
Exemple #9
0
    def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero(self):
        raw_data = [{'name': '  foo    '}]
        pos_data = [{'name': 'FOO'}]

        pf_callable = self.mocker.mock()
        pf_callable(mocker.ANY, 5)
        self.mocker.result(pos_data)
        self.mocker.replay()

        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B(), prefetch_callable=pf_callable)
        post_data = ppl.run(raw_data, prefetch=5)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #10
0
    def test_prefetch_callable_is_called_when_prefetch_arg_is_greater_than_zero(
            self):
        raw_data = [{'name': '  foo    '}]
        pos_data = [{'name': 'FOO'}]

        pf_callable = self.mocker.mock()
        pf_callable(mocker.ANY, 5)
        self.mocker.result(pos_data)
        self.mocker.replay()

        from plumber import Pipeline
        A = self._makeOneA()
        B = self._makeOneB()

        ppl = Pipeline(A(), B(), prefetch_callable=pf_callable)
        post_data = ppl.run(raw_data, prefetch=5)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #11
0
    def test_pipes_are_run_in_right_order(self):
        from plumber import Pipeline
        a = self.mocker.mock()
        b = self.mocker.mock()

        with self.mocker.order():
            a.feed(mocker.ANY)
            self.mocker.result(a)

            b.feed(mocker.ANY)
            self.mocker.result(b)

            iter(b)
            self.mocker.result(b)

        self.mocker.replay()

        ppl = Pipeline(a, b)
        post_data = ppl.run([{'name': None}])  # placebo input value

        for pd in post_data:
            self.assertNone(pd)
Exemple #12
0
    def test_pipes_are_run_in_right_order(self):
        from plumber import Pipeline
        a = self.mocker.mock()
        b = self.mocker.mock()

        with self.mocker.order():
            a.feed(mocker.ANY)
            self.mocker.result(a)

            b.feed(mocker.ANY)
            self.mocker.result(b)

            iter(b)
            self.mocker.result(b)

        self.mocker.replay()

        ppl = Pipeline(a, b)
        post_data = ppl.run([{'name': None}])  # placebo input value

        for pd in post_data:
            self.assertNone(pd)
Exemple #13
0
    def test_prefetching_generators(self):
        from plumber import Pipeline
        import time

        def makeA():
            from plumber import Pipe

            class A(Pipe):
                def transform(self, data):
                    data['name'] = data['name'].strip()
                    time.sleep(0.3)
                    return data

            return A

        def makeB():
            from plumber import Pipe

            class B(Pipe):
                def transform(self, data):
                    data['name'] = data['name'].upper()
                    return data

            return B

        raw_data = ({'name': '  foo    '} for i in range(10))

        A = makeA()
        B = makeB()

        ppl = Pipeline(A(), B())

        self.assertTrue(hasattr(raw_data, 'next'))

        post_data = ppl.run(raw_data, prefetch=2)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #14
0
    def test_processing_custom_objects(self):
        class Foo(object):
            def __init__(self):
                self.name = u'Foo name'

        raw_data = Foo()
        pos_data = [{'name': 'FOO NAME'}]

        from plumber import Pipeline, Pipe
        class A(Pipe):
            def transform(self, data):
                return {'name': data.name}

        class B(Pipe):
            def transform(self, data):
                data = {k:v.upper() for k, v in data.items()}
                return data

        ppl = Pipeline(A(), B())
        post_data = ppl.run(raw_data, rewrap=True)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO NAME'})
Exemple #15
0
    def test_prefetching_generators(self):
        from plumber import Pipeline
        import time
        def makeA():
            from plumber import Pipe

            class A(Pipe):
                def transform(self, data):
                    data['name'] = data['name'].strip()
                    time.sleep(0.3)
                    return data

            return A

        def makeB():
            from plumber import Pipe

            class B(Pipe):
                def transform(self, data):
                    data['name'] = data['name'].upper()
                    return data

            return B

        raw_data = ({'name': '  foo    '} for i in range(10))

        A = makeA()
        B = makeB()

        ppl = Pipeline(A(), B())

        self.assertTrue(hasattr(raw_data, 'next'))

        post_data = ppl.run(raw_data, prefetch=2)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO'})
Exemple #16
0
    def test_processing_custom_objects(self):
        class Foo(object):
            def __init__(self):
                self.name = u'Foo name'

        raw_data = Foo()
        pos_data = [{'name': 'FOO NAME'}]

        from plumber import Pipeline, Pipe

        class A(Pipe):
            def transform(self, data):
                return {'name': data.name}

        class B(Pipe):
            def transform(self, data):
                data = {k: v.upper() for k, v in data.items()}
                return data

        ppl = Pipeline(A(), B())
        post_data = ppl.run(raw_data, rewrap=True)

        for pd in post_data:
            self.assertEqual(pd, {'name': 'FOO NAME'})