Esempio n. 1
0
class TestHttpInlet(inlet_tester.InletTester):
    def setUp(self):
        super().setUp()
        set_response(b'{"asdf":"12"}')

    def get_inlet(self):
        return HttpInlet('https://jsonplaceholder.typicode.com/todos/1')

    @patch(fqname(Update))
    def test_json(self, update):
        records = asyncio.run(self.inlet._pull(update))

        self.assertIsInstance(records, (list))

    @patch(fqname(Update))
    def test_html(self, update):
        self.inlet.json = False

        records = asyncio.run(self.inlet._pull(update))

        self.assertIsInstance(records, (list))

    @patch(fqname(Update))
    def test_invalid_json(self, update):
        set_response(b'<html>')
        self.inlet.json = True

        self.assertRaisesRegex(ValueError,
                               'Response does not contain valid JSON',
                               asyncio.run, self.inlet._pull(update))

    @patch(fqname(Update))
    def test_invalid_html(self, update):
        set_response(None)
        self.inlet.json = False

        self.assertRaisesRegex(
            AttributeError, '\'NoneType\' object has no attribute \'decode\'',
            asyncio.run, self.inlet._pull(update))
Esempio n. 2
0
class TestPrintOutlet(TestCase):
    @patch(fqname(Record), spec=Record)
    @patch(fqname(Update), spec=Update)
    def setUp(self, update, record):
        self.print_outlet = PrintOutlet()
        self.update = update
        self.record = record

        record.payload = 'test'
        record.metadata.return_value = {}

        record.__repr__ = lambda x: 'TestRecord(test)'
        update.__repr__ = lambda x: 'TestUpdate()'

    @patch('sys.stdout', new_callable=io.StringIO)
    def test_push(self, stdout):

        asyncio.run(self.print_outlet._push([self.record], self.update))

        self.assertEqual(stdout.getvalue(), 'TestUpdate() TestRecord(test)\n')

    @patch('sys.stdout', new_callable=io.StringIO)
    def test_push_only_payload(self, stdout):
        self.print_outlet.only_payload = True

        asyncio.run(self.print_outlet._push([self.record], self.update))

        self.assertEqual(stdout.getvalue(), 'TestUpdate() test\n')

    @patch('sys.stdout', new_callable=io.StringIO)
    def test_push_skip_update(self, stdout):
        self.print_outlet.skip_update = True

        asyncio.run(self.print_outlet._push([self.record], self.update))

        self.assertEqual(stdout.getvalue(), 'TestRecord(test)\n')
Esempio n. 3
0
class TestLink(TestCase):
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_transfer(self, inlet, outlet):
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    tags='test_update')

        link.transfer()

        inlet._pull.assert_called()
        outlet._push.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_run(self, inlet, outlet):
        async def task():
            link = Link([inlet], [outlet],
                        timedelta(seconds=1),
                        tags='test_run',
                        copy_records=False)

            inlet_result = await inlet._pull(None)
            await link._run()

            inlet._pull.assert_called()
            outlet._push.assert_called_with(inlet_result, mock.ANY)

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_inlet(self, inlet, outlet):
        inlet._pull.side_effect = DummyException('Test exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    ignore_exceptions=False,
                    tags='test_exception_inlet')

        self.assertRaises(DummyException, link.transfer)

        inlet._pull.assert_called()
        outlet._push.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_outlet(self, inlet, outlet):
        # inlet._pull.return_value, _ = self.inlet_return()
        # inlet._pull.side_effect = pull_mock
        # inlet._pull.return_value = Future()
        outlet._push.side_effect = DummyException('Test exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    ignore_exceptions=False,
                    tags='test_exception_outlet')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    ignore_exceptions=False,
                    tags='test_exception_outlet')

        self.assertRaises(DummyException, link.transfer)

        inlet._pull.assert_called()
        outlet._push.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_caught(self, inlet, outlet):
        logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
        inlet._pull.side_effect = DummyException('Test inlet exception')
        outlet._push.side_effect = DummyException('Test outlet exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    tags='test_exception_caught',
                    ignore_exceptions=True)

        try:
            link.transfer()
        except Exception as e:
            self.fail(f'Should not raise exception: {e}')

        inlet._pull.assert_called()
        outlet._push.assert_called()

    # check that one exception doesn't halt other inlets or outlets
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_ignore_partial_exception(self, inlet1, inlet2, outlet1, outlet2):
        logging.getLogger('databay.Link').setLevel(logging.CRITICAL)

        async def task():
            # inlet_future = Future()
            inlet1._pull.side_effect = DummyException('Test inlet1 exception')
            outlet1._push.side_effect = DummyException(
                'Test outlet1 exception')
            # inlet1._pull.return_value = inlet_future
            # inlet2._pull.return_value = inlet_future
            link = Link([inlet1, inlet2], [outlet1, outlet2],
                        timedelta(seconds=1),
                        tags='test_ignore_partial_exception',
                        copy_records=False,
                        ignore_exceptions=True)

            # results = [object()]
            results = await inlet2._pull(None)
            # inlet_future.set_result(results)

            await link._run()

            inlet1._pull.assert_called()
            inlet2._pull.assert_called()
            outlet1._push.assert_called_with(results, mock.ANY)
            outlet2._push.assert_called_with(results, mock.ANY)

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_start(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=False)
        type(outlet1).active = mock.PropertyMock(return_value=False)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start')

        link.on_start()

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_start_already_active(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=True)
        type(outlet1).active = mock.PropertyMock(return_value=True)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start_already_active')

        link.on_start()

        inlet1.on_start.assert_not_called()
        outlet1.on_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_shutdown(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=True)
        type(outlet1).active = mock.PropertyMock(return_value=True)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown')

        link.on_shutdown()

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_already_inactive(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=False)
        type(outlet1).active = mock.PropertyMock(return_value=False)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown_already_inactive')

        link.on_shutdown()

        inlet1.on_shutdown.assert_not_called()
        outlet1.on_shutdown.assert_not_called()

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet(self, inlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_inlet')

        link.add_inlets(inlet1)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet_multiple(self, inlet1, inlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_add_inlet_multiple')

        link.add_inlets([inlet1, inlet2])

        self.assertEqual(link.inlets, [inlet1, inlet2])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet_same(self, inlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same')

        link.add_inlets(inlet1)
        self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_remove_inlet(self, inlet1, inlet2):
        link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet')

        link.add_inlets([inlet1, inlet2])
        link.remove_inlets(inlet2)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_remove_inlet_invalid(self, inlet1, inlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_remove_inlet_invalid')

        link.add_inlets([inlet1])

        self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet(self, outlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_outlet')

        link.add_outlets(outlet1)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet_multiple(self, outlet1, outlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_add_outlet_multiple')

        link.add_outlets([outlet1, outlet2])

        self.assertEqual(link.outlets, [outlet1, outlet2])

    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet_same(self, outlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same')

        link.add_outlets(outlet1)
        self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_remove_outlet(self, outlet1, outlet2):
        link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet')

        link.add_outlets([outlet1, outlet2])
        link.remove_outlets(outlet2)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_remove_outlet_invalid(self, outlet1, outlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_remove_outlet_invalid')

        link.add_outlets([outlet1])

        self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
        self.assertEqual(link.outlets, [outlet1])

    # this rv is invalid, should be a list
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object()))
    def xtest_non_iterable_raised(self, inlet1):
        logging.getLogger('databay.Link').setLevel(logging.ERROR)
        link = Link([inlet1], [],
                    timedelta(seconds=1),
                    tags='test_non_iterable_raised')
        with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
            link.transfer()

    # this rv will raise DummyException
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable()))
    def test_generic_error_raised(self, inlet1):
        logging.getLogger('databay.Link').setLevel(logging.ERROR)
        link = Link([inlet1], [],
                    timedelta(seconds=1),
                    tags='test_generic_error_raised')
        # with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION):
        self.assertRaises(DummyException, link.transfer)

    def test_integer_to_timedelta(self):
        link = Link([], [], 1, name='test_integer_interval_coerced')
        self.assertEqual(link._interval, timedelta(seconds=1))

    def test_float_to_timedelta(self):
        link = Link([], [], 1.5, name='test_float_interval_coerced')
        self.assertEqual(link._interval, timedelta(seconds=1.5))

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_inlet_exception_raise(self, inlet1, outlet1):
        inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start')

        self.assertRaises(RuntimeError, link.on_start)

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_inlet_exception_catch(self, inlet1, outlet1):
        logging.getLogger('databay.Link').setLevel(logging.WARNING)
        inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_start()
        self.assertTrue(
            'on_start inlet exception: "" for inlet:' in ';'.join(cm.output))

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_outlet_exception_raise(self, inlet1, outlet1, outlet2):
        outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_start')

        self.assertRaises(RuntimeError, link.on_start)

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()
        outlet2.try_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_outlet_exception_catch(self, inlet1, outlet1, outlet2):
        logging.getLogger('databay.Link').setLevel(logging.WARNING)
        outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_start',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_start()
        self.assertTrue(
            'on_start outlet exception: "" for outlet:' in ';'.join(cm.output),
            cm.output)

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()
        outlet2.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_inlet_exception_raise(self, inlet1, outlet1):
        inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown')

        self.assertRaises(RuntimeError, link.on_shutdown)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_inlet_exception_catch(self, inlet1, outlet1):
        logging.getLogger('databay.Link').setLevel(logging.WARNING)
        inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_shutdown()
        self.assertTrue(
            'on_shutdown inlet exception: "" for inlet:'
            in ';'.join(cm.output), cm.output)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_outlet_exception_raise(self, inlet1, outlet1,
                                                outlet2):
        outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
                                                        )
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_shutdown')

        self.assertRaises(RuntimeError, link.on_shutdown)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()
        outlet2.try_shutdown.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_outlet_exception_catch(self, inlet1, outlet1,
                                                outlet2):
        logging.getLogger('databay.Link').setLevel(logging.WARNING)
        outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
                                                        )
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_shutdown',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_shutdown()
        self.assertTrue(
            'on_shutdown outlet exception: "" for outlet:'
            in ';'.join(cm.output), cm.output)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()
        outlet2.try_shutdown.assert_called()

    def test_single_tag(self):
        tag = 'tagA'
        link = Link([], [], timedelta(seconds=1), tags=tag)
        self.assertEqual(link.tags, [tag])

    def test_multiple_tags(self):
        tagA = 'tagA'
        tagB = 'tagB'
        link = Link([], [], timedelta(seconds=1), tags=[tagA, tagB])
        self.assertEqual(link.tags, [tagA, tagB])

    def test_tag_as_name(self):
        link_name = 'link_name'
        link = Link([], [], timedelta(seconds=1), name=link_name)
        self.assertEqual(link_name, link.tags[0])

    def test_name_from_tag(self):
        link_name = 'link_name'
        link = Link([], [], timedelta(seconds=1), tags=[link_name])
        self.assertEqual(link.name, link.tags[0])
Esempio n. 4
0
class TestBasePlanner(TestCase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        logging.getLogger('databay').setLevel(logging.WARNING)

    @patch.multiple(BasePlanner, __abstractmethods__=set())
    def setUp(self):
        self.planner = BasePlanner()
        self.planner._schedule = mock.Mock(
            side_effect=lambda link: link.set_job(object()))
        self.planner._unschedule = mock.Mock(
            side_effect=lambda link: link.set_job(None))
        self.planner._start_planner = mock.Mock()
        self.planner._shutdown_planner = mock.Mock()

    @patch(fqname(Link), spec=Link)
    def test_add_links(self, link):
        def set_job(job):
            link.job = job

        link.set_job.side_effect = set_job
        link.job = None

        self.planner.add_links(link)
        self.assertIsNotNone(link.job, 'Link should contain a job')
        self.assertTrue(link in self.planner.links,
                        'Planner should contain the link')

    @patch(fqname(Link), spec=Link)
    def test_add_links_array(self, link):
        def set_job(job):
            link.job = job

        link.set_job.side_effect = set_job
        link.job = None

        self.planner.add_links([link])
        self.assertIsNotNone(link.job, 'Link should contain a job')
        self.assertTrue(link in self.planner.links,
                        'Planner should contain the link')

    @patch(fqname(Link), spec=Link)
    def test_remove_links(self, link):
        def set_job(job):
            link.job = job

        link.set_job.side_effect = set_job
        link.job = None

        self.planner.add_links(link)
        self.planner.remove_links(link)
        self.assertIsNone(link.job, 'Link should not contain a job')
        self.assertTrue(link not in self.planner.links,
                        'Planner should not contain the link')

    @patch(fqname(Link), spec=Link)
    def test_remove_invalid_link(self, link):
        def set_job(job):
            link.job = job

        link.set_job.side_effect = set_job
        link.job = None

        self.assertRaises(MissingLinkError, self.planner.remove_links, link)
        self.assertIsNone(link.job, 'Link should not contain a job')
        self.assertTrue(link not in self.planner.links,
                        'Planner should not contain the link')

    @patch(fqname(Link), spec=Link)
    def test_start(self, link):
        self.planner.add_links(link)
        self.planner.start()
        link.on_start.assert_called()
        self.planner._start_planner.assert_called()

    @patch(fqname(Link), spec=Link)
    def test_shutdown(self, link):
        self.planner.add_links(link)
        self.planner.shutdown()
        link.on_shutdown.assert_called()
        self.planner._shutdown_planner.assert_called()

    @patch(fqname(Link), spec=Link)
    def test_start_order(self, link):
        # on_start should be called before _start_planner
        link.on_start.side_effect = lambda: self.planner._start_planner.assert_not_called(
        )

        self.planner.add_links(link)
        self.planner.start()

        # finally both should be called
        link.on_start.assert_called()
        self.planner._start_planner.assert_called()

    @patch(fqname(Link), spec=Link)
    def test_shutdown_order(self, link):
        # on_shutdown should be called after _shutdown_planner
        self.planner._shutdown_planner.side_effect = lambda wait: link.on_shutdown.assert_not_called(
        )

        self.planner.add_links(link)
        self.planner.shutdown()

        # finally both should be called
        link.on_shutdown.assert_called()
        self.planner._shutdown_planner.assert_called()
Esempio n. 5
0
class TestMongoOutlet(TestCase):

    def setUp(self):
        self.outlet = MongoOutlet()

    @mongomock.patch(servers=(('localhost', 27017),))
    def test_connect(self):

        result = self.outlet.connect()

        self.assertFalse(result)
        self.assertEqual(self.outlet._db.name, 'databay')
        self.assertEqual(self.outlet._client.address[0], 'localhost')
        self.assertEqual(self.outlet._client.address[1], 27017)

    @mongomock.patch(servers=(('test', 1234),))
    def test_connect_custom_host(self):
        self.outlet = MongoOutlet(host='test', port=1234)

        self.outlet.connect()

        self.assertEqual(self.outlet._client.host, 'test')
        self.assertEqual(self.outlet._client.port, 1234)

    @mongomock.patch(servers=(('localhost', 27017),))
    def test_connect_custom_database(self):
        result = self.outlet.connect('my_database')

        self.assertFalse(result)
        self.assertEqual(self.outlet._db.name, 'my_database')

    @mongomock.patch(servers=(('localhost', 27017),))
    def test_connect_twice_same(self):
        result = self.outlet.connect()
        self.assertFalse(result)
        result = self.outlet.connect()
        self.assertTrue(result)
        self.assertEqual(self.outlet._db.name, 'databay')
        self.assertEqual(self.outlet._client.address[0], 'localhost')
        self.assertEqual(self.outlet._client.address[1], 27017)

    @mongomock.patch(servers=(('localhost', 27017),))
    def test_connect_twice_different(self):
        result = self.outlet.connect()
        self.assertFalse(result)
        self.assertEqual(self.outlet._db.name, 'databay')

        result = self.outlet.connect('test')
        self.assertFalse(result)
        self.assertEqual(self.outlet._db.name, 'test')
        self.assertEqual(self.outlet._client.address[0], 'localhost')
        self.assertEqual(self.outlet._client.address[1], 27017)

    def test_disconnect(self):
        close_mock = MagicMock()
        self.outlet._client = MagicMock(close=close_mock)
        self.outlet._db = MagicMock()
        self.outlet.disconnect()

        close_mock.assert_called_once()
        self.assertIsNone(self.outlet._client)
        self.assertIsNone(self.outlet._db)

    def test_on_start(self):
        self.outlet.connect = MagicMock()
        self.outlet.on_start()
        self.outlet.connect.assert_called_once()

    def test_on_shutdown(self):
        self.outlet.disconnect = MagicMock()
        self.outlet.on_shutdown()
        self.outlet.disconnect.assert_called_once()

    @mongomock.patch(servers=(('localhost', 27017),))
    def test__add_collection(self):
        name = 'test_collection'
        self.outlet._add_collection(name)
        self.assertTrue(name in self.outlet._db.list_collection_names(
        ), f'Database should contain collection \'{name}\'')
        self.assertIsInstance(self.outlet._db[name], Collection)

    @mongomock.patch(servers=(('localhost', 27017),))
    def test__add_collection_invalid(self):
        name = 'test_collection'
        self.outlet._add_collection(None)
        self.assertFalse(name in self.outlet._db.list_collection_names(
        ), f'Database should not contain collection \'{name}\'')

    @mongomock.patch(servers=(('localhost', 27017),))
    def test__get_collection(self):
        name = 'test_collection'
        self.outlet._add_collection(name)
        self.assertTrue(name in self.outlet._db.list_collection_names(
        ), f'Database should contain collection \'{name}\'')
        self.assertIsInstance(self.outlet._db[name], Collection)

        collection = self.outlet._get_collection(name)
        self.assertIsInstance(collection, Collection)

    @mongomock.patch(servers=(('localhost', 27017),))
    def test__get_collection_invalid(self):
        name = 'test_collection'
        self.outlet._add_collection(name)
        self.assertTrue(name in self.outlet._db.list_collection_names(
        ), f'Database should contain collection \'{name}\'')
        self.assertIsInstance(self.outlet._db[name], Collection)

        self.assertRaises(MongoCollectionNotFound,
                          self.outlet._get_collection, 'invalid_name')

    @mongomock.patch(servers=(('localhost', 27017),))
    @patch(fqname(Record), spec=Record)
    @patch(fqname(Record), spec=Record)
    @patch(fqname(Record), spec=Record)
    def test__group_by_collection(self, recordA, recordB, recordB2):
        recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'}
        recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
        recordB2.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
        recordA.payload = 'testA'
        recordB.payload = 'testB'
        recordB2.payload = 'testB2'

        target = {'A': ['testA'], 'B': ['testB', 'testB2']}

        records = [recordA, recordB, recordB2]

        collections = self.outlet._group_by_collection(records)
        self.assertEqual(collections.keys(), target.keys())
        self.assertEqual(collections['A'], [recordA.payload])
        self.assertEqual(collections['B'], [recordB.payload, recordB2.payload])

    @mongomock.patch(servers=(('localhost', 27017),))
    @patch(fqname(Record), spec=Record)
    @patch(fqname(Record), spec=Record)
    def test__group_by_collection_list_payload(self, recordA, recordB):
        recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'}
        recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
        recordA.payload = 'testA'
        recordB.payload = ['testB', 'testB2']

        target = {'A': ['testA'], 'B': ['testB', 'testB2']}

        records = [recordA, recordB]

        collections = self.outlet._group_by_collection(records)
        self.assertEqual(collections.keys(), target.keys())
        self.assertEqual(collections['A'], [recordA.payload])
        self.assertEqual(collections['B'], [*recordB.payload])

    @mongomock.patch(servers=(('localhost', 27017),))
    @patch(fqname(Update), spec=Update)
    @patch(fqname(Record), spec=Record)
    @patch(fqname(Record), spec=Record)
    def test_push(self, recordA, recordB, update):
        # recordA = Record(1)
        # recordB = Record(1)
        recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'}
        recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
        recordA.payload = {'testA': 1}
        recordB.payload = [{'testB': 2}, {'testB2': 3}]

        records = [recordA, recordB]
        self.outlet.try_start()
        self.outlet.push(records, update)

        for record in records:
            collection = self.outlet._get_collection(
                record.metadata.get(MongoOutlet.MONGODB_COLLECTION))
            payload = record.payload
            if not isinstance(payload, list):
                payload = [payload]

            for p in payload:
                result = collection.find_one({'_id': p.get('_id')})
                self.assertEqual(p, result)

        self.outlet.try_shutdown()

    @mongomock.patch(servers=(('localhost', 27017),))
    @patch(fqname(Update), spec=Update)
    @patch(fqname(Record), spec=Record)
    def test_push_inactive(self, record, update):
        result = self.outlet.push([record], update)
        self.assertFalse(result, 'Push should have stopped and returned False')

    @mongomock.patch(servers=(('localhost', 27017),))
    def test_ensure_connection(self):
        mock = MagicMock(foo=MagicMock(), _db=None)

        ensure_connection(mock.foo)(mock)

        mock.connect.assert_called_once()
        mock.foo.assert_called_once()
Esempio n. 6
0
class TestCsvOutlet(TestCase):
    @classmethod
    def setUpClass(cls):
        cls.this_filepath = os.path.abspath(os.path.dirname(__file__))
        cls.target_filepath = os.path.join(cls.this_filepath,
                                           '_csv_outlet_target.csv')
        cls.attempt_filepath = os.path.join(cls.this_filepath,
                                            '_csv_outlet_attempt.csv')
        cls.custom_filepath = os.path.join(cls.this_filepath,
                                           '_csv_outlet_custom_attempt.csv')
        with open(cls.target_filepath, 'r') as f:
            cls.target = f.read()

    @patch(fqname(Record), spec=Record)
    @patch(fqname(Update), spec=Update)
    def setUp(self, update, record):
        if os.path.exists(self.attempt_filepath):
            os.remove(self.attempt_filepath)

        if os.path.exists(self.custom_filepath):
            os.remove(self.custom_filepath)

        self.csv_outlet = CsvOutlet(self.attempt_filepath)
        self.update = update
        self.record = record

        record.payload = {'foo': 'bar', 'baz': 'qux'}
        record.metadata = {}

        record.__repr__ = lambda x: 'TestRecord(test)'
        update.__repr__ = lambda x: 'TestUpdate()'

        self.records = [copy.copy(record) for i in range(0, 4)]

    def test_push_append_mode(self):
        asyncio.run(self.csv_outlet._push(self.records, self.update))

        self.assertTrue(os.path.exists(self.attempt_filepath),
                        'File should exist')

        with open(self.attempt_filepath, 'r') as f:
            self.assertEqual(f.read(), self.target)

        os.remove(self.attempt_filepath)

    def test_push_write_mode(self):

        for record in self.records:
            record.metadata = {CsvOutlet.FILE_MODE: 'w'}

        asyncio.run(self.csv_outlet._push(self.records, self.update))

        self.assertTrue(os.path.exists(self.attempt_filepath),
                        'File should exist')

        with open(self.attempt_filepath, 'r') as f:
            self.assertEqual(
                f.read(), 'foo,baz\nbar,qux\n',
                'Each write should have overridden the previous.')

        os.remove(self.attempt_filepath)

    def test_push_custom_files(self):
        self.records[0].metadata = {CsvOutlet.CSV_FILE: self.custom_filepath}

        asyncio.run(self.csv_outlet._push(self.records, self.update))

        self.assertTrue(os.path.exists(self.attempt_filepath),
                        'File should exist')
        self.assertTrue(os.path.exists(self.custom_filepath),
                        'File should exist')

        with open(self.attempt_filepath, 'r') as f:
            self.assertEqual(f.read(), 'foo,baz\nbar,qux\nbar,qux\nbar,qux\n',
                             'Should miss one record')

        with open(self.custom_filepath, 'r') as f:
            self.assertEqual(f.read(), 'foo,baz\nbar,qux\n',
                             'Should contain the one record')

        os.remove(self.attempt_filepath)
        os.remove(self.custom_filepath)
Esempio n. 7
0
class InletTester(TestCase):
    """
    Utility class used for testing concrete implementations of :any:`Inlet`.
    """
    def get_inlet(self):  # pragma: no cover
        """Implement this method to return instances of your inlet class."""
        return NullInlet()

    def setUp(self):
        self.gmetadata = {'global': 'global'}
        self.inlets = self.get_inlet()
        if not isinstance(self.inlets, list):
            self.inlets = [self.inlets]

        self.inlet = self.inlets[0]
        self.inlet._metadata = {**self.inlet._metadata, **self.gmetadata}

    @for_each_inlet
    def test_new_record(self):
        """
        |decorated| :any:`for_each_inlet`

        Test creating new records and passing local metadata.
        """
        payload = {'test': 123}
        metadata = {'metadata': 321}
        record = self.inlet.new_record(payload=payload, metadata=metadata)
        self.assertIsInstance(record, Record)
        self.assertEqual(record.payload, payload)
        self.assertLessEqual(
            metadata.items(), record.metadata.items(),
            'Local metadata should be contained in record.metadata')
        self.assertLessEqual(
            self.gmetadata.items(), record.metadata.items(),
            'Global metadata should be contained in record.metadata')

    @for_each_inlet
    def test_new_record_override_global(self):
        """
        |decorated| :any:`for_each_inlet`

        Test creating new records and overriding global metadata.
        """
        payload = {'test': 123}
        metadata = {'global': 'local'}
        record = self.inlet.new_record(payload=payload, metadata=metadata)
        self.assertIsInstance(record, Record)
        self.assertEqual(record.payload, payload)
        self.assertLessEqual(
            metadata.items(), record.metadata.items(),
            'Local metadata should be contained in record.metadata')
        self.assertEqual(
            record.metadata['global'], 'local',
            'Global metadata should be overridden by local metadata')

    @patch(fqname(Update))
    @for_each_inlet
    def test_dont_read_metadata(self, update):
        """
        |decorated| :any:`for_each_inlet`

        Test creating new records and overriding global metadata.
        """

        meta = MagicMock()
        meta.keys.return_value = self.gmetadata.keys()
        meta.__getitem__.side_effect = lambda x: self.gmetadata.__getitem__(x)

        self.inlet._metadata = meta
        self.inlet.on_start()
        records = asyncio.run(self.inlet._pull(update))
        self.inlet.on_shutdown()
        self.inlet._metadata.get.assert_not_called()
        self.assertEqual(self.inlet._metadata.__getitem__.call_count,
                         len(self.gmetadata))

        self.assertIsInstance(records, (list))
        for record in records:
            self.assertIsInstance(record, Record)
            self.assertIsNotNone(record.payload)
            self.assertLessEqual(
                self.gmetadata.items(), record.metadata.items(),
                'Global metadata should be contained in record.metadata')

    @for_each_inlet
    @patch(fqname(Update))
    def test_pull(self, update):
        """
        |decorated| :any:`for_each_inlet`

        Test pulling data from the inlet.
        """
        self.inlet.on_start()
        records = asyncio.run(self.inlet._pull(update))
        self.inlet.on_shutdown()

        self.assertIsInstance(records, (list))
        for record in records:
            self.assertIsInstance(record, Record)
            self.assertIsNotNone(record.payload)
            self.assertLessEqual(
                self.gmetadata.items(), record.metadata.items(),
                'Global metadata should be contained in record.metadata')
Esempio n. 8
0
class TestLink(TestCase):
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_transfer(self, inlet, outlet):
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    name='test_update')

        link.transfer()

        inlet._pull.assert_called()
        outlet._push.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_run(self, inlet, outlet):
        async def task():
            link = Link([inlet], [outlet],
                        timedelta(seconds=1),
                        name='test_run',
                        copy_records=False)

            inlet_result = await inlet._pull(None)
            await link._run()

            inlet._pull.assert_called()
            outlet._push.assert_called_with(inlet_result, mock.ANY)

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_inlet(self, inlet, outlet):
        inlet._pull.side_effect = DummyException('Test exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    catch_exceptions=False,
                    name='test_exception_inlet')

        self.assertRaises(DummyException, link.transfer)

        inlet._pull.assert_called()
        outlet._push.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_outlet(self, inlet, outlet):
        # inlet._pull.return_value, _ = self.inlet_return()
        # inlet._pull.side_effect = pull_mock
        # inlet._pull.return_value = Future()
        outlet._push.side_effect = DummyException('Test exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    catch_exceptions=False,
                    name='test_exception_outlet')

        self.assertRaises(DummyException, link.transfer)

        inlet._pull.assert_called()
        outlet._push.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_caught(self, inlet, outlet):
        logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
        inlet._pull.side_effect = DummyException('Test inlet exception')
        outlet._push.side_effect = DummyException('Test outlet exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    name='test_exception_caught',
                    catch_exceptions=True)

        try:
            link.transfer()
        except Exception as e:
            self.fail(f'Should not raise exception: {e}')

        inlet._pull.assert_called()
        outlet._push.assert_called()

    # check that one exception doesn't halt other inlets or outlets
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_catch_partial_exception(self, inlet1, inlet2, outlet1, outlet2):
        logging.getLogger('databay.Link').setLevel(logging.CRITICAL)

        async def task():
            # inlet_future = Future()
            inlet1._pull.side_effect = DummyException('Test inlet1 exception')
            outlet1._push.side_effect = DummyException(
                'Test outlet1 exception')
            # inlet1._pull.return_value = inlet_future
            # inlet2._pull.return_value = inlet_future
            link = Link([inlet1, inlet2], [outlet1, outlet2],
                        timedelta(seconds=1),
                        name='test_catch_partial_exception',
                        copy_records=False,
                        catch_exceptions=True)

            # results = [object()]
            results = await inlet2._pull(None)
            # inlet_future.set_result(results)

            await link._run()

            inlet1._pull.assert_called()
            inlet2._pull.assert_called()
            outlet1._push.assert_called_with(results, mock.ANY)
            outlet2._push.assert_called_with(results, mock.ANY)

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_start(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=False)
        type(outlet1).active = mock.PropertyMock(return_value=False)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    name='test_on_start')

        link.on_start()

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_start_already_active(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=True)
        type(outlet1).active = mock.PropertyMock(return_value=True)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    name='test_on_start_already_active')

        link.on_start()

        inlet1.on_start.assert_not_called()
        outlet1.on_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_shutdown(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=True)
        type(outlet1).active = mock.PropertyMock(return_value=True)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    name='test_on_shutdown')

        link.on_shutdown()

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_already_inactive(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=False)
        type(outlet1).active = mock.PropertyMock(return_value=False)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    name='test_on_shutdown_already_inactive')

        link.on_shutdown()

        inlet1.on_shutdown.assert_not_called()
        outlet1.on_shutdown.assert_not_called()

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet(self, inlet1):
        link = Link([], [], timedelta(seconds=1), name='test_add_inlet')

        link.add_inlets(inlet1)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet_multiple(self, inlet1, inlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    name='test_add_inlet_multiple')

        link.add_inlets([inlet1, inlet2])

        self.assertEqual(link.inlets, [inlet1, inlet2])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet_same(self, inlet1):
        link = Link([], [], timedelta(seconds=1), name='test_add_inlet_same')

        link.add_inlets(inlet1)
        self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_remove_inlet(self, inlet1, inlet2):
        link = Link([], [], timedelta(seconds=1), name='test_remove_inlet')

        link.add_inlets([inlet1, inlet2])
        link.remove_inlets(inlet2)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_remove_inlet_invalid(self, inlet1, inlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    name='test_remove_inlet_invalid')

        link.add_inlets([inlet1])

        self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet(self, outlet1):
        link = Link([], [], timedelta(seconds=1), name='test_add_outlet')

        link.add_outlets(outlet1)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet_multiple(self, outlet1, outlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    name='test_add_outlet_multiple')

        link.add_outlets([outlet1, outlet2])

        self.assertEqual(link.outlets, [outlet1, outlet2])

    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet_same(self, outlet1):
        link = Link([], [], timedelta(seconds=1), name='test_add_outlet_same')

        link.add_outlets(outlet1)
        self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_remove_outlet(self, outlet1, outlet2):
        link = Link([], [], timedelta(seconds=1), name='test_remove_outlet')

        link.add_outlets([outlet1, outlet2])
        link.remove_outlets(outlet2)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_remove_outlet_invalid(self, outlet1, outlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    name='test_remove_outlet_invalid')

        link.add_outlets([outlet1])

        self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Inlet), spec=Inlet,
           _pull=pull_mock(object()))  # this rv is invalid, should be a list
    def xtest_non_iterable_raised(self, inlet1):
        logging.getLogger('databay.Link').setLevel(logging.ERROR)
        link = Link([inlet1], [],
                    timedelta(seconds=1),
                    name='test_non_iterable_raised')
        with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
            link.transfer()

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable())
           )  # this rv will raise DummyException
    def test_generic_error_raised(self, inlet1):
        logging.getLogger('databay.Link').setLevel(logging.ERROR)
        link = Link([inlet1], [],
                    timedelta(seconds=1),
                    name='test_generic_error_raised')
        # with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION):
        self.assertRaises(DummyException, link.transfer)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet,
           _pull=pull_mock(object()))  # this rv is invalid, should be a list
    def xtest_non_iterable_caught(self, inlet1, inlet2, outlet1):
        logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
        link = Link([inlet1, inlet2], [outlet1],
                    timedelta(seconds=1),
                    name='test_non_iterable_caught',
                    catch_exceptions=True)
        results = asyncio.run(inlet2._pull(None))
        link.transfer()
        outlet1._push.assert_called_with(results, mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable())
           )  # this rv will raise DummyException
    def xtest_generic_error_caught(self, inlet1, inlet2, outlet1):
        logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
        link = Link([inlet1, inlet2], [outlet1],
                    timedelta(seconds=1),
                    name='test_generic_error_caught',
                    catch_exceptions=True)
        results = asyncio.run(inlet2._pull(None))
        link.transfer()
        outlet1._push.assert_called_with(results, mock.ANY)
Esempio n. 9
0
class TestBuffers(TestCase):
    @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
    @patch(fqname(Inlet), spec=Inlet)
    def test_buffer_count(self, inlet, outlet):
        buffer = Buffer(count_threshold=3)

        payload = [1, 2, 3, 4]
        records = [Record(payload=p) for p in payload]

        link = Link(inlet,
                    outlet,
                    interval=1,
                    processors=buffer,
                    copy_records=False)

        inlet._pull = pull_mock(records[:2])
        link.transfer()
        outlet._push.assert_called_with(
            [], mock.ANY)  # after first call we shouldn't have any records

        inlet._pull = pull_mock(records[2:])
        link.transfer()
        outlet._push.assert_called_with(
            records, mock.ANY)  # all records should be returned here

    @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
    @patch(fqname(Inlet), spec=Inlet)
    def test_buffer_time(self, inlet, outlet):
        buffer = Buffer(time_threshold=0.02)

        payload = [1, 2, 3, 4]
        records = [Record(payload=p) for p in payload]

        link = Link(inlet,
                    outlet,
                    interval=1,
                    processors=buffer,
                    copy_records=False)

        inlet._pull = pull_mock(records[:2])
        link.transfer()
        outlet._push.assert_called_with(
            [], mock.ANY)  # not enough time have passed

        inlet._pull = pull_mock(records[2:])
        link.transfer()
        outlet._push.assert_called_with(
            [], mock.ANY)  # not enough time have passed

        time.sleep(0.02)

        inlet._pull = pull_mock([])
        link.transfer()
        outlet._push.assert_called_with(
            records, mock.ANY)  # all records should be returned here

    @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
    @patch(fqname(Inlet), spec=Inlet)
    def test_flush(self, inlet, outlet):
        buffer = Buffer(count_threshold=100, time_threshold=10)

        payload = [1, 2, 3, 4]
        records = [Record(payload=p) for p in payload]

        link = Link(inlet,
                    outlet,
                    interval=1,
                    processors=buffer,
                    copy_records=False)

        inlet._pull = pull_mock(records[:2])
        link.transfer()
        outlet._push.assert_called_with([], mock.ANY)  # no records yet

        inlet._pull = pull_mock(records[2:])
        link.transfer()
        outlet._push.assert_called_with([], mock.ANY)  # no records yet

        buffer.flush = True
        inlet._pull = pull_mock([])
        link.transfer()
        outlet._push.assert_called_with(
            records, mock.ANY)  # all records should be flushed

    @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
    @patch(fqname(Inlet), spec=Inlet)
    def test_flush_after_shutdown(self, inlet, outlet):
        buffer = Buffer(count_threshold=100, time_threshold=10)

        counter_dict = {'counter': 0, 'records': []}

        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    processors=buffer,
                    copy_records=False)
        planner = SchedulePlanner(link, refresh_interval=0.01)

        async def pull_coro(_):
            counter_dict['counter'] += 1
            record = Record(payload=counter_dict['counter'])
            counter_dict['records'].append(record)
            return [record]

        mock_pull = MagicMock(side_effect=pull_coro)
        inlet._pull = mock_pull

        th = Thread(target=planner.start, daemon=True)
        th.start()
        time.sleep(0.1)

        planner.shutdown()
        th.join()

        calls = outlet._push.call_args_list
        for c in calls:
            self.assertEqual(c(), [],
                             'Should only contain empty record lists.')
        self.assertEqual(buffer.records, counter_dict['records'],
                         'All records should be stored in the buffer')

        planner.force_transfer()
        self.assertEqual(outlet._push.call_args(), [],
                         'Should return empty record list')

        buffer.flush = True
        planner.force_transfer()
        self.assertEqual(outlet._push.call_args[0][0], counter_dict['records'],
                         'Should return all records')
Esempio n. 10
0
class TestAPSPlanner(TestCase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        logging.getLogger('databay').setLevel(logging.WARNING)

    @patch(fqname(Link), spec=Link)
    def setUp(self, link):
        self.planner = APSPlanner()

        def set_job(job):
            link.job = job

        link.interval.total_seconds.return_value = 0.02
        link.set_job.side_effect = set_job
        link.job = None
        self.link = link

    def test__schedule(self):
        self.planner._schedule(self.link)
        self.assertIsNotNone(self.link.job, 'Link should contain a job')
        asp_job = self.planner._scheduler.get_jobs()[0]
        self.assertEqual(self.link.job, asp_job,
                         'Link\'s job should be same as scheduler\'s')

    def test__unschedule(self):
        self.planner._schedule(self.link)
        self.planner._unschedule(self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertEqual(len(self.planner._scheduler.get_jobs()), 0,
                         'Scheduler should not have any jobs')

    def test__unschedule_invalid(self):
        self.planner._unschedule(self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertEqual(len(self.planner._scheduler.get_jobs()), 0,
                         'Scheduler should not have any jobs')

    def test_add_links(self):
        self.planner.add_links(self.link)
        self.assertIsNotNone(self.link.job, 'Link should contain a job')
        self.assertTrue(self.link in self.planner.links,
                        'Planner should contain the link')

    def test_add_links_on_init(self):
        self.planner = APSPlanner(self.link)
        self.assertIsNotNone(self.link.job, 'Link should contain a job')
        self.assertTrue(self.link in self.planner.links,
                        'Planner should contain the link')

    def test_remove_links(self):
        self.planner.add_links(self.link)
        self.planner.remove_links(self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertTrue(self.link not in self.planner.links,
                        'Planner should not contain the link')

    def test_remove_invalid_link(self):
        self.assertRaises(MissingLinkError, self.planner.remove_links,
                          self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertTrue(self.link not in self.planner.links,
                        'Planner should not contain the link')

    def test_start(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.assertTrue(self.planner.running, 'Scheduler should be running')
        self.planner.shutdown(False)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    # todo: APS is currently broken for this test case, wait for an update
    def xtest_start_paused(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.pause()
        self.assertRaises(SchedulerAlreadyRunningError, self.planner.start)
        self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
                         'Scheduler should be paused')
        self.planner.shutdown(False)

        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_shutdown(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.shutdown(False)
        self.assertFalse(self.planner.running,
                         'Scheduler should not be running')
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_pause(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.pause()
        self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
                         'Scheduler should be paused')
        self.planner.shutdown(False)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_resume(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.pause()
        self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
                         'Scheduler should be paused')
        self.planner.resume()
        self.assertTrue(self.planner.running, 'Scheduler should not be paused')
        self.planner.shutdown(False)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_shutdown_paused(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.pause()
        self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
                         'Scheduler should be paused')
        self.planner.shutdown(False)
        self.assertFalse(self.planner.running,
                         'Scheduler should not be running')
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_pause_shutdown(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.shutdown(False)
        self.assertRaises(SchedulerNotRunningError, self.planner.pause)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_resume_shutdown(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.shutdown(False)
        self.assertRaises(SchedulerNotRunningError, self.planner.resume)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_start_shutdown(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.shutdown(False)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread 1 should be stopped.')
        self.assertFalse(self.planner.running,
                         'Scheduler should not be running')

        th2 = Thread(target=self.planner.start, daemon=True)
        th2.start()
        self.assertTrue(self.planner.running, 'Scheduler should be running')
        self.planner.shutdown(False)
        th2.join(timeout=2)
        self.assertFalse(th2.is_alive(), 'Thread 2 should be stopped.')

    def test_add_and_run(self):
        self.link.interval.total_seconds.return_value = 0.02
        self.planner.add_links(self.link)

        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        time.sleep(0.04)
        self.link.transfer.assert_called()

        self.planner.shutdown(False)
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def _with_exception(self, link, catch_exceptions):
        logging.getLogger('databay').setLevel(logging.CRITICAL)
        # logging.getLogger('databay').setLevel(logging.INFO)
        self.planner = APSPlanner(catch_exceptions=catch_exceptions)

        link.transfer.side_effect = DummyException()
        link.interval.total_seconds.return_value = 0.02
        self.planner.add_links(link)

        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        time.sleep(0.04)
        link.transfer.assert_called()

        if catch_exceptions:
            self.assertTrue(self.planner.running,
                            'Scheduler should be running')
            self.planner.shutdown(False)
            th.join(timeout=2)
            self.assertFalse(th.is_alive(), 'Thread should be stopped.')

        self.assertFalse(self.planner.running, 'Scheduler should be stopped')

    def test_catch_exception(self):
        self._with_exception(self.link, True)

    def test_raise_exception(self):
        self._with_exception(self.link, False)

    def test_uncommon_exception(self):
        logging.getLogger('databay').setLevel(logging.CRITICAL)

        self.link.transfer.side_effect = DummyUnusualException(123, True)
        self.link.interval.total_seconds.return_value = 0.02
        self.planner.add_links(self.link)

        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        time.sleep(0.04)
        self.link.transfer.assert_called()

        self.assertFalse(self.planner.running, 'Scheduler should be stopped')
Esempio n. 11
0
class TestLink(TestCase):
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_transfer(self, inlet, outlet):
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    tags='test_update')

        link.transfer()

        inlet._pull.assert_called()
        outlet._push.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_run(self, inlet, outlet):
        async def task():
            link = Link([inlet], [outlet],
                        timedelta(seconds=1),
                        tags='test_run',
                        copy_records=False)

            inlet_result = await inlet._pull(None)
            await link._run()

            inlet._pull.assert_called()
            outlet._push.assert_called_with(inlet_result, mock.ANY)

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_inlet(self, inlet, outlet):
        inlet._pull.side_effect = DummyException('Test exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    ignore_exceptions=False,
                    tags='test_exception_inlet')

        self.assertRaises(DummyException, link.transfer)

        inlet._pull.assert_called()
        outlet._push.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_outlet(self, inlet, outlet):
        # inlet._pull.return_value, _ = self.inlet_return()
        # inlet._pull.side_effect = pull_mock
        # inlet._pull.return_value = Future()
        outlet._push.side_effect = DummyException('Test exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    ignore_exceptions=False,
                    tags='test_exception_outlet')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    ignore_exceptions=False,
                    tags='test_exception_outlet')

        self.assertRaises(DummyException, link.transfer)

        inlet._pull.assert_called()
        outlet._push.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_exception_caught(self, inlet, outlet):
        inlet._pull.side_effect = DummyException('Test inlet exception')
        outlet._push.side_effect = DummyException('Test outlet exception')
        link = Link([inlet], [outlet],
                    timedelta(seconds=1),
                    tags='test_exception_caught',
                    ignore_exceptions=True)

        try:
            with self.assertLogs(logging.getLogger('databay.Link'),
                                 level='WARNING') as cm:
                link.transfer()
                self.assertTrue('Test inlet exception' in ';'.join(cm.output))
                self.assertTrue('Test outlet exception' in ';'.join(cm.output))
        except Exception as e:
            self.fail(f'Should not raise exception: {e}')

        inlet._pull.assert_called()
        outlet._push.assert_called()

    # check that one exception doesn't halt other inlets or outlets
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_ignore_partial_exception(self, inlet1, inlet2, outlet1, outlet2):
        async def task():
            # inlet_future = Future()
            inlet1._pull.side_effect = DummyException('Test inlet1 exception')
            outlet1._push.side_effect = DummyException(
                'Test outlet1 exception')
            # inlet1._pull.return_value = inlet_future
            # inlet2._pull.return_value = inlet_future
            link = Link([inlet1, inlet2], [outlet1, outlet2],
                        timedelta(seconds=1),
                        tags='test_ignore_partial_exception',
                        copy_records=False,
                        ignore_exceptions=True)

            # results = [object()]
            results = await inlet2._pull(None)
            # inlet_future.set_result(results)

            with self.assertLogs(logging.getLogger('databay.Link'),
                                 level='WARNING') as cm:
                await link._run()
                self.assertTrue('Test inlet1 exception' in ';'.join(cm.output))
                self.assertTrue(
                    'Test outlet1 exception' in ';'.join(cm.output))

            inlet1._pull.assert_called()
            inlet2._pull.assert_called()
            outlet1._push.assert_called_with(results, mock.ANY)
            outlet2._push.assert_called_with(results, mock.ANY)

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_start(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=False)
        type(outlet1).active = mock.PropertyMock(return_value=False)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start')

        link.on_start()

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_start_already_active(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=True)
        type(outlet1).active = mock.PropertyMock(return_value=True)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start_already_active')

        link.on_start()

        inlet1.on_start.assert_not_called()
        outlet1.on_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_on_shutdown(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=True)
        type(outlet1).active = mock.PropertyMock(return_value=True)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown')

        link.on_shutdown()

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_already_inactive(self, inlet1, outlet1):
        type(inlet1).active = mock.PropertyMock(return_value=False)
        type(outlet1).active = mock.PropertyMock(return_value=False)
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown_already_inactive')

        link.on_shutdown()

        inlet1.on_shutdown.assert_not_called()
        outlet1.on_shutdown.assert_not_called()

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet(self, inlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_inlet')

        link.add_inlets(inlet1)

        self.assertEqual(link.inlets, [inlet1])

    def test_add_inlet_invalid(self):
        link = Link([], [], timedelta(seconds=1), tags='test_add_inlet')
        self.assertRaises(TypeError, link.add_inlets, "invalid_inlet")

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet_multiple(self, inlet1, inlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_add_inlet_multiple')

        link.add_inlets([inlet1, inlet2])

        self.assertEqual(link.inlets, [inlet1, inlet2])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_add_inlet_same(self, inlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same')

        link.add_inlets(inlet1)
        self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_remove_inlet(self, inlet1, inlet2):
        link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet')

        link.add_inlets([inlet1, inlet2])
        link.remove_inlets(inlet2)

        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
    def test_remove_inlet_invalid(self, inlet1, inlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_remove_inlet_invalid')

        link.add_inlets([inlet1])

        self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
        self.assertEqual(link.inlets, [inlet1])

    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet(self, outlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_outlet')

        link.add_outlets(outlet1)

        self.assertEqual(link.outlets, [outlet1])

    def test_add_outlet_invalid(self):
        link = Link([], [], timedelta(seconds=1), tags='test_add_outlet')
        self.assertRaises(TypeError, link.add_outlets, "invalid_outlet")

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet_multiple(self, outlet1, outlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_add_outlet_multiple')

        link.add_outlets([outlet1, outlet2])

        self.assertEqual(link.outlets, [outlet1, outlet2])

    @patch(fqname(Outlet), spec=Outlet)
    def test_add_outlet_same(self, outlet1):
        link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same')

        link.add_outlets(outlet1)
        self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_remove_outlet(self, outlet1, outlet2):
        link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet')

        link.add_outlets([outlet1, outlet2])
        link.remove_outlets(outlet2)

        self.assertEqual(link.outlets, [outlet1])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    def test_remove_outlet_invalid(self, outlet1, outlet2):
        link = Link([], [],
                    timedelta(seconds=1),
                    tags='test_remove_outlet_invalid')

        link.add_outlets([outlet1])

        self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
        self.assertEqual(link.outlets, [outlet1])

    # this rv is invalid, should be a list
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object()))
    def xtest_non_iterable_raised(self, inlet1):
        logging.getLogger('databay.Link').setLevel(logging.ERROR)
        link = Link([inlet1], [],
                    timedelta(seconds=1),
                    tags='test_non_iterable_raised')
        with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
            link.transfer()

    # this rv will raise DummyException
    @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable()))
    def test_generic_error_raised(self, inlet1):
        link = Link([inlet1], [],
                    timedelta(seconds=1),
                    tags='test_generic_error_raised')
        # with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION):
        self.assertRaises(DummyException, link.transfer)

    def test_integer_to_timedelta(self):
        link = Link([], [], 1, name='test_integer_interval_coerced')
        self.assertEqual(link._interval, timedelta(seconds=1))

    def test_float_to_timedelta(self):
        link = Link([], [], 1.5, name='test_float_interval_coerced')
        self.assertEqual(link._interval, timedelta(seconds=1.5))

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_inlet_exception_raise(self, inlet1, outlet1):
        inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start')

        self.assertRaises(RuntimeError, link.on_start)

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_inlet_exception_catch(self, inlet1, outlet1):
        inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_start',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_start()
            self.assertTrue('on_start inlet exception: "" for inlet:' in
                            ';'.join(cm.output))

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_outlet_exception_raise(self, inlet1, outlet1, outlet2):
        outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_start')

        self.assertRaises(RuntimeError, link.on_start)

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()
        outlet2.try_start.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_start_outlet_exception_catch(self, inlet1, outlet1, outlet2):
        outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_start',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_start()
            self.assertTrue(
                'on_start outlet exception: "" for outlet:'
                in ';'.join(cm.output), cm.output)

        inlet1.try_start.assert_called()
        outlet1.try_start.assert_called()
        outlet2.try_start.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_inlet_exception_raise(self, inlet1, outlet1):
        inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown')

        self.assertRaises(RuntimeError, link.on_shutdown)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_inlet_exception_catch(self, inlet1, outlet1):
        inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
        link = Link([inlet1], [outlet1],
                    timedelta(seconds=1),
                    tags='test_on_shutdown',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_shutdown()
            self.assertTrue(
                'on_shutdown inlet exception: "" for inlet:'
                in ';'.join(cm.output), cm.output)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_outlet_exception_raise(self, inlet1, outlet1,
                                                outlet2):
        outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
                                                        )
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_shutdown')

        self.assertRaises(RuntimeError, link.on_shutdown)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()
        outlet2.try_shutdown.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_on_shutdown_outlet_exception_catch(self, inlet1, outlet1,
                                                outlet2):
        outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
                                                        )
        link = Link([inlet1], [outlet1, outlet2],
                    timedelta(seconds=1),
                    tags='test_on_shutdown',
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.on_shutdown()
            self.assertTrue(
                'on_shutdown outlet exception: "" for outlet:'
                in ';'.join(cm.output), cm.output)

        inlet1.try_shutdown.assert_called()
        outlet1.try_shutdown.assert_called()
        outlet2.try_shutdown.assert_called()

    def test_single_tag(self):
        tag = 'tagA'
        link = Link([], [], timedelta(seconds=1), tags=tag)
        self.assertEqual(link.tags, [tag])

    def test_multiple_tags(self):
        tagA = 'tagA'
        tagB = 'tagB'
        link = Link([], [], timedelta(seconds=1), tags=[tagA, tagB])
        self.assertEqual(link.tags, [tagA, tagB])

    def test_tag_as_name(self):
        link_name = 'link_name'
        link = Link([], [], timedelta(seconds=1), name=link_name)
        self.assertEqual(link_name, link.tags[0])

    def test_name_from_tag(self):
        link_name = 'link_name'
        link = Link([], [], timedelta(seconds=1), tags=[link_name])
        self.assertEqual(link.name, link.tags[0])

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    @patch(fqname(Inlet), spec=Inlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_inlet_concurrency(self, inlet1, inlet2, inlet3, outlet):
        counter = {'value': 0}

        # this will increment the counter on each async call, and check if they exceed the concurrency value
        async def slow_pull(_):
            counter['value'] += 1
            self.assertLessEqual(counter['value'], 2,
                                 "Only 2 inlets should pull at a time")
            await asyncio.sleep(0.01)
            counter['value'] -= 1
            return [123]

        inlet1._pull = slow_pull
        inlet2._pull = slow_pull
        inlet3._pull = slow_pull

        async def task():
            link = Link([inlet1, inlet2, inlet3], [outlet],
                        timedelta(seconds=1),
                        tags='test_inlet_concurrency',
                        copy_records=False,
                        inlet_concurrency=2)

            await link._run()

        asyncio.run(task())

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_processors_one(self, inlet, outlet):
        records = [2, 3]
        inlet._pull = pull_mock(records)
        processor = MagicMock(side_effect=lambda r: r)
        link = Link(inlet, outlet, interval=0.01, processors=processor)
        link.transfer()
        processor.assert_called_with(records)
        outlet._push.assert_called_with(records, mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_processors_many(self, inlet, outlet):
        records = [2, 3]
        inlet._pull = pull_mock(records)
        processorA = MagicMock(side_effect=lambda x: x)
        processorB = MagicMock(side_effect=lambda x: x)
        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    processors=[processorA, processorB])
        link.transfer()
        processorA.assert_called_with(records)
        processorB.assert_called_with(records)
        outlet._push.assert_called_with(records, mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_processors_change_records(self, inlet, outlet):
        records = [2, 3]
        inlet._pull = pull_mock(records)
        processorA = MagicMock(
            side_effect=lambda r: list(map(lambda y: y * y, r)))
        processorB = MagicMock(
            side_effect=lambda r: list(map(lambda y: -y, r)))
        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    processors=[processorA, processorB])
        link.transfer()
        processorA.assert_called_with(records)
        processorB.assert_called_with([4, 9])
        outlet._push.assert_called_with([-4, -9], mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_processors_filter_records(self, inlet, outlet):
        records = [2, 3, 4]
        inlet._pull = pull_mock(records)
        processorA = MagicMock(
            side_effect=lambda r: list(filter(lambda y: y > 2, r)))
        processorB = MagicMock(
            side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r)))
        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    processors=[processorA, processorB])
        link.transfer()
        processorA.assert_called_with(records)
        processorB.assert_called_with([3, 4])
        outlet._push.assert_called_with([4], mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_groupers_one_passive(self, inlet, outlet):
        records = [1, 2, 3, 4]
        inlet._pull = pull_mock(records)
        grouper = MagicMock(side_effect=lambda r: r)  # does nothing on purpose
        link = Link(inlet, outlet, interval=0.01, groupers=grouper)
        link.transfer()
        grouper.assert_called_with([records])
        outlet._push.assert_called_with(records, mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_groupers_one_active(self, inlet, outlet):
        records = [1, 2, 3, 4]
        inlet._pull = pull_mock(records)

        # makes [[1,2], [3,4]]
        grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])

        link = Link(inlet, outlet, interval=0.01, groupers=grouper)
        link.transfer()
        grouper.assert_called_with([records])
        calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
        outlet._push.assert_has_calls(calls)  # expects [[1,2], [3,4]]

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_groupers_many_active(self, inlet, outlet):
        records = [1, 2, 3, 4]
        inlet._pull = pull_mock(records)

        # makes [[1,2], [3,4]]
        grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])

        # makes [[1], [2], [3], [4]]
        grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] +
                             [[sub] for sub in r[1]])

        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    groupers=[grouperA, grouperB])
        link.transfer()

        grouperA.assert_called_with([records])

        callsA = [call([records[:2], records[2:]])]  # expects [[1,2], [3,4]]
        grouperB.assert_has_calls(callsA)

        callsB = [
            call([records[0]], mock.ANY),
            call([records[1]], mock.ANY),
            call([records[2]], mock.ANY),
            call([records[3]], mock.ANY)
        ]
        outlet._push.assert_has_calls(callsB)  # expects [[1], [2], [3], [4]]

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_processor_exception(self, inlet, outlet):
        records = [2, 3]
        inlet._pull = pull_mock(records)
        processor = MagicMock(
            side_effect=DummyException('Processor exception'))
        link = Link(inlet, outlet, interval=0.01, processors=processor)
        self.assertRaises(DummyException, link.transfer)
        processor.assert_called_with(records)
        outlet._push.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_processor_exception_ignored(self, inlet, outlet):
        records = [2, 3]
        inlet._pull = pull_mock(records)
        processorA = MagicMock(
            side_effect=DummyException('Processor exception'))
        processorB = MagicMock(
            side_effect=lambda r: list(map(lambda y: y * y, r)))
        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    processors=[processorA, processorB],
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.transfer()
            self.assertTrue('Processor exception:' in ';'.join(cm.output),
                            cm.output)

        processorA.assert_called_with(records)
        processorB.assert_called_with(records)
        outlet._push.assert_called_with([4, 9], mock.ANY)

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_grouper_exception(self, inlet, outlet):
        records = [2, 3]
        inlet._pull = pull_mock(records)
        grouper = MagicMock(side_effect=DummyException('Grouper exception'))
        link = Link(inlet, outlet, interval=0.01, groupers=grouper)
        self.assertRaises(DummyException, link.transfer)
        grouper.assert_called_with([records])
        outlet._push.assert_not_called()

    @patch(fqname(Outlet), spec=Outlet)
    @patch(fqname(Inlet), spec=Inlet)
    def test_grouper_exception_ignored(self, inlet, outlet):
        records = [1, 2, 3, 4]
        inlet._pull = pull_mock(records)
        grouperA = MagicMock(side_effect=DummyException('Grouper exception'))
        grouperB = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
        link = Link(inlet,
                    outlet,
                    interval=0.01,
                    groupers=[grouperA, grouperB],
                    ignore_exceptions=True)

        with self.assertLogs(logging.getLogger('databay.Link'),
                             level='ERROR') as cm:
            link.transfer()
            self.assertTrue('Grouper exception:' in ';'.join(cm.output),
                            cm.output)

        grouperA.assert_called_with([records])
        grouperB.assert_called_with([records])
        calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
        outlet._push.assert_has_calls(calls)  # expects [[1,2], [3,4]]
Esempio n. 12
0
class TestSchedulePlanner(TestCase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        logging.getLogger('databay').setLevel(logging.WARNING)

    @patch(fqname(Link), spec=Link)
    def setUp(self, link):
        self.planner = SchedulePlanner(refresh_interval=0.02)

        def set_job(job):
            link.job = job

        link.interval.total_seconds.return_value = 0.02
        link.set_job.side_effect = set_job
        link.job = None
        self.link = link

    def tearDown(self):
        if len(schedule.jobs) > 0:
            schedule.clear()

    def test__run_job(self):
        self.planner._create_thread_pool()
        self.planner._run_job(self.link)
        self.link.transfer.assert_called_once()
        self.planner._destroy_thread_pool()

    def test__schedule(self):
        self.planner._schedule(self.link)
        self.assertIsNotNone(self.link.job, 'Link should contain a job')
        schedule_job = schedule.jobs[0]
        self.assertEqual(self.link.job, schedule_job,
                         'Link\'s job should be same as schedule\'s')
        # self.planner._unschedule(link)

    def test__unschedule(self):
        self.planner._schedule(self.link)
        self.planner._unschedule(self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertEqual(len(schedule.jobs), 0,
                         'Schedule should not have any jobs')

    def test__unschedule_invalid(self):
        self.planner._unschedule(self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertEqual(len(schedule.jobs), 0,
                         'Scheduler should not have any jobs')

    def test_add_links(self):
        self.planner.add_links(self.link)
        self.assertIsNotNone(self.link.job, 'Link should contain a job')
        self.assertTrue(self.link in self.planner.links,
                        'Planner should contain the link')

    def test_add_links_on_init(self):
        self.planner = SchedulePlanner(self.link, refresh_interval=0.02)
        self.assertIsNotNone(self.link.job, 'Link should contain a job')
        self.assertTrue(self.link in self.planner.links,
                        'Planner should contain the link')

    def test_remove_links(self):
        self.planner.add_links(self.link)
        self.planner.remove_links(self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertTrue(self.link not in self.planner.links,
                        'Planner should not contain the link')

    def test_remove_invalid_link(self):
        self.assertRaises(MissingLinkError, self.planner.remove_links,
                          self.link)
        self.assertIsNone(self.link.job, 'Link should not contain a job')
        self.assertTrue(self.link not in self.planner.links,
                        'Planner should not contain the link')

    def test_start(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.assertTrue(self.planner._running, 'Planner should be running')
        self.planner.shutdown()
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_shutdown(self):
        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        self.planner.shutdown()
        self.assertFalse(self.planner._running,
                         'Planner should be not running')
        self.assertIsNone(self.planner._thread_pool,
                          'Planner should not have a thread pool')
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_add_and_run(self):
        self.link.interval.total_seconds.return_value = 0.02
        self.planner._refresh_interval = 0.02
        self.planner.add_links(self.link)

        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        time.sleep(0.04)
        self.link.transfer.assert_called()

        self.planner.shutdown()
        th.join(timeout=2)
        self.assertFalse(th.is_alive(), 'Thread should be stopped.')

    def test_invalid_interval(self):
        self.link.interval.total_seconds.return_value = 0.1
        self.planner._refresh_interval = 0.2

        self.assertRaises(ScheduleIntervalError, self.planner.add_links,
                          self.link)

    def _with_exception(self, link, catch_exceptions):
        logging.getLogger('databay').setLevel(logging.CRITICAL)
        self.planner = SchedulePlanner(catch_exceptions=catch_exceptions)
        link.transfer.side_effect = DummyException()
        link.interval.total_seconds.return_value = 0.02
        self.planner._refresh_interval = 0.02

        link.transfer.side_effect = DummyException()
        link.interval.total_seconds.return_value = 0.02
        self.planner.add_links(link)

        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        time.sleep(0.04)
        link.transfer.assert_called()

        if catch_exceptions:
            self.assertTrue(self.planner.running,
                            'Scheduler should be running')
            self.planner.shutdown(False)
            th.join(timeout=2)
            self.assertFalse(th.is_alive(), 'Thread should be stopped.')

        self.assertFalse(self.planner.running, 'Scheduler should be stopped')

    def test_catch_exception(self):
        self._with_exception(self.link, True)

    def test_raise_exception(self):
        self._with_exception(self.link, False)

    def test_uncommon_exception(self):
        logging.getLogger('databay').setLevel(logging.CRITICAL)

        self.link.transfer.side_effect = DummyUnusualException(argA=123,
                                                               argB=True)
        self.link.interval.total_seconds.return_value = 0.02
        self.planner.add_links(self.link)

        th = Thread(target=self.planner.start, daemon=True)
        th.start()
        time.sleep(0.04)
        self.link.transfer.assert_called()

        self.assertFalse(self.planner.running, 'Scheduler should be stopped')