class TestHttpInlet(inlet_tester.InletTester): def setUp(self): super().setUp() set_response(b'{"asdf":"12"}') def get_inlet(self): return HttpInlet('https://jsonplaceholder.typicode.com/todos/1') @patch(fqname(Update)) def test_json(self, update): records = asyncio.run(self.inlet._pull(update)) self.assertIsInstance(records, (list)) @patch(fqname(Update)) def test_html(self, update): self.inlet.json = False records = asyncio.run(self.inlet._pull(update)) self.assertIsInstance(records, (list)) @patch(fqname(Update)) def test_invalid_json(self, update): set_response(b'<html>') self.inlet.json = True self.assertRaisesRegex(ValueError, 'Response does not contain valid JSON', asyncio.run, self.inlet._pull(update)) @patch(fqname(Update)) def test_invalid_html(self, update): set_response(None) self.inlet.json = False self.assertRaisesRegex( AttributeError, '\'NoneType\' object has no attribute \'decode\'', asyncio.run, self.inlet._pull(update))
class TestPrintOutlet(TestCase): @patch(fqname(Record), spec=Record) @patch(fqname(Update), spec=Update) def setUp(self, update, record): self.print_outlet = PrintOutlet() self.update = update self.record = record record.payload = 'test' record.metadata.return_value = {} record.__repr__ = lambda x: 'TestRecord(test)' update.__repr__ = lambda x: 'TestUpdate()' @patch('sys.stdout', new_callable=io.StringIO) def test_push(self, stdout): asyncio.run(self.print_outlet._push([self.record], self.update)) self.assertEqual(stdout.getvalue(), 'TestUpdate() TestRecord(test)\n') @patch('sys.stdout', new_callable=io.StringIO) def test_push_only_payload(self, stdout): self.print_outlet.only_payload = True asyncio.run(self.print_outlet._push([self.record], self.update)) self.assertEqual(stdout.getvalue(), 'TestUpdate() test\n') @patch('sys.stdout', new_callable=io.StringIO) def test_push_skip_update(self, stdout): self.print_outlet.skip_update = True asyncio.run(self.print_outlet._push([self.record], self.update)) self.assertEqual(stdout.getvalue(), 'TestRecord(test)\n')
class TestLink(TestCase): @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_transfer(self, inlet, outlet): link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_update') link.transfer() inlet._pull.assert_called() outlet._push.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_run(self, inlet, outlet): async def task(): link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_run', copy_records=False) inlet_result = await inlet._pull(None) await link._run() inlet._pull.assert_called() outlet._push.assert_called_with(inlet_result, mock.ANY) asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_inlet(self, inlet, outlet): inlet._pull.side_effect = DummyException('Test exception') link = Link([inlet], [outlet], timedelta(seconds=1), ignore_exceptions=False, tags='test_exception_inlet') self.assertRaises(DummyException, link.transfer) inlet._pull.assert_called() outlet._push.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_outlet(self, inlet, outlet): # inlet._pull.return_value, _ = self.inlet_return() # inlet._pull.side_effect = pull_mock # inlet._pull.return_value = Future() outlet._push.side_effect = DummyException('Test exception') link = Link([inlet], [outlet], timedelta(seconds=1), ignore_exceptions=False, tags='test_exception_outlet') link = Link([inlet], [outlet], timedelta(seconds=1), ignore_exceptions=False, tags='test_exception_outlet') self.assertRaises(DummyException, link.transfer) inlet._pull.assert_called() outlet._push.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_caught(self, inlet, outlet): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) inlet._pull.side_effect = DummyException('Test inlet exception') outlet._push.side_effect = DummyException('Test outlet exception') link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_exception_caught', ignore_exceptions=True) try: link.transfer() except Exception as e: self.fail(f'Should not raise exception: {e}') inlet._pull.assert_called() outlet._push.assert_called() # check that one exception doesn't halt other inlets or outlets @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_ignore_partial_exception(self, inlet1, inlet2, outlet1, outlet2): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) async def task(): # inlet_future = Future() inlet1._pull.side_effect = DummyException('Test inlet1 exception') outlet1._push.side_effect = DummyException( 'Test outlet1 exception') # inlet1._pull.return_value = inlet_future # inlet2._pull.return_value = inlet_future link = Link([inlet1, inlet2], [outlet1, outlet2], timedelta(seconds=1), tags='test_ignore_partial_exception', copy_records=False, ignore_exceptions=True) # results = [object()] results = await inlet2._pull(None) # inlet_future.set_result(results) await link._run() inlet1._pull.assert_called() inlet2._pull.assert_called() outlet1._push.assert_called_with(results, mock.ANY) outlet2._push.assert_called_with(results, mock.ANY) asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_start(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start') link.on_start() inlet1.try_start.assert_called() outlet1.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_start_already_active(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start_already_active') link.on_start() inlet1.on_start.assert_not_called() outlet1.on_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_shutdown(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown') link.on_shutdown() inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_already_inactive(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown_already_inactive') link.on_shutdown() inlet1.on_shutdown.assert_not_called() outlet1.on_shutdown.assert_not_called() @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet(self, inlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet') link.add_inlets(inlet1) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet_multiple(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_multiple') link.add_inlets([inlet1, inlet2]) self.assertEqual(link.inlets, [inlet1, inlet2]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet_same(self, inlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same') link.add_inlets(inlet1) self.assertRaises(InvalidNodeError, link.add_inlets, inlet1) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_remove_inlet(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet') link.add_inlets([inlet1, inlet2]) link.remove_inlets(inlet2) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_remove_inlet_invalid(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet_invalid') link.add_inlets([inlet1]) self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet(self, outlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet') link.add_outlets(outlet1) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet_multiple(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_multiple') link.add_outlets([outlet1, outlet2]) self.assertEqual(link.outlets, [outlet1, outlet2]) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet_same(self, outlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same') link.add_outlets(outlet1) self.assertRaises(InvalidNodeError, link.add_outlets, outlet1) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_remove_outlet(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet') link.add_outlets([outlet1, outlet2]) link.remove_outlets(outlet2) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_remove_outlet_invalid(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet_invalid') link.add_outlets([outlet1]) self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2) self.assertEqual(link.outlets, [outlet1]) # this rv is invalid, should be a list @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object())) def xtest_non_iterable_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), tags='test_non_iterable_raised') with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'): link.transfer() # this rv will raise DummyException @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable())) def test_generic_error_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), tags='test_generic_error_raised') # with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION): self.assertRaises(DummyException, link.transfer) def test_integer_to_timedelta(self): link = Link([], [], 1, name='test_integer_interval_coerced') self.assertEqual(link._interval, timedelta(seconds=1)) def test_float_to_timedelta(self): link = Link([], [], 1.5, name='test_float_interval_coerced') self.assertEqual(link._interval, timedelta(seconds=1.5)) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_inlet_exception_raise(self, inlet1, outlet1): inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start') self.assertRaises(RuntimeError, link.on_start) inlet1.try_start.assert_called() outlet1.try_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_inlet_exception_catch(self, inlet1, outlet1): logging.getLogger('databay.Link').setLevel(logging.WARNING) inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_start() self.assertTrue( 'on_start inlet exception: "" for inlet:' in ';'.join(cm.output)) inlet1.try_start.assert_called() outlet1.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_outlet_exception_raise(self, inlet1, outlet1, outlet2): outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_start') self.assertRaises(RuntimeError, link.on_start) inlet1.try_start.assert_called() outlet1.try_start.assert_called() outlet2.try_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_outlet_exception_catch(self, inlet1, outlet1, outlet2): logging.getLogger('databay.Link').setLevel(logging.WARNING) outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_start', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_start() self.assertTrue( 'on_start outlet exception: "" for outlet:' in ';'.join(cm.output), cm.output) inlet1.try_start.assert_called() outlet1.try_start.assert_called() outlet2.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_inlet_exception_raise(self, inlet1, outlet1): inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown') self.assertRaises(RuntimeError, link.on_shutdown) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_inlet_exception_catch(self, inlet1, outlet1): logging.getLogger('databay.Link').setLevel(logging.WARNING) inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_shutdown() self.assertTrue( 'on_shutdown inlet exception: "" for inlet:' in ';'.join(cm.output), cm.output) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_outlet_exception_raise(self, inlet1, outlet1, outlet2): outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())' ) link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_shutdown') self.assertRaises(RuntimeError, link.on_shutdown) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() outlet2.try_shutdown.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_outlet_exception_catch(self, inlet1, outlet1, outlet2): logging.getLogger('databay.Link').setLevel(logging.WARNING) outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())' ) link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_shutdown', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_shutdown() self.assertTrue( 'on_shutdown outlet exception: "" for outlet:' in ';'.join(cm.output), cm.output) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() outlet2.try_shutdown.assert_called() def test_single_tag(self): tag = 'tagA' link = Link([], [], timedelta(seconds=1), tags=tag) self.assertEqual(link.tags, [tag]) def test_multiple_tags(self): tagA = 'tagA' tagB = 'tagB' link = Link([], [], timedelta(seconds=1), tags=[tagA, tagB]) self.assertEqual(link.tags, [tagA, tagB]) def test_tag_as_name(self): link_name = 'link_name' link = Link([], [], timedelta(seconds=1), name=link_name) self.assertEqual(link_name, link.tags[0]) def test_name_from_tag(self): link_name = 'link_name' link = Link([], [], timedelta(seconds=1), tags=[link_name]) self.assertEqual(link.name, link.tags[0])
class TestBasePlanner(TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logging.getLogger('databay').setLevel(logging.WARNING) @patch.multiple(BasePlanner, __abstractmethods__=set()) def setUp(self): self.planner = BasePlanner() self.planner._schedule = mock.Mock( side_effect=lambda link: link.set_job(object())) self.planner._unschedule = mock.Mock( side_effect=lambda link: link.set_job(None)) self.planner._start_planner = mock.Mock() self.planner._shutdown_planner = mock.Mock() @patch(fqname(Link), spec=Link) def test_add_links(self, link): def set_job(job): link.job = job link.set_job.side_effect = set_job link.job = None self.planner.add_links(link) self.assertIsNotNone(link.job, 'Link should contain a job') self.assertTrue(link in self.planner.links, 'Planner should contain the link') @patch(fqname(Link), spec=Link) def test_add_links_array(self, link): def set_job(job): link.job = job link.set_job.side_effect = set_job link.job = None self.planner.add_links([link]) self.assertIsNotNone(link.job, 'Link should contain a job') self.assertTrue(link in self.planner.links, 'Planner should contain the link') @patch(fqname(Link), spec=Link) def test_remove_links(self, link): def set_job(job): link.job = job link.set_job.side_effect = set_job link.job = None self.planner.add_links(link) self.planner.remove_links(link) self.assertIsNone(link.job, 'Link should not contain a job') self.assertTrue(link not in self.planner.links, 'Planner should not contain the link') @patch(fqname(Link), spec=Link) def test_remove_invalid_link(self, link): def set_job(job): link.job = job link.set_job.side_effect = set_job link.job = None self.assertRaises(MissingLinkError, self.planner.remove_links, link) self.assertIsNone(link.job, 'Link should not contain a job') self.assertTrue(link not in self.planner.links, 'Planner should not contain the link') @patch(fqname(Link), spec=Link) def test_start(self, link): self.planner.add_links(link) self.planner.start() link.on_start.assert_called() self.planner._start_planner.assert_called() @patch(fqname(Link), spec=Link) def test_shutdown(self, link): self.planner.add_links(link) self.planner.shutdown() link.on_shutdown.assert_called() self.planner._shutdown_planner.assert_called() @patch(fqname(Link), spec=Link) def test_start_order(self, link): # on_start should be called before _start_planner link.on_start.side_effect = lambda: self.planner._start_planner.assert_not_called( ) self.planner.add_links(link) self.planner.start() # finally both should be called link.on_start.assert_called() self.planner._start_planner.assert_called() @patch(fqname(Link), spec=Link) def test_shutdown_order(self, link): # on_shutdown should be called after _shutdown_planner self.planner._shutdown_planner.side_effect = lambda wait: link.on_shutdown.assert_not_called( ) self.planner.add_links(link) self.planner.shutdown() # finally both should be called link.on_shutdown.assert_called() self.planner._shutdown_planner.assert_called()
class TestMongoOutlet(TestCase): def setUp(self): self.outlet = MongoOutlet() @mongomock.patch(servers=(('localhost', 27017),)) def test_connect(self): result = self.outlet.connect() self.assertFalse(result) self.assertEqual(self.outlet._db.name, 'databay') self.assertEqual(self.outlet._client.address[0], 'localhost') self.assertEqual(self.outlet._client.address[1], 27017) @mongomock.patch(servers=(('test', 1234),)) def test_connect_custom_host(self): self.outlet = MongoOutlet(host='test', port=1234) self.outlet.connect() self.assertEqual(self.outlet._client.host, 'test') self.assertEqual(self.outlet._client.port, 1234) @mongomock.patch(servers=(('localhost', 27017),)) def test_connect_custom_database(self): result = self.outlet.connect('my_database') self.assertFalse(result) self.assertEqual(self.outlet._db.name, 'my_database') @mongomock.patch(servers=(('localhost', 27017),)) def test_connect_twice_same(self): result = self.outlet.connect() self.assertFalse(result) result = self.outlet.connect() self.assertTrue(result) self.assertEqual(self.outlet._db.name, 'databay') self.assertEqual(self.outlet._client.address[0], 'localhost') self.assertEqual(self.outlet._client.address[1], 27017) @mongomock.patch(servers=(('localhost', 27017),)) def test_connect_twice_different(self): result = self.outlet.connect() self.assertFalse(result) self.assertEqual(self.outlet._db.name, 'databay') result = self.outlet.connect('test') self.assertFalse(result) self.assertEqual(self.outlet._db.name, 'test') self.assertEqual(self.outlet._client.address[0], 'localhost') self.assertEqual(self.outlet._client.address[1], 27017) def test_disconnect(self): close_mock = MagicMock() self.outlet._client = MagicMock(close=close_mock) self.outlet._db = MagicMock() self.outlet.disconnect() close_mock.assert_called_once() self.assertIsNone(self.outlet._client) self.assertIsNone(self.outlet._db) def test_on_start(self): self.outlet.connect = MagicMock() self.outlet.on_start() self.outlet.connect.assert_called_once() def test_on_shutdown(self): self.outlet.disconnect = MagicMock() self.outlet.on_shutdown() self.outlet.disconnect.assert_called_once() @mongomock.patch(servers=(('localhost', 27017),)) def test__add_collection(self): name = 'test_collection' self.outlet._add_collection(name) self.assertTrue(name in self.outlet._db.list_collection_names( ), f'Database should contain collection \'{name}\'') self.assertIsInstance(self.outlet._db[name], Collection) @mongomock.patch(servers=(('localhost', 27017),)) def test__add_collection_invalid(self): name = 'test_collection' self.outlet._add_collection(None) self.assertFalse(name in self.outlet._db.list_collection_names( ), f'Database should not contain collection \'{name}\'') @mongomock.patch(servers=(('localhost', 27017),)) def test__get_collection(self): name = 'test_collection' self.outlet._add_collection(name) self.assertTrue(name in self.outlet._db.list_collection_names( ), f'Database should contain collection \'{name}\'') self.assertIsInstance(self.outlet._db[name], Collection) collection = self.outlet._get_collection(name) self.assertIsInstance(collection, Collection) @mongomock.patch(servers=(('localhost', 27017),)) def test__get_collection_invalid(self): name = 'test_collection' self.outlet._add_collection(name) self.assertTrue(name in self.outlet._db.list_collection_names( ), f'Database should contain collection \'{name}\'') self.assertIsInstance(self.outlet._db[name], Collection) self.assertRaises(MongoCollectionNotFound, self.outlet._get_collection, 'invalid_name') @mongomock.patch(servers=(('localhost', 27017),)) @patch(fqname(Record), spec=Record) @patch(fqname(Record), spec=Record) @patch(fqname(Record), spec=Record) def test__group_by_collection(self, recordA, recordB, recordB2): recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'} recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'} recordB2.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'} recordA.payload = 'testA' recordB.payload = 'testB' recordB2.payload = 'testB2' target = {'A': ['testA'], 'B': ['testB', 'testB2']} records = [recordA, recordB, recordB2] collections = self.outlet._group_by_collection(records) self.assertEqual(collections.keys(), target.keys()) self.assertEqual(collections['A'], [recordA.payload]) self.assertEqual(collections['B'], [recordB.payload, recordB2.payload]) @mongomock.patch(servers=(('localhost', 27017),)) @patch(fqname(Record), spec=Record) @patch(fqname(Record), spec=Record) def test__group_by_collection_list_payload(self, recordA, recordB): recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'} recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'} recordA.payload = 'testA' recordB.payload = ['testB', 'testB2'] target = {'A': ['testA'], 'B': ['testB', 'testB2']} records = [recordA, recordB] collections = self.outlet._group_by_collection(records) self.assertEqual(collections.keys(), target.keys()) self.assertEqual(collections['A'], [recordA.payload]) self.assertEqual(collections['B'], [*recordB.payload]) @mongomock.patch(servers=(('localhost', 27017),)) @patch(fqname(Update), spec=Update) @patch(fqname(Record), spec=Record) @patch(fqname(Record), spec=Record) def test_push(self, recordA, recordB, update): # recordA = Record(1) # recordB = Record(1) recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'} recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'} recordA.payload = {'testA': 1} recordB.payload = [{'testB': 2}, {'testB2': 3}] records = [recordA, recordB] self.outlet.try_start() self.outlet.push(records, update) for record in records: collection = self.outlet._get_collection( record.metadata.get(MongoOutlet.MONGODB_COLLECTION)) payload = record.payload if not isinstance(payload, list): payload = [payload] for p in payload: result = collection.find_one({'_id': p.get('_id')}) self.assertEqual(p, result) self.outlet.try_shutdown() @mongomock.patch(servers=(('localhost', 27017),)) @patch(fqname(Update), spec=Update) @patch(fqname(Record), spec=Record) def test_push_inactive(self, record, update): result = self.outlet.push([record], update) self.assertFalse(result, 'Push should have stopped and returned False') @mongomock.patch(servers=(('localhost', 27017),)) def test_ensure_connection(self): mock = MagicMock(foo=MagicMock(), _db=None) ensure_connection(mock.foo)(mock) mock.connect.assert_called_once() mock.foo.assert_called_once()
class TestCsvOutlet(TestCase): @classmethod def setUpClass(cls): cls.this_filepath = os.path.abspath(os.path.dirname(__file__)) cls.target_filepath = os.path.join(cls.this_filepath, '_csv_outlet_target.csv') cls.attempt_filepath = os.path.join(cls.this_filepath, '_csv_outlet_attempt.csv') cls.custom_filepath = os.path.join(cls.this_filepath, '_csv_outlet_custom_attempt.csv') with open(cls.target_filepath, 'r') as f: cls.target = f.read() @patch(fqname(Record), spec=Record) @patch(fqname(Update), spec=Update) def setUp(self, update, record): if os.path.exists(self.attempt_filepath): os.remove(self.attempt_filepath) if os.path.exists(self.custom_filepath): os.remove(self.custom_filepath) self.csv_outlet = CsvOutlet(self.attempt_filepath) self.update = update self.record = record record.payload = {'foo': 'bar', 'baz': 'qux'} record.metadata = {} record.__repr__ = lambda x: 'TestRecord(test)' update.__repr__ = lambda x: 'TestUpdate()' self.records = [copy.copy(record) for i in range(0, 4)] def test_push_append_mode(self): asyncio.run(self.csv_outlet._push(self.records, self.update)) self.assertTrue(os.path.exists(self.attempt_filepath), 'File should exist') with open(self.attempt_filepath, 'r') as f: self.assertEqual(f.read(), self.target) os.remove(self.attempt_filepath) def test_push_write_mode(self): for record in self.records: record.metadata = {CsvOutlet.FILE_MODE: 'w'} asyncio.run(self.csv_outlet._push(self.records, self.update)) self.assertTrue(os.path.exists(self.attempt_filepath), 'File should exist') with open(self.attempt_filepath, 'r') as f: self.assertEqual( f.read(), 'foo,baz\nbar,qux\n', 'Each write should have overridden the previous.') os.remove(self.attempt_filepath) def test_push_custom_files(self): self.records[0].metadata = {CsvOutlet.CSV_FILE: self.custom_filepath} asyncio.run(self.csv_outlet._push(self.records, self.update)) self.assertTrue(os.path.exists(self.attempt_filepath), 'File should exist') self.assertTrue(os.path.exists(self.custom_filepath), 'File should exist') with open(self.attempt_filepath, 'r') as f: self.assertEqual(f.read(), 'foo,baz\nbar,qux\nbar,qux\nbar,qux\n', 'Should miss one record') with open(self.custom_filepath, 'r') as f: self.assertEqual(f.read(), 'foo,baz\nbar,qux\n', 'Should contain the one record') os.remove(self.attempt_filepath) os.remove(self.custom_filepath)
class InletTester(TestCase): """ Utility class used for testing concrete implementations of :any:`Inlet`. """ def get_inlet(self): # pragma: no cover """Implement this method to return instances of your inlet class.""" return NullInlet() def setUp(self): self.gmetadata = {'global': 'global'} self.inlets = self.get_inlet() if not isinstance(self.inlets, list): self.inlets = [self.inlets] self.inlet = self.inlets[0] self.inlet._metadata = {**self.inlet._metadata, **self.gmetadata} @for_each_inlet def test_new_record(self): """ |decorated| :any:`for_each_inlet` Test creating new records and passing local metadata. """ payload = {'test': 123} metadata = {'metadata': 321} record = self.inlet.new_record(payload=payload, metadata=metadata) self.assertIsInstance(record, Record) self.assertEqual(record.payload, payload) self.assertLessEqual( metadata.items(), record.metadata.items(), 'Local metadata should be contained in record.metadata') self.assertLessEqual( self.gmetadata.items(), record.metadata.items(), 'Global metadata should be contained in record.metadata') @for_each_inlet def test_new_record_override_global(self): """ |decorated| :any:`for_each_inlet` Test creating new records and overriding global metadata. """ payload = {'test': 123} metadata = {'global': 'local'} record = self.inlet.new_record(payload=payload, metadata=metadata) self.assertIsInstance(record, Record) self.assertEqual(record.payload, payload) self.assertLessEqual( metadata.items(), record.metadata.items(), 'Local metadata should be contained in record.metadata') self.assertEqual( record.metadata['global'], 'local', 'Global metadata should be overridden by local metadata') @patch(fqname(Update)) @for_each_inlet def test_dont_read_metadata(self, update): """ |decorated| :any:`for_each_inlet` Test creating new records and overriding global metadata. """ meta = MagicMock() meta.keys.return_value = self.gmetadata.keys() meta.__getitem__.side_effect = lambda x: self.gmetadata.__getitem__(x) self.inlet._metadata = meta self.inlet.on_start() records = asyncio.run(self.inlet._pull(update)) self.inlet.on_shutdown() self.inlet._metadata.get.assert_not_called() self.assertEqual(self.inlet._metadata.__getitem__.call_count, len(self.gmetadata)) self.assertIsInstance(records, (list)) for record in records: self.assertIsInstance(record, Record) self.assertIsNotNone(record.payload) self.assertLessEqual( self.gmetadata.items(), record.metadata.items(), 'Global metadata should be contained in record.metadata') @for_each_inlet @patch(fqname(Update)) def test_pull(self, update): """ |decorated| :any:`for_each_inlet` Test pulling data from the inlet. """ self.inlet.on_start() records = asyncio.run(self.inlet._pull(update)) self.inlet.on_shutdown() self.assertIsInstance(records, (list)) for record in records: self.assertIsInstance(record, Record) self.assertIsNotNone(record.payload) self.assertLessEqual( self.gmetadata.items(), record.metadata.items(), 'Global metadata should be contained in record.metadata')
class TestLink(TestCase): @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_transfer(self, inlet, outlet): link = Link([inlet], [outlet], timedelta(seconds=1), name='test_update') link.transfer() inlet._pull.assert_called() outlet._push.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_run(self, inlet, outlet): async def task(): link = Link([inlet], [outlet], timedelta(seconds=1), name='test_run', copy_records=False) inlet_result = await inlet._pull(None) await link._run() inlet._pull.assert_called() outlet._push.assert_called_with(inlet_result, mock.ANY) asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_inlet(self, inlet, outlet): inlet._pull.side_effect = DummyException('Test exception') link = Link([inlet], [outlet], timedelta(seconds=1), catch_exceptions=False, name='test_exception_inlet') self.assertRaises(DummyException, link.transfer) inlet._pull.assert_called() outlet._push.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_outlet(self, inlet, outlet): # inlet._pull.return_value, _ = self.inlet_return() # inlet._pull.side_effect = pull_mock # inlet._pull.return_value = Future() outlet._push.side_effect = DummyException('Test exception') link = Link([inlet], [outlet], timedelta(seconds=1), catch_exceptions=False, name='test_exception_outlet') self.assertRaises(DummyException, link.transfer) inlet._pull.assert_called() outlet._push.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_caught(self, inlet, outlet): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) inlet._pull.side_effect = DummyException('Test inlet exception') outlet._push.side_effect = DummyException('Test outlet exception') link = Link([inlet], [outlet], timedelta(seconds=1), name='test_exception_caught', catch_exceptions=True) try: link.transfer() except Exception as e: self.fail(f'Should not raise exception: {e}') inlet._pull.assert_called() outlet._push.assert_called() # check that one exception doesn't halt other inlets or outlets @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_catch_partial_exception(self, inlet1, inlet2, outlet1, outlet2): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) async def task(): # inlet_future = Future() inlet1._pull.side_effect = DummyException('Test inlet1 exception') outlet1._push.side_effect = DummyException( 'Test outlet1 exception') # inlet1._pull.return_value = inlet_future # inlet2._pull.return_value = inlet_future link = Link([inlet1, inlet2], [outlet1, outlet2], timedelta(seconds=1), name='test_catch_partial_exception', copy_records=False, catch_exceptions=True) # results = [object()] results = await inlet2._pull(None) # inlet_future.set_result(results) await link._run() inlet1._pull.assert_called() inlet2._pull.assert_called() outlet1._push.assert_called_with(results, mock.ANY) outlet2._push.assert_called_with(results, mock.ANY) asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_start(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), name='test_on_start') link.on_start() inlet1.try_start.assert_called() outlet1.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_start_already_active(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), name='test_on_start_already_active') link.on_start() inlet1.on_start.assert_not_called() outlet1.on_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_shutdown(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), name='test_on_shutdown') link.on_shutdown() inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_already_inactive(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), name='test_on_shutdown_already_inactive') link.on_shutdown() inlet1.on_shutdown.assert_not_called() outlet1.on_shutdown.assert_not_called() @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet(self, inlet1): link = Link([], [], timedelta(seconds=1), name='test_add_inlet') link.add_inlets(inlet1) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet_multiple(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), name='test_add_inlet_multiple') link.add_inlets([inlet1, inlet2]) self.assertEqual(link.inlets, [inlet1, inlet2]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet_same(self, inlet1): link = Link([], [], timedelta(seconds=1), name='test_add_inlet_same') link.add_inlets(inlet1) self.assertRaises(InvalidNodeError, link.add_inlets, inlet1) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_remove_inlet(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), name='test_remove_inlet') link.add_inlets([inlet1, inlet2]) link.remove_inlets(inlet2) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_remove_inlet_invalid(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), name='test_remove_inlet_invalid') link.add_inlets([inlet1]) self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet(self, outlet1): link = Link([], [], timedelta(seconds=1), name='test_add_outlet') link.add_outlets(outlet1) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet_multiple(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), name='test_add_outlet_multiple') link.add_outlets([outlet1, outlet2]) self.assertEqual(link.outlets, [outlet1, outlet2]) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet_same(self, outlet1): link = Link([], [], timedelta(seconds=1), name='test_add_outlet_same') link.add_outlets(outlet1) self.assertRaises(InvalidNodeError, link.add_outlets, outlet1) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_remove_outlet(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), name='test_remove_outlet') link.add_outlets([outlet1, outlet2]) link.remove_outlets(outlet2) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_remove_outlet_invalid(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), name='test_remove_outlet_invalid') link.add_outlets([outlet1]) self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object())) # this rv is invalid, should be a list def xtest_non_iterable_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), name='test_non_iterable_raised') with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'): link.transfer() @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable()) ) # this rv will raise DummyException def test_generic_error_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), name='test_generic_error_raised') # with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION): self.assertRaises(DummyException, link.transfer) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object())) # this rv is invalid, should be a list def xtest_non_iterable_caught(self, inlet1, inlet2, outlet1): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) link = Link([inlet1, inlet2], [outlet1], timedelta(seconds=1), name='test_non_iterable_caught', catch_exceptions=True) results = asyncio.run(inlet2._pull(None)) link.transfer() outlet1._push.assert_called_with(results, mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable()) ) # this rv will raise DummyException def xtest_generic_error_caught(self, inlet1, inlet2, outlet1): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) link = Link([inlet1, inlet2], [outlet1], timedelta(seconds=1), name='test_generic_error_caught', catch_exceptions=True) results = asyncio.run(inlet2._pull(None)) link.transfer() outlet1._push.assert_called_with(results, mock.ANY)
class TestBuffers(TestCase): @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock()) @patch(fqname(Inlet), spec=Inlet) def test_buffer_count(self, inlet, outlet): buffer = Buffer(count_threshold=3) payload = [1, 2, 3, 4] records = [Record(payload=p) for p in payload] link = Link(inlet, outlet, interval=1, processors=buffer, copy_records=False) inlet._pull = pull_mock(records[:2]) link.transfer() outlet._push.assert_called_with( [], mock.ANY) # after first call we shouldn't have any records inlet._pull = pull_mock(records[2:]) link.transfer() outlet._push.assert_called_with( records, mock.ANY) # all records should be returned here @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock()) @patch(fqname(Inlet), spec=Inlet) def test_buffer_time(self, inlet, outlet): buffer = Buffer(time_threshold=0.02) payload = [1, 2, 3, 4] records = [Record(payload=p) for p in payload] link = Link(inlet, outlet, interval=1, processors=buffer, copy_records=False) inlet._pull = pull_mock(records[:2]) link.transfer() outlet._push.assert_called_with( [], mock.ANY) # not enough time have passed inlet._pull = pull_mock(records[2:]) link.transfer() outlet._push.assert_called_with( [], mock.ANY) # not enough time have passed time.sleep(0.02) inlet._pull = pull_mock([]) link.transfer() outlet._push.assert_called_with( records, mock.ANY) # all records should be returned here @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock()) @patch(fqname(Inlet), spec=Inlet) def test_flush(self, inlet, outlet): buffer = Buffer(count_threshold=100, time_threshold=10) payload = [1, 2, 3, 4] records = [Record(payload=p) for p in payload] link = Link(inlet, outlet, interval=1, processors=buffer, copy_records=False) inlet._pull = pull_mock(records[:2]) link.transfer() outlet._push.assert_called_with([], mock.ANY) # no records yet inlet._pull = pull_mock(records[2:]) link.transfer() outlet._push.assert_called_with([], mock.ANY) # no records yet buffer.flush = True inlet._pull = pull_mock([]) link.transfer() outlet._push.assert_called_with( records, mock.ANY) # all records should be flushed @patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock()) @patch(fqname(Inlet), spec=Inlet) def test_flush_after_shutdown(self, inlet, outlet): buffer = Buffer(count_threshold=100, time_threshold=10) counter_dict = {'counter': 0, 'records': []} link = Link(inlet, outlet, interval=0.01, processors=buffer, copy_records=False) planner = SchedulePlanner(link, refresh_interval=0.01) async def pull_coro(_): counter_dict['counter'] += 1 record = Record(payload=counter_dict['counter']) counter_dict['records'].append(record) return [record] mock_pull = MagicMock(side_effect=pull_coro) inlet._pull = mock_pull th = Thread(target=planner.start, daemon=True) th.start() time.sleep(0.1) planner.shutdown() th.join() calls = outlet._push.call_args_list for c in calls: self.assertEqual(c(), [], 'Should only contain empty record lists.') self.assertEqual(buffer.records, counter_dict['records'], 'All records should be stored in the buffer') planner.force_transfer() self.assertEqual(outlet._push.call_args(), [], 'Should return empty record list') buffer.flush = True planner.force_transfer() self.assertEqual(outlet._push.call_args[0][0], counter_dict['records'], 'Should return all records')
class TestAPSPlanner(TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logging.getLogger('databay').setLevel(logging.WARNING) @patch(fqname(Link), spec=Link) def setUp(self, link): self.planner = APSPlanner() def set_job(job): link.job = job link.interval.total_seconds.return_value = 0.02 link.set_job.side_effect = set_job link.job = None self.link = link def test__schedule(self): self.planner._schedule(self.link) self.assertIsNotNone(self.link.job, 'Link should contain a job') asp_job = self.planner._scheduler.get_jobs()[0] self.assertEqual(self.link.job, asp_job, 'Link\'s job should be same as scheduler\'s') def test__unschedule(self): self.planner._schedule(self.link) self.planner._unschedule(self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertEqual(len(self.planner._scheduler.get_jobs()), 0, 'Scheduler should not have any jobs') def test__unschedule_invalid(self): self.planner._unschedule(self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertEqual(len(self.planner._scheduler.get_jobs()), 0, 'Scheduler should not have any jobs') def test_add_links(self): self.planner.add_links(self.link) self.assertIsNotNone(self.link.job, 'Link should contain a job') self.assertTrue(self.link in self.planner.links, 'Planner should contain the link') def test_add_links_on_init(self): self.planner = APSPlanner(self.link) self.assertIsNotNone(self.link.job, 'Link should contain a job') self.assertTrue(self.link in self.planner.links, 'Planner should contain the link') def test_remove_links(self): self.planner.add_links(self.link) self.planner.remove_links(self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertTrue(self.link not in self.planner.links, 'Planner should not contain the link') def test_remove_invalid_link(self): self.assertRaises(MissingLinkError, self.planner.remove_links, self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertTrue(self.link not in self.planner.links, 'Planner should not contain the link') def test_start(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.assertTrue(self.planner.running, 'Scheduler should be running') self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') # todo: APS is currently broken for this test case, wait for an update def xtest_start_paused(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.pause() self.assertRaises(SchedulerAlreadyRunningError, self.planner.start) self.assertEqual(self.planner._scheduler.state, STATE_PAUSED, 'Scheduler should be paused') self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_shutdown(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.shutdown(False) self.assertFalse(self.planner.running, 'Scheduler should not be running') th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_pause(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.pause() self.assertEqual(self.planner._scheduler.state, STATE_PAUSED, 'Scheduler should be paused') self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_resume(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.pause() self.assertEqual(self.planner._scheduler.state, STATE_PAUSED, 'Scheduler should be paused') self.planner.resume() self.assertTrue(self.planner.running, 'Scheduler should not be paused') self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_shutdown_paused(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.pause() self.assertEqual(self.planner._scheduler.state, STATE_PAUSED, 'Scheduler should be paused') self.planner.shutdown(False) self.assertFalse(self.planner.running, 'Scheduler should not be running') th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_pause_shutdown(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.shutdown(False) self.assertRaises(SchedulerNotRunningError, self.planner.pause) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_resume_shutdown(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.shutdown(False) self.assertRaises(SchedulerNotRunningError, self.planner.resume) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_start_shutdown(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread 1 should be stopped.') self.assertFalse(self.planner.running, 'Scheduler should not be running') th2 = Thread(target=self.planner.start, daemon=True) th2.start() self.assertTrue(self.planner.running, 'Scheduler should be running') self.planner.shutdown(False) th2.join(timeout=2) self.assertFalse(th2.is_alive(), 'Thread 2 should be stopped.') def test_add_and_run(self): self.link.interval.total_seconds.return_value = 0.02 self.planner.add_links(self.link) th = Thread(target=self.planner.start, daemon=True) th.start() time.sleep(0.04) self.link.transfer.assert_called() self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def _with_exception(self, link, catch_exceptions): logging.getLogger('databay').setLevel(logging.CRITICAL) # logging.getLogger('databay').setLevel(logging.INFO) self.planner = APSPlanner(catch_exceptions=catch_exceptions) link.transfer.side_effect = DummyException() link.interval.total_seconds.return_value = 0.02 self.planner.add_links(link) th = Thread(target=self.planner.start, daemon=True) th.start() time.sleep(0.04) link.transfer.assert_called() if catch_exceptions: self.assertTrue(self.planner.running, 'Scheduler should be running') self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') self.assertFalse(self.planner.running, 'Scheduler should be stopped') def test_catch_exception(self): self._with_exception(self.link, True) def test_raise_exception(self): self._with_exception(self.link, False) def test_uncommon_exception(self): logging.getLogger('databay').setLevel(logging.CRITICAL) self.link.transfer.side_effect = DummyUnusualException(123, True) self.link.interval.total_seconds.return_value = 0.02 self.planner.add_links(self.link) th = Thread(target=self.planner.start, daemon=True) th.start() time.sleep(0.04) self.link.transfer.assert_called() self.assertFalse(self.planner.running, 'Scheduler should be stopped')
class TestLink(TestCase): @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_transfer(self, inlet, outlet): link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_update') link.transfer() inlet._pull.assert_called() outlet._push.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_run(self, inlet, outlet): async def task(): link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_run', copy_records=False) inlet_result = await inlet._pull(None) await link._run() inlet._pull.assert_called() outlet._push.assert_called_with(inlet_result, mock.ANY) asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_inlet(self, inlet, outlet): inlet._pull.side_effect = DummyException('Test exception') link = Link([inlet], [outlet], timedelta(seconds=1), ignore_exceptions=False, tags='test_exception_inlet') self.assertRaises(DummyException, link.transfer) inlet._pull.assert_called() outlet._push.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_outlet(self, inlet, outlet): # inlet._pull.return_value, _ = self.inlet_return() # inlet._pull.side_effect = pull_mock # inlet._pull.return_value = Future() outlet._push.side_effect = DummyException('Test exception') link = Link([inlet], [outlet], timedelta(seconds=1), ignore_exceptions=False, tags='test_exception_outlet') link = Link([inlet], [outlet], timedelta(seconds=1), ignore_exceptions=False, tags='test_exception_outlet') self.assertRaises(DummyException, link.transfer) inlet._pull.assert_called() outlet._push.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_exception_caught(self, inlet, outlet): inlet._pull.side_effect = DummyException('Test inlet exception') outlet._push.side_effect = DummyException('Test outlet exception') link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_exception_caught', ignore_exceptions=True) try: with self.assertLogs(logging.getLogger('databay.Link'), level='WARNING') as cm: link.transfer() self.assertTrue('Test inlet exception' in ';'.join(cm.output)) self.assertTrue('Test outlet exception' in ';'.join(cm.output)) except Exception as e: self.fail(f'Should not raise exception: {e}') inlet._pull.assert_called() outlet._push.assert_called() # check that one exception doesn't halt other inlets or outlets @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_ignore_partial_exception(self, inlet1, inlet2, outlet1, outlet2): async def task(): # inlet_future = Future() inlet1._pull.side_effect = DummyException('Test inlet1 exception') outlet1._push.side_effect = DummyException( 'Test outlet1 exception') # inlet1._pull.return_value = inlet_future # inlet2._pull.return_value = inlet_future link = Link([inlet1, inlet2], [outlet1, outlet2], timedelta(seconds=1), tags='test_ignore_partial_exception', copy_records=False, ignore_exceptions=True) # results = [object()] results = await inlet2._pull(None) # inlet_future.set_result(results) with self.assertLogs(logging.getLogger('databay.Link'), level='WARNING') as cm: await link._run() self.assertTrue('Test inlet1 exception' in ';'.join(cm.output)) self.assertTrue( 'Test outlet1 exception' in ';'.join(cm.output)) inlet1._pull.assert_called() inlet2._pull.assert_called() outlet1._push.assert_called_with(results, mock.ANY) outlet2._push.assert_called_with(results, mock.ANY) asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_start(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start') link.on_start() inlet1.try_start.assert_called() outlet1.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_start_already_active(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start_already_active') link.on_start() inlet1.on_start.assert_not_called() outlet1.on_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_on_shutdown(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown') link.on_shutdown() inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_already_inactive(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown_already_inactive') link.on_shutdown() inlet1.on_shutdown.assert_not_called() outlet1.on_shutdown.assert_not_called() @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet(self, inlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet') link.add_inlets(inlet1) self.assertEqual(link.inlets, [inlet1]) def test_add_inlet_invalid(self): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet') self.assertRaises(TypeError, link.add_inlets, "invalid_inlet") @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet_multiple(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_multiple') link.add_inlets([inlet1, inlet2]) self.assertEqual(link.inlets, [inlet1, inlet2]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_add_inlet_same(self, inlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same') link.add_inlets(inlet1) self.assertRaises(InvalidNodeError, link.add_inlets, inlet1) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_remove_inlet(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet') link.add_inlets([inlet1, inlet2]) link.remove_inlets(inlet2) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock()) def test_remove_inlet_invalid(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet_invalid') link.add_inlets([inlet1]) self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2) self.assertEqual(link.inlets, [inlet1]) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet(self, outlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet') link.add_outlets(outlet1) self.assertEqual(link.outlets, [outlet1]) def test_add_outlet_invalid(self): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet') self.assertRaises(TypeError, link.add_outlets, "invalid_outlet") @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet_multiple(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_multiple') link.add_outlets([outlet1, outlet2]) self.assertEqual(link.outlets, [outlet1, outlet2]) @patch(fqname(Outlet), spec=Outlet) def test_add_outlet_same(self, outlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same') link.add_outlets(outlet1) self.assertRaises(InvalidNodeError, link.add_outlets, outlet1) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_remove_outlet(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet') link.add_outlets([outlet1, outlet2]) link.remove_outlets(outlet2) self.assertEqual(link.outlets, [outlet1]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) def test_remove_outlet_invalid(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet_invalid') link.add_outlets([outlet1]) self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2) self.assertEqual(link.outlets, [outlet1]) # this rv is invalid, should be a list @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object())) def xtest_non_iterable_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), tags='test_non_iterable_raised') with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'): link.transfer() # this rv will raise DummyException @patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable())) def test_generic_error_raised(self, inlet1): link = Link([inlet1], [], timedelta(seconds=1), tags='test_generic_error_raised') # with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION): self.assertRaises(DummyException, link.transfer) def test_integer_to_timedelta(self): link = Link([], [], 1, name='test_integer_interval_coerced') self.assertEqual(link._interval, timedelta(seconds=1)) def test_float_to_timedelta(self): link = Link([], [], 1.5, name='test_float_interval_coerced') self.assertEqual(link._interval, timedelta(seconds=1.5)) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_inlet_exception_raise(self, inlet1, outlet1): inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start') self.assertRaises(RuntimeError, link.on_start) inlet1.try_start.assert_called() outlet1.try_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_inlet_exception_catch(self, inlet1, outlet1): inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_start() self.assertTrue('on_start inlet exception: "" for inlet:' in ';'.join(cm.output)) inlet1.try_start.assert_called() outlet1.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_outlet_exception_raise(self, inlet1, outlet1, outlet2): outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_start') self.assertRaises(RuntimeError, link.on_start) inlet1.try_start.assert_called() outlet1.try_start.assert_called() outlet2.try_start.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_start_outlet_exception_catch(self, inlet1, outlet1, outlet2): outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_start', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_start() self.assertTrue( 'on_start outlet exception: "" for outlet:' in ';'.join(cm.output), cm.output) inlet1.try_start.assert_called() outlet1.try_start.assert_called() outlet2.try_start.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_inlet_exception_raise(self, inlet1, outlet1): inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown') self.assertRaises(RuntimeError, link.on_shutdown) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_inlet_exception_catch(self, inlet1, outlet1): inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_shutdown() self.assertTrue( 'on_shutdown inlet exception: "" for inlet:' in ';'.join(cm.output), cm.output) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_outlet_exception_raise(self, inlet1, outlet1, outlet2): outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())' ) link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_shutdown') self.assertRaises(RuntimeError, link.on_shutdown) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() outlet2.try_shutdown.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_on_shutdown_outlet_exception_catch(self, inlet1, outlet1, outlet2): outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())' ) link = Link([inlet1], [outlet1, outlet2], timedelta(seconds=1), tags='test_on_shutdown', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_shutdown() self.assertTrue( 'on_shutdown outlet exception: "" for outlet:' in ';'.join(cm.output), cm.output) inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called() outlet2.try_shutdown.assert_called() def test_single_tag(self): tag = 'tagA' link = Link([], [], timedelta(seconds=1), tags=tag) self.assertEqual(link.tags, [tag]) def test_multiple_tags(self): tagA = 'tagA' tagB = 'tagB' link = Link([], [], timedelta(seconds=1), tags=[tagA, tagB]) self.assertEqual(link.tags, [tagA, tagB]) def test_tag_as_name(self): link_name = 'link_name' link = Link([], [], timedelta(seconds=1), name=link_name) self.assertEqual(link_name, link.tags[0]) def test_name_from_tag(self): link_name = 'link_name' link = Link([], [], timedelta(seconds=1), tags=[link_name]) self.assertEqual(link.name, link.tags[0]) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) @patch(fqname(Inlet), spec=Inlet) @patch(fqname(Inlet), spec=Inlet) def test_inlet_concurrency(self, inlet1, inlet2, inlet3, outlet): counter = {'value': 0} # this will increment the counter on each async call, and check if they exceed the concurrency value async def slow_pull(_): counter['value'] += 1 self.assertLessEqual(counter['value'], 2, "Only 2 inlets should pull at a time") await asyncio.sleep(0.01) counter['value'] -= 1 return [123] inlet1._pull = slow_pull inlet2._pull = slow_pull inlet3._pull = slow_pull async def task(): link = Link([inlet1, inlet2, inlet3], [outlet], timedelta(seconds=1), tags='test_inlet_concurrency', copy_records=False, inlet_concurrency=2) await link._run() asyncio.run(task()) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_processors_one(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processor = MagicMock(side_effect=lambda r: r) link = Link(inlet, outlet, interval=0.01, processors=processor) link.transfer() processor.assert_called_with(records) outlet._push.assert_called_with(records, mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_processors_many(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock(side_effect=lambda x: x) processorB = MagicMock(side_effect=lambda x: x) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with(records) outlet._push.assert_called_with(records, mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_processors_change_records(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=lambda r: list(map(lambda y: y * y, r))) processorB = MagicMock( side_effect=lambda r: list(map(lambda y: -y, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with([4, 9]) outlet._push.assert_called_with([-4, -9], mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_processors_filter_records(self, inlet, outlet): records = [2, 3, 4] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=lambda r: list(filter(lambda y: y > 2, r))) processorB = MagicMock( side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with([3, 4]) outlet._push.assert_called_with([4], mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_groupers_one_passive(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) grouper = MagicMock(side_effect=lambda r: r) # does nothing on purpose link = Link(inlet, outlet, interval=0.01, groupers=grouper) link.transfer() grouper.assert_called_with([records]) outlet._push.assert_called_with(records, mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_groupers_one_active(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) # makes [[1,2], [3,4]] grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) link = Link(inlet, outlet, interval=0.01, groupers=grouper) link.transfer() grouper.assert_called_with([records]) calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)] outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]] @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_groupers_many_active(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) # makes [[1,2], [3,4]] grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) # makes [[1], [2], [3], [4]] grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] + [[sub] for sub in r[1]]) link = Link(inlet, outlet, interval=0.01, groupers=[grouperA, grouperB]) link.transfer() grouperA.assert_called_with([records]) callsA = [call([records[:2], records[2:]])] # expects [[1,2], [3,4]] grouperB.assert_has_calls(callsA) callsB = [ call([records[0]], mock.ANY), call([records[1]], mock.ANY), call([records[2]], mock.ANY), call([records[3]], mock.ANY) ] outlet._push.assert_has_calls(callsB) # expects [[1], [2], [3], [4]] @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_processor_exception(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processor = MagicMock( side_effect=DummyException('Processor exception')) link = Link(inlet, outlet, interval=0.01, processors=processor) self.assertRaises(DummyException, link.transfer) processor.assert_called_with(records) outlet._push.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_processor_exception_ignored(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=DummyException('Processor exception')) processorB = MagicMock( side_effect=lambda r: list(map(lambda y: y * y, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB], ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.transfer() self.assertTrue('Processor exception:' in ';'.join(cm.output), cm.output) processorA.assert_called_with(records) processorB.assert_called_with(records) outlet._push.assert_called_with([4, 9], mock.ANY) @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_grouper_exception(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) grouper = MagicMock(side_effect=DummyException('Grouper exception')) link = Link(inlet, outlet, interval=0.01, groupers=grouper) self.assertRaises(DummyException, link.transfer) grouper.assert_called_with([records]) outlet._push.assert_not_called() @patch(fqname(Outlet), spec=Outlet) @patch(fqname(Inlet), spec=Inlet) def test_grouper_exception_ignored(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) grouperA = MagicMock(side_effect=DummyException('Grouper exception')) grouperB = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) link = Link(inlet, outlet, interval=0.01, groupers=[grouperA, grouperB], ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.transfer() self.assertTrue('Grouper exception:' in ';'.join(cm.output), cm.output) grouperA.assert_called_with([records]) grouperB.assert_called_with([records]) calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)] outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
class TestSchedulePlanner(TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logging.getLogger('databay').setLevel(logging.WARNING) @patch(fqname(Link), spec=Link) def setUp(self, link): self.planner = SchedulePlanner(refresh_interval=0.02) def set_job(job): link.job = job link.interval.total_seconds.return_value = 0.02 link.set_job.side_effect = set_job link.job = None self.link = link def tearDown(self): if len(schedule.jobs) > 0: schedule.clear() def test__run_job(self): self.planner._create_thread_pool() self.planner._run_job(self.link) self.link.transfer.assert_called_once() self.planner._destroy_thread_pool() def test__schedule(self): self.planner._schedule(self.link) self.assertIsNotNone(self.link.job, 'Link should contain a job') schedule_job = schedule.jobs[0] self.assertEqual(self.link.job, schedule_job, 'Link\'s job should be same as schedule\'s') # self.planner._unschedule(link) def test__unschedule(self): self.planner._schedule(self.link) self.planner._unschedule(self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertEqual(len(schedule.jobs), 0, 'Schedule should not have any jobs') def test__unschedule_invalid(self): self.planner._unschedule(self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertEqual(len(schedule.jobs), 0, 'Scheduler should not have any jobs') def test_add_links(self): self.planner.add_links(self.link) self.assertIsNotNone(self.link.job, 'Link should contain a job') self.assertTrue(self.link in self.planner.links, 'Planner should contain the link') def test_add_links_on_init(self): self.planner = SchedulePlanner(self.link, refresh_interval=0.02) self.assertIsNotNone(self.link.job, 'Link should contain a job') self.assertTrue(self.link in self.planner.links, 'Planner should contain the link') def test_remove_links(self): self.planner.add_links(self.link) self.planner.remove_links(self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertTrue(self.link not in self.planner.links, 'Planner should not contain the link') def test_remove_invalid_link(self): self.assertRaises(MissingLinkError, self.planner.remove_links, self.link) self.assertIsNone(self.link.job, 'Link should not contain a job') self.assertTrue(self.link not in self.planner.links, 'Planner should not contain the link') def test_start(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.assertTrue(self.planner._running, 'Planner should be running') self.planner.shutdown() th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_shutdown(self): th = Thread(target=self.planner.start, daemon=True) th.start() self.planner.shutdown() self.assertFalse(self.planner._running, 'Planner should be not running') self.assertIsNone(self.planner._thread_pool, 'Planner should not have a thread pool') th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_add_and_run(self): self.link.interval.total_seconds.return_value = 0.02 self.planner._refresh_interval = 0.02 self.planner.add_links(self.link) th = Thread(target=self.planner.start, daemon=True) th.start() time.sleep(0.04) self.link.transfer.assert_called() self.planner.shutdown() th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') def test_invalid_interval(self): self.link.interval.total_seconds.return_value = 0.1 self.planner._refresh_interval = 0.2 self.assertRaises(ScheduleIntervalError, self.planner.add_links, self.link) def _with_exception(self, link, catch_exceptions): logging.getLogger('databay').setLevel(logging.CRITICAL) self.planner = SchedulePlanner(catch_exceptions=catch_exceptions) link.transfer.side_effect = DummyException() link.interval.total_seconds.return_value = 0.02 self.planner._refresh_interval = 0.02 link.transfer.side_effect = DummyException() link.interval.total_seconds.return_value = 0.02 self.planner.add_links(link) th = Thread(target=self.planner.start, daemon=True) th.start() time.sleep(0.04) link.transfer.assert_called() if catch_exceptions: self.assertTrue(self.planner.running, 'Scheduler should be running') self.planner.shutdown(False) th.join(timeout=2) self.assertFalse(th.is_alive(), 'Thread should be stopped.') self.assertFalse(self.planner.running, 'Scheduler should be stopped') def test_catch_exception(self): self._with_exception(self.link, True) def test_raise_exception(self): self._with_exception(self.link, False) def test_uncommon_exception(self): logging.getLogger('databay').setLevel(logging.CRITICAL) self.link.transfer.side_effect = DummyUnusualException(argA=123, argB=True) self.link.interval.total_seconds.return_value = 0.02 self.planner.add_links(self.link) th = Thread(target=self.planner.start, daemon=True) th.start() time.sleep(0.04) self.link.transfer.assert_called() self.assertFalse(self.planner.running, 'Scheduler should be stopped')