class TestHttpInlet(inlet_tester.InletTester):
def setUp(self):
super().setUp()
set_response(b'{"asdf":"12"}')
def get_inlet(self):
return HttpInlet('https://jsonplaceholder.typicode.com/todos/1')
@patch(fqname(Update))
def test_json(self, update):
records = asyncio.run(self.inlet._pull(update))
self.assertIsInstance(records, (list))
@patch(fqname(Update))
def test_html(self, update):
self.inlet.json = False
records = asyncio.run(self.inlet._pull(update))
self.assertIsInstance(records, (list))
@patch(fqname(Update))
def test_invalid_json(self, update):
set_response(b'<html>')
self.inlet.json = True
self.assertRaisesRegex(ValueError,
'Response does not contain valid JSON',
asyncio.run, self.inlet._pull(update))
@patch(fqname(Update))
def test_invalid_html(self, update):
set_response(None)
self.inlet.json = False
self.assertRaisesRegex(
AttributeError, '\'NoneType\' object has no attribute \'decode\'',
asyncio.run, self.inlet._pull(update))
class TestPrintOutlet(TestCase):
@patch(fqname(Record), spec=Record)
@patch(fqname(Update), spec=Update)
def setUp(self, update, record):
self.print_outlet = PrintOutlet()
self.update = update
self.record = record
record.payload = 'test'
record.metadata.return_value = {}
record.__repr__ = lambda x: 'TestRecord(test)'
update.__repr__ = lambda x: 'TestUpdate()'
@patch('sys.stdout', new_callable=io.StringIO)
def test_push(self, stdout):
asyncio.run(self.print_outlet._push([self.record], self.update))
self.assertEqual(stdout.getvalue(), 'TestUpdate() TestRecord(test)\n')
@patch('sys.stdout', new_callable=io.StringIO)
def test_push_only_payload(self, stdout):
self.print_outlet.only_payload = True
asyncio.run(self.print_outlet._push([self.record], self.update))
self.assertEqual(stdout.getvalue(), 'TestUpdate() test\n')
@patch('sys.stdout', new_callable=io.StringIO)
def test_push_skip_update(self, stdout):
self.print_outlet.skip_update = True
asyncio.run(self.print_outlet._push([self.record], self.update))
self.assertEqual(stdout.getvalue(), 'TestRecord(test)\n')
class TestLink(TestCase):
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_transfer(self, inlet, outlet):
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_update')
link.transfer()
inlet._pull.assert_called()
outlet._push.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_run(self, inlet, outlet):
async def task():
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_run',
copy_records=False)
inlet_result = await inlet._pull(None)
await link._run()
inlet._pull.assert_called()
outlet._push.assert_called_with(inlet_result, mock.ANY)
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_inlet(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_inlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_outlet(self, inlet, outlet):
# inlet._pull.return_value, _ = self.inlet_return()
# inlet._pull.side_effect = pull_mock
# inlet._pull.return_value = Future()
outlet._push.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_outlet')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_outlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_caught(self, inlet, outlet):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_exception_caught',
ignore_exceptions=True)
try:
link.transfer()
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
# check that one exception doesn't halt other inlets or outlets
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_ignore_partial_exception(self, inlet1, inlet2, outlet1, outlet2):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
async def task():
# inlet_future = Future()
inlet1._pull.side_effect = DummyException('Test inlet1 exception')
outlet1._push.side_effect = DummyException(
'Test outlet1 exception')
# inlet1._pull.return_value = inlet_future
# inlet2._pull.return_value = inlet_future
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_ignore_partial_exception',
copy_records=False,
ignore_exceptions=True)
# results = [object()]
results = await inlet2._pull(None)
# inlet_future.set_result(results)
await link._run()
inlet1._pull.assert_called()
inlet2._pull.assert_called()
outlet1._push.assert_called_with(results, mock.ANY)
outlet2._push.assert_called_with(results, mock.ANY)
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_start(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start')
link.on_start()
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_start_already_active(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start_already_active')
link.on_start()
inlet1.on_start.assert_not_called()
outlet1.on_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_shutdown(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown')
link.on_shutdown()
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_already_inactive(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown_already_inactive')
link.on_shutdown()
inlet1.on_shutdown.assert_not_called()
outlet1.on_shutdown.assert_not_called()
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet(self, inlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet')
link.add_inlets(inlet1)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet_multiple(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_add_inlet_multiple')
link.add_inlets([inlet1, inlet2])
self.assertEqual(link.inlets, [inlet1, inlet2])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet_same(self, inlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same')
link.add_inlets(inlet1)
self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_remove_inlet(self, inlet1, inlet2):
link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet')
link.add_inlets([inlet1, inlet2])
link.remove_inlets(inlet2)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_remove_inlet_invalid(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_remove_inlet_invalid')
link.add_inlets([inlet1])
self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet(self, outlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_outlet')
link.add_outlets(outlet1)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet_multiple(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_add_outlet_multiple')
link.add_outlets([outlet1, outlet2])
self.assertEqual(link.outlets, [outlet1, outlet2])
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet_same(self, outlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same')
link.add_outlets(outlet1)
self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_remove_outlet(self, outlet1, outlet2):
link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet')
link.add_outlets([outlet1, outlet2])
link.remove_outlets(outlet2)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_remove_outlet_invalid(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_remove_outlet_invalid')
link.add_outlets([outlet1])
self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
self.assertEqual(link.outlets, [outlet1])
# this rv is invalid, should be a list
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object()))
def xtest_non_iterable_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
tags='test_non_iterable_raised')
with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
link.transfer()
# this rv will raise DummyException
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable()))
def test_generic_error_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
tags='test_generic_error_raised')
# with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION):
self.assertRaises(DummyException, link.transfer)
def test_integer_to_timedelta(self):
link = Link([], [], 1, name='test_integer_interval_coerced')
self.assertEqual(link._interval, timedelta(seconds=1))
def test_float_to_timedelta(self):
link = Link([], [], 1.5, name='test_float_interval_coerced')
self.assertEqual(link._interval, timedelta(seconds=1.5))
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_inlet_exception_raise(self, inlet1, outlet1):
inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start')
self.assertRaises(RuntimeError, link.on_start)
inlet1.try_start.assert_called()
outlet1.try_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_inlet_exception_catch(self, inlet1, outlet1):
logging.getLogger('databay.Link').setLevel(logging.WARNING)
inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_start()
self.assertTrue(
'on_start inlet exception: "" for inlet:' in ';'.join(cm.output))
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_outlet_exception_raise(self, inlet1, outlet1, outlet2):
outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_start')
self.assertRaises(RuntimeError, link.on_start)
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
outlet2.try_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_outlet_exception_catch(self, inlet1, outlet1, outlet2):
logging.getLogger('databay.Link').setLevel(logging.WARNING)
outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_start',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_start()
self.assertTrue(
'on_start outlet exception: "" for outlet:' in ';'.join(cm.output),
cm.output)
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
outlet2.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_inlet_exception_raise(self, inlet1, outlet1):
inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown')
self.assertRaises(RuntimeError, link.on_shutdown)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_inlet_exception_catch(self, inlet1, outlet1):
logging.getLogger('databay.Link').setLevel(logging.WARNING)
inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_shutdown()
self.assertTrue(
'on_shutdown inlet exception: "" for inlet:'
in ';'.join(cm.output), cm.output)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_outlet_exception_raise(self, inlet1, outlet1,
outlet2):
outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
)
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_shutdown')
self.assertRaises(RuntimeError, link.on_shutdown)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
outlet2.try_shutdown.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_outlet_exception_catch(self, inlet1, outlet1,
outlet2):
logging.getLogger('databay.Link').setLevel(logging.WARNING)
outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
)
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_shutdown',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_shutdown()
self.assertTrue(
'on_shutdown outlet exception: "" for outlet:'
in ';'.join(cm.output), cm.output)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
outlet2.try_shutdown.assert_called()
def test_single_tag(self):
tag = 'tagA'
link = Link([], [], timedelta(seconds=1), tags=tag)
self.assertEqual(link.tags, [tag])
def test_multiple_tags(self):
tagA = 'tagA'
tagB = 'tagB'
link = Link([], [], timedelta(seconds=1), tags=[tagA, tagB])
self.assertEqual(link.tags, [tagA, tagB])
def test_tag_as_name(self):
link_name = 'link_name'
link = Link([], [], timedelta(seconds=1), name=link_name)
self.assertEqual(link_name, link.tags[0])
def test_name_from_tag(self):
link_name = 'link_name'
link = Link([], [], timedelta(seconds=1), tags=[link_name])
self.assertEqual(link.name, link.tags[0])
class TestBasePlanner(TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
logging.getLogger('databay').setLevel(logging.WARNING)
@patch.multiple(BasePlanner, __abstractmethods__=set())
def setUp(self):
self.planner = BasePlanner()
self.planner._schedule = mock.Mock(
side_effect=lambda link: link.set_job(object()))
self.planner._unschedule = mock.Mock(
side_effect=lambda link: link.set_job(None))
self.planner._start_planner = mock.Mock()
self.planner._shutdown_planner = mock.Mock()
@patch(fqname(Link), spec=Link)
def test_add_links(self, link):
def set_job(job):
link.job = job
link.set_job.side_effect = set_job
link.job = None
self.planner.add_links(link)
self.assertIsNotNone(link.job, 'Link should contain a job')
self.assertTrue(link in self.planner.links,
'Planner should contain the link')
@patch(fqname(Link), spec=Link)
def test_add_links_array(self, link):
def set_job(job):
link.job = job
link.set_job.side_effect = set_job
link.job = None
self.planner.add_links([link])
self.assertIsNotNone(link.job, 'Link should contain a job')
self.assertTrue(link in self.planner.links,
'Planner should contain the link')
@patch(fqname(Link), spec=Link)
def test_remove_links(self, link):
def set_job(job):
link.job = job
link.set_job.side_effect = set_job
link.job = None
self.planner.add_links(link)
self.planner.remove_links(link)
self.assertIsNone(link.job, 'Link should not contain a job')
self.assertTrue(link not in self.planner.links,
'Planner should not contain the link')
@patch(fqname(Link), spec=Link)
def test_remove_invalid_link(self, link):
def set_job(job):
link.job = job
link.set_job.side_effect = set_job
link.job = None
self.assertRaises(MissingLinkError, self.planner.remove_links, link)
self.assertIsNone(link.job, 'Link should not contain a job')
self.assertTrue(link not in self.planner.links,
'Planner should not contain the link')
@patch(fqname(Link), spec=Link)
def test_start(self, link):
self.planner.add_links(link)
self.planner.start()
link.on_start.assert_called()
self.planner._start_planner.assert_called()
@patch(fqname(Link), spec=Link)
def test_shutdown(self, link):
self.planner.add_links(link)
self.planner.shutdown()
link.on_shutdown.assert_called()
self.planner._shutdown_planner.assert_called()
@patch(fqname(Link), spec=Link)
def test_start_order(self, link):
# on_start should be called before _start_planner
link.on_start.side_effect = lambda: self.planner._start_planner.assert_not_called(
)
self.planner.add_links(link)
self.planner.start()
# finally both should be called
link.on_start.assert_called()
self.planner._start_planner.assert_called()
@patch(fqname(Link), spec=Link)
def test_shutdown_order(self, link):
# on_shutdown should be called after _shutdown_planner
self.planner._shutdown_planner.side_effect = lambda wait: link.on_shutdown.assert_not_called(
)
self.planner.add_links(link)
self.planner.shutdown()
# finally both should be called
link.on_shutdown.assert_called()
self.planner._shutdown_planner.assert_called()
class TestMongoOutlet(TestCase):
def setUp(self):
self.outlet = MongoOutlet()
@mongomock.patch(servers=(('localhost', 27017),))
def test_connect(self):
result = self.outlet.connect()
self.assertFalse(result)
self.assertEqual(self.outlet._db.name, 'databay')
self.assertEqual(self.outlet._client.address[0], 'localhost')
self.assertEqual(self.outlet._client.address[1], 27017)
@mongomock.patch(servers=(('test', 1234),))
def test_connect_custom_host(self):
self.outlet = MongoOutlet(host='test', port=1234)
self.outlet.connect()
self.assertEqual(self.outlet._client.host, 'test')
self.assertEqual(self.outlet._client.port, 1234)
@mongomock.patch(servers=(('localhost', 27017),))
def test_connect_custom_database(self):
result = self.outlet.connect('my_database')
self.assertFalse(result)
self.assertEqual(self.outlet._db.name, 'my_database')
@mongomock.patch(servers=(('localhost', 27017),))
def test_connect_twice_same(self):
result = self.outlet.connect()
self.assertFalse(result)
result = self.outlet.connect()
self.assertTrue(result)
self.assertEqual(self.outlet._db.name, 'databay')
self.assertEqual(self.outlet._client.address[0], 'localhost')
self.assertEqual(self.outlet._client.address[1], 27017)
@mongomock.patch(servers=(('localhost', 27017),))
def test_connect_twice_different(self):
result = self.outlet.connect()
self.assertFalse(result)
self.assertEqual(self.outlet._db.name, 'databay')
result = self.outlet.connect('test')
self.assertFalse(result)
self.assertEqual(self.outlet._db.name, 'test')
self.assertEqual(self.outlet._client.address[0], 'localhost')
self.assertEqual(self.outlet._client.address[1], 27017)
def test_disconnect(self):
close_mock = MagicMock()
self.outlet._client = MagicMock(close=close_mock)
self.outlet._db = MagicMock()
self.outlet.disconnect()
close_mock.assert_called_once()
self.assertIsNone(self.outlet._client)
self.assertIsNone(self.outlet._db)
def test_on_start(self):
self.outlet.connect = MagicMock()
self.outlet.on_start()
self.outlet.connect.assert_called_once()
def test_on_shutdown(self):
self.outlet.disconnect = MagicMock()
self.outlet.on_shutdown()
self.outlet.disconnect.assert_called_once()
@mongomock.patch(servers=(('localhost', 27017),))
def test__add_collection(self):
name = 'test_collection'
self.outlet._add_collection(name)
self.assertTrue(name in self.outlet._db.list_collection_names(
), f'Database should contain collection \'{name}\'')
self.assertIsInstance(self.outlet._db[name], Collection)
@mongomock.patch(servers=(('localhost', 27017),))
def test__add_collection_invalid(self):
name = 'test_collection'
self.outlet._add_collection(None)
self.assertFalse(name in self.outlet._db.list_collection_names(
), f'Database should not contain collection \'{name}\'')
@mongomock.patch(servers=(('localhost', 27017),))
def test__get_collection(self):
name = 'test_collection'
self.outlet._add_collection(name)
self.assertTrue(name in self.outlet._db.list_collection_names(
), f'Database should contain collection \'{name}\'')
self.assertIsInstance(self.outlet._db[name], Collection)
collection = self.outlet._get_collection(name)
self.assertIsInstance(collection, Collection)
@mongomock.patch(servers=(('localhost', 27017),))
def test__get_collection_invalid(self):
name = 'test_collection'
self.outlet._add_collection(name)
self.assertTrue(name in self.outlet._db.list_collection_names(
), f'Database should contain collection \'{name}\'')
self.assertIsInstance(self.outlet._db[name], Collection)
self.assertRaises(MongoCollectionNotFound,
self.outlet._get_collection, 'invalid_name')
@mongomock.patch(servers=(('localhost', 27017),))
@patch(fqname(Record), spec=Record)
@patch(fqname(Record), spec=Record)
@patch(fqname(Record), spec=Record)
def test__group_by_collection(self, recordA, recordB, recordB2):
recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'}
recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
recordB2.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
recordA.payload = 'testA'
recordB.payload = 'testB'
recordB2.payload = 'testB2'
target = {'A': ['testA'], 'B': ['testB', 'testB2']}
records = [recordA, recordB, recordB2]
collections = self.outlet._group_by_collection(records)
self.assertEqual(collections.keys(), target.keys())
self.assertEqual(collections['A'], [recordA.payload])
self.assertEqual(collections['B'], [recordB.payload, recordB2.payload])
@mongomock.patch(servers=(('localhost', 27017),))
@patch(fqname(Record), spec=Record)
@patch(fqname(Record), spec=Record)
def test__group_by_collection_list_payload(self, recordA, recordB):
recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'}
recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
recordA.payload = 'testA'
recordB.payload = ['testB', 'testB2']
target = {'A': ['testA'], 'B': ['testB', 'testB2']}
records = [recordA, recordB]
collections = self.outlet._group_by_collection(records)
self.assertEqual(collections.keys(), target.keys())
self.assertEqual(collections['A'], [recordA.payload])
self.assertEqual(collections['B'], [*recordB.payload])
@mongomock.patch(servers=(('localhost', 27017),))
@patch(fqname(Update), spec=Update)
@patch(fqname(Record), spec=Record)
@patch(fqname(Record), spec=Record)
def test_push(self, recordA, recordB, update):
# recordA = Record(1)
# recordB = Record(1)
recordA.metadata = {MongoOutlet.MONGODB_COLLECTION: 'A'}
recordB.metadata = {MongoOutlet.MONGODB_COLLECTION: 'B'}
recordA.payload = {'testA': 1}
recordB.payload = [{'testB': 2}, {'testB2': 3}]
records = [recordA, recordB]
self.outlet.try_start()
self.outlet.push(records, update)
for record in records:
collection = self.outlet._get_collection(
record.metadata.get(MongoOutlet.MONGODB_COLLECTION))
payload = record.payload
if not isinstance(payload, list):
payload = [payload]
for p in payload:
result = collection.find_one({'_id': p.get('_id')})
self.assertEqual(p, result)
self.outlet.try_shutdown()
@mongomock.patch(servers=(('localhost', 27017),))
@patch(fqname(Update), spec=Update)
@patch(fqname(Record), spec=Record)
def test_push_inactive(self, record, update):
result = self.outlet.push([record], update)
self.assertFalse(result, 'Push should have stopped and returned False')
@mongomock.patch(servers=(('localhost', 27017),))
def test_ensure_connection(self):
mock = MagicMock(foo=MagicMock(), _db=None)
ensure_connection(mock.foo)(mock)
mock.connect.assert_called_once()
mock.foo.assert_called_once()
class TestCsvOutlet(TestCase):
@classmethod
def setUpClass(cls):
cls.this_filepath = os.path.abspath(os.path.dirname(__file__))
cls.target_filepath = os.path.join(cls.this_filepath,
'_csv_outlet_target.csv')
cls.attempt_filepath = os.path.join(cls.this_filepath,
'_csv_outlet_attempt.csv')
cls.custom_filepath = os.path.join(cls.this_filepath,
'_csv_outlet_custom_attempt.csv')
with open(cls.target_filepath, 'r') as f:
cls.target = f.read()
@patch(fqname(Record), spec=Record)
@patch(fqname(Update), spec=Update)
def setUp(self, update, record):
if os.path.exists(self.attempt_filepath):
os.remove(self.attempt_filepath)
if os.path.exists(self.custom_filepath):
os.remove(self.custom_filepath)
self.csv_outlet = CsvOutlet(self.attempt_filepath)
self.update = update
self.record = record
record.payload = {'foo': 'bar', 'baz': 'qux'}
record.metadata = {}
record.__repr__ = lambda x: 'TestRecord(test)'
update.__repr__ = lambda x: 'TestUpdate()'
self.records = [copy.copy(record) for i in range(0, 4)]
def test_push_append_mode(self):
asyncio.run(self.csv_outlet._push(self.records, self.update))
self.assertTrue(os.path.exists(self.attempt_filepath),
'File should exist')
with open(self.attempt_filepath, 'r') as f:
self.assertEqual(f.read(), self.target)
os.remove(self.attempt_filepath)
def test_push_write_mode(self):
for record in self.records:
record.metadata = {CsvOutlet.FILE_MODE: 'w'}
asyncio.run(self.csv_outlet._push(self.records, self.update))
self.assertTrue(os.path.exists(self.attempt_filepath),
'File should exist')
with open(self.attempt_filepath, 'r') as f:
self.assertEqual(
f.read(), 'foo,baz\nbar,qux\n',
'Each write should have overridden the previous.')
os.remove(self.attempt_filepath)
def test_push_custom_files(self):
self.records[0].metadata = {CsvOutlet.CSV_FILE: self.custom_filepath}
asyncio.run(self.csv_outlet._push(self.records, self.update))
self.assertTrue(os.path.exists(self.attempt_filepath),
'File should exist')
self.assertTrue(os.path.exists(self.custom_filepath),
'File should exist')
with open(self.attempt_filepath, 'r') as f:
self.assertEqual(f.read(), 'foo,baz\nbar,qux\nbar,qux\nbar,qux\n',
'Should miss one record')
with open(self.custom_filepath, 'r') as f:
self.assertEqual(f.read(), 'foo,baz\nbar,qux\n',
'Should contain the one record')
os.remove(self.attempt_filepath)
os.remove(self.custom_filepath)
class InletTester(TestCase):
"""
Utility class used for testing concrete implementations of :any:`Inlet`.
"""
def get_inlet(self): # pragma: no cover
"""Implement this method to return instances of your inlet class."""
return NullInlet()
def setUp(self):
self.gmetadata = {'global': 'global'}
self.inlets = self.get_inlet()
if not isinstance(self.inlets, list):
self.inlets = [self.inlets]
self.inlet = self.inlets[0]
self.inlet._metadata = {**self.inlet._metadata, **self.gmetadata}
@for_each_inlet
def test_new_record(self):
"""
|decorated| :any:`for_each_inlet`
Test creating new records and passing local metadata.
"""
payload = {'test': 123}
metadata = {'metadata': 321}
record = self.inlet.new_record(payload=payload, metadata=metadata)
self.assertIsInstance(record, Record)
self.assertEqual(record.payload, payload)
self.assertLessEqual(
metadata.items(), record.metadata.items(),
'Local metadata should be contained in record.metadata')
self.assertLessEqual(
self.gmetadata.items(), record.metadata.items(),
'Global metadata should be contained in record.metadata')
@for_each_inlet
def test_new_record_override_global(self):
"""
|decorated| :any:`for_each_inlet`
Test creating new records and overriding global metadata.
"""
payload = {'test': 123}
metadata = {'global': 'local'}
record = self.inlet.new_record(payload=payload, metadata=metadata)
self.assertIsInstance(record, Record)
self.assertEqual(record.payload, payload)
self.assertLessEqual(
metadata.items(), record.metadata.items(),
'Local metadata should be contained in record.metadata')
self.assertEqual(
record.metadata['global'], 'local',
'Global metadata should be overridden by local metadata')
@patch(fqname(Update))
@for_each_inlet
def test_dont_read_metadata(self, update):
"""
|decorated| :any:`for_each_inlet`
Test creating new records and overriding global metadata.
"""
meta = MagicMock()
meta.keys.return_value = self.gmetadata.keys()
meta.__getitem__.side_effect = lambda x: self.gmetadata.__getitem__(x)
self.inlet._metadata = meta
self.inlet.on_start()
records = asyncio.run(self.inlet._pull(update))
self.inlet.on_shutdown()
self.inlet._metadata.get.assert_not_called()
self.assertEqual(self.inlet._metadata.__getitem__.call_count,
len(self.gmetadata))
self.assertIsInstance(records, (list))
for record in records:
self.assertIsInstance(record, Record)
self.assertIsNotNone(record.payload)
self.assertLessEqual(
self.gmetadata.items(), record.metadata.items(),
'Global metadata should be contained in record.metadata')
@for_each_inlet
@patch(fqname(Update))
def test_pull(self, update):
"""
|decorated| :any:`for_each_inlet`
Test pulling data from the inlet.
"""
self.inlet.on_start()
records = asyncio.run(self.inlet._pull(update))
self.inlet.on_shutdown()
self.assertIsInstance(records, (list))
for record in records:
self.assertIsInstance(record, Record)
self.assertIsNotNone(record.payload)
self.assertLessEqual(
self.gmetadata.items(), record.metadata.items(),
'Global metadata should be contained in record.metadata')
class TestLink(TestCase):
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_transfer(self, inlet, outlet):
link = Link([inlet], [outlet],
timedelta(seconds=1),
name='test_update')
link.transfer()
inlet._pull.assert_called()
outlet._push.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_run(self, inlet, outlet):
async def task():
link = Link([inlet], [outlet],
timedelta(seconds=1),
name='test_run',
copy_records=False)
inlet_result = await inlet._pull(None)
await link._run()
inlet._pull.assert_called()
outlet._push.assert_called_with(inlet_result, mock.ANY)
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_inlet(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
catch_exceptions=False,
name='test_exception_inlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_outlet(self, inlet, outlet):
# inlet._pull.return_value, _ = self.inlet_return()
# inlet._pull.side_effect = pull_mock
# inlet._pull.return_value = Future()
outlet._push.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
catch_exceptions=False,
name='test_exception_outlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_caught(self, inlet, outlet):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
name='test_exception_caught',
catch_exceptions=True)
try:
link.transfer()
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
# check that one exception doesn't halt other inlets or outlets
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_catch_partial_exception(self, inlet1, inlet2, outlet1, outlet2):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
async def task():
# inlet_future = Future()
inlet1._pull.side_effect = DummyException('Test inlet1 exception')
outlet1._push.side_effect = DummyException(
'Test outlet1 exception')
# inlet1._pull.return_value = inlet_future
# inlet2._pull.return_value = inlet_future
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1),
name='test_catch_partial_exception',
copy_records=False,
catch_exceptions=True)
# results = [object()]
results = await inlet2._pull(None)
# inlet_future.set_result(results)
await link._run()
inlet1._pull.assert_called()
inlet2._pull.assert_called()
outlet1._push.assert_called_with(results, mock.ANY)
outlet2._push.assert_called_with(results, mock.ANY)
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_start(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
name='test_on_start')
link.on_start()
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_start_already_active(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
name='test_on_start_already_active')
link.on_start()
inlet1.on_start.assert_not_called()
outlet1.on_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_shutdown(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
name='test_on_shutdown')
link.on_shutdown()
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_already_inactive(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
name='test_on_shutdown_already_inactive')
link.on_shutdown()
inlet1.on_shutdown.assert_not_called()
outlet1.on_shutdown.assert_not_called()
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet(self, inlet1):
link = Link([], [], timedelta(seconds=1), name='test_add_inlet')
link.add_inlets(inlet1)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet_multiple(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
name='test_add_inlet_multiple')
link.add_inlets([inlet1, inlet2])
self.assertEqual(link.inlets, [inlet1, inlet2])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet_same(self, inlet1):
link = Link([], [], timedelta(seconds=1), name='test_add_inlet_same')
link.add_inlets(inlet1)
self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_remove_inlet(self, inlet1, inlet2):
link = Link([], [], timedelta(seconds=1), name='test_remove_inlet')
link.add_inlets([inlet1, inlet2])
link.remove_inlets(inlet2)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_remove_inlet_invalid(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
name='test_remove_inlet_invalid')
link.add_inlets([inlet1])
self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet(self, outlet1):
link = Link([], [], timedelta(seconds=1), name='test_add_outlet')
link.add_outlets(outlet1)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet_multiple(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
name='test_add_outlet_multiple')
link.add_outlets([outlet1, outlet2])
self.assertEqual(link.outlets, [outlet1, outlet2])
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet_same(self, outlet1):
link = Link([], [], timedelta(seconds=1), name='test_add_outlet_same')
link.add_outlets(outlet1)
self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_remove_outlet(self, outlet1, outlet2):
link = Link([], [], timedelta(seconds=1), name='test_remove_outlet')
link.add_outlets([outlet1, outlet2])
link.remove_outlets(outlet2)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_remove_outlet_invalid(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
name='test_remove_outlet_invalid')
link.add_outlets([outlet1])
self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Inlet), spec=Inlet,
_pull=pull_mock(object())) # this rv is invalid, should be a list
def xtest_non_iterable_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
name='test_non_iterable_raised')
with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
link.transfer()
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable())
) # this rv will raise DummyException
def test_generic_error_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
name='test_generic_error_raised')
# with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION):
self.assertRaises(DummyException, link.transfer)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet,
_pull=pull_mock(object())) # this rv is invalid, should be a list
def xtest_non_iterable_caught(self, inlet1, inlet2, outlet1):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
link = Link([inlet1, inlet2], [outlet1],
timedelta(seconds=1),
name='test_non_iterable_caught',
catch_exceptions=True)
results = asyncio.run(inlet2._pull(None))
link.transfer()
outlet1._push.assert_called_with(results, mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable())
) # this rv will raise DummyException
def xtest_generic_error_caught(self, inlet1, inlet2, outlet1):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
link = Link([inlet1, inlet2], [outlet1],
timedelta(seconds=1),
name='test_generic_error_caught',
catch_exceptions=True)
results = asyncio.run(inlet2._pull(None))
link.transfer()
outlet1._push.assert_called_with(results, mock.ANY)
class TestBuffers(TestCase):
@patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
@patch(fqname(Inlet), spec=Inlet)
def test_buffer_count(self, inlet, outlet):
buffer = Buffer(count_threshold=3)
payload = [1, 2, 3, 4]
records = [Record(payload=p) for p in payload]
link = Link(inlet,
outlet,
interval=1,
processors=buffer,
copy_records=False)
inlet._pull = pull_mock(records[:2])
link.transfer()
outlet._push.assert_called_with(
[], mock.ANY) # after first call we shouldn't have any records
inlet._pull = pull_mock(records[2:])
link.transfer()
outlet._push.assert_called_with(
records, mock.ANY) # all records should be returned here
@patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
@patch(fqname(Inlet), spec=Inlet)
def test_buffer_time(self, inlet, outlet):
buffer = Buffer(time_threshold=0.02)
payload = [1, 2, 3, 4]
records = [Record(payload=p) for p in payload]
link = Link(inlet,
outlet,
interval=1,
processors=buffer,
copy_records=False)
inlet._pull = pull_mock(records[:2])
link.transfer()
outlet._push.assert_called_with(
[], mock.ANY) # not enough time have passed
inlet._pull = pull_mock(records[2:])
link.transfer()
outlet._push.assert_called_with(
[], mock.ANY) # not enough time have passed
time.sleep(0.02)
inlet._pull = pull_mock([])
link.transfer()
outlet._push.assert_called_with(
records, mock.ANY) # all records should be returned here
@patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
@patch(fqname(Inlet), spec=Inlet)
def test_flush(self, inlet, outlet):
buffer = Buffer(count_threshold=100, time_threshold=10)
payload = [1, 2, 3, 4]
records = [Record(payload=p) for p in payload]
link = Link(inlet,
outlet,
interval=1,
processors=buffer,
copy_records=False)
inlet._pull = pull_mock(records[:2])
link.transfer()
outlet._push.assert_called_with([], mock.ANY) # no records yet
inlet._pull = pull_mock(records[2:])
link.transfer()
outlet._push.assert_called_with([], mock.ANY) # no records yet
buffer.flush = True
inlet._pull = pull_mock([])
link.transfer()
outlet._push.assert_called_with(
records, mock.ANY) # all records should be flushed
@patch(fqname(Outlet), spec=Outlet, _push=CoroutineMock())
@patch(fqname(Inlet), spec=Inlet)
def test_flush_after_shutdown(self, inlet, outlet):
buffer = Buffer(count_threshold=100, time_threshold=10)
counter_dict = {'counter': 0, 'records': []}
link = Link(inlet,
outlet,
interval=0.01,
processors=buffer,
copy_records=False)
planner = SchedulePlanner(link, refresh_interval=0.01)
async def pull_coro(_):
counter_dict['counter'] += 1
record = Record(payload=counter_dict['counter'])
counter_dict['records'].append(record)
return [record]
mock_pull = MagicMock(side_effect=pull_coro)
inlet._pull = mock_pull
th = Thread(target=planner.start, daemon=True)
th.start()
time.sleep(0.1)
planner.shutdown()
th.join()
calls = outlet._push.call_args_list
for c in calls:
self.assertEqual(c(), [],
'Should only contain empty record lists.')
self.assertEqual(buffer.records, counter_dict['records'],
'All records should be stored in the buffer')
planner.force_transfer()
self.assertEqual(outlet._push.call_args(), [],
'Should return empty record list')
buffer.flush = True
planner.force_transfer()
self.assertEqual(outlet._push.call_args[0][0], counter_dict['records'],
'Should return all records')
class TestAPSPlanner(TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
logging.getLogger('databay').setLevel(logging.WARNING)
@patch(fqname(Link), spec=Link)
def setUp(self, link):
self.planner = APSPlanner()
def set_job(job):
link.job = job
link.interval.total_seconds.return_value = 0.02
link.set_job.side_effect = set_job
link.job = None
self.link = link
def test__schedule(self):
self.planner._schedule(self.link)
self.assertIsNotNone(self.link.job, 'Link should contain a job')
asp_job = self.planner._scheduler.get_jobs()[0]
self.assertEqual(self.link.job, asp_job,
'Link\'s job should be same as scheduler\'s')
def test__unschedule(self):
self.planner._schedule(self.link)
self.planner._unschedule(self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertEqual(len(self.planner._scheduler.get_jobs()), 0,
'Scheduler should not have any jobs')
def test__unschedule_invalid(self):
self.planner._unschedule(self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertEqual(len(self.planner._scheduler.get_jobs()), 0,
'Scheduler should not have any jobs')
def test_add_links(self):
self.planner.add_links(self.link)
self.assertIsNotNone(self.link.job, 'Link should contain a job')
self.assertTrue(self.link in self.planner.links,
'Planner should contain the link')
def test_add_links_on_init(self):
self.planner = APSPlanner(self.link)
self.assertIsNotNone(self.link.job, 'Link should contain a job')
self.assertTrue(self.link in self.planner.links,
'Planner should contain the link')
def test_remove_links(self):
self.planner.add_links(self.link)
self.planner.remove_links(self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertTrue(self.link not in self.planner.links,
'Planner should not contain the link')
def test_remove_invalid_link(self):
self.assertRaises(MissingLinkError, self.planner.remove_links,
self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertTrue(self.link not in self.planner.links,
'Planner should not contain the link')
def test_start(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.assertTrue(self.planner.running, 'Scheduler should be running')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
# todo: APS is currently broken for this test case, wait for an update
def xtest_start_paused(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.pause()
self.assertRaises(SchedulerAlreadyRunningError, self.planner.start)
self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
'Scheduler should be paused')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_shutdown(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.shutdown(False)
self.assertFalse(self.planner.running,
'Scheduler should not be running')
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_pause(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.pause()
self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
'Scheduler should be paused')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_resume(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.pause()
self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
'Scheduler should be paused')
self.planner.resume()
self.assertTrue(self.planner.running, 'Scheduler should not be paused')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_shutdown_paused(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.pause()
self.assertEqual(self.planner._scheduler.state, STATE_PAUSED,
'Scheduler should be paused')
self.planner.shutdown(False)
self.assertFalse(self.planner.running,
'Scheduler should not be running')
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_pause_shutdown(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.shutdown(False)
self.assertRaises(SchedulerNotRunningError, self.planner.pause)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_resume_shutdown(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.shutdown(False)
self.assertRaises(SchedulerNotRunningError, self.planner.resume)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_start_shutdown(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread 1 should be stopped.')
self.assertFalse(self.planner.running,
'Scheduler should not be running')
th2 = Thread(target=self.planner.start, daemon=True)
th2.start()
self.assertTrue(self.planner.running, 'Scheduler should be running')
self.planner.shutdown(False)
th2.join(timeout=2)
self.assertFalse(th2.is_alive(), 'Thread 2 should be stopped.')
def test_add_and_run(self):
self.link.interval.total_seconds.return_value = 0.02
self.planner.add_links(self.link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
self.link.transfer.assert_called()
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def _with_exception(self, link, catch_exceptions):
logging.getLogger('databay').setLevel(logging.CRITICAL)
# logging.getLogger('databay').setLevel(logging.INFO)
self.planner = APSPlanner(catch_exceptions=catch_exceptions)
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner.add_links(link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
link.transfer.assert_called()
if catch_exceptions:
self.assertTrue(self.planner.running,
'Scheduler should be running')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
self.assertFalse(self.planner.running, 'Scheduler should be stopped')
def test_catch_exception(self):
self._with_exception(self.link, True)
def test_raise_exception(self):
self._with_exception(self.link, False)
def test_uncommon_exception(self):
logging.getLogger('databay').setLevel(logging.CRITICAL)
self.link.transfer.side_effect = DummyUnusualException(123, True)
self.link.interval.total_seconds.return_value = 0.02
self.planner.add_links(self.link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
self.link.transfer.assert_called()
self.assertFalse(self.planner.running, 'Scheduler should be stopped')
class TestLink(TestCase):
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_transfer(self, inlet, outlet):
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_update')
link.transfer()
inlet._pull.assert_called()
outlet._push.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_run(self, inlet, outlet):
async def task():
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_run',
copy_records=False)
inlet_result = await inlet._pull(None)
await link._run()
inlet._pull.assert_called()
outlet._push.assert_called_with(inlet_result, mock.ANY)
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_inlet(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_inlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_outlet(self, inlet, outlet):
# inlet._pull.return_value, _ = self.inlet_return()
# inlet._pull.side_effect = pull_mock
# inlet._pull.return_value = Future()
outlet._push.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_outlet')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_outlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_exception_caught(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_exception_caught',
ignore_exceptions=True)
try:
with self.assertLogs(logging.getLogger('databay.Link'),
level='WARNING') as cm:
link.transfer()
self.assertTrue('Test inlet exception' in ';'.join(cm.output))
self.assertTrue('Test outlet exception' in ';'.join(cm.output))
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
# check that one exception doesn't halt other inlets or outlets
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_ignore_partial_exception(self, inlet1, inlet2, outlet1, outlet2):
async def task():
# inlet_future = Future()
inlet1._pull.side_effect = DummyException('Test inlet1 exception')
outlet1._push.side_effect = DummyException(
'Test outlet1 exception')
# inlet1._pull.return_value = inlet_future
# inlet2._pull.return_value = inlet_future
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_ignore_partial_exception',
copy_records=False,
ignore_exceptions=True)
# results = [object()]
results = await inlet2._pull(None)
# inlet_future.set_result(results)
with self.assertLogs(logging.getLogger('databay.Link'),
level='WARNING') as cm:
await link._run()
self.assertTrue('Test inlet1 exception' in ';'.join(cm.output))
self.assertTrue(
'Test outlet1 exception' in ';'.join(cm.output))
inlet1._pull.assert_called()
inlet2._pull.assert_called()
outlet1._push.assert_called_with(results, mock.ANY)
outlet2._push.assert_called_with(results, mock.ANY)
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_start(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start')
link.on_start()
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_start_already_active(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start_already_active')
link.on_start()
inlet1.on_start.assert_not_called()
outlet1.on_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_on_shutdown(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown')
link.on_shutdown()
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_already_inactive(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown_already_inactive')
link.on_shutdown()
inlet1.on_shutdown.assert_not_called()
outlet1.on_shutdown.assert_not_called()
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet(self, inlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet')
link.add_inlets(inlet1)
self.assertEqual(link.inlets, [inlet1])
def test_add_inlet_invalid(self):
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet')
self.assertRaises(TypeError, link.add_inlets, "invalid_inlet")
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet_multiple(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_add_inlet_multiple')
link.add_inlets([inlet1, inlet2])
self.assertEqual(link.inlets, [inlet1, inlet2])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_add_inlet_same(self, inlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same')
link.add_inlets(inlet1)
self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_remove_inlet(self, inlet1, inlet2):
link = Link([], [], timedelta(seconds=1), tags='test_remove_inlet')
link.add_inlets([inlet1, inlet2])
link.remove_inlets(inlet2)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock())
def test_remove_inlet_invalid(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_remove_inlet_invalid')
link.add_inlets([inlet1])
self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
self.assertEqual(link.inlets, [inlet1])
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet(self, outlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_outlet')
link.add_outlets(outlet1)
self.assertEqual(link.outlets, [outlet1])
def test_add_outlet_invalid(self):
link = Link([], [], timedelta(seconds=1), tags='test_add_outlet')
self.assertRaises(TypeError, link.add_outlets, "invalid_outlet")
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet_multiple(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_add_outlet_multiple')
link.add_outlets([outlet1, outlet2])
self.assertEqual(link.outlets, [outlet1, outlet2])
@patch(fqname(Outlet), spec=Outlet)
def test_add_outlet_same(self, outlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same')
link.add_outlets(outlet1)
self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_remove_outlet(self, outlet1, outlet2):
link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet')
link.add_outlets([outlet1, outlet2])
link.remove_outlets(outlet2)
self.assertEqual(link.outlets, [outlet1])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
def test_remove_outlet_invalid(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_remove_outlet_invalid')
link.add_outlets([outlet1])
self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
self.assertEqual(link.outlets, [outlet1])
# this rv is invalid, should be a list
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(object()))
def xtest_non_iterable_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
tags='test_non_iterable_raised')
with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
link.transfer()
# this rv will raise DummyException
@patch(fqname(Inlet), spec=Inlet, _pull=pull_mock(DummyIterable()))
def test_generic_error_raised(self, inlet1):
link = Link([inlet1], [],
timedelta(seconds=1),
tags='test_generic_error_raised')
# with self.assertRaisesRegex(TypeError, databay.link._ITERABLE_EXCEPTION):
self.assertRaises(DummyException, link.transfer)
def test_integer_to_timedelta(self):
link = Link([], [], 1, name='test_integer_interval_coerced')
self.assertEqual(link._interval, timedelta(seconds=1))
def test_float_to_timedelta(self):
link = Link([], [], 1.5, name='test_float_interval_coerced')
self.assertEqual(link._interval, timedelta(seconds=1.5))
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_inlet_exception_raise(self, inlet1, outlet1):
inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start')
self.assertRaises(RuntimeError, link.on_start)
inlet1.try_start.assert_called()
outlet1.try_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_inlet_exception_catch(self, inlet1, outlet1):
inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_start()
self.assertTrue('on_start inlet exception: "" for inlet:' in
';'.join(cm.output))
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_outlet_exception_raise(self, inlet1, outlet1, outlet2):
outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_start')
self.assertRaises(RuntimeError, link.on_start)
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
outlet2.try_start.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_start_outlet_exception_catch(self, inlet1, outlet1, outlet2):
outlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_start',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_start()
self.assertTrue(
'on_start outlet exception: "" for outlet:'
in ';'.join(cm.output), cm.output)
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
outlet2.try_start.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_inlet_exception_raise(self, inlet1, outlet1):
inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown')
self.assertRaises(RuntimeError, link.on_shutdown)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_inlet_exception_catch(self, inlet1, outlet1):
inlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_shutdown()
self.assertTrue(
'on_shutdown inlet exception: "" for inlet:'
in ';'.join(cm.output), cm.output)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_outlet_exception_raise(self, inlet1, outlet1,
outlet2):
outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
)
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_shutdown')
self.assertRaises(RuntimeError, link.on_shutdown)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
outlet2.try_shutdown.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_on_shutdown_outlet_exception_catch(self, inlet1, outlet1,
outlet2):
outlet1.try_shutdown.side_effect = lambda: exec('raise(RuntimeError())'
)
link = Link([inlet1], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_on_shutdown',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_shutdown()
self.assertTrue(
'on_shutdown outlet exception: "" for outlet:'
in ';'.join(cm.output), cm.output)
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
outlet2.try_shutdown.assert_called()
def test_single_tag(self):
tag = 'tagA'
link = Link([], [], timedelta(seconds=1), tags=tag)
self.assertEqual(link.tags, [tag])
def test_multiple_tags(self):
tagA = 'tagA'
tagB = 'tagB'
link = Link([], [], timedelta(seconds=1), tags=[tagA, tagB])
self.assertEqual(link.tags, [tagA, tagB])
def test_tag_as_name(self):
link_name = 'link_name'
link = Link([], [], timedelta(seconds=1), name=link_name)
self.assertEqual(link_name, link.tags[0])
def test_name_from_tag(self):
link_name = 'link_name'
link = Link([], [], timedelta(seconds=1), tags=[link_name])
self.assertEqual(link.name, link.tags[0])
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
@patch(fqname(Inlet), spec=Inlet)
@patch(fqname(Inlet), spec=Inlet)
def test_inlet_concurrency(self, inlet1, inlet2, inlet3, outlet):
counter = {'value': 0}
# this will increment the counter on each async call, and check if they exceed the concurrency value
async def slow_pull(_):
counter['value'] += 1
self.assertLessEqual(counter['value'], 2,
"Only 2 inlets should pull at a time")
await asyncio.sleep(0.01)
counter['value'] -= 1
return [123]
inlet1._pull = slow_pull
inlet2._pull = slow_pull
inlet3._pull = slow_pull
async def task():
link = Link([inlet1, inlet2, inlet3], [outlet],
timedelta(seconds=1),
tags='test_inlet_concurrency',
copy_records=False,
inlet_concurrency=2)
await link._run()
asyncio.run(task())
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_processors_one(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processor = MagicMock(side_effect=lambda r: r)
link = Link(inlet, outlet, interval=0.01, processors=processor)
link.transfer()
processor.assert_called_with(records)
outlet._push.assert_called_with(records, mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_processors_many(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(side_effect=lambda x: x)
processorB = MagicMock(side_effect=lambda x: x)
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with(records)
outlet._push.assert_called_with(records, mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_processors_change_records(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=lambda r: list(map(lambda y: y * y, r)))
processorB = MagicMock(
side_effect=lambda r: list(map(lambda y: -y, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with([4, 9])
outlet._push.assert_called_with([-4, -9], mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_processors_filter_records(self, inlet, outlet):
records = [2, 3, 4]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=lambda r: list(filter(lambda y: y > 2, r)))
processorB = MagicMock(
side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with([3, 4])
outlet._push.assert_called_with([4], mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_groupers_one_passive(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
grouper = MagicMock(side_effect=lambda r: r) # does nothing on purpose
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
link.transfer()
grouper.assert_called_with([records])
outlet._push.assert_called_with(records, mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_groupers_one_active(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
# makes [[1,2], [3,4]]
grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
link.transfer()
grouper.assert_called_with([records])
calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_groupers_many_active(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
# makes [[1,2], [3,4]]
grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
# makes [[1], [2], [3], [4]]
grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] +
[[sub] for sub in r[1]])
link = Link(inlet,
outlet,
interval=0.01,
groupers=[grouperA, grouperB])
link.transfer()
grouperA.assert_called_with([records])
callsA = [call([records[:2], records[2:]])] # expects [[1,2], [3,4]]
grouperB.assert_has_calls(callsA)
callsB = [
call([records[0]], mock.ANY),
call([records[1]], mock.ANY),
call([records[2]], mock.ANY),
call([records[3]], mock.ANY)
]
outlet._push.assert_has_calls(callsB) # expects [[1], [2], [3], [4]]
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_processor_exception(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processor = MagicMock(
side_effect=DummyException('Processor exception'))
link = Link(inlet, outlet, interval=0.01, processors=processor)
self.assertRaises(DummyException, link.transfer)
processor.assert_called_with(records)
outlet._push.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_processor_exception_ignored(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=DummyException('Processor exception'))
processorB = MagicMock(
side_effect=lambda r: list(map(lambda y: y * y, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB],
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.transfer()
self.assertTrue('Processor exception:' in ';'.join(cm.output),
cm.output)
processorA.assert_called_with(records)
processorB.assert_called_with(records)
outlet._push.assert_called_with([4, 9], mock.ANY)
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_grouper_exception(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
grouper = MagicMock(side_effect=DummyException('Grouper exception'))
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
self.assertRaises(DummyException, link.transfer)
grouper.assert_called_with([records])
outlet._push.assert_not_called()
@patch(fqname(Outlet), spec=Outlet)
@patch(fqname(Inlet), spec=Inlet)
def test_grouper_exception_ignored(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
grouperA = MagicMock(side_effect=DummyException('Grouper exception'))
grouperB = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
link = Link(inlet,
outlet,
interval=0.01,
groupers=[grouperA, grouperB],
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.transfer()
self.assertTrue('Grouper exception:' in ';'.join(cm.output),
cm.output)
grouperA.assert_called_with([records])
grouperB.assert_called_with([records])
calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
class TestSchedulePlanner(TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
logging.getLogger('databay').setLevel(logging.WARNING)
@patch(fqname(Link), spec=Link)
def setUp(self, link):
self.planner = SchedulePlanner(refresh_interval=0.02)
def set_job(job):
link.job = job
link.interval.total_seconds.return_value = 0.02
link.set_job.side_effect = set_job
link.job = None
self.link = link
def tearDown(self):
if len(schedule.jobs) > 0:
schedule.clear()
def test__run_job(self):
self.planner._create_thread_pool()
self.planner._run_job(self.link)
self.link.transfer.assert_called_once()
self.planner._destroy_thread_pool()
def test__schedule(self):
self.planner._schedule(self.link)
self.assertIsNotNone(self.link.job, 'Link should contain a job')
schedule_job = schedule.jobs[0]
self.assertEqual(self.link.job, schedule_job,
'Link\'s job should be same as schedule\'s')
# self.planner._unschedule(link)
def test__unschedule(self):
self.planner._schedule(self.link)
self.planner._unschedule(self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertEqual(len(schedule.jobs), 0,
'Schedule should not have any jobs')
def test__unschedule_invalid(self):
self.planner._unschedule(self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertEqual(len(schedule.jobs), 0,
'Scheduler should not have any jobs')
def test_add_links(self):
self.planner.add_links(self.link)
self.assertIsNotNone(self.link.job, 'Link should contain a job')
self.assertTrue(self.link in self.planner.links,
'Planner should contain the link')
def test_add_links_on_init(self):
self.planner = SchedulePlanner(self.link, refresh_interval=0.02)
self.assertIsNotNone(self.link.job, 'Link should contain a job')
self.assertTrue(self.link in self.planner.links,
'Planner should contain the link')
def test_remove_links(self):
self.planner.add_links(self.link)
self.planner.remove_links(self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertTrue(self.link not in self.planner.links,
'Planner should not contain the link')
def test_remove_invalid_link(self):
self.assertRaises(MissingLinkError, self.planner.remove_links,
self.link)
self.assertIsNone(self.link.job, 'Link should not contain a job')
self.assertTrue(self.link not in self.planner.links,
'Planner should not contain the link')
def test_start(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.assertTrue(self.planner._running, 'Planner should be running')
self.planner.shutdown()
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_shutdown(self):
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.planner.shutdown()
self.assertFalse(self.planner._running,
'Planner should be not running')
self.assertIsNone(self.planner._thread_pool,
'Planner should not have a thread pool')
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_add_and_run(self):
self.link.interval.total_seconds.return_value = 0.02
self.planner._refresh_interval = 0.02
self.planner.add_links(self.link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
self.link.transfer.assert_called()
self.planner.shutdown()
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
def test_invalid_interval(self):
self.link.interval.total_seconds.return_value = 0.1
self.planner._refresh_interval = 0.2
self.assertRaises(ScheduleIntervalError, self.planner.add_links,
self.link)
def _with_exception(self, link, catch_exceptions):
logging.getLogger('databay').setLevel(logging.CRITICAL)
self.planner = SchedulePlanner(catch_exceptions=catch_exceptions)
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner._refresh_interval = 0.02
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner.add_links(link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
link.transfer.assert_called()
if catch_exceptions:
self.assertTrue(self.planner.running,
'Scheduler should be running')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
self.assertFalse(self.planner.running, 'Scheduler should be stopped')
def test_catch_exception(self):
self._with_exception(self.link, True)
def test_raise_exception(self):
self._with_exception(self.link, False)
def test_uncommon_exception(self):
logging.getLogger('databay').setLevel(logging.CRITICAL)
self.link.transfer.side_effect = DummyUnusualException(argA=123,
argB=True)
self.link.interval.total_seconds.return_value = 0.02
self.planner.add_links(self.link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
self.link.transfer.assert_called()
self.assertFalse(self.planner.running, 'Scheduler should be stopped')
