def _with_exception(self, link, ignore_exceptions):
self.planner = SchedulePlanner(ignore_exceptions=ignore_exceptions)
self.planner.immediate_transfer = False # otherwise planner will never start
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner._refresh_interval = 0.02
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner.add_links(link)
with self.assertLogs(logging.getLogger('databay.BasePlanner'),
level='WARNING') as cm:
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
link.transfer.assert_called()
if ignore_exceptions:
self.assertTrue(self.planner.running,
'Planner should be running')
self.planner.shutdown(wait=False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
self.assertFalse(self.planner.running, 'Planner should be stopped')
self.assertTrue('I\'m a dummy exception' in ';'.join(cm.output))
async def task():
# inlet_future = Future()
inlet1._pull.side_effect = DummyException('Test inlet1 exception')
outlet1._push.side_effect = DummyException(
'Test outlet1 exception')
# inlet1._pull.return_value = inlet_future
# inlet2._pull.return_value = inlet_future
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_ignore_partial_exception',
copy_records=False,
ignore_exceptions=True)
# results = [object()]
results = await inlet2._pull(None)
# inlet_future.set_result(results)
with self.assertLogs(logging.getLogger('databay.Link'),
level='WARNING') as cm:
await link._run()
self.assertTrue('Test inlet1 exception' in ';'.join(cm.output))
self.assertTrue(
'Test outlet1 exception' in ';'.join(cm.output))
inlet1._pull.assert_called()
inlet2._pull.assert_called()
outlet1._push.assert_called_with(results, mock.ANY)
outlet2._push.assert_called_with(results, mock.ANY)
def test_exception_caught(self, inlet, outlet):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_exception_caught',
ignore_exceptions=True)
try:
link.transfer()
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
def test_grouper_exception(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
grouper = MagicMock(side_effect=DummyException('Grouper exception'))
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
self.assertRaises(DummyException, link.transfer)
grouper.assert_called_with([records])
outlet._push.assert_not_called()
def test_processor_exception(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processor = MagicMock(
side_effect=DummyException('Processor exception'))
link = Link(inlet, outlet, interval=0.01, processors=processor)
self.assertRaises(DummyException, link.transfer)
processor.assert_called_with(records)
outlet._push.assert_not_called()
def test_exception_caught(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_exception_caught',
ignore_exceptions=True)
try:
with self.assertLogs(logging.getLogger('databay.Link'),
level='WARNING') as cm:
link.transfer()
self.assertTrue('Test inlet exception' in ';'.join(cm.output))
self.assertTrue('Test outlet exception' in ';'.join(cm.output))
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
def test_exception_inlet(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
ignore_exceptions=False,
tags='test_exception_inlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_not_called()
async def task():
# inlet_future = Future()
inlet1._pull.side_effect = DummyException('Test inlet1 exception')
outlet1._push.side_effect = DummyException(
'Test outlet1 exception')
# inlet1._pull.return_value = inlet_future
# inlet2._pull.return_value = inlet_future
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1),
tags='test_ignore_partial_exception',
copy_records=False,
ignore_exceptions=True)
# results = [object()]
results = await inlet2._pull(None)
# inlet_future.set_result(results)
await link._run()
inlet1._pull.assert_called()
inlet2._pull.assert_called()
outlet1._push.assert_called_with(results, mock.ANY)
outlet2._push.assert_called_with(results, mock.ANY)
def test_exception_outlet(self, inlet, outlet):
# inlet._pull.return_value, _ = self.inlet_return()
# inlet._pull.side_effect = pull_mock
# inlet._pull.return_value = Future()
outlet._push.side_effect = DummyException('Test exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
catch_exceptions=False,
name='test_exception_outlet')
self.assertRaises(DummyException, link.transfer)
inlet._pull.assert_called()
outlet._push.assert_called()
def _with_exception(self, link, catch_exceptions):
logging.getLogger('databay').setLevel(logging.CRITICAL)
self.planner = SchedulePlanner(catch_exceptions=catch_exceptions)
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner._refresh_interval = 0.02
link.transfer.side_effect = DummyException()
link.interval.total_seconds.return_value = 0.02
self.planner.add_links(link)
th = Thread(target=self.planner.start, daemon=True)
th.start()
time.sleep(0.04)
link.transfer.assert_called()
if catch_exceptions:
self.assertTrue(self.planner.running,
'Scheduler should be running')
self.planner.shutdown(False)
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
self.assertFalse(self.planner.running, 'Scheduler should be stopped')
def test_immediate_transfer_exception(self):
self.link.interval.total_seconds.return_value = 10
self.planner._ignore_exceptions = False
self.link.transfer.side_effect = DummyException(
'First transfer exception!')
self.planner.add_links(self.link)
with self.assertLogs(logging.getLogger('databay.BasePlanner'),
level='WARNING') as cm:
th = Thread(target=self.planner.start, daemon=True)
th.start()
self.link.transfer.assert_called_once()
self.assertFalse(self.planner.running,
'Planner should not have started')
th.join(timeout=2)
self.assertFalse(th.is_alive(), 'Thread should be stopped.')
self.assertTrue('First transfer exception!' in ';'.join(cm.output))
def test_custom_exception(self):
custom_controller = MagicMock(
side_effect=DummyException('Custom controller exception!'))
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
buffer = Buffer(count_threshold=10,
custom_controllers=custom_controller)
with self.assertLogs(logging.getLogger('databay.Buffer'),
level='DEBUG') as cm:
rvA = buffer(records[:6])
rvB = buffer(records[6:])
self.assertTrue(
'Custom controller exception!' in ';'.join(cm.output))
self.assertEqual(rvA, [], 'Should not contain any records yet')
self.assertEqual(rvB, records, 'Should contain all records')
def test_grouper_exception_ignored(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
grouperA = MagicMock(side_effect=DummyException('Grouper exception'))
grouperB = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
link = Link(inlet,
outlet,
interval=0.01,
groupers=[grouperA, grouperB],
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.transfer()
self.assertTrue('Grouper exception:' in ';'.join(cm.output),
cm.output)
grouperA.assert_called_with([records])
grouperB.assert_called_with([records])
calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_processor_exception_ignored(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=DummyException('Processor exception'))
processorB = MagicMock(
side_effect=lambda r: list(map(lambda y: y * y, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB],
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.transfer()
self.assertTrue('Processor exception:' in ';'.join(cm.output),
cm.output)
processorA.assert_called_with(records)
processorB.assert_called_with(records)
outlet._push.assert_called_with([4, 9], mock.ANY)
def __iter__(self):
raise DummyException()