def test_groupers_many_active(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
# makes [[1,2], [3,4]]
grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
# makes [[1], [2], [3], [4]]
grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] +
[[sub] for sub in r[1]])
link = Link(inlet,
outlet,
interval=0.01,
groupers=[grouperA, grouperB])
link.transfer()
grouperA.assert_called_with([records])
callsA = [call([records[:2], records[2:]])] # expects [[1,2], [3,4]]
grouperB.assert_has_calls(callsA)
callsB = [
call([records[0]], mock.ANY),
call([records[1]], mock.ANY),
call([records[2]], mock.ANY),
call([records[3]], mock.ANY)
]
outlet._push.assert_has_calls(callsB) # expects [[1], [2], [3], [4]]
def xtest_non_iterable_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
tags='test_non_iterable_raised')
with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
link.transfer()
def test_processors_one(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processor = MagicMock(side_effect=lambda r: r)
link = Link(inlet, outlet, interval=0.01, processors=processor)
link.transfer()
processor.assert_called_with(records)
outlet._push.assert_called_with(records, mock.ANY)
def test_groupers_one_passive(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
grouper = MagicMock(side_effect=lambda r: r) # does nothing on purpose
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
link.transfer()
grouper.assert_called_with([records])
outlet._push.assert_called_with(records, mock.ANY)
def test_metadata_global(self):
inlet1 = DummyInlet(metadata={'secret': 'global'})
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(seconds=1))
link.transfer()
self.assertEqual(outlet1.records[0].metadata['secret'],
inlet1.metadata['secret'])
def test_update_single(self):
inlet1 = DummyInlet()
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(
seconds=1), copy_records=False)
link.transfer()
self.assertEqual(outlet1.records[0], inlet1.record)
def xtest_generic_error_caught(self, inlet1, inlet2, outlet1):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
link = Link([inlet1, inlet2], [outlet1],
timedelta(seconds=1),
name='test_generic_error_caught',
catch_exceptions=True)
results = asyncio.run(inlet2._pull(None))
link.transfer()
outlet1._push.assert_called_with(results, mock.ANY)
def test_transfer(self, inlet, outlet):
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_update')
link.transfer()
inlet._pull.assert_called()
outlet._push.assert_called()
def test_update_multiple(self):
inlet1 = DummyInlet()
inlet2 = DummyInlet()
outlet1 = DummyOutlet()
outlet2 = DummyOutlet()
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1), copy_records=False)
link.transfer()
self.assertEqual(outlet1.records, [inlet1.record, inlet2.record])
self.assertEqual(outlet2.records, [inlet1.record, inlet2.record])
def test_groupers_one_active(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
# makes [[1,2], [3,4]]
grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
link.transfer()
grouper.assert_called_with([records])
calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_processors_many(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(side_effect=lambda x: x)
processorB = MagicMock(side_effect=lambda x: x)
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with(records)
outlet._push.assert_called_with(records, mock.ANY)
def test_metadata_local(self):
inlet1 = DummyInlet(metadata={'secret': 'global'})
inlet1.record = inlet1.new_record(
{'test': 20}, metadata={'secret': 'local', 'key': 'value'})
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(seconds=1))
link.transfer()
self.assertEqual(
outlet1.records[0].metadata['secret'], inlet1.record.metadata['secret'])
self.assertEqual(
outlet1.records[0].metadata['key'], inlet1.record.metadata['key'])
self.assertNotEqual(
outlet1.records[0].metadata['secret'], inlet1.metadata['secret'])
def test_processors_filter_records(self, inlet, outlet):
records = [2, 3, 4]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=lambda r: list(filter(lambda y: y > 2, r)))
processorB = MagicMock(
side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with([3, 4])
outlet._push.assert_called_with([4], mock.ANY)
def test_processors_change_records(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=lambda r: list(map(lambda y: y * y, r)))
processorB = MagicMock(
side_effect=lambda r: list(map(lambda y: -y, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with([4, 9])
outlet._push.assert_called_with([-4, -9], mock.ANY)
def test_exception_caught(self, inlet, outlet):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_exception_caught',
ignore_exceptions=True)
try:
link.transfer()
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
def test_await_pull_multiple_sync(self):
inlet1 = DummyAwaitInlet(False)
inlet2 = DummyAwaitInlet(False)
inlet3 = DummyAwaitInlet(False)
outlet1 = DummyOutlet()
link = Link([inlet1, inlet2, inlet3], [outlet1],
timedelta(seconds=0.), copy_records=False)
start_time = datetime.now()
link.transfer()
end_time = datetime.now()
diff = end_time - start_time
self.assertEqual(outlet1.records, [
inlet1.record, inlet2.record, inlet3.record])
total_wait_time = inlet1.wait_time + inlet2.wait_time + inlet3.wait_time
self.assertGreaterEqual(diff.total_seconds(), total_wait_time)
def test_exception_caught(self, inlet, outlet):
inlet._pull.side_effect = DummyException('Test inlet exception')
outlet._push.side_effect = DummyException('Test outlet exception')
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_exception_caught',
ignore_exceptions=True)
try:
with self.assertLogs(logging.getLogger('databay.Link'),
level='WARNING') as cm:
link.transfer()
self.assertTrue('Test inlet exception' in ';'.join(cm.output))
self.assertTrue('Test outlet exception' in ';'.join(cm.output))
except Exception as e:
self.fail(f'Should not raise exception: {e}')
inlet._pull.assert_called()
outlet._push.assert_called()
def test_grouper_exception_ignored(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
grouperA = MagicMock(side_effect=DummyException('Grouper exception'))
grouperB = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
link = Link(inlet,
outlet,
interval=0.01,
groupers=[grouperA, grouperB],
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.transfer()
self.assertTrue('Grouper exception:' in ';'.join(cm.output),
cm.output)
grouperA.assert_called_with([records])
grouperB.assert_called_with([records])
calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_processor_exception_ignored(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=DummyException('Processor exception'))
processorB = MagicMock(
side_effect=lambda r: list(map(lambda y: y * y, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB],
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.transfer()
self.assertTrue('Processor exception:' in ';'.join(cm.output),
cm.output)
processorA.assert_called_with(records)
processorB.assert_called_with(records)
outlet._push.assert_called_with([4, 9], mock.ANY)