def test_groupers_many_active(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
# makes [[1,2], [3,4]]
grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
# makes [[1], [2], [3], [4]]
grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] +
[[sub] for sub in r[1]])
link = Link(inlet,
outlet,
interval=0.01,
groupers=[grouperA, grouperB])
link.transfer()
grouperA.assert_called_with([records])
callsA = [call([records[:2], records[2:]])] # expects [[1,2], [3,4]]
grouperB.assert_has_calls(callsA)
callsB = [
call([records[0]], mock.ANY),
call([records[1]], mock.ANY),
call([records[2]], mock.ANY),
call([records[3]], mock.ANY)
]
outlet._push.assert_has_calls(callsB) # expects [[1], [2], [3], [4]]
def test_add_inlet_same(self, inlet1):
link = Link([], [], timedelta(seconds=1), name='test_add_inlet_same')
link.add_inlets(inlet1)
self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)
self.assertEqual(link.inlets, [inlet1])
def test_add_outlet_same(self, outlet1):
link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same')
link.add_outlets(outlet1)
self.assertRaises(InvalidNodeError, link.add_outlets, outlet1)
self.assertEqual(link.outlets, [outlet1])
def xtest_non_iterable_raised(self, inlet1):
logging.getLogger('databay.Link').setLevel(logging.ERROR)
link = Link([inlet1], [],
timedelta(seconds=1),
tags='test_non_iterable_raised')
with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'):
link.transfer()
def test_add_inlet_multiple(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
name='test_add_inlet_multiple')
link.add_inlets([inlet1, inlet2])
self.assertEqual(link.inlets, [inlet1, inlet2])
def test_add_inlet_same(self):
inlet1 = DummyInlet()
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same')
link.add_inlets(inlet1)
self.assertRaises(InvalidNodeError, link.add_inlets, inlet1)
self.assertEqual(link.inlets, [inlet1])
def test_processors_one(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processor = MagicMock(side_effect=lambda r: r)
link = Link(inlet, outlet, interval=0.01, processors=processor)
link.transfer()
processor.assert_called_with(records)
outlet._push.assert_called_with(records, mock.ANY)
def test_groupers_one_passive(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
grouper = MagicMock(side_effect=lambda r: r) # does nothing on purpose
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
link.transfer()
grouper.assert_called_with([records])
outlet._push.assert_called_with(records, mock.ANY)
def test_add_outlet_multiple(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_add_outlet_multiple')
link.add_outlets([outlet1, outlet2])
self.assertEqual(link.outlets, [outlet1, outlet2])
def test_remove_inlet_invalid(self, inlet1, inlet2):
link = Link([], [],
timedelta(seconds=1),
name='test_remove_inlet_invalid')
link.add_inlets([inlet1])
self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2)
self.assertEqual(link.inlets, [inlet1])
def xtest_generic_error_caught(self, inlet1, inlet2, outlet1):
logging.getLogger('databay.Link').setLevel(logging.CRITICAL)
link = Link([inlet1, inlet2], [outlet1],
timedelta(seconds=1),
name='test_generic_error_caught',
catch_exceptions=True)
results = asyncio.run(inlet2._pull(None))
link.transfer()
outlet1._push.assert_called_with(results, mock.ANY)
def test_update_single(self):
inlet1 = DummyInlet()
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(
seconds=1), copy_records=False)
link.transfer()
self.assertEqual(outlet1.records[0], inlet1.record)
def test_transfer(self, inlet, outlet):
link = Link([inlet], [outlet],
timedelta(seconds=1),
tags='test_update')
link.transfer()
inlet._pull.assert_called()
outlet._push.assert_called()
def test_add_multiple_inlets_same(self):
inlet1 = DummyInlet()
inlet2 = DummyInlet()
link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same')
link.add_inlets(inlet1)
link.add_inlets(inlet2)
self.assertEqual(link.inlets, [inlet1, inlet2])
def test_metadata_global(self):
inlet1 = DummyInlet(metadata={'secret': 'global'})
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(seconds=1))
link.transfer()
self.assertEqual(outlet1.records[0].metadata['secret'],
inlet1.metadata['secret'])
def test_remove_outlet_invalid(self, outlet1, outlet2):
link = Link([], [],
timedelta(seconds=1),
tags='test_remove_outlet_invalid')
link.add_outlets([outlet1])
self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2)
self.assertEqual(link.outlets, [outlet1])
def test_on_start(self):
inlet1 = DummyInlet()
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(seconds=1))
self.assertFalse(inlet1.active)
self.assertFalse(outlet1.active)
link.on_start()
self.assertTrue(inlet1.active)
self.assertTrue(outlet1.active)
def test_on_shutdown_already_inactive(self):
inlet1 = DummyInlet()
outlet1 = DummyOutlet()
inlet1.on_shutdown = mock.Mock()
outlet1.on_shutdown = mock.Mock()
link = Link([inlet1], [outlet1], timedelta(seconds=1))
link.on_shutdown()
inlet1.on_shutdown.assert_not_called()
outlet1.on_shutdown.assert_not_called()
def test_on_start_already_active(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
name='test_on_start_already_active')
link.on_start()
inlet1.on_start.assert_not_called()
outlet1.on_start.assert_not_called()
def test_on_shutdown_already_inactive(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=False)
type(outlet1).active = mock.PropertyMock(return_value=False)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown_already_inactive')
link.on_shutdown()
inlet1.on_shutdown.assert_not_called()
outlet1.on_shutdown.assert_not_called()
def test_on_shutdown(self, inlet1, outlet1):
type(inlet1).active = mock.PropertyMock(return_value=True)
type(outlet1).active = mock.PropertyMock(return_value=True)
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_shutdown')
link.on_shutdown()
inlet1.try_shutdown.assert_called()
outlet1.try_shutdown.assert_called()
def test_groupers_one_active(self, inlet, outlet):
records = [1, 2, 3, 4]
inlet._pull = pull_mock(records)
# makes [[1,2], [3,4]]
grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]])
link = Link(inlet, outlet, interval=0.01, groupers=grouper)
link.transfer()
grouper.assert_called_with([records])
calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)]
outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_update_multiple(self):
inlet1 = DummyInlet()
inlet2 = DummyInlet()
outlet1 = DummyOutlet()
outlet2 = DummyOutlet()
link = Link([inlet1, inlet2], [outlet1, outlet2],
timedelta(seconds=1), copy_records=False)
link.transfer()
self.assertEqual(outlet1.records, [inlet1.record, inlet2.record])
self.assertEqual(outlet2.records, [inlet1.record, inlet2.record])
def test_processors_many(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(side_effect=lambda x: x)
processorB = MagicMock(side_effect=lambda x: x)
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with(records)
outlet._push.assert_called_with(records, mock.ANY)
def test_on_shutdown(self):
inlet1 = DummyInlet()
outlet1 = DummyOutlet()
inlet1._active = True
outlet1._active = True
link = Link(inlet1, outlet1, timedelta(seconds=1))
self.assertTrue(inlet1.active)
self.assertTrue(outlet1.active)
link.on_shutdown()
self.assertFalse(inlet1.active)
self.assertFalse(outlet1.active)
def test_on_start_already_active(self):
inlet1 = DummyInlet()
outlet1 = DummyOutlet()
inlet1.on_start = mock.Mock()
outlet1.on_start = mock.Mock()
inlet1._active = True
outlet1._active = True
link = Link([inlet1], [outlet1], timedelta(seconds=1))
link.on_start()
inlet1.on_start.assert_not_called()
outlet1.on_start.assert_not_called()
def test_on_start_inlet_exception_catch(self, inlet1, outlet1):
inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())')
link = Link([inlet1], [outlet1],
timedelta(seconds=1),
tags='test_on_start',
ignore_exceptions=True)
with self.assertLogs(logging.getLogger('databay.Link'),
level='ERROR') as cm:
link.on_start()
self.assertTrue('on_start inlet exception: "" for inlet:' in
';'.join(cm.output))
inlet1.try_start.assert_called()
outlet1.try_start.assert_called()
def test_processors_change_records(self, inlet, outlet):
records = [2, 3]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=lambda r: list(map(lambda y: y * y, r)))
processorB = MagicMock(
side_effect=lambda r: list(map(lambda y: -y, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with([4, 9])
outlet._push.assert_called_with([-4, -9], mock.ANY)
def test_processors_filter_records(self, inlet, outlet):
records = [2, 3, 4]
inlet._pull = pull_mock(records)
processorA = MagicMock(
side_effect=lambda r: list(filter(lambda y: y > 2, r)))
processorB = MagicMock(
side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r)))
link = Link(inlet,
outlet,
interval=0.01,
processors=[processorA, processorB])
link.transfer()
processorA.assert_called_with(records)
processorB.assert_called_with([3, 4])
outlet._push.assert_called_with([4], mock.ANY)
def test_metadata_local(self):
inlet1 = DummyInlet(metadata={'secret': 'global'})
inlet1.record = inlet1.new_record(
{'test': 20}, metadata={'secret': 'local', 'key': 'value'})
outlet1 = DummyOutlet()
link = Link([inlet1], [outlet1], timedelta(seconds=1))
link.transfer()
self.assertEqual(
outlet1.records[0].metadata['secret'], inlet1.record.metadata['secret'])
self.assertEqual(
outlet1.records[0].metadata['key'], inlet1.record.metadata['key'])
self.assertNotEqual(
outlet1.records[0].metadata['secret'], inlet1.metadata['secret'])
