def test_groupers_many_active(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) # makes [[1,2], [3,4]] grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) # makes [[1], [2], [3], [4]] grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] + [[sub] for sub in r[1]]) link = Link(inlet, outlet, interval=0.01, groupers=[grouperA, grouperB]) link.transfer() grouperA.assert_called_with([records]) callsA = [call([records[:2], records[2:]])] # expects [[1,2], [3,4]] grouperB.assert_has_calls(callsA) callsB = [ call([records[0]], mock.ANY), call([records[1]], mock.ANY), call([records[2]], mock.ANY), call([records[3]], mock.ANY) ] outlet._push.assert_has_calls(callsB) # expects [[1], [2], [3], [4]]
def test_add_inlet_same(self, inlet1): link = Link([], [], timedelta(seconds=1), name='test_add_inlet_same') link.add_inlets(inlet1) self.assertRaises(InvalidNodeError, link.add_inlets, inlet1) self.assertEqual(link.inlets, [inlet1])
def test_add_outlet_same(self, outlet1): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_same') link.add_outlets(outlet1) self.assertRaises(InvalidNodeError, link.add_outlets, outlet1) self.assertEqual(link.outlets, [outlet1])
def xtest_non_iterable_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), tags='test_non_iterable_raised') with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'): link.transfer()
def test_add_inlet_multiple(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), name='test_add_inlet_multiple') link.add_inlets([inlet1, inlet2]) self.assertEqual(link.inlets, [inlet1, inlet2])
def test_add_inlet_same(self): inlet1 = DummyInlet() link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same') link.add_inlets(inlet1) self.assertRaises(InvalidNodeError, link.add_inlets, inlet1) self.assertEqual(link.inlets, [inlet1])
def test_processors_one(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processor = MagicMock(side_effect=lambda r: r) link = Link(inlet, outlet, interval=0.01, processors=processor) link.transfer() processor.assert_called_with(records) outlet._push.assert_called_with(records, mock.ANY)
def test_groupers_one_passive(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) grouper = MagicMock(side_effect=lambda r: r) # does nothing on purpose link = Link(inlet, outlet, interval=0.01, groupers=grouper) link.transfer() grouper.assert_called_with([records]) outlet._push.assert_called_with(records, mock.ANY)
def test_add_outlet_multiple(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_add_outlet_multiple') link.add_outlets([outlet1, outlet2]) self.assertEqual(link.outlets, [outlet1, outlet2])
def test_remove_inlet_invalid(self, inlet1, inlet2): link = Link([], [], timedelta(seconds=1), name='test_remove_inlet_invalid') link.add_inlets([inlet1]) self.assertRaises(InvalidNodeError, link.remove_inlets, inlet2) self.assertEqual(link.inlets, [inlet1])
def xtest_generic_error_caught(self, inlet1, inlet2, outlet1): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) link = Link([inlet1, inlet2], [outlet1], timedelta(seconds=1), name='test_generic_error_caught', catch_exceptions=True) results = asyncio.run(inlet2._pull(None)) link.transfer() outlet1._push.assert_called_with(results, mock.ANY)
def test_update_single(self): inlet1 = DummyInlet() outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta( seconds=1), copy_records=False) link.transfer() self.assertEqual(outlet1.records[0], inlet1.record)
def test_transfer(self, inlet, outlet): link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_update') link.transfer() inlet._pull.assert_called() outlet._push.assert_called()
def test_add_multiple_inlets_same(self): inlet1 = DummyInlet() inlet2 = DummyInlet() link = Link([], [], timedelta(seconds=1), tags='test_add_inlet_same') link.add_inlets(inlet1) link.add_inlets(inlet2) self.assertEqual(link.inlets, [inlet1, inlet2])
def test_metadata_global(self): inlet1 = DummyInlet(metadata={'secret': 'global'}) outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta(seconds=1)) link.transfer() self.assertEqual(outlet1.records[0].metadata['secret'], inlet1.metadata['secret'])
def test_remove_outlet_invalid(self, outlet1, outlet2): link = Link([], [], timedelta(seconds=1), tags='test_remove_outlet_invalid') link.add_outlets([outlet1]) self.assertRaises(InvalidNodeError, link.remove_outlets, outlet2) self.assertEqual(link.outlets, [outlet1])
def test_on_start(self): inlet1 = DummyInlet() outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta(seconds=1)) self.assertFalse(inlet1.active) self.assertFalse(outlet1.active) link.on_start() self.assertTrue(inlet1.active) self.assertTrue(outlet1.active)
def test_on_shutdown_already_inactive(self): inlet1 = DummyInlet() outlet1 = DummyOutlet() inlet1.on_shutdown = mock.Mock() outlet1.on_shutdown = mock.Mock() link = Link([inlet1], [outlet1], timedelta(seconds=1)) link.on_shutdown() inlet1.on_shutdown.assert_not_called() outlet1.on_shutdown.assert_not_called()
def test_on_start_already_active(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), name='test_on_start_already_active') link.on_start() inlet1.on_start.assert_not_called() outlet1.on_start.assert_not_called()
def test_on_shutdown_already_inactive(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=False) type(outlet1).active = mock.PropertyMock(return_value=False) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown_already_inactive') link.on_shutdown() inlet1.on_shutdown.assert_not_called() outlet1.on_shutdown.assert_not_called()
def test_on_shutdown(self, inlet1, outlet1): type(inlet1).active = mock.PropertyMock(return_value=True) type(outlet1).active = mock.PropertyMock(return_value=True) link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_shutdown') link.on_shutdown() inlet1.try_shutdown.assert_called() outlet1.try_shutdown.assert_called()
def test_groupers_one_active(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) # makes [[1,2], [3,4]] grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) link = Link(inlet, outlet, interval=0.01, groupers=grouper) link.transfer() grouper.assert_called_with([records]) calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)] outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_update_multiple(self): inlet1 = DummyInlet() inlet2 = DummyInlet() outlet1 = DummyOutlet() outlet2 = DummyOutlet() link = Link([inlet1, inlet2], [outlet1, outlet2], timedelta(seconds=1), copy_records=False) link.transfer() self.assertEqual(outlet1.records, [inlet1.record, inlet2.record]) self.assertEqual(outlet2.records, [inlet1.record, inlet2.record])
def test_processors_many(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock(side_effect=lambda x: x) processorB = MagicMock(side_effect=lambda x: x) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with(records) outlet._push.assert_called_with(records, mock.ANY)
def test_on_shutdown(self): inlet1 = DummyInlet() outlet1 = DummyOutlet() inlet1._active = True outlet1._active = True link = Link(inlet1, outlet1, timedelta(seconds=1)) self.assertTrue(inlet1.active) self.assertTrue(outlet1.active) link.on_shutdown() self.assertFalse(inlet1.active) self.assertFalse(outlet1.active)
def test_on_start_already_active(self): inlet1 = DummyInlet() outlet1 = DummyOutlet() inlet1.on_start = mock.Mock() outlet1.on_start = mock.Mock() inlet1._active = True outlet1._active = True link = Link([inlet1], [outlet1], timedelta(seconds=1)) link.on_start() inlet1.on_start.assert_not_called() outlet1.on_start.assert_not_called()
def test_on_start_inlet_exception_catch(self, inlet1, outlet1): inlet1.try_start.side_effect = lambda: exec('raise(RuntimeError())') link = Link([inlet1], [outlet1], timedelta(seconds=1), tags='test_on_start', ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.on_start() self.assertTrue('on_start inlet exception: "" for inlet:' in ';'.join(cm.output)) inlet1.try_start.assert_called() outlet1.try_start.assert_called()
def test_processors_change_records(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=lambda r: list(map(lambda y: y * y, r))) processorB = MagicMock( side_effect=lambda r: list(map(lambda y: -y, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with([4, 9]) outlet._push.assert_called_with([-4, -9], mock.ANY)
def test_processors_filter_records(self, inlet, outlet): records = [2, 3, 4] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=lambda r: list(filter(lambda y: y > 2, r))) processorB = MagicMock( side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with([3, 4]) outlet._push.assert_called_with([4], mock.ANY)
def test_metadata_local(self): inlet1 = DummyInlet(metadata={'secret': 'global'}) inlet1.record = inlet1.new_record( {'test': 20}, metadata={'secret': 'local', 'key': 'value'}) outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta(seconds=1)) link.transfer() self.assertEqual( outlet1.records[0].metadata['secret'], inlet1.record.metadata['secret']) self.assertEqual( outlet1.records[0].metadata['key'], inlet1.record.metadata['key']) self.assertNotEqual( outlet1.records[0].metadata['secret'], inlet1.metadata['secret'])