def test_groupers_many_active(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) # makes [[1,2], [3,4]] grouperA = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) # makes [[1], [2], [3], [4]] grouperB = MagicMock(side_effect=lambda r: [[sub] for sub in r[0]] + [[sub] for sub in r[1]]) link = Link(inlet, outlet, interval=0.01, groupers=[grouperA, grouperB]) link.transfer() grouperA.assert_called_with([records]) callsA = [call([records[:2], records[2:]])] # expects [[1,2], [3,4]] grouperB.assert_has_calls(callsA) callsB = [ call([records[0]], mock.ANY), call([records[1]], mock.ANY), call([records[2]], mock.ANY), call([records[3]], mock.ANY) ] outlet._push.assert_has_calls(callsB) # expects [[1], [2], [3], [4]]
def xtest_non_iterable_raised(self, inlet1): logging.getLogger('databay.Link').setLevel(logging.ERROR) link = Link([inlet1], [], timedelta(seconds=1), tags='test_non_iterable_raised') with self.assertRaisesRegex(TypeError, 'Inlets must return iterable'): link.transfer()
def test_processors_one(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processor = MagicMock(side_effect=lambda r: r) link = Link(inlet, outlet, interval=0.01, processors=processor) link.transfer() processor.assert_called_with(records) outlet._push.assert_called_with(records, mock.ANY)
def test_groupers_one_passive(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) grouper = MagicMock(side_effect=lambda r: r) # does nothing on purpose link = Link(inlet, outlet, interval=0.01, groupers=grouper) link.transfer() grouper.assert_called_with([records]) outlet._push.assert_called_with(records, mock.ANY)
def test_metadata_global(self): inlet1 = DummyInlet(metadata={'secret': 'global'}) outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta(seconds=1)) link.transfer() self.assertEqual(outlet1.records[0].metadata['secret'], inlet1.metadata['secret'])
def test_update_single(self): inlet1 = DummyInlet() outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta( seconds=1), copy_records=False) link.transfer() self.assertEqual(outlet1.records[0], inlet1.record)
def xtest_generic_error_caught(self, inlet1, inlet2, outlet1): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) link = Link([inlet1, inlet2], [outlet1], timedelta(seconds=1), name='test_generic_error_caught', catch_exceptions=True) results = asyncio.run(inlet2._pull(None)) link.transfer() outlet1._push.assert_called_with(results, mock.ANY)
def test_transfer(self, inlet, outlet): link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_update') link.transfer() inlet._pull.assert_called() outlet._push.assert_called()
def test_update_multiple(self): inlet1 = DummyInlet() inlet2 = DummyInlet() outlet1 = DummyOutlet() outlet2 = DummyOutlet() link = Link([inlet1, inlet2], [outlet1, outlet2], timedelta(seconds=1), copy_records=False) link.transfer() self.assertEqual(outlet1.records, [inlet1.record, inlet2.record]) self.assertEqual(outlet2.records, [inlet1.record, inlet2.record])
def test_groupers_one_active(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) # makes [[1,2], [3,4]] grouper = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) link = Link(inlet, outlet, interval=0.01, groupers=grouper) link.transfer() grouper.assert_called_with([records]) calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)] outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_processors_many(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock(side_effect=lambda x: x) processorB = MagicMock(side_effect=lambda x: x) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with(records) outlet._push.assert_called_with(records, mock.ANY)
def test_metadata_local(self): inlet1 = DummyInlet(metadata={'secret': 'global'}) inlet1.record = inlet1.new_record( {'test': 20}, metadata={'secret': 'local', 'key': 'value'}) outlet1 = DummyOutlet() link = Link([inlet1], [outlet1], timedelta(seconds=1)) link.transfer() self.assertEqual( outlet1.records[0].metadata['secret'], inlet1.record.metadata['secret']) self.assertEqual( outlet1.records[0].metadata['key'], inlet1.record.metadata['key']) self.assertNotEqual( outlet1.records[0].metadata['secret'], inlet1.metadata['secret'])
def test_processors_filter_records(self, inlet, outlet): records = [2, 3, 4] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=lambda r: list(filter(lambda y: y > 2, r))) processorB = MagicMock( side_effect=lambda r: list(filter(lambda y: y % 2 == 0, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with([3, 4]) outlet._push.assert_called_with([4], mock.ANY)
def test_processors_change_records(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=lambda r: list(map(lambda y: y * y, r))) processorB = MagicMock( side_effect=lambda r: list(map(lambda y: -y, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB]) link.transfer() processorA.assert_called_with(records) processorB.assert_called_with([4, 9]) outlet._push.assert_called_with([-4, -9], mock.ANY)
def test_exception_caught(self, inlet, outlet): logging.getLogger('databay.Link').setLevel(logging.CRITICAL) inlet._pull.side_effect = DummyException('Test inlet exception') outlet._push.side_effect = DummyException('Test outlet exception') link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_exception_caught', ignore_exceptions=True) try: link.transfer() except Exception as e: self.fail(f'Should not raise exception: {e}') inlet._pull.assert_called() outlet._push.assert_called()
def test_await_pull_multiple_sync(self): inlet1 = DummyAwaitInlet(False) inlet2 = DummyAwaitInlet(False) inlet3 = DummyAwaitInlet(False) outlet1 = DummyOutlet() link = Link([inlet1, inlet2, inlet3], [outlet1], timedelta(seconds=0.), copy_records=False) start_time = datetime.now() link.transfer() end_time = datetime.now() diff = end_time - start_time self.assertEqual(outlet1.records, [ inlet1.record, inlet2.record, inlet3.record]) total_wait_time = inlet1.wait_time + inlet2.wait_time + inlet3.wait_time self.assertGreaterEqual(diff.total_seconds(), total_wait_time)
def test_exception_caught(self, inlet, outlet): inlet._pull.side_effect = DummyException('Test inlet exception') outlet._push.side_effect = DummyException('Test outlet exception') link = Link([inlet], [outlet], timedelta(seconds=1), tags='test_exception_caught', ignore_exceptions=True) try: with self.assertLogs(logging.getLogger('databay.Link'), level='WARNING') as cm: link.transfer() self.assertTrue('Test inlet exception' in ';'.join(cm.output)) self.assertTrue('Test outlet exception' in ';'.join(cm.output)) except Exception as e: self.fail(f'Should not raise exception: {e}') inlet._pull.assert_called() outlet._push.assert_called()
def test_grouper_exception_ignored(self, inlet, outlet): records = [1, 2, 3, 4] inlet._pull = pull_mock(records) grouperA = MagicMock(side_effect=DummyException('Grouper exception')) grouperB = MagicMock(side_effect=lambda r: [r[0][:2], r[0][2:]]) link = Link(inlet, outlet, interval=0.01, groupers=[grouperA, grouperB], ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.transfer() self.assertTrue('Grouper exception:' in ';'.join(cm.output), cm.output) grouperA.assert_called_with([records]) grouperB.assert_called_with([records]) calls = [call(records[:2], mock.ANY), call(records[2:], mock.ANY)] outlet._push.assert_has_calls(calls) # expects [[1,2], [3,4]]
def test_processor_exception_ignored(self, inlet, outlet): records = [2, 3] inlet._pull = pull_mock(records) processorA = MagicMock( side_effect=DummyException('Processor exception')) processorB = MagicMock( side_effect=lambda r: list(map(lambda y: y * y, r))) link = Link(inlet, outlet, interval=0.01, processors=[processorA, processorB], ignore_exceptions=True) with self.assertLogs(logging.getLogger('databay.Link'), level='ERROR') as cm: link.transfer() self.assertTrue('Processor exception:' in ';'.join(cm.output), cm.output) processorA.assert_called_with(records) processorB.assert_called_with(records) outlet._push.assert_called_with([4, 9], mock.ANY)