def test_buffer_time(self, inlet, outlet):
buffer = Buffer(time_threshold=0.02)
payload = [1, 2, 3, 4]
records = [Record(payload=p) for p in payload]
link = Link(inlet,
outlet,
interval=1,
processors=buffer,
copy_records=False)
inlet._pull = pull_mock(records[:2])
link.transfer()
outlet._push.assert_called_with(
[], mock.ANY) # not enough time have passed
inlet._pull = pull_mock(records[2:])
link.transfer()
outlet._push.assert_called_with(
[], mock.ANY) # not enough time have passed
time.sleep(0.02)
inlet._pull = pull_mock([])
link.transfer()
outlet._push.assert_called_with(
records, mock.ANY) # all records should be returned here
def test_flush(self, inlet, outlet):
buffer = Buffer(count_threshold=100, time_threshold=10)
payload = [1, 2, 3, 4]
records = [Record(payload=p) for p in payload]
link = Link(inlet,
outlet,
interval=1,
processors=buffer,
copy_records=False)
inlet._pull = pull_mock(records[:2])
link.transfer()
outlet._push.assert_called_with([], mock.ANY) # no records yet
inlet._pull = pull_mock(records[2:])
link.transfer()
outlet._push.assert_called_with([], mock.ANY) # no records yet
buffer.flush = True
inlet._pull = pull_mock([])
link.transfer()
outlet._push.assert_called_with(
records, mock.ANY) # all records should be flushed
def test_processors_one(self):
payload = [2, 3]
records = [Record(payload=p) for p in payload]
processor = MagicMock(side_effect=lambda r: r)
outlet = DummyOutlet(processors=processor)
asyncio.run(outlet._push(records, update_mock))
processor.assert_called_with(records)
def test_execute_count(self):
buffer = Buffer(count_threshold=10)
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
rvA = buffer(records[:6])
rvB = buffer(records[6:])
self.assertEqual(rvA, [], 'Should not contain any records yet')
self.assertEqual(rvB, records, 'Should contain all records')
def test_processors_many(self):
payload = [2, 3]
records = [Record(payload=p) for p in payload]
processorA = MagicMock(side_effect=lambda x: x)
processorB = MagicMock(side_effect=lambda x: x)
outlet = DummyOutlet(processors=[processorA, processorB])
asyncio.run(outlet._push(records, update_mock))
processorA.assert_called_with(records)
processorB.assert_called_with(records)
def test_flush(self):
buffer = Buffer(count_threshold=100, time_threshold=10)
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
buffer.flush = False
rvA = buffer(records[:6])
buffer.flush = True
rvB = buffer(records[6:])
self.assertEqual(rvA, [], 'Should not contain any records yet')
self.assertEqual(rvB, records, 'Should contain all records')
def test_count_controller(self):
buffer = Buffer(count_threshold=10)
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
result_false = buffer._count_controller(records[:6])
result_true = buffer._count_controller(records)
self.assertFalse(
result_false,
'Count controller should return False with 6 records')
self.assertTrue(result_true,
'Count controller should return True with 11 records')
def test_on_reset(self):
custom_controller = MagicMock()
on_reset = MagicMock()
buffer = Buffer(custom_controllers=custom_controller,
on_reset=on_reset)
payload = [1, 2]
records = [Record(payload=p) for p in payload]
buffer(records)
custom_controller.assert_called_with(records)
self.assertEqual(on_reset.call_count, 1)
buffer.reset()
self.assertEqual(on_reset.call_count, 2)
def test_execute_custom(self):
# checks if number 11 is in the any of the records' payload
custom_controller = lambda x: len(
list(filter(lambda v: v.payload == 11, x)))
buffer = Buffer(custom_controllers=custom_controller)
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
rvA = buffer(records[:6])
rvB = buffer(records[6:])
self.assertEqual(rvA, [], 'Should not contain any records yet')
self.assertEqual(rvB, records, 'Should contain all records')
def test_execute_disjoint(self):
custom_controller = MagicMock(side_effect=lambda x: False)
buffer = Buffer(count_threshold=1,
time_threshold=1,
custom_controllers=custom_controller)
buffer._count_controller = MagicMock(side_effect=lambda x: False)
buffer._time_controller = MagicMock(side_effect=lambda x: False)
payload = [1, 2, 3]
records = [Record(payload=p) for p in payload]
buffer._execute(records)
buffer._count_controller.assert_called_with(records)
buffer._time_controller.assert_called_with(records)
custom_controller.assert_called_with(records)
def test_time_controller(self):
buffer = Buffer(time_threshold=0.01)
payload = [1, 2, 3]
records = [Record(payload=p) for p in payload]
result_false = buffer._time_controller(records)
time.sleep(0.01)
result_true = buffer._time_controller(records)
self.assertFalse(
result_false,
'Time controller should return False after less than 0.01 seconds')
self.assertTrue(
result_true,
'Time controller should return True after sleeping for 0.01 seconds'
)
def test_execute_count_before_time(self):
buffer = Buffer(count_threshold=10, time_threshold=1)
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
rv = []
for r in records:
rv = buffer([r])
if rv != []:
break
self.assertEqual(rv, records, 'Should contain all records')
self.assertEqual(buffer.records, [],
'Buffer.records should be reset to []')
self.assertIsNone(buffer.time_start,
'Buffer.time_start should be reset to None')
def test_execute_conjoint_true(self):
custom_controller = MagicMock(side_effect=lambda x: True)
buffer = Buffer(count_threshold=1,
time_threshold=1,
custom_controllers=custom_controller,
conjugate_controllers=True)
buffer._count_controller = MagicMock(side_effect=lambda x: True)
buffer._time_controller = MagicMock(side_effect=lambda x: True)
payload = [1, 2, 3]
records = [Record(payload=p) for p in payload]
result = buffer._execute(records)
buffer._count_controller.assert_called_with(records)
buffer._time_controller.assert_called_with(records)
custom_controller.assert_called_with(records)
self.assertEqual(result, records)
def test_custom_exception(self):
custom_controller = MagicMock(
side_effect=DummyException('Custom controller exception!'))
payload = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
records = [Record(payload=p) for p in payload]
buffer = Buffer(count_threshold=10,
custom_controllers=custom_controller)
with self.assertLogs(logging.getLogger('databay.Buffer'),
level='DEBUG') as cm:
rvA = buffer(records[:6])
rvB = buffer(records[6:])
self.assertTrue(
'Custom controller exception!' in ';'.join(cm.output))
self.assertEqual(rvA, [], 'Should not contain any records yet')
self.assertEqual(rvB, records, 'Should contain all records')
def new_record(self, payload, metadata: dict = None) -> Record:
"""
Create a new :any:`Record`. This should be the preferred way of creating new records.
:type payload: Any
:param payload: Data produced by this inlet.
:type metadata: dict
:param metadata: Local metadata that will override and/or append to the global metadata. It will be attached to the new record.
|default| :code:`None`
:returns: New record created
:rtype: :any:`Record`
"""
full_metadata = {
**self._metadata,
**(metadata if metadata is not None else {})
}
full_metadata['__inlet__'] = str(self)
return Record(payload=payload, metadata=full_metadata)
def test_buffer_count(self, inlet, outlet):
buffer = Buffer(count_threshold=3)
payload = [1, 2, 3, 4]
records = [Record(payload=p) for p in payload]
link = Link(inlet,
outlet,
interval=1,
processors=buffer,
copy_records=False)
inlet._pull = pull_mock(records[:2])
link.transfer()
outlet._push.assert_called_with(
[], mock.ANY) # after first call we shouldn't have any records
inlet._pull = pull_mock(records[2:])
link.transfer()
outlet._push.assert_called_with(
records, mock.ANY) # all records should be returned here
def test_push_async(self):
outlet = DummyAsyncOutlet()
records = [Record(None), Record(None)]
asyncio.run(outlet._push(records, None))
self.assertEqual(outlet.records, records)
def test_push(self):
outlet = DummyOutlet()
records = [Record(None), Record(None)]
asyncio.run(outlet._push(records, update_mock))
self.assertEqual(outlet.records, records)
async def pull_coro(_):
counter_dict['counter'] += 1
record = Record(payload=counter_dict['counter'])
counter_dict['records'].append(record)
return [record]