def test_multiple_batches__old_records__by_rows():
stream_oldest = CatStream(100, version=0)
stream_middle_aged = CatStream(100, version=5)
stream_latest = CatStream(100, version=10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'],
max_rows=20)
assert len(singer_stream.peek_buffer()) == 0
while not singer_stream.buffer_full:
singer_stream.add_record_message(
stream_oldest.generate_record_message())
assert len(singer_stream.peek_buffer()) == 20
singer_stream.flush_buffer()
assert len(singer_stream.peek_buffer()) == 0
singer_stream.add_record_message(stream_latest.generate_record_message())
assert len(singer_stream.peek_buffer()) == 1
reasonable_cutoff = 1000
while not singer_stream.buffer_full and reasonable_cutoff != 0:
singer_stream.add_record_message(
stream_middle_aged.generate_record_message())
reasonable_cutoff -= 1
assert reasonable_cutoff == 0
assert len(singer_stream.peek_buffer()) == 1
assert [] == missing_sdc_properties(singer_stream)
def test_multiple_batches__by_memory():
stream = CatStream(100)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'],
max_buffer_size=1024)
assert len(singer_stream.peek_buffer()) == 0
while not singer_stream.buffer_full:
singer_stream.add_record_message(stream.generate_record_message())
assert len(singer_stream.peek_buffer()) == 1
assert [] == missing_sdc_properties(singer_stream)
singer_stream.flush_buffer()
assert len(singer_stream.peek_buffer()) == 0