def test_flush_when_segment_empty(self):
current_segment_id = 123
backend = Mock()
log = Log(current_segment_id, backend)
log.flush()
self.assertEqual(log.get_current_segment_id(), current_segment_id)
backend.put_segment.assert_not_called()
backend.flush.assert_called_once_with()
def test_written_blocks_should_be_in_s3_after_flush(self):
current_segment_id = 123
block_size = 64
blocks_per_segment = 8
address = BlockAddress(current_segment_id, 0)
bucket = S3Bucket(self.BUCKET_NAME)
expected_block_bytes = block_size * b'a'
# Write a single block and flush
with AsyncWriter(bucket, 8, 4) as async_writer:
with DiskCache(async_writer, 32) as disk_cache:
memory_cache = MemoryCache(disk_cache, 8)
log = Log(current_segment_id,
memory_cache,
block_size=block_size,
blocks_per_segment=blocks_per_segment)
log.write_data_block(expected_block_bytes)
log.flush()
# Attempt read from a fresh cache
with AsyncWriter(bucket, 8, 4) as async_writer:
with DiskCache(async_writer, 32) as disk_cache:
memory_cache = MemoryCache(disk_cache, 8)
log = Log(current_segment_id + 1,
memory_cache,
block_size=block_size,
blocks_per_segment=blocks_per_segment)
result = log.read_block(address)
self.assertEqual(result, expected_block_bytes)
def test_write_inode_should_write_to_the_current_segment(self):
current_segment_id = 123
backend = Mock()
log = Log(current_segment_id, backend)
bytes = b'abc'
block_address = log.write_inode(bytes, 123)
self.assertEqual(block_address.segmentid, current_segment_id)
self.assertEqual(log.get_current_segment_id(), current_segment_id)
backend.put_segment.assert_not_called()
def test_read_block_from_a_previous_segment_should_return_the_block(self):
block_size = 64
address = BlockAddress(123, 1)
block_bytes = b'abc'
segment = ReadWriteSegment(123, block_size=block_size)
segment.write_data(block_size * b'x')
segment.write_data(block_bytes)
backend = Mock()
backend.get_segment.return_value = segment.to_bytes()
log = Log(999, backend, block_size=block_size)
result = log.read_block(address)
self.assertEqual(bytes(result[:len(block_bytes)]), block_bytes)
backend.get_segment.assert_called_once_with(address.segmentid)
def test_write_inode_when_segment_is_full_should_put_the_segment(self):
current_segment_id = 123
block_size = 128
blocks_per_segment = 64
block_bytes = block_size * b'x'
backend = Mock()
log = Log(current_segment_id,
backend,
block_size=block_size,
blocks_per_segment=blocks_per_segment)
for block_number in range(blocks_per_segment):
block_address = log.write_inode(block_bytes, 123)
self.assertEqual(block_address.offset, block_number)
# test that the current segment id is the starting id + 1
self.assertEqual(log.get_current_segment_id(), current_segment_id + 1)
backend.put_segment.assert_called_once_with(current_segment_id, ANY)
def test_read_block_from_current_segment_should_return_the_block(self):
block_size = 64
address = BlockAddress(123, 2)
block_bytes = block_size * b'a'
backend = Mock()
log = Log(address.segmentid, backend, block_size=block_size)
for _ in range(address.offset):
log.write_data_block(block_size * b'x')
log.write_data_block(block_bytes)
result = log.read_block(address)
self.assertEqual(bytes(result), block_bytes)
backend.get_segment.assert_not_called()
def test_write_and_read_to_s3(self):
current_segment_id = 123
block_size = 64
blocks_per_segment = 8
address = BlockAddress(current_segment_id, blocks_per_segment // 2)
bucket = S3Bucket(self.BUCKET_NAME)
expected_block_bytes = block_size * b'a'
# First, write enough blocks to put a segment to S3
with AsyncWriter(bucket, 8, 4) as async_writer:
with DiskCache(async_writer, 32) as disk_cache:
memory_cache = MemoryCache(disk_cache, 8)
log = Log(current_segment_id,
memory_cache,
block_size=block_size,
blocks_per_segment=blocks_per_segment)
for i in range(blocks_per_segment):
if i == address.offset:
block_bytes = expected_block_bytes
else:
block_bytes = block_size * b'x'
log.write_data_block(block_bytes)
result = log.read_block(address) # Should be read from cache
self.assertEqual(result, expected_block_bytes)
# Attempt read from a fresh cache
with AsyncWriter(bucket, 8, 4) as async_writer:
with DiskCache(async_writer, 32) as disk_cache:
memory_cache = MemoryCache(disk_cache, 8)
log = Log(current_segment_id + 1,
memory_cache,
block_size=block_size,
blocks_per_segment=blocks_per_segment)
result = log.read_block(address)
self.assertEqual(result, expected_block_bytes)
def test_flush(self):
current_segment_id = 123
block_size = 128
blocks_per_segment = 8
backend = Mock()
log = Log(current_segment_id,
backend,
block_size=block_size,
blocks_per_segment=blocks_per_segment)
block = block_size * b'a'
for i in range(blocks_per_segment):
block_address = log.write_data_block(block)
log.flush()
self.assertEqual(log.get_current_segment_id(), current_segment_id + 1)
backend.put_segment.assert_called_once_with(current_segment_id, ANY)
backend.flush.assert_called_once_with()