def test_resolve_mispositioned_record_with_positioned_disk_control(
_mocked_exists, mocked_data_file, _mocked_index_file, header_index,
position, disk_control_position):
"""Check different combinations of headers sequences with non empty disk_control.
TODO(karapuz): add test to this test case.
:param int header_index:
:param int position:
:param int disk_control_position:
"""
mocked_data_file.return_value.read_disk_control.return_value = make_header(
position=disk_control_position)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
blob_repairer._index_headers = [
make_header(disk_size=5, position=3),
make_header(position=4),
]
assert not blob_repairer.resolve_mispositioned_record(
header_index, position, [])
def test_resolve_mispositioned_record_with_empty_disk_control(
_mocked_exists, mocked_data_file, _mocked_index_file, header_index,
position, headers, expect_result):
"""Check different combinations of headers sequences.
Check different combinations of headers sequences with
BlobRepairer.resolve_mispositioned_record and empty disk control.
TODO(karapuz): not all branches are tested within:
eblob_kit.BlobRepairer.resolve_mispositioned_record
:param int header_index:
:param int position:
:param List[eblob_kit.DiskControl] headers:
:param bool expect_result:
"""
mocked_data_file.return_value.read_disk_control.return_value = make_header(
)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
blob_repairer._index_headers = headers
assert blob_repairer.resolve_mispositioned_record(header_index, position,
[]) == expect_result
def test_check_hole(_mock_exist, data_mock, _index_mock, mocker, position, end,
header_position, holes, holes_size, check_header_called):
"""Check generated DiskControl header with BlobRepairer.check_hole.
TODO(karapuz): try to move common for all test patch code to BlobRepairer fixture.
:param int position:
:param int end:
:param int header_position:
:param int holes:
:param int holes_size:
:param int check_header_called:
"""
# DataFile mock
dummy_header = make_header(position=header_position)
data_mock.return_value.read_disk_control.return_value = dummy_header
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
mocker.spy(blob_repairer, 'check_header')
blob_repairer.check_hole(position, end)
assert blob_repairer.stat.holes == holes
assert blob_repairer.stat.holes_size == holes_size
if check_header_called:
blob_repairer.check_header.assert_called_with(dummy_header)
assert blob_repairer.check_header.call_count == check_header_called
def test_resolve_mispositioned_record_with_empty_disk_control(_mocked_exists,
mocked_data_file,
_mocked_index_file,
header_index,
position,
headers,
expect_result):
"""Check different combinations of headers sequences.
Check different combinations of headers sequences with
BlobRepairer.resolve_mispositioned_record and empty disk control.
TODO(karapuz): not all branches are tested within:
eblob_kit.BlobRepairer.resolve_mispositioned_record
:param int header_index:
:param int position:
:param List[eblob_kit.DiskControl] headers:
:param bool expect_result:
"""
mocked_data_file.return_value.read_disk_control.return_value = make_header()
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
blob_repairer._index_headers = headers
assert blob_repairer.resolve_mispositioned_record(header_index, position, []) == expect_result
def test_get_checksum_size_for_nocsum(data_size):
"""Count checksum for headers with CHUNKED_CSUM flag.
No metter what is data_size of header, checksum size should be zero.
"""
header = make_header(data_size=data_size, flags=RecordFlags(RecordFlags.NOCSUM))
assert get_checksum_size(header) == 0
def test_get_checksum_size_for_nocsum(data_size):
"""Count checksum for headers with CHUNKED_CSUM flag.
No metter what is data_size of header, checksum size should be zero.
"""
header = make_header(data_size=data_size,
flags=RecordFlags(RecordFlags.NOCSUM))
assert get_checksum_size(header) == 0
def test_resolve_mismatch(_mocked_exists, _mocked_data_file, _mocked_index_file):
"""Check generated headers sequence with BlobRepairer.resolve_mismatch."""
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
flags = RecordFlags(0)
args_list = [make_header(key='', data_size=x, disk_size=2 * x, flags=flags, position=3 * x)
for x in xrange(3)]
blob_repairer.resolve_mismatch(*args_list)
assert not blob_repairer.valid
assert blob_repairer.stat.mismatched_headers == [tuple(args_list)[:2]]
def test_check_header_with_flags_nodata_size(_mocked_exists, mocked_data_file, _mocked_index, flags):
"""Tested valid headers with different flags combination.
:param eblob_kit.RecordFlags flags: bit flags to be used with eblob_kit.DiskControl
"""
# DataFile mock
mocked_data_file.return_value.__len__.return_value = 512
header = make_header(flags=RecordFlags(flags), disk_size=128)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.check_header(header)
def test_check_header_with_flags_nodata_size(_mocked_exists, mocked_data_file,
_mocked_index, flags):
"""Tested valid headers with different flags combination.
:param eblob_kit.RecordFlags flags: bit flags to be used with eblob_kit.DiskControl
"""
# DataFile mock
mocked_data_file.return_value.__len__.return_value = 512
header = make_header(flags=RecordFlags(flags), disk_size=128)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.check_header(header)
def test_resolve_mispositioned_record_with_positioned_disk_control(_mocked_exists,
mocked_data_file,
_mocked_index_file,
header_index,
position,
disk_control_position):
"""Check different combinations of headers sequences with non empty disk_control.
TODO(karapuz): add test to this test case.
:param int header_index:
:param int position:
:param int disk_control_position:
"""
mocked_data_file.return_value.read_disk_control.return_value = make_header(position=disk_control_position)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
blob_repairer._index_headers = [make_header(disk_size=5, position=3), make_header(position=4), ]
assert not blob_repairer.resolve_mispositioned_record(header_index, position, [])
def test_fix(mocker):
"""Tests for BlobRepairer.fix.
TODO(karapuz): function test for 'fix' command, following test not completed.
"""
# IndexFile mock
index_file_class = mocker.patch('eblob_kit.IndexFile', autospec=True)
index_header = make_header(key='',
data_size=1,
disk_size=2,
position=3,
flags=4)
index_file_class.return_value.__iter__.return_value = iter([index_header])
# TODO(karapuz): add test for sorted case
type(index_file_class.return_value).sorted = mocker.PropertyMock(
return_value=False)
# DataFile mock
data_file_class = mocker.patch('eblob_kit.DataFile', autospec=True)
# Blob mock
mocker.patch('eblob_kit.Blob.create', )
mocker.patch('eblob_kit.is_destination_writable', return_value=True)
mocker.patch('os.path.exists', return_value=True)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
mocker.spy(blob_repairer, 'check')
mocker.spy(blob_repairer, 'check_index')
mocker.spy(blob_repairer, 'print_check_report')
blob_repairer.fix(TEST_DESTINATION_PATH, noprompt=True)
# Check the self.check method
assert blob_repairer.check.call_count == 1
blob_repairer.check.assert_called_with(verify_csum=True, fast=False)
assert blob_repairer.check_index.call_count == 1
blob_repairer.check_index.assert_called_with(fast=False)
assert blob_repairer.print_check_report.call_count == 1
def test_resolve_mismatch(_mocked_exists, _mocked_data_file,
_mocked_index_file):
"""Check generated headers sequence with BlobRepairer.resolve_mismatch."""
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
flags = RecordFlags(0)
args_list = [
make_header(key='',
data_size=x,
disk_size=2 * x,
flags=flags,
position=3 * x) for x in xrange(3)
]
blob_repairer.resolve_mismatch(*args_list)
assert not blob_repairer.valid
assert blob_repairer.stat.mismatched_headers == [tuple(args_list)[:2]]
def test_check_header_with_exthdr_and_datasize(_mocked_exists,
mocked_data_file, _index_file,
data_size, disk_size, expect):
"""Check header with EXTHDR and incorrect data_size.
Checks for:
header.data_size + EllipticsHeader.size(48) + checksum_size(64) > header.disk_size
"""
header = make_header(data_size=data_size,
disk_size=disk_size,
flags=RecordFlags(RecordFlags.EXTHDR))
# Return big enough constant to suppress preliminary checks in BlobRepairer.check_header.
mocked_data_file.return_value.__len__.return_value = 512
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.check_header(header) == expect
def test_resolve_mispositioned_record_assert(_mocked_exists, mocked_data_file,
_index_file_mock, header_index,
position, headers):
"""Check for resolve_mispositioned_record fail on assert.
:param int header_index:
:param int position:
:param List[eblob_kit.DiskControl] headers:
"""
mocked_data_file.return_value.read_disk_control.return_value = make_header(
)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
with pytest.raises(AssertionError):
blob_repairer._index_headers = headers
blob_repairer.resolve_mispositioned_record(header_index, position, [])
def test_check_header_with_exthdr_and_datasize(_mocked_exists,
mocked_data_file,
_index_file,
data_size,
disk_size,
expect):
"""Check header with EXTHDR and incorrect data_size.
Checks for:
header.data_size + EllipticsHeader.size(48) + checksum_size(64) > header.disk_size
"""
header = make_header(data_size=data_size, disk_size=disk_size, flags=RecordFlags(RecordFlags.EXTHDR))
# Return big enough constant to suppress preliminary checks in BlobRepairer.check_header.
mocked_data_file.return_value.__len__.return_value = 512
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.check_header(header) == expect
def test_resolve_mispositioned_record_assert(_mocked_exists,
mocked_data_file,
_index_file_mock,
header_index,
position,
headers):
"""Check for resolve_mispositioned_record fail on assert.
:param int header_index:
:param int position:
:param List[eblob_kit.DiskControl] headers:
"""
mocked_data_file.return_value.read_disk_control.return_value = make_header()
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
with pytest.raises(AssertionError):
blob_repairer._index_headers = headers
blob_repairer.resolve_mispositioned_record(header_index, position, [])
def test_check_hole(_mock_exist,
data_mock,
_index_mock,
mocker,
position,
end,
header_position,
holes,
holes_size,
check_header_called):
"""Check generated DiskControl header with BlobRepairer.check_hole.
TODO(karapuz): try to move common for all test patch code to BlobRepairer fixture.
:param int position:
:param int end:
:param int header_position:
:param int holes:
:param int holes_size:
:param int check_header_called:
"""
# DataFile mock
dummy_header = make_header(position=header_position)
data_mock.return_value.read_disk_control.return_value = dummy_header
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
assert blob_repairer.valid
mocker.spy(blob_repairer, 'check_header')
blob_repairer.check_hole(position, end)
assert blob_repairer.stat.holes == holes
assert blob_repairer.stat.holes_size == holes_size
if check_header_called:
blob_repairer.check_header.assert_called_with(dummy_header)
assert blob_repairer.check_header.call_count == check_header_called
def test_fix(mocker):
"""Tests for BlobRepairer.fix.
TODO(karapuz): function test for 'fix' command, following test not completed.
"""
# IndexFile mock
index_file_class = mocker.patch('eblob_kit.IndexFile', autospec=True)
index_header = make_header(key='', data_size=1, disk_size=2, position=3, flags=4)
index_file_class.return_value.__iter__.return_value = iter([index_header])
# TODO(karapuz): add test for sorted case
type(index_file_class.return_value).sorted = mocker.PropertyMock(return_value=False)
# DataFile mock
data_file_class = mocker.patch('eblob_kit.DataFile', autospec=True)
# Blob mock
mocker.patch('eblob_kit.Blob.create', )
mocker.patch('eblob_kit.is_destination_writable', return_value=True)
mocker.patch('os.path.exists', return_value=True)
blob_repairer = BlobRepairer(TEST_BLOB_PATH)
mocker.spy(blob_repairer, 'check')
mocker.spy(blob_repairer, 'check_index')
mocker.spy(blob_repairer, 'print_check_report')
blob_repairer.fix(TEST_DESTINATION_PATH, noprompt=True)
# Check the self.check method
assert blob_repairer.check.call_count == 1
blob_repairer.check.assert_called_with(verify_csum=True, fast=False)
assert blob_repairer.check_index.call_count == 1
blob_repairer.check_index.assert_called_with(fast=False)
assert blob_repairer.print_check_report.call_count == 1
def test_get_checksum_size_noflags(data_size):
"""Count checksum for headers with no influencing flags set."""
header = make_header(data_size=data_size, flags=RecordFlags(0))
assert get_checksum_size(header) == 64
def test_get_checksum_size_for_chunked_csum(data_size, checksum_size):
"""Count checksum for headers with CHUNKED_CSUM flag."""
header = make_header(data_size=data_size, flags=RecordFlags(RecordFlags.CHUNKED_CSUM))
assert get_checksum_size(header) == checksum_size
def test_get_checksum_size_for_chunked_csum(data_size, checksum_size):
"""Count checksum for headers with CHUNKED_CSUM flag."""
header = make_header(data_size=data_size,
flags=RecordFlags(RecordFlags.CHUNKED_CSUM))
assert get_checksum_size(header) == checksum_size
TEST_DESTINATION_PATH = 'dst/path'
TEST_INDEX_PATH = TEST_BLOB_PATH + '/' + 'test_data-0.0.index'
TEST_INDEX_SORTED_PATH = TEST_BLOB_PATH + '/' + 'test_data-0.0.index.sorted'
TEST_DATA_PATH = TEST_BLOB_PATH + '/' + TEST_BLOB_PREFIX
if sys.version_info < (3, 0):
OPEN_TO_PATCH = '__builtin__.open'
else:
OPEN_TO_PATCH = 'builtins.open'
@pytest.mark.parametrize('header, data_len, check_result', [
(make_header(data_size=1, disk_size=128), 512, True),
# Malformed header has empty disk_size.
(make_header(data_size=1, position=1), 512, False),
# Malformed header has out of range position (position + disk_size > total data len).
(make_header(data_size=1, disk_size=128, position=1), 128, False),
# Malformed header has empty data_size, but it is committed and not removed.
(make_header(disk_size=128), 512, False),
])
@mock.patch('eblob_kit.IndexFile', autospec=True)
@mock.patch('eblob_kit.DataFile', autospec=True)
@mock.patch('os.path.exists', return_value=True)
def test_check_header_without_flags(_mocked_index,
mocked_data_file,
mocked_exist,
header,
data_len,
def test_get_checksum_size_noflags(data_size):
"""Count checksum for headers with no influencing flags set."""
header = make_header(data_size=data_size, flags=RecordFlags(0))
assert get_checksum_size(header) == 64
TEST_INDEX_PATH = TEST_BLOB_PATH + '/' + 'test_data-0.0.index'
TEST_INDEX_SORTED_PATH = TEST_BLOB_PATH + '/' + 'test_data-0.0.index.sorted'
TEST_DATA_PATH = TEST_BLOB_PATH + '/' + TEST_BLOB_PREFIX
if sys.version_info < (3, 0):
OPEN_TO_PATCH = '__builtin__.open'
else:
OPEN_TO_PATCH = 'builtins.open'
@pytest.mark.parametrize(
'header, data_len, check_result',
[
(make_header(data_size=1, disk_size=128), 512, True),
# Malformed header has empty disk_size.
(make_header(data_size=1, position=1), 512, False),
# Malformed header has out of range position (position + disk_size > total data len).
(make_header(data_size=1, disk_size=128, position=1), 128, False),
# Malformed header has empty data_size, but it is committed and not removed.
(make_header(disk_size=128), 512, False),
])
@mock.patch('eblob_kit.IndexFile', autospec=True)
@mock.patch('eblob_kit.DataFile', autospec=True)
@mock.patch('os.path.exists', return_value=True)
def test_check_header_without_flags(_mocked_index, mocked_data_file,
mocked_exist, header, data_len,
check_result):
"""Check generated header whith BlobRepairer.check_header.