def test_files_list_blocks(monkeypatch):
with mock.patch("socket.gethostbyname", return_value="127.0.0.1"):
files = Files()
def get_block_range(my_range, start, end):
return ["block1", "block2", "block3"]
monkeypatch.setattr(files.redis, "zrange", get_block_range)
blocks = files.list_blocks()
assert isinstance(blocks, list)
assert len(blocks) == 3
assert blocks == ["block1", "block2", "block3"]
def audit():
"""
Downloads and checks the integrity of the blocks stored on the storage nodes.
Failing blocks that cannot be recovered by copying from a replica are grouped
by documents and added to a queue returned by the function.
Returns:
dict(str, set(int)): Documents and the indices of the blocks that need
to be reconstructed
"""
reconstruction_needed = []
dispatcher = Dispatcher(get_dispatcher_configuration())
files = Files(host="metadata")
# List blocks
blocks = files.get_blocks(files.list_blocks())
# For each block
for block in blocks:
LOGGER.debug("Looking at block {:s}".format(block.key))
# For each replica of the block
providers = list(set(block.providers))
for provider_name in providers:
LOGGER.debug("Looking at replica of {:s} on {:s}".format(block.key, provider_name))
# Download replica
replica = dispatcher.providers[provider_name].get(block.key)
computed_checksum = None
if replica:
computed_checksum = hashlib.sha256(replica).digest()
# If the replica does not match its checksum
if not replica or computed_checksum != block.checksum:
repaired = False
if not replica:
LOGGER.warn("Could not load replica of {:s} on {:s}".format(block.key, provider_name))
if computed_checksum:
LOGGER.warn("Replica of {:s} on {:s} does not match expected checksum (actual = {:s}, expected = {:s})".format(block.key, provider_name, convert_binary_to_hex_digest(computed_checksum), convert_binary_to_hex_digest(block.checksum)))
# Look for sane replicas
other_providers = list(set(providers).difference(set([provider_name])))
if other_providers:
for other_provider in other_providers:
candidate_replica = dispatcher.providers[other_provider].get(block.key)
if not candidate_replica:
continue
candidate_checksum = hashlib.sha256(candidate_replica).digest()
if candidate_checksum == block.checksum:
# Copy the new valid replica
dispatcher.providers[provider_name].put(candidate_replica, block.key)
repaired = True
break
# Otherwise
if not repaired:
# Queue the block for reconstruction
LOGGER.warn("Replica of {:s} on {:s} must be reconstructed".format(block.key, provider_name))
reconstruction_needed.append(block.key)
else:
LOGGER.debug("Replica of {:s} on {:s} is OK".format(block.key, provider_name))
return group_blocks_by_path(reconstruction_needed)