def test_files_list_blocks(monkeypatch): with mock.patch("socket.gethostbyname", return_value="127.0.0.1"): files = Files() def get_block_range(my_range, start, end): return ["block1", "block2", "block3"] monkeypatch.setattr(files.redis, "zrange", get_block_range) blocks = files.list_blocks() assert isinstance(blocks, list) assert len(blocks) == 3 assert blocks == ["block1", "block2", "block3"]
def audit(): """ Downloads and checks the integrity of the blocks stored on the storage nodes. Failing blocks that cannot be recovered by copying from a replica are grouped by documents and added to a queue returned by the function. Returns: dict(str, set(int)): Documents and the indices of the blocks that need to be reconstructed """ reconstruction_needed = [] dispatcher = Dispatcher(get_dispatcher_configuration()) files = Files(host="metadata") # List blocks blocks = files.get_blocks(files.list_blocks()) # For each block for block in blocks: LOGGER.debug("Looking at block {:s}".format(block.key)) # For each replica of the block providers = list(set(block.providers)) for provider_name in providers: LOGGER.debug("Looking at replica of {:s} on {:s}".format(block.key, provider_name)) # Download replica replica = dispatcher.providers[provider_name].get(block.key) computed_checksum = None if replica: computed_checksum = hashlib.sha256(replica).digest() # If the replica does not match its checksum if not replica or computed_checksum != block.checksum: repaired = False if not replica: LOGGER.warn("Could not load replica of {:s} on {:s}".format(block.key, provider_name)) if computed_checksum: LOGGER.warn("Replica of {:s} on {:s} does not match expected checksum (actual = {:s}, expected = {:s})".format(block.key, provider_name, convert_binary_to_hex_digest(computed_checksum), convert_binary_to_hex_digest(block.checksum))) # Look for sane replicas other_providers = list(set(providers).difference(set([provider_name]))) if other_providers: for other_provider in other_providers: candidate_replica = dispatcher.providers[other_provider].get(block.key) if not candidate_replica: continue candidate_checksum = hashlib.sha256(candidate_replica).digest() if candidate_checksum == block.checksum: # Copy the new valid replica dispatcher.providers[provider_name].put(candidate_replica, block.key) repaired = True break # Otherwise if not repaired: # Queue the block for reconstruction LOGGER.warn("Replica of {:s} on {:s} must be reconstructed".format(block.key, provider_name)) reconstruction_needed.append(block.key) else: LOGGER.debug("Replica of {:s} on {:s} is OK".format(block.key, provider_name)) return group_blocks_by_path(reconstruction_needed)