def test_dispatcher_store_returns_a_Metadata_object():
conf = copy.deepcopy(VALID_DISPATCHER_CONF)
del conf["providers"]["redis"]
with mock.patch("socket.gethostbyname", return_value="127.0.0.1"):
dispatcher = dsp.Dispatcher(conf)
encoded_file = playcloud_pb2.File()
encoded_file.original_size = 0
meta = dispatcher.put(DEFAULT_PATH, encoded_file)
assert isinstance(meta, metadata.MetaDocument)
assert meta.path == DEFAULT_PATH
def test_Dispatcher_list_files(mocked_socket):
conf = copy.deepcopy(VALID_DISPATCHER_CONF)
del conf["providers"]["redis"]
dispatcher = dsp.Dispatcher(conf)
with mock.patch("pyproxy.metadata.Files.values", return_value=[]):
result = dispatcher.list()
assert isinstance(result, list)
assert len(result) == 0
path = "key"
expected_metadata = metadata.MetaDocument(path)
with mock.patch("pyproxy.metadata.Files.values",
return_value=[expected_metadata]):
result = dispatcher.list()
assert isinstance(result, list)
assert len(result) == 1
assert isinstance(result[0], metadata.MetaDocument)
assert result[0].path == path
def rebuild(configuration_path):
"""
Rebuilds metadata by data from the storage nodes
Args:
configuration_path(str): Path to the dispatcher configuration file
Returns:
list(metadata.MetaDocument): The rebuilt metadata information
"""
if not configuration_path or not isinstance(configuration_path, str):
raise ValueError(
"configuration_path argument must be a non-empty string")
with open(configuration_path, "r") as handle:
dispatcher_configuration = json.load(handle)
dispatcher = d.Dispatcher(configuration=dispatcher_configuration)
documents = {}
for provider_name in dispatcher.providers:
provider = dispatcher.providers[provider_name]
documents = rebuild_node(provider, provider_name, documents=documents)
return documents
def delete_block(block, host="metadata", port=6379):
"""
Removes location of replicas from metadata.
Args:
block(MetaBlock): The block whose replicas need to be destroyed
host(str, optional): The host metadata server
port(int, optional): The port the metadata server is listening on
Returns:
bool: Whether the block was deleted
Raises:
ValueError: if the block is not a MetaBlock instance
"""
if not block or not isinstance(block, mtdt.MetaBlock):
raise ValueError("block argument must be a valid MetaBlock")
LOGGER.debug("delete_block: block={:s}, host={:s}, port={:d}".format(
block.key, host, port))
files = mtdt.Files(host=host, port=port)
filename = dsp.extract_path_from_key(block.key)
with open("./dispatcher.json", "r") as handle:
dispatcher_configuration = json.load(handle)
dispatcher = dsp.Dispatcher(configuration=dispatcher_configuration)
hostname = os.uname()[1]
kazoo_resource = os.path.join("/", filename)
kazoo_identifier = "repair-{:s}".format(hostname)
with KAZOO_CLIENT.WriteLock(kazoo_resource, kazoo_identifier):
metadata = files.get(filename)
for metablock in metadata.blocks:
if metablock.key == block.key:
metablock.providers = metablock.providers[:1]
for provider_name in metablock.providers[1:]:
dispatcher.providers[provider_name].delete(metablock.key)
LOGGER.debug(
"delete_block: Removed replica of {:s} from {:s}".
format(metablock.key, provider_name))
break
files.put(metadata.path, metadata)
return len(files.get_block(block.key).providers) == 1
def rebuild(configuration_path):
"""
Rebuilds metadata by data from the storage nodes
Args:
configuration_path(str): Path to the dispatcher configuration file
Returns:
list(metadata.MetaDocument): The rebuilt metadata information
"""
if not configuration_path or not isinstance(configuration_path, str):
raise ValueError(
"configuration_path argument must be a non-empty string")
with open(configuration_path, "r") as handle:
dispatcher_configuration = json.load(handle)
metadata_server = metadata.Files()
original_documents = metadata_server.get_files(metadata_server.keys())
total_documents = len(original_documents)
total_blocks = count_blocks(original_documents)
dispatcher = d.Dispatcher(configuration=dispatcher_configuration)
completion = {
"documents": [0],
"documents_availability": [0],
"blocks": [0],
"blocks_availability": [0]
}
documents = {}
provider_names = dispatcher.providers.keys()
random.shuffle(provider_names)
sys.stderr.write("total_blocks: {:d}\n".format(total_blocks))
available_blocks = set()
for provider_name in provider_names:
blocks_score = 0.0
provider = dispatcher.providers[provider_name]
documents = rebuild_node(provider, provider_name, documents=documents)
completion["documents"].append(float(len(documents)) / total_documents)
current_block_count = count_blocks(documents.values())
sys.stderr.write("{:d} blocks read\n".format(current_block_count))
completion["blocks"].append(float(current_block_count / total_blocks))
for document in documents.values():
original_blocks = {
b.key: b
for b in metadata_server.get(document.path).blocks
}
for block in document.blocks:
score_for_block = compute_block_completion(
block, original_blocks[block.key])
blocks_score += score_for_block
if score_for_block == 1.0:
available_blocks.add(block.key)
documents_availability = 0
sys.stderr.write("There are {:d} blocks available\n".format(
len(available_blocks)))
for doc_key in documents:
document = metadata_server.get(doc_key)
if is_document_available(document, available_blocks):
documents_availability += 1
sys.stderr.write(
"Document {:s} is available\n".format(doc_key))
completion["documents_availability"].append(
float(documents_availability) / total_documents)
completion["blocks_availability"].append(
float(blocks_score) / total_blocks)
return completion