def test_result_into_recursive_path(root_path, ckpt_suffix, gt_indices):
"""Test that the recursive parsing of path into a result dict works as intended.
It should remove the common root leaving only a relative path to the testing
checkpoint, then convert each layer of the path into a dict index.
e.g.
ckpt=/my/root/path/with/some/checkpoint.onnx, root=/my/root/path
results = {
"with": {
"some": {
"checkpoint.onnx": {"my": "result"}
}
}
}
"""
random.seed(1984)
result = {"f1": random.random()}
ckpt_path = os.path.join(root_path, ckpt_suffix)
result_dict = validation.recursive_defaultdict()
validation.result_into_recursive_path(result_dict, ckpt_path, root_path,
result)
# Convert back to a dict so we don't inadvertently create indices while testing the result
assert reduce(getitem, gt_indices, dict(result_dict)) == result
def mock_pooled_validation_run(checkpoint_paths,
root_path,
all_results,
bert_args,
config,
initializers,
paths_to_process,
num_processes=1,
available_ipus=16):
results = validation.recursive_defaultdict()
for path in checkpoint_paths:
validation.result_into_recursive_path(results, path, root_path,
all_results[path])
return results
def test_merge_hierarchy_results(num_processes, num_sweeps, num_repeats):
"""Ensure that nested directory paths are handled correctly by the validation result pooler"""
# Generate a number of mocked paths that would go as input to validation
paths = []
root = '/mock/root/'
path_pieces = [[f'sweep-{i}' for i in range(num_sweeps)],
[f'repeat-{i}' for i in range(num_repeats)]]
for p in path_pieces[0]:
for q in path_pieces[1]:
paths.append(str(Path(root) / 'ckpt' / p / q))
# Now pool them as we would if running pooled validation
checkpoint_paths_pooled = (paths[i * len(paths) // num_processes:(i + 1) *
len(paths) // num_processes]
for i in range(num_processes))
# Mock the results out (we don't care about the value)
pooled_results = []
for process_paths in checkpoint_paths_pooled:
result_dict = validation.recursive_defaultdict()
for path in process_paths:
validation.result_into_recursive_path(result_dict, path, root,
"A_MOCK_RESULT")
pooled_results.append(result_dict)
# Code under test
mock_results = validation.merge_pooled_results(pooled_results)
# Now check that the number of results provided in the test code exactly
# matches the number of results given above.
def count_leaves(d):
count = 0
for i in d.values():
if isinstance(i, dict):
count += count_leaves(i)
else:
count += 1
return count
assert count_leaves(mock_results) == len(paths)
