def test_result_into_recursive_path(root_path, ckpt_suffix, gt_indices): """Test that the recursive parsing of path into a result dict works as intended. It should remove the common root leaving only a relative path to the testing checkpoint, then convert each layer of the path into a dict index. e.g. ckpt=/my/root/path/with/some/checkpoint.onnx, root=/my/root/path results = { "with": { "some": { "checkpoint.onnx": {"my": "result"} } } } """ random.seed(1984) result = {"f1": random.random()} ckpt_path = os.path.join(root_path, ckpt_suffix) result_dict = validation.recursive_defaultdict() validation.result_into_recursive_path(result_dict, ckpt_path, root_path, result) # Convert back to a dict so we don't inadvertently create indices while testing the result assert reduce(getitem, gt_indices, dict(result_dict)) == result
def mock_pooled_validation_run(checkpoint_paths, root_path, all_results, bert_args, config, initializers, paths_to_process, num_processes=1, available_ipus=16): results = validation.recursive_defaultdict() for path in checkpoint_paths: validation.result_into_recursive_path(results, path, root_path, all_results[path]) return results
def test_merge_hierarchy_results(num_processes, num_sweeps, num_repeats): """Ensure that nested directory paths are handled correctly by the validation result pooler""" # Generate a number of mocked paths that would go as input to validation paths = [] root = '/mock/root/' path_pieces = [[f'sweep-{i}' for i in range(num_sweeps)], [f'repeat-{i}' for i in range(num_repeats)]] for p in path_pieces[0]: for q in path_pieces[1]: paths.append(str(Path(root) / 'ckpt' / p / q)) # Now pool them as we would if running pooled validation checkpoint_paths_pooled = (paths[i * len(paths) // num_processes:(i + 1) * len(paths) // num_processes] for i in range(num_processes)) # Mock the results out (we don't care about the value) pooled_results = [] for process_paths in checkpoint_paths_pooled: result_dict = validation.recursive_defaultdict() for path in process_paths: validation.result_into_recursive_path(result_dict, path, root, "A_MOCK_RESULT") pooled_results.append(result_dict) # Code under test mock_results = validation.merge_pooled_results(pooled_results) # Now check that the number of results provided in the test code exactly # matches the number of results given above. def count_leaves(d): count = 0 for i in d.values(): if isinstance(i, dict): count += count_leaves(i) else: count += 1 return count assert count_leaves(mock_results) == len(paths)