def test_recursive_list_dir_with_nested_files(self):
root_dir = self.create_tempdir()
file_paths = [
os.path.join(root_dir, *p)
for p in [("a", "1.md"), ("a", "2.md"), ("a", "b", "1.md")]
]
for path in file_paths:
self.create_tempfile(path)
actual_files = list(filesystem_utils.recursive_list_dir(root_dir))
self.assertCountEqual(actual_files, file_paths)
def validate_tag_dir(directory: str) -> Dict[str, TagDefinitionError]:
"""Validate all tag definition files in the given directory.
Args:
directory: Directory path over which should be walked to find all tag files.
Returns:
file_to_error: Dict mapping from file path to occuring error.
"""
file_paths = list(filesystem_utils.recursive_list_dir(directory))
return validate_tag_files(file_paths)
def validate_documentation_files(documentation_dir: str,
files_to_validate: List[str] = None):
"""Validate documentation files in a directory."""
file_paths = list(filesystem_utils.recursive_list_dir(documentation_dir))
do_smoke_test = bool(files_to_validate)
validated = 0
for file_path in file_paths:
if files_to_validate and file_path[len(documentation_dir) +
1:] not in files_to_validate:
continue
logging.info("Validating %s.", file_path)
documentation_parser = DocumentationParser(documentation_dir)
documentation_parser.validate(file_path, do_smoke_test)
validated += 1
logging.info("Found %d matching files - all validated successfully.",
validated)
if not do_smoke_test:
logging.info(
"No models were smoke tested. To download and smoke test a specific "
"model, specify files directly in the command line, for example: "
"'python tools/validator.py vtab/models/wae-ukl/1.md'")
return validated
def validate_documentation_dir(root_dir: str,
relative_docs_path: str = DOCS_PATH):
"""Validate Markdown files in `root_dir/relative_docs_path`.
Args:
root_dir: Absolute path to the top-level dir that contains Markdown files
and YAML config files.
relative_docs_path: Relative path under `root_dir` containing the Markdown
files. Defaults to "assets/docs".
Raises:
MarkdownDocumentationError: if invalid Markdown files have been found.
"""
documentation_dir = os.path.join(root_dir, relative_docs_path)
logging.info("Validating all files in %s.", documentation_dir)
relative_paths = [
os.path.relpath(file_path, documentation_dir)
for file_path in filesystem_utils.recursive_list_dir(documentation_dir)
]
validate_documentation_files(
root_dir,
relative_paths,
relative_docs_path=relative_docs_path,
do_smoke_test=False)
def test_recursive_list_dir_with_empty_nested_directories(self): root_dir = self.create_tempdir() pathlib.Path(os.path.join(root_dir, "a", "model1")).mkdir(parents=True) pathlib.Path(os.path.join(root_dir, "a", "model2")).mkdir(parents=True) files = list(filesystem_utils.recursive_list_dir(root_dir)) self.assertEmpty(files)
def test_recursive_list_dir_with_empty_directory(self): root_dir = self.create_tempdir() files = list(filesystem_utils.recursive_list_dir(root_dir)) self.assertEmpty(files)
def _validate_file_paths(model_dir: str):
valid_path_regex = re.compile(r"(/[\w-][!',_\w\.\-=:% ]*)+")
for filepath in filesystem_utils.recursive_list_dir(model_dir):
if not valid_path_regex.fullmatch(filepath):
raise MarkdownDocumentationError(f"Invalid filepath in asset: {filepath}")
