def test_recursive_list_dir_with_nested_files(self): root_dir = self.create_tempdir() file_paths = [ os.path.join(root_dir, *p) for p in [("a", "1.md"), ("a", "2.md"), ("a", "b", "1.md")] ] for path in file_paths: self.create_tempfile(path) actual_files = list(filesystem_utils.recursive_list_dir(root_dir)) self.assertCountEqual(actual_files, file_paths)
def validate_tag_dir(directory: str) -> Dict[str, TagDefinitionError]: """Validate all tag definition files in the given directory. Args: directory: Directory path over which should be walked to find all tag files. Returns: file_to_error: Dict mapping from file path to occuring error. """ file_paths = list(filesystem_utils.recursive_list_dir(directory)) return validate_tag_files(file_paths)
def validate_documentation_files(documentation_dir: str, files_to_validate: List[str] = None): """Validate documentation files in a directory.""" file_paths = list(filesystem_utils.recursive_list_dir(documentation_dir)) do_smoke_test = bool(files_to_validate) validated = 0 for file_path in file_paths: if files_to_validate and file_path[len(documentation_dir) + 1:] not in files_to_validate: continue logging.info("Validating %s.", file_path) documentation_parser = DocumentationParser(documentation_dir) documentation_parser.validate(file_path, do_smoke_test) validated += 1 logging.info("Found %d matching files - all validated successfully.", validated) if not do_smoke_test: logging.info( "No models were smoke tested. To download and smoke test a specific " "model, specify files directly in the command line, for example: " "'python tools/validator.py vtab/models/wae-ukl/1.md'") return validated
def validate_documentation_dir(root_dir: str, relative_docs_path: str = DOCS_PATH): """Validate Markdown files in `root_dir/relative_docs_path`. Args: root_dir: Absolute path to the top-level dir that contains Markdown files and YAML config files. relative_docs_path: Relative path under `root_dir` containing the Markdown files. Defaults to "assets/docs". Raises: MarkdownDocumentationError: if invalid Markdown files have been found. """ documentation_dir = os.path.join(root_dir, relative_docs_path) logging.info("Validating all files in %s.", documentation_dir) relative_paths = [ os.path.relpath(file_path, documentation_dir) for file_path in filesystem_utils.recursive_list_dir(documentation_dir) ] validate_documentation_files( root_dir, relative_paths, relative_docs_path=relative_docs_path, do_smoke_test=False)
def test_recursive_list_dir_with_empty_nested_directories(self): root_dir = self.create_tempdir() pathlib.Path(os.path.join(root_dir, "a", "model1")).mkdir(parents=True) pathlib.Path(os.path.join(root_dir, "a", "model2")).mkdir(parents=True) files = list(filesystem_utils.recursive_list_dir(root_dir)) self.assertEmpty(files)
def test_recursive_list_dir_with_empty_directory(self): root_dir = self.create_tempdir() files = list(filesystem_utils.recursive_list_dir(root_dir)) self.assertEmpty(files)
def _validate_file_paths(model_dir: str): valid_path_regex = re.compile(r"(/[\w-][!',_\w\.\-=:% ]*)+") for filepath in filesystem_utils.recursive_list_dir(model_dir): if not valid_path_regex.fullmatch(filepath): raise MarkdownDocumentationError(f"Invalid filepath in asset: {filepath}")