def test_local_and_visible(tmp_path) -> None:
"""Test if file is local (not symlink) and visible (not hidden)."""
local_file = tmp_path / 'local.md'
local_file.touch()
if file_utils.is_windows():
link_file = tmp_path / 'not_local.lnk'
link_file.touch()
else:
link_file = tmp_path / 'linked.md'
link_file.symlink_to(local_file)
assert file_utils.is_local_and_visible(local_file)
assert not file_utils.is_local_and_visible(link_file)
def test_make_hidden_file(tmp_path: pathlib.Path) -> None:
"""Test make hidden files."""
file_path = tmp_path / '.keep'
file_utils.make_hidden_file(file_path)
file_path2 = tmp_path / 'hidden.txt'
file_utils.make_hidden_file(file_path2)
assert file_path.exists(
) and not file_utils.is_local_and_visible(file_path)
if file_utils.is_windows():
assert file_path2.exists(
) and not file_utils.is_local_and_visible(file_path2)
else:
assert (tmp_path / '.hidden.txt').exists(
) and not file_utils.is_local_and_visible(tmp_path / '.hidden.txt')
def template_validate(self, validate_header: bool,
validate_only_header: bool, heading: str,
readme_validate: bool) -> int:
"""Validate that the template is acceptable markdown."""
if not self.template_dir.is_dir():
raise TrestleError(
f'Template directory {self.rel_dir(self.template_dir)} for task {self.task_name} does not exist.'
)
# get list of files:
template_files = self.template_dir.rglob('*')
for template_file in template_files:
try:
if not file_utils.is_local_and_visible(template_file):
continue
elif template_file.is_dir():
continue
elif template_file.suffix.lower() == '.md':
if not readme_validate and template_file.name == 'readme.md':
raise TrestleError(
'Template directory contains a readme.md file and readme validation is off.'
)
md_api = MarkdownAPI()
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, heading)
elif template_file.suffix.lower().lstrip('.') == 'drawio':
_ = draw_io.DrawIOMetadataValidator(template_file)
else:
logger.info(
f'File: {self.rel_dir(template_file)} within the template directory was ignored'
+ 'as it is not markdown.')
except Exception as ex:
raise TrestleError(
f'Template file {self.rel_dir(template_file)} for task {self.task_name}'
+ f' failed to validate due to {ex}')
logger.info(f'TEMPLATES VALID: {self.task_name}.')
return CmdReturnCodes.SUCCESS.value
def _validate_dir(self, candidate_dir: pathlib.Path, recurse: bool,
readme_validate: bool,
relative_exclusions: List[pathlib.Path],
template_version: str, ignore: str) -> bool:
"""Validate a directory within the trestle project."""
all_versioned_templates = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
instances = list(candidate_dir.iterdir())
if recurse:
instances = candidate_dir.rglob('*')
if ignore:
p = re.compile(ignore)
instances = list(
filter(
lambda f: len(
list(
filter(
p.match,
str(f.relative_to(candidate_dir)).split(
'/')))) == 0, instances))
for instance_file in instances:
if not file_utils.is_local_and_visible(instance_file):
continue
if instance_file.name.lower(
) == 'readme.md' and not readme_validate:
continue
if instance_file.is_dir() and not recurse:
continue
if any(
str(ex) in str(instance_file)
for ex in relative_exclusions):
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(candidate_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == '.md':
md_api = MarkdownAPI()
versioned_template_dir = None
if template_version != '':
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = {}
all_versioned_templates[instance_version]['drawio'] = list(
filter(lambda p: p.suffix == '.drawio', templates))[0]
all_versioned_templates[instance_version]['md'] = list(
filter(lambda p: p.suffix == '.md', templates))[0]
# validate
md_api.load_validator_with_template(
all_versioned_templates[instance_version]['md'], True,
False)
status = md_api.validate_instance(instance_file)
if not status:
logger.info(f'INVALID: {self.rel_dir(instance_file)}')
return False
else:
logger.info(f'VALID: {self.rel_dir(instance_file)}')
elif instance_file.suffix == '.drawio':
drawio = DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[
author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = {}
all_versioned_templates[instance_version]['drawio'] = list(
filter(lambda p: p.suffix == '.drawio', templates))[0]
all_versioned_templates[instance_version]['md'] = list(
filter(lambda p: p.suffix == '.md', templates))[0]
# validate
drawio_validator = DrawIOMetadataValidator(
all_versioned_templates[instance_version]['drawio'])
status = drawio_validator.validate(instance_file)
if not status:
logger.info(f'INVALID: {self.rel_dir(instance_file)}')
return False
else:
logger.info(f'VALID: {self.rel_dir(instance_file)}')
else:
logger.debug(
f'Unsupported extension of the instance file: {instance_file}, will not be validated.'
)
return True
def _validate_dir(self,
governed_heading: str,
md_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
recurse: bool,
readme_validate: bool,
template_version: Optional[str] = None,
ignore: Optional[str] = None) -> int:
"""
Validate md files in a directory with option to recurse.
Template version will be fetched from the instance header.
"""
# status is a linux returncode
status = 0
for item_path in md_dir.iterdir():
if file_utils.is_local_and_visible(item_path):
if item_path.is_file():
if not item_path.suffix == '.md':
logger.info(
f'Unexpected file {self.rel_dir(item_path)} in folder {self.rel_dir(md_dir)}, skipping.'
)
continue
if not readme_validate and item_path.name.lower(
) == 'readme.md':
continue
if ignore:
p = re.compile(ignore)
matched = p.match(item_path.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {item_path} from validation.')
continue
md_api = MarkdownAPI()
if template_version != '':
template_file = self.template_dir / self.template_name
else:
instance_version = md_api.processor.fetch_value_from_header(
item_path, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1'
versione_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versione_template_dir / self.template_name
if not template_file.is_file():
raise TrestleError(
f'Required template file: {self.rel_dir(template_file)} does not exist. Exiting.'
)
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, governed_heading)
if not md_api.validate_instance(item_path):
logger.info(f'INVALID: {self.rel_dir(item_path)}')
status = 1
else:
logger.info(f'VALID: {self.rel_dir(item_path)}')
elif recurse:
if ignore:
p = re.compile(ignore)
if len(
list(
filter(
p.match,
str(item_path.relative_to(
md_dir)).split('/')))) > 0:
logger.info(
f'Ignoring directory {item_path} from validation.'
)
continue
rc = self._validate_dir(governed_heading, item_path,
validate_header,
validate_only_header, recurse,
readme_validate, template_version,
ignore)
if rc != 0:
status = rc
return status
def _measure_template_folder(self, instance_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
governed_heading: str, readme_validate: bool,
template_version: str, ignore: str) -> bool:
"""
Validate instances against templates.
Validation will succeed iff:
1. All template files from the specified version are present in the task
2. All of the instances are valid
"""
all_versioned_templates = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
for instance_file in instance_dir.iterdir():
if not file_utils.is_local_and_visible(instance_file):
continue
if not instance_file.is_file():
continue
if instance_file.name.lower(
) == 'readme.md' and not readme_validate:
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(instance_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == '.md':
md_api = MarkdownAPI()
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = dict.fromkeys([
t.relative_to(versioned_template_dir)
for t in templates
], False)
if instance_file_name in all_versioned_templates[
instance_version]:
# validate
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, governed_heading)
status = md_api.validate_instance(instance_file)
if not status:
logger.warning(
f'INVALID: Markdown file {instance_file} failed validation against'
+ f' {template_file}')
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][
instance_file_name] = True
elif instance_file.suffix == '.drawio':
drawio = draw_io.DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[
author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = dict.fromkeys([
t.relative_to(versioned_template_dir)
for t in templates
], False)
if instance_file_name in all_versioned_templates[
instance_version]:
# validate
drawio_validator = draw_io.DrawIOMetadataValidator(
template_file)
status = drawio_validator.validate(instance_file)
if not status:
logger.warning(
f'INVALID: Drawio file {instance_file} failed validation against'
+ f' {template_file}')
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][
instance_file_name] = True
else:
logger.debug(
f'Unsupported extension of the instance file: {instance_file}, will not be validated.'
)
# Check that all template files are present
for version in all_versioned_templates.keys():
for template in all_versioned_templates[version]:
if not all_versioned_templates[version][template]:
logger.warning(
f'Required template file {template} does not exist in measured instance'
+ f'{instance_dir}')
return False
return True