def template_validate(self) -> int:
"""Validate the integrity of the template files."""
logger.info('Checking template file integrity')
for template_file in self.template_dir.iterdir():
if (template_file.name
not in author_const.REFERENCE_TEMPLATES.values()
and template_file.name.lower() != 'readme.md'):
raise TrestleError(
f'Unexpected template file {self.rel_dir(template_file)}')
if template_file.suffix == '.md':
try:
md_api = MarkdownAPI()
md_api.load_validator_with_template(
template_file, True, False)
except Exception as ex:
raise TrestleError(
f'Template for task {self.task_name} failed to validate due to {ex}'
)
elif template_file.suffix == '.drawio':
try:
_ = DrawIOMetadataValidator(template_file)
except Exception as ex:
raise TrestleError(
f'Template for task {self.task_name} failed to validate due to {ex}'
)
logger.info('Templates validated')
return CmdReturnCodes.SUCCESS.value
def test_bad_unicode_in_parsetree(tmp_path: pathlib.Path):
"""Test error on read of bad unicode in parsetree."""
bad_file = tmp_path / 'bad_unicode.md'
with open(bad_file, 'wb') as f:
f.write(b'\x81')
with pytest.raises(err.TrestleError):
md_api = MarkdownAPI()
md_api.load_validator_with_template(bad_file, False, False)
def test_instance_template_version(template_path: pathlib.Path,
instance_path: pathlib.Path,
status: bool) -> None:
"""Test for x-trestle-template-version header in instances."""
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_path, False, True)
result = md_api.validate_instance(instance_path)
assert result == status
def test_md_validator_with_md_header(template_path: pathlib.Path,
instance_path: pathlib.Path, status: bool,
yaml_header_validate: bool) -> None:
"""Test with validation of heading."""
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_path, yaml_header_validate,
False, 'Governed Document')
result = md_api.validate_instance(instance_path)
assert result == status
def test_md_validator_substitutions(template_path: pathlib.Path,
instance_path: pathlib.Path, status: bool,
header_validate: bool,
validate_md_body: bool) -> None:
"""Run markdown validator to expected outcome."""
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_path, header_validate,
validate_md_body)
result = md_api.validate_instance(instance_path)
assert result == status
def test_validate_for_governed_header(template_path: pathlib.Path,
instance_path: pathlib.Path,
status: bool,
governed_header: str) -> None:
"""Test scenarios for validate w.r.t the governed header."""
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_path, False, False,
governed_header)
result = md_api.validate_instance(instance_path)
assert result == status
def write_versioned_template(resource_name: str,
task_path: Path,
target_file: Path,
version: Optional[str] = None) -> None:
"""
Write a template with the header or metadata of a specified version.
If no version was given the latest version for the task will be used.
Args:
resource_name: Template resource name
task_path: Task path
target_file: File path where template will be written
version: return a resource of a specific version
Returns:
A dotted path of a versioned template, list of all available versions
"""
TemplateVersioning._check_if_exists_and_dir(task_path)
try:
templates_resource_path = TRESTLE_RESOURCES + '.templates'
generic_template = Path(
resource_filename(templates_resource_path,
resource_name)).resolve()
if version is None:
_, version = TemplateVersioning.get_latest_version_for_task(
task_path)
# modify header/metadata in the template
if generic_template.suffix == '.md':
md_api = MarkdownAPI()
header, md_body = md_api.processor.read_markdown_wo_processing(
generic_template)
header[TEMPLATE_VERSION_HEADER] = version
md_api.write_markdown_with_header(target_file, header, md_body)
logger.debug(
f'Successfully written template markdown to {target_file}')
elif generic_template.suffix == '.drawio':
drawio = DrawIO(generic_template)
metadata = drawio.get_metadata()[0]
metadata[TEMPLATE_VERSION_HEADER] = version
drawio.write_drawio_with_metadata(generic_template, metadata,
0, target_file)
logger.debug(
f'Successfully written template drawio to {target_file}')
else:
raise TrestleError(
f'Unsupported template file extension {generic_template.suffix}'
)
except OSError as e:
raise TrestleError(f'Error while updating template folder: {e}')
def test_md_by_hand() -> None:
"""Simpler test to enable debugging."""
template_path = pathlib.Path(
'tests/data/author/0.0.1/test_3_md_hand_edited/template.md')
instance_path = pathlib.Path(
'tests/data/author/0.0.1/test_3_md_hand_edited/decisions_000.md')
header_validate = False
status = True
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_path, header_validate, False,
'Governed Document')
result = md_api.validate_instance(instance_path)
assert result == status
def test_ssp_generate(import_cat, specify_sections,
tmp_trestle_dir: pathlib.Path) -> None:
"""Test the ssp generator."""
args, sections, yaml_path = setup_for_ssp(True, False, tmp_trestle_dir,
prof_name, ssp_name, import_cat)
if specify_sections:
args.allowed_sections = 'ImplGuidance,ExpectedEvidence'
ssp_cmd = SSPGenerate()
# run the command for happy path
assert ssp_cmd._run(args) == 0
ac_dir = tmp_trestle_dir / (ssp_name + '/ac')
ac_1 = ac_dir / 'ac-1.md'
ac_2 = ac_dir / 'ac-2.md'
assert ac_1.exists()
assert ac_2.exists()
assert ac_1.stat().st_size > 1000
assert ac_2.stat().st_size > 2000
with open(yaml_path, 'r', encoding=const.FILE_ENCODING) as f:
yaml = YAML()
expected_header = yaml.load(f)
sections_dict = sections_to_dict(sections)
expected_header[const.SECTIONS_TAG] = sections_dict
assert test_utils.confirm_text_in_file(
ac_1, '## Control', '## Control Guidance') != specify_sections
md_api = MarkdownAPI()
header, tree = md_api.processor.process_markdown(ac_1)
assert tree is not None
assert expected_header == header
header, tree = md_api.processor.process_markdown(ac_2)
assert tree is not None
assert expected_header == header
def test_modify_md_node_header_lvl(testdata_dir: pathlib.Path,
tmp_trestle_dir: pathlib.Path) -> None:
"""Test that header modification works."""
markdown_file = testdata_dir / 'markdown/valid_levels_no_text.md'
expected_file = testdata_dir / 'markdown/valid_levels_no_text_increased_level.md'
md_api = MarkdownAPI()
_, tree = md_api.processor.process_markdown(markdown_file)
tree.change_header_level_by(1)
_, tree_expected = md_api.processor.process_markdown(expected_file)
assert tree_expected.content.raw_text == tree.content.raw_text
assert tree_expected.content.text == tree.content.text
assert len(list(tree.get_all_headers_for_level(1))) == len(
list(tree_expected.get_all_headers_for_level(1)))
assert len(list(tree.get_all_headers_for_level(2))) == len(
list(tree_expected.get_all_headers_for_level(2)))
assert len(list(tree.get_all_headers_for_level(3))) == len(
list(tree_expected.get_all_headers_for_level(3)))
assert tree.get_node_for_key(
'####### Header 3.2.1.1.1.1'
).content.raw_text == tree_expected.get_node_for_key(
'####### Header 3.2.1.1.1.1').content.raw_text
assert len(
tree.get_node_for_key('## Header 3 a deeper tree').subnodes) == len(
tree_expected.get_node_for_key(
'## Header 3 a deeper tree').subnodes)
for key in tree_expected.content.subnodes_keys:
node = tree.get_node_for_key(key)
assert node is not None
expected_node = tree_expected.get_node_for_key(key)
assert node.content.raw_text == expected_node.content.raw_text
assert len(node.subnodes) == len(expected_node.subnodes)
tree.change_header_level_by(-1)
_, tree_expected = md_api.processor.process_markdown(markdown_file)
assert tree_expected.content.raw_text == tree.content.raw_text
assert tree_expected.content.text == tree.content.text
assert len(list(tree.get_all_headers_for_level(1))) == len(
list(tree_expected.get_all_headers_for_level(1)))
assert len(list(tree.get_all_headers_for_level(2))) == len(
list(tree_expected.get_all_headers_for_level(2)))
assert len(list(tree.get_all_headers_for_level(3))) == len(
list(tree_expected.get_all_headers_for_level(3)))
assert len(
tree.get_node_for_key('# Header 3 a deeper tree').subnodes) == len(
tree_expected.get_node_for_key(
'# Header 3 a deeper tree').subnodes)
for key in tree_expected.content.subnodes_keys:
node = tree.get_node_for_key(key)
assert node is not None
expected_node = tree_expected.get_node_for_key(key)
assert node.content.raw_text == expected_node.content.raw_text
assert len(node.subnodes) == len(expected_node.subnodes)
def template_validate(self, heading: str, validate_header: bool,
validate_only_header: bool) -> int:
"""Validate that the template is acceptable markdown."""
template_file = self.template_dir / self.template_name
if not self._validate_template_dir():
raise TrestleError('Aborting setup')
if not template_file.is_file():
raise TrestleError(
f'Required template file: {self.rel_dir(template_file)} does not exist. Exiting.'
)
try:
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_file, validate_header,
validate_only_header, heading)
except Exception as ex:
raise TrestleError(
f'Template for task {self.task_name} failed to validate due to {ex}'
)
logger.info(f'TEMPLATES VALID: {self.task_name}')
return CmdReturnCodes.SUCCESS.value
def template_validate(self, validate_header: bool,
validate_only_header: bool, heading: str,
readme_validate: bool) -> int:
"""Validate that the template is acceptable markdown."""
if not self.template_dir.is_dir():
raise TrestleError(
f'Template directory {self.rel_dir(self.template_dir)} for task {self.task_name} does not exist.'
)
# get list of files:
template_files = self.template_dir.rglob('*')
for template_file in template_files:
try:
if not file_utils.is_local_and_visible(template_file):
continue
elif template_file.is_dir():
continue
elif template_file.suffix.lower() == '.md':
if not readme_validate and template_file.name == 'readme.md':
raise TrestleError(
'Template directory contains a readme.md file and readme validation is off.'
)
md_api = MarkdownAPI()
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, heading)
elif template_file.suffix.lower().lstrip('.') == 'drawio':
_ = draw_io.DrawIOMetadataValidator(template_file)
else:
logger.info(
f'File: {self.rel_dir(template_file)} within the template directory was ignored'
+ 'as it is not markdown.')
except Exception as ex:
raise TrestleError(
f'Template file {self.rel_dir(template_file)} for task {self.task_name}'
+ f' failed to validate due to {ex}')
logger.info(f'TEMPLATES VALID: {self.task_name}.')
return CmdReturnCodes.SUCCESS.value
def test_write_versioned_template(tmp_path: pathlib.Path) -> None:
"""Test writing a template to the folder."""
task_path = tmp_path.joinpath('trestle/author/sample_task/')
with pytest.raises(TrestleError):
TemplateVersioning.get_versioned_template_dir(task_path)
task_path.mkdir(parents=True)
tmp_path_01 = task_path.joinpath('0.0.1')
tmp_path_02 = task_path.joinpath('0.0.2')
tmp_path_01.mkdir(parents=True)
tmp_path_02.mkdir(parents=True)
template = tmp_path_01.joinpath('template.md')
TemplateVersioning.write_versioned_template('template.md', tmp_path_01,
template, None)
assert task_path.joinpath(START_TEMPLATE_VERSION).joinpath(
'template.md').exists()
assert template.exists()
template2 = tmp_path_01.joinpath('template2.md')
template3 = tmp_path_02.joinpath('template3.md')
template4 = tmp_path_02.joinpath('template4.md')
TemplateVersioning.write_versioned_template('template.md', tmp_path_01,
template2, '0.0.1')
TemplateVersioning.write_versioned_template('template.md', tmp_path_02,
template3, '0.0.2')
TemplateVersioning.write_versioned_template('template.md', tmp_path_02,
template4, None)
assert tmp_path_01.joinpath('template.md').exists()
assert tmp_path_01.joinpath('template2.md').exists()
assert tmp_path_02.joinpath('template3.md').exists()
assert tmp_path_02.joinpath('template4.md').exists()
assert template.exists()
md_api = MarkdownAPI()
header, _ = md_api.processor.read_markdown_wo_processing(
tmp_path_02.joinpath('template3.md'))
assert header[TEMPLATE_VERSION_HEADER] == '0.0.2'
template_drawio = tmp_path_02.joinpath('template.drawio')
TemplateVersioning.write_versioned_template('template.drawio', tmp_path_02,
template_drawio, '0.0.2')
drawio = DrawIO(template_drawio)
metadata = drawio.get_metadata()[0]
assert metadata[TEMPLATE_VERSION_HEADER] == '0.0.2'
def test_ssp_generate_header_edit(yaml_header: bool,
tmp_trestle_dir: pathlib.Path) -> None:
"""Test ssp generate does not overwrite header edits."""
# always start by creating the markdown with the yaml header
args, sections, yaml_path = setup_for_ssp(True, False, tmp_trestle_dir,
prof_name, ssp_name)
ssp_cmd = SSPGenerate()
assert ssp_cmd._run(args) == 0
ac_dir = tmp_trestle_dir / (ssp_name + '/ac')
ac_1 = ac_dir / 'ac-1.md'
with open(yaml_path, 'r', encoding=const.FILE_ENCODING) as f:
yaml = YAML()
yaml_header = yaml.load(f)
md_api = MarkdownAPI()
header, tree = md_api.processor.process_markdown(ac_1)
assert tree is not None
# remove the sections that were added to original header so we can check other changes in header
header.pop(const.SECTIONS_TAG)
assert yaml_header == header
# edit the header by adding a list item and removing a value
assert test_utils.insert_text_in_file(ac_1, 'System Specific',
' - My new edits\n')
assert test_utils.delete_line_in_file(ac_1, 'Corporate')
# if the yaml header is not written out, the new header should be the one currently in the control
# if the yaml header is written out, it is merged with the current header giving priority to current header
# so if not written out, the header should have one item added and another deleted due to edits in this test
# if written out, it should just have the one added item because the deleted one will be put back in
# tell it not to add the yaml header
if not yaml_header:
args.yaml_header = None
assert ssp_cmd._run(args) == 0
header, tree = md_api.processor.process_markdown(ac_1)
assert tree is not None
assert len(header['control-origination']) == 2
if not yaml_header:
assert 'new' in header['control-origination'][0]
else:
assert 'new' not in header['control-origination'][0]
def test_modify_subtree(testdata_dir: pathlib.Path,
tmp_trestle_dir: pathlib.Path) -> None:
"""Test modification of the subtree."""
markdown_file = testdata_dir / 'markdown/valid_no_lvl1_headers.md'
md_api = MarkdownAPI()
_, tree = md_api.processor.process_markdown(markdown_file)
assert tree.get_node_header_lvl() is None
subtree = tree.get_node_for_key('## Header root child 2')
subtree.change_header_level_by(-1)
assert len(subtree.content.subnodes_keys) == 10
assert subtree.key == '# Header root child 2'
assert len(list(subtree.get_all_headers_for_level(2))) == 2
assert len(list(subtree.get_all_headers_for_level(3))) == 4
assert len(list(subtree.get_all_headers_for_level(4))) == 4
assert subtree.get_node_header_lvl() == 1
def test_ssp_generate_no_header(tmp_trestle_dir: pathlib.Path) -> None:
"""Test the ssp generator with no yaml header."""
args, _, _ = setup_for_ssp(False, False, tmp_trestle_dir, prof_name,
ssp_name)
ssp_cmd = SSPGenerate()
args.sections = None
# run the command for happy path
assert ssp_cmd._run(args) == 0
ac_dir = tmp_trestle_dir / (ssp_name + '/ac')
ac_1 = ac_dir / 'ac-1.md'
ac_2 = ac_dir / 'ac-2.md'
assert ac_1.exists()
assert ac_2.exists()
assert ac_1.stat().st_size > 1000
assert ac_2.stat().st_size > 2000
md_api = MarkdownAPI()
header, tree = md_api.processor.process_markdown(ac_1)
assert tree is not None
assert not header
header, tree = md_api.processor.process_markdown(ac_2)
assert tree is not None
assert not header
def test_modify_md_node_remove_restore_headers(
testdata_dir: pathlib.Path, tmp_trestle_dir: pathlib.Path) -> None:
"""Test that header modification works."""
markdown_file = testdata_dir / 'markdown/valid_no_lvl1_headers.md'
expected_file = testdata_dir / 'markdown/valid_no_headers.md'
md_api = MarkdownAPI()
_, tree = md_api.processor.process_markdown(markdown_file)
tree.change_header_level_by(-99)
_, tree_expected = md_api.processor.process_markdown(expected_file)
assert tree_expected.content.raw_text == tree.content.raw_text
assert len(list(tree.get_all_headers_for_level(1))) == len(
list(tree_expected.get_all_headers_for_level(1)))
assert len(list(tree.get_all_headers_for_level(2))) == len(
list(tree_expected.get_all_headers_for_level(2)))
assert len(list(tree.get_all_headers_for_level(3))) == len(
list(tree_expected.get_all_headers_for_level(3)))
tree.change_header_level_by(4)
assert len(list(tree.get_all_headers_for_level(4))) == 14
def _validate_dir(self,
governed_heading: str,
md_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
recurse: bool,
readme_validate: bool,
template_version: Optional[str] = None,
ignore: Optional[str] = None) -> int:
"""
Validate md files in a directory with option to recurse.
Template version will be fetched from the instance header.
"""
# status is a linux returncode
status = 0
for item_path in md_dir.iterdir():
if file_utils.is_local_and_visible(item_path):
if item_path.is_file():
if not item_path.suffix == '.md':
logger.info(
f'Unexpected file {self.rel_dir(item_path)} in folder {self.rel_dir(md_dir)}, skipping.'
)
continue
if not readme_validate and item_path.name.lower(
) == 'readme.md':
continue
if ignore:
p = re.compile(ignore)
matched = p.match(item_path.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {item_path} from validation.')
continue
md_api = MarkdownAPI()
if template_version != '':
template_file = self.template_dir / self.template_name
else:
instance_version = md_api.processor.fetch_value_from_header(
item_path, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1'
versione_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versione_template_dir / self.template_name
if not template_file.is_file():
raise TrestleError(
f'Required template file: {self.rel_dir(template_file)} does not exist. Exiting.'
)
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, governed_heading)
if not md_api.validate_instance(item_path):
logger.info(f'INVALID: {self.rel_dir(item_path)}')
status = 1
else:
logger.info(f'VALID: {self.rel_dir(item_path)}')
elif recurse:
if ignore:
p = re.compile(ignore)
if len(
list(
filter(
p.match,
str(item_path.relative_to(
md_dir)).split('/')))) > 0:
logger.info(
f'Ignoring directory {item_path} from validation.'
)
continue
rc = self._validate_dir(governed_heading, item_path,
validate_header,
validate_only_header, recurse,
readme_validate, template_version,
ignore)
if rc != 0:
status = rc
return status
def test_bad_file_path(tmp_path: pathlib.Path):
"""Check errors are thrown with bad files."""
no_file = tmp_path / 'non_existent.md'
with pytest.raises(err.TrestleError):
md_api = MarkdownAPI()
md_api.load_validator_with_template(no_file, False, False)
def test_broken_yaml_header(testdata_dir: pathlib.Path):
"""Test for a bad markdown header."""
bad_file = testdata_dir / 'author' / 'bad_md_header.md'
with pytest.raises(err.TrestleError):
md_api = MarkdownAPI()
md_api.load_validator_with_template(bad_file, True, False)
def test_template_path_mismatch(template_path: pathlib.Path) -> None:
"""Test template path and version mismatch."""
with pytest.raises(err.TrestleError):
md_api = MarkdownAPI()
md_api.load_validator_with_template(template_path, False, True)
def _validate_dir(self, candidate_dir: pathlib.Path, recurse: bool,
readme_validate: bool,
relative_exclusions: List[pathlib.Path],
template_version: str, ignore: str) -> bool:
"""Validate a directory within the trestle project."""
all_versioned_templates = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
instances = list(candidate_dir.iterdir())
if recurse:
instances = candidate_dir.rglob('*')
if ignore:
p = re.compile(ignore)
instances = list(
filter(
lambda f: len(
list(
filter(
p.match,
str(f.relative_to(candidate_dir)).split(
'/')))) == 0, instances))
for instance_file in instances:
if not file_utils.is_local_and_visible(instance_file):
continue
if instance_file.name.lower(
) == 'readme.md' and not readme_validate:
continue
if instance_file.is_dir() and not recurse:
continue
if any(
str(ex) in str(instance_file)
for ex in relative_exclusions):
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(candidate_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == '.md':
md_api = MarkdownAPI()
versioned_template_dir = None
if template_version != '':
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = {}
all_versioned_templates[instance_version]['drawio'] = list(
filter(lambda p: p.suffix == '.drawio', templates))[0]
all_versioned_templates[instance_version]['md'] = list(
filter(lambda p: p.suffix == '.md', templates))[0]
# validate
md_api.load_validator_with_template(
all_versioned_templates[instance_version]['md'], True,
False)
status = md_api.validate_instance(instance_file)
if not status:
logger.info(f'INVALID: {self.rel_dir(instance_file)}')
return False
else:
logger.info(f'VALID: {self.rel_dir(instance_file)}')
elif instance_file.suffix == '.drawio':
drawio = DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[
author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = {}
all_versioned_templates[instance_version]['drawio'] = list(
filter(lambda p: p.suffix == '.drawio', templates))[0]
all_versioned_templates[instance_version]['md'] = list(
filter(lambda p: p.suffix == '.md', templates))[0]
# validate
drawio_validator = DrawIOMetadataValidator(
all_versioned_templates[instance_version]['drawio'])
status = drawio_validator.validate(instance_file)
if not status:
logger.info(f'INVALID: {self.rel_dir(instance_file)}')
return False
else:
logger.info(f'VALID: {self.rel_dir(instance_file)}')
else:
logger.debug(
f'Unsupported extension of the instance file: {instance_file}, will not be validated.'
)
return True
def _measure_template_folder(self, instance_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
governed_heading: str, readme_validate: bool,
template_version: str, ignore: str) -> bool:
"""
Validate instances against templates.
Validation will succeed iff:
1. All template files from the specified version are present in the task
2. All of the instances are valid
"""
all_versioned_templates = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
for instance_file in instance_dir.iterdir():
if not file_utils.is_local_and_visible(instance_file):
continue
if not instance_file.is_file():
continue
if instance_file.name.lower(
) == 'readme.md' and not readme_validate:
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(instance_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == '.md':
md_api = MarkdownAPI()
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = dict.fromkeys([
t.relative_to(versioned_template_dir)
for t in templates
], False)
if instance_file_name in all_versioned_templates[
instance_version]:
# validate
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, governed_heading)
status = md_api.validate_instance(instance_file)
if not status:
logger.warning(
f'INVALID: Markdown file {instance_file} failed validation against'
+ f' {template_file}')
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][
instance_file_name] = True
elif instance_file.suffix == '.drawio':
drawio = draw_io.DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[
author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = dict.fromkeys([
t.relative_to(versioned_template_dir)
for t in templates
], False)
if instance_file_name in all_versioned_templates[
instance_version]:
# validate
drawio_validator = draw_io.DrawIOMetadataValidator(
template_file)
status = drawio_validator.validate(instance_file)
if not status:
logger.warning(
f'INVALID: Drawio file {instance_file} failed validation against'
+ f' {template_file}')
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][
instance_file_name] = True
else:
logger.debug(
f'Unsupported extension of the instance file: {instance_file}, will not be validated.'
)
# Check that all template files are present
for version in all_versioned_templates.keys():
for template in all_versioned_templates[version]:
if not all_versioned_templates[version][template]:
logger.warning(
f'Required template file {template} does not exist in measured instance'
+ f'{instance_dir}')
return False
return True
