def setup_template_governed_docs(self, template_version: str) -> int:
"""Create structure to allow markdown template enforcement.
Returns:
Unix return code.
"""
if not self.task_path.exists():
self.task_path.mkdir(exist_ok=True, parents=True)
elif self.task_path.is_file():
raise TrestleError(
f'Task path: {self.rel_dir(self.task_path)} is a file not a directory.'
)
if not self.template_dir.exists():
self.template_dir.mkdir(exist_ok=True, parents=True)
elif self.template_dir.is_file():
raise TrestleError(
f'Template path: {self.rel_dir(self.template_dir)} is a file not a directory.'
)
logger.debug(self.template_dir)
if not self._validate_template_dir():
raise TrestleError('Aborting setup')
template_file = self.template_dir / self.template_name
if template_file.is_file():
return CmdReturnCodes.SUCCESS.value
TemplateVersioning.write_versioned_template('template.md',
self.template_dir,
template_file,
template_version)
logger.info(
f'Template file setup for task {self.task_name} at {self.rel_dir(template_file)}'
)
logger.info(f'Task directory is {self.rel_dir(self.task_path)}')
return CmdReturnCodes.SUCCESS.value
def setup(self, template_version: str) -> int:
"""Create template directory and templates."""
# Step 1 - validation
if self.task_name and not self.task_path.exists():
self.task_path.mkdir(exist_ok=True, parents=True)
elif self.task_name and self.task_path.is_file():
raise TrestleError(
f'Task path: {self.rel_dir(self.task_path)} is a file not a directory.'
)
if not self.template_dir.exists():
self.template_dir.mkdir(exist_ok=True, parents=True)
logger.info(
f'Populating template files to {self.rel_dir(self.template_dir)}')
for template in author_const.REFERENCE_TEMPLATES.values():
destination_path = self.template_dir / template
TemplateVersioning.write_versioned_template(
template, self.template_dir, destination_path,
template_version)
logger.info(
f'Template directory populated {self.rel_dir(destination_path)}'
)
return CmdReturnCodes.SUCCESS.value
def test_template_versioning(tmp_path: pathlib.Path) -> None:
"""Full on testing of updating the template folder structure and retrieving it back."""
task_path = tmp_path.joinpath('trestle/author/sample_task/')
task_path.mkdir(parents=True)
task_path.joinpath('0.0.1').mkdir(parents=True)
task_path.joinpath('0.0.2').mkdir(parents=True)
template = task_path.joinpath('0.0.2/template.md')
template2 = task_path.joinpath('0.0.2/template2.md')
old_template = task_path.joinpath('old_template.md')
old_template.touch()
TemplateVersioning.update_template_folder_structure(task_path)
TemplateVersioning.write_versioned_template('template.md',
task_path.joinpath('0.0.2'),
template, '0.0.2')
TemplateVersioning.write_versioned_template('template.md',
task_path.joinpath('0.0.2'),
template2, None)
v1_dir = TemplateVersioning.get_versioned_template_dir(
task_path, START_TEMPLATE_VERSION)
latest_dir, version = TemplateVersioning.get_latest_version_for_task(
task_path)
v2_dir = TemplateVersioning.get_versioned_template_dir(task_path, '0.0.2')
assert not old_template.exists()
assert task_path.joinpath(START_TEMPLATE_VERSION).joinpath(
'old_template.md').exists()
assert task_path.joinpath('0.0.2').joinpath('template.md').exists()
assert task_path.joinpath('0.0.2').joinpath('template2.md').exists()
assert v1_dir.exists()
assert v2_dir == latest_dir
assert version == '0.0.2'
def test_get_versioned_template(tmp_path: pathlib.Path) -> None:
"""Test get template of the specified version."""
task_path = tmp_path.joinpath('trestle/author/sample_task/')
with pytest.raises(TrestleError):
TemplateVersioning.get_versioned_template_dir(task_path)
task_path.mkdir(parents=True)
with pytest.raises(TrestleError):
TemplateVersioning.get_versioned_template_dir(task_path)
v1_dir = task_path.joinpath(START_TEMPLATE_VERSION)
v2_dir = task_path.joinpath('0.1.5/')
v3_dir = task_path.joinpath('10.02.1/')
v1_dir.mkdir(parents=True)
v2_dir.mkdir(parents=True)
v3_dir.mkdir(parents=True)
first_version = TemplateVersioning.get_versioned_template_dir(
task_path, START_TEMPLATE_VERSION)
assert first_version == v1_dir
latest_version = TemplateVersioning.get_versioned_template_dir(task_path)
assert latest_version == v3_dir
second_version = TemplateVersioning.get_versioned_template_dir(
task_path, '0.1.5')
assert second_version == v2_dir
with pytest.raises(TrestleError):
TemplateVersioning.get_versioned_template_dir(task_path, '6.7.8')
def _setup_template_dir(self, args: argparse.Namespace) -> int:
"""Set template directory and update to new format."""
if not self.global_ and self.task_name is None:
logger.error(
'At least a global flag or a task name should be provided.')
return CmdReturnCodes.INCORRECT_ARGS.value
if self.global_:
old_template_dir = self.trestle_root / TRESTLE_CONFIG_DIR / 'author' / '__global__'
self._set_template_version_to_latest(args, old_template_dir)
self.template_dir = old_template_dir / args.template_version
elif self.task_name and not self.global_:
old_template_dir = self.trestle_root / TRESTLE_CONFIG_DIR / 'author' / self.task_name
self._set_template_version_to_latest(args, old_template_dir)
self.template_dir = old_template_dir / args.template_version
if old_template_dir.exists():
TemplateVersioning.validate_template_folder(old_template_dir)
TemplateVersioning.update_template_folder_structure(
old_template_dir)
return CmdReturnCodes.SUCCESS.value
def setup_template(self, template_version: str) -> int:
"""Create structure to allow markdown template enforcement."""
if not self.task_path.exists():
self.task_path.mkdir(exist_ok=True, parents=True)
elif self.task_path.is_file():
raise TrestleError(
f'Task path: {self.rel_dir(self.task_path)} is a file not a directory.'
)
if not self.template_dir.exists():
self.template_dir.mkdir(exist_ok=True, parents=True)
elif self.template_dir.is_file():
raise TrestleError(
f'Template path: {self.rel_dir(self.template_dir)} is a file not a directory.'
)
template_file_a_md = self.template_dir / 'a_template.md'
template_file_another_md = self.template_dir / 'another_template.md'
template_file_drawio = self.template_dir / 'architecture.drawio'
TemplateVersioning.write_versioned_template('template.md',
self.template_dir,
template_file_a_md,
template_version)
TemplateVersioning.write_versioned_template('template.md',
self.template_dir,
template_file_another_md,
template_version)
TemplateVersioning.write_versioned_template('template.drawio',
self.template_dir,
template_file_drawio,
template_version)
return CmdReturnCodes.SUCCESS.value
def test_valid_version() -> None:
"""Test is valid verion."""
assert TemplateVersioning.is_valid_version('0.0.1')
assert not TemplateVersioning.is_valid_version("'0.0.1'")
assert not TemplateVersioning.is_valid_version('not_valid.0.0.1')
assert not TemplateVersioning.is_valid_version('0.1')
assert not TemplateVersioning.is_valid_version('1')
assert not TemplateVersioning.is_valid_version('0.0.0.1')
def _set_template_version_to_latest(self, args: argparse.Namespace,
template_dir: pathlib.Path):
"""Set template version argument to the latest version if none was given."""
if not TemplateVersioning.is_valid_version(args.template_version):
raise TrestleError(
f'Version {args.template_version} is invalid, version format should be: 0.0.1'
)
if args.template_version is None and args.mode == ARG_VALIDATE:
# in validate mode no version will validate instances based on header version
args.template_version = ''
if args.template_version is None:
args.template_version = START_TEMPLATE_VERSION
if template_dir.exists():
all_versions = TemplateVersioning.get_all_versions_for_task(
template_dir)
if all_versions:
args.template_version = max(all_versions)
if args.template_version == '':
logger.info(
'Instances will be validated against template version specified in their headers.'
)
else:
logger.info(f'Set template version to {args.template_version}.')
def test_template_version_folder_update(tmp_path: pathlib.Path) -> None:
"""Test template folder update from old structure to new."""
task_path = tmp_path.joinpath('trestle/author/sample_task/')
task_path.mkdir(parents=True)
old_template = task_path.joinpath('template.md')
old_template.touch()
with pytest.raises(TrestleError):
TemplateVersioning.update_template_folder_structure(old_template)
# make sure file gets put to first version
TemplateVersioning.update_template_folder_structure(task_path)
assert task_path.joinpath('0.0.1/').joinpath('template.md').exists()
assert not old_template.exists()
# make sure new file also gets put to the first version
another_template = task_path.joinpath('missplaced_template.md')
another_template.touch()
TemplateVersioning.update_template_folder_structure(task_path)
assert task_path.joinpath('0.0.1/').joinpath('template.md').exists()
assert task_path.joinpath('0.0.1/').joinpath(
'missplaced_template.md').exists()
assert not another_template.exists()
# make sure other versions get no modifications
v2_dir = task_path.joinpath('11.12.113')
v2_dir.mkdir()
yet_another_template = task_path.joinpath('new_template.md')
yet_another_template.touch()
TemplateVersioning.update_template_folder_structure(task_path)
assert len(list(v2_dir.iterdir())) == 0
assert task_path.joinpath('0.0.1/').joinpath('template.md').exists()
assert task_path.joinpath('0.0.1/').joinpath(
'missplaced_template.md').exists()
assert task_path.joinpath('0.0.1/').joinpath('new_template.md').exists()
assert not yet_another_template.exists()
def test_get_latest_version(tmp_path: pathlib.Path) -> None:
"""Test get latest version of template."""
task_path = tmp_path.joinpath('trestle/author/sample_task/')
with pytest.raises(TrestleError):
TemplateVersioning.get_latest_version_for_task(task_path)
task_path.mkdir(parents=True)
latest_path, version = TemplateVersioning.get_latest_version_for_task(
task_path)
assert latest_path == task_path.joinpath('0.0.1')
assert version == '0.0.1'
v1_dir = task_path.joinpath('0.0.1/')
v2_dir = task_path.joinpath('11.10.1234/')
v1_dir.mkdir(parents=True)
v2_dir.mkdir(parents=True)
latest_path, version = TemplateVersioning.get_latest_version_for_task(
task_path)
assert latest_path == v2_dir
assert version == '11.10.1234'
template_v1 = v1_dir.joinpath('template.md')
template_v2 = v2_dir.joinpath('template.md')
template_v1.touch()
template_v2.touch()
latest_path, version = TemplateVersioning.get_latest_version_for_task(
task_path)
assert latest_path == v2_dir
assert version == '11.10.1234'
with pytest.raises(TrestleError):
TemplateVersioning.get_latest_version_for_task(template_v1)
def _validate_dir(self, candidate_dir: pathlib.Path, recurse: bool,
readme_validate: bool,
relative_exclusions: List[pathlib.Path],
template_version: str, ignore: str) -> bool:
"""Validate a directory within the trestle project."""
all_versioned_templates = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
instances = list(candidate_dir.iterdir())
if recurse:
instances = candidate_dir.rglob('*')
if ignore:
p = re.compile(ignore)
instances = list(
filter(
lambda f: len(
list(
filter(
p.match,
str(f.relative_to(candidate_dir)).split(
'/')))) == 0, instances))
for instance_file in instances:
if not file_utils.is_local_and_visible(instance_file):
continue
if instance_file.name.lower(
) == 'readme.md' and not readme_validate:
continue
if instance_file.is_dir() and not recurse:
continue
if any(
str(ex) in str(instance_file)
for ex in relative_exclusions):
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(candidate_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == '.md':
md_api = MarkdownAPI()
versioned_template_dir = None
if template_version != '':
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = {}
all_versioned_templates[instance_version]['drawio'] = list(
filter(lambda p: p.suffix == '.drawio', templates))[0]
all_versioned_templates[instance_version]['md'] = list(
filter(lambda p: p.suffix == '.md', templates))[0]
# validate
md_api.load_validator_with_template(
all_versioned_templates[instance_version]['md'], True,
False)
status = md_api.validate_instance(instance_file)
if not status:
logger.info(f'INVALID: {self.rel_dir(instance_file)}')
return False
else:
logger.info(f'VALID: {self.rel_dir(instance_file)}')
elif instance_file.suffix == '.drawio':
drawio = DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[
author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = {}
all_versioned_templates[instance_version]['drawio'] = list(
filter(lambda p: p.suffix == '.drawio', templates))[0]
all_versioned_templates[instance_version]['md'] = list(
filter(lambda p: p.suffix == '.md', templates))[0]
# validate
drawio_validator = DrawIOMetadataValidator(
all_versioned_templates[instance_version]['drawio'])
status = drawio_validator.validate(instance_file)
if not status:
logger.info(f'INVALID: {self.rel_dir(instance_file)}')
return False
else:
logger.info(f'VALID: {self.rel_dir(instance_file)}')
else:
logger.debug(
f'Unsupported extension of the instance file: {instance_file}, will not be validated.'
)
return True
def _validate_dir(self,
governed_heading: str,
md_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
recurse: bool,
readme_validate: bool,
template_version: Optional[str] = None,
ignore: Optional[str] = None) -> int:
"""
Validate md files in a directory with option to recurse.
Template version will be fetched from the instance header.
"""
# status is a linux returncode
status = 0
for item_path in md_dir.iterdir():
if file_utils.is_local_and_visible(item_path):
if item_path.is_file():
if not item_path.suffix == '.md':
logger.info(
f'Unexpected file {self.rel_dir(item_path)} in folder {self.rel_dir(md_dir)}, skipping.'
)
continue
if not readme_validate and item_path.name.lower(
) == 'readme.md':
continue
if ignore:
p = re.compile(ignore)
matched = p.match(item_path.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {item_path} from validation.')
continue
md_api = MarkdownAPI()
if template_version != '':
template_file = self.template_dir / self.template_name
else:
instance_version = md_api.processor.fetch_value_from_header(
item_path, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1'
versione_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versione_template_dir / self.template_name
if not template_file.is_file():
raise TrestleError(
f'Required template file: {self.rel_dir(template_file)} does not exist. Exiting.'
)
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, governed_heading)
if not md_api.validate_instance(item_path):
logger.info(f'INVALID: {self.rel_dir(item_path)}')
status = 1
else:
logger.info(f'VALID: {self.rel_dir(item_path)}')
elif recurse:
if ignore:
p = re.compile(ignore)
if len(
list(
filter(
p.match,
str(item_path.relative_to(
md_dir)).split('/')))) > 0:
logger.info(
f'Ignoring directory {item_path} from validation.'
)
continue
rc = self._validate_dir(governed_heading, item_path,
validate_header,
validate_only_header, recurse,
readme_validate, template_version,
ignore)
if rc != 0:
status = rc
return status
def _measure_template_folder(self, instance_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
governed_heading: str, readme_validate: bool,
template_version: str, ignore: str) -> bool:
"""
Validate instances against templates.
Validation will succeed iff:
1. All template files from the specified version are present in the task
2. All of the instances are valid
"""
all_versioned_templates = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
for instance_file in instance_dir.iterdir():
if not file_utils.is_local_and_visible(instance_file):
continue
if not instance_file.is_file():
continue
if instance_file.name.lower(
) == 'readme.md' and not readme_validate:
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(
f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(instance_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == '.md':
md_api = MarkdownAPI()
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = dict.fromkeys([
t.relative_to(versioned_template_dir)
for t in templates
], False)
if instance_file_name in all_versioned_templates[
instance_version]:
# validate
md_api.load_validator_with_template(
template_file, validate_header,
not validate_only_header, governed_heading)
status = md_api.validate_instance(instance_file)
if not status:
logger.warning(
f'INVALID: Markdown file {instance_file} failed validation against'
+ f' {template_file}')
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][
instance_file_name] = True
elif instance_file.suffix == '.drawio':
drawio = draw_io.DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[
author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = list(
filter(lambda p: file_utils.is_local_and_visible(p),
versioned_template_dir.iterdir()))
if not readme_validate:
templates = list(
filter(lambda p: p.name.lower() != 'readme.md',
templates))
all_versioned_templates[instance_version] = dict.fromkeys([
t.relative_to(versioned_template_dir)
for t in templates
], False)
if instance_file_name in all_versioned_templates[
instance_version]:
# validate
drawio_validator = draw_io.DrawIOMetadataValidator(
template_file)
status = drawio_validator.validate(instance_file)
if not status:
logger.warning(
f'INVALID: Drawio file {instance_file} failed validation against'
+ f' {template_file}')
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][
instance_file_name] = True
else:
logger.debug(
f'Unsupported extension of the instance file: {instance_file}, will not be validated.'
)
# Check that all template files are present
for version in all_versioned_templates.keys():
for template in all_versioned_templates[version]:
if not all_versioned_templates[version][template]:
logger.warning(
f'Required template file {template} does not exist in measured instance'
+ f'{instance_dir}')
return False
return True
def test_write_versioned_template(tmp_path: pathlib.Path) -> None:
"""Test writing a template to the folder."""
task_path = tmp_path.joinpath('trestle/author/sample_task/')
with pytest.raises(TrestleError):
TemplateVersioning.get_versioned_template_dir(task_path)
task_path.mkdir(parents=True)
tmp_path_01 = task_path.joinpath('0.0.1')
tmp_path_02 = task_path.joinpath('0.0.2')
tmp_path_01.mkdir(parents=True)
tmp_path_02.mkdir(parents=True)
template = tmp_path_01.joinpath('template.md')
TemplateVersioning.write_versioned_template('template.md', tmp_path_01,
template, None)
assert task_path.joinpath(START_TEMPLATE_VERSION).joinpath(
'template.md').exists()
assert template.exists()
template2 = tmp_path_01.joinpath('template2.md')
template3 = tmp_path_02.joinpath('template3.md')
template4 = tmp_path_02.joinpath('template4.md')
TemplateVersioning.write_versioned_template('template.md', tmp_path_01,
template2, '0.0.1')
TemplateVersioning.write_versioned_template('template.md', tmp_path_02,
template3, '0.0.2')
TemplateVersioning.write_versioned_template('template.md', tmp_path_02,
template4, None)
assert tmp_path_01.joinpath('template.md').exists()
assert tmp_path_01.joinpath('template2.md').exists()
assert tmp_path_02.joinpath('template3.md').exists()
assert tmp_path_02.joinpath('template4.md').exists()
assert template.exists()
md_api = MarkdownAPI()
header, _ = md_api.processor.read_markdown_wo_processing(
tmp_path_02.joinpath('template3.md'))
assert header[TEMPLATE_VERSION_HEADER] == '0.0.2'
template_drawio = tmp_path_02.joinpath('template.drawio')
TemplateVersioning.write_versioned_template('template.drawio', tmp_path_02,
template_drawio, '0.0.2')
drawio = DrawIO(template_drawio)
metadata = drawio.get_metadata()[0]
assert metadata[TEMPLATE_VERSION_HEADER] == '0.0.2'