def test_write_control_header_params(overwrite_header_values,
tmp_path: pathlib.Path) -> None:
"""Test write/read of control header params."""
# orig file just has one param ac-1_prm_3
src_control_path = pathlib.Path(
'tests/data/author/controls/control_with_components_and_params.md')
# header has two params - 3 and 4
header = {
const.SET_PARAMS_TAG: {
'ac-1_prm_3': {
'values': 'new prm_3 val from input header'
},
'ac-1_prm_4': {
'values': 'new prm_4 val from input header'
}
},
'foo': 'new bar',
'new-reviewer': 'James',
'special': 'new value to ignore',
'none-thing': 'none value to ignore'
}
control_path = tmp_path / 'ac-1.md'
shutil.copyfile(src_control_path, control_path)
markdown_processor = MarkdownProcessor()
# header_1 should have one param: 3
header_1, _ = markdown_processor.read_markdown_wo_processing(control_path)
assert len(header_1.keys()) == 8
orig_control_read, group_title = ControlIOReader.read_control(
control_path, True)
assert group_title == 'Access Control'
control_writer = ControlIOWriter()
# write the control back out with the test header
control_writer.write_control(tmp_path, orig_control_read, group_title,
header, None, False, False, None,
overwrite_header_values, None, None)
# header_2 should have 2 params: 3 and 4
header_2, _ = markdown_processor.read_markdown_wo_processing(control_path)
assert len(header_2.keys()) == 9
assert header_2['new-reviewer'] == 'James'
assert len(header_2[const.SET_PARAMS_TAG]) == 2
assert 'new' in header_2[const.SET_PARAMS_TAG]['ac-1_prm_4']['values']
if not overwrite_header_values:
assert 'orig' in header_2[const.SET_PARAMS_TAG]['ac-1_prm_3']['values']
assert header_2['foo'] == 'bar'
assert header_2['special'] == ''
assert header_2['none-thing'] is None
else:
assert 'new' in header_2[const.SET_PARAMS_TAG]['ac-1_prm_3']['values']
assert header_2['foo'] == 'new bar'
assert header_2['special'] == 'new value to ignore'
assert header_2['none-thing'] == 'none value to ignore'
assert 'orig' in orig_control_read.params[0].values[0].__root__
new_control_read, _ = ControlIOReader.read_control(control_path, True)
# insert the new param in the orig control so we can compare the two controls
orig_control_read.params.append(new_control_read.params[1])
if overwrite_header_values:
orig_control_read.params[0] = new_control_read.params[0]
assert test_utils.controls_equivalent(orig_control_read, new_control_read)
def _parameter_table(self, control_id: str, level: int) -> str:
"""Print Param_id | Default (aka label) | Value or set to 'none'."""
if not self._ssp:
raise TrestleError('Cannot get parameter table, set SSP first.')
writer = ControlIOWriter()
control = self._catalog_interface.get_control(control_id)
if not control:
return ''
params_lines = writer.get_params(control)
tree = MarkdownNode.build_tree_from_markdown(params_lines)
tree.change_header_level_by(level)
return tree.content.raw_text
def get_statement_label_if_exists(self, control_id: str,
statement_id: str) -> Tuple[Optional[str], Optional[common.Part]]:
"""Get statement label if given."""
def does_part_exists(part: common.Part) -> bool:
does_match = False
if part.name and part.name in {'statement', 'item'} and part.id == statement_id:
does_match = True
return does_match
control = self.get_control(control_id)
if not control:
return '', None
label = None
found_part = None
if control.parts:
for part in as_list(control.parts):
# Performance OSCAL assumption, ids are nested so recurse only if prefix
if part.id and statement_id.startswith(part.id):
part = self.find_part_with_condition(part, does_part_exists)
if part:
label = ControlIOWriter.get_label(part)
found_part = part
break
return label, found_part
def delete_withdrawn_controls(self) -> None:
"""Delete all withdrawn controls from the catalog."""
delete_list = []
for control in self.get_all_controls_from_dict():
if ControlIOWriter.is_withdrawn(control):
delete_list.append(control.id)
for id_ in delete_list:
self.delete_control(id_)
def test_control_objective(tmp_path: pathlib.Path) -> None:
"""Test read and write of control with objective."""
# write the control directly as raw markdown text
md_path = tmp_path / 'xy-9.md'
with open(md_path, 'w') as f:
f.write(control_text)
# read it in as markdown to an OSCAL control in memory
control, group_title = ControlIOReader.read_control(md_path, True)
assert group_title == 'My Group Title'
sub_dir = tmp_path / 'sub_dir'
sub_dir.mkdir(exist_ok=True)
# write it out as markdown in a separate directory to avoid name clash
control_writer = ControlIOWriter()
control_writer.write_control(sub_dir, control, 'My Group Title', None,
None, False, False, None, False, None, None)
# confirm the newly written markdown text is identical to what was read originally
assert test_utils.text_files_equal(md_path, sub_dir / 'xy-9.md')
def get_control_part_prose(self, control_id: str, part_name: str) -> str:
"""
Get the prose for a named part in the control.
Args:
control_id: id of the control
part_name: name of the part
Returns:
Single string concatenating prose from all parts and sub-parts in control with that name.
"""
control = self.get_control(control_id)
return ControlIOWriter.get_part_prose(control, part_name)
def get_control_statement(self, control_id: str, level: int) -> str:
"""
Get the control statement for an ssp - to be printed in markdown as a structured list.
Args:
control_id: The control_id to use.
Returns:
A markdown blob as a string.
"""
if not self._resolved_catalog:
raise TrestleError(
'Cannot get control statement, set resolved catalog first.')
writer = ControlIOWriter()
control = self._catalog_interface.get_control(control_id)
if not control:
return ''
control_lines = writer.get_control_statement(control)
return self._build_tree_and_adjust(control_lines, level)
def test_merge_dicts_deep(overwrite_header_values) -> None:
"""Test deep merge of dicts."""
dest = {
'trestle': {
'foo': {
'hello': 1
}
},
'fedramp': {
'roles': [5, 6],
'values': 8
},
'orig': 11
}
src = {
'trestle': {
'foo': {
'hello': 3
},
'bar': 4
},
'fedramp': {
'roles': 7,
'values': 10
},
'extra': 12
}
ControlIOWriter.merge_dicts_deep(dest, src, overwrite_header_values)
if not overwrite_header_values:
assert dest['trestle'] == {'foo': {'hello': 1}, 'bar': 4}
assert dest['fedramp'] == {'roles': [5, 6], 'values': 8}
assert dest['orig'] == 11
assert dest['extra'] == 12
else:
assert dest['trestle'] == {'foo': {'hello': 3}, 'bar': 4}
assert dest['fedramp'] == {'roles': 7, 'values': 10}
assert dest['orig'] == 11
assert dest['extra'] == 12
def test_control_failures(tmp_path: pathlib.Path) -> None:
"""Test various failure modes."""
part = common.Part(name='foo')
assert ControlIOWriter.get_label(part) == ''
assert ControlIOReader._strip_to_make_ncname('1a@foo') == 'afoo'
with pytest.raises(TrestleError):
ControlIOReader._strip_to_make_ncname('1@')
with pytest.raises(TrestleError):
ControlIOReader._indent('')
with pytest.raises(TrestleError):
ControlIOReader._indent(' foo')
def test_merge_dicts_deep_empty() -> None:
"""Test that empty items are left alone."""
dest = {'foo': ''}
src = {'foo': 'fancy value'}
ControlIOWriter.merge_dicts_deep(dest, src, False)
assert dest['foo'] == ''
dest['foo'] = None
ControlIOWriter.merge_dicts_deep(dest, src, False)
assert dest['foo'] is None
ControlIOWriter.merge_dicts_deep(dest, src, True)
assert dest['foo'] == 'fancy value'
def write_catalog_as_markdown(
self,
md_path: pathlib.Path,
yaml_header: dict,
sections_dict: Optional[Dict[str, str]],
prompt_responses: bool,
additional_content: bool = False,
profile: Optional[prof.Profile] = None,
overwrite_header_values: bool = False,
set_parameters: bool = False,
required_sections: Optional[str] = None,
allowed_sections: Optional[str] = None
) -> None:
"""
Write out the catalog controls from dict as markdown files to the specified directory.
Args:
md_path: Path to directory in which to write the markdown
yaml_header: Dictionary to write into the yaml header of the controls
sections_dict: Optional dict mapping section short names to long
prompt_responses: Whether to prompt for responses in the control markdown
additional_content: Should the additional content be printed corresponding to profile adds
profile: Optional profile containing the adds making up additional content
overwrite_header_values: Overwrite existing values in markdown header content but add new content
set_parameters: Set header values based on params in the control and in the profile
required_sections: Optional string containing list of sections that should be prompted for prose
allowed_sections: Optional string containing list of sections that should be included in markdown
Returns:
None
"""
writer = ControlIOWriter()
required_section_list = required_sections.split(',') if required_sections else []
allowed_section_list = allowed_sections.split(',') if allowed_sections else []
# create the directory in which to write the control markdown files
md_path.mkdir(exist_ok=True, parents=True)
catalog_interface = CatalogInterface(self._catalog)
# get the list of SetParams for this profile
full_profile_param_dict = CatalogInterface._get_full_profile_param_dict(profile) if profile else {}
# write out the controls
for control in catalog_interface.get_all_controls_from_catalog(True):
# make copy of incoming yaml header
new_header = copy.deepcopy(yaml_header)
# here we do special handling of how set-parameters merge with the yaml header
if set_parameters:
# get all params for this control
control_param_dict = ControlIOReader.get_control_param_dict(control, False)
set_param_dict: Dict[str, str] = {}
for param_id, param_dict in control_param_dict.items():
# if the param is in the profile set_params, load its contents first and mark as profile-values
if param_id in full_profile_param_dict:
# get the param from the profile set_param
param = full_profile_param_dict[param_id]
# assign its contents to the dict
new_dict = ModelUtils.parameter_to_dict(param, True)
profile_values = new_dict.get(const.VALUES, None)
if profile_values:
new_dict[const.PROFILE_VALUES] = profile_values
new_dict.pop(const.VALUES)
# then insert the original, incoming values as values
if param_id in control_param_dict:
orig_param = control_param_dict[param_id]
orig_dict = ModelUtils.parameter_to_dict(orig_param, True)
new_dict[const.VALUES] = orig_dict.get(const.VALUES, None)
# merge contents from the two sources with priority to the profile-param
for item in ['select', 'label']:
if item in orig_dict and item not in new_dict:
new_dict[item] = orig_dict[item]
else:
new_dict = ModelUtils.parameter_to_dict(param_dict, True)
new_dict.pop('id')
set_param_dict[param_id] = new_dict
if set_param_dict:
if const.SET_PARAMS_TAG not in new_header:
new_header[const.SET_PARAMS_TAG] = {}
if overwrite_header_values:
# update the control params with new values
for key, value in new_header[const.SET_PARAMS_TAG].items():
if key in control_param_dict:
set_param_dict[key] = value
else:
# update the control params with any values in yaml header not set in control
# need to maintain order in the set_param_dict
for key, value in new_header[const.SET_PARAMS_TAG].items():
if key in control_param_dict and key not in set_param_dict:
set_param_dict[key] = value
new_header[const.SET_PARAMS_TAG] = set_param_dict
elif const.SET_PARAMS_TAG in new_header:
# need to cull any params that are not in control
pop_list: List[str] = []
for key in new_header[const.SET_PARAMS_TAG].keys():
if key not in control_param_dict:
pop_list.append(key)
for pop in pop_list:
new_header[const.SET_PARAMS_TAG].pop(pop)
_, group_title, _ = catalog_interface.get_group_info_by_control(control.id)
# control could be in sub-group of group so build path to it
group_dir = md_path
control_path = catalog_interface._get_control_path(control.id)
for sub_dir in control_path:
group_dir = group_dir / sub_dir
if not group_dir.exists():
group_dir.mkdir(parents=True, exist_ok=True)
writer.write_control(
group_dir,
control,
group_title,
new_header,
sections_dict,
additional_content,
prompt_responses,
profile,
overwrite_header_values,
required_section_list,
allowed_section_list
)
def jinja_ify(
trestle_root: pathlib.Path,
r_input_file: pathlib.Path,
r_output_file: pathlib.Path,
ssp: Optional[str],
profile: Optional[str],
lut: Optional[Dict[str, Any]] = None,
number_captions: Optional[bool] = False,
parameters_formatting: Optional[str] = None
) -> int:
"""Run jinja over an input file with additional booleans."""
if lut is None:
lut = {}
template_folder = pathlib.Path.cwd()
jinja_env = Environment(
loader=FileSystemLoader(template_folder),
extensions=[MDSectionInclude, MDCleanInclude, MDDatestamp],
trim_blocks=True,
autoescape=True
)
template = jinja_env.get_template(str(r_input_file))
# create boolean dict
if operator.xor(bool(ssp), bool(profile)):
raise TrestleIncorrectArgsError('Both SSP and profile should be provided or not at all')
if ssp:
# name lookup
ssp_data, _ = ModelUtils.load_top_level_model(trestle_root, ssp, SystemSecurityPlan)
lut['ssp'] = ssp_data
_, profile_path = ModelUtils.load_top_level_model(trestle_root, profile, Profile)
profile_resolver = ProfileResolver()
resolved_catalog = profile_resolver.get_resolved_profile_catalog(
trestle_root, profile_path, False, False, parameters_formatting
)
ssp_writer = SSPMarkdownWriter(trestle_root)
ssp_writer.set_ssp(ssp_data)
ssp_writer.set_catalog(resolved_catalog)
lut['catalog'] = resolved_catalog
lut['catalog_interface'] = CatalogInterface(resolved_catalog)
lut['control_io_writer'] = ControlIOWriter()
lut['ssp_md_writer'] = ssp_writer
new_output = template.render(**lut)
output = ''
# This recursion allows nesting within expressions (e.g. an expression can contain jinja templates).
error_countdown = JinjaCmd.max_recursion_depth
while new_output != output and error_countdown > 0:
error_countdown = error_countdown - 1
output = new_output
random_name = uuid.uuid4() # Should be random and not used.
dict_loader = DictLoader({str(random_name): new_output})
jinja_env = Environment(
loader=ChoiceLoader([dict_loader, FileSystemLoader(template_folder)]),
extensions=[MDCleanInclude, MDSectionInclude, MDDatestamp],
autoescape=True,
trim_blocks=True
)
template = jinja_env.get_template(str(random_name))
new_output = template.render(**lut)
output_file = trestle_root / r_output_file
if number_captions:
output_file.open('w', encoding=const.FILE_ENCODING).write(_number_captions(output))
else:
output_file.open('w', encoding=const.FILE_ENCODING).write(output)
return CmdReturnCodes.SUCCESS.value
part_b2.parts = [part_b2i]
part_b.parts = [part_b1, part_b2]
parts = [part_a, part_b, part_c]
elif case == case_3:
part_b2.parts = [part_b2i]
part_b.parts = [part_b1, part_b2]
parts = [part_a, part_b]
else:
parts = None
statement_part.parts = parts
control.parts = [statement_part]
if sections:
control.parts.extend([sec_1, sec_2])
writer = ControlIOWriter()
writer.write_control(tmp_path, control, 'My Group Title', None, None,
additional_content, False, None, False, None, None)
md_path = tmp_path / f'{control.id}.md'
reader = ControlIOReader()
new_control, group_title = reader.read_control(md_path, False)
new_control.title = dummy_title
assert group_title == 'My Group Title'
assert len(new_control.parts) == len(control.parts)
assert control.parts[0].prose == new_control.parts[0].prose
assert control.parts[0].parts == new_control.parts[0].parts
assert control == new_control
def test_control_objective(tmp_path: pathlib.Path) -> None: