def test_get_control_param_dict(tmp_trestle_dir: pathlib.Path) -> None:
"""Test getting the param dict of a control."""
test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True)
prof_a_path = ModelUtils.path_for_top_level_model(tmp_trestle_dir,
'test_profile_a',
prof.Profile,
FileContentType.JSON)
catalog = ProfileResolver.get_resolved_profile_catalog(
tmp_trestle_dir, prof_a_path)
catalog_interface = CatalogInterface(catalog)
control = catalog_interface.get_control('ac-1')
param_dict = ControlIOReader.get_control_param_dict(control, False)
# confirm profile value is used
assert ControlIOReader.param_values_as_str(
param_dict['ac-1_prm_1']) == 'all alert personnel'
# confirm original param label is used since no value was assigned
assert ControlIOReader.param_to_str(param_dict['ac-1_prm_7'],
ParameterRep.VALUE_OR_LABEL_OR_CHOICES
) == 'organization-defined events'
param = control.params[0]
param.values = None
param.select = common.ParameterSelection(
how_many=common.HowMany.one_or_more, choice=['choice 1', 'choice 2'])
param_dict = ControlIOReader.get_control_param_dict(control, False)
assert ControlIOReader.param_to_str(
param_dict['ac-1_prm_1'],
ParameterRep.VALUE_OR_LABEL_OR_CHOICES) == 'choice 1, choice 2'
def generate_markdown(self, trestle_root: pathlib.Path,
profile_path: pathlib.Path,
markdown_path: pathlib.Path, yaml_header: dict,
overwrite_header_values: bool,
sections_dict: Optional[Dict[str, str]],
required_sections: Optional[str]) -> int:
"""Generate markdown for the controls in the profile.
Args:
trestle_root: Root directory of the trestle workspace
profile_path: Path of the profile json file
markdown_path: Path to the directory into which the markdown will be written
yaml_header: Dict to merge into the yaml header of the control markdown
overwrite_header_values: Overwrite values in the markdown header but allow new items to be added
sections_dict: Optional dict mapping section short names to long
required_sections: Optional comma-sep list of sections that get prompted for prose if not in the profile
Returns:
0 on success, 1 on error
"""
try:
if sections_dict and 'statement' in sections_dict:
logger.warning('statement is not allowed as a section name.')
return CmdReturnCodes.COMMAND_ERROR.value
_, _, profile = ModelUtils.load_distributed(
profile_path, trestle_root)
catalog = ProfileResolver().get_resolved_profile_catalog(
trestle_root, profile_path, True, True, None,
ParameterRep.LEAVE_MOUSTACHE)
catalog_interface = CatalogInterface(catalog)
catalog_interface.write_catalog_as_markdown(
md_path=markdown_path,
yaml_header=yaml_header,
sections_dict=sections_dict,
prompt_responses=False,
additional_content=True,
profile=profile,
overwrite_header_values=overwrite_header_values,
set_parameters=True,
required_sections=required_sections,
allowed_sections=None)
except TrestleNotFoundError as e:
raise TrestleError(f'Profile {profile_path} not found, error {e}')
except TrestleError as e:
raise TrestleError(
f'Error generating the catalog as markdown: {e}')
return CmdReturnCodes.SUCCESS.value
def test_get_control_and_group_info_from_catalog(tmp_trestle_dir: pathlib.Path) -> None:
"""Test get all groups from the catalog."""
test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True)
prof_a_path = ModelUtils.path_for_top_level_model(
tmp_trestle_dir, 'test_profile_a', prof.Profile, FileContentType.JSON
)
catalog = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_a_path)
cat_interface = CatalogInterface(catalog)
all_groups_top = cat_interface.get_all_controls_from_catalog(recurse=False)
assert len(list(all_groups_top)) == 6
all_groups_rec = cat_interface.get_all_controls_from_catalog(recurse=True)
assert len(list(all_groups_rec)) == 7
all_group_ids = cat_interface.get_group_ids()
assert len(all_group_ids) == 1
statement_label, part = cat_interface.get_statement_label_if_exists('ac-1', 'ac-1_smt.c.2')
assert statement_label == '2.'
assert part.id == 'ac-1_smt.c.2'
cat_path = cat_interface._get_control_path('ac-2')
assert cat_path[0] == 'ac'
assert len(cat_path) == 1
def test_parameter_resolution(tmp_trestle_dir: pathlib.Path) -> None:
"""Test whether expected order of operations is preserved for parameter substution."""
test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True)
prof_e_path = ModelUtils.path_for_top_level_model(
tmp_trestle_dir, 'test_profile_e', prof.Profile, FileContentType.JSON
)
profile_e_parameter_string = '## Override value ##'
profile_a_value = 'all alert personnel'
# based on 800-53 rev 5
cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_e_path)
interface = CatalogInterface(cat)
control = interface.get_control('ac-1')
locations = interface.find_string_in_control(control, profile_e_parameter_string)
locations_a = interface.find_string_in_control(control, profile_a_value)
assert len(locations) == 1
assert len(locations_a) == 0
assert len(control.params[1].constraints) == 1
def test_profile_resolver(tmp_trestle_dir: pathlib.Path) -> None:
"""Test the resolver."""
test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True)
prof_a_path = ModelUtils.path_for_top_level_model(
tmp_trestle_dir, 'test_profile_a', prof.Profile, FileContentType.JSON
)
cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_a_path)
interface = CatalogInterface(cat)
# added part ac-1_expevid from prof a
list1 = find_string_in_all_controls_prose(interface, 'Detailed evidence logs')
# modify param ac-3.3_prm_2 in prof b
list2 = find_string_in_all_controls_prose(interface, 'full and complete compliance')
assert len(list1) == 1
assert len(list2) == 1
assert interface.get_count_of_controls_in_catalog(False) == 6
assert interface.get_count_of_controls_in_catalog(True) == 7
assert len(cat.controls) == 4
assert interface.get_dependent_control_ids('ac-3') == ['ac-3.3']
control = interface.get_control('a-1')
assert control.parts[0].parts[0].id == 'a-1_deep'
assert control.parts[0].parts[0].prose == 'Extra added part in subpart'
def test_add_props(tmp_trestle_dir: pathlib.Path) -> None:
"""Test all types of property additions."""
test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True)
prof_f_path = ModelUtils.path_for_top_level_model(
tmp_trestle_dir, 'test_profile_f', prof.Profile, FileContentType.JSON
)
cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_f_path)
interface = CatalogInterface(cat)
ac_3 = interface.get_control('ac-3')
assert len(ac_3.props) == 6
assert ac_3.props[-1].value == 'four'
for part in ac_3.parts:
if part.id == 'ac-3_stmt':
assert len(part.props) == 4
ac_5 = interface.get_control('ac-5')
for part in ac_5.parts:
if part.id == 'ac-5_stmt':
for sub_part in part.parts:
if sub_part.id == 'ac-5_smt.a':
assert len(sub_part.props) == 4
def test_all_positions_for_alter_can_be_resolved(tmp_trestle_dir: pathlib.Path) -> None:
"""Test that all alter adds positions can be resolved."""
cat_path = test_utils.JSON_TEST_DATA_PATH / test_utils.SIMPLIFIED_NIST_CATALOG_NAME
repo = Repository(tmp_trestle_dir)
repo.load_and_import_model(cat_path, 'nist_cat')
prof_d_path = test_utils.JSON_TEST_DATA_PATH / 'test_profile_d.json'
repo.load_and_import_model(prof_d_path, 'test_profile_d')
cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_d_path)
interface = CatalogInterface(cat)
control_a1 = interface.get_control('ac-1')
control_a2 = interface.get_control('ac-2')
assert control_a1.parts[0].id == 'ac-1_first_lev1'
assert control_a1.parts[1].parts[3].id == 'ac-1_last_lev2'
assert control_a1.parts[2].id == 'ac-1_after1_ac-1_smt_lev1'
assert control_a1.parts[3].id == 'ac-1_after2_ac-1_smt_lev1'
assert control_a1.parts[1].parts[0].parts[1].id == 'ac-1_smt_before1_a.2_lev3'
assert control_a1.parts[1].parts[0].parts[2].id == 'ac-1_smt_before2_a.2_lev3'
assert control_a1.parts[1].parts[0].parts[3].parts[0].id == 'ac-1_smt_inside1_at_the_end_a.2_lev4'
assert control_a1.parts[1].parts[0].parts[3].parts[1].id == 'ac-1_smt_inside2_at_the_end_a.2_lev4'
assert control_a2.parts[0].id == 'ac-2_implgdn_lev1'
def catalog_interface_equivalent(cat_int_a: CatalogInterface, cat_b: cat.Catalog, strong=True) -> bool:
"""Test equivalence of catalog dict contents in various ways."""
cat_int_b = CatalogInterface(cat_b)
if cat_int_b.get_count_of_controls_in_dict() != cat_int_a.get_count_of_controls_in_dict():
logger.error('count of controls is different')
return False
for a in cat_int_a.get_all_controls_from_dict():
try:
b = cat_int_b.get_control(a.id)
except Exception as e:
logger.error(f'error finding control {a.id} {e}')
if not controls_equivalent(a, b, strong):
logger.error(f'controls differ: {a.id}')
return False
return True
def _modify_controls(self, catalog: cat.Catalog) -> cat.Catalog:
"""Modify the controls based on the profile."""
logger.debug(
f'modify specify catalog {catalog.metadata.title} for profile {self._profile.metadata.title}'
)
self._catalog_interface = CatalogInterface(catalog)
alters: Optional[List[prof.Alter]] = None
# find the modify and alters
if self._profile.modify:
# change all parameter values
if self._profile.modify.set_parameters and not self._block_params:
set_param_list = self._profile.modify.set_parameters
for set_param in set_param_list:
self._set_parameter_in_control(set_param)
alters = self._profile.modify.alters
if alters is not None:
title = self._profile.metadata.title
for alter in alters:
if alter.control_id is None:
logger.warning(
f'Alter must have control id specified in profile {title}.'
)
continue
id_ = alter.control_id
if alter.removes is not None:
logger.warning(
f'Alter not supported for removes in profile {title} control {id_}'
)
continue
# we want a warning about adds even if adds are blocked, as in profile generate
if alter.adds is None:
logger.warning(
f'Alter has no adds in profile {title} control {id_}')
continue
if not self._block_adds:
for add in alter.adds:
if add.position is None and add.parts is not None:
msg = f'Alter/Add position is not specified in profile {title} control {id_}'
msg += ' when adding part, so defaulting to ending.'
logger.warning(msg)
add.position = prof.Position.ending
control = self._catalog_interface.get_control(id_)
if control is None:
logger.warning(
f'Alter/Add refers to control {id_} but it is not found in the import '
+
f'for profile {self._profile.metadata.title}')
else:
self._add_to_control(control, add)
self._catalog_interface.replace_control(control)
if self._change_prose:
# go through all controls and fix the prose based on param values
self._change_prose_with_param_values()
catalog = self._catalog_interface.get_catalog()
# update the original profile metadata with new contents
# roles and responsible-parties will be pulled in with new uuid's
new_metadata = self._profile.metadata
new_metadata.title = f'{catalog.metadata.title}: Resolved by profile {self._profile.metadata.title}'
links: List[common.Link] = []
for import_ in self._profile.imports:
links.append(
common.Link(**{
'href': import_.href,
'rel': 'resolution-source'
}))
new_metadata.links = links
# move catalog controls from dummy group '' into the catalog
for group in as_list(catalog.groups):
if not group.id:
catalog.controls = group.controls
catalog.groups.remove(group)
break
catalog.metadata = new_metadata
return catalog
def test_profile_generate_assemble(add_header: bool, guid_dict: Dict,
use_cli: bool, dir_exists: bool,
set_parameters: bool,
tmp_trestle_dir: pathlib.Path,
monkeypatch: MonkeyPatch) -> None:
"""Test the profile markdown generator."""
ac1_path, assembled_prof_dir, profile_path, markdown_path = setup_profile_generate(
tmp_trestle_dir)
yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml'
# convert resolved profile catalog to markdown then assemble it after adding an item to a control
if use_cli:
test_args = f'trestle author profile-generate -n {prof_name} -o {md_name} -rs NeededExtra'.split(
)
if add_header:
test_args.extend(['-y', str(yaml_header_path)])
test_args.extend(['-s', all_sections_str])
monkeypatch.setattr(sys, 'argv', test_args)
assert Trestle().run() == 0
edit_files(ac1_path, set_parameters, guid_dict)
test_args = f'trestle author profile-assemble -n {prof_name} -m {md_name} -o {assembled_prof_name}'.split(
)
if set_parameters:
test_args.append('-sp')
if dir_exists:
assembled_prof_dir.mkdir()
monkeypatch.setattr(sys, 'argv', test_args)
assert Trestle().run() == 0
else:
profile_generate = ProfileGenerate()
yaml_header = {}
if add_header:
yaml = YAML()
yaml_header = yaml.load(yaml_header_path.open('r'))
sections_dict = sections_to_dict(all_sections_str)
profile_generate.generate_markdown(tmp_trestle_dir, profile_path,
markdown_path, yaml_header, False,
sections_dict, 'NeededExtra')
edit_files(ac1_path, set_parameters, guid_dict)
if dir_exists:
assembled_prof_dir.mkdir()
assert ProfileAssemble.assemble_profile(tmp_trestle_dir, prof_name,
md_name, assembled_prof_name,
set_parameters, False, None,
None, None) == 0
# check the assembled profile is as expected
profile: prof.Profile
profile, _ = ModelUtils.load_top_level_model(tmp_trestle_dir,
assembled_prof_name,
prof.Profile,
FileContentType.JSON)
assert ModelUtils.model_age(profile) < test_utils.NEW_MODEL_AGE_SECONDS
# get the set_params in the assembled profile
set_params = profile.modify.set_parameters
if set_parameters:
assert set_params[0].param_id == 'ac-1_prm_1'
assert set_params[0].values[0].__root__ == 'all personnel'
assert set_params[1].param_id == 'ac-1_prm_2'
assert set_params[1].values[0].__root__ == 'Organization-level'
assert set_params[1].values[1].__root__ == 'System-level'
assert set_params[2].param_id == 'ac-1_prm_3'
assert set_params[2].values[0].__root__ == 'new value'
else:
assert len(set_params) == 15
# now create the resolved profile catalog from the assembled json profile and confirm the addition is there
catalog = ProfileResolver.get_resolved_profile_catalog(
tmp_trestle_dir, assembled_prof_dir / 'profile.json')
catalog_interface = CatalogInterface(catalog)
# confirm presence of all expected strings in the control named parts
for name, exp_str in guid_dict['name_exp']:
prose = catalog_interface.get_control_part_prose('ac-1', name)
assert prose.find(exp_str) >= 0
def test_profile_ohv(required_sections: Optional[str], success: bool,
ohv: bool, tmp_trestle_dir: pathlib.Path) -> None:
"""Test profile generate assemble with overwrite-header-values."""
ac1_path, assembled_prof_dir, profile_path, markdown_path = setup_profile_generate(
tmp_trestle_dir)
yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml'
new_version = '1.2.3'
# convert resolved profile catalog to markdown then assemble it after adding an item to a control
# if set_parameters is true, the yaml header will contain all the parameters
profile_generate = ProfileGenerate()
yaml = YAML()
yaml_header = yaml.load(yaml_header_path.open('r'))
profile_generate.generate_markdown(tmp_trestle_dir, profile_path,
markdown_path, yaml_header, ohv, None,
None)
edit_files(ac1_path, True, multi_guidance_dict)
markdown_path = tmp_trestle_dir / md_name
# change guidance in the other two controls but don't change header
ac2_path = markdown_path / 'ac/ac-2.md'
ac21_path = markdown_path / 'ac/ac-2.1.md'
edit_files(ac2_path, False, multi_guidance_dict)
edit_files(ac21_path, False, multi_guidance_dict)
if success:
assert ProfileAssemble.assemble_profile(tmp_trestle_dir, prof_name,
md_name, assembled_prof_name,
True, False, new_version,
required_sections, None) == 0
# check the assembled profile is as expected
profile: prof.Profile
profile, _ = ModelUtils.load_top_level_model(tmp_trestle_dir,
assembled_prof_name,
prof.Profile,
FileContentType.JSON)
set_params = profile.modify.set_parameters
assert len(set_params) == 14
assert set_params[0].values[0].__root__ == 'all personnel'
assert set_params[1].param_id == 'ac-1_prm_2'
assert set_params[1].values[0].__root__ == 'Organization-level'
assert set_params[1].values[1].__root__ == 'System-level'
assert set_params[2].values[0].__root__ == 'new value'
assert profile.metadata.version.__root__ == new_version
if ohv:
assert set_params[3].values[0].__root__ == 'no meetings'
assert set_params[3].label == 'meetings cancelled'
else:
assert set_params[3].values[0].__root__ == 'all meetings'
assert set_params[3].label == 'organization-defined events'
catalog = ProfileResolver.get_resolved_profile_catalog(
tmp_trestle_dir, assembled_prof_dir / 'profile.json')
catalog_interface = CatalogInterface(catalog)
# confirm presence of all expected strings in the control named parts
for name, exp_str in multi_guidance_dict['name_exp']:
prose = catalog_interface.get_control_part_prose('ac-1', name)
assert prose.find(exp_str) >= 0
else:
with pytest.raises(TrestleError):
ProfileAssemble.assemble_profile(tmp_trestle_dir, prof_name,
md_name, assembled_prof_name,
True, False, new_version,
required_sections, None)
def filter_ssp(self, trestle_root: pathlib.Path, ssp_name: str,
profile_name: str, out_name: str, regenerate: bool,
version: Optional[str]) -> int:
"""
Filter the ssp based on controls included by the profile and output new ssp.
Args:
trestle_root: root directory of the trestle project
ssp_name: name of the ssp model
profile_name: name of the profile model used for filtering
out_name: name of the output ssp model with filtered controls
regenerate: whether to regenerate the uuid's in the ssp
version: new version for the model
Returns:
0 on success, 1 otherwise
"""
ssp: ossp.SystemSecurityPlan
ssp, _ = ModelUtils.load_top_level_model(trestle_root, ssp_name,
ossp.SystemSecurityPlan,
FileContentType.JSON)
profile_path = ModelUtils.path_for_top_level_model(
trestle_root, profile_name, prof.Profile, FileContentType.JSON)
prof_resolver = ProfileResolver()
catalog = prof_resolver.get_resolved_profile_catalog(
trestle_root, profile_path)
catalog_interface = CatalogInterface(catalog)
# The input ssp should reference a superset of the controls referenced by the profile
# Need to cull references in the ssp to controls not in the profile
# Also make sure the output ssp contains imp reqs for all controls in the profile
control_imp = ssp.control_implementation
ssp_control_ids: Set[str] = set()
new_set_params: List[ossp.SetParameter] = []
for set_param in as_list(control_imp.set_parameters):
control = catalog_interface.get_control_by_param_id(
set_param.param_id)
if control is not None:
new_set_params.append(set_param)
ssp_control_ids.add(control.id)
control_imp.set_parameters = new_set_params if new_set_params else None
new_imp_requirements: List[ossp.ImplementedRequirement] = []
for imp_requirement in as_list(control_imp.implemented_requirements):
control = catalog_interface.get_control(imp_requirement.control_id)
if control is not None:
new_imp_requirements.append(imp_requirement)
ssp_control_ids.add(control.id)
control_imp.implemented_requirements = new_imp_requirements
# make sure all controls in the profile have implemented reqs in the final ssp
if not ssp_control_ids.issuperset(catalog_interface.get_control_ids()):
raise TrestleError(
'Unable to filter the ssp because the profile references controls not in it.'
)
ssp.control_implementation = control_imp
if regenerate:
ssp, _, _ = ModelUtils.regenerate_uuids(ssp)
if version:
ssp.metadata.version = com.Version(__root__=version)
ModelUtils.update_last_modified(ssp)
ModelUtils.save_top_level_model(ssp, trestle_root, out_name,
FileContentType.JSON)
return CmdReturnCodes.SUCCESS.value
def _run(self, args: argparse.Namespace) -> int:
try:
log.set_log_level_from_args(args)
trestle_root = args.trestle_root
if not file_utils.is_directory_name_allowed(args.output):
raise TrestleError(
f'{args.output} is not an allowed directory name')
profile_path = trestle_root / f'profiles/{args.profile}/profile.json'
yaml_header: dict = {}
if args.yaml_header:
try:
logging.debug(
f'Loading yaml header file {args.yaml_header}')
yaml = YAML()
yaml_header = yaml.load(
pathlib.Path(args.yaml_header).open('r'))
except YAMLError as e:
raise TrestleError(
f'YAML error loading yaml header for ssp generation: {e}'
)
markdown_path = trestle_root / args.output
profile_resolver = ProfileResolver()
resolved_catalog = profile_resolver.get_resolved_profile_catalog(
trestle_root, profile_path)
catalog_interface = CatalogInterface(resolved_catalog)
sections_dict: Dict[str, str] = {}
if args.sections:
sections_dict = sections_to_dict(args.sections)
if 'statement' in sections_dict:
raise TrestleError(
'Statement is not allowed as a section name.')
# add any existing sections from the controls but only have short names
control_section_short_names = catalog_interface.get_sections()
for short_name in control_section_short_names:
if short_name not in sections_dict:
sections_dict[short_name] = short_name
logger.debug(f'ssp sections dict: {sections_dict}')
catalog_interface.write_catalog_as_markdown(
md_path=markdown_path,
yaml_header=yaml_header,
sections_dict=sections_dict,
prompt_responses=True,
additional_content=False,
profile=None,
overwrite_header_values=args.overwrite_header_values,
set_parameters=False,
required_sections=None,
allowed_sections=args.allowed_sections)
return CmdReturnCodes.SUCCESS.value
except Exception as e: # pragma: no cover
return handle_generic_command_exception(
e, logger, 'Error while writing markdown from catalog')
def test_deep_catalog() -> None:
"""Test ssp generation with deep catalog."""
catalog = test_utils.generate_complex_catalog()
interface = CatalogInterface(catalog)
assert interface.get_count_of_controls_in_catalog(False) == 13
assert interface.get_count_of_controls_in_catalog(True) == 16
def _run(self, args: argparse.Namespace) -> int:
try:
log.set_log_level_from_args(args)
trestle_root = pathlib.Path(args.trestle_root)
md_path = trestle_root / args.markdown
# the original, reference ssp name defaults to same as output if name not specified
# thus in cyclic editing you are reading and writing same json ssp
orig_ssp_name = args.output
if args.name:
orig_ssp_name = args.name
new_ssp_name = args.output
# if orig ssp exists - need to load it rather than instantiate new one
orig_ssp_path = ModelUtils.path_for_top_level_model(
trestle_root, orig_ssp_name, ossp.SystemSecurityPlan,
FileContentType.JSON)
# if output ssp already exists, load it to see if new one is different
existing_ssp: Optional[ossp.SystemSecurityPlan] = None
new_ssp_path = ModelUtils.path_for_top_level_model(
trestle_root, new_ssp_name, ossp.SystemSecurityPlan,
FileContentType.JSON)
if new_ssp_path.exists():
_, _, existing_ssp = ModelUtils.load_distributed(
new_ssp_path, trestle_root)
ssp: ossp.SystemSecurityPlan
comp_dict: Dict[str, ossp.SystemComponent] = {}
# need to load imp_reqs from markdown but need component first
if orig_ssp_path.exists():
# load the existing json ssp
_, _, ssp = ModelUtils.load_distributed(
orig_ssp_path, trestle_root)
for component in ssp.system_implementation.components:
comp_dict[component.title] = component
# read the new imp reqs from markdown and have them reference existing components
imp_reqs = CatalogInterface.read_catalog_imp_reqs(
md_path, comp_dict)
self._merge_imp_reqs(ssp, imp_reqs)
else:
# create a sample ssp to hold all the parts
ssp = gens.generate_sample_model(ossp.SystemSecurityPlan)
# load the imp_reqs from markdown and create components as needed, referenced by ### headers
imp_reqs = CatalogInterface.read_catalog_imp_reqs(
md_path, comp_dict)
# create system implementation
system_imp: ossp.SystemImplementation = gens.generate_sample_model(
ossp.SystemImplementation)
ssp.system_implementation = system_imp
# create a control implementation to hold the implementated requirements
control_imp: ossp.ControlImplementation = gens.generate_sample_model(
ossp.ControlImplementation)
control_imp.implemented_requirements = imp_reqs
control_imp.description = const.SSP_SYSTEM_CONTROL_IMPLEMENTATION_TEXT
# insert the parts into the ssp
ssp.control_implementation = control_imp
ssp.system_implementation = system_imp
# we don't have access to the original profile so we don't know the href
import_profile: ossp.ImportProfile = gens.generate_sample_model(
ossp.ImportProfile)
import_profile.href = 'REPLACE_ME'
ssp.import_profile = import_profile
# now that we know the complete list of needed components, add them to the sys_imp
# TODO if the ssp already existed then components may need to be removed if not ref'd by imp_reqs
component_list: List[ossp.SystemComponent] = []
for comp in comp_dict.values():
component_list.append(comp)
if ssp.system_implementation.components:
# reconstruct list with same order as existing, but add/remove components as needed
new_list: List[ossp.SystemComponent] = []
for comp in ssp.system_implementation.components:
if comp in component_list:
new_list.append(comp)
for comp in component_list:
if comp not in new_list:
new_list.append(comp)
ssp.system_implementation.components = new_list
elif component_list:
ssp.system_implementation.components = component_list
self._generate_roles_in_metadata(ssp)
if args.version:
ssp.metadata.version = com.Version(__root__=args.version)
if existing_ssp == ssp:
logger.info(
'No changes to assembled ssp so ssp not written out.')
return CmdReturnCodes.SUCCESS.value
if args.regenerate:
ssp, _, _ = ModelUtils.regenerate_uuids(ssp)
ModelUtils.update_last_modified(ssp)
# write out the ssp as json
ModelUtils.save_top_level_model(ssp, trestle_root, new_ssp_name,
FileContentType.JSON)
return CmdReturnCodes.SUCCESS.value
except Exception as e: # pragma: no cover
return handle_generic_command_exception(
e, logger, 'Error while assembling SSP')
def assemble_profile(trestle_root: pathlib.Path, orig_profile_name: str,
md_name: str, new_profile_name: str,
set_parameters: bool, regenerate: bool,
version: Optional[str],
required_sections: Optional[str],
allowed_sections: Optional[List[str]]) -> int:
"""
Assemble the markdown directory into a json profile model file.
Args:
trestle_root: The trestle root directory
orig_profile_name: Optional name of profile used to generate the markdown (default is new_profile_name)
md_name: The name of the directory containing the markdown control files for the profile
new_profile_name: The name of the new json profile. It can be the same as original to overwrite
set_parameters: Use the parameters in the yaml header to specify values for setparameters in the profile
regenerate: Whether to regenerate the uuid's in the profile
version: Optional version for the assembled profile
required_sections: Optional List of required sections in assembled profile, as comma-separated short names
allowed_sections: Optional list of section short names that are allowed, as comma-separated short names
Returns:
0 on success, 1 otherwise
Notes:
There must already be a json profile model and it will either be updated or a new json profile created.
The generated markdown shows the current values for parameters of controls being imported, as set by
the original catalog and any intermediate profiles. It also shows the current SetParameters being applied
by this profile. That list of SetParameters can be edited by changing the assigned values and adding or
removing SetParameters from that list. During assembly that list will be used to create the SetParameters
in the assembled profile if the --set-parameters option is specified.
"""
# first load the original json profile but call it 'new' because it is being updated with new items
profile_path = trestle_root / f'profiles/{orig_profile_name}/profile.json'
_, _, new_profile = ModelUtils.load_distributed(
profile_path, trestle_root)
required_sections_list = required_sections.split(
',') if required_sections else []
# load the editable sections of the markdown and create Adds for them
# then overwrite the Adds in the existing profile with the new ones
# keep track if any changes were made
md_dir = trestle_root / md_name
found_alters, param_dict = CatalogInterface.read_additional_content(
md_dir, required_sections_list)
if allowed_sections:
for alter in found_alters:
for add in alter.adds:
for part in add.parts:
if part.name not in allowed_sections:
raise TrestleError(
f'Profile has alter with name {part.name} not in allowed sections.'
)
ProfileAssemble._replace_alter_adds(new_profile, found_alters)
if set_parameters:
ProfileAssemble._replace_modify_set_params(new_profile, param_dict)
new_profile_dir = trestle_root / f'profiles/{new_profile_name}'
new_profile_path = new_profile_dir / 'profile.json'
if version:
new_profile.metadata.version = com.Version(__root__=version)
if new_profile_path.exists():
# first do simple and fast tests of changes so that the full equality check is only done as last resort
_, _, existing_profile = ModelUtils.load_distributed(
new_profile_path, trestle_root)
# need special handling in case modify is None in either or both profiles
no_changes = not existing_profile.modify and not new_profile.modify
both_modifies_present = existing_profile.modify and new_profile.modify
if both_modifies_present:
no_changes = (existing_profile.modify.alters
== new_profile.modify.alters
and existing_profile.modify.set_parameters
== new_profile.modify.set_parameters)
if no_changes and existing_profile.metadata.version != new_profile.metadata.version:
no_changes = False
# at this point if no known changes then do the full and possibly expensive comparison to confirm
if no_changes and existing_profile == new_profile:
logger.info(
'Assembled profile has no changes so no update of existing file.'
)
return CmdReturnCodes.SUCCESS.value
if regenerate:
new_profile, _, _ = ModelUtils.regenerate_uuids(new_profile)
ModelUtils.update_last_modified(new_profile)
if new_profile_dir.exists():
logger.info(
'Creating profile from markdown and destination profile directory exists, so updating.'
)
try:
shutil.rmtree(str(new_profile_dir))
except OSError as e:
raise TrestleError(
f'OSError deleting existing catalog directory with rmtree {new_profile_dir}: {e}'
)
try:
new_profile_dir.mkdir()
new_profile.oscal_write(new_profile_dir / 'profile.json')
except OSError as e:
raise TrestleError(
f'OSError writing profile from markdown to {new_profile_dir}: {e}'
)
return CmdReturnCodes.SUCCESS.value
def find_string_in_all_controls_prose(interface: CatalogInterface, seek_str: str) -> List[Tuple[str, str]]:
"""Find all instances of this string in catalog prose and return with control id."""
hits: List[Tuple[str, str]] = []
for control in interface.get_all_controls_from_catalog(True):
hits.extend(interface.find_string_in_control(control, seek_str))
return hits
class Modify(Pipeline.Filter):
"""Modify the controls based on the profile."""
def __init__(
self,
profile: prof.Profile,
change_prose: bool = False,
block_adds: bool = False,
block_params: bool = False,
params_format: str = None,
param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES
) -> None:
"""Initialize the filter."""
self._profile = profile
self._catalog_interface: Optional[CatalogInterface] = None
self._block_adds = block_adds
self._block_params = block_params
self._change_prose = change_prose
self._params_format = params_format
self._param_rep = param_rep
logger.debug(
f'modify initialize filter with profile {profile.metadata.title}')
@staticmethod
def _replace_ids_with_text(prose: str, param_rep: ParameterRep,
param_dict: Dict[str, common.Parameter]) -> str:
"""Find all instances of param_ids in prose and replace each with corresponding parameter representation.
Need to check all values in dict for a match
Reject matches where the string has an adjacent alphanumeric char: param_1 and param_10 or aparam_1
"""
for param in param_dict.values():
if param.id not in prose:
continue
# create the replacement text for the param_id
param_str = ControlIOReader.param_to_str(param, param_rep)
# non-capturing groups are odd in re.sub so capture all 3 groups and replace the middle one
pattern = r'(^|[^a-zA-Z0-9_])' + param.id + r'($|[^a-zA-Z0-9_])'
prose = re.sub(pattern, r'\1' + param_str + r'\2', prose)
return prose
@staticmethod
def _replace_params(
text: str,
param_dict: Dict[str, common.Parameter],
params_format: Optional[str] = None,
param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES
) -> str:
"""
Replace params found in moustaches with values from the param_dict.
A single line of prose may contain multiple moustaches.
"""
# first check if there are any moustache patterns in the text
if param_rep == ParameterRep.LEAVE_MOUSTACHE:
return text
staches: List[str] = re.findall(r'{{.*?}}', text)
if not staches:
return text
# now have list of all staches including braces, e.g. ['{{foo}}', '{{bar}}']
# clean the staches so they just have the param ids
param_ids = []
for stache in staches:
# remove braces so these are just param_ids but may have extra chars
stache_contents = stache[2:(-2)]
param_id = stache_contents.replace('insert: param,', '').strip()
param_ids.append(param_id)
# now replace original stache text with param values
for i, _ in enumerate(staches):
# A moustache may refer to a param_id not listed in the control's params
if param_ids[i] not in param_dict:
logger.warning(
f'Control prose references param {param_ids[i]} not found in the control.'
)
elif param_dict[param_ids[i]] is not None:
param = param_dict[param_ids[i]]
param_str = ControlIOReader.param_to_str(
param, param_rep, False, False, params_format)
text = text.replace(staches[i], param_str, 1)
else:
logger.warning(
f'Control prose references param {param_ids[i]} with no specified value.'
)
return text
@staticmethod
def _replace_part_prose(
control: cat.Control,
part: common.Part,
param_dict: Dict[str, common.Parameter],
params_format: Optional[str] = None,
param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES
) -> None:
"""Replace the part prose according to set_param."""
if part.prose is not None:
fixed_prose = Modify._replace_params(part.prose, param_dict,
params_format, param_rep)
# change the prose in the control itself
part.prose = fixed_prose
for prt in as_list(part.parts):
Modify._replace_part_prose(control, prt, param_dict, params_format,
param_rep)
for sub_control in as_list(control.controls):
for prt in as_list(sub_control.parts):
Modify._replace_part_prose(sub_control, prt, param_dict,
params_format, param_rep)
@staticmethod
def _replace_control_prose(
control: cat.Control,
param_dict: Dict[str, common.Parameter],
params_format: Optional[str] = None,
param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES
) -> None:
"""Replace the control prose according to set_param."""
for part in as_list(control.parts):
if part.prose is not None:
fixed_prose = Modify._replace_params(part.prose, param_dict,
params_format, param_rep)
# change the prose in the control itself
part.prose = fixed_prose
for prt in as_list(part.parts):
Modify._replace_part_prose(control, prt, param_dict,
params_format, param_rep)
for sub_control in as_list(control.controls):
prts: List[common.Part] = as_list(sub_control.parts)
for prt in prts:
Modify._replace_part_prose(sub_control, prt, param_dict,
params_format, param_rep)
@staticmethod
def _add_contents_as_list(add: prof.Add) -> List[OBT]:
add_list = []
add_list.extend(as_list(add.props))
add_list.extend(as_list(add.parts))
add_list.extend(as_list(add.links))
return add_list
@staticmethod
def _add_adds_to_part(part: common.Part, add: prof.Add) -> None:
for attr in ['params', 'props', 'parts', 'links']:
add_list = getattr(add, attr, None)
if add_list:
Modify._add_attr_to_part(part, add_list, attr, add.position)
@staticmethod
def _add_to_list(input_list: List[OBT], add: prof.Add) -> bool:
"""Add the contents of the add according to its by_id and position.
Return True on success or False if id needed and not found.
This is only called when by_id is not None.
The add will be inserted if the id is found, or return False if not.
This allows a separate recursive routine to search sub-lists for the id.
"""
add_list = Modify._add_contents_as_list(add)
# Test here for matched by_id attribute.
try:
for index in range(len(input_list)):
if input_list[index].id == add.by_id:
if add.position == prof.Position.after:
for offset, new_item in enumerate(add_list):
input_list.insert(index + 1 + offset, new_item)
return True
elif add.position == prof.Position.before:
for offset, new_item in enumerate(add_list):
input_list.insert(index + offset, new_item)
return True
# if starting or ending, the adds go directly into this part according to type
Modify._add_adds_to_part(input_list[index], add)
return True
except AttributeError:
raise TrestleError(
'Cannot use "after" or "before" modifications for a list where elements'
+ ' do not contain the referenced by_id attribute.')
return False
@staticmethod
def _add_to_parts(parts: List[common.Part], add: prof.Add) -> bool:
"""
Add the add to the parts.
This is only called if add.by_id is not None.
"""
if Modify._add_to_list(parts, add):
return True
for part in parts:
if part.parts is not None and Modify._add_to_parts(
part.parts, add):
return True
return False
@staticmethod
def _add_attr_to_part(part: common.Part, items: List[OBT], attr: str,
position: prof.Position) -> None:
attr_list = as_list(getattr(part, attr, None))
if position in [prof.Position.starting, prof.Position.before]:
items.extend(attr_list)
attr_list = items
else:
attr_list.extend(items)
setattr(part, attr, attr_list)
@staticmethod
def _add_attr_to_control(control: cat.Control, items: List[OBT], attr: str,
position: prof.Position) -> None:
attr_list = as_list(getattr(control, attr, None))
if position in [prof.Position.starting, prof.Position.before]:
items.extend(attr_list)
attr_list = items
else:
attr_list.extend(items)
setattr(control, attr, attr_list)
@staticmethod
def _add_to_control(control: cat.Control, add: prof.Add) -> None:
"""First step in applying Add to control."""
control.parts = as_list(control.parts)
if add.by_id is None or add.by_id == control.id:
# add contents will be added to the control directly and with no recursion
for attr in ['params', 'props', 'parts', 'links']:
add_list = getattr(add, attr, None)
if add_list:
Modify._add_attr_to_control(control, add_list, attr,
add.position)
return
else:
# this is only called if by_id is not None
if not Modify._add_to_parts(control.parts, add):
logger.warning(
f'Could not find id for add in control {control.id}: {add.by_id}'
)
def _set_parameter_in_control(self, set_param: prof.SetParameter) -> None:
"""
Find the control with the param_id in it and set the parameter value.
This does not recurse because expectation is that only top level params will be set.
"""
control = self._catalog_interface.get_control_by_param_id(
set_param.param_id)
if control is None:
raise TrestleError(
f'Set parameter object in profile does not have a corresponding param-id: "{set_param.param_id}"'
)
control.params = as_list(control.params)
param_ids = [param.id for param in control.params]
if set_param.param_id not in param_ids:
raise TrestleNotFoundError(
f'Param id {set_param.param_id} not found in control {control.id}'
)
index = param_ids.index(set_param.param_id)
param = control.params[index]
param.values = set_param.values
param.constraints = set_param.constraints
param.guidelines = set_param.guidelines
param.links = set_param.links
param.props = set_param.props
param.select = set_param.select
param.usage = set_param.usage
control.params[index] = param
self._catalog_interface.replace_control(control)
def _change_prose_with_param_values(self):
"""Go through all controls and change prose based on param values."""
param_dict: Dict[str, common.Paramter] = {}
# build the full mapping of params to values
for control in self._catalog_interface.get_all_controls_from_dict():
param_dict.update(
ControlIOReader.get_control_param_dict(control, False))
# insert param values into prose of all controls
for control in self._catalog_interface.get_all_controls_from_dict():
self._replace_control_prose(control, param_dict,
self._params_format, self._param_rep)
def _modify_controls(self, catalog: cat.Catalog) -> cat.Catalog:
"""Modify the controls based on the profile."""
logger.debug(
f'modify specify catalog {catalog.metadata.title} for profile {self._profile.metadata.title}'
)
self._catalog_interface = CatalogInterface(catalog)
alters: Optional[List[prof.Alter]] = None
# find the modify and alters
if self._profile.modify:
# change all parameter values
if self._profile.modify.set_parameters and not self._block_params:
set_param_list = self._profile.modify.set_parameters
for set_param in set_param_list:
self._set_parameter_in_control(set_param)
alters = self._profile.modify.alters
if alters is not None:
title = self._profile.metadata.title
for alter in alters:
if alter.control_id is None:
logger.warning(
f'Alter must have control id specified in profile {title}.'
)
continue
id_ = alter.control_id
if alter.removes is not None:
logger.warning(
f'Alter not supported for removes in profile {title} control {id_}'
)
continue
# we want a warning about adds even if adds are blocked, as in profile generate
if alter.adds is None:
logger.warning(
f'Alter has no adds in profile {title} control {id_}')
continue
if not self._block_adds:
for add in alter.adds:
if add.position is None and add.parts is not None:
msg = f'Alter/Add position is not specified in profile {title} control {id_}'
msg += ' when adding part, so defaulting to ending.'
logger.warning(msg)
add.position = prof.Position.ending
control = self._catalog_interface.get_control(id_)
if control is None:
logger.warning(
f'Alter/Add refers to control {id_} but it is not found in the import '
+
f'for profile {self._profile.metadata.title}')
else:
self._add_to_control(control, add)
self._catalog_interface.replace_control(control)
if self._change_prose:
# go through all controls and fix the prose based on param values
self._change_prose_with_param_values()
catalog = self._catalog_interface.get_catalog()
# update the original profile metadata with new contents
# roles and responsible-parties will be pulled in with new uuid's
new_metadata = self._profile.metadata
new_metadata.title = f'{catalog.metadata.title}: Resolved by profile {self._profile.metadata.title}'
links: List[common.Link] = []
for import_ in self._profile.imports:
links.append(
common.Link(**{
'href': import_.href,
'rel': 'resolution-source'
}))
new_metadata.links = links
# move catalog controls from dummy group '' into the catalog
for group in as_list(catalog.groups):
if not group.id:
catalog.controls = group.controls
catalog.groups.remove(group)
break
catalog.metadata = new_metadata
return catalog
def process(self,
catalog_iter: Iterator[cat.Catalog]) -> Iterator[cat.Catalog]:
"""Make the modifications to the controls based on the profile."""
catalog = next(catalog_iter)
logger.debug(
f'modify process with catalog {catalog.metadata.title} using profile {self._profile.metadata.title}'
)
yield self._modify_controls(catalog)
def jinja_ify(
trestle_root: pathlib.Path,
r_input_file: pathlib.Path,
r_output_file: pathlib.Path,
ssp: Optional[str],
profile: Optional[str],
lut: Optional[Dict[str, Any]] = None,
number_captions: Optional[bool] = False,
parameters_formatting: Optional[str] = None
) -> int:
"""Run jinja over an input file with additional booleans."""
if lut is None:
lut = {}
template_folder = pathlib.Path.cwd()
jinja_env = Environment(
loader=FileSystemLoader(template_folder),
extensions=[MDSectionInclude, MDCleanInclude, MDDatestamp],
trim_blocks=True,
autoescape=True
)
template = jinja_env.get_template(str(r_input_file))
# create boolean dict
if operator.xor(bool(ssp), bool(profile)):
raise TrestleIncorrectArgsError('Both SSP and profile should be provided or not at all')
if ssp:
# name lookup
ssp_data, _ = ModelUtils.load_top_level_model(trestle_root, ssp, SystemSecurityPlan)
lut['ssp'] = ssp_data
_, profile_path = ModelUtils.load_top_level_model(trestle_root, profile, Profile)
profile_resolver = ProfileResolver()
resolved_catalog = profile_resolver.get_resolved_profile_catalog(
trestle_root, profile_path, False, False, parameters_formatting
)
ssp_writer = SSPMarkdownWriter(trestle_root)
ssp_writer.set_ssp(ssp_data)
ssp_writer.set_catalog(resolved_catalog)
lut['catalog'] = resolved_catalog
lut['catalog_interface'] = CatalogInterface(resolved_catalog)
lut['control_io_writer'] = ControlIOWriter()
lut['ssp_md_writer'] = ssp_writer
new_output = template.render(**lut)
output = ''
# This recursion allows nesting within expressions (e.g. an expression can contain jinja templates).
error_countdown = JinjaCmd.max_recursion_depth
while new_output != output and error_countdown > 0:
error_countdown = error_countdown - 1
output = new_output
random_name = uuid.uuid4() # Should be random and not used.
dict_loader = DictLoader({str(random_name): new_output})
jinja_env = Environment(
loader=ChoiceLoader([dict_loader, FileSystemLoader(template_folder)]),
extensions=[MDCleanInclude, MDSectionInclude, MDDatestamp],
autoescape=True,
trim_blocks=True
)
template = jinja_env.get_template(str(random_name))
new_output = template.render(**lut)
output_file = trestle_root / r_output_file
if number_captions:
output_file.open('w', encoding=const.FILE_ENCODING).write(_number_captions(output))
else:
output_file.open('w', encoding=const.FILE_ENCODING).write(output)
return CmdReturnCodes.SUCCESS.value
def test_profile_resolver_merge(sample_catalog_rich_controls: cat.Catalog) -> None:
"""Test profile resolver merge."""
profile = gens.generate_sample_model(prof.Profile)
method = prof.Method.merge
combine = prof.Combine(method=method)
profile.merge = prof.Merge(combine=combine)
merge = Merge(profile)
# merge into empty catalog
merged = gens.generate_sample_model(cat.Catalog)
new_merged = merge._merge_catalog(merged, sample_catalog_rich_controls)
catalog_interface = CatalogInterface(new_merged)
assert catalog_interface.get_count_of_controls_in_catalog(True) == 5
# add part to first control and merge, then make sure it is there
part = com.Part(name='foo', title='added part')
control_id = sample_catalog_rich_controls.controls[0].id
cat_with_added_part = copy.deepcopy(sample_catalog_rich_controls)
cat_with_added_part.controls[0].parts.append(part)
final_merged = merge._merge_catalog(sample_catalog_rich_controls, cat_with_added_part)
catalog_interface = CatalogInterface(final_merged)
assert catalog_interface.get_count_of_controls_in_catalog(True) == 5
assert catalog_interface.get_control(control_id).parts[-1].name == 'foo'
# add part to first control and merge but with use-first. The part should not be there at end.
method = prof.Method.use_first
combine = prof.Combine(method=method)
profile.merge = prof.Merge(combine=combine)
merge = Merge(profile)
final_merged = merge._merge_catalog(sample_catalog_rich_controls, cat_with_added_part)
catalog_interface = CatalogInterface(final_merged)
assert catalog_interface.get_count_of_controls_in_catalog(True) == 5
assert len(catalog_interface.get_control(control_id).parts) == 1
# now force a merge with keep
profile.merge = None
merge_keep = Merge(profile)
merged_keep = merge_keep._merge_catalog(new_merged, sample_catalog_rich_controls)
assert CatalogInterface(merged_keep).get_count_of_controls_in_catalog(True) == 10