def test_get_control_param_dict(tmp_trestle_dir: pathlib.Path) -> None: """Test getting the param dict of a control.""" test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True) prof_a_path = ModelUtils.path_for_top_level_model(tmp_trestle_dir, 'test_profile_a', prof.Profile, FileContentType.JSON) catalog = ProfileResolver.get_resolved_profile_catalog( tmp_trestle_dir, prof_a_path) catalog_interface = CatalogInterface(catalog) control = catalog_interface.get_control('ac-1') param_dict = ControlIOReader.get_control_param_dict(control, False) # confirm profile value is used assert ControlIOReader.param_values_as_str( param_dict['ac-1_prm_1']) == 'all alert personnel' # confirm original param label is used since no value was assigned assert ControlIOReader.param_to_str(param_dict['ac-1_prm_7'], ParameterRep.VALUE_OR_LABEL_OR_CHOICES ) == 'organization-defined events' param = control.params[0] param.values = None param.select = common.ParameterSelection( how_many=common.HowMany.one_or_more, choice=['choice 1', 'choice 2']) param_dict = ControlIOReader.get_control_param_dict(control, False) assert ControlIOReader.param_to_str( param_dict['ac-1_prm_1'], ParameterRep.VALUE_OR_LABEL_OR_CHOICES) == 'choice 1, choice 2'
def generate_markdown(self, trestle_root: pathlib.Path, profile_path: pathlib.Path, markdown_path: pathlib.Path, yaml_header: dict, overwrite_header_values: bool, sections_dict: Optional[Dict[str, str]], required_sections: Optional[str]) -> int: """Generate markdown for the controls in the profile. Args: trestle_root: Root directory of the trestle workspace profile_path: Path of the profile json file markdown_path: Path to the directory into which the markdown will be written yaml_header: Dict to merge into the yaml header of the control markdown overwrite_header_values: Overwrite values in the markdown header but allow new items to be added sections_dict: Optional dict mapping section short names to long required_sections: Optional comma-sep list of sections that get prompted for prose if not in the profile Returns: 0 on success, 1 on error """ try: if sections_dict and 'statement' in sections_dict: logger.warning('statement is not allowed as a section name.') return CmdReturnCodes.COMMAND_ERROR.value _, _, profile = ModelUtils.load_distributed( profile_path, trestle_root) catalog = ProfileResolver().get_resolved_profile_catalog( trestle_root, profile_path, True, True, None, ParameterRep.LEAVE_MOUSTACHE) catalog_interface = CatalogInterface(catalog) catalog_interface.write_catalog_as_markdown( md_path=markdown_path, yaml_header=yaml_header, sections_dict=sections_dict, prompt_responses=False, additional_content=True, profile=profile, overwrite_header_values=overwrite_header_values, set_parameters=True, required_sections=required_sections, allowed_sections=None) except TrestleNotFoundError as e: raise TrestleError(f'Profile {profile_path} not found, error {e}') except TrestleError as e: raise TrestleError( f'Error generating the catalog as markdown: {e}') return CmdReturnCodes.SUCCESS.value
def test_get_control_and_group_info_from_catalog(tmp_trestle_dir: pathlib.Path) -> None: """Test get all groups from the catalog.""" test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True) prof_a_path = ModelUtils.path_for_top_level_model( tmp_trestle_dir, 'test_profile_a', prof.Profile, FileContentType.JSON ) catalog = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_a_path) cat_interface = CatalogInterface(catalog) all_groups_top = cat_interface.get_all_controls_from_catalog(recurse=False) assert len(list(all_groups_top)) == 6 all_groups_rec = cat_interface.get_all_controls_from_catalog(recurse=True) assert len(list(all_groups_rec)) == 7 all_group_ids = cat_interface.get_group_ids() assert len(all_group_ids) == 1 statement_label, part = cat_interface.get_statement_label_if_exists('ac-1', 'ac-1_smt.c.2') assert statement_label == '2.' assert part.id == 'ac-1_smt.c.2' cat_path = cat_interface._get_control_path('ac-2') assert cat_path[0] == 'ac' assert len(cat_path) == 1
def test_parameter_resolution(tmp_trestle_dir: pathlib.Path) -> None: """Test whether expected order of operations is preserved for parameter substution.""" test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True) prof_e_path = ModelUtils.path_for_top_level_model( tmp_trestle_dir, 'test_profile_e', prof.Profile, FileContentType.JSON ) profile_e_parameter_string = '## Override value ##' profile_a_value = 'all alert personnel' # based on 800-53 rev 5 cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_e_path) interface = CatalogInterface(cat) control = interface.get_control('ac-1') locations = interface.find_string_in_control(control, profile_e_parameter_string) locations_a = interface.find_string_in_control(control, profile_a_value) assert len(locations) == 1 assert len(locations_a) == 0 assert len(control.params[1].constraints) == 1
def test_profile_resolver(tmp_trestle_dir: pathlib.Path) -> None: """Test the resolver.""" test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True) prof_a_path = ModelUtils.path_for_top_level_model( tmp_trestle_dir, 'test_profile_a', prof.Profile, FileContentType.JSON ) cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_a_path) interface = CatalogInterface(cat) # added part ac-1_expevid from prof a list1 = find_string_in_all_controls_prose(interface, 'Detailed evidence logs') # modify param ac-3.3_prm_2 in prof b list2 = find_string_in_all_controls_prose(interface, 'full and complete compliance') assert len(list1) == 1 assert len(list2) == 1 assert interface.get_count_of_controls_in_catalog(False) == 6 assert interface.get_count_of_controls_in_catalog(True) == 7 assert len(cat.controls) == 4 assert interface.get_dependent_control_ids('ac-3') == ['ac-3.3'] control = interface.get_control('a-1') assert control.parts[0].parts[0].id == 'a-1_deep' assert control.parts[0].parts[0].prose == 'Extra added part in subpart'
def test_add_props(tmp_trestle_dir: pathlib.Path) -> None: """Test all types of property additions.""" test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True) prof_f_path = ModelUtils.path_for_top_level_model( tmp_trestle_dir, 'test_profile_f', prof.Profile, FileContentType.JSON ) cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_f_path) interface = CatalogInterface(cat) ac_3 = interface.get_control('ac-3') assert len(ac_3.props) == 6 assert ac_3.props[-1].value == 'four' for part in ac_3.parts: if part.id == 'ac-3_stmt': assert len(part.props) == 4 ac_5 = interface.get_control('ac-5') for part in ac_5.parts: if part.id == 'ac-5_stmt': for sub_part in part.parts: if sub_part.id == 'ac-5_smt.a': assert len(sub_part.props) == 4
def test_all_positions_for_alter_can_be_resolved(tmp_trestle_dir: pathlib.Path) -> None: """Test that all alter adds positions can be resolved.""" cat_path = test_utils.JSON_TEST_DATA_PATH / test_utils.SIMPLIFIED_NIST_CATALOG_NAME repo = Repository(tmp_trestle_dir) repo.load_and_import_model(cat_path, 'nist_cat') prof_d_path = test_utils.JSON_TEST_DATA_PATH / 'test_profile_d.json' repo.load_and_import_model(prof_d_path, 'test_profile_d') cat = ProfileResolver.get_resolved_profile_catalog(tmp_trestle_dir, prof_d_path) interface = CatalogInterface(cat) control_a1 = interface.get_control('ac-1') control_a2 = interface.get_control('ac-2') assert control_a1.parts[0].id == 'ac-1_first_lev1' assert control_a1.parts[1].parts[3].id == 'ac-1_last_lev2' assert control_a1.parts[2].id == 'ac-1_after1_ac-1_smt_lev1' assert control_a1.parts[3].id == 'ac-1_after2_ac-1_smt_lev1' assert control_a1.parts[1].parts[0].parts[1].id == 'ac-1_smt_before1_a.2_lev3' assert control_a1.parts[1].parts[0].parts[2].id == 'ac-1_smt_before2_a.2_lev3' assert control_a1.parts[1].parts[0].parts[3].parts[0].id == 'ac-1_smt_inside1_at_the_end_a.2_lev4' assert control_a1.parts[1].parts[0].parts[3].parts[1].id == 'ac-1_smt_inside2_at_the_end_a.2_lev4' assert control_a2.parts[0].id == 'ac-2_implgdn_lev1'
def catalog_interface_equivalent(cat_int_a: CatalogInterface, cat_b: cat.Catalog, strong=True) -> bool: """Test equivalence of catalog dict contents in various ways.""" cat_int_b = CatalogInterface(cat_b) if cat_int_b.get_count_of_controls_in_dict() != cat_int_a.get_count_of_controls_in_dict(): logger.error('count of controls is different') return False for a in cat_int_a.get_all_controls_from_dict(): try: b = cat_int_b.get_control(a.id) except Exception as e: logger.error(f'error finding control {a.id} {e}') if not controls_equivalent(a, b, strong): logger.error(f'controls differ: {a.id}') return False return True
def _modify_controls(self, catalog: cat.Catalog) -> cat.Catalog: """Modify the controls based on the profile.""" logger.debug( f'modify specify catalog {catalog.metadata.title} for profile {self._profile.metadata.title}' ) self._catalog_interface = CatalogInterface(catalog) alters: Optional[List[prof.Alter]] = None # find the modify and alters if self._profile.modify: # change all parameter values if self._profile.modify.set_parameters and not self._block_params: set_param_list = self._profile.modify.set_parameters for set_param in set_param_list: self._set_parameter_in_control(set_param) alters = self._profile.modify.alters if alters is not None: title = self._profile.metadata.title for alter in alters: if alter.control_id is None: logger.warning( f'Alter must have control id specified in profile {title}.' ) continue id_ = alter.control_id if alter.removes is not None: logger.warning( f'Alter not supported for removes in profile {title} control {id_}' ) continue # we want a warning about adds even if adds are blocked, as in profile generate if alter.adds is None: logger.warning( f'Alter has no adds in profile {title} control {id_}') continue if not self._block_adds: for add in alter.adds: if add.position is None and add.parts is not None: msg = f'Alter/Add position is not specified in profile {title} control {id_}' msg += ' when adding part, so defaulting to ending.' logger.warning(msg) add.position = prof.Position.ending control = self._catalog_interface.get_control(id_) if control is None: logger.warning( f'Alter/Add refers to control {id_} but it is not found in the import ' + f'for profile {self._profile.metadata.title}') else: self._add_to_control(control, add) self._catalog_interface.replace_control(control) if self._change_prose: # go through all controls and fix the prose based on param values self._change_prose_with_param_values() catalog = self._catalog_interface.get_catalog() # update the original profile metadata with new contents # roles and responsible-parties will be pulled in with new uuid's new_metadata = self._profile.metadata new_metadata.title = f'{catalog.metadata.title}: Resolved by profile {self._profile.metadata.title}' links: List[common.Link] = [] for import_ in self._profile.imports: links.append( common.Link(**{ 'href': import_.href, 'rel': 'resolution-source' })) new_metadata.links = links # move catalog controls from dummy group '' into the catalog for group in as_list(catalog.groups): if not group.id: catalog.controls = group.controls catalog.groups.remove(group) break catalog.metadata = new_metadata return catalog
def test_profile_generate_assemble(add_header: bool, guid_dict: Dict, use_cli: bool, dir_exists: bool, set_parameters: bool, tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: """Test the profile markdown generator.""" ac1_path, assembled_prof_dir, profile_path, markdown_path = setup_profile_generate( tmp_trestle_dir) yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml' # convert resolved profile catalog to markdown then assemble it after adding an item to a control if use_cli: test_args = f'trestle author profile-generate -n {prof_name} -o {md_name} -rs NeededExtra'.split( ) if add_header: test_args.extend(['-y', str(yaml_header_path)]) test_args.extend(['-s', all_sections_str]) monkeypatch.setattr(sys, 'argv', test_args) assert Trestle().run() == 0 edit_files(ac1_path, set_parameters, guid_dict) test_args = f'trestle author profile-assemble -n {prof_name} -m {md_name} -o {assembled_prof_name}'.split( ) if set_parameters: test_args.append('-sp') if dir_exists: assembled_prof_dir.mkdir() monkeypatch.setattr(sys, 'argv', test_args) assert Trestle().run() == 0 else: profile_generate = ProfileGenerate() yaml_header = {} if add_header: yaml = YAML() yaml_header = yaml.load(yaml_header_path.open('r')) sections_dict = sections_to_dict(all_sections_str) profile_generate.generate_markdown(tmp_trestle_dir, profile_path, markdown_path, yaml_header, False, sections_dict, 'NeededExtra') edit_files(ac1_path, set_parameters, guid_dict) if dir_exists: assembled_prof_dir.mkdir() assert ProfileAssemble.assemble_profile(tmp_trestle_dir, prof_name, md_name, assembled_prof_name, set_parameters, False, None, None, None) == 0 # check the assembled profile is as expected profile: prof.Profile profile, _ = ModelUtils.load_top_level_model(tmp_trestle_dir, assembled_prof_name, prof.Profile, FileContentType.JSON) assert ModelUtils.model_age(profile) < test_utils.NEW_MODEL_AGE_SECONDS # get the set_params in the assembled profile set_params = profile.modify.set_parameters if set_parameters: assert set_params[0].param_id == 'ac-1_prm_1' assert set_params[0].values[0].__root__ == 'all personnel' assert set_params[1].param_id == 'ac-1_prm_2' assert set_params[1].values[0].__root__ == 'Organization-level' assert set_params[1].values[1].__root__ == 'System-level' assert set_params[2].param_id == 'ac-1_prm_3' assert set_params[2].values[0].__root__ == 'new value' else: assert len(set_params) == 15 # now create the resolved profile catalog from the assembled json profile and confirm the addition is there catalog = ProfileResolver.get_resolved_profile_catalog( tmp_trestle_dir, assembled_prof_dir / 'profile.json') catalog_interface = CatalogInterface(catalog) # confirm presence of all expected strings in the control named parts for name, exp_str in guid_dict['name_exp']: prose = catalog_interface.get_control_part_prose('ac-1', name) assert prose.find(exp_str) >= 0
def test_profile_ohv(required_sections: Optional[str], success: bool, ohv: bool, tmp_trestle_dir: pathlib.Path) -> None: """Test profile generate assemble with overwrite-header-values.""" ac1_path, assembled_prof_dir, profile_path, markdown_path = setup_profile_generate( tmp_trestle_dir) yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml' new_version = '1.2.3' # convert resolved profile catalog to markdown then assemble it after adding an item to a control # if set_parameters is true, the yaml header will contain all the parameters profile_generate = ProfileGenerate() yaml = YAML() yaml_header = yaml.load(yaml_header_path.open('r')) profile_generate.generate_markdown(tmp_trestle_dir, profile_path, markdown_path, yaml_header, ohv, None, None) edit_files(ac1_path, True, multi_guidance_dict) markdown_path = tmp_trestle_dir / md_name # change guidance in the other two controls but don't change header ac2_path = markdown_path / 'ac/ac-2.md' ac21_path = markdown_path / 'ac/ac-2.1.md' edit_files(ac2_path, False, multi_guidance_dict) edit_files(ac21_path, False, multi_guidance_dict) if success: assert ProfileAssemble.assemble_profile(tmp_trestle_dir, prof_name, md_name, assembled_prof_name, True, False, new_version, required_sections, None) == 0 # check the assembled profile is as expected profile: prof.Profile profile, _ = ModelUtils.load_top_level_model(tmp_trestle_dir, assembled_prof_name, prof.Profile, FileContentType.JSON) set_params = profile.modify.set_parameters assert len(set_params) == 14 assert set_params[0].values[0].__root__ == 'all personnel' assert set_params[1].param_id == 'ac-1_prm_2' assert set_params[1].values[0].__root__ == 'Organization-level' assert set_params[1].values[1].__root__ == 'System-level' assert set_params[2].values[0].__root__ == 'new value' assert profile.metadata.version.__root__ == new_version if ohv: assert set_params[3].values[0].__root__ == 'no meetings' assert set_params[3].label == 'meetings cancelled' else: assert set_params[3].values[0].__root__ == 'all meetings' assert set_params[3].label == 'organization-defined events' catalog = ProfileResolver.get_resolved_profile_catalog( tmp_trestle_dir, assembled_prof_dir / 'profile.json') catalog_interface = CatalogInterface(catalog) # confirm presence of all expected strings in the control named parts for name, exp_str in multi_guidance_dict['name_exp']: prose = catalog_interface.get_control_part_prose('ac-1', name) assert prose.find(exp_str) >= 0 else: with pytest.raises(TrestleError): ProfileAssemble.assemble_profile(tmp_trestle_dir, prof_name, md_name, assembled_prof_name, True, False, new_version, required_sections, None)
def filter_ssp(self, trestle_root: pathlib.Path, ssp_name: str, profile_name: str, out_name: str, regenerate: bool, version: Optional[str]) -> int: """ Filter the ssp based on controls included by the profile and output new ssp. Args: trestle_root: root directory of the trestle project ssp_name: name of the ssp model profile_name: name of the profile model used for filtering out_name: name of the output ssp model with filtered controls regenerate: whether to regenerate the uuid's in the ssp version: new version for the model Returns: 0 on success, 1 otherwise """ ssp: ossp.SystemSecurityPlan ssp, _ = ModelUtils.load_top_level_model(trestle_root, ssp_name, ossp.SystemSecurityPlan, FileContentType.JSON) profile_path = ModelUtils.path_for_top_level_model( trestle_root, profile_name, prof.Profile, FileContentType.JSON) prof_resolver = ProfileResolver() catalog = prof_resolver.get_resolved_profile_catalog( trestle_root, profile_path) catalog_interface = CatalogInterface(catalog) # The input ssp should reference a superset of the controls referenced by the profile # Need to cull references in the ssp to controls not in the profile # Also make sure the output ssp contains imp reqs for all controls in the profile control_imp = ssp.control_implementation ssp_control_ids: Set[str] = set() new_set_params: List[ossp.SetParameter] = [] for set_param in as_list(control_imp.set_parameters): control = catalog_interface.get_control_by_param_id( set_param.param_id) if control is not None: new_set_params.append(set_param) ssp_control_ids.add(control.id) control_imp.set_parameters = new_set_params if new_set_params else None new_imp_requirements: List[ossp.ImplementedRequirement] = [] for imp_requirement in as_list(control_imp.implemented_requirements): control = catalog_interface.get_control(imp_requirement.control_id) if control is not None: new_imp_requirements.append(imp_requirement) ssp_control_ids.add(control.id) control_imp.implemented_requirements = new_imp_requirements # make sure all controls in the profile have implemented reqs in the final ssp if not ssp_control_ids.issuperset(catalog_interface.get_control_ids()): raise TrestleError( 'Unable to filter the ssp because the profile references controls not in it.' ) ssp.control_implementation = control_imp if regenerate: ssp, _, _ = ModelUtils.regenerate_uuids(ssp) if version: ssp.metadata.version = com.Version(__root__=version) ModelUtils.update_last_modified(ssp) ModelUtils.save_top_level_model(ssp, trestle_root, out_name, FileContentType.JSON) return CmdReturnCodes.SUCCESS.value
def _run(self, args: argparse.Namespace) -> int: try: log.set_log_level_from_args(args) trestle_root = args.trestle_root if not file_utils.is_directory_name_allowed(args.output): raise TrestleError( f'{args.output} is not an allowed directory name') profile_path = trestle_root / f'profiles/{args.profile}/profile.json' yaml_header: dict = {} if args.yaml_header: try: logging.debug( f'Loading yaml header file {args.yaml_header}') yaml = YAML() yaml_header = yaml.load( pathlib.Path(args.yaml_header).open('r')) except YAMLError as e: raise TrestleError( f'YAML error loading yaml header for ssp generation: {e}' ) markdown_path = trestle_root / args.output profile_resolver = ProfileResolver() resolved_catalog = profile_resolver.get_resolved_profile_catalog( trestle_root, profile_path) catalog_interface = CatalogInterface(resolved_catalog) sections_dict: Dict[str, str] = {} if args.sections: sections_dict = sections_to_dict(args.sections) if 'statement' in sections_dict: raise TrestleError( 'Statement is not allowed as a section name.') # add any existing sections from the controls but only have short names control_section_short_names = catalog_interface.get_sections() for short_name in control_section_short_names: if short_name not in sections_dict: sections_dict[short_name] = short_name logger.debug(f'ssp sections dict: {sections_dict}') catalog_interface.write_catalog_as_markdown( md_path=markdown_path, yaml_header=yaml_header, sections_dict=sections_dict, prompt_responses=True, additional_content=False, profile=None, overwrite_header_values=args.overwrite_header_values, set_parameters=False, required_sections=None, allowed_sections=args.allowed_sections) return CmdReturnCodes.SUCCESS.value except Exception as e: # pragma: no cover return handle_generic_command_exception( e, logger, 'Error while writing markdown from catalog')
def test_deep_catalog() -> None: """Test ssp generation with deep catalog.""" catalog = test_utils.generate_complex_catalog() interface = CatalogInterface(catalog) assert interface.get_count_of_controls_in_catalog(False) == 13 assert interface.get_count_of_controls_in_catalog(True) == 16
def _run(self, args: argparse.Namespace) -> int: try: log.set_log_level_from_args(args) trestle_root = pathlib.Path(args.trestle_root) md_path = trestle_root / args.markdown # the original, reference ssp name defaults to same as output if name not specified # thus in cyclic editing you are reading and writing same json ssp orig_ssp_name = args.output if args.name: orig_ssp_name = args.name new_ssp_name = args.output # if orig ssp exists - need to load it rather than instantiate new one orig_ssp_path = ModelUtils.path_for_top_level_model( trestle_root, orig_ssp_name, ossp.SystemSecurityPlan, FileContentType.JSON) # if output ssp already exists, load it to see if new one is different existing_ssp: Optional[ossp.SystemSecurityPlan] = None new_ssp_path = ModelUtils.path_for_top_level_model( trestle_root, new_ssp_name, ossp.SystemSecurityPlan, FileContentType.JSON) if new_ssp_path.exists(): _, _, existing_ssp = ModelUtils.load_distributed( new_ssp_path, trestle_root) ssp: ossp.SystemSecurityPlan comp_dict: Dict[str, ossp.SystemComponent] = {} # need to load imp_reqs from markdown but need component first if orig_ssp_path.exists(): # load the existing json ssp _, _, ssp = ModelUtils.load_distributed( orig_ssp_path, trestle_root) for component in ssp.system_implementation.components: comp_dict[component.title] = component # read the new imp reqs from markdown and have them reference existing components imp_reqs = CatalogInterface.read_catalog_imp_reqs( md_path, comp_dict) self._merge_imp_reqs(ssp, imp_reqs) else: # create a sample ssp to hold all the parts ssp = gens.generate_sample_model(ossp.SystemSecurityPlan) # load the imp_reqs from markdown and create components as needed, referenced by ### headers imp_reqs = CatalogInterface.read_catalog_imp_reqs( md_path, comp_dict) # create system implementation system_imp: ossp.SystemImplementation = gens.generate_sample_model( ossp.SystemImplementation) ssp.system_implementation = system_imp # create a control implementation to hold the implementated requirements control_imp: ossp.ControlImplementation = gens.generate_sample_model( ossp.ControlImplementation) control_imp.implemented_requirements = imp_reqs control_imp.description = const.SSP_SYSTEM_CONTROL_IMPLEMENTATION_TEXT # insert the parts into the ssp ssp.control_implementation = control_imp ssp.system_implementation = system_imp # we don't have access to the original profile so we don't know the href import_profile: ossp.ImportProfile = gens.generate_sample_model( ossp.ImportProfile) import_profile.href = 'REPLACE_ME' ssp.import_profile = import_profile # now that we know the complete list of needed components, add them to the sys_imp # TODO if the ssp already existed then components may need to be removed if not ref'd by imp_reqs component_list: List[ossp.SystemComponent] = [] for comp in comp_dict.values(): component_list.append(comp) if ssp.system_implementation.components: # reconstruct list with same order as existing, but add/remove components as needed new_list: List[ossp.SystemComponent] = [] for comp in ssp.system_implementation.components: if comp in component_list: new_list.append(comp) for comp in component_list: if comp not in new_list: new_list.append(comp) ssp.system_implementation.components = new_list elif component_list: ssp.system_implementation.components = component_list self._generate_roles_in_metadata(ssp) if args.version: ssp.metadata.version = com.Version(__root__=args.version) if existing_ssp == ssp: logger.info( 'No changes to assembled ssp so ssp not written out.') return CmdReturnCodes.SUCCESS.value if args.regenerate: ssp, _, _ = ModelUtils.regenerate_uuids(ssp) ModelUtils.update_last_modified(ssp) # write out the ssp as json ModelUtils.save_top_level_model(ssp, trestle_root, new_ssp_name, FileContentType.JSON) return CmdReturnCodes.SUCCESS.value except Exception as e: # pragma: no cover return handle_generic_command_exception( e, logger, 'Error while assembling SSP')
def assemble_profile(trestle_root: pathlib.Path, orig_profile_name: str, md_name: str, new_profile_name: str, set_parameters: bool, regenerate: bool, version: Optional[str], required_sections: Optional[str], allowed_sections: Optional[List[str]]) -> int: """ Assemble the markdown directory into a json profile model file. Args: trestle_root: The trestle root directory orig_profile_name: Optional name of profile used to generate the markdown (default is new_profile_name) md_name: The name of the directory containing the markdown control files for the profile new_profile_name: The name of the new json profile. It can be the same as original to overwrite set_parameters: Use the parameters in the yaml header to specify values for setparameters in the profile regenerate: Whether to regenerate the uuid's in the profile version: Optional version for the assembled profile required_sections: Optional List of required sections in assembled profile, as comma-separated short names allowed_sections: Optional list of section short names that are allowed, as comma-separated short names Returns: 0 on success, 1 otherwise Notes: There must already be a json profile model and it will either be updated or a new json profile created. The generated markdown shows the current values for parameters of controls being imported, as set by the original catalog and any intermediate profiles. It also shows the current SetParameters being applied by this profile. That list of SetParameters can be edited by changing the assigned values and adding or removing SetParameters from that list. During assembly that list will be used to create the SetParameters in the assembled profile if the --set-parameters option is specified. """ # first load the original json profile but call it 'new' because it is being updated with new items profile_path = trestle_root / f'profiles/{orig_profile_name}/profile.json' _, _, new_profile = ModelUtils.load_distributed( profile_path, trestle_root) required_sections_list = required_sections.split( ',') if required_sections else [] # load the editable sections of the markdown and create Adds for them # then overwrite the Adds in the existing profile with the new ones # keep track if any changes were made md_dir = trestle_root / md_name found_alters, param_dict = CatalogInterface.read_additional_content( md_dir, required_sections_list) if allowed_sections: for alter in found_alters: for add in alter.adds: for part in add.parts: if part.name not in allowed_sections: raise TrestleError( f'Profile has alter with name {part.name} not in allowed sections.' ) ProfileAssemble._replace_alter_adds(new_profile, found_alters) if set_parameters: ProfileAssemble._replace_modify_set_params(new_profile, param_dict) new_profile_dir = trestle_root / f'profiles/{new_profile_name}' new_profile_path = new_profile_dir / 'profile.json' if version: new_profile.metadata.version = com.Version(__root__=version) if new_profile_path.exists(): # first do simple and fast tests of changes so that the full equality check is only done as last resort _, _, existing_profile = ModelUtils.load_distributed( new_profile_path, trestle_root) # need special handling in case modify is None in either or both profiles no_changes = not existing_profile.modify and not new_profile.modify both_modifies_present = existing_profile.modify and new_profile.modify if both_modifies_present: no_changes = (existing_profile.modify.alters == new_profile.modify.alters and existing_profile.modify.set_parameters == new_profile.modify.set_parameters) if no_changes and existing_profile.metadata.version != new_profile.metadata.version: no_changes = False # at this point if no known changes then do the full and possibly expensive comparison to confirm if no_changes and existing_profile == new_profile: logger.info( 'Assembled profile has no changes so no update of existing file.' ) return CmdReturnCodes.SUCCESS.value if regenerate: new_profile, _, _ = ModelUtils.regenerate_uuids(new_profile) ModelUtils.update_last_modified(new_profile) if new_profile_dir.exists(): logger.info( 'Creating profile from markdown and destination profile directory exists, so updating.' ) try: shutil.rmtree(str(new_profile_dir)) except OSError as e: raise TrestleError( f'OSError deleting existing catalog directory with rmtree {new_profile_dir}: {e}' ) try: new_profile_dir.mkdir() new_profile.oscal_write(new_profile_dir / 'profile.json') except OSError as e: raise TrestleError( f'OSError writing profile from markdown to {new_profile_dir}: {e}' ) return CmdReturnCodes.SUCCESS.value
def find_string_in_all_controls_prose(interface: CatalogInterface, seek_str: str) -> List[Tuple[str, str]]: """Find all instances of this string in catalog prose and return with control id.""" hits: List[Tuple[str, str]] = [] for control in interface.get_all_controls_from_catalog(True): hits.extend(interface.find_string_in_control(control, seek_str)) return hits
class Modify(Pipeline.Filter): """Modify the controls based on the profile.""" def __init__( self, profile: prof.Profile, change_prose: bool = False, block_adds: bool = False, block_params: bool = False, params_format: str = None, param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES ) -> None: """Initialize the filter.""" self._profile = profile self._catalog_interface: Optional[CatalogInterface] = None self._block_adds = block_adds self._block_params = block_params self._change_prose = change_prose self._params_format = params_format self._param_rep = param_rep logger.debug( f'modify initialize filter with profile {profile.metadata.title}') @staticmethod def _replace_ids_with_text(prose: str, param_rep: ParameterRep, param_dict: Dict[str, common.Parameter]) -> str: """Find all instances of param_ids in prose and replace each with corresponding parameter representation. Need to check all values in dict for a match Reject matches where the string has an adjacent alphanumeric char: param_1 and param_10 or aparam_1 """ for param in param_dict.values(): if param.id not in prose: continue # create the replacement text for the param_id param_str = ControlIOReader.param_to_str(param, param_rep) # non-capturing groups are odd in re.sub so capture all 3 groups and replace the middle one pattern = r'(^|[^a-zA-Z0-9_])' + param.id + r'($|[^a-zA-Z0-9_])' prose = re.sub(pattern, r'\1' + param_str + r'\2', prose) return prose @staticmethod def _replace_params( text: str, param_dict: Dict[str, common.Parameter], params_format: Optional[str] = None, param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES ) -> str: """ Replace params found in moustaches with values from the param_dict. A single line of prose may contain multiple moustaches. """ # first check if there are any moustache patterns in the text if param_rep == ParameterRep.LEAVE_MOUSTACHE: return text staches: List[str] = re.findall(r'{{.*?}}', text) if not staches: return text # now have list of all staches including braces, e.g. ['{{foo}}', '{{bar}}'] # clean the staches so they just have the param ids param_ids = [] for stache in staches: # remove braces so these are just param_ids but may have extra chars stache_contents = stache[2:(-2)] param_id = stache_contents.replace('insert: param,', '').strip() param_ids.append(param_id) # now replace original stache text with param values for i, _ in enumerate(staches): # A moustache may refer to a param_id not listed in the control's params if param_ids[i] not in param_dict: logger.warning( f'Control prose references param {param_ids[i]} not found in the control.' ) elif param_dict[param_ids[i]] is not None: param = param_dict[param_ids[i]] param_str = ControlIOReader.param_to_str( param, param_rep, False, False, params_format) text = text.replace(staches[i], param_str, 1) else: logger.warning( f'Control prose references param {param_ids[i]} with no specified value.' ) return text @staticmethod def _replace_part_prose( control: cat.Control, part: common.Part, param_dict: Dict[str, common.Parameter], params_format: Optional[str] = None, param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES ) -> None: """Replace the part prose according to set_param.""" if part.prose is not None: fixed_prose = Modify._replace_params(part.prose, param_dict, params_format, param_rep) # change the prose in the control itself part.prose = fixed_prose for prt in as_list(part.parts): Modify._replace_part_prose(control, prt, param_dict, params_format, param_rep) for sub_control in as_list(control.controls): for prt in as_list(sub_control.parts): Modify._replace_part_prose(sub_control, prt, param_dict, params_format, param_rep) @staticmethod def _replace_control_prose( control: cat.Control, param_dict: Dict[str, common.Parameter], params_format: Optional[str] = None, param_rep: ParameterRep = ParameterRep.VALUE_OR_LABEL_OR_CHOICES ) -> None: """Replace the control prose according to set_param.""" for part in as_list(control.parts): if part.prose is not None: fixed_prose = Modify._replace_params(part.prose, param_dict, params_format, param_rep) # change the prose in the control itself part.prose = fixed_prose for prt in as_list(part.parts): Modify._replace_part_prose(control, prt, param_dict, params_format, param_rep) for sub_control in as_list(control.controls): prts: List[common.Part] = as_list(sub_control.parts) for prt in prts: Modify._replace_part_prose(sub_control, prt, param_dict, params_format, param_rep) @staticmethod def _add_contents_as_list(add: prof.Add) -> List[OBT]: add_list = [] add_list.extend(as_list(add.props)) add_list.extend(as_list(add.parts)) add_list.extend(as_list(add.links)) return add_list @staticmethod def _add_adds_to_part(part: common.Part, add: prof.Add) -> None: for attr in ['params', 'props', 'parts', 'links']: add_list = getattr(add, attr, None) if add_list: Modify._add_attr_to_part(part, add_list, attr, add.position) @staticmethod def _add_to_list(input_list: List[OBT], add: prof.Add) -> bool: """Add the contents of the add according to its by_id and position. Return True on success or False if id needed and not found. This is only called when by_id is not None. The add will be inserted if the id is found, or return False if not. This allows a separate recursive routine to search sub-lists for the id. """ add_list = Modify._add_contents_as_list(add) # Test here for matched by_id attribute. try: for index in range(len(input_list)): if input_list[index].id == add.by_id: if add.position == prof.Position.after: for offset, new_item in enumerate(add_list): input_list.insert(index + 1 + offset, new_item) return True elif add.position == prof.Position.before: for offset, new_item in enumerate(add_list): input_list.insert(index + offset, new_item) return True # if starting or ending, the adds go directly into this part according to type Modify._add_adds_to_part(input_list[index], add) return True except AttributeError: raise TrestleError( 'Cannot use "after" or "before" modifications for a list where elements' + ' do not contain the referenced by_id attribute.') return False @staticmethod def _add_to_parts(parts: List[common.Part], add: prof.Add) -> bool: """ Add the add to the parts. This is only called if add.by_id is not None. """ if Modify._add_to_list(parts, add): return True for part in parts: if part.parts is not None and Modify._add_to_parts( part.parts, add): return True return False @staticmethod def _add_attr_to_part(part: common.Part, items: List[OBT], attr: str, position: prof.Position) -> None: attr_list = as_list(getattr(part, attr, None)) if position in [prof.Position.starting, prof.Position.before]: items.extend(attr_list) attr_list = items else: attr_list.extend(items) setattr(part, attr, attr_list) @staticmethod def _add_attr_to_control(control: cat.Control, items: List[OBT], attr: str, position: prof.Position) -> None: attr_list = as_list(getattr(control, attr, None)) if position in [prof.Position.starting, prof.Position.before]: items.extend(attr_list) attr_list = items else: attr_list.extend(items) setattr(control, attr, attr_list) @staticmethod def _add_to_control(control: cat.Control, add: prof.Add) -> None: """First step in applying Add to control.""" control.parts = as_list(control.parts) if add.by_id is None or add.by_id == control.id: # add contents will be added to the control directly and with no recursion for attr in ['params', 'props', 'parts', 'links']: add_list = getattr(add, attr, None) if add_list: Modify._add_attr_to_control(control, add_list, attr, add.position) return else: # this is only called if by_id is not None if not Modify._add_to_parts(control.parts, add): logger.warning( f'Could not find id for add in control {control.id}: {add.by_id}' ) def _set_parameter_in_control(self, set_param: prof.SetParameter) -> None: """ Find the control with the param_id in it and set the parameter value. This does not recurse because expectation is that only top level params will be set. """ control = self._catalog_interface.get_control_by_param_id( set_param.param_id) if control is None: raise TrestleError( f'Set parameter object in profile does not have a corresponding param-id: "{set_param.param_id}"' ) control.params = as_list(control.params) param_ids = [param.id for param in control.params] if set_param.param_id not in param_ids: raise TrestleNotFoundError( f'Param id {set_param.param_id} not found in control {control.id}' ) index = param_ids.index(set_param.param_id) param = control.params[index] param.values = set_param.values param.constraints = set_param.constraints param.guidelines = set_param.guidelines param.links = set_param.links param.props = set_param.props param.select = set_param.select param.usage = set_param.usage control.params[index] = param self._catalog_interface.replace_control(control) def _change_prose_with_param_values(self): """Go through all controls and change prose based on param values.""" param_dict: Dict[str, common.Paramter] = {} # build the full mapping of params to values for control in self._catalog_interface.get_all_controls_from_dict(): param_dict.update( ControlIOReader.get_control_param_dict(control, False)) # insert param values into prose of all controls for control in self._catalog_interface.get_all_controls_from_dict(): self._replace_control_prose(control, param_dict, self._params_format, self._param_rep) def _modify_controls(self, catalog: cat.Catalog) -> cat.Catalog: """Modify the controls based on the profile.""" logger.debug( f'modify specify catalog {catalog.metadata.title} for profile {self._profile.metadata.title}' ) self._catalog_interface = CatalogInterface(catalog) alters: Optional[List[prof.Alter]] = None # find the modify and alters if self._profile.modify: # change all parameter values if self._profile.modify.set_parameters and not self._block_params: set_param_list = self._profile.modify.set_parameters for set_param in set_param_list: self._set_parameter_in_control(set_param) alters = self._profile.modify.alters if alters is not None: title = self._profile.metadata.title for alter in alters: if alter.control_id is None: logger.warning( f'Alter must have control id specified in profile {title}.' ) continue id_ = alter.control_id if alter.removes is not None: logger.warning( f'Alter not supported for removes in profile {title} control {id_}' ) continue # we want a warning about adds even if adds are blocked, as in profile generate if alter.adds is None: logger.warning( f'Alter has no adds in profile {title} control {id_}') continue if not self._block_adds: for add in alter.adds: if add.position is None and add.parts is not None: msg = f'Alter/Add position is not specified in profile {title} control {id_}' msg += ' when adding part, so defaulting to ending.' logger.warning(msg) add.position = prof.Position.ending control = self._catalog_interface.get_control(id_) if control is None: logger.warning( f'Alter/Add refers to control {id_} but it is not found in the import ' + f'for profile {self._profile.metadata.title}') else: self._add_to_control(control, add) self._catalog_interface.replace_control(control) if self._change_prose: # go through all controls and fix the prose based on param values self._change_prose_with_param_values() catalog = self._catalog_interface.get_catalog() # update the original profile metadata with new contents # roles and responsible-parties will be pulled in with new uuid's new_metadata = self._profile.metadata new_metadata.title = f'{catalog.metadata.title}: Resolved by profile {self._profile.metadata.title}' links: List[common.Link] = [] for import_ in self._profile.imports: links.append( common.Link(**{ 'href': import_.href, 'rel': 'resolution-source' })) new_metadata.links = links # move catalog controls from dummy group '' into the catalog for group in as_list(catalog.groups): if not group.id: catalog.controls = group.controls catalog.groups.remove(group) break catalog.metadata = new_metadata return catalog def process(self, catalog_iter: Iterator[cat.Catalog]) -> Iterator[cat.Catalog]: """Make the modifications to the controls based on the profile.""" catalog = next(catalog_iter) logger.debug( f'modify process with catalog {catalog.metadata.title} using profile {self._profile.metadata.title}' ) yield self._modify_controls(catalog)
def jinja_ify( trestle_root: pathlib.Path, r_input_file: pathlib.Path, r_output_file: pathlib.Path, ssp: Optional[str], profile: Optional[str], lut: Optional[Dict[str, Any]] = None, number_captions: Optional[bool] = False, parameters_formatting: Optional[str] = None ) -> int: """Run jinja over an input file with additional booleans.""" if lut is None: lut = {} template_folder = pathlib.Path.cwd() jinja_env = Environment( loader=FileSystemLoader(template_folder), extensions=[MDSectionInclude, MDCleanInclude, MDDatestamp], trim_blocks=True, autoescape=True ) template = jinja_env.get_template(str(r_input_file)) # create boolean dict if operator.xor(bool(ssp), bool(profile)): raise TrestleIncorrectArgsError('Both SSP and profile should be provided or not at all') if ssp: # name lookup ssp_data, _ = ModelUtils.load_top_level_model(trestle_root, ssp, SystemSecurityPlan) lut['ssp'] = ssp_data _, profile_path = ModelUtils.load_top_level_model(trestle_root, profile, Profile) profile_resolver = ProfileResolver() resolved_catalog = profile_resolver.get_resolved_profile_catalog( trestle_root, profile_path, False, False, parameters_formatting ) ssp_writer = SSPMarkdownWriter(trestle_root) ssp_writer.set_ssp(ssp_data) ssp_writer.set_catalog(resolved_catalog) lut['catalog'] = resolved_catalog lut['catalog_interface'] = CatalogInterface(resolved_catalog) lut['control_io_writer'] = ControlIOWriter() lut['ssp_md_writer'] = ssp_writer new_output = template.render(**lut) output = '' # This recursion allows nesting within expressions (e.g. an expression can contain jinja templates). error_countdown = JinjaCmd.max_recursion_depth while new_output != output and error_countdown > 0: error_countdown = error_countdown - 1 output = new_output random_name = uuid.uuid4() # Should be random and not used. dict_loader = DictLoader({str(random_name): new_output}) jinja_env = Environment( loader=ChoiceLoader([dict_loader, FileSystemLoader(template_folder)]), extensions=[MDCleanInclude, MDSectionInclude, MDDatestamp], autoescape=True, trim_blocks=True ) template = jinja_env.get_template(str(random_name)) new_output = template.render(**lut) output_file = trestle_root / r_output_file if number_captions: output_file.open('w', encoding=const.FILE_ENCODING).write(_number_captions(output)) else: output_file.open('w', encoding=const.FILE_ENCODING).write(output) return CmdReturnCodes.SUCCESS.value
def test_profile_resolver_merge(sample_catalog_rich_controls: cat.Catalog) -> None: """Test profile resolver merge.""" profile = gens.generate_sample_model(prof.Profile) method = prof.Method.merge combine = prof.Combine(method=method) profile.merge = prof.Merge(combine=combine) merge = Merge(profile) # merge into empty catalog merged = gens.generate_sample_model(cat.Catalog) new_merged = merge._merge_catalog(merged, sample_catalog_rich_controls) catalog_interface = CatalogInterface(new_merged) assert catalog_interface.get_count_of_controls_in_catalog(True) == 5 # add part to first control and merge, then make sure it is there part = com.Part(name='foo', title='added part') control_id = sample_catalog_rich_controls.controls[0].id cat_with_added_part = copy.deepcopy(sample_catalog_rich_controls) cat_with_added_part.controls[0].parts.append(part) final_merged = merge._merge_catalog(sample_catalog_rich_controls, cat_with_added_part) catalog_interface = CatalogInterface(final_merged) assert catalog_interface.get_count_of_controls_in_catalog(True) == 5 assert catalog_interface.get_control(control_id).parts[-1].name == 'foo' # add part to first control and merge but with use-first. The part should not be there at end. method = prof.Method.use_first combine = prof.Combine(method=method) profile.merge = prof.Merge(combine=combine) merge = Merge(profile) final_merged = merge._merge_catalog(sample_catalog_rich_controls, cat_with_added_part) catalog_interface = CatalogInterface(final_merged) assert catalog_interface.get_count_of_controls_in_catalog(True) == 5 assert len(catalog_interface.get_control(control_id).parts) == 1 # now force a merge with keep profile.merge = None merge_keep = Merge(profile) merged_keep = merge_keep._merge_catalog(new_merged, sample_catalog_rich_controls) assert CatalogInterface(merged_keep).get_count_of_controls_in_catalog(True) == 10