def test_update_action(sample_nist_component_def): """Test update action.""" element = Element(sample_nist_component_def) metadata = common.Metadata( **{ 'title': 'My simple catalog', 'last-modified': datetime.now().astimezone(), 'version': '0.0.0', 'oscal-version': OSCAL_VERSION }) sub_element_path = ElementPath('component-definition.metadata') prev_metadata = element.get_at(sub_element_path) uac = UpdateAction(metadata, element, sub_element_path) uac.execute() assert element.get_at(sub_element_path) is not prev_metadata assert element.get_at(sub_element_path) == metadata uac.rollback() assert element.get_at(sub_element_path) == prev_metadata assert element.get_at(sub_element_path) is not metadata
def test_update_action(sample_target): """Test update action.""" element = Element(sample_target) metadata = target.Metadata( **{ 'title': 'My simple catalog', 'last-modified': datetime.now().astimezone(), 'version': '0.0.0', 'oscal-version': '1.0.0-Milestone3' }) sub_element_path = ElementPath('metadata') prev_metadata = element.get_at(sub_element_path) uac = UpdateAction(metadata, element, sub_element_path) uac.execute() assert element.get_at(sub_element_path) is not prev_metadata assert element.get_at(sub_element_path) == metadata uac.rollback() assert element.get_at(sub_element_path) == prev_metadata assert element.get_at(sub_element_path) is not metadata
def test_add(tmp_path: pathlib.Path, keep_cwd: pathlib.Path) -> None: """Test Add.add() method used by trestle CreateCmd.""" file_path = pathlib.Path( test_utils.JSON_TEST_DATA_PATH) / 'minimal_catalog_missing_roles.json' minimal_catalog_missing_roles = Catalog.oscal_read(file_path) # expected catalog after first add of Role file_path = pathlib.Path.joinpath(test_utils.JSON_TEST_DATA_PATH, 'minimal_catalog_roles.json') expected_catalog_roles1 = Element(Catalog.oscal_read(file_path)) # expected catalog after second add of Role file_path = pathlib.Path.joinpath(test_utils.JSON_TEST_DATA_PATH, 'minimal_catalog_roles_double.json') expected_catalog_roles2 = Element(Catalog.oscal_read(file_path)) # expected catalog after add of Responsible-Party file_path = pathlib.Path.joinpath(test_utils.JSON_TEST_DATA_PATH, 'minimal_catalog_roles_double_rp.json') expected_catalog_roles2_rp = Element(Catalog.oscal_read(file_path)) content_type = FileContentType.JSON _, _ = test_utils.prepare_trestle_project_dir( tmp_path, content_type, minimal_catalog_missing_roles, test_utils.CATALOGS_DIR) # Execute first _add element_path = ElementPath('catalog.metadata.roles') catalog_element = Element(minimal_catalog_missing_roles) expected_update_action_1 = UpdateAction( expected_catalog_roles1.get_at(element_path), catalog_element, element_path) actual_update_action, actual_catalog_roles = Add.add( element_path, catalog_element, False) assert actual_catalog_roles == expected_catalog_roles1 assert actual_update_action == expected_update_action_1 # Execute second _add - this time roles already exists, so this adds a roles object to roles array catalog_element = actual_catalog_roles expected_update_action_2 = UpdateAction( expected_catalog_roles2.get_at(element_path), catalog_element, element_path) actual_update_action2, actual_catalog_roles2 = Add.add( element_path, catalog_element, False) assert actual_catalog_roles2 == expected_catalog_roles2 assert actual_update_action2 == expected_update_action_2 # Execute _add for responsible-parties to the same catalog element_path = ElementPath('catalog.metadata.responsible-parties') catalog_element = actual_catalog_roles2 expected_update_action_3 = UpdateAction( expected_catalog_roles2_rp.get_at(element_path), catalog_element, element_path) actual_update_action3, actual_catalog_roles2_rp = Add.add( element_path, catalog_element, False) assert actual_catalog_roles2_rp == expected_catalog_roles2_rp assert actual_update_action3 == expected_update_action_3
def test_split_chained_sub_model_plans( tmp_path: pathlib.Path, simplified_nist_catalog: oscatalog.Catalog, keep_cwd: pathlib.Path) -> None: """Test for split_model method with chained sum models like catalog.metadata.parties.*.""" # Assume we are running a command like below # trestle split -f catalog.json -e catalog.metadata.parties.* # see https://github.com/IBM/compliance-trestle/issues/172 content_type = FileContentType.JSON # prepare trestle project dir with the file catalog_dir, catalog_file = test_utils.prepare_trestle_project_dir( tmp_path, content_type, simplified_nist_catalog, test_utils.CATALOGS_DIR) # read the model from file catalog = oscatalog.Catalog.oscal_read(catalog_file) element = Element(catalog) element_args = ['catalog.metadata.parties.*'] element_paths = cmd_utils.parse_element_args( None, element_args, catalog_dir.relative_to(tmp_path)) assert 2 == len(element_paths) expected_plan = Plan() # prepare to extract metadata and parties metadata_file = catalog_dir / element_paths[0].to_file_path(content_type) metadata_field_alias = element_paths[0].get_element_name() metadata = element.get_at(element_paths[0]) meta_element = Element(metadata, metadata_field_alias) # extract parties parties_dir = catalog_dir / 'catalog/metadata/parties' for i, party in enumerate(meta_element.get_at(element_paths[1], False)): prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH) sub_model_actions = SplitCmd.prepare_sub_model_split_actions( party, parties_dir, prefix, content_type) expected_plan.add_actions(sub_model_actions) # stripped metadata stripped_metadata = metadata.stripped_instance( stripped_fields_aliases=['parties']) expected_plan.add_action(CreatePathAction(metadata_file)) expected_plan.add_action( WriteFileAction(metadata_file, Element(stripped_metadata, metadata_field_alias), content_type)) # stripped catalog root_file = catalog_dir / element_paths[0].to_root_path(content_type) remaining_root = element.get().stripped_instance(metadata_field_alias) expected_plan.add_action(CreatePathAction(root_file, True)) expected_plan.add_action( WriteFileAction(root_file, Element(remaining_root), content_type)) split_plan = SplitCmd.split_model(catalog, element_paths, catalog_dir, content_type, '', None) assert expected_plan == split_plan
def prepare_element(sample_target_def): """Prepare a target element for remove tests.""" element = Element(sample_target_def) parties: List[target.Party] = [] parties.append( target.Party( **{ 'uuid': 'ff47836c-877c-4007-bbf3-c9d9bd805000', 'party-name': 'TEST1', 'type': 'organization' })) parties.append( target.Party( **{ 'uuid': 'ee88836c-877c-4007-bbf3-c9d9bd805000', 'party-name': 'TEST2', 'type': 'organization' })) sub_element_path = ElementPath('target-definition.metadata.parties.*') ac = UpdateAction(parties, element, sub_element_path) ac.execute() assert element.get_at(sub_element_path) == parties return element
def test_update_list_sub_element_action(sample_target): """Test setting a list.""" element = Element(sample_target) parties: List[target.Party] = [] parties.append( target.Party( **{ 'uuid': 'ff47836c-877c-4007-bbf3-c9d9bd805000', 'party-name': 'TEST1', 'type': 'organization' })) parties.append( target.Party( **{ 'uuid': 'ee88836c-877c-4007-bbf3-c9d9bd805000', 'party-name': 'TEST2', 'type': 'organization' })) sub_element_path = ElementPath('metadata.parties.*') uac = UpdateAction(parties, element, sub_element_path) uac.execute() assert element.get_at(sub_element_path) == parties
def add(element_path: ElementPath, parent_element: Element, include_optional: bool) -> None: """For a element_path, add a child model to the parent_element of a given parent_model. Args: element_path: element path of the item to create within the model parent_element: the parent element that will host the created element include_optional: whether to create optional attributes in the created element Notes: First we find the child model at the specified element path and instantiate it with default values. Then we check if there's already existing element at that path, in which case we append the child model to the existing list of dict. Then we set up an action plan to update the model (specified by file_path) in memory, create a file at the same location and write the file. We update the parent_element to prepare for next adds in the chain """ if '*' in element_path.get_full_path_parts(): raise err.TrestleError('trestle add does not support Wildcard element path.') # Get child model try: child_model = element_path.get_type(type(parent_element.get())) # Create child element with sample values child_object = gens.generate_sample_model(child_model, include_optional=include_optional) if parent_element.get_at(element_path) is not None: # The element already exists if type(parent_element.get_at(element_path)) is list: child_object = parent_element.get_at(element_path) + child_object elif type(parent_element.get_at(element_path)) is dict: child_object = {**parent_element.get_at(element_path), **child_object} else: raise err.TrestleError('Already exists and is not a list or dictionary.') except Exception as e: raise err.TrestleError(f'Bad element path. {str(e)}') update_action = UpdateAction( sub_element=child_object, dest_element=parent_element, sub_element_path=element_path ) parent_element = parent_element.set_at(element_path, child_object) return update_action, parent_element
def test_element_get_at(sample_target: target.TargetDefinition): """Test element get method.""" element = Element(sample_target) assert element.get() == sample_target assert element.get_at() == element.get() assert element.get_at(ElementPath('metadata')) == sample_target.metadata assert element.get_at( ElementPath('metadata.title')) == sample_target.metadata.title assert element.get_at(ElementPath('targets')) == sample_target.targets assert element.get_at(ElementPath('targets.*')) == sample_target.targets assert element.get_at( ElementPath('metadata.parties.*')) == sample_target.metadata.parties assert element.get_at( ElementPath('metadata.parties.0')) == sample_target.metadata.parties[0] assert element.get_at(ElementPath( 'metadata.parties.0.uuid')) == sample_target.metadata.parties[0].uuid # invalid indexing assert element.get_at(ElementPath('metadata.title.0')) is None
def test_split_model_plans( tmp_path: pathlib.Path, sample_nist_component_def: component.ComponentDefinition) -> None: """Test for split_model method.""" # Assume we are running a command like below # trestle split -f component-definition.yaml -e component-definition.metadata content_type = FileContentType.YAML # prepare trestle project dir with the file component_def_dir, component_def_file = test_utils.prepare_trestle_project_dir( tmp_path, content_type, sample_nist_component_def, test_utils.COMPONENT_DEF_DIR) # read the model from file component_def = component.ComponentDefinition.oscal_read( component_def_file) element = Element(component_def) element_args = ['component-definition.metadata'] element_paths = cmd_utils.parse_element_args(None, element_args) # extract values metadata_file = component_def_dir / element_paths[0].to_file_path( content_type) metadata = element.get_at(element_paths[0]) root_file = component_def_dir / element_paths[0].to_root_path(content_type) remaining_root = element.get().stripped_instance( element_paths[0].get_element_name()) # prepare the plan expected_plan = Plan() expected_plan.add_action(CreatePathAction(metadata_file)) expected_plan.add_action( WriteFileAction(metadata_file, Element(metadata), content_type)) expected_plan.add_action(CreatePathAction(root_file, True)) expected_plan.add_action( WriteFileAction(root_file, Element(remaining_root), content_type)) split_plan = SplitCmd.split_model(component_def, element_paths, component_def_dir, content_type, '', None) assert expected_plan == split_plan
def remove(cls, element_path: ElementPath, parent_model: Type[OscalBaseModel], parent_element: Element) -> Tuple[RemoveAction, Element]: """For the element_path, remove a model from the parent_element of a given parent_model. First we check if there is an existing element at that path If not, we complain. Then we set up an action plan to update the model (specified by file_path) in memory, return the action and return the parent_element. LIMITATIONS: 1. This does not remove elements of a list or dict. Instead, the entire list or dict is removed. 2. This cannot remove arbitrarily named elements that are not specified in the schema. For example, "responsible-parties" contains named elements, e.g., "organisation". The tool will not remove the "organisation" as it is not in the schema, but one can remove its elements, e.g., "party-uuids". """ element_path_list = element_path.get_full_path_parts() if '*' in element_path_list: raise err.TrestleError('trestle remove does not support Wildcard element path.') deleting_element = parent_element.get_at(element_path) if deleting_element is not None: # The element already exists if type(deleting_element) is list: logger.warning( 'Warning: trestle remove does not support removing elements of a list: ' 'this removes the entire list' ) elif type(deleting_element) is dict: logger.warning( 'Warning: trestle remove does not support removing dict elements: ' 'this removes the entire dict element' ) else: raise err.TrestleError(f'Bad element path: {str(element_path)}') remove_action = RemoveAction(parent_element, element_path) return remove_action, parent_element
def test_split_model(tmp_dir, sample_target_def: ostarget.TargetDefinition): """Test for split_model method.""" # Assume we are running a command like below # trestle split -f target-definition.yaml -e target-definition.metadata content_type = FileContentType.YAML # prepare trestle project dir with the file target_def_dir, target_def_file = test_utils.prepare_trestle_project_dir( tmp_dir, content_type, sample_target_def, test_utils.TARGET_DEFS_DIR) # read the model from file target_def = ostarget.TargetDefinition.oscal_read(target_def_file) element = Element(target_def) element_args = ['target-definition.metadata'] element_paths = cmd_utils.parse_element_args(element_args) # extract values metadata_file = target_def_dir / element_paths[0].to_file_path( content_type) metadata = element.get_at(element_paths[0]) root_file = target_def_dir / element_paths[0].to_root_path(content_type) remaining_root = element.get().stripped_instance( element_paths[0].get_element_name()) # prepare the plan expected_plan = Plan() expected_plan.add_action(CreatePathAction(metadata_file)) expected_plan.add_action( WriteFileAction(metadata_file, Element(metadata), content_type)) expected_plan.add_action(CreatePathAction(root_file, True)) expected_plan.add_action( WriteFileAction(root_file, Element(remaining_root), content_type)) split_plan = SplitCmd.split_model(target_def, element_paths, target_def_dir, content_type) assert expected_plan == split_plan
def test_element_get_at(sample_target_def: target.TargetDefinition): """Test element get method.""" element = Element(sample_target_def) # field alias should succeed assert element.get_at( ElementPath('target-definition.metadata.last-modified') ) == sample_target_def.metadata.last_modified # field name should fail assert element.get_at(ElementPath('target-definition.metadata.last_modified')) is None assert element.get() == sample_target_def assert element.get_at() == element.get() assert element.get_at(ElementPath('target-definition.metadata')) == sample_target_def.metadata assert element.get_at(ElementPath('target-definition.metadata.title')) == sample_target_def.metadata.title assert element.get_at(ElementPath('target-definition.targets')) == sample_target_def.targets assert element.get_at(ElementPath('target-definition.targets.*')) == sample_target_def.targets assert element.get_at(ElementPath('target-definition.metadata.parties.*')) == sample_target_def.metadata.parties assert element.get_at(ElementPath('target-definition.metadata.parties.0')) == sample_target_def.metadata.parties[0] assert element.get_at(ElementPath('target-definition.metadata.parties.0.uuid') ) == sample_target_def.metadata.parties[0].uuid for uuid in sample_target_def.targets: path_str = f'target-definition.targets.{uuid}' assert element.get_at(ElementPath(path_str)) == sample_target_def.targets[uuid] # invalid indexing assert element.get_at(ElementPath('target-definition.metadata.title.0')) is None # invalid path with missing root model assert element.get_at(ElementPath('metadata.title')) is None # element_path with parent path parent_path = ElementPath('target-definition.metadata') element_path = ElementPath('metadta.parties.*', parent_path) assert element.get_at(element_path) == sample_target_def.metadata.parties # element_path with parent path parent_path = ElementPath('target-definition.targets.*') element_path = ElementPath('target.target-control-implementations.*', parent_path) targets = element.get_at(parent_path) for key in targets: target = targets[key] target_element = Element(target) assert target_element.get_at(element_path) == target.target_control_implementations # element_path in a list with parent path parent_path = ElementPath('target-definition.targets.*') element_path = ElementPath('target.target-control-implementations.0', parent_path) targets = element.get_at(parent_path) for key in targets: target = targets[key] target_element = Element(target) assert target_element.get_at(element_path) == target.target_control_implementations[0]
def describe(cls, file_path: pathlib.Path, element_path_str: str, trestle_root: pathlib.Path) -> List[str]: """Describe the contents of the file. Args: file_path: pathlib.Path Path for model file to describe. element_path_str: Element path of element in model to describe. Can be ''. Returns: The list of lines of text in the description, or an empty list on failure """ # figure out the model type so we can read it try: model_type, _ = ModelUtils.get_stripped_model_type(file_path, trestle_root) model: OscalBaseModel = model_type.oscal_read(file_path) except TrestleError as e: logger.warning(f'Error loading model {file_path} to describe: {e}') return [] sub_model = model # if an element path was provided, follow the path chain to the desired sub_model if element_path_str: if '*' in element_path_str or ',' in element_path_str: logger.warning('Wildcards and commas are not allowed in the element path for describe.') return [] if '.' not in element_path_str: logger.warning('The element path for describe must either be omitted or contain at least 2 parts.') return [] element_paths = utils.parse_element_arg(model, element_path_str) sub_model_element = Element(model) for element_path in element_paths: sub_model = sub_model_element.get_at(element_path, False) sub_model_element = Element(sub_model) # now that we have the desired sub_model we can describe it text_out: List[str] = [] # create top level text depending on whether an element path was used element_text = '' if not element_path_str else f' at element path {element_path_str}' if type(sub_model) is list: text = f'Model file {file_path}{element_text} is a {cls._description_text(sub_model)}' text_out.append(text) logger.info(text) else: text = f'Model file {file_path}{element_text} is of type ' text += f'{cls._clean_type_string(str(type(sub_model)))} and contains:' text_out.append(text) logger.info(text) for key in sub_model.__fields__.keys(): value = getattr(sub_model, key, None) text = f' {key}: {cls._description_text(value)}' text_out.append(text) logger.info(text) return text_out
def split_model_at_path_chain( cls, model_obj: OscalBaseModel, element_paths: List[ElementPath], base_dir: pathlib.Path, content_type: FileContentType, cur_path_index: int, split_plan: Plan, strip_root: bool, root_file_name: str, aliases_to_strip: Dict[str, AliasTracker], last_one: bool = True ) -> int: """Recursively split the model at the provided chain of element paths. It assumes that a chain of element paths starts at the cur_path_index with the first path ending with a wildcard (*) If the wildcard follows an element that is inherently a list of items, the list of items is extracted. But if the wildcard follows a generic model than members of that model class found in the model will be split off. But only the non-trivial elements are removed, i.e. not str, int, datetime, etc. Args: model_obj: The OscalBaseModel to be split element_paths: The List[ElementPath] of elements to split, including embedded wildcards base_dir: pathlib.Path of the file being split content_type: json or yaml files cur_path_index: Index into the list of element paths for the current split operation split_plan: The accumulated plan of actions needed to perform the split strip_root: Whether to strip elements from the root object root_file_name: Filename of root file that gets split into a list of items aliases_to_strip: AliasTracker previously loaded with aliases that need to be split from each element last_one: bool indicating last item in array has been split and stripped model can now be written Returns: int representing the index where the chain of the path ends. Examples: For example, element paths could have a list of paths as below for a `ComponentDefinition` model where the first path is the start of the chain. For each of the sub model described by the first element path (e.g component-defintion.components.*) in the chain, the subsequent paths (e.g component.control-implementations.*) will be applied recursively to retrieve the sub-sub models: [ 'component-definition.component.*', 'component.control-implementations.*' ] for a command like below: trestle split -f component.yaml -e component-definition.components.*.control-implementations.* """ if split_plan is None: raise TrestleError('Split plan must have been initialized') if cur_path_index < 0: raise TrestleError('Current index of the chain of paths cannot be less than 0') # if there are no more element_paths, return the current plan if cur_path_index >= len(element_paths): return cur_path_index # initialize local variables element = Element(model_obj) stripped_field_alias: List[str] = [] # get the sub_model specified by the element_path of this round element_path = element_paths[cur_path_index] # does the next element_path point back at me is_parent = cur_path_index + 1 < len(element_paths) and element_paths[cur_path_index + 1].get_parent() == element_path # root dir name for sub models dir # 00000__group.json will have the root_dir name as 00000__group for sub models of group # catalog.json will have the root_dir name as catalog root_dir = '' if root_file_name != '': root_dir = str(pathlib.Path(root_file_name).with_suffix('')) sub_models = element.get_at(element_path, False) # we call sub_models as in plural, but it can be just one # assume cur_path_index is the end of the chain # value of this variable may change during recursive split of the sub-models below path_chain_end = cur_path_index # if wildcard is present in the element_path and the next path in the chain has current path as the parent, # Then deal with case of list, or split of arbitrary oscalbasemodel if is_parent and element_path.get_last() is not ElementPath.WILDCARD: # create dir for all sub model items sub_models_dir = base_dir / element_path.to_root_path() sub_model_plan = Plan() path_chain_end = cls.split_model_at_path_chain( sub_models, element_paths, sub_models_dir, content_type, cur_path_index + 1, sub_model_plan, True, '', aliases_to_strip ) sub_model_actions = sub_model_plan.get_actions() split_plan.add_actions(sub_model_actions) elif element_path.get_last() == ElementPath.WILDCARD: # extract sub-models into a dict with appropriate prefix sub_model_items: Dict[str, OscalBaseModel] = {} sub_models_dir = base_dir / element_path.to_file_path(root_dir=root_dir) if isinstance(sub_models, list): for i, sub_model_item in enumerate(sub_models): # e.g. `groups/00000_groups/` prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH) sub_model_items[prefix] = sub_model_item # process list sub model items count = 0 for key, sub_model_item in sub_model_items.items(): count += 1 # recursively split the sub-model if there are more element paths to traverse # e.g. split component.control-implementations.* require_recursive_split = cur_path_index + 1 < len(element_paths) and element_paths[ cur_path_index + 1].get_parent() == element_path if require_recursive_split: # prepare individual directory for each sub-model sub_root_file_name = cmd_utils.to_model_file_name(sub_model_item, key, content_type) sub_model_plan = Plan() last_one: bool = count == len(sub_model_items) path_chain_end = cls.split_model_at_path_chain( sub_model_item, element_paths, sub_models_dir, content_type, cur_path_index + 1, sub_model_plan, True, sub_root_file_name, aliases_to_strip, last_one ) sub_model_actions = sub_model_plan.get_actions() else: sub_model_actions = cls.prepare_sub_model_split_actions( sub_model_item, sub_models_dir, key, content_type ) split_plan.add_actions(sub_model_actions) else: # the chain of path ends at the current index. # so no recursive call. Let's just write the sub model to the file and get out if sub_models is not None: sub_model_file = base_dir / element_path.to_file_path(content_type, root_dir=root_dir) split_plan.add_action(CreatePathAction(sub_model_file)) split_plan.add_action( WriteFileAction(sub_model_file, Element(sub_models, element_path.get_element_name()), content_type) ) # Strip the root model and add a WriteAction for the updated model object in the plan if strip_root: full_path = element_path.get_full() path = '.'.join(full_path.split('.')[:-1]) aliases = [element_path.get_element_name()] need_to_write = True use_alias_dict = aliases_to_strip is not None and path in aliases_to_strip if use_alias_dict: aliases = aliases_to_strip[path].get_aliases() need_to_write = aliases_to_strip[path].needs_writing() stripped_model = model_obj.stripped_instance(stripped_fields_aliases=aliases) # can mark it written even if it doesn't need writing since it is empty # but if an array only mark it written if it's the last one if last_one and use_alias_dict: aliases_to_strip[path].mark_written() # If it's an empty model after stripping the fields, don't create path and don't write field_list = [x for x in model_obj.__fields__.keys() if model_obj.__fields__[x] is not None] if set(field_list) == set(stripped_field_alias): return path_chain_end if need_to_write: if root_file_name != '': root_file = base_dir / root_file_name else: root_file = base_dir / element_path.to_root_path(content_type) split_plan.add_action(CreatePathAction(root_file)) wrapper_alias = classname_to_alias(stripped_model.__class__.__name__, AliasMode.JSON) split_plan.add_action(WriteFileAction(root_file, Element(stripped_model, wrapper_alias), content_type)) # return the end of the current path chain return path_chain_end
def test_split_multi_level_dict( tmp_path: pathlib.Path, sample_target_def: ostarget.TargetDefinition) -> None: """Test for split_model method.""" # Assume we are running a command like below # trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.* content_type = FileContentType.YAML # prepare trestle project dir with the file target_def_dir, target_def_file = test_utils.prepare_trestle_project_dir( tmp_path, content_type, sample_target_def, test_utils.TARGET_DEFS_DIR) file_ext = FileContentType.to_file_extension(content_type) # read the model from file target_def: ostarget.TargetDefinition = ostarget.TargetDefinition.oscal_read( target_def_file) element = Element(target_def) element_args = [ 'target-definition.targets.*.target-control-implementations.*' ] element_paths = test_utils.prepare_element_paths(target_def_dir, element_args) expected_plan = Plan() # extract values targets: dict = element.get_at(element_paths[0]) targets_dir = target_def_dir / element_paths[0].to_file_path() # split every targets for key in targets: # individual target dir target: ostarget.Target = targets[key] target_element = Element(targets[key]) model_type = utils.classname_to_alias(type(target).__name__, 'json') dir_prefix = key target_dir_name = f'{dir_prefix}{const.IDX_SEP}{model_type}' target_file = targets_dir / f'{target_dir_name}{file_ext}' # target control impl dir for the target target_ctrl_impls: dict = target_element.get_at(element_paths[1]) targets_ctrl_dir = targets_dir / element_paths[1].to_file_path( root_dir=target_dir_name) for i, target_ctrl_impl in enumerate(target_ctrl_impls): model_type = utils.classname_to_alias( type(target_ctrl_impl).__name__, 'json') file_prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH) file_name = f'{file_prefix}{const.IDX_SEP}{model_type}{file_ext}' file_path = targets_ctrl_dir / file_name expected_plan.add_action(CreatePathAction(file_path)) expected_plan.add_action( WriteFileAction(file_path, Element(target_ctrl_impl), content_type)) # write stripped target model stripped_target = target.stripped_instance( stripped_fields_aliases=[element_paths[1].get_element_name()]) expected_plan.add_action(CreatePathAction(target_file)) expected_plan.add_action( WriteFileAction(target_file, Element(stripped_target), content_type)) root_file = target_def_dir / f'target-definition{file_ext}' remaining_root = element.get().stripped_instance( stripped_fields_aliases=[element_paths[0].get_element_name()]) expected_plan.add_action(CreatePathAction(root_file, True)) expected_plan.add_action( WriteFileAction(root_file, Element(remaining_root), content_type)) split_plan = SplitCmd.split_model(target_def, element_paths, target_def_dir, content_type) assert expected_plan == split_plan
def test_subsequent_split_model( tmp_path: pathlib.Path, sample_target_def: ostarget.TargetDefinition) -> None: """Test subsequent split of sub models.""" # Assume we are running a command like below # trestle split -f target-definition.yaml -e target-definition.metadata content_type = FileContentType.YAML # prepare trestle project dir with the file target_def_dir, target_def_file = test_utils.prepare_trestle_project_dir( tmp_path, content_type, sample_target_def, test_utils.TARGET_DEFS_DIR) # first split the target-def into metadata target_def = ostarget.TargetDefinition.oscal_read(target_def_file) element = Element(target_def, 'target-definition') element_args = ['target-definition.metadata'] element_paths = test_utils.prepare_element_paths(target_def_dir, element_args) metadata_file = target_def_dir / element_paths[0].to_file_path( content_type) metadata: ostarget.Metadata = element.get_at(element_paths[0]) root_file = target_def_dir / element_paths[0].to_root_path(content_type) metadata_field_alias = element_paths[0].get_element_name() stripped_root = element.get().stripped_instance( stripped_fields_aliases=[metadata_field_alias]) root_wrapper_alias = utils.classname_to_alias( stripped_root.__class__.__name__, 'json') first_plan = Plan() first_plan.add_action(CreatePathAction(metadata_file)) first_plan.add_action( WriteFileAction(metadata_file, Element(metadata, metadata_field_alias), content_type)) first_plan.add_action(CreatePathAction(root_file, True)) first_plan.add_action( WriteFileAction(root_file, Element(stripped_root, root_wrapper_alias), content_type)) first_plan.execute() # this will split the files in the temp directory # now, prepare the expected plan to split metadta at parties second_plan = Plan() metadata_file_dir = target_def_dir / element_paths[0].to_root_path() metadata2 = ostarget.Metadata.oscal_read(metadata_file) element = Element(metadata2, metadata_field_alias) element_args = ['metadata.parties.*'] element_paths = test_utils.prepare_element_paths(target_def_dir, element_args) parties_dir = metadata_file_dir / element_paths[0].to_file_path() for i, party in enumerate(element.get_at(element_paths[0])): prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH) sub_model_actions = SplitCmd.prepare_sub_model_split_actions( party, parties_dir, prefix, content_type) second_plan.add_actions(sub_model_actions) # stripped metadata stripped_metadata = metadata2.stripped_instance( stripped_fields_aliases=['parties']) second_plan.add_action(CreatePathAction(metadata_file, True)) second_plan.add_action( WriteFileAction(metadata_file, Element(stripped_metadata, metadata_field_alias), content_type)) # call the split command and compare the plans split_plan = SplitCmd.split_model(metadata, element_paths, metadata_file_dir, content_type) assert second_plan == split_plan
def test_split_multi_level_dict_plans( tmp_path: pathlib.Path, sample_nist_component_def: component.ComponentDefinition, keep_cwd) -> None: """Test for split_model method.""" # Assume we are running a command like below # trestle split -f target.yaml -e component-definition.components.*.control-implementations.* content_type = FileContentType.YAML # prepare trestle project dir with the file component_def_dir, component_def_file = test_utils.prepare_trestle_project_dir( tmp_path, content_type, sample_nist_component_def, test_utils.COMPONENT_DEF_DIR) file_ext = FileContentType.to_file_extension(content_type) # read the model from file component_def: component.ComponentDefinition = component.ComponentDefinition.oscal_read( component_def_file) element = Element(component_def) element_args = [ 'component-definition.components.*.control-implementations.*' ] element_paths = cmd_utils.parse_element_args( None, element_args, component_def_dir.relative_to(tmp_path)) expected_plan = Plan() # extract values components: list = element.get_at(element_paths[0]) components_dir = component_def_dir / element_paths[0].to_file_path() # split every targets for index, comp_obj in enumerate(components): # individual target dir component_element = Element(comp_obj) model_type = str_utils.classname_to_alias( type(comp_obj).__name__, AliasMode.JSON) dir_prefix = str(index).zfill(const.FILE_DIGIT_PREFIX_LENGTH) component_dir_name = f'{dir_prefix}{const.IDX_SEP}{model_type}' component_file = components_dir / f'{component_dir_name}{file_ext}' # target control impl dir for the target component_ctrl_impls: list = component_element.get_at(element_paths[1]) component_ctrl_dir = components_dir / element_paths[1].to_file_path( root_dir=component_dir_name) for i, component_ctrl_impl in enumerate(component_ctrl_impls): model_type = str_utils.classname_to_alias( type(component_ctrl_impl).__name__, AliasMode.JSON) file_prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH) file_name = f'{file_prefix}{const.IDX_SEP}{model_type}{file_ext}' file_path = component_ctrl_dir / file_name expected_plan.add_action(CreatePathAction(file_path)) expected_plan.add_action( WriteFileAction(file_path, Element(component_ctrl_impl), content_type)) # write stripped target model stripped_target = comp_obj.stripped_instance( stripped_fields_aliases=[element_paths[1].get_element_name()]) expected_plan.add_action(CreatePathAction(component_file)) expected_plan.add_action( WriteFileAction(component_file, Element(stripped_target), content_type)) root_file = component_def_dir / f'component-definition{file_ext}' remaining_root = element.get().stripped_instance( stripped_fields_aliases=[element_paths[0].get_element_name()]) expected_plan.add_action(CreatePathAction(root_file, True)) expected_plan.add_action( WriteFileAction(root_file, Element(remaining_root), content_type)) split_plan = SplitCmd.split_model(component_def, element_paths, component_def_dir, content_type, '', None) assert expected_plan == split_plan
def split_model_at_path_chain(cls, model_obj: OscalBaseModel, element_paths: List[ElementPath], base_dir: pathlib.Path, content_type: FileContentType, cur_path_index: int, split_plan: Plan, strip_root: bool, root_file_name: str = '') -> int: """Recursively split the model at the provided chain of element paths. It assumes that a chain of element paths starts at the cur_path_index with the first path ending with a wildcard (*) It returns the index where the chain of path ends. For example, element paths could have a list of paths as below for a `TargetDefinition` model where the first path is the start of the chain. For each of the sub model described by the first element path (e.g target-defintion.targets.*) in the chain, the subsequent paths (e.g. target.target-control-implementations.*) will be applied recursively to retrieve the sub-sub models: [ 'target-definition.targets.*', 'target.target-control-implementations.*' ] for a command like below: trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.* """ # assume we ran the command below: # trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.* if split_plan is None: raise TrestleError('Split plan must have been initialized') if cur_path_index < 0: raise TrestleError( 'Current index of the chain of paths cannot be less than 0') # if there are no more element_paths, return the current plan if cur_path_index >= len(element_paths): return cur_path_index # initialize local variables element = Element(model_obj) stripped_field_alias = [] # get the sub_model specified by the element_path of this round element_path = element_paths[cur_path_index] is_parent = cur_path_index + 1 < len(element_paths) and element_paths[ cur_path_index + 1].get_parent() == element_path # root dir name for sub models dir # 00000__group.json will have the root_dir name as 00000__group for sub models of group # catalog.json will have the root_dir name as catalog sub models root_dir = '' if root_file_name != '': root_dir = pathlib.Path(root_file_name).stem # check that the path is not multiple level deep path_parts = element_path.get() if path_parts[-1] == ElementPath.WILDCARD: path_parts = path_parts[:-1] if len(path_parts) > 2: msg = 'Trestle supports split of first level children only, ' msg += f'found path "{element_path}" with level = {len(path_parts)}' raise TrestleError(msg) sub_models = element.get_at( element_path, False) # we call sub_models as in plural, but it can be just one if sub_models is None: return cur_path_index # assume cur_path_index is the end of the chain # value of this variable may change during recursive split of the sub-models below path_chain_end = cur_path_index # if wildcard is present in the element_path and the next path in the chain has current path as the parent, # we need to split recursively and create separate file for each sub item # for example, in the first round we get the `targets` using the path `target-definition.targets.*` # so, now we need to split each of the target recursively. Note that target is an instance of dict # However, there can be other sub_model, which is of type list if is_parent and element_path.get_last() is not ElementPath.WILDCARD: # create dir for all sub model items sub_models_dir = base_dir / element_path.to_root_path() sub_model_plan = Plan() path_chain_end = cls.split_model_at_path_chain( sub_models, element_paths, sub_models_dir, content_type, cur_path_index + 1, sub_model_plan, True) sub_model_actions = sub_model_plan.get_actions() split_plan.add_actions(sub_model_actions) elif element_path.get_last() == ElementPath.WILDCARD: # extract sub-models into a dict with appropriate prefix sub_model_items: Dict[str, OscalBaseModel] = {} sub_models_dir = base_dir / element_path.to_file_path( root_dir=root_dir) if isinstance(sub_models, list): for i, sub_model_item in enumerate(sub_models): # e.g. `groups/00000_groups/` prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH) sub_model_items[prefix] = sub_model_item elif isinstance(sub_models, dict): # prefix is the key of the dict sub_model_items = sub_models else: # unexpected sub model type for multi-level split with wildcard raise TrestleError( f'Sub element at {element_path} is not of type list or dict for further split' ) # process list sub model items for key in sub_model_items: prefix = key sub_model_item = sub_model_items[key] # recursively split the sub-model if there are more element paths to traverse # e.g. split target.target-control-implementations.* require_recursive_split = cur_path_index + 1 < len( element_paths) and element_paths[ cur_path_index + 1].get_parent() == element_path if require_recursive_split: # prepare individual directory for each sub-model # e.g. `targets/<UUID>__target/` sub_root_file_name = cmd_utils.to_model_file_name( sub_model_item, prefix, content_type) sub_model_plan = Plan() path_chain_end = cls.split_model_at_path_chain( sub_model_item, element_paths, sub_models_dir, content_type, cur_path_index + 1, sub_model_plan, True, sub_root_file_name) sub_model_actions = sub_model_plan.get_actions() else: sub_model_actions = cls.prepare_sub_model_split_actions( sub_model_item, sub_models_dir, prefix, content_type) split_plan.add_actions(sub_model_actions) else: # the chain of path ends at the current index. # so no recursive call. Let's just write the sub model to the file and get out sub_model_file = base_dir / element_path.to_file_path( content_type, root_dir=root_dir) split_plan.add_action(CreatePathAction(sub_model_file)) split_plan.add_action( WriteFileAction( sub_model_file, Element(sub_models, element_path.get_element_name()), content_type)) # Strip the root model and add a WriteAction for the updated model object in the plan if strip_root: stripped_field_alias.append(element_path.get_element_name()) stripped_root = model_obj.stripped_instance( stripped_fields_aliases=stripped_field_alias) if root_file_name != '': root_file = base_dir / root_file_name else: root_file = base_dir / element_path.to_root_path(content_type) split_plan.add_action(CreatePathAction(root_file)) wrapper_alias = utils.classname_to_alias( stripped_root.__class__.__name__, 'json') split_plan.add_action( WriteFileAction(root_file, Element(stripped_root, wrapper_alias), content_type)) # return the end of the current path chain return path_chain_end
def test_element_get_at(sample_nist_component_def: component.ComponentDefinition): """Test element get method.""" element = Element(sample_nist_component_def) # field alias should succeed assert element.get_at( ElementPath('component-definition.metadata.last-modified') ) == sample_nist_component_def.metadata.last_modified # field name should fail assert element.get_at(ElementPath('component-definition.metadata.last_modified')) is None assert element.get() == sample_nist_component_def assert element.get_at() == element.get() assert element.get_at(ElementPath('component-definition.metadata')) == sample_nist_component_def.metadata assert element.get_at( ElementPath('component-definition.metadata.title') ) == sample_nist_component_def.metadata.title assert element.get_at(ElementPath('component-definition.components')) == sample_nist_component_def.components assert element.get_at(ElementPath('component-definition.components.*')) == sample_nist_component_def.components assert element.get_at( ElementPath('component-definition.metadata.parties.*') ) == sample_nist_component_def.metadata.parties assert element.get_at(ElementPath('component-definition.metadata.parties.0') ) == sample_nist_component_def.metadata.parties[0] assert element.get_at(ElementPath('component-definition.metadata.parties.0.uuid') ) == sample_nist_component_def.metadata.parties[0].uuid for index in range(len(sample_nist_component_def.components)): path_str = f'component-definition.components.{index}' assert element.get_at(ElementPath(path_str)) == sample_nist_component_def.components[index] # invalid indexing assert element.get_at(ElementPath('component-definition.metadata.title.0')) is None # invalid path with missing root model assert element.get_at(ElementPath('metadata.title')) is None # element_path with parent path parent_path = ElementPath('component-definition.metadata') element_path = ElementPath('metadata.parties.*', parent_path) assert element.get_at(element_path) == sample_nist_component_def.metadata.parties # element_path with parent path parent_path = ElementPath('component-definition.components.*') element_path = ElementPath('component.control-implementations.*', parent_path) component_list = element.get_at(parent_path) for component_item in component_list: component_element = Element(component_item) assert component_element.get_at(element_path) == component_item.control_implementations # element_path in a list with parent path parent_path = ElementPath('component-definition.components.*') element_path = ElementPath('component.control-implementations.0', parent_path) component_list = element.get_at(parent_path) for component_item in component_list: component_element = Element(component_item) assert component_element.get_at(element_path) == component_item.control_implementations[0]