def test_update_action(sample_nist_component_def):
"""Test update action."""
element = Element(sample_nist_component_def)
metadata = common.Metadata(
**{
'title': 'My simple catalog',
'last-modified': datetime.now().astimezone(),
'version': '0.0.0',
'oscal-version': OSCAL_VERSION
})
sub_element_path = ElementPath('component-definition.metadata')
prev_metadata = element.get_at(sub_element_path)
uac = UpdateAction(metadata, element, sub_element_path)
uac.execute()
assert element.get_at(sub_element_path) is not prev_metadata
assert element.get_at(sub_element_path) == metadata
uac.rollback()
assert element.get_at(sub_element_path) == prev_metadata
assert element.get_at(sub_element_path) is not metadata
def test_update_action(sample_target):
"""Test update action."""
element = Element(sample_target)
metadata = target.Metadata(
**{
'title': 'My simple catalog',
'last-modified': datetime.now().astimezone(),
'version': '0.0.0',
'oscal-version': '1.0.0-Milestone3'
})
sub_element_path = ElementPath('metadata')
prev_metadata = element.get_at(sub_element_path)
uac = UpdateAction(metadata, element, sub_element_path)
uac.execute()
assert element.get_at(sub_element_path) is not prev_metadata
assert element.get_at(sub_element_path) == metadata
uac.rollback()
assert element.get_at(sub_element_path) == prev_metadata
assert element.get_at(sub_element_path) is not metadata
def test_add(tmp_path: pathlib.Path, keep_cwd: pathlib.Path) -> None:
"""Test Add.add() method used by trestle CreateCmd."""
file_path = pathlib.Path(
test_utils.JSON_TEST_DATA_PATH) / 'minimal_catalog_missing_roles.json'
minimal_catalog_missing_roles = Catalog.oscal_read(file_path)
# expected catalog after first add of Role
file_path = pathlib.Path.joinpath(test_utils.JSON_TEST_DATA_PATH,
'minimal_catalog_roles.json')
expected_catalog_roles1 = Element(Catalog.oscal_read(file_path))
# expected catalog after second add of Role
file_path = pathlib.Path.joinpath(test_utils.JSON_TEST_DATA_PATH,
'minimal_catalog_roles_double.json')
expected_catalog_roles2 = Element(Catalog.oscal_read(file_path))
# expected catalog after add of Responsible-Party
file_path = pathlib.Path.joinpath(test_utils.JSON_TEST_DATA_PATH,
'minimal_catalog_roles_double_rp.json')
expected_catalog_roles2_rp = Element(Catalog.oscal_read(file_path))
content_type = FileContentType.JSON
_, _ = test_utils.prepare_trestle_project_dir(
tmp_path, content_type, minimal_catalog_missing_roles,
test_utils.CATALOGS_DIR)
# Execute first _add
element_path = ElementPath('catalog.metadata.roles')
catalog_element = Element(minimal_catalog_missing_roles)
expected_update_action_1 = UpdateAction(
expected_catalog_roles1.get_at(element_path), catalog_element,
element_path)
actual_update_action, actual_catalog_roles = Add.add(
element_path, catalog_element, False)
assert actual_catalog_roles == expected_catalog_roles1
assert actual_update_action == expected_update_action_1
# Execute second _add - this time roles already exists, so this adds a roles object to roles array
catalog_element = actual_catalog_roles
expected_update_action_2 = UpdateAction(
expected_catalog_roles2.get_at(element_path), catalog_element,
element_path)
actual_update_action2, actual_catalog_roles2 = Add.add(
element_path, catalog_element, False)
assert actual_catalog_roles2 == expected_catalog_roles2
assert actual_update_action2 == expected_update_action_2
# Execute _add for responsible-parties to the same catalog
element_path = ElementPath('catalog.metadata.responsible-parties')
catalog_element = actual_catalog_roles2
expected_update_action_3 = UpdateAction(
expected_catalog_roles2_rp.get_at(element_path), catalog_element,
element_path)
actual_update_action3, actual_catalog_roles2_rp = Add.add(
element_path, catalog_element, False)
assert actual_catalog_roles2_rp == expected_catalog_roles2_rp
assert actual_update_action3 == expected_update_action_3
def test_split_chained_sub_model_plans(
tmp_path: pathlib.Path, simplified_nist_catalog: oscatalog.Catalog,
keep_cwd: pathlib.Path) -> None:
"""Test for split_model method with chained sum models like catalog.metadata.parties.*."""
# Assume we are running a command like below
# trestle split -f catalog.json -e catalog.metadata.parties.*
# see https://github.com/IBM/compliance-trestle/issues/172
content_type = FileContentType.JSON
# prepare trestle project dir with the file
catalog_dir, catalog_file = test_utils.prepare_trestle_project_dir(
tmp_path, content_type, simplified_nist_catalog,
test_utils.CATALOGS_DIR)
# read the model from file
catalog = oscatalog.Catalog.oscal_read(catalog_file)
element = Element(catalog)
element_args = ['catalog.metadata.parties.*']
element_paths = cmd_utils.parse_element_args(
None, element_args, catalog_dir.relative_to(tmp_path))
assert 2 == len(element_paths)
expected_plan = Plan()
# prepare to extract metadata and parties
metadata_file = catalog_dir / element_paths[0].to_file_path(content_type)
metadata_field_alias = element_paths[0].get_element_name()
metadata = element.get_at(element_paths[0])
meta_element = Element(metadata, metadata_field_alias)
# extract parties
parties_dir = catalog_dir / 'catalog/metadata/parties'
for i, party in enumerate(meta_element.get_at(element_paths[1], False)):
prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
sub_model_actions = SplitCmd.prepare_sub_model_split_actions(
party, parties_dir, prefix, content_type)
expected_plan.add_actions(sub_model_actions)
# stripped metadata
stripped_metadata = metadata.stripped_instance(
stripped_fields_aliases=['parties'])
expected_plan.add_action(CreatePathAction(metadata_file))
expected_plan.add_action(
WriteFileAction(metadata_file,
Element(stripped_metadata, metadata_field_alias),
content_type))
# stripped catalog
root_file = catalog_dir / element_paths[0].to_root_path(content_type)
remaining_root = element.get().stripped_instance(metadata_field_alias)
expected_plan.add_action(CreatePathAction(root_file, True))
expected_plan.add_action(
WriteFileAction(root_file, Element(remaining_root), content_type))
split_plan = SplitCmd.split_model(catalog, element_paths, catalog_dir,
content_type, '', None)
assert expected_plan == split_plan
def prepare_element(sample_target_def):
"""Prepare a target element for remove tests."""
element = Element(sample_target_def)
parties: List[target.Party] = []
parties.append(
target.Party(
**{
'uuid': 'ff47836c-877c-4007-bbf3-c9d9bd805000',
'party-name': 'TEST1',
'type': 'organization'
}))
parties.append(
target.Party(
**{
'uuid': 'ee88836c-877c-4007-bbf3-c9d9bd805000',
'party-name': 'TEST2',
'type': 'organization'
}))
sub_element_path = ElementPath('target-definition.metadata.parties.*')
ac = UpdateAction(parties, element, sub_element_path)
ac.execute()
assert element.get_at(sub_element_path) == parties
return element
def test_update_list_sub_element_action(sample_target):
"""Test setting a list."""
element = Element(sample_target)
parties: List[target.Party] = []
parties.append(
target.Party(
**{
'uuid': 'ff47836c-877c-4007-bbf3-c9d9bd805000',
'party-name': 'TEST1',
'type': 'organization'
}))
parties.append(
target.Party(
**{
'uuid': 'ee88836c-877c-4007-bbf3-c9d9bd805000',
'party-name': 'TEST2',
'type': 'organization'
}))
sub_element_path = ElementPath('metadata.parties.*')
uac = UpdateAction(parties, element, sub_element_path)
uac.execute()
assert element.get_at(sub_element_path) == parties
def add(element_path: ElementPath, parent_element: Element, include_optional: bool) -> None:
"""For a element_path, add a child model to the parent_element of a given parent_model.
Args:
element_path: element path of the item to create within the model
parent_element: the parent element that will host the created element
include_optional: whether to create optional attributes in the created element
Notes:
First we find the child model at the specified element path and instantiate it with default values.
Then we check if there's already existing element at that path, in which case we append the child model
to the existing list of dict.
Then we set up an action plan to update the model (specified by file_path) in memory, create a file
at the same location and write the file.
We update the parent_element to prepare for next adds in the chain
"""
if '*' in element_path.get_full_path_parts():
raise err.TrestleError('trestle add does not support Wildcard element path.')
# Get child model
try:
child_model = element_path.get_type(type(parent_element.get()))
# Create child element with sample values
child_object = gens.generate_sample_model(child_model, include_optional=include_optional)
if parent_element.get_at(element_path) is not None:
# The element already exists
if type(parent_element.get_at(element_path)) is list:
child_object = parent_element.get_at(element_path) + child_object
elif type(parent_element.get_at(element_path)) is dict:
child_object = {**parent_element.get_at(element_path), **child_object}
else:
raise err.TrestleError('Already exists and is not a list or dictionary.')
except Exception as e:
raise err.TrestleError(f'Bad element path. {str(e)}')
update_action = UpdateAction(
sub_element=child_object, dest_element=parent_element, sub_element_path=element_path
)
parent_element = parent_element.set_at(element_path, child_object)
return update_action, parent_element
def test_element_get_at(sample_target: target.TargetDefinition):
"""Test element get method."""
element = Element(sample_target)
assert element.get() == sample_target
assert element.get_at() == element.get()
assert element.get_at(ElementPath('metadata')) == sample_target.metadata
assert element.get_at(
ElementPath('metadata.title')) == sample_target.metadata.title
assert element.get_at(ElementPath('targets')) == sample_target.targets
assert element.get_at(ElementPath('targets.*')) == sample_target.targets
assert element.get_at(
ElementPath('metadata.parties.*')) == sample_target.metadata.parties
assert element.get_at(
ElementPath('metadata.parties.0')) == sample_target.metadata.parties[0]
assert element.get_at(ElementPath(
'metadata.parties.0.uuid')) == sample_target.metadata.parties[0].uuid
# invalid indexing
assert element.get_at(ElementPath('metadata.title.0')) is None
def test_split_model_plans(
tmp_path: pathlib.Path,
sample_nist_component_def: component.ComponentDefinition) -> None:
"""Test for split_model method."""
# Assume we are running a command like below
# trestle split -f component-definition.yaml -e component-definition.metadata
content_type = FileContentType.YAML
# prepare trestle project dir with the file
component_def_dir, component_def_file = test_utils.prepare_trestle_project_dir(
tmp_path, content_type, sample_nist_component_def,
test_utils.COMPONENT_DEF_DIR)
# read the model from file
component_def = component.ComponentDefinition.oscal_read(
component_def_file)
element = Element(component_def)
element_args = ['component-definition.metadata']
element_paths = cmd_utils.parse_element_args(None, element_args)
# extract values
metadata_file = component_def_dir / element_paths[0].to_file_path(
content_type)
metadata = element.get_at(element_paths[0])
root_file = component_def_dir / element_paths[0].to_root_path(content_type)
remaining_root = element.get().stripped_instance(
element_paths[0].get_element_name())
# prepare the plan
expected_plan = Plan()
expected_plan.add_action(CreatePathAction(metadata_file))
expected_plan.add_action(
WriteFileAction(metadata_file, Element(metadata), content_type))
expected_plan.add_action(CreatePathAction(root_file, True))
expected_plan.add_action(
WriteFileAction(root_file, Element(remaining_root), content_type))
split_plan = SplitCmd.split_model(component_def, element_paths,
component_def_dir, content_type, '',
None)
assert expected_plan == split_plan
def remove(cls, element_path: ElementPath, parent_model: Type[OscalBaseModel],
parent_element: Element) -> Tuple[RemoveAction, Element]:
"""For the element_path, remove a model from the parent_element of a given parent_model.
First we check if there is an existing element at that path
If not, we complain.
Then we set up an action plan to update the model (specified by file_path) in memory,
return the action and return the parent_element.
LIMITATIONS:
1. This does not remove elements of a list or dict. Instead, the entire list or dict is removed.
2. This cannot remove arbitrarily named elements that are not specified in the schema.
For example, "responsible-parties" contains named elements, e.g., "organisation". The tool will not
remove the "organisation" as it is not in the schema, but one can remove its elements, e.g., "party-uuids".
"""
element_path_list = element_path.get_full_path_parts()
if '*' in element_path_list:
raise err.TrestleError('trestle remove does not support Wildcard element path.')
deleting_element = parent_element.get_at(element_path)
if deleting_element is not None:
# The element already exists
if type(deleting_element) is list:
logger.warning(
'Warning: trestle remove does not support removing elements of a list: '
'this removes the entire list'
)
elif type(deleting_element) is dict:
logger.warning(
'Warning: trestle remove does not support removing dict elements: '
'this removes the entire dict element'
)
else:
raise err.TrestleError(f'Bad element path: {str(element_path)}')
remove_action = RemoveAction(parent_element, element_path)
return remove_action, parent_element
def test_split_model(tmp_dir, sample_target_def: ostarget.TargetDefinition):
"""Test for split_model method."""
# Assume we are running a command like below
# trestle split -f target-definition.yaml -e target-definition.metadata
content_type = FileContentType.YAML
# prepare trestle project dir with the file
target_def_dir, target_def_file = test_utils.prepare_trestle_project_dir(
tmp_dir, content_type, sample_target_def, test_utils.TARGET_DEFS_DIR)
# read the model from file
target_def = ostarget.TargetDefinition.oscal_read(target_def_file)
element = Element(target_def)
element_args = ['target-definition.metadata']
element_paths = cmd_utils.parse_element_args(element_args)
# extract values
metadata_file = target_def_dir / element_paths[0].to_file_path(
content_type)
metadata = element.get_at(element_paths[0])
root_file = target_def_dir / element_paths[0].to_root_path(content_type)
remaining_root = element.get().stripped_instance(
element_paths[0].get_element_name())
# prepare the plan
expected_plan = Plan()
expected_plan.add_action(CreatePathAction(metadata_file))
expected_plan.add_action(
WriteFileAction(metadata_file, Element(metadata), content_type))
expected_plan.add_action(CreatePathAction(root_file, True))
expected_plan.add_action(
WriteFileAction(root_file, Element(remaining_root), content_type))
split_plan = SplitCmd.split_model(target_def, element_paths,
target_def_dir, content_type)
assert expected_plan == split_plan
def test_element_get_at(sample_target_def: target.TargetDefinition):
"""Test element get method."""
element = Element(sample_target_def)
# field alias should succeed
assert element.get_at(
ElementPath('target-definition.metadata.last-modified')
) == sample_target_def.metadata.last_modified
# field name should fail
assert element.get_at(ElementPath('target-definition.metadata.last_modified')) is None
assert element.get() == sample_target_def
assert element.get_at() == element.get()
assert element.get_at(ElementPath('target-definition.metadata')) == sample_target_def.metadata
assert element.get_at(ElementPath('target-definition.metadata.title')) == sample_target_def.metadata.title
assert element.get_at(ElementPath('target-definition.targets')) == sample_target_def.targets
assert element.get_at(ElementPath('target-definition.targets.*')) == sample_target_def.targets
assert element.get_at(ElementPath('target-definition.metadata.parties.*')) == sample_target_def.metadata.parties
assert element.get_at(ElementPath('target-definition.metadata.parties.0')) == sample_target_def.metadata.parties[0]
assert element.get_at(ElementPath('target-definition.metadata.parties.0.uuid')
) == sample_target_def.metadata.parties[0].uuid
for uuid in sample_target_def.targets:
path_str = f'target-definition.targets.{uuid}'
assert element.get_at(ElementPath(path_str)) == sample_target_def.targets[uuid]
# invalid indexing
assert element.get_at(ElementPath('target-definition.metadata.title.0')) is None
# invalid path with missing root model
assert element.get_at(ElementPath('metadata.title')) is None
# element_path with parent path
parent_path = ElementPath('target-definition.metadata')
element_path = ElementPath('metadta.parties.*', parent_path)
assert element.get_at(element_path) == sample_target_def.metadata.parties
# element_path with parent path
parent_path = ElementPath('target-definition.targets.*')
element_path = ElementPath('target.target-control-implementations.*', parent_path)
targets = element.get_at(parent_path)
for key in targets:
target = targets[key]
target_element = Element(target)
assert target_element.get_at(element_path) == target.target_control_implementations
# element_path in a list with parent path
parent_path = ElementPath('target-definition.targets.*')
element_path = ElementPath('target.target-control-implementations.0', parent_path)
targets = element.get_at(parent_path)
for key in targets:
target = targets[key]
target_element = Element(target)
assert target_element.get_at(element_path) == target.target_control_implementations[0]
def describe(cls, file_path: pathlib.Path, element_path_str: str, trestle_root: pathlib.Path) -> List[str]:
"""Describe the contents of the file.
Args:
file_path: pathlib.Path Path for model file to describe.
element_path_str: Element path of element in model to describe. Can be ''.
Returns:
The list of lines of text in the description, or an empty list on failure
"""
# figure out the model type so we can read it
try:
model_type, _ = ModelUtils.get_stripped_model_type(file_path, trestle_root)
model: OscalBaseModel = model_type.oscal_read(file_path)
except TrestleError as e:
logger.warning(f'Error loading model {file_path} to describe: {e}')
return []
sub_model = model
# if an element path was provided, follow the path chain to the desired sub_model
if element_path_str:
if '*' in element_path_str or ',' in element_path_str:
logger.warning('Wildcards and commas are not allowed in the element path for describe.')
return []
if '.' not in element_path_str:
logger.warning('The element path for describe must either be omitted or contain at least 2 parts.')
return []
element_paths = utils.parse_element_arg(model, element_path_str)
sub_model_element = Element(model)
for element_path in element_paths:
sub_model = sub_model_element.get_at(element_path, False)
sub_model_element = Element(sub_model)
# now that we have the desired sub_model we can describe it
text_out: List[str] = []
# create top level text depending on whether an element path was used
element_text = '' if not element_path_str else f' at element path {element_path_str}'
if type(sub_model) is list:
text = f'Model file {file_path}{element_text} is a {cls._description_text(sub_model)}'
text_out.append(text)
logger.info(text)
else:
text = f'Model file {file_path}{element_text} is of type '
text += f'{cls._clean_type_string(str(type(sub_model)))} and contains:'
text_out.append(text)
logger.info(text)
for key in sub_model.__fields__.keys():
value = getattr(sub_model, key, None)
text = f' {key}: {cls._description_text(value)}'
text_out.append(text)
logger.info(text)
return text_out
def split_model_at_path_chain(
cls,
model_obj: OscalBaseModel,
element_paths: List[ElementPath],
base_dir: pathlib.Path,
content_type: FileContentType,
cur_path_index: int,
split_plan: Plan,
strip_root: bool,
root_file_name: str,
aliases_to_strip: Dict[str, AliasTracker],
last_one: bool = True
) -> int:
"""Recursively split the model at the provided chain of element paths.
It assumes that a chain of element paths starts at the cur_path_index with the first path ending
with a wildcard (*)
If the wildcard follows an element that is inherently a list of items, the list of items is extracted.
But if the wildcard follows a generic model than members of that model class found in the model will be
split off. But only the non-trivial elements are removed, i.e. not str, int, datetime, etc.
Args:
model_obj: The OscalBaseModel to be split
element_paths: The List[ElementPath] of elements to split, including embedded wildcards
base_dir: pathlib.Path of the file being split
content_type: json or yaml files
cur_path_index: Index into the list of element paths for the current split operation
split_plan: The accumulated plan of actions needed to perform the split
strip_root: Whether to strip elements from the root object
root_file_name: Filename of root file that gets split into a list of items
aliases_to_strip: AliasTracker previously loaded with aliases that need to be split from each element
last_one: bool indicating last item in array has been split and stripped model can now be written
Returns:
int representing the index where the chain of the path ends.
Examples:
For example, element paths could have a list of paths as below for a `ComponentDefinition` model where
the first path is the start of the chain.
For each of the sub model described by the first element path (e.g component-defintion.components.*) in the
chain, the subsequent paths (e.g component.control-implementations.*) will be applied recursively
to retrieve the sub-sub models:
[
'component-definition.component.*',
'component.control-implementations.*'
]
for a command like below:
trestle split -f component.yaml -e component-definition.components.*.control-implementations.*
"""
if split_plan is None:
raise TrestleError('Split plan must have been initialized')
if cur_path_index < 0:
raise TrestleError('Current index of the chain of paths cannot be less than 0')
# if there are no more element_paths, return the current plan
if cur_path_index >= len(element_paths):
return cur_path_index
# initialize local variables
element = Element(model_obj)
stripped_field_alias: List[str] = []
# get the sub_model specified by the element_path of this round
element_path = element_paths[cur_path_index]
# does the next element_path point back at me
is_parent = cur_path_index + 1 < len(element_paths) and element_paths[cur_path_index
+ 1].get_parent() == element_path
# root dir name for sub models dir
# 00000__group.json will have the root_dir name as 00000__group for sub models of group
# catalog.json will have the root_dir name as catalog
root_dir = ''
if root_file_name != '':
root_dir = str(pathlib.Path(root_file_name).with_suffix(''))
sub_models = element.get_at(element_path, False) # we call sub_models as in plural, but it can be just one
# assume cur_path_index is the end of the chain
# value of this variable may change during recursive split of the sub-models below
path_chain_end = cur_path_index
# if wildcard is present in the element_path and the next path in the chain has current path as the parent,
# Then deal with case of list, or split of arbitrary oscalbasemodel
if is_parent and element_path.get_last() is not ElementPath.WILDCARD:
# create dir for all sub model items
sub_models_dir = base_dir / element_path.to_root_path()
sub_model_plan = Plan()
path_chain_end = cls.split_model_at_path_chain(
sub_models,
element_paths,
sub_models_dir,
content_type,
cur_path_index + 1,
sub_model_plan,
True,
'',
aliases_to_strip
)
sub_model_actions = sub_model_plan.get_actions()
split_plan.add_actions(sub_model_actions)
elif element_path.get_last() == ElementPath.WILDCARD:
# extract sub-models into a dict with appropriate prefix
sub_model_items: Dict[str, OscalBaseModel] = {}
sub_models_dir = base_dir / element_path.to_file_path(root_dir=root_dir)
if isinstance(sub_models, list):
for i, sub_model_item in enumerate(sub_models):
# e.g. `groups/00000_groups/`
prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
sub_model_items[prefix] = sub_model_item
# process list sub model items
count = 0
for key, sub_model_item in sub_model_items.items():
count += 1
# recursively split the sub-model if there are more element paths to traverse
# e.g. split component.control-implementations.*
require_recursive_split = cur_path_index + 1 < len(element_paths) and element_paths[
cur_path_index + 1].get_parent() == element_path
if require_recursive_split:
# prepare individual directory for each sub-model
sub_root_file_name = cmd_utils.to_model_file_name(sub_model_item, key, content_type)
sub_model_plan = Plan()
last_one: bool = count == len(sub_model_items)
path_chain_end = cls.split_model_at_path_chain(
sub_model_item,
element_paths,
sub_models_dir,
content_type,
cur_path_index + 1,
sub_model_plan,
True,
sub_root_file_name,
aliases_to_strip,
last_one
)
sub_model_actions = sub_model_plan.get_actions()
else:
sub_model_actions = cls.prepare_sub_model_split_actions(
sub_model_item, sub_models_dir, key, content_type
)
split_plan.add_actions(sub_model_actions)
else:
# the chain of path ends at the current index.
# so no recursive call. Let's just write the sub model to the file and get out
if sub_models is not None:
sub_model_file = base_dir / element_path.to_file_path(content_type, root_dir=root_dir)
split_plan.add_action(CreatePathAction(sub_model_file))
split_plan.add_action(
WriteFileAction(sub_model_file, Element(sub_models, element_path.get_element_name()), content_type)
)
# Strip the root model and add a WriteAction for the updated model object in the plan
if strip_root:
full_path = element_path.get_full()
path = '.'.join(full_path.split('.')[:-1])
aliases = [element_path.get_element_name()]
need_to_write = True
use_alias_dict = aliases_to_strip is not None and path in aliases_to_strip
if use_alias_dict:
aliases = aliases_to_strip[path].get_aliases()
need_to_write = aliases_to_strip[path].needs_writing()
stripped_model = model_obj.stripped_instance(stripped_fields_aliases=aliases)
# can mark it written even if it doesn't need writing since it is empty
# but if an array only mark it written if it's the last one
if last_one and use_alias_dict:
aliases_to_strip[path].mark_written()
# If it's an empty model after stripping the fields, don't create path and don't write
field_list = [x for x in model_obj.__fields__.keys() if model_obj.__fields__[x] is not None]
if set(field_list) == set(stripped_field_alias):
return path_chain_end
if need_to_write:
if root_file_name != '':
root_file = base_dir / root_file_name
else:
root_file = base_dir / element_path.to_root_path(content_type)
split_plan.add_action(CreatePathAction(root_file))
wrapper_alias = classname_to_alias(stripped_model.__class__.__name__, AliasMode.JSON)
split_plan.add_action(WriteFileAction(root_file, Element(stripped_model, wrapper_alias), content_type))
# return the end of the current path chain
return path_chain_end
def test_split_multi_level_dict(
tmp_path: pathlib.Path,
sample_target_def: ostarget.TargetDefinition) -> None:
"""Test for split_model method."""
# Assume we are running a command like below
# trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.*
content_type = FileContentType.YAML
# prepare trestle project dir with the file
target_def_dir, target_def_file = test_utils.prepare_trestle_project_dir(
tmp_path, content_type, sample_target_def, test_utils.TARGET_DEFS_DIR)
file_ext = FileContentType.to_file_extension(content_type)
# read the model from file
target_def: ostarget.TargetDefinition = ostarget.TargetDefinition.oscal_read(
target_def_file)
element = Element(target_def)
element_args = [
'target-definition.targets.*.target-control-implementations.*'
]
element_paths = test_utils.prepare_element_paths(target_def_dir,
element_args)
expected_plan = Plan()
# extract values
targets: dict = element.get_at(element_paths[0])
targets_dir = target_def_dir / element_paths[0].to_file_path()
# split every targets
for key in targets:
# individual target dir
target: ostarget.Target = targets[key]
target_element = Element(targets[key])
model_type = utils.classname_to_alias(type(target).__name__, 'json')
dir_prefix = key
target_dir_name = f'{dir_prefix}{const.IDX_SEP}{model_type}'
target_file = targets_dir / f'{target_dir_name}{file_ext}'
# target control impl dir for the target
target_ctrl_impls: dict = target_element.get_at(element_paths[1])
targets_ctrl_dir = targets_dir / element_paths[1].to_file_path(
root_dir=target_dir_name)
for i, target_ctrl_impl in enumerate(target_ctrl_impls):
model_type = utils.classname_to_alias(
type(target_ctrl_impl).__name__, 'json')
file_prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
file_name = f'{file_prefix}{const.IDX_SEP}{model_type}{file_ext}'
file_path = targets_ctrl_dir / file_name
expected_plan.add_action(CreatePathAction(file_path))
expected_plan.add_action(
WriteFileAction(file_path, Element(target_ctrl_impl),
content_type))
# write stripped target model
stripped_target = target.stripped_instance(
stripped_fields_aliases=[element_paths[1].get_element_name()])
expected_plan.add_action(CreatePathAction(target_file))
expected_plan.add_action(
WriteFileAction(target_file, Element(stripped_target),
content_type))
root_file = target_def_dir / f'target-definition{file_ext}'
remaining_root = element.get().stripped_instance(
stripped_fields_aliases=[element_paths[0].get_element_name()])
expected_plan.add_action(CreatePathAction(root_file, True))
expected_plan.add_action(
WriteFileAction(root_file, Element(remaining_root), content_type))
split_plan = SplitCmd.split_model(target_def, element_paths,
target_def_dir, content_type)
assert expected_plan == split_plan
def test_subsequent_split_model(
tmp_path: pathlib.Path,
sample_target_def: ostarget.TargetDefinition) -> None:
"""Test subsequent split of sub models."""
# Assume we are running a command like below
# trestle split -f target-definition.yaml -e target-definition.metadata
content_type = FileContentType.YAML
# prepare trestle project dir with the file
target_def_dir, target_def_file = test_utils.prepare_trestle_project_dir(
tmp_path, content_type, sample_target_def, test_utils.TARGET_DEFS_DIR)
# first split the target-def into metadata
target_def = ostarget.TargetDefinition.oscal_read(target_def_file)
element = Element(target_def, 'target-definition')
element_args = ['target-definition.metadata']
element_paths = test_utils.prepare_element_paths(target_def_dir,
element_args)
metadata_file = target_def_dir / element_paths[0].to_file_path(
content_type)
metadata: ostarget.Metadata = element.get_at(element_paths[0])
root_file = target_def_dir / element_paths[0].to_root_path(content_type)
metadata_field_alias = element_paths[0].get_element_name()
stripped_root = element.get().stripped_instance(
stripped_fields_aliases=[metadata_field_alias])
root_wrapper_alias = utils.classname_to_alias(
stripped_root.__class__.__name__, 'json')
first_plan = Plan()
first_plan.add_action(CreatePathAction(metadata_file))
first_plan.add_action(
WriteFileAction(metadata_file, Element(metadata, metadata_field_alias),
content_type))
first_plan.add_action(CreatePathAction(root_file, True))
first_plan.add_action(
WriteFileAction(root_file, Element(stripped_root, root_wrapper_alias),
content_type))
first_plan.execute() # this will split the files in the temp directory
# now, prepare the expected plan to split metadta at parties
second_plan = Plan()
metadata_file_dir = target_def_dir / element_paths[0].to_root_path()
metadata2 = ostarget.Metadata.oscal_read(metadata_file)
element = Element(metadata2, metadata_field_alias)
element_args = ['metadata.parties.*']
element_paths = test_utils.prepare_element_paths(target_def_dir,
element_args)
parties_dir = metadata_file_dir / element_paths[0].to_file_path()
for i, party in enumerate(element.get_at(element_paths[0])):
prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
sub_model_actions = SplitCmd.prepare_sub_model_split_actions(
party, parties_dir, prefix, content_type)
second_plan.add_actions(sub_model_actions)
# stripped metadata
stripped_metadata = metadata2.stripped_instance(
stripped_fields_aliases=['parties'])
second_plan.add_action(CreatePathAction(metadata_file, True))
second_plan.add_action(
WriteFileAction(metadata_file,
Element(stripped_metadata, metadata_field_alias),
content_type))
# call the split command and compare the plans
split_plan = SplitCmd.split_model(metadata, element_paths,
metadata_file_dir, content_type)
assert second_plan == split_plan
def test_split_multi_level_dict_plans(
tmp_path: pathlib.Path,
sample_nist_component_def: component.ComponentDefinition,
keep_cwd) -> None:
"""Test for split_model method."""
# Assume we are running a command like below
# trestle split -f target.yaml -e component-definition.components.*.control-implementations.*
content_type = FileContentType.YAML
# prepare trestle project dir with the file
component_def_dir, component_def_file = test_utils.prepare_trestle_project_dir(
tmp_path, content_type, sample_nist_component_def,
test_utils.COMPONENT_DEF_DIR)
file_ext = FileContentType.to_file_extension(content_type)
# read the model from file
component_def: component.ComponentDefinition = component.ComponentDefinition.oscal_read(
component_def_file)
element = Element(component_def)
element_args = [
'component-definition.components.*.control-implementations.*'
]
element_paths = cmd_utils.parse_element_args(
None, element_args, component_def_dir.relative_to(tmp_path))
expected_plan = Plan()
# extract values
components: list = element.get_at(element_paths[0])
components_dir = component_def_dir / element_paths[0].to_file_path()
# split every targets
for index, comp_obj in enumerate(components):
# individual target dir
component_element = Element(comp_obj)
model_type = str_utils.classname_to_alias(
type(comp_obj).__name__, AliasMode.JSON)
dir_prefix = str(index).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
component_dir_name = f'{dir_prefix}{const.IDX_SEP}{model_type}'
component_file = components_dir / f'{component_dir_name}{file_ext}'
# target control impl dir for the target
component_ctrl_impls: list = component_element.get_at(element_paths[1])
component_ctrl_dir = components_dir / element_paths[1].to_file_path(
root_dir=component_dir_name)
for i, component_ctrl_impl in enumerate(component_ctrl_impls):
model_type = str_utils.classname_to_alias(
type(component_ctrl_impl).__name__, AliasMode.JSON)
file_prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
file_name = f'{file_prefix}{const.IDX_SEP}{model_type}{file_ext}'
file_path = component_ctrl_dir / file_name
expected_plan.add_action(CreatePathAction(file_path))
expected_plan.add_action(
WriteFileAction(file_path, Element(component_ctrl_impl),
content_type))
# write stripped target model
stripped_target = comp_obj.stripped_instance(
stripped_fields_aliases=[element_paths[1].get_element_name()])
expected_plan.add_action(CreatePathAction(component_file))
expected_plan.add_action(
WriteFileAction(component_file, Element(stripped_target),
content_type))
root_file = component_def_dir / f'component-definition{file_ext}'
remaining_root = element.get().stripped_instance(
stripped_fields_aliases=[element_paths[0].get_element_name()])
expected_plan.add_action(CreatePathAction(root_file, True))
expected_plan.add_action(
WriteFileAction(root_file, Element(remaining_root), content_type))
split_plan = SplitCmd.split_model(component_def, element_paths,
component_def_dir, content_type, '',
None)
assert expected_plan == split_plan
def split_model_at_path_chain(cls,
model_obj: OscalBaseModel,
element_paths: List[ElementPath],
base_dir: pathlib.Path,
content_type: FileContentType,
cur_path_index: int,
split_plan: Plan,
strip_root: bool,
root_file_name: str = '') -> int:
"""Recursively split the model at the provided chain of element paths.
It assumes that a chain of element paths starts at the cur_path_index with the first path ending
with a wildcard (*)
It returns the index where the chain of path ends.
For example, element paths could have a list of paths as below for a `TargetDefinition` model where
the first path is the start of the chain.
For each of the sub model described by the first element path (e.g target-defintion.targets.*) in the chain,
the subsequent paths (e.g. target.target-control-implementations.*) will be applied recursively to retrieve
the sub-sub models:
[
'target-definition.targets.*',
'target.target-control-implementations.*'
]
for a command like below:
trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.*
"""
# assume we ran the command below:
# trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.*
if split_plan is None:
raise TrestleError('Split plan must have been initialized')
if cur_path_index < 0:
raise TrestleError(
'Current index of the chain of paths cannot be less than 0')
# if there are no more element_paths, return the current plan
if cur_path_index >= len(element_paths):
return cur_path_index
# initialize local variables
element = Element(model_obj)
stripped_field_alias = []
# get the sub_model specified by the element_path of this round
element_path = element_paths[cur_path_index]
is_parent = cur_path_index + 1 < len(element_paths) and element_paths[
cur_path_index + 1].get_parent() == element_path
# root dir name for sub models dir
# 00000__group.json will have the root_dir name as 00000__group for sub models of group
# catalog.json will have the root_dir name as catalog sub models
root_dir = ''
if root_file_name != '':
root_dir = pathlib.Path(root_file_name).stem
# check that the path is not multiple level deep
path_parts = element_path.get()
if path_parts[-1] == ElementPath.WILDCARD:
path_parts = path_parts[:-1]
if len(path_parts) > 2:
msg = 'Trestle supports split of first level children only, '
msg += f'found path "{element_path}" with level = {len(path_parts)}'
raise TrestleError(msg)
sub_models = element.get_at(
element_path,
False) # we call sub_models as in plural, but it can be just one
if sub_models is None:
return cur_path_index
# assume cur_path_index is the end of the chain
# value of this variable may change during recursive split of the sub-models below
path_chain_end = cur_path_index
# if wildcard is present in the element_path and the next path in the chain has current path as the parent,
# we need to split recursively and create separate file for each sub item
# for example, in the first round we get the `targets` using the path `target-definition.targets.*`
# so, now we need to split each of the target recursively. Note that target is an instance of dict
# However, there can be other sub_model, which is of type list
if is_parent and element_path.get_last() is not ElementPath.WILDCARD:
# create dir for all sub model items
sub_models_dir = base_dir / element_path.to_root_path()
sub_model_plan = Plan()
path_chain_end = cls.split_model_at_path_chain(
sub_models, element_paths, sub_models_dir, content_type,
cur_path_index + 1, sub_model_plan, True)
sub_model_actions = sub_model_plan.get_actions()
split_plan.add_actions(sub_model_actions)
elif element_path.get_last() == ElementPath.WILDCARD:
# extract sub-models into a dict with appropriate prefix
sub_model_items: Dict[str, OscalBaseModel] = {}
sub_models_dir = base_dir / element_path.to_file_path(
root_dir=root_dir)
if isinstance(sub_models, list):
for i, sub_model_item in enumerate(sub_models):
# e.g. `groups/00000_groups/`
prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
sub_model_items[prefix] = sub_model_item
elif isinstance(sub_models, dict):
# prefix is the key of the dict
sub_model_items = sub_models
else:
# unexpected sub model type for multi-level split with wildcard
raise TrestleError(
f'Sub element at {element_path} is not of type list or dict for further split'
)
# process list sub model items
for key in sub_model_items:
prefix = key
sub_model_item = sub_model_items[key]
# recursively split the sub-model if there are more element paths to traverse
# e.g. split target.target-control-implementations.*
require_recursive_split = cur_path_index + 1 < len(
element_paths) and element_paths[
cur_path_index + 1].get_parent() == element_path
if require_recursive_split:
# prepare individual directory for each sub-model
# e.g. `targets/<UUID>__target/`
sub_root_file_name = cmd_utils.to_model_file_name(
sub_model_item, prefix, content_type)
sub_model_plan = Plan()
path_chain_end = cls.split_model_at_path_chain(
sub_model_item, element_paths, sub_models_dir,
content_type, cur_path_index + 1, sub_model_plan, True,
sub_root_file_name)
sub_model_actions = sub_model_plan.get_actions()
else:
sub_model_actions = cls.prepare_sub_model_split_actions(
sub_model_item, sub_models_dir, prefix, content_type)
split_plan.add_actions(sub_model_actions)
else:
# the chain of path ends at the current index.
# so no recursive call. Let's just write the sub model to the file and get out
sub_model_file = base_dir / element_path.to_file_path(
content_type, root_dir=root_dir)
split_plan.add_action(CreatePathAction(sub_model_file))
split_plan.add_action(
WriteFileAction(
sub_model_file,
Element(sub_models, element_path.get_element_name()),
content_type))
# Strip the root model and add a WriteAction for the updated model object in the plan
if strip_root:
stripped_field_alias.append(element_path.get_element_name())
stripped_root = model_obj.stripped_instance(
stripped_fields_aliases=stripped_field_alias)
if root_file_name != '':
root_file = base_dir / root_file_name
else:
root_file = base_dir / element_path.to_root_path(content_type)
split_plan.add_action(CreatePathAction(root_file))
wrapper_alias = utils.classname_to_alias(
stripped_root.__class__.__name__, 'json')
split_plan.add_action(
WriteFileAction(root_file, Element(stripped_root,
wrapper_alias),
content_type))
# return the end of the current path chain
return path_chain_end
def test_element_get_at(sample_nist_component_def: component.ComponentDefinition):
"""Test element get method."""
element = Element(sample_nist_component_def)
# field alias should succeed
assert element.get_at(
ElementPath('component-definition.metadata.last-modified')
) == sample_nist_component_def.metadata.last_modified
# field name should fail
assert element.get_at(ElementPath('component-definition.metadata.last_modified')) is None
assert element.get() == sample_nist_component_def
assert element.get_at() == element.get()
assert element.get_at(ElementPath('component-definition.metadata')) == sample_nist_component_def.metadata
assert element.get_at(
ElementPath('component-definition.metadata.title')
) == sample_nist_component_def.metadata.title
assert element.get_at(ElementPath('component-definition.components')) == sample_nist_component_def.components
assert element.get_at(ElementPath('component-definition.components.*')) == sample_nist_component_def.components
assert element.get_at(
ElementPath('component-definition.metadata.parties.*')
) == sample_nist_component_def.metadata.parties
assert element.get_at(ElementPath('component-definition.metadata.parties.0')
) == sample_nist_component_def.metadata.parties[0]
assert element.get_at(ElementPath('component-definition.metadata.parties.0.uuid')
) == sample_nist_component_def.metadata.parties[0].uuid
for index in range(len(sample_nist_component_def.components)):
path_str = f'component-definition.components.{index}'
assert element.get_at(ElementPath(path_str)) == sample_nist_component_def.components[index]
# invalid indexing
assert element.get_at(ElementPath('component-definition.metadata.title.0')) is None
# invalid path with missing root model
assert element.get_at(ElementPath('metadata.title')) is None
# element_path with parent path
parent_path = ElementPath('component-definition.metadata')
element_path = ElementPath('metadata.parties.*', parent_path)
assert element.get_at(element_path) == sample_nist_component_def.metadata.parties
# element_path with parent path
parent_path = ElementPath('component-definition.components.*')
element_path = ElementPath('component.control-implementations.*', parent_path)
component_list = element.get_at(parent_path)
for component_item in component_list:
component_element = Element(component_item)
assert component_element.get_at(element_path) == component_item.control_implementations
# element_path in a list with parent path
parent_path = ElementPath('component-definition.components.*')
element_path = ElementPath('component.control-implementations.0', parent_path)
component_list = element.get_at(parent_path)
for component_item in component_list:
component_element = Element(component_item)
assert component_element.get_at(element_path) == component_item.control_implementations[0]
