def test_stripped_instance(sample_target_def: OscalBaseModel) -> None:
"""Test stripped_instance method."""
assert hasattr(sample_target_def, 'metadata')
sc_instance = sample_target_def.stripped_instance(
stripped_fields_aliases=['metadata'])
assert not hasattr(sc_instance, 'metadata')
sc_instance = sample_target_def.stripped_instance(
stripped_fields=['metadata'])
assert not hasattr(sc_instance, 'metadata')
with pytest.raises(err.TrestleError):
sc_instance = sample_target_def.stripped_instance(
stripped_fields_aliases=['invalid'])
if isinstance(sample_target_def, ostarget.TargetDefinition):
metadata = sample_target_def.metadata
assert hasattr(metadata, 'last_modified')
instance = metadata.stripped_instance(
stripped_fields_aliases=['last-modified'])
assert not hasattr(instance, 'last_modified')
instance = metadata.stripped_instance(
stripped_fields=['last_modified'])
assert not hasattr(sc_instance, 'last_modified')
else:
raise Exception('Test failure')
def create_trestle_project_with_model(
top_dir: pathlib.Path, model_obj: OscalBaseModel, model_name: str, monkeypatch: MonkeyPatch
) -> pathlib.Path:
"""Create initialized trestle project and import the model into it."""
cur_dir = pathlib.Path.cwd()
# create subdirectory for trestle project
trestle_root = top_dir / 'my_trestle'
trestle_root.mkdir()
os.chdir(trestle_root)
try:
testargs = ['trestle', 'init']
monkeypatch.setattr(sys, 'argv', testargs)
assert Trestle().run() == 0
# place model object in top directory outside trestle project
# so it can be imported
tmp_model_path = top_dir / (model_name + '.json')
model_obj.oscal_write(tmp_model_path)
i = ImportCmd()
args = argparse.Namespace(
trestle_root=trestle_root, file=str(tmp_model_path), output=model_name, verbose=0, regenerate=False
)
assert i._run(args) == 0
except Exception as e:
raise TrestleError(f'Error creating trestle project with model: {e}')
finally:
os.chdir(cur_dir)
return trestle_root
def prepare_trestle_project_dir(
repo_dir: pathlib.Path, content_type: FileContentType, model_obj: OscalBaseModel, models_dir_name: str
):
"""Prepare a temp directory with an example OSCAL model."""
ensure_trestle_config_dir(repo_dir)
model_alias = str_utils.classname_to_alias(model_obj.__class__.__name__, AliasMode.JSON)
file_ext = FileContentType.to_file_extension(content_type)
models_full_path = repo_dir / models_dir_name / 'my_test_model'
model_def_file = models_full_path / f'{model_alias}{file_ext}'
models_full_path.mkdir(exist_ok=True, parents=True)
model_obj.oscal_write(model_def_file)
return models_full_path, model_def_file
def prepare_trestle_project_dir(
tmp_dir, content_type: FileContentType, model_obj: OscalBaseModel, models_dir_name: str
):
"""Prepare a temp directory with an example OSCAL model."""
ensure_trestle_config_dir(tmp_dir)
model_alias = utils.classname_to_alias(model_obj.__class__.__name__, 'json')
file_ext = FileContentType.to_file_extension(content_type)
models_full_path = tmp_dir / models_dir_name / 'my_test_model'
model_def_file = models_full_path / f'{model_alias}{file_ext}'
fs.ensure_directory(models_full_path)
model_obj.oscal_write(model_def_file)
return models_full_path, model_def_file
def split_model(
cls,
model_obj: OscalBaseModel,
element_paths: List[ElementPath],
base_dir: pathlib.Path,
content_type: FileContentType,
root_file_name: str,
aliases_to_strip: Dict[str, AliasTracker]
) -> Plan:
"""Split the model at the provided element paths.
It returns a plan for the operation
"""
# initialize plan
split_plan = Plan()
# loop through the element path list and update the split_plan
stripped_field_alias = []
cur_path_index = 0
while cur_path_index < len(element_paths):
# extract the sub element name for each of the root path of the path chain
element_path = element_paths[cur_path_index]
if element_path.get_parent() is None and len(element_path.get()) > 1:
stripped_part = element_path.get()[1]
if stripped_part == ElementPath.WILDCARD:
stripped_field_alias.append('__root__')
else:
if stripped_part not in stripped_field_alias:
stripped_field_alias.append(stripped_part)
# split model at the path chain
cur_path_index = cls.split_model_at_path_chain(
model_obj,
element_paths,
base_dir,
content_type,
cur_path_index,
split_plan,
False,
root_file_name,
aliases_to_strip
)
cur_path_index += 1
# strip the root model object and add a WriteAction
stripped_root = model_obj.stripped_instance(stripped_fields_aliases=stripped_field_alias)
# If it's an empty model after stripping the fields, don't create path and don't write
if set(model_obj.__fields__.keys()) == set(stripped_field_alias):
return split_plan
if root_file_name != '':
root_file = base_dir / root_file_name
else:
root_file = base_dir / element_paths[0].to_root_path(content_type)
split_plan.add_action(CreatePathAction(root_file, True))
wrapper_alias = classname_to_alias(stripped_root.__class__.__name__, AliasMode.JSON)
split_plan.add_action(WriteFileAction(root_file, Element(stripped_root, wrapper_alias), content_type))
return split_plan
def split_model(cls,
model_obj: OscalBaseModel,
element_paths: List[ElementPath],
base_dir: pathlib.Path,
content_type: FileContentType,
root_file_name: str = '') -> Plan:
"""Split the model at the provided element paths.
It returns a plan for the operation
"""
# assume we ran the command below:
# trestle split -f target.yaml
# -e 'target-definition.metadata,
# target-definition.targets.*.target-control-implementations.*'
# initialize plan
split_plan = Plan()
# loop through the element path list and update the split_plan
stripped_field_alias = []
cur_path_index = 0
while cur_path_index < len(element_paths):
# extract the sub element name for each of the root path of the path chain
element_path = element_paths[cur_path_index]
if element_path.get_parent() is None and len(
element_path.get()) > 1:
stripped_part = element_path.get()[1]
if stripped_part == ElementPath.WILDCARD:
stripped_field_alias.append('__root__')
else:
stripped_field_alias.append(stripped_part)
# split model at the path chain
cur_path_index = cls.split_model_at_path_chain(
model_obj, element_paths, base_dir, content_type,
cur_path_index, split_plan, False, root_file_name)
cur_path_index += 1
# strip the root model object and add a WriteAction
stripped_root = model_obj.stripped_instance(
stripped_fields_aliases=stripped_field_alias)
if root_file_name != '':
root_file = base_dir / root_file_name
else:
root_file = base_dir / element_paths[0].to_root_path(content_type)
split_plan.add_action(CreatePathAction(root_file, True))
wrapper_alias = utils.classname_to_alias(
stripped_root.__class__.__name__, 'json')
split_plan.add_action(
WriteFileAction(root_file, Element(stripped_root, wrapper_alias),
content_type))
return split_plan
def verify_file_content(file_path: pathlib.Path, model: OscalBaseModel):
"""Verify that the file contains the correct model data."""
model.oscal_read(file_path)
def split_model_at_path_chain(cls,
model_obj: OscalBaseModel,
element_paths: List[ElementPath],
base_dir: pathlib.Path,
content_type: FileContentType,
cur_path_index: int,
split_plan: Plan,
strip_root: bool,
root_file_name: str = '') -> int:
"""Recursively split the model at the provided chain of element paths.
It assumes that a chain of element paths starts at the cur_path_index with the first path ending
with a wildcard (*)
It returns the index where the chain of path ends.
For example, element paths could have a list of paths as below for a `TargetDefinition` model where
the first path is the start of the chain.
For each of the sub model described by the first element path (e.g target-defintion.targets.*) in the chain,
the subsequent paths (e.g. target.target-control-implementations.*) will be applied recursively to retrieve
the sub-sub models:
[
'target-definition.targets.*',
'target.target-control-implementations.*'
]
for a command like below:
trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.*
"""
# assume we ran the command below:
# trestle split -f target.yaml -e target-definition.targets.*.target-control-implementations.*
if split_plan is None:
raise TrestleError('Split plan must have been initialized')
if cur_path_index < 0:
raise TrestleError(
'Current index of the chain of paths cannot be less than 0')
# if there are no more element_paths, return the current plan
if cur_path_index >= len(element_paths):
return cur_path_index
# initialize local variables
element = Element(model_obj)
stripped_field_alias = []
# get the sub_model specified by the element_path of this round
element_path = element_paths[cur_path_index]
is_parent = cur_path_index + 1 < len(element_paths) and element_paths[
cur_path_index + 1].get_parent() == element_path
# root dir name for sub models dir
# 00000__group.json will have the root_dir name as 00000__group for sub models of group
# catalog.json will have the root_dir name as catalog sub models
root_dir = ''
if root_file_name != '':
root_dir = pathlib.Path(root_file_name).stem
# check that the path is not multiple level deep
path_parts = element_path.get()
if path_parts[-1] == ElementPath.WILDCARD:
path_parts = path_parts[:-1]
if len(path_parts) > 2:
msg = 'Trestle supports split of first level children only, '
msg += f'found path "{element_path}" with level = {len(path_parts)}'
raise TrestleError(msg)
sub_models = element.get_at(
element_path,
False) # we call sub_models as in plural, but it can be just one
if sub_models is None:
return cur_path_index
# assume cur_path_index is the end of the chain
# value of this variable may change during recursive split of the sub-models below
path_chain_end = cur_path_index
# if wildcard is present in the element_path and the next path in the chain has current path as the parent,
# we need to split recursively and create separate file for each sub item
# for example, in the first round we get the `targets` using the path `target-definition.targets.*`
# so, now we need to split each of the target recursively. Note that target is an instance of dict
# However, there can be other sub_model, which is of type list
if is_parent and element_path.get_last() is not ElementPath.WILDCARD:
# create dir for all sub model items
sub_models_dir = base_dir / element_path.to_root_path()
sub_model_plan = Plan()
path_chain_end = cls.split_model_at_path_chain(
sub_models, element_paths, sub_models_dir, content_type,
cur_path_index + 1, sub_model_plan, True)
sub_model_actions = sub_model_plan.get_actions()
split_plan.add_actions(sub_model_actions)
elif element_path.get_last() == ElementPath.WILDCARD:
# extract sub-models into a dict with appropriate prefix
sub_model_items: Dict[str, OscalBaseModel] = {}
sub_models_dir = base_dir / element_path.to_file_path(
root_dir=root_dir)
if isinstance(sub_models, list):
for i, sub_model_item in enumerate(sub_models):
# e.g. `groups/00000_groups/`
prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
sub_model_items[prefix] = sub_model_item
elif isinstance(sub_models, dict):
# prefix is the key of the dict
sub_model_items = sub_models
else:
# unexpected sub model type for multi-level split with wildcard
raise TrestleError(
f'Sub element at {element_path} is not of type list or dict for further split'
)
# process list sub model items
for key in sub_model_items:
prefix = key
sub_model_item = sub_model_items[key]
# recursively split the sub-model if there are more element paths to traverse
# e.g. split target.target-control-implementations.*
require_recursive_split = cur_path_index + 1 < len(
element_paths) and element_paths[
cur_path_index + 1].get_parent() == element_path
if require_recursive_split:
# prepare individual directory for each sub-model
# e.g. `targets/<UUID>__target/`
sub_root_file_name = cmd_utils.to_model_file_name(
sub_model_item, prefix, content_type)
sub_model_plan = Plan()
path_chain_end = cls.split_model_at_path_chain(
sub_model_item, element_paths, sub_models_dir,
content_type, cur_path_index + 1, sub_model_plan, True,
sub_root_file_name)
sub_model_actions = sub_model_plan.get_actions()
else:
sub_model_actions = cls.prepare_sub_model_split_actions(
sub_model_item, sub_models_dir, prefix, content_type)
split_plan.add_actions(sub_model_actions)
else:
# the chain of path ends at the current index.
# so no recursive call. Let's just write the sub model to the file and get out
sub_model_file = base_dir / element_path.to_file_path(
content_type, root_dir=root_dir)
split_plan.add_action(CreatePathAction(sub_model_file))
split_plan.add_action(
WriteFileAction(
sub_model_file,
Element(sub_models, element_path.get_element_name()),
content_type))
# Strip the root model and add a WriteAction for the updated model object in the plan
if strip_root:
stripped_field_alias.append(element_path.get_element_name())
stripped_root = model_obj.stripped_instance(
stripped_fields_aliases=stripped_field_alias)
if root_file_name != '':
root_file = base_dir / root_file_name
else:
root_file = base_dir / element_path.to_root_path(content_type)
split_plan.add_action(CreatePathAction(root_file))
wrapper_alias = utils.classname_to_alias(
stripped_root.__class__.__name__, 'json')
split_plan.add_action(
WriteFileAction(root_file, Element(stripped_root,
wrapper_alias),
content_type))
# return the end of the current path chain
return path_chain_end
def split_model_at_path_chain(
cls,
model_obj: OscalBaseModel,
element_paths: List[ElementPath],
base_dir: pathlib.Path,
content_type: FileContentType,
cur_path_index: int,
split_plan: Plan,
strip_root: bool,
root_file_name: str,
aliases_to_strip: Dict[str, AliasTracker],
last_one: bool = True
) -> int:
"""Recursively split the model at the provided chain of element paths.
It assumes that a chain of element paths starts at the cur_path_index with the first path ending
with a wildcard (*)
If the wildcard follows an element that is inherently a list of items, the list of items is extracted.
But if the wildcard follows a generic model than members of that model class found in the model will be
split off. But only the non-trivial elements are removed, i.e. not str, int, datetime, etc.
Args:
model_obj: The OscalBaseModel to be split
element_paths: The List[ElementPath] of elements to split, including embedded wildcards
base_dir: pathlib.Path of the file being split
content_type: json or yaml files
cur_path_index: Index into the list of element paths for the current split operation
split_plan: The accumulated plan of actions needed to perform the split
strip_root: Whether to strip elements from the root object
root_file_name: Filename of root file that gets split into a list of items
aliases_to_strip: AliasTracker previously loaded with aliases that need to be split from each element
last_one: bool indicating last item in array has been split and stripped model can now be written
Returns:
int representing the index where the chain of the path ends.
Examples:
For example, element paths could have a list of paths as below for a `ComponentDefinition` model where
the first path is the start of the chain.
For each of the sub model described by the first element path (e.g component-defintion.components.*) in the
chain, the subsequent paths (e.g component.control-implementations.*) will be applied recursively
to retrieve the sub-sub models:
[
'component-definition.component.*',
'component.control-implementations.*'
]
for a command like below:
trestle split -f component.yaml -e component-definition.components.*.control-implementations.*
"""
if split_plan is None:
raise TrestleError('Split plan must have been initialized')
if cur_path_index < 0:
raise TrestleError('Current index of the chain of paths cannot be less than 0')
# if there are no more element_paths, return the current plan
if cur_path_index >= len(element_paths):
return cur_path_index
# initialize local variables
element = Element(model_obj)
stripped_field_alias: List[str] = []
# get the sub_model specified by the element_path of this round
element_path = element_paths[cur_path_index]
# does the next element_path point back at me
is_parent = cur_path_index + 1 < len(element_paths) and element_paths[cur_path_index
+ 1].get_parent() == element_path
# root dir name for sub models dir
# 00000__group.json will have the root_dir name as 00000__group for sub models of group
# catalog.json will have the root_dir name as catalog
root_dir = ''
if root_file_name != '':
root_dir = str(pathlib.Path(root_file_name).with_suffix(''))
sub_models = element.get_at(element_path, False) # we call sub_models as in plural, but it can be just one
# assume cur_path_index is the end of the chain
# value of this variable may change during recursive split of the sub-models below
path_chain_end = cur_path_index
# if wildcard is present in the element_path and the next path in the chain has current path as the parent,
# Then deal with case of list, or split of arbitrary oscalbasemodel
if is_parent and element_path.get_last() is not ElementPath.WILDCARD:
# create dir for all sub model items
sub_models_dir = base_dir / element_path.to_root_path()
sub_model_plan = Plan()
path_chain_end = cls.split_model_at_path_chain(
sub_models,
element_paths,
sub_models_dir,
content_type,
cur_path_index + 1,
sub_model_plan,
True,
'',
aliases_to_strip
)
sub_model_actions = sub_model_plan.get_actions()
split_plan.add_actions(sub_model_actions)
elif element_path.get_last() == ElementPath.WILDCARD:
# extract sub-models into a dict with appropriate prefix
sub_model_items: Dict[str, OscalBaseModel] = {}
sub_models_dir = base_dir / element_path.to_file_path(root_dir=root_dir)
if isinstance(sub_models, list):
for i, sub_model_item in enumerate(sub_models):
# e.g. `groups/00000_groups/`
prefix = str(i).zfill(const.FILE_DIGIT_PREFIX_LENGTH)
sub_model_items[prefix] = sub_model_item
# process list sub model items
count = 0
for key, sub_model_item in sub_model_items.items():
count += 1
# recursively split the sub-model if there are more element paths to traverse
# e.g. split component.control-implementations.*
require_recursive_split = cur_path_index + 1 < len(element_paths) and element_paths[
cur_path_index + 1].get_parent() == element_path
if require_recursive_split:
# prepare individual directory for each sub-model
sub_root_file_name = cmd_utils.to_model_file_name(sub_model_item, key, content_type)
sub_model_plan = Plan()
last_one: bool = count == len(sub_model_items)
path_chain_end = cls.split_model_at_path_chain(
sub_model_item,
element_paths,
sub_models_dir,
content_type,
cur_path_index + 1,
sub_model_plan,
True,
sub_root_file_name,
aliases_to_strip,
last_one
)
sub_model_actions = sub_model_plan.get_actions()
else:
sub_model_actions = cls.prepare_sub_model_split_actions(
sub_model_item, sub_models_dir, key, content_type
)
split_plan.add_actions(sub_model_actions)
else:
# the chain of path ends at the current index.
# so no recursive call. Let's just write the sub model to the file and get out
if sub_models is not None:
sub_model_file = base_dir / element_path.to_file_path(content_type, root_dir=root_dir)
split_plan.add_action(CreatePathAction(sub_model_file))
split_plan.add_action(
WriteFileAction(sub_model_file, Element(sub_models, element_path.get_element_name()), content_type)
)
# Strip the root model and add a WriteAction for the updated model object in the plan
if strip_root:
full_path = element_path.get_full()
path = '.'.join(full_path.split('.')[:-1])
aliases = [element_path.get_element_name()]
need_to_write = True
use_alias_dict = aliases_to_strip is not None and path in aliases_to_strip
if use_alias_dict:
aliases = aliases_to_strip[path].get_aliases()
need_to_write = aliases_to_strip[path].needs_writing()
stripped_model = model_obj.stripped_instance(stripped_fields_aliases=aliases)
# can mark it written even if it doesn't need writing since it is empty
# but if an array only mark it written if it's the last one
if last_one and use_alias_dict:
aliases_to_strip[path].mark_written()
# If it's an empty model after stripping the fields, don't create path and don't write
field_list = [x for x in model_obj.__fields__.keys() if model_obj.__fields__[x] is not None]
if set(field_list) == set(stripped_field_alias):
return path_chain_end
if need_to_write:
if root_file_name != '':
root_file = base_dir / root_file_name
else:
root_file = base_dir / element_path.to_root_path(content_type)
split_plan.add_action(CreatePathAction(root_file))
wrapper_alias = classname_to_alias(stripped_model.__class__.__name__, AliasMode.JSON)
split_plan.add_action(WriteFileAction(root_file, Element(stripped_model, wrapper_alias), content_type))
# return the end of the current path chain
return path_chain_end