def _parse_dict(data: Dict[str, Any], model_name: str) -> OscalBaseModel:
"""Load a model from the data dict.
This functionality is provided for situations when the OSCAL data type is not known ahead of time. Here the model
has been loaded into memory using json loads or similar and passed as a dict.
Args:
data: Oscal data loaded into memory in a dictionary with the `root key` removed.
model_name: it should be of the form <module>.<class> from trestle.oscal.* modules
Returns:
The oscal model of the desired model.
"""
if data is None:
raise TrestleError('data name is required')
if model_name is None:
raise TrestleError('model_name is required')
parts = model_name.split('.')
class_name = parts.pop()
module_name = '.'.join(parts)
logger.debug(f'Loading class "{class_name}" from "{module_name}"')
module = importlib.import_module(module_name)
mclass = getattr(module, class_name)
if mclass is None:
raise TrestleError(
f'class "{class_name}" could not be found in "{module_name}"')
instance = mclass.parse_obj(data)
return instance
def parse_dict(data: dict, model_name: str):
"""Load a model from the data dict.
Argument:
model_name: it should be of the form <module>.<class>
<class> should be a Pydantic class that supports `parse_obj` method
"""
warnings.warn('trestle.parser functions are deprecated',
DeprecationWarning)
if data is None:
raise TrestleError('data name is required')
if model_name is None:
raise TrestleError('model_name is required')
parts = model_name.split('.')
class_name = parts.pop()
module_name = '.'.join(parts)
logger.debug(f'Loading class "{class_name}" from "{module_name}"')
module = importlib.import_module(module_name)
mclass = getattr(module, class_name)
if mclass is None:
raise TrestleError(
f'class "{class_name}" could not be found in "{module_name}"')
instance = mclass.parse_obj(data)
return instance
def __init__(self,
sub_path: pathlib.Path,
clear_content: bool = False) -> None:
"""Initialize a create path action.
It creates all the missing directories in the path.
If it is a file, then it also creates an empty file with the name provided
Arguments:
sub_path: this is the desired file or directory path that needs to be created under the project root
"""
if not isinstance(sub_path, pathlib.Path):
raise TrestleError('Sub path must be of type pathlib.Path')
self._trestle_project_root = fs.get_trestle_project_root(sub_path)
if self._trestle_project_root is None:
raise TrestleError(
f'Sub path "{sub_path}" should be child of a valid trestle project'
)
self._sub_path = sub_path
self._created_paths: List[pathlib.Path] = []
# variables for handling with file content
self._clear_content = clear_content
self._old_file_content = None
super().__init__(ActionType.CREATE_PATH, True)
def _run(self, args) -> None:
"""Validate an OSCAL file in different modes."""
if args.file is None:
raise TrestleError(f'Argument "-{const.ARG_FILE_SHORT}" is required')
if args.mode is None:
raise TrestleError(f'Argument "-{const.ARG_MODE_SHORT}" is required')
mode = args.mode
if mode != const.VAL_MODE_DUPLICATES:
raise TrestleError(f'Mode value "{mode}" is not recognized.')
if args.item is None:
raise TrestleError(f'Argument "-{const.ARG_ITEM_SHORT}" is required')
item = args.item
file_path = pathlib.Path(args.file).absolute()
model_type, _ = fs.get_contextual_model_type(file_path)
model: OscalBaseModel = model_type.oscal_read(file_path)
loe = validator.find_values_by_name(model, item)
if loe:
nitems = len(loe)
is_valid = nitems == len(set(loe))
if is_valid:
self.out(f'The model is valid and contains no duplicates of item {args.item}')
else:
self.out(f'The model is invalid and contains duplicates of item {args.item}')
raise TrestleValidationError(f'Model {args.file} is invalid with duplicate values of {args.item}')
else:
self.out(f'The model is valid but contains no items of name {args.item}')
def set_at(self, element_path: ElementPath, sub_element: OscalBaseModel) -> 'Element':
"""Set a sub_element at the path in the current element.
Sub element can be Element, OscalBaseModel, list or None type
It returns the element itself so that chaining operation can be done such as
`element.set_at(path, sub-element).get()`.
"""
# convert the element_path to ElementPath if needed
if isinstance(element_path, str):
element_path = ElementPath(element_path)
# convert sub-element to OscalBaseModel if needed
model_obj = self._get_sub_element_obj(sub_element)
# find the root-model and element path parts
_, path_parts = self._split_element_path(element_path)
# TODO validate that self._elem is of same type as root_model
# If wildcard is present, check the input type and determine the preceding element
if element_path.get_last() == ElementPath.WILDCARD:
# validate the type is either list or OscalBaseModel
if not isinstance(model_obj, list) and not isinstance(model_obj, OscalBaseModel):
raise TrestleError(
f'The model object needs to be a List or OscalBaseModel for path with "{ElementPath.WILDCARD}"'
)
# since wildcard * is there, we need to go one level up for preceding element in the path
preceding_elm = self.get_preceding_element(element_path.get_preceding_path())
else:
# get the preceding element in the path
preceding_elm = self.get_preceding_element(element_path)
if preceding_elm is None:
raise TrestleError(f'Invalid sub element path {element_path} with no valid preceding element')
# check if it can be a valid sub_element of the parent
sub_element_name = element_path.get_element_name().replace('-', '_')
if hasattr(preceding_elm, sub_element_name) is False:
raise TrestleError(
f'Element "{preceding_elm.__class__}" does not have the attribute "{sub_element_name}" \
of type "{model_obj.__class__}"'
)
# set the sub-element
try:
setattr(preceding_elm, sub_element_name, model_obj)
except ValidationError:
sub_element_class = self.get_sub_element_class(preceding_elm, sub_element_name)
raise TrestleError(
f'Validation error: {sub_element_name} is expected to be "{sub_element_class}", \
but found "{model_obj.__class__}"'
)
# returning self will allow to do 'chaining' of commands after set
return self
def parse_element_arg(element_arg: str, contextual_mode: bool = True) -> List[ElementPath]:
"""Parse an element arg string into a list of ElementPath.
contextual_mode specifies if the path is a valid project model path or not. For example,
if we are processing a metadata.parties.*, we need to know which metadata we are processing. If we pass
contextual_mode=True, we can infer the root model by inspecting the file directory
If contextual_mode=False, then the path must include the full path, e.g. catalog.metadata.parties.* instead of just
metadata.parties.*
One option for caller to utilize this utility function: fs.is_valid_project_model_path(pathlib.Path.cwd())
"""
element_paths: List[ElementPath] = []
element_arg = element_arg.strip()
# search for wildcards and create paths with its parent path
path_parts = element_arg.split(ElementPath.PATH_SEPARATOR)
if len(path_parts) <= 0:
raise TrestleError(f'Invalid element path "{element_arg}" without any path separator')
prev_element_path = None
parent_model = path_parts[0]
i = 1
while i < len(path_parts):
p = path_parts[i]
if p == ElementPath.WILDCARD and len(element_paths) > 0:
# append wildcard to the latest element path
latest_path = element_paths.pop()
if latest_path.get_last() == ElementPath.WILDCARD:
raise TrestleError(f'Invalid element path with consecutive {ElementPath.WILDCARD}')
latest_path_str = ElementPath.PATH_SEPARATOR.join([latest_path.to_string(), p])
element_path = ElementPath(latest_path_str, latest_path.get_parent())
else:
# create and append elment_path
p = ElementPath.PATH_SEPARATOR.join([parent_model, p])
element_path = ElementPath(p, parent_path=prev_element_path)
# if the path has wildcard and there is more parts later,
# get the parent model for the alias path
if element_path.get_last() == ElementPath.WILDCARD:
full_path_str = ElementPath.PATH_SEPARATOR.join(element_path.get_full_path_parts()[:-1])
parent_model = fs.get_singular_alias(full_path_str, contextual_mode)
else:
parent_model = element_path.get_element_name()
# store values for next cycle
prev_element_path = element_path
element_paths.append(element_path)
i += 1
if len(element_paths) <= 0:
raise TrestleError(f'Invalid element path "{element_arg}" without any path separator')
return element_paths
def execute(self) -> None:
"""Execute the action."""
if self._element is None:
raise TrestleError('Element is empty and cannot write')
if not self._is_writer_valid():
raise TrestleError('Writer is not provided or closed')
self._writer.write(self._encode())
self._writer.flush()
self._mark_executed()
def rollback(self) -> None:
"""Rollback the action."""
if not self._is_writer_valid():
raise TrestleError('Writer is not provided or closed')
if self._lastStreamPos < 0:
raise TrestleError('Last stream position is not available to rollback to')
if self.has_executed():
self._writer.seek(self._lastStreamPos)
self._writer.truncate()
self._mark_rollback()
def test_trestle_error():
"""Test trestle error."""
msg = 'Custom error'
try:
raise TrestleError(msg)
except TrestleError as err:
assert err.msg == msg
def _run(self, args):
"""Validate an OSCAL file in different modes."""
# get the Model
if args[const.ARG_FILE] is None:
raise TrestleError(f'Argument "-{const.ARG_FILE_SHORT}" is required')
model: OscalBaseModel = cmd_utils.get_model(args[const.ARG_FILE])
element_paths: List[ElementPath] = cmd_utils.parse_element_args(args[const.ARG_ELEMENT])
split_plan = self._split_model(model, element_paths)
try:
split_plan.execute()
except Exception as ex:
split_plan.rollback()
raise TrestleError(f'Could not perform operation: {ex}')
def to_content_type(cls, file_extension: str) -> 'FileContentType':
"""Get content type form file extension."""
if file_extension == '.json':
return FileContentType.JSON
elif file_extension == '.yaml' or file_extension == '.yml':
return FileContentType.YAML
raise TrestleError(f'Unsupported file extension {file_extension}')
def to_file_extension(cls, content_type: 'FileContentType') -> str:
"""Get file extension for the type."""
if content_type == FileContentType.YAML:
return '.yaml'
elif content_type == FileContentType.JSON:
return '.json'
raise TrestleError(f'Invalid file content type {content_type}')
def rollback(self) -> None:
"""Execute the rollback action."""
if not self._file_path.exists():
raise TrestleError(f'File at {self._file_path} does not exist')
with open(self._file_path, 'a+') as writer:
self._writer = writer
super().rollback()
def root_key(data: dict):
"""Find root model name in the data."""
warnings.warn('trestle.parser functions are deprecated',
DeprecationWarning)
if len(data.items()) == 1:
return next(iter(data))
raise TrestleError('data does not contain a root key')
def _encode(self) -> str:
"""Encode the element to appropriate content type."""
if self._content_type == FileContentType.YAML:
return self._element.to_yaml()
elif self._content_type == FileContentType.JSON:
return self._element.to_json()
raise TrestleError(f'Invalid content type {self._content_type}')
def __init__(self, file_path: pathlib.Path, element: Element, content_type: FileContentType) -> None:
"""Initialize a write file action.
It opens the file in append mode. Therefore the file needs to exist even if it is a new file.
"""
if not isinstance(file_path, pathlib.Path):
raise TrestleError('file_path should be of type pathlib.Path')
if not self._valid_file_extension(file_path, content_type):
raise TrestleError(
f'Invalid file type extension in path "{file_path}" for content type "{content_type.name}"'
)
self._file_path = file_path
# initialize super without writer for now
# Note, execute and rollback sets the writer as appropriate
super().__init__(None, element, content_type)
def _parse(self, element_path) -> List[str]:
"""Parse the element path and validate."""
parts: List[str] = element_path.split(self.PATH_SEPARATOR)
for i, part in enumerate(parts):
if part == '':
raise TrestleError(
f'Invalid path "{element_path}" because having empty path parts between "{self.PATH_SEPARATOR}" \
or in the beginning')
elif part == self.WILDCARD and i != len(parts) - 1:
raise TrestleError(
f'Invalid path. Wildcard "{self.WILDCARD}" can only be at the end'
)
if parts[-1] == self.WILDCARD and len(parts) == 1:
raise TrestleError(f'Invalid path {element_path}')
return parts
def __init__(self, sub_path: pathlib.Path) -> None:
"""Initialize a remove path action.
It removes the file or directory recursively into trash.
Arguments:
sub_path: this is the desired file or directory path that needs to be removed under the project root
"""
if not isinstance(sub_path, pathlib.Path):
raise TrestleError('Sub path must be of type pathlib.Path')
self._trestle_project_root = fs.get_trestle_project_root(sub_path)
if self._trestle_project_root is None:
raise TrestleError(f'Sub path "{sub_path}" should be child of a valid trestle project')
self._sub_path = sub_path
super().__init__(ActionType.REMOVE_PATH, True)
def load_file(file_name: str) -> Dict[str, Any]:
"""Load JSON or YAML file content into a dict."""
_, file_extension = os.path.splitext(file_name)
with open(file_name) as f:
if file_extension == '.yaml':
return yaml.load(f, yaml.FullLoader)
elif file_extension == '.json':
return json.load(f)
else:
raise TrestleError(f'Invalid file extension "{file_extension}"')
def __init__(self, file_path: pathlib.Path, element: Element,
content_type: FileContentType) -> None:
"""Initialize a write file action.
It opens the file in append mode. Therefore the file needs to exist even if it is a new file.
"""
if not isinstance(file_path, pathlib.Path):
raise TrestleError('file_path should be of type pathlib.Path')
inferred_content_type = FileContentType.to_content_type(
file_path.suffix)
if inferred_content_type != content_type:
raise TrestleError(
f'Mismatch between stated content type {content_type.name} and file path {file_path}'
)
self._file_path = file_path
# initialize super without writer for now
# Note, execute and rollback sets the writer as appropriate
super().__init__(None, element, content_type)
def _get_sub_element_obj(self, sub_element):
"""Convert sub element into allowed model obj."""
if not self.is_allowed_sub_element_type(sub_element):
raise TrestleError(
f'Sub element must be one of "{self.get_allowed_sub_element_types()}", found "{sub_element.__class__}"'
)
model_obj = sub_element
if isinstance(sub_element, Element):
model_obj = sub_element.get()
return model_obj
def __init__(self, writer: Optional[io.TextIOWrapper], element: Element, content_type: FileContentType) -> None:
"""Initialize an write file action."""
super().__init__(ActionType.WRITE, True)
if writer is not None and not issubclass(io.TextIOWrapper, writer.__class__):
raise TrestleError(f'Writer must be of io.TextIOWrapper, given f{writer.__class__}')
self._writer: Optional[io.TextIOWrapper] = writer
self._element: Element = element
self._content_type: FileContentType = content_type
self._lastStreamPos = -1
if self._writer is not None:
self._lastStreamPos = self._writer.tell()
def execute(self) -> None:
"""Execute the action."""
if not self._file_path.exists():
raise TrestleError(f'File at {self._file_path} does not exist')
with open(self._file_path, 'a+') as writer:
if self._lastStreamPos < 0:
self._lastStreamPos = writer.tell()
else:
writer.seek(self._lastStreamPos)
self._writer = writer
super().execute()
def _parse(self, element_path: str) -> List[str]:
"""Parse the element path and validate."""
parts: List[str] = element_path.split(self.PATH_SEPARATOR)
for i, part in enumerate(parts):
if part == '':
raise TrestleError(
f'Invalid path "{element_path}" because having empty path parts between "{self.PATH_SEPARATOR}" \
or in the beginning'
)
elif part == self.WILDCARD and i != len(parts) - 1:
raise TrestleError(f'Invalid path. Wildcard "{self.WILDCARD}" can only be at the end')
if parts[-1] == self.WILDCARD:
if len(parts) == 1:
raise TrestleError(f'Invalid path {element_path} with wildcard.')
if len(parts) <= 1:
raise TrestleError(
'Element path must have at least two parts with the first part being the model root name \
like "target-definition.metadata"'
)
return parts
def class_to_oscal(class_name: str, mode: str) -> str:
"""
Return oscal json or field element name based on class name.
This is applicable when asking for a singular element.
"""
warnings.warn(
'trestle.parser functions are deprecated. class_to_oscal now contained in utils',
DeprecationWarning)
parts = pascal_case_split(class_name)
if mode == 'json':
return '-'.join(map(str.lower, parts))
elif mode == 'field':
return '_'.join(map(str.lower, parts))
else:
raise TrestleError('Bad option')
def _run(self, args: argparse.Namespace) -> int:
"""Merge elements into the parent oscal model."""
log.set_log_level_from_args(args)
try:
# Handle multiple element paths: element_paths = args.element.split(',')
if len(args.element.split(',')) > 1:
raise TrestleError(
'Trestle merge -e/-element currently takes only 1 element.'
)
plan = self.merge(ElementPath(args.element))
plan.simulate()
plan.execute()
except BaseException as err:
logger.error(f'Merge failed: {err}')
return 1
return 0
def parse_file(file_name: pathlib.Path,
model_name: Optional[str]) -> OscalBaseModel:
"""
Load an oscal file from the file system where the oscal model type is not known.
Args:
file_name: File path
model_name: it should be of the form module.class which is derived from OscalBaseModel
"""
if file_name is None:
raise TrestleError('file_name is required')
data = fs.load_file(file_name)
rkey = root_key(data)
if model_name is None:
model_name = to_full_model_name(rkey)
return _parse_dict(data[rkey], model_name)
def load_file(file_name: pathlib.Path) -> Dict[str, Any]:
"""
Load JSON or YAML file content into a dict.
This is not intended to be the default load mechanism. It should only be used
if a OSCAL object type is unknown but the context a user is in.
"""
content_type = FileContentType.to_content_type(file_name.suffix)
with file_name.open('r', encoding=const.FILE_ENCODING) as f:
if content_type == FileContentType.YAML:
return yaml.load(f, yaml.FullLoader)
elif content_type == FileContentType.JSON:
return json.load(f)
else:
logger.debug(
f'Invalid file extension "{file_name.suffix}" in load')
raise TrestleError(f'Invalid file extension "{file_name.suffix}"')
def __init__(self, sub_element, dest_element: Element, sub_element_path: ElementPath) -> None:
"""Initialize an add element action.
Sub element can be OscalBaseModel, Element, list or None
"""
super().__init__(ActionType.UPDATE, True)
if not Element.is_allowed_sub_element_type(sub_element):
allowed_types = Element.get_allowed_sub_element_types()
raise TrestleError(
f'Sub element "{sub_element.__class__} is not a allowed sub element types in "{allowed_types}"'
)
self._sub_element = sub_element
self._dest_element: Element = dest_element
self._sub_element_path: ElementPath = sub_element_path
self._prev_sub_element = None
def parse_file(file_name: str, model_name: str):
"""Load a model from the file.
Argument:
model_name: it should be of the form <module>.<class>
<class> should be a Pydantic class that supports `parse_obj` method
"""
warnings.warn('trestle.parser functions are deprecated',
DeprecationWarning)
if file_name is None:
raise TrestleError('file_name is required')
data = fs.load_file(file_name)
rkey = root_key(data)
if model_name is None:
model_name = to_full_model_name(rkey)
return parse_dict(data[rkey], model_name)