def get_singular_alias(alias_path: str, contextual_mode: bool = False) -> str:
"""
Get the alias in the singular form from a jsonpath.
If contextual_mode is True and contextual_path is None, it assumes alias_path is relative to the directory the user
is running trestle from.
"""
if len(alias_path.strip()) == 0:
raise err.TrestleError('Invalid jsonpath.')
singular_alias: str = ''
full_alias_path = alias_path
if contextual_mode:
_, full_model_alias = get_contextual_model_type()
first_alias_a = full_model_alias.split('.')[-1]
first_alias_b = alias_path.split('.')[0]
if first_alias_a == first_alias_b:
full_model_alias = '.'.join(full_model_alias.split('.')[:-1])
full_alias_path = '.'.join([full_model_alias, alias_path]).strip('.')
path_parts = full_alias_path.split(const.ALIAS_PATH_SEPARATOR)
if len(path_parts) < 2:
raise err.TrestleError('Invalid jsonpath.')
model_types = []
root_model_alias = path_parts[0]
found = False
for module_name in const.MODELTYPE_TO_MODELMODULE.values():
model_type, model_alias = utils.get_root_model(module_name)
if root_model_alias == model_alias:
found = True
model_types.append(model_type)
break
if not found:
raise err.TrestleError(
f'{root_model_alias} is an invalid root model alias.')
model_type = model_types[0]
for i in range(1, len(path_parts)):
if utils.is_collection_field_type(model_type):
model_type = utils.get_inner_type(model_type)
i = i + 1
else:
model_type = model_type.alias_to_field_map()[
path_parts[i]].outer_type_
model_types.append(model_type)
if not utils.is_collection_field_type(model_type):
raise err.TrestleError('Not a valid generic collection model.')
last_alias = path_parts[-1]
parent_model_type = model_types[-2]
singular_alias = utils.classname_to_alias(
utils.get_inner_type(parent_model_type.alias_to_field_map()
[last_alias].outer_type_).__name__, 'json')
return singular_alias
def test_get_target_model() -> None:
"""Test utils method get_target_model."""
assert mutils.is_collection_field_type(
mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)
) is True
assert (mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)).__origin__ is list
assert mutils.get_inner_type(
mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)
) is catalog.Role
assert mutils.is_collection_field_type(
mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog)
) is True
assert mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog).__origin__ is dict
assert mutils.get_inner_type(
mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog)
) is catalog.ResponsibleParty
assert mutils.is_collection_field_type(
mutils.get_target_model(['catalog', 'metadata', 'responsible-parties', 'creator'], catalog.Catalog)
) is False
assert mutils.get_target_model(
['catalog', 'metadata', 'responsible-parties', 'creator'], catalog.Catalog
) is catalog.ResponsibleParty
assert mutils.get_target_model(['catalog', 'metadata'], catalog.Catalog) is catalog.Metadata
with pytest.raises(err.TrestleError):
mutils.get_target_model(['catalog', 'metadata', 'bad_element'], catalog.Catalog)
def generate_sample_model(model: Type[Any]) -> OscalBaseModel:
"""Given a model class, generate an object of that class with sample values."""
# FIXME: Should be in separate generator module as it inherits EVERYTHING
model_type = model
if utils.is_collection_field_type(model):
model_type = model.__origin__
model = utils.get_inner_type(model)
else:
model = model
model_dict = {}
for field in model.__fields__:
outer_type = model.__fields__[field].outer_type_
# Check for unions. This is awkward due to allow support for python 3.7
# It also does not inspect for which union we want. Should be removable with oscal 1.0.0
if getattr(outer_type, '__origin__', None) == Union:
outer_type = outer_type.__args__[0]
if model.__fields__[field].required:
""" FIXME: This type_ could be a List or a Dict """
if utils.is_collection_field_type(outer_type) or issubclass(
outer_type, BaseModel):
model_dict[field] = generate_sample_model(outer_type)
else:
model_dict[field] = generate_sample_value_by_type(
outer_type, field, model)
if model_type is list:
return [model(**model_dict)]
elif model_type is dict:
return {'REPLACE_ME': model(**model_dict)}
return model(**model_dict)
def generate_sample_model(model: Union[Type[TG], List[TG], Dict[str, TG]]) -> TG:
"""Given a model class, generate an object of that class with sample values."""
# FIXME: Typing is wrong.
# TODO: The typing here is very generic - which may cause some pain. It may be more appropriate to create a wrapper
# Function for the to level execution. This would imply restructuring some other parts of the code.
model_type = model
# This block normalizes model type down to
if utils.is_collection_field_type(model): # type: ignore
model_type = utils.get_origin(model) # type: ignore
model = utils.get_inner_type(model) # type: ignore
model = cast(TG, model) # type: ignore
model_dict = {}
# this block is needed to avoid situations where an inbuilt is inside a list / dict.
if issubclass(model, OscalBaseModel):
for field in model.__fields__:
outer_type = model.__fields__[field].outer_type_
# Check for unions. This is awkward due to allow support for python 3.7
# It also does not inspect for which union we want. Should be removable with oscal 1.0.0
if utils.get_origin(outer_type) == Union:
outer_type = outer_type.__args__[0]
if model.__fields__[field].required:
""" FIXME: This type_ could be a List or a Dict """
if utils.is_collection_field_type(outer_type) or issubclass(outer_type, OscalBaseModel):
model_dict[field] = generate_sample_model(outer_type)
else:
model_dict[field] = generate_sample_value_by_type(outer_type, field)
# Note: this assumes list constrains in oscal are always 1 as a minimum size. if two this may still fail.
else:
# There is set of circumstances where a m
if model_type is list:
return [generate_sample_value_by_type(model, '')]
elif model_type is dict:
return {'REPLACE_ME': generate_sample_value_by_type(model, '')}
err.TrestleError('Unhandled collection type.')
if model_type is list:
return [model(**model_dict)]
elif model_type is dict:
return {'REPLACE_ME': model(**model_dict)}
return model(**model_dict)
def get_stripped_contextual_model(
path: pathlib.Path = None,
aliases_not_to_be_stripped: List[str] = None
) -> Tuple[Type[OscalBaseModel], str]:
"""
Get the stripped contextual model class and alias based on the contextual path.
This function relies on the directory structure of the trestle model being edited to determine, based on the
existing files and folder, which fields should be stripped from the model type represented by the path passed in as
a parameter.
"""
if path is None:
path = pathlib.Path.cwd()
if aliases_not_to_be_stripped is None:
aliases_not_to_be_stripped = []
singular_model_type, model_alias = get_contextual_model_type(path)
# Stripped models do not apply to collection types such as List[] and Dict{}
# if model type is a list or dict, generate a new wrapping model for it
if utils.is_collection_field_type(singular_model_type):
malias = model_alias.split('.')[-1]
class_name = utils.alias_to_classname(malias, 'json')
model_type = create_model(class_name,
__base__=OscalBaseModel,
__root__=(singular_model_type, ...))
model_type = cast(Type[OscalBaseModel], model_type)
return model_type, model_alias
malias = model_alias.split('.')[-1]
if path.is_dir() and malias != extract_alias(path):
split_subdir = path / malias
else:
split_subdir = path.parent / path.with_suffix('').name
aliases_to_be_stripped = set()
if split_subdir.exists():
for f in split_subdir.iterdir():
alias = extract_alias(f)
if alias not in aliases_not_to_be_stripped:
aliases_to_be_stripped.add(alias)
if len(aliases_to_be_stripped) > 0:
model_type = singular_model_type.create_stripped_model_type(
stripped_fields_aliases=list(aliases_to_be_stripped))
return model_type, model_alias
else:
return singular_model_type, model_alias
def __init__(self, elem: OscalBaseModel, wrapper_alias: str = ''):
"""Initialize an element wrapper.
wrapper_alias is the OSCAL alias for the given elem object and used for seriazation in to_json() method.
For example,
- List[Catalog.Group] element should have wrapper alias 'groups'
- Catalog element should have wrapper alias 'catalog'
wrapper_alias is mandatory for collection type object
if wrapper_alias = IGNORE_WRAPPER_ALIAS, then it is ignored and assumed to be json-serializable during to_json()
"""
self._elem: OscalBaseModel = elem
if wrapper_alias == '' and wrapper_alias != self.IGNORE_WRAPPER_ALIAS:
if utils.is_collection_field_type(elem):
raise TrestleError('wrapper_alias is required for a collection type object')
else:
wrapper_alias = utils.classname_to_alias(elem.__class__.__name__, 'json')
self._wrapper_alias: str = wrapper_alias
def get_contextual_model_type(
path: pathlib.Path = None) -> Tuple[Type[OscalBaseModel], str]:
"""Get the full contextual model class and full jsonpath for the alias based on the contextual path."""
if path is None:
path = pathlib.Path.cwd()
if not is_valid_project_model_path(path):
raise err.TrestleError(f'Trestle project not found at {path}')
root_path = get_trestle_project_root(path)
project_model_path = get_project_model_path(path)
if root_path is None or project_model_path is None:
raise err.TrestleError('Trestle project not found')
relative_path = path.relative_to(str(root_path))
project_type = relative_path.parts[0] # catalogs, profiles, etc
module_name = const.MODELTYPE_TO_MODELMODULE[project_type]
model_relative_path = pathlib.Path(*relative_path.parts[2:])
model_type, model_alias = utils.get_root_model(module_name)
full_alias = model_alias
for i in range(len(model_relative_path.parts)):
tmp_path = root_path.joinpath(*relative_path.parts[:2],
*model_relative_path.parts[:i + 1])
alias = extract_alias(tmp_path)
if i > 0 or model_alias != alias:
model_alias = alias
full_alias = f'{full_alias}.{model_alias}'
if utils.is_collection_field_type(model_type):
model_type = utils.get_inner_type(model_type)
else:
model_type = model_type.alias_to_field_map()[alias].outer_type_
return model_type, full_alias
def test_is_collection_field_type() -> None:
"""Test for checking whether the type of a field in an OscalBaseModel object is a collection field."""
good_catalog = load_good_catalog()
assert mutils.is_collection_field_type(type('this is a string')) is False
assert mutils.is_collection_field_type(type(good_catalog)) is False # Catalog
catalog_field = catalog.Model.alias_to_field_map()['catalog']
assert mutils.is_collection_field_type(catalog_field.outer_type_) is False # Catalog
assert mutils.is_collection_field_type(type(good_catalog.metadata)) is False # Metadata
metadata_field = catalog.Catalog.alias_to_field_map()['metadata']
assert mutils.is_collection_field_type(metadata_field.outer_type_) is False # Metadata
assert mutils.is_collection_field_type(type(good_catalog.metadata.roles)) is False # list
roles_field = catalog.Metadata.alias_to_field_map()['roles']
assert mutils.is_collection_field_type(roles_field.outer_type_) is True # List[Role]
assert mutils.is_collection_field_type(roles_field.type_) is False # Role
assert mutils.is_collection_field_type(type(good_catalog.metadata.responsible_parties)) is False # list
responsible_parties_field = catalog.Metadata.alias_to_field_map()['responsible-parties']
assert mutils.is_collection_field_type(responsible_parties_field.outer_type_) is True # Dict[str, ResponsibleParty]
assert mutils.is_collection_field_type(responsible_parties_field.type_) is False # ResponsibleParty
assert mutils.is_collection_field_type(
type(good_catalog.metadata.parties[0].addresses[0].addr_lines)
) is False # list
postal_address_field = catalog.Address.alias_to_field_map()['addr-lines']
assert mutils.is_collection_field_type(postal_address_field.outer_type_) is True # List[AddrLine]
assert mutils.is_collection_field_type(postal_address_field.type_) is False # AddrLine
def load_distributed(
file_path: Path,
collection_type: Optional[Type[Any]] = None
) -> Tuple[Type[OscalBaseModel], str, Union[
OscalBaseModel, List[OscalBaseModel], Dict[str, OscalBaseModel]]]:
"""
Given path to a model, load the model.
If the model is decomposed/split/distributed,the decomposed models are loaded recursively.
Args:
file_path (pathlib.Path): The path to the file/directory to be loaded.
collection_type (Type[Any], optional): The type of collection model, if it is a collection model.
typing.List if the model is a list, typing.Dict if the model is additionalProperty.
Defaults to None.
Returns:
Tuple[Type[OscalBaseModel], str, Union[OscalBaseModel, List[OscalBaseModel], Dict[str, OscalBaseModel]]]: Return
a tuple of Model Type (e.g. class 'trestle.oscal.catalog.Catalog'), Model Alias (e.g. 'catalog.metadata'),
and Instance of the Model. If the model is decomposed/split/distributed, the instance of the model contains
the decomposed models loaded recursively.
"""
# If the path contains a list type model
if collection_type is list:
return _load_list(file_path)
# If the path contains a dict type model
if collection_type is dict:
return _load_dict(file_path)
# Get current model
primary_model_type, primary_model_alias = fs.get_stripped_contextual_model(
file_path.absolute())
primary_model_instance = primary_model_type.oscal_read(file_path)
primary_model_dict = primary_model_instance.__dict__
# Is model decomposed?
file_dir = file_path.parent
decomposed_dir = file_dir / file_path.parts[-1].split('.')[0]
if decomposed_dir.exists():
aliases_not_to_be_stripped = []
instances_to_be_merged: List[OscalBaseModel] = []
for path in sorted(Path.iterdir(decomposed_dir)):
if path.is_file():
model_type, model_alias, model_instance = load_distributed(
path)
aliases_not_to_be_stripped.append(model_alias.split('.')[-1])
instances_to_be_merged.append(model_instance)
elif path.is_dir():
model_type, model_alias = fs.get_stripped_contextual_model(
path.absolute())
# Only load the directory if it is a collection model. Otherwise do nothing - it gets loaded when
# iterating over the model file
if '__root__' in model_type.__fields__.keys(
) and utils.is_collection_field_type(
model_type.__fields__['__root__'].outer_type_):
# This directory is a decomposed List or Dict
collection_type = utils.get_origin(
model_type.__fields__['__root__'].outer_type_)
model_type, model_alias, model_instance = load_distributed(
path, collection_type)
aliases_not_to_be_stripped.append(
model_alias.split('.')[-1])
instances_to_be_merged.append(model_instance)
for i in range(len(aliases_not_to_be_stripped)):
alias = aliases_not_to_be_stripped[i]
instance = instances_to_be_merged[i]
if hasattr(instance, '__dict__'
) and '__root__' in instance.__dict__ and isinstance(
instance, OscalBaseModel):
instance = instance.__dict__['__root__']
primary_model_dict[alias] = instance
merged_model_type, merged_model_alias = fs.get_stripped_contextual_model(
file_path.absolute(), aliases_not_to_be_stripped)
merged_model_instance = merged_model_type(
**primary_model_dict) # type: ignore
return merged_model_type, merged_model_alias, merged_model_instance
else:
return primary_model_type, primary_model_alias, primary_model_instance
def _list_options_for_merge(
self,
cwd: Path,
current_alias: str,
current_model: Type[OscalBaseModel],
current_filename: str,
initial_path: Path = None,
visited_elements: Set[str] = None
):
"""List paths that can be used in the -e option for the merge operation."""
if initial_path is None:
initial_path = cwd
if visited_elements is None:
visited_elements = set()
path_sep = '.' if current_alias else ''
# List options for merge
if not utils.is_collection_field_type(current_model):
malias = current_alias.split('.')[-1]
if cwd.is_dir() and malias != fs.extract_alias(cwd):
split_subdir = cwd / malias
else:
split_subdir = cwd.parent / cwd.with_suffix('').name
# Go through each file or subdirectory in the cwd
fields_by_alias = current_model.alias_to_field_map()
for filepath in Path.iterdir(split_subdir):
if filepath.is_file() and cwd == initial_path:
continue
alias = filepath.with_suffix('').name
if alias in fields_by_alias:
visited_element = f'{current_alias}{path_sep}{alias}'
if visited_element not in visited_elements:
visited_elements.add(visited_element)
self.out(f"{visited_element} (merges \'{filepath.name}\' into \'{cwd / current_filename}\')")
# If it is subdirectory, call this function recursively
if Path.is_dir(filepath):
self._list_options_for_merge(
filepath,
f'{current_alias}{path_sep}{alias}',
fields_by_alias[alias].outer_type_,
f'{alias}.json',
initial_path=initial_path,
visited_elements=visited_elements
)
else:
# List merge option for collection at the base level
destination_dir = cwd.parent if len(initial_path.parts) < len(cwd.parts) else cwd
destination = destination_dir / current_filename
visited_element = f'{current_alias}{path_sep}{const.ELEMENT_WILDCARD}'
if visited_element not in visited_elements:
visited_elements.add(visited_element)
self.out(f"{visited_element} (merges all files/subdirectories under {cwd} into \'{destination}\')")
# Go through each subdirectory in the collection and look for nested merge options
singular_alias = fs.get_singular_alias(current_alias, False)
singular_model = utils.get_inner_type(current_model)
for filename in sorted(cwd.glob(f'*{const.IDX_SEP}{singular_alias}')):
if Path.is_dir(filename):
self._list_options_for_merge(
filename, # f'{current_alias}{path_sep}*',
f'{current_alias}{path_sep}{singular_alias}',
singular_model,
f'{singular_alias}.json',
initial_path=initial_path,
visited_elements=visited_elements
)
