def test_get_inner_type() -> None:
"""Test retrievel of inner type of a model field representing a collection."""
good_catalog = load_good_catalog()
with pytest.raises(err.TrestleError):
# Type of catalog is not a collection field type
mutils.get_inner_type(type(good_catalog))
with pytest.raises(err.TrestleError):
# Type of field catalog is not a collection field type
catalog_field = catalog.Model.alias_to_field_map()['catalog']
mutils.get_inner_type(catalog_field.outer_type_)
with pytest.raises(err.TrestleError):
# Type of roles object is not a collection field type
mutils.get_inner_type(type(good_catalog.metadata.roles))
# Type of field roles is a collection field type
roles_field = catalog.Metadata.alias_to_field_map()['roles']
role_type = mutils.get_inner_type(roles_field.outer_type_)
assert role_type == catalog.Role
with pytest.raises(err.TrestleError):
# Type of responsible_parties object is not a collection field type
mutils.get_inner_type(type(good_catalog.metadata.responsible_parties))
# Type of field responsible-parties is a collection field type
responsible_parties_field = catalog.Metadata.alias_to_field_map()['responsible-parties']
responsible_party_type = mutils.get_inner_type(responsible_parties_field.outer_type_)
assert responsible_party_type == catalog.ResponsibleParty
def get_singular_alias(alias_path: str, contextual_mode: bool = False) -> str:
"""
Get the alias in the singular form from a jsonpath.
If contextual_mode is True and contextual_path is None, it assumes alias_path is relative to the directory the user
is running trestle from.
"""
if len(alias_path.strip()) == 0:
raise err.TrestleError('Invalid jsonpath.')
singular_alias: str = ''
full_alias_path = alias_path
if contextual_mode:
_, full_model_alias = get_contextual_model_type()
first_alias_a = full_model_alias.split('.')[-1]
first_alias_b = alias_path.split('.')[0]
if first_alias_a == first_alias_b:
full_model_alias = '.'.join(full_model_alias.split('.')[:-1])
full_alias_path = '.'.join([full_model_alias, alias_path]).strip('.')
path_parts = full_alias_path.split(const.ALIAS_PATH_SEPARATOR)
if len(path_parts) < 2:
raise err.TrestleError('Invalid jsonpath.')
model_types = []
root_model_alias = path_parts[0]
found = False
for module_name in const.MODELTYPE_TO_MODELMODULE.values():
model_type, model_alias = utils.get_root_model(module_name)
if root_model_alias == model_alias:
found = True
model_types.append(model_type)
break
if not found:
raise err.TrestleError(
f'{root_model_alias} is an invalid root model alias.')
model_type = model_types[0]
for i in range(1, len(path_parts)):
if utils.is_collection_field_type(model_type):
model_type = utils.get_inner_type(model_type)
i = i + 1
else:
model_type = model_type.alias_to_field_map()[
path_parts[i]].outer_type_
model_types.append(model_type)
if not utils.is_collection_field_type(model_type):
raise err.TrestleError('Not a valid generic collection model.')
last_alias = path_parts[-1]
parent_model_type = model_types[-2]
singular_alias = utils.classname_to_alias(
utils.get_inner_type(parent_model_type.alias_to_field_map()
[last_alias].outer_type_).__name__, 'json')
return singular_alias
def test_get_target_model() -> None:
"""Test utils method get_target_model."""
assert mutils.is_collection_field_type(
mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)
) is True
assert (mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)).__origin__ is list
assert mutils.get_inner_type(
mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)
) is catalog.Role
assert mutils.is_collection_field_type(
mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog)
) is True
assert mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog).__origin__ is dict
assert mutils.get_inner_type(
mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog)
) is catalog.ResponsibleParty
assert mutils.is_collection_field_type(
mutils.get_target_model(['catalog', 'metadata', 'responsible-parties', 'creator'], catalog.Catalog)
) is False
assert mutils.get_target_model(
['catalog', 'metadata', 'responsible-parties', 'creator'], catalog.Catalog
) is catalog.ResponsibleParty
assert mutils.get_target_model(['catalog', 'metadata'], catalog.Catalog) is catalog.Metadata
with pytest.raises(err.TrestleError):
mutils.get_target_model(['catalog', 'metadata', 'bad_element'], catalog.Catalog)
def generate_sample_model(model: Type[Any]) -> OscalBaseModel:
"""Given a model class, generate an object of that class with sample values."""
# FIXME: Should be in separate generator module as it inherits EVERYTHING
model_type = model
if utils.is_collection_field_type(model):
model_type = model.__origin__
model = utils.get_inner_type(model)
else:
model = model
model_dict = {}
for field in model.__fields__:
outer_type = model.__fields__[field].outer_type_
# Check for unions. This is awkward due to allow support for python 3.7
# It also does not inspect for which union we want. Should be removable with oscal 1.0.0
if getattr(outer_type, '__origin__', None) == Union:
outer_type = outer_type.__args__[0]
if model.__fields__[field].required:
""" FIXME: This type_ could be a List or a Dict """
if utils.is_collection_field_type(outer_type) or issubclass(
outer_type, BaseModel):
model_dict[field] = generate_sample_model(outer_type)
else:
model_dict[field] = generate_sample_value_by_type(
outer_type, field, model)
if model_type is list:
return [model(**model_dict)]
elif model_type is dict:
return {'REPLACE_ME': model(**model_dict)}
return model(**model_dict)
def generate_sample_model(model: Union[Type[TG], List[TG], Dict[str, TG]]) -> TG:
"""Given a model class, generate an object of that class with sample values."""
# FIXME: Typing is wrong.
# TODO: The typing here is very generic - which may cause some pain. It may be more appropriate to create a wrapper
# Function for the to level execution. This would imply restructuring some other parts of the code.
model_type = model
# This block normalizes model type down to
if utils.is_collection_field_type(model): # type: ignore
model_type = utils.get_origin(model) # type: ignore
model = utils.get_inner_type(model) # type: ignore
model = cast(TG, model) # type: ignore
model_dict = {}
# this block is needed to avoid situations where an inbuilt is inside a list / dict.
if issubclass(model, OscalBaseModel):
for field in model.__fields__:
outer_type = model.__fields__[field].outer_type_
# Check for unions. This is awkward due to allow support for python 3.7
# It also does not inspect for which union we want. Should be removable with oscal 1.0.0
if utils.get_origin(outer_type) == Union:
outer_type = outer_type.__args__[0]
if model.__fields__[field].required:
""" FIXME: This type_ could be a List or a Dict """
if utils.is_collection_field_type(outer_type) or issubclass(outer_type, OscalBaseModel):
model_dict[field] = generate_sample_model(outer_type)
else:
model_dict[field] = generate_sample_value_by_type(outer_type, field)
# Note: this assumes list constrains in oscal are always 1 as a minimum size. if two this may still fail.
else:
# There is set of circumstances where a m
if model_type is list:
return [generate_sample_value_by_type(model, '')]
elif model_type is dict:
return {'REPLACE_ME': generate_sample_value_by_type(model, '')}
err.TrestleError('Unhandled collection type.')
if model_type is list:
return [model(**model_dict)]
elif model_type is dict:
return {'REPLACE_ME': model(**model_dict)}
return model(**model_dict)
def get_contextual_model_type(
path: pathlib.Path = None) -> Tuple[Type[OscalBaseModel], str]:
"""Get the full contextual model class and full jsonpath for the alias based on the contextual path."""
if path is None:
path = pathlib.Path.cwd()
if not is_valid_project_model_path(path):
raise err.TrestleError(f'Trestle project not found at {path}')
root_path = get_trestle_project_root(path)
project_model_path = get_project_model_path(path)
if root_path is None or project_model_path is None:
raise err.TrestleError('Trestle project not found')
relative_path = path.relative_to(str(root_path))
project_type = relative_path.parts[0] # catalogs, profiles, etc
module_name = const.MODELTYPE_TO_MODELMODULE[project_type]
model_relative_path = pathlib.Path(*relative_path.parts[2:])
model_type, model_alias = utils.get_root_model(module_name)
full_alias = model_alias
for i in range(len(model_relative_path.parts)):
tmp_path = root_path.joinpath(*relative_path.parts[:2],
*model_relative_path.parts[:i + 1])
alias = extract_alias(tmp_path)
if i > 0 or model_alias != alias:
model_alias = alias
full_alias = f'{full_alias}.{model_alias}'
if utils.is_collection_field_type(model_type):
model_type = utils.get_inner_type(model_type)
else:
model_type = model_type.alias_to_field_map()[alias].outer_type_
return model_type, full_alias
def test_get_contextual_model_type(tmp_path: pathlib.Path) -> None:
"""Test get model type and alias based on filesystem context."""
import trestle.core.utils as cutils
with pytest.raises(TrestleError):
fs.get_contextual_model_type(tmp_path / 'invalidpath')
with pytest.raises(TrestleError):
fs.get_contextual_model_type(tmp_path)
create_sample_catalog_project(tmp_path)
catalogs_dir = tmp_path / 'catalogs'
mycatalog_dir = catalogs_dir / 'mycatalog'
catalog_dir = mycatalog_dir / 'catalog'
metadata_dir = catalog_dir / 'metadata'
roles_dir = metadata_dir / 'roles'
rps_dir = metadata_dir / 'responsible-parties'
props_dir = metadata_dir / 'props'
groups_dir = mycatalog_dir / 'groups'
group_dir = groups_dir / f'00000{IDX_SEP}group'
controls_dir = group_dir / 'controls'
with pytest.raises(TrestleError):
assert fs.get_contextual_model_type(catalogs_dir) is None
assert fs.get_contextual_model_type(mycatalog_dir) == (catalog.Catalog,
'catalog')
assert fs.get_contextual_model_type(
mycatalog_dir / 'catalog.json') == (catalog.Catalog, 'catalog')
assert fs.get_contextual_model_type(
catalog_dir / 'back-matter.json') == (catalog.BackMatter,
'catalog.back-matter')
assert fs.get_contextual_model_type(
catalog_dir / 'metadata.yaml') == (catalog.Metadata,
'catalog.metadata')
assert fs.get_contextual_model_type(metadata_dir) == (catalog.Metadata,
'catalog.metadata')
# The line below is no longer possible to execute in many situations due to the constrained lists
# assert fs.get_contextual_model_type(roles_dir) == (List[catalog.Role], 'catalog.metadata.roles') # noqa: E800
(type_, element) = fs.get_contextual_model_type(roles_dir)
assert cutils.get_origin(type_) == list
assert element == 'catalog.metadata.roles'
assert fs.get_contextual_model_type(
roles_dir / '00000__role.json') == (catalog.Role,
'catalog.metadata.roles.role')
assert fs.get_contextual_model_type(rps_dir) == (
Dict[str,
catalog.ResponsibleParty], 'catalog.metadata.responsible-parties')
assert fs.get_contextual_model_type(
rps_dir / 'creator__responsible-party.json') == (
catalog.ResponsibleParty,
'catalog.metadata.responsible-parties.responsible-party')
(type_, element) = fs.get_contextual_model_type(props_dir)
assert cutils.get_origin(type_) == list
assert cutils.get_inner_type(type_) == catalog.Property
assert element == 'catalog.metadata.props'
(expected_type, expected_json_path) = fs.get_contextual_model_type(
props_dir / f'00000{IDX_SEP}property.json')
assert expected_type == catalog.Property
assert expected_json_path == 'catalog.metadata.props.property'
assert cutils.get_origin(type_) == list
assert fs.get_contextual_model_type(
groups_dir / f'00000{IDX_SEP}group.json') == (catalog.Group,
'catalog.groups.group')
assert fs.get_contextual_model_type(group_dir) == (catalog.Group,
'catalog.groups.group')
assert fs.get_contextual_model_type(
controls_dir / f'00000{IDX_SEP}control.json') == (
catalog.Control, 'catalog.groups.group.controls.control')
def _list_options_for_merge(
self,
cwd: Path,
current_alias: str,
current_model: Type[OscalBaseModel],
current_filename: str,
initial_path: Path = None,
visited_elements: Set[str] = None
):
"""List paths that can be used in the -e option for the merge operation."""
if initial_path is None:
initial_path = cwd
if visited_elements is None:
visited_elements = set()
path_sep = '.' if current_alias else ''
# List options for merge
if not utils.is_collection_field_type(current_model):
malias = current_alias.split('.')[-1]
if cwd.is_dir() and malias != fs.extract_alias(cwd):
split_subdir = cwd / malias
else:
split_subdir = cwd.parent / cwd.with_suffix('').name
# Go through each file or subdirectory in the cwd
fields_by_alias = current_model.alias_to_field_map()
for filepath in Path.iterdir(split_subdir):
if filepath.is_file() and cwd == initial_path:
continue
alias = filepath.with_suffix('').name
if alias in fields_by_alias:
visited_element = f'{current_alias}{path_sep}{alias}'
if visited_element not in visited_elements:
visited_elements.add(visited_element)
self.out(f"{visited_element} (merges \'{filepath.name}\' into \'{cwd / current_filename}\')")
# If it is subdirectory, call this function recursively
if Path.is_dir(filepath):
self._list_options_for_merge(
filepath,
f'{current_alias}{path_sep}{alias}',
fields_by_alias[alias].outer_type_,
f'{alias}.json',
initial_path=initial_path,
visited_elements=visited_elements
)
else:
# List merge option for collection at the base level
destination_dir = cwd.parent if len(initial_path.parts) < len(cwd.parts) else cwd
destination = destination_dir / current_filename
visited_element = f'{current_alias}{path_sep}{const.ELEMENT_WILDCARD}'
if visited_element not in visited_elements:
visited_elements.add(visited_element)
self.out(f"{visited_element} (merges all files/subdirectories under {cwd} into \'{destination}\')")
# Go through each subdirectory in the collection and look for nested merge options
singular_alias = fs.get_singular_alias(current_alias, False)
singular_model = utils.get_inner_type(current_model)
for filename in sorted(cwd.glob(f'*{const.IDX_SEP}{singular_alias}')):
if Path.is_dir(filename):
self._list_options_for_merge(
filename, # f'{current_alias}{path_sep}*',
f'{current_alias}{path_sep}{singular_alias}',
singular_model,
f'{singular_alias}.json',
initial_path=initial_path,
visited_elements=visited_elements
)
