def get_singular_alias(alias_path: str, contextual_mode: bool = False) -> str: """ Get the alias in the singular form from a jsonpath. If contextual_mode is True and contextual_path is None, it assumes alias_path is relative to the directory the user is running trestle from. """ if len(alias_path.strip()) == 0: raise err.TrestleError('Invalid jsonpath.') singular_alias: str = '' full_alias_path = alias_path if contextual_mode: _, full_model_alias = get_contextual_model_type() first_alias_a = full_model_alias.split('.')[-1] first_alias_b = alias_path.split('.')[0] if first_alias_a == first_alias_b: full_model_alias = '.'.join(full_model_alias.split('.')[:-1]) full_alias_path = '.'.join([full_model_alias, alias_path]).strip('.') path_parts = full_alias_path.split(const.ALIAS_PATH_SEPARATOR) if len(path_parts) < 2: raise err.TrestleError('Invalid jsonpath.') model_types = [] root_model_alias = path_parts[0] found = False for module_name in const.MODELTYPE_TO_MODELMODULE.values(): model_type, model_alias = utils.get_root_model(module_name) if root_model_alias == model_alias: found = True model_types.append(model_type) break if not found: raise err.TrestleError( f'{root_model_alias} is an invalid root model alias.') model_type = model_types[0] for i in range(1, len(path_parts)): if utils.is_collection_field_type(model_type): model_type = utils.get_inner_type(model_type) i = i + 1 else: model_type = model_type.alias_to_field_map()[ path_parts[i]].outer_type_ model_types.append(model_type) if not utils.is_collection_field_type(model_type): raise err.TrestleError('Not a valid generic collection model.') last_alias = path_parts[-1] parent_model_type = model_types[-2] singular_alias = utils.classname_to_alias( utils.get_inner_type(parent_model_type.alias_to_field_map() [last_alias].outer_type_).__name__, 'json') return singular_alias
def test_get_target_model() -> None: """Test utils method get_target_model.""" assert mutils.is_collection_field_type( mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog) ) is True assert (mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog)).__origin__ is list assert mutils.get_inner_type( mutils.get_target_model(['catalog', 'metadata', 'roles'], catalog.Catalog) ) is catalog.Role assert mutils.is_collection_field_type( mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog) ) is True assert mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog).__origin__ is dict assert mutils.get_inner_type( mutils.get_target_model(['catalog', 'metadata', 'responsible-parties'], catalog.Catalog) ) is catalog.ResponsibleParty assert mutils.is_collection_field_type( mutils.get_target_model(['catalog', 'metadata', 'responsible-parties', 'creator'], catalog.Catalog) ) is False assert mutils.get_target_model( ['catalog', 'metadata', 'responsible-parties', 'creator'], catalog.Catalog ) is catalog.ResponsibleParty assert mutils.get_target_model(['catalog', 'metadata'], catalog.Catalog) is catalog.Metadata with pytest.raises(err.TrestleError): mutils.get_target_model(['catalog', 'metadata', 'bad_element'], catalog.Catalog)
def generate_sample_model(model: Type[Any]) -> OscalBaseModel: """Given a model class, generate an object of that class with sample values.""" # FIXME: Should be in separate generator module as it inherits EVERYTHING model_type = model if utils.is_collection_field_type(model): model_type = model.__origin__ model = utils.get_inner_type(model) else: model = model model_dict = {} for field in model.__fields__: outer_type = model.__fields__[field].outer_type_ # Check for unions. This is awkward due to allow support for python 3.7 # It also does not inspect for which union we want. Should be removable with oscal 1.0.0 if getattr(outer_type, '__origin__', None) == Union: outer_type = outer_type.__args__[0] if model.__fields__[field].required: """ FIXME: This type_ could be a List or a Dict """ if utils.is_collection_field_type(outer_type) or issubclass( outer_type, BaseModel): model_dict[field] = generate_sample_model(outer_type) else: model_dict[field] = generate_sample_value_by_type( outer_type, field, model) if model_type is list: return [model(**model_dict)] elif model_type is dict: return {'REPLACE_ME': model(**model_dict)} return model(**model_dict)
def generate_sample_model(model: Union[Type[TG], List[TG], Dict[str, TG]]) -> TG: """Given a model class, generate an object of that class with sample values.""" # FIXME: Typing is wrong. # TODO: The typing here is very generic - which may cause some pain. It may be more appropriate to create a wrapper # Function for the to level execution. This would imply restructuring some other parts of the code. model_type = model # This block normalizes model type down to if utils.is_collection_field_type(model): # type: ignore model_type = utils.get_origin(model) # type: ignore model = utils.get_inner_type(model) # type: ignore model = cast(TG, model) # type: ignore model_dict = {} # this block is needed to avoid situations where an inbuilt is inside a list / dict. if issubclass(model, OscalBaseModel): for field in model.__fields__: outer_type = model.__fields__[field].outer_type_ # Check for unions. This is awkward due to allow support for python 3.7 # It also does not inspect for which union we want. Should be removable with oscal 1.0.0 if utils.get_origin(outer_type) == Union: outer_type = outer_type.__args__[0] if model.__fields__[field].required: """ FIXME: This type_ could be a List or a Dict """ if utils.is_collection_field_type(outer_type) or issubclass(outer_type, OscalBaseModel): model_dict[field] = generate_sample_model(outer_type) else: model_dict[field] = generate_sample_value_by_type(outer_type, field) # Note: this assumes list constrains in oscal are always 1 as a minimum size. if two this may still fail. else: # There is set of circumstances where a m if model_type is list: return [generate_sample_value_by_type(model, '')] elif model_type is dict: return {'REPLACE_ME': generate_sample_value_by_type(model, '')} err.TrestleError('Unhandled collection type.') if model_type is list: return [model(**model_dict)] elif model_type is dict: return {'REPLACE_ME': model(**model_dict)} return model(**model_dict)
def get_stripped_contextual_model( path: pathlib.Path = None, aliases_not_to_be_stripped: List[str] = None ) -> Tuple[Type[OscalBaseModel], str]: """ Get the stripped contextual model class and alias based on the contextual path. This function relies on the directory structure of the trestle model being edited to determine, based on the existing files and folder, which fields should be stripped from the model type represented by the path passed in as a parameter. """ if path is None: path = pathlib.Path.cwd() if aliases_not_to_be_stripped is None: aliases_not_to_be_stripped = [] singular_model_type, model_alias = get_contextual_model_type(path) # Stripped models do not apply to collection types such as List[] and Dict{} # if model type is a list or dict, generate a new wrapping model for it if utils.is_collection_field_type(singular_model_type): malias = model_alias.split('.')[-1] class_name = utils.alias_to_classname(malias, 'json') model_type = create_model(class_name, __base__=OscalBaseModel, __root__=(singular_model_type, ...)) model_type = cast(Type[OscalBaseModel], model_type) return model_type, model_alias malias = model_alias.split('.')[-1] if path.is_dir() and malias != extract_alias(path): split_subdir = path / malias else: split_subdir = path.parent / path.with_suffix('').name aliases_to_be_stripped = set() if split_subdir.exists(): for f in split_subdir.iterdir(): alias = extract_alias(f) if alias not in aliases_not_to_be_stripped: aliases_to_be_stripped.add(alias) if len(aliases_to_be_stripped) > 0: model_type = singular_model_type.create_stripped_model_type( stripped_fields_aliases=list(aliases_to_be_stripped)) return model_type, model_alias else: return singular_model_type, model_alias
def __init__(self, elem: OscalBaseModel, wrapper_alias: str = ''): """Initialize an element wrapper. wrapper_alias is the OSCAL alias for the given elem object and used for seriazation in to_json() method. For example, - List[Catalog.Group] element should have wrapper alias 'groups' - Catalog element should have wrapper alias 'catalog' wrapper_alias is mandatory for collection type object if wrapper_alias = IGNORE_WRAPPER_ALIAS, then it is ignored and assumed to be json-serializable during to_json() """ self._elem: OscalBaseModel = elem if wrapper_alias == '' and wrapper_alias != self.IGNORE_WRAPPER_ALIAS: if utils.is_collection_field_type(elem): raise TrestleError('wrapper_alias is required for a collection type object') else: wrapper_alias = utils.classname_to_alias(elem.__class__.__name__, 'json') self._wrapper_alias: str = wrapper_alias
def get_contextual_model_type( path: pathlib.Path = None) -> Tuple[Type[OscalBaseModel], str]: """Get the full contextual model class and full jsonpath for the alias based on the contextual path.""" if path is None: path = pathlib.Path.cwd() if not is_valid_project_model_path(path): raise err.TrestleError(f'Trestle project not found at {path}') root_path = get_trestle_project_root(path) project_model_path = get_project_model_path(path) if root_path is None or project_model_path is None: raise err.TrestleError('Trestle project not found') relative_path = path.relative_to(str(root_path)) project_type = relative_path.parts[0] # catalogs, profiles, etc module_name = const.MODELTYPE_TO_MODELMODULE[project_type] model_relative_path = pathlib.Path(*relative_path.parts[2:]) model_type, model_alias = utils.get_root_model(module_name) full_alias = model_alias for i in range(len(model_relative_path.parts)): tmp_path = root_path.joinpath(*relative_path.parts[:2], *model_relative_path.parts[:i + 1]) alias = extract_alias(tmp_path) if i > 0 or model_alias != alias: model_alias = alias full_alias = f'{full_alias}.{model_alias}' if utils.is_collection_field_type(model_type): model_type = utils.get_inner_type(model_type) else: model_type = model_type.alias_to_field_map()[alias].outer_type_ return model_type, full_alias
def test_is_collection_field_type() -> None: """Test for checking whether the type of a field in an OscalBaseModel object is a collection field.""" good_catalog = load_good_catalog() assert mutils.is_collection_field_type(type('this is a string')) is False assert mutils.is_collection_field_type(type(good_catalog)) is False # Catalog catalog_field = catalog.Model.alias_to_field_map()['catalog'] assert mutils.is_collection_field_type(catalog_field.outer_type_) is False # Catalog assert mutils.is_collection_field_type(type(good_catalog.metadata)) is False # Metadata metadata_field = catalog.Catalog.alias_to_field_map()['metadata'] assert mutils.is_collection_field_type(metadata_field.outer_type_) is False # Metadata assert mutils.is_collection_field_type(type(good_catalog.metadata.roles)) is False # list roles_field = catalog.Metadata.alias_to_field_map()['roles'] assert mutils.is_collection_field_type(roles_field.outer_type_) is True # List[Role] assert mutils.is_collection_field_type(roles_field.type_) is False # Role assert mutils.is_collection_field_type(type(good_catalog.metadata.responsible_parties)) is False # list responsible_parties_field = catalog.Metadata.alias_to_field_map()['responsible-parties'] assert mutils.is_collection_field_type(responsible_parties_field.outer_type_) is True # Dict[str, ResponsibleParty] assert mutils.is_collection_field_type(responsible_parties_field.type_) is False # ResponsibleParty assert mutils.is_collection_field_type( type(good_catalog.metadata.parties[0].addresses[0].addr_lines) ) is False # list postal_address_field = catalog.Address.alias_to_field_map()['addr-lines'] assert mutils.is_collection_field_type(postal_address_field.outer_type_) is True # List[AddrLine] assert mutils.is_collection_field_type(postal_address_field.type_) is False # AddrLine
def load_distributed( file_path: Path, collection_type: Optional[Type[Any]] = None ) -> Tuple[Type[OscalBaseModel], str, Union[ OscalBaseModel, List[OscalBaseModel], Dict[str, OscalBaseModel]]]: """ Given path to a model, load the model. If the model is decomposed/split/distributed,the decomposed models are loaded recursively. Args: file_path (pathlib.Path): The path to the file/directory to be loaded. collection_type (Type[Any], optional): The type of collection model, if it is a collection model. typing.List if the model is a list, typing.Dict if the model is additionalProperty. Defaults to None. Returns: Tuple[Type[OscalBaseModel], str, Union[OscalBaseModel, List[OscalBaseModel], Dict[str, OscalBaseModel]]]: Return a tuple of Model Type (e.g. class 'trestle.oscal.catalog.Catalog'), Model Alias (e.g. 'catalog.metadata'), and Instance of the Model. If the model is decomposed/split/distributed, the instance of the model contains the decomposed models loaded recursively. """ # If the path contains a list type model if collection_type is list: return _load_list(file_path) # If the path contains a dict type model if collection_type is dict: return _load_dict(file_path) # Get current model primary_model_type, primary_model_alias = fs.get_stripped_contextual_model( file_path.absolute()) primary_model_instance = primary_model_type.oscal_read(file_path) primary_model_dict = primary_model_instance.__dict__ # Is model decomposed? file_dir = file_path.parent decomposed_dir = file_dir / file_path.parts[-1].split('.')[0] if decomposed_dir.exists(): aliases_not_to_be_stripped = [] instances_to_be_merged: List[OscalBaseModel] = [] for path in sorted(Path.iterdir(decomposed_dir)): if path.is_file(): model_type, model_alias, model_instance = load_distributed( path) aliases_not_to_be_stripped.append(model_alias.split('.')[-1]) instances_to_be_merged.append(model_instance) elif path.is_dir(): model_type, model_alias = fs.get_stripped_contextual_model( path.absolute()) # Only load the directory if it is a collection model. Otherwise do nothing - it gets loaded when # iterating over the model file if '__root__' in model_type.__fields__.keys( ) and utils.is_collection_field_type( model_type.__fields__['__root__'].outer_type_): # This directory is a decomposed List or Dict collection_type = utils.get_origin( model_type.__fields__['__root__'].outer_type_) model_type, model_alias, model_instance = load_distributed( path, collection_type) aliases_not_to_be_stripped.append( model_alias.split('.')[-1]) instances_to_be_merged.append(model_instance) for i in range(len(aliases_not_to_be_stripped)): alias = aliases_not_to_be_stripped[i] instance = instances_to_be_merged[i] if hasattr(instance, '__dict__' ) and '__root__' in instance.__dict__ and isinstance( instance, OscalBaseModel): instance = instance.__dict__['__root__'] primary_model_dict[alias] = instance merged_model_type, merged_model_alias = fs.get_stripped_contextual_model( file_path.absolute(), aliases_not_to_be_stripped) merged_model_instance = merged_model_type( **primary_model_dict) # type: ignore return merged_model_type, merged_model_alias, merged_model_instance else: return primary_model_type, primary_model_alias, primary_model_instance
def _list_options_for_merge( self, cwd: Path, current_alias: str, current_model: Type[OscalBaseModel], current_filename: str, initial_path: Path = None, visited_elements: Set[str] = None ): """List paths that can be used in the -e option for the merge operation.""" if initial_path is None: initial_path = cwd if visited_elements is None: visited_elements = set() path_sep = '.' if current_alias else '' # List options for merge if not utils.is_collection_field_type(current_model): malias = current_alias.split('.')[-1] if cwd.is_dir() and malias != fs.extract_alias(cwd): split_subdir = cwd / malias else: split_subdir = cwd.parent / cwd.with_suffix('').name # Go through each file or subdirectory in the cwd fields_by_alias = current_model.alias_to_field_map() for filepath in Path.iterdir(split_subdir): if filepath.is_file() and cwd == initial_path: continue alias = filepath.with_suffix('').name if alias in fields_by_alias: visited_element = f'{current_alias}{path_sep}{alias}' if visited_element not in visited_elements: visited_elements.add(visited_element) self.out(f"{visited_element} (merges \'{filepath.name}\' into \'{cwd / current_filename}\')") # If it is subdirectory, call this function recursively if Path.is_dir(filepath): self._list_options_for_merge( filepath, f'{current_alias}{path_sep}{alias}', fields_by_alias[alias].outer_type_, f'{alias}.json', initial_path=initial_path, visited_elements=visited_elements ) else: # List merge option for collection at the base level destination_dir = cwd.parent if len(initial_path.parts) < len(cwd.parts) else cwd destination = destination_dir / current_filename visited_element = f'{current_alias}{path_sep}{const.ELEMENT_WILDCARD}' if visited_element not in visited_elements: visited_elements.add(visited_element) self.out(f"{visited_element} (merges all files/subdirectories under {cwd} into \'{destination}\')") # Go through each subdirectory in the collection and look for nested merge options singular_alias = fs.get_singular_alias(current_alias, False) singular_model = utils.get_inner_type(current_model) for filename in sorted(cwd.glob(f'*{const.IDX_SEP}{singular_alias}')): if Path.is_dir(filename): self._list_options_for_merge( filename, # f'{current_alias}{path_sep}*', f'{current_alias}{path_sep}{singular_alias}', singular_model, f'{singular_alias}.json', initial_path=initial_path, visited_elements=visited_elements )