def require_scalar(self, *args: Type) -> None:
"""Require the node to be a scalar.
If additional arguments are passed, these are taken as a list \
of valid types; if the node matches one of these, then it is \
accepted.
Example:
# Match either an int or a string
node.require_scalar(int, str)
Arguments:
args: One or more types to match one of.
"""
node = Node(self.yaml_node)
if len(args) == 0:
if not node.is_scalar():
raise RecognitionError(('{}{}A scalar is required').format(
self.yaml_node.start_mark, os.linesep))
else:
for typ in args:
if node.is_scalar(typ):
return
raise RecognitionError(
('{}{}A scalar of type {} is required').format(
self.yaml_node.start_mark, os.linesep, args))
def __check_no_missing_attributes(self, node: yaml.Node,
mapping: CommentedMap) -> None:
"""Checks that all required attributes are present.
Also checks that they're of the correct type.
Args:
mapping: The mapping with subobjects of this object.
Raises:
RecognitionError: if an attribute is missing or the type
is incorrect.
"""
logger.debug('Checking presence of required attributes')
for name, type_, required in class_subobjects(self.class_):
if required and name not in mapping:
raise RecognitionError(('{}{}Missing attribute "{}" needed for'
' constructing a {}').format(
node.start_mark, os.linesep, name,
self.class_.__name__))
if name in mapping and not self.__type_matches(
mapping[name], type_):
raise RecognitionError(('{}{}Attribute "{}" has incorrect type'
' {}, expecting a {}').format(
node.start_mark, os.linesep, name,
type(mapping[name]), type_))
def require_attribute(self,
attribute: str,
typ: Union[None, Type] = _Any) -> None:
"""Require an attribute on the node to exist.
This implies that the node must be a mapping.
If ``typ`` is given, the attribute must have this type.
Args:
attribute: The name of the attribute / mapping key.
typ: The type the attribute must have.
"""
self.require_mapping()
attr_nodes = [
value_node for key_node, value_node in self.yaml_node.value
if key_node.value == attribute
]
if len(attr_nodes) == 0:
raise RecognitionError(
('{}{}Missing required attribute "{}"').format(
self.yaml_node.start_mark, os.linesep, attribute))
attr_node = attr_nodes[0]
if typ != _Any:
recognized_types, message = self.__recognizer.recognize(
attr_node, cast(Type, typ))
if len(recognized_types) == 0:
raise RecognitionError(message)
def __call__(
self, loader: 'Loader', node: yaml.Node
) -> Generator[Any, None, None]:
"""Construct a Path object from a yaml node.
This expects the node to contain a string with the path.
Args:
loader: The yatiml.loader that is creating this object.
node: The node to construct from.
Yields:
The incomplete constructed object.
"""
logger.debug('Constructing an object of type pathlib.Path')
if not isinstance(node, yaml.ScalarNode) or not isinstance(
node.value, str):
raise RecognitionError(
('{}{}Expected a string containing a Path.').format(
node.start_mark, os.linesep))
# ruamel.yaml expects us to yield an incomplete object, but Paths are
# special, so we'll have to make the whole thing right away.
new_obj = pathlib.Path(node.value)
yield new_obj
def __call__(self, loader: 'Loader',
node: yaml.Node) -> Generator[Any, None, None]:
"""Construct an user string value from a yaml node.
This constructs an object of the user-defined string class that
this is the constructor for. It is registered with the yaml
library, and called by it.
Note that this yields rather than returns, in a somewhat odd
way. That's directly from the PyYAML/ruamel.yaml documentation.
Args:
loader: The yatiml.loader that is creating this object.
node: The node to construct from.
Yields:
The incomplete constructed object.
"""
logger.debug('Constructing an object of type {}'.format(
self.class_.__name__))
if not isinstance(node, yaml.ScalarNode) or not isinstance(
node.value, str):
raise RecognitionError(
('{}{}Expected a string matching a {}.').format(
node.start_mark, os.linesep, self.class_.__name__))
# ruamel.yaml expects us to yield an incomplete object, but strings are
# immutable, so we'll have to make the whole thing right away.
new_obj = self.class_(node.value)
yield new_obj
def __strip_extra_attributes(self, node: yaml.Node,
known_attrs: List[str]) -> None:
"""Strips tags from extra attributes.
This prevents nodes under attributes that are not part of our
data model from being converted to objects. They'll be plain
CommentedMaps instead, which then get converted to OrderedDicts
for the user.
Args:
node: The node to process
known_attrs: The attributes to not strip
"""
known_keys = list(known_attrs)
if 'self' not in known_keys:
raise RuntimeError('The __init__ method of {} does not have a'
' "self" attribute! Please add one, this is'
' not a valid constructor.'.format(
self.class_.__name__))
known_keys.remove('self')
if '_yatiml_extra' in known_keys:
known_keys.remove('_yatiml_extra')
for key_node, value_node in node.value:
if (not isinstance(key_node, yaml.ScalarNode)
or key_node.tag != 'tag:yaml.org,2002:str'):
raise RecognitionError(
('{}{}Mapping keys that are not of type'
' string are not supported by YAtiML.').format(
node.start_mark, os.linesep))
if key_node.value not in known_keys:
strip_tags(self.__loader, value_node)
def __strip_extra_attributes(self, node: yaml.Node,
known_attrs: List[str]) -> None:
"""Strips tags from extra attributes.
This prevents nodes under attributes that are not part of our \
data model from being converted to objects. They'll be plain \
CommentedMaps instead, which then get converted to OrderedDicts \
for the user.
Args:
node: The node to process
known_attrs: The attributes to not strip
"""
known_keys = list(known_attrs)
known_keys.remove('self')
if 'yatiml_extra' in known_keys:
known_keys.remove('yatiml_extra')
for key_node, value_node in node.value:
if (not isinstance(key_node, yaml.ScalarNode)
or key_node.tag != 'tag:yaml.org,2002:str'):
raise RecognitionError(
('{}{}Mapping keys that are not of type'
' string are not supported by YAtiML.').format(
node.start_mark, os.linesep))
if key_node.value not in known_keys:
self.__strip_tags(value_node)
def require_attribute_value(
self, attribute: str, value: Union[int, str, float, bool,
None]) -> None:
"""Require an attribute on the node to have a particular value.
This requires the attribute to exist, and to have the given value \
and corresponding type. Handy for in your yatiml_recognize() \
function.
Args:
attribute: The name of the attribute / mapping key.
value: The value the attribute must have to recognize an \
object of this type.
Raises:
RecognitionError: If the attribute does not exist, or does \
not have the required value.
"""
found = False
for key_node, value_node in self.yaml_node.value:
if (key_node.tag == 'tag:yaml.org,2002:str'
and key_node.value == attribute):
found = True
node = Node(value_node)
if not node.is_scalar(type(value)):
raise RecognitionError(
('{}{}Incorrect attribute type where value {}'
' of type {} was required').format(
self.yaml_node.start_mark, os.linesep, value,
type(value)))
if node.get_value() != value:
raise RecognitionError(
('{}{}Incorrect attribute value'
' {} where {} was required').format(
self.yaml_node.start_mark, os.linesep,
value_node.value, value))
if not found:
raise RecognitionError(
('{}{}Required attribute {} not found').format(
self.yaml_node.start_mark, os.linesep, attribute))
def __type_check_attributes(self, node: yaml.Node, mapping: CommentedMap,
argspec: inspect.FullArgSpec) -> None:
"""Ensure all attributes have a matching constructor argument.
This checks that there is a constructor argument with a
matching type for each existing attribute.
If the class has a _yatiml_extra attribute, then extra
attributes are okay and no error will be raised if they exist.
Args:
node: The node we're processing
mapping: The mapping with constructed subobjects
constructor_attrs: The attributes of the constructor,
including self and _yatiml_extra, if applicable
"""
logger.debug('Checking for extraneous attributes')
logger.debug('Constructor arguments: {}, mapping: {}'.format(
argspec.args, list(mapping.keys())))
for key, value in mapping.items():
if not isinstance(key, str):
raise RecognitionError(('{}{}YAtiML only supports strings'
' for mapping keys').format(
node.start_mark, os.linesep))
if key not in argspec.args and '_yatiml_extra' not in argspec.args:
raise RecognitionError(
('{}{}Found additional attributes ("{}")'
' and {} does not support those').format(
node.start_mark, os.linesep, key,
self.class_.__name__))
if key in argspec.args and key in argspec.annotations:
if not self.__type_matches(value, argspec.annotations[key]):
raise RecognitionError(
('{}{}Expected attribute "{}" to be of type {}'
' but it is a(n) {}').format(
node.start_mark, os.linesep, key,
argspec.annotations[key], type(value)))
def recognize(self, node: yaml.Node, expected_type: Type) -> RecResult:
"""Figure out how to interpret this node.
This is not quite a type check. This function makes a list of
all types that match the expected type and also the node, and
returns that list. The goal here is not to test validity, but
to determine how to process this node further.
That said, it will recognize built-in types only in case of
an exact match.
Args:
node: The YAML node to recognize.
expected_type: The type we expect this node to be, based
on the context provided by our type definitions.
Returns:
A list of matching types.
"""
logger.debug('Recognizing {} as a {}'.format(node, expected_type))
recognized_types = None # type: Any
if expected_type in (str, int, float, bool, bool_union_fix, date, None,
type(None)):
recognized_types, message = self.__recognize_scalar(
node, expected_type)
elif expected_type in self.__additional_classes:
recognized_types, message = self.__recognize_additional(
node, expected_type)
elif is_generic_union(expected_type):
recognized_types, message = self.__recognize_union(
node, expected_type)
elif is_generic_sequence(expected_type):
recognized_types, message = self.__recognize_list(
node, expected_type)
elif is_generic_mapping(expected_type):
recognized_types, message = self.__recognize_dict(
node, expected_type)
elif expected_type in self.__registered_classes.values():
recognized_types, message = self.__recognize_user_classes(
node, expected_type)
elif expected_type in (Any, ):
recognized_types, message = [Any], ''
if recognized_types is None:
raise RecognitionError(
('Could not recognize for type {},'
' is it registered?').format(expected_type.__name__))
logger.debug('Recognized types {} matching {}'.format(
recognized_types, expected_type))
return recognized_types, message
def require_attribute_value_not(
self, attribute: str, value: Union[int, str, float, bool,
None]) -> None:
"""Require an attribute on the node to not have a given value.
This requires the attribute to exist, and to not have the given
value.
Args:
attribute: The name of the attribute / mapping key.
value: The value the attribute must not have to recognize an
object of this type.
Raises:
yatiml.RecognitionError: If the attribute does not exist,
or has the required value.
"""
found = False
for key_node, value_node in self.yaml_node.value:
if (key_node.tag == 'tag:yaml.org,2002:str'
and key_node.value == attribute):
found = True
node = Node(value_node)
if not node.is_scalar(type(value)):
return
if node.get_value() == value:
raise RecognitionError(
('{}{}Incorrect attribute value'
' {} where {} was not allowed').format(
self.yaml_node.start_mark, os.linesep,
value_node.value, value))
if not found:
raise RecognitionError(
('{}{}Required attribute "{}" not found').format(
self.yaml_node.start_mark, os.linesep, attribute))
def require_sequence(self) -> None:
"""Require the node to be a sequence."""
if not isinstance(self.yaml_node, yaml.SequenceNode):
raise RecognitionError(('{}{}A sequence is required here').format(
self.yaml_node.start_mark, os.linesep))
def require_mapping(self) -> None:
"""Require the node to be a mapping."""
if not isinstance(self.yaml_node, yaml.MappingNode):
raise RecognitionError(('{}{}A mapping is required here').format(
self.yaml_node.start_mark, os.linesep))
def __call__(self, loader: 'Loader',
node: yaml.Node) -> Generator[Any, None, None]:
"""Construct an object from a yaml node.
This constructs an object of the user-defined type that this
is the constructor for. It is registered with the yaml library,
and called by it. Recursion is handled by calling the yaml
library, so we only need to construct an object using the keys
and values of the given MappingNode, and those values have been
converted recursively for us.
Since Python does not do type checks, we do a type check
manually, to ensure that the class's constructor gets the types
it expects. This avoids confusing errors, but moreover is a
security feature that ensures that regardless of the content
of the YAML file, we produce the objects that the programmer
defined and expects.
Note that this yields rather than returns, in a somewhat odd
way. That's directly from the PyYAML/ruamel.yaml documentation.
Args:
loader: The yatiml.loader that is creating this object.
node: The node to construct from.
Yields:
The incomplete constructed object.
"""
logger.debug('Constructing an object of type {}'.format(
self.class_.__name__))
if not isinstance(node, yaml.MappingNode):
raise RecognitionError(
('{}{}Expected a MappingNode. There'
' is probably something wrong with your _yatiml_savorize()'
' function.').format(node.start_mark, os.linesep))
self.__loader = loader
# figure out which keys are extra and strip them of tags
# to prevent constructing objects we haven't type checked
argspec = inspect.getfullargspec(self.class_.__init__)
self.__strip_extra_attributes(node, argspec.args)
# create object and let yaml lib construct subobjects
new_obj = self.class_.__new__(self.class_) # type: ignore
yield new_obj
mapping = CommentedMap()
loader.construct_mapping(node, mapping, deep=True)
# Convert ruamel.yaml's round-trip types to list and OrderedDict,
# recursively for each attribute value in our mapping. Note that
# mapping itself is still a CommentedMap.
for key, value in mapping.copy().items():
if (isinstance(value, CommentedMap)
or isinstance(value, CommentedSeq)):
mapping[key] = self.__to_plain_containers(value)
# do type check
self.__check_no_missing_attributes(node, mapping)
self.__type_check_attributes(node, mapping, argspec)
# construct object, this should work now
try:
logger.debug('Calling __init__')
if '_yatiml_extra' in argspec.args:
attrs = self.__split_off_extra_attributes(
mapping, argspec.args)
new_obj.__init__(**attrs)
else:
new_obj.__init__(**mapping)
except Exception as e:
tb = traceback.format_exc()
raise RecognitionError((
'{}{}Could not construct object of class {}: {}\n{}'
).format(
node.start_mark, os.linesep, self.class_.__name__,
e, tb))
logger.debug('Done constructing {}'.format(self.class_.__name__))
def __process_node(self, node: yaml.Node,
expected_type: Type) -> yaml.Node:
"""Processes a node.
This is the main function that implements yatiml's \
functionality. It figures out how to interpret this node \
(recognition), then applies syntactic sugar, and finally \
recurses to the subnodes, if any.
Args:
node: The node to process.
expected_type: The type we expect this node to be.
Returns:
The transformed node, or a transformed copy.
"""
logger.info('Processing node {} expecting type {}'.format(
node, expected_type))
# figure out how to interpret this node
recognized_types, message = self.__recognizer.recognize(
node, expected_type)
if len(recognized_types) != 1:
raise RecognitionError(message)
recognized_type = recognized_types[0]
# remove syntactic sugar
logger.debug('Savorizing node {}'.format(node))
if recognized_type in self._registered_classes.values():
node = self.__savorize(node, recognized_type)
logger.debug('Savorized, now {}'.format(node))
# process subnodes
logger.debug('Recursing into subnodes')
if is_generic_list(recognized_type):
if node.tag != 'tag:yaml.org,2002:seq':
raise RecognitionError('{}{}Expected a {} here'.format(
node.start_mark, os.linesep, type_to_desc(expected_type)))
for item in node.value:
self.__process_node(item,
generic_type_args(recognized_type)[0])
elif is_generic_dict(recognized_type):
if node.tag != 'tag:yaml.org,2002:map':
raise RecognitionError('{}{}Expected a {} here'.format(
node.start_mark, os.linesep, type_to_desc(expected_type)))
for _, value_node in node.value:
self.__process_node(value_node,
generic_type_args(recognized_type)[1])
elif recognized_type in self._registered_classes.values():
if (not issubclass(recognized_type, enum.Enum)
and not issubclass(recognized_type, str)
and not issubclass(recognized_type, UserString)):
for attr_name, type_, _ in class_subobjects(recognized_type):
cnode = Node(node)
if cnode.has_attribute(attr_name):
subnode = cnode.get_attribute(attr_name)
new_subnode = self.__process_node(
subnode.yaml_node, type_)
cnode.set_attribute(attr_name, new_subnode)
else:
logger.debug('Not a generic class or a user-defined class, not'
' recursing')
node.tag = self.__type_to_tag(recognized_type)
logger.debug('Finished processing node {}'.format(node))
return node