def normalize(cls, normalizer, value, instance, schema): r"""Method to normalize the instance based on the property value. Args: normalizer (Normalizer): Normalizer class. value (object): Property value. instance (object): Object to normalize. schema (dict): Schema containing this property. Returns: object: Normalized object. """ if isinstance(value, (list, tuple)): v0 = value[0] for v in value: t = get_type_class(v) # if normalizer.is_type(instance, v): if t.validate(v): v0 = v break type_cls = get_type_class(v0) else: type_cls = get_type_class(value) kws = {} if (((type_cls.name in ['class', 'function']) and normalizer._working_dir_stack)): kws['working_dir'] = normalizer._working_dir_stack[-1] return type_cls.normalize(instance, **kws)
def normalize(cls, normalizer, value, instance, schema): r"""Method to normalize the instance based on the property value. Args: normalizer (Normalizer): Normalizer class. value (object): Property value. instance (object): Object to normalize. schema (dict): Schema containing this property. Returns: object: Normalized object. """ if isinstance(value, (list, tuple)): v0 = value[0] for v in value: t = get_type_class(v) # if normalizer.is_type(instance, v): if t.validate(v): v0 = v break type_cls = get_type_class(v0) else: type_cls = get_type_class(value) return type_cls.normalize(instance)
def decode(cls, metadata, data, typedef=None, typedef_validated=False, dont_check=False): r"""Decode an object. Args: metadata (dict): Meta data describing the data. data (bytes): Encoded data. typedef (dict, optional): Type properties that decoded object should be tested against. Defaults to None and object may have any values for the type properties (so long as they match the schema). typedef_validated (bool, optional): If True, the type definition is taken as already having been validated and will not be validated again during the encoding process. Defaults to False. dont_check (bool, optional): If True, the metadata will not be checked against the type definition. Defaults to False. Returns: object: Decoded object. Raises: ValueError: If the metadata does not match the type definition. ValueError: If the decoded object does not match type definition. """ conv_func = None if isinstance(metadata, dict): metatype = metadata.get('type', None) if (metatype not in [None, 'bytes']) and (typedef == { 'type': 'bytes' }): new_cls = get_type_class(metatype) return new_cls.decode(metadata, data, dont_check=dont_check) if metatype != cls.name: conv_func = conversions.get_conversion(metatype, cls.name) if (((conv_func is None) and (len(metadata.get('items', [])) == 1) and cls.check_encoded(metadata['items'][0], typedef))): conv_func = _get_single_array_element if (not conv_func) and (not dont_check): cls.check_encoded(metadata, typedef, raise_errors=True, typedef_validated=typedef_validated) if conv_func: new_cls = get_type_class(metadata['type']) out = conv_func( new_cls.decode(metadata, data, dont_check=dont_check)) else: out = cls.decode_data(data, metadata) out = cls.transform_type(out, typedef) return out
def transform_type(cls, obj, typedef=None): r"""Transform an object based on type info. Args: obj (object): Object to transform. typedef (dict): Type definition that should be used to transform the object. Returns: object: Transformed object. """ if not (isinstance(typedef, dict) and isinstance( typedef.get(cls._json_property, None), cls.python_types) and isinstance(obj, cls.python_types)): return obj map_typedef = typedef[cls._json_property] map_out = cls._container_type() for k, v in cls._iterate(obj): if cls._has_element(map_typedef, k): cls._assign( map_out, k, get_type_class(map_typedef[k]['type']).transform_type( v, typedef=map_typedef[k])) else: cls._assign(map_out, k, v) return map_out
def update_typedef(self, **kwargs): r"""Update the current typedef with new values. Args: **kwargs: All keyword arguments are considered to be new type definitions. If they are a valid definition property, they will be copied to the typedef associated with the instance. Returns: dict: A dictionary of keyword arguments that were not added to the type definition. """ map = kwargs.get(self._json_property, None) map_out = self._container_type() if isinstance(map, self.python_types): for k, v in self._iterate(map): v_typedef = complete_typedef(v) if self._has_element(self._typecls, k): self._assign(map_out, k, self._typecls[k].update_typedef(**v_typedef)) else: self._assign( self._typecls, k, get_type_class(v_typedef['type'])(**v_typedef)) self._assign(map, k, self._typecls[k]._typedef) kwargs[self._json_property] = map out = super(ContainerMetaschemaType, self).update_typedef(**kwargs) if map_out: out[self._json_property] = map_out return out
def coerce_type(cls, obj, typedef=None, **kwargs): r"""Coerce objects of specific types to match the data type. Args: obj (object): Object to be coerced. typedef (dict, optional): Type defintion that object should be coerced to. Defaults to None. **kwargs: Additional keyword arguments are metadata entries that may aid in coercing the type. Returns: object: Coerced object. Raises: RuntimeError: If obj is a dictionary, but key_order is not provided. """ if not (isinstance(typedef, dict) and isinstance( typedef.get(cls._json_property, None), cls.python_types) and isinstance(obj, cls.python_types)): return obj map_typedef = typedef[cls._json_property] map_out = cls._container_type() for k, v in cls._iterate(obj): if cls._has_element(map_typedef, k): cls._assign( map_out, k, get_type_class(map_typedef[k]['type']).coerce_type( v, typedef=map_typedef[k])) else: cls._assign(map_out, k, v) return map_out
def _normalize_schema(validator, ref, instance, schema): r"""Normalize a schema at the root to handle case where only type string specified.""" # if isinstance(instance, str): # instance = dict(type=instance) # return instance if isinstance(instance, str) and (instance in _type_registry): instance = {'type': instance} elif isinstance(instance, dict): if len(instance) == 0: pass elif 'type' not in instance: valid_types = None for k in instance.keys(): prop_class = get_metaschema_property(k, skip_generic=True) if prop_class is None: continue if valid_types is None: valid_types = set(prop_class.types) else: valid_types = (valid_types & set(prop_class.types)) if (valid_types is None) or (len(valid_types) == 0): # There were not any recorded properties so this must be a # dictionary of properties instance = {'type': 'object', 'properties': instance} else: if len(valid_types) > 1: valid_type_classes = sorted( [_type_registry[t] for t in valid_types], key=_specificity_sort_key) s_max = valid_type_classes[0].specificity valid_types = [] for tcls in valid_type_classes: if tcls.specificity > s_max: break valid_types.append(tcls.name) if 'scalar' in valid_types: for t in ['1darray', 'ndarray']: if t in valid_types: valid_types.remove(t) if len(valid_types) > 1: raise Exception("Multiple possible classes: %s" % valid_types) instance['type'] = valid_types[0] elif isinstance(instance, (list, tuple)): # If inside validation of items as a schema, don't assume a # list is a malformed schema. Doing so results in infinite # recursion. if not ((len(validator._schema_path_stack) >= 2) and (validator._schema_path_stack[-2:] == ['items', 0])): instance = {'type': 'array', 'items': instance} if isinstance(instance, dict) and ('type' in instance): typecls = get_type_class(instance['type']) instance = typecls.normalize_definition(instance) return instance
def get_test_data(typename): r"""Determine a test data set for the specified type. Args: typename (str): Name of datatype. Returns: object: Example of specified datatype. """ typeclass = get_type_class(typename) return typeclass.get_test_data()
def get_test_data(cls, typename): r"""Determine a test data set for the specified type. Args: typename (str): Name of datatype. Returns: object: Example of specified datatype. """ typeclass = get_type_class(typename) testclass = typeclass.import_test_class() out = testclass._valid_decoded[0] return out
def validate_metadata(cls, obj): r"""Validates an encoded object. Args: obj (string): Encoded object to validate. """ if ((isinstance(obj, dict) and ('type' in obj) and (obj['type'] != cls.name))): type_cls = get_type_class(obj['type']) if type_cls.is_fixed and type_cls.issubtype(cls.name): obj = type_cls.typedef_fixed2base(obj) # jsonschema.validate(obj, cls.metaschema(), cls=cls.validator()) jsonschema.validate(obj, cls.metadata_schema(), cls=cls.validator())
def _generate_data(cls, typedef): r"""Generate mock data for the specified type. Args: typedef (dict): Type definition. Returns: object: Python object of the specified type. """ out = cls._container_type() for k, v in cls._iterate(typedef[cls._json_property]): vcls = get_type_class(v['type']) cls._assign(out, k, vcls.generate_data(v)) return out
def validate_metadata(cls, obj, **kwargs): r"""Validates an encoded object. Args: obj (string): Encoded object to validate. **kwargs: Additional keyword arguments are passed to the validator. """ if ((isinstance(obj, dict) and ('type' in obj) and (obj['type'] != cls.name))): type_cls = get_type_class(obj['type']) if type_cls.is_fixed and type_cls.issubtype(cls.name): obj = type_cls.typedef_fixed2base(obj) # jsonschema.validate(obj, cls.metaschema(), cls=cls.validator()) # jsonschema.validate(obj, cls.metadata_schema(), cls=cls.validator()) return validate_instance(obj, cls.metadata_schema(), **kwargs)
def compare(cls, prop1, prop2, root1=None, root2=None): r"""Method to determine compatiblity of one property value with another. This method is not necessarily symmetric in that the second value may not be compatible with the first even if the first is compatible with the second. Args: prop1 (object): Property value to compare against prop2. prop2 (object): Property value to compare against. Yields: str: Comparision failure messages. """ type_cls = get_type_class(prop1) if not type_cls.issubtype(prop2): yield "Type '%s' is not a subtype of type '%s'" % (prop1, prop2)
def is_type(self, instance, types): r"""Determine if an object is an example of the given type. Args: instance (object): Object to test against to the type. type (str, list): Name of single type or a list of types that instance should be tested against. Returns: bool: True if the instance is of the specified type(s). False otherwise. """ out = super(Normalizer, self).is_type(instance, types) if (_jsonschema_ver_maj < 3) and out: out = get_type_class(types).validate(instance) return out
def decode_data(cls, obj, typedef): r"""Decode an object. Args: obj (string): Encoded object to decode. typedef (dict): Type definition that should be used to decode the object. Returns: object: Decoded object. """ container = cls._container_type() for k, v in cls._iterate(obj): vtypedef = cls._get_element(typedef[cls._json_property], k, {}) vcls = get_type_class(vtypedef['type']) cls._assign(container, k, vcls.decode_data(v, vtypedef)) return container
def create_multitype_class(types): r"""Create a MultiMetaschemaType class that wraps multiple classes. Args: types (list): List of names of types. Returns: class: Subclass of MultiMetaschemaType that add classes. """ type_classes = OrderedDict() type_name = '_'.join(types) class_name = str('MultiMetaschemaType_%s' % type_name) for t in types: type_classes[t] = get_type_class(t) out = type(class_name, (MultiMetaschemaType, ), {'type_classes': type_classes, 'name': type_name}) return out
def check_received_data(cls, typename, x_recv): r"""Check that the received message is equivalent to the test data for the specified type. Args: typename (str): Name of datatype. x_recv (object): Received object. Raises: AssertionError: If the received message is not equivalent to the received message. """ typeclass = get_type_class(typename) testclass = typeclass.import_test_class() x_sent = cls.get_test_data(typename) print('RECEIVED:') pprint.pprint(x_recv) print('EXPECTED:') pprint.pprint(x_sent) testclass.assert_result_equal(x_recv, x_sent)
def extract_typedef(cls, metadata): r"""Extract the minimum typedef required for this type from the provided metadata. Args: metadata (dict): Message metadata. Returns: dict: Encoded type definition with unncessary properties removed. """ out = super(ContainerMetaschemaType, cls).extract_typedef(metadata) if cls._json_property in out: contents = out[cls._json_property] if isinstance(contents, cls.python_types): for k, v in cls._iterate(contents): if 'type' in v: vcls = get_type_class(v['type']) cls._assign(contents, k, vcls.extract_typedef(v)) out[cls._json_property] = contents return out
def update_serializer(self, extract=False, skip_type=False, **kwargs): r"""Update serializer with provided information. Args: extract (bool, optional): If True, the updated typedef will be the bare minimum as extracted from total set of provided keywords, otherwise the entire set will be sued. Defaults to False. skip_type (bool, optional): If True, everything is updated except the data type. Defaults to False. **kwargs: Additional keyword arguments are processed as part of they type definition and are parsed for old-style keywords. Raises: RuntimeError: If there are keywords that are not valid typedef keywords (currect or old-style). """ old_datatype = None if self.initialized: old_datatype = copy.deepcopy(self.datatype) _metaschema = get_metaschema() # Raise an error if the types are not compatible seritype = kwargs.pop('seritype', self.seritype) if (seritype != self._seritype) and (seritype != 'default'): # pragma: debug raise Exception("Cannot change types form %s to %s." % (self._seritype, seritype)) # Remove metadata keywords unrelated to serialization # TODO: Find a better way of tracking these _remove_kws = [ 'body', 'address', 'size', 'id', 'incomplete', 'raw', 'commtype', 'filetype', 'response_address', 'request_id', 'append', 'in_temp', 'is_series', 'working_dir', 'fmts', 'model_driver', 'env', 'send_converter', 'recv_converter', 'typedef_base' ] kws = list(kwargs.keys()) for k in kws: if (k in _remove_kws) or k.startswith('zmq'): kwargs.pop(k) # Set attributes and remove unused metadata keys for k in self._schema_properties.keys(): if (k in kwargs) and (k != 'datatype'): setattr(self, k, kwargs.pop(k)) # Create preliminary typedef typedef = kwargs.pop('datatype', {}) for k in _metaschema['properties'].keys(): if k in kwargs: typedef[k] = kwargs.pop(k) # Update extra keywords if (len(kwargs) > 0): self.extra_kwargs.update(kwargs) self.debug("Extra kwargs: %s" % str(self.extra_kwargs)) # Update type if not skip_type: # Update typedef from oldstyle keywords in extra_kwargs typedef = self.update_typedef_from_oldstyle(typedef) if typedef.get('type', None): if extract: cls = get_type_class(typedef['type']) typedef = cls.extract_typedef(typedef) self.datatype = get_type_from_def(typedef) # Check to see if new datatype is compatible with new one if old_datatype is not None: errors = list( compare_schema(self.typedef, old_datatype._typedef) or ()) if errors: raise RuntimeError(( "Updated datatype is not compatible with the existing one." + " New:\n%s\nOld:\n%s\n") % (pprint.pformat(self.typedef), pprint.pformat(old_datatype._typedef))) # Enfore that strings used with messages are in bytes for k in self._attr_conv: v = getattr(self, k, None) if isinstance(v, (str, bytes)): setattr(self, k, tools.str2bytes(v))
def test_get_type_class(): r"""Test get_type_class.""" for v in _valid_objects.keys(): datatypes.get_type_class(v) with pytest.raises(ValueError): datatypes.get_type_class('invalid')
def get_native_type(cls, **kwargs): r"""Get the native type. Args: type (str, optional): Name of |yggdrasil| extended JSON type or JSONSchema dictionary defining a datatype. **kwargs: Additional keyword arguments may be used in determining the precise declaration that should be used. Returns: str: The native type. """ out = super(FortranModelDriver, cls).get_native_type(**kwargs) intent_regex = r'(,\s*intent\(.+?\))' for x in re.finditer(intent_regex, out): out = out.replace(x.group(0), '') type_match = re.search(cls.function_param['type_regex'], out) if type_match: type_match = type_match.groupdict() if type_match.get('shape_var', None): # pragma: debug if ('pointer' not in out) and ('allocatable' not in out): out += ', allocatable' if type_match['shape_var'][0] == '*': out = out.replace('*', ':') # raise Exception("Used default native_type, but need alias") elif type_match.get('length_var', None): if ((('pointer' not in out) and ('allocatable' not in out) and (type_match['length_var'] != 'X'))): out += ', allocatable' if type_match['length_var'] == '*': out = out.replace('*', ':') if not ((out == '*') or ('X' in out)): if out.startswith('ygg'): out = 'type(%s)' % out return out from yggdrasil.metaschema.datatypes import get_type_class json_type = kwargs.get('datatype', kwargs.get('type', 'bytes')) if isinstance(json_type, str): # pragma: no cover json_type = {'type': json_type} if 'type' in kwargs: # pragma: no cover json_type.update(kwargs) assert (isinstance(json_type, dict)) json_type = get_type_class( json_type['type']).normalize_definition(json_type) if out == '*': dim_str = '' if json_type['type'] == '1darray': if 'length' in json_type: dim_str = ', dimension(%s)' % str(json_type['length']) elif json_type['type'] == 'ndarray': if 'shape' in json_type: dim_str = ', dimension(%s)' % ','.join( [str(x) for x in json_type['shape']]) json_subtype = copy.deepcopy(json_type) json_subtype['type'] = json_subtype.pop('subtype') out = cls.get_native_type(datatype=json_subtype) + dim_str if not dim_str: json_subtype['type'] = out.split('(')[0] if json_subtype['type'] == 'character': # pragma: debug json_subtype['precision'] = '' raise RuntimeError("Character array requires precision.") else: json_subtype['precision'] = int(json_subtype['precision'] / 8) json_subtype.setdefault('ndim', 'n') out = 'type(%s)' % cls.get_native_type(type=( '%s_pointer' % json_type['type'])).format(**json_subtype) elif 'X' in out: if cls.allows_realloc(kwargs): out = 'type(yggchar_r)' else: if out.startswith('ygguint'): out = 'type(%s)' % out if out.startswith('logical'): precision = json_type.get('precision', 8) elif out.startswith('complex'): precision = json_type['precision'] / 2 elif json_type.get('subtype', json_type['type']) == 'unicode': precision = json_type['precision'] / 4 else: precision = json_type['precision'] out = out.replace('X', str(int(precision / 8))) return out
def test_get_type_class(): r"""Test get_type_class.""" for v in _valid_objects.keys(): datatypes.get_type_class(v) assert_raises(ValueError, datatypes.get_type_class, 'invalid')