def resolve_item(item, addr=None): if Serializable.is_serializable(item): hydrated_args = {'address': addr} if addr else {} # Recurse on the Serializable's values and hydrates any addressables found. This unwinds # from the leaves thus hydrating item's closure in the inline case. for key, value in item._asdict().items(): is_addressable = AddressableDescriptor.is_addressable(item, key) def maybe_addr(x): return parse_addr(x) if is_addressable and isinstance(x, six.string_types) else x if isinstance(value, collections.MutableMapping): container_type = type(value) container = container_type() container.update((k, resolve_item(maybe_addr(v))) for k, v in value.items()) hydrated_args[key] = container elif isinstance(value, collections.MutableSequence): container_type = type(value) hydrated_args[key] = container_type(resolve_item(maybe_addr(v)) for v in value) else: hydrated_args[key] = resolve_item(maybe_addr(value)) # Re-build the thin Serializable with either fully hydrated objects or Resolvables # substituted for all Address values; ie: Only ever expose fully resolved or resolvable # closures for requested addresses. return self._hydrate(type(item), **hydrated_args) elif isinstance(item, Address): if self._inline: return self._resolve_recursively(item, resolve_path) else: # TODO(John Sirois): Implement lazy cycle checks across Resolver chains. return Resolver(self, address=item) else: return item
def parse(cls, path, symbol_table_cls): parse_globals = cls._get_globals(symbol_table_cls) python = _read(path) symbols = {} six.exec_(python, parse_globals, symbols) objects = [] for name, obj in symbols.items(): if isinstance(obj, type): # Allow type imports continue if not Serializable.is_serializable(obj): raise ParseError('Found a non-serializable top-level object: {}'.format(obj)) attributes = obj._asdict() if 'name' in attributes: attributes = attributes.copy() redundant_name = attributes.pop('name', None) if redundant_name and redundant_name != name: raise ParseError('The object named {!r} is assigned to a mismatching name {!r}' .format(redundant_name, name)) obj_type = type(obj) named_obj = obj_type(name=name, **attributes) objects.append(named_obj) return objects
def parse(cls, path, symbol_table_cls, parser_cls): """Parses a source for addressable Serializable objects. No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any objects they point to in other namespaces or even in the same namespace but from a seperate source are left as unresolved pointers. :param string path: The path to the byte source containing serialized objects. :param parser_cls: The parser cls to use. :type parser_cls: A :class:`pants.engine.exp.parser.Parser` """ objects = parser_cls.parse(path, symbol_table_cls) objects_by_name = {} for obj in objects: if not Serializable.is_serializable(obj): raise UnaddressableObjectError('Parsed a non-serializable object: {!r}'.format(obj)) attributes = obj._asdict() name = attributes.get('name') if not name: raise UnaddressableObjectError('Parsed a non-addressable object: {!r}'.format(obj)) if name in objects_by_name: raise DuplicateNameError('An object already exists at {!r} with name {!r}: {!r}. Cannot ' 'map {!r}'.format(path, name, objects_by_name[name], obj)) objects_by_name[name] = obj return cls(path, objects_by_name)
def parse(cls, path, symbol_table_cls): parse_globals = cls._get_globals(symbol_table_cls) python = _read(path) symbols = {} six.exec_(python, parse_globals, symbols) objects = [] for name, obj in symbols.items(): if isinstance(obj, type): # Allow type imports continue if not Serializable.is_serializable(obj): raise ParseError( 'Found a non-serializable top-level object: {}'.format( obj)) attributes = obj._asdict() if 'name' in attributes: attributes = attributes.copy() redundant_name = attributes.pop('name', None) if redundant_name and redundant_name != name: raise ParseError( 'The object named {!r} is assigned to a mismatching name {!r}' .format(redundant_name, name)) obj_type = type(obj) named_obj = obj_type(name=name, **attributes) objects.append(named_obj) return objects
def _checked_value(self, instance, value): # We allow four forms of value: # 1. An opaque (to us) string address pointing to a value that can be resolved by external # means. # 2. A `Resolvable` value that we can lazily resolve and type-check in `__get__`. # 3. A concrete instance that meets our type constraint. # 4. A dict when our type constraint has exactly one Serializable subject type - we convert the # dict into an instance of that type. if value is None: return None if isinstance(value, (six.string_types, Resolvable)): return value # Support untyped dicts that we deserialize on-demand here into the required type. # This feature allows for more brevity in the JSON form (local type inference) and an alternate # construction style in the python forms. type_constraint = self._get_type_constraint(instance) if (isinstance(value, dict) and len(type_constraint.types) == 1 and Serializable.is_serializable_type(type_constraint.types[0])): if not value: # TODO(John Sirois): Is this the right thing to do? Or should an empty serializable_type # be constructed? return None # {} -> None. else: serializable_type = type_constraint.types[0] return serializable_type(**value) if not type_constraint.satisfied_by(value): raise TypeConstraintError( 'Got {} of type {} for {} attribute of {} but expected {!r}'. format(value, type(value).__name__, self._name, instance, type_constraint)) return value
def parse_python_assignments(python, symbol_table=None): """Parses the given python code into a list of top-level addressable Serializable objects found. Only Serializable objects assigned to top-level variables will be collected and returned. These objects will be addressable via their top-level variable names in the parsed namespace. :param string python: A python build file blob. :returns: A list of decoded addressable, Serializable objects. :rtype: list :raises: :class:`ParseError` if there were any problems encountered parsing the given `python`. """ def aliased(type_name, object_type, **kwargs): return object_type(typename=type_name, **kwargs) parse_globals = {} for alias, symbol in (symbol_table or {}).items(): parse_globals[alias] = functools.partial(aliased, alias, symbol) symbols = {} six.exec_(python, parse_globals, symbols) objects = [] for name, obj in symbols.items(): if Serializable.is_serializable(obj): attributes = obj._asdict() redundant_name = attributes.pop('name', name) if redundant_name and redundant_name != name: raise ParseError( 'The object named {!r} is assigned to a mismatching name {!r}' .format(redundant_name, name)) obj_type = type(obj) named_obj = obj_type(name=name, **attributes) objects.append(named_obj) return objects
def parse(cls, path, parser=None): """Parses a source for addressable Serializable objects. By default an enhanced JSON parser is used. The parser admits extra blank lines, comment lines and more than one top-level JSON object. See :`pants.engine.exp.parsers.parse_json` for more details on the modified JSON format and the schema for Serializable json objects. No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any objects they point to in other namespaces or even in the same namespace but from a seperate source are left as unresolved pointers. :param string path: The path to the byte source containing serialized objects. :param parser: The parser to use; by default a json parser. :type parser: :class:`collection.Callable` that accepts a file path and returns a list of all addressable Serializable objects parsed from it. """ parse = parser or parsers.parse_json objects = parse(path) objects_by_name = {} for obj in objects: if not Serializable.is_serializable(obj): raise UnaddressableObjectError('Parsed a non-serializable object: {!r}'.format(obj)) attributes = obj._asdict() name = attributes.get('name') if not name: raise UnaddressableObjectError('Parsed a non-addressable object: {!r}'.format(obj)) if name in objects_by_name: raise DuplicateNameError('An object already exists at {!r} with name {!r}: {!r}. Cannot ' 'map {!r}'.format(path, name, objects_by_name[name], obj)) objects_by_name[name] = obj return cls(path, objects_by_name)
def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls): """Parses a source for addressable Serializable objects. No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any objects they point to in other namespaces or even in the same namespace but from a seperate source are left as unresolved pointers. :param string path: The path to the byte source containing serialized objects. :param parser_cls: The parser cls to use. :type parser_cls: A :class:`pants.engine.exp.parser.Parser` """ try: objects = parser_cls.parse(filepath, filecontent, symbol_table_cls) except Exception as e: raise MappingError('Failed to parse {}:\n{}'.format(filepath, e)) objects_by_name = {} for obj in objects: if not Serializable.is_serializable(obj): raise UnaddressableObjectError( 'Parsed a non-serializable object: {!r}'.format(obj)) attributes = obj._asdict() name = attributes.get('name') if not name: raise UnaddressableObjectError( 'Parsed a non-addressable object: {!r}'.format(obj)) if name in objects_by_name: raise DuplicateNameError( 'An object already exists at {!r} with name {!r}: {!r}. Cannot ' 'map {!r}'.format(filepath, name, objects_by_name[name], obj)) objects_by_name[name] = obj return cls(filepath, OrderedDict(sorted(objects_by_name.items())))
def parse(cls, path, parse=None): """Parses a source for addressable Serializable objects. By default an enhanced JSON parser is used. The parser admits extra blank lines, comment lines and more than one top-level JSON object. See :`pants.engine.exp.parsers.parse_json` for more details on the modified JSON format and the schema for Serializable json objects. No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any objects they point to in other namespaces or even in the same namespace but from a seperate source are left as unresolved pointers. :param string path: The path to the byte source containing serialized objects. :param parse: The parse function to use; by default a json parser. :type parse: :class:`collection.Callable` that accepts a byte source and returns a list of all addressable Serializable objects parsed from it. """ parse = parse or parsers.parse_json with open(path, 'r') as fp: objects = parse(fp.read()) objects_by_name = {} for obj in objects: if not Serializable.is_serializable( obj) or not obj._asdict().get('name'): raise UnaddressableObjectError( 'Parsed a non-addressable object: {!r}'.format(obj)) attributes = obj._asdict() name = attributes['name'] if name in objects_by_name: raise DuplicateNameError( 'An object already exists at {!r} with name {!r}: {!r}. Cannot ' 'map {!r}'.format(path, name, objects_by_name[name], obj)) objects_by_name[name] = obj return cls(path, objects_by_name)
def registered(type_name, object_type, name=None, **kwargs): if name: obj = object_type(name=name, type_alias=type_name, **kwargs) if Serializable.is_serializable(obj): objects.append(obj) return obj else: return object_type(type_alias=type_name, **kwargs)
def _object_encoder(o): if not Serializable.is_serializable(o): raise ParseError( 'Can only encode Serializable objects in JSON, given {!r} of type {}' .format(o, type(o).__name__)) encoded = o._asdict() if 'typename' not in encoded: encoded['typename'] = '{}.{}'.format( inspect.getmodule(o).__name__, type(o).__name__) return encoded
def resolve_item(item, addr=None): if Serializable.is_serializable(item): hydrated_args = {'address': addr} if addr else {} # Recurse on the Serializable's values and hydrate any `Addressed` found. This unwinds from # the leaves thus hydrating item's closure. for key, value in item._asdict().items(): if isinstance(value, collections.MutableMapping): container_type = type(value) container = container_type() container.update( (k, resolve_item(v)) for k, v in value.items()) hydrated_args[key] = container elif isinstance(value, collections.Iterable) and not isinstance( value, six.string_types): container_type = type(value) hydrated_args[key] = container_type( resolve_item(v) for v in value) else: hydrated_args[key] = resolve_item(value) # Re-build the thin Serializable with fully hydrated objects substituted for all Addressed # values; ie: Only ever expose full resolved closures for requested addresses. item_type = type(item) hydrated_item = item_type(**hydrated_args) # Let factories replace the hydrated object. if isinstance(hydrated_item, SerializableFactory): hydrated_item = hydrated_item.create() # Finally make sure objects that can self-validate get a chance to do so before we cache # them as the pointee of `hydrated_item.address`. if isinstance(hydrated_item, Validatable): hydrated_item.validate() return hydrated_item elif isinstance(item, Addressed): referenced_address = Address.parse( spec=item.address_spec, relative_to=address.spec_path) referenced_item = self._resolve_recursively( referenced_address, resolve_path) if not item.type_constraint.satisfied_by(referenced_item): raise ResolvedTypeMismatchError( 'Found a {} when resolving {} for {}, expected a {!r}'. format( type(referenced_item).__name__, referenced_address, address, item.type_constraint)) return referenced_item else: return item
def _object_encoder(obj, inline): if isinstance(obj, Resolvable): return obj.resolve() if inline else obj.address if isinstance(obj, Address): return obj.reference() if not Serializable.is_serializable(obj): raise ParseError('Can only encode Serializable objects in JSON, given {!r} of type {}' .format(obj, type(obj).__name__)) encoded = obj._asdict() if 'type_alias' not in encoded: encoded = encoded.copy() encoded['type_alias'] = '{}.{}'.format(inspect.getmodule(obj).__name__, type(obj).__name__) return {k: v for k, v in encoded.items() if v}
def resolve_item(item, addr=None): if Serializable.is_serializable(item): hydrated_args = {'address': addr} if addr else {} # Recurse on the Serializable's values and hydrate any `Addressed` found. This unwinds from # the leaves thus hydrating item's closure. for key, value in item._asdict().items(): if isinstance(value, collections.MutableMapping): container_type = type(value) container = container_type() container.update((k, resolve_item(v)) for k, v in value.items()) hydrated_args[key] = container elif isinstance(value, collections.Iterable) and not isinstance(value, six.string_types): container_type = type(value) hydrated_args[key] = container_type(resolve_item(v) for v in value) else: hydrated_args[key] = resolve_item(value) # Re-build the thin Serializable with fully hydrated objects substituted for all Addressed # values; ie: Only ever expose full resolved closures for requested addresses. item_type = type(item) hydrated_item = item_type(**hydrated_args) # Let factories replace the hydrated object. if isinstance(hydrated_item, SerializableFactory): hydrated_item = hydrated_item.create() # Finally make sure objects that can self-validate get a chance to do so before we cache # them as the pointee of `hydrated_item.address`. if isinstance(hydrated_item, Validatable): hydrated_item.validate() return hydrated_item elif isinstance(item, Addressed): referenced_address = Address.parse(spec=item.address_spec, relative_to=address.spec_path) referenced_item = self._resolve_recursively(referenced_address, resolve_path) if not item.type_constraint.satisfied_by(referenced_item): raise ResolvedTypeMismatchError('Found a {} when resolving {} for {}, expected a {!r}' .format(type(referenced_item).__name__, referenced_address, address, item.type_constraint)) return referenced_item else: return item
def _object_encoder(obj, inline): if isinstance(obj, Resolvable): return obj.resolve() if inline else obj.address if isinstance(obj, Address): return obj.reference() if not Serializable.is_serializable(obj): raise ParseError( 'Can only encode Serializable objects in JSON, given {!r} of type {}' .format(obj, type(obj).__name__)) encoded = obj._asdict() if 'type_alias' not in encoded: encoded = encoded.copy() encoded['type_alias'] = '{}.{}'.format( inspect.getmodule(obj).__name__, type(obj).__name__) return {k: v for k, v in encoded.items() if v}
def __set__(self, instance, value): if not Serializable.is_serializable(instance): raise NotSerializableError( "The addressable descriptor {} can only be applied to methods or " "properties of Serializable objects, applied to method {} of " "type {}".format(type(self).__name__, self._name, type(instance).__name__) ) instance_dict = instance._asdict() if self._name in instance_dict: raise MutationError( "Attribute {} of {} has already been set to {}, rejecting attempt to " "re-set with {}".format(self._name, instance, instance_dict[self._name], value) ) value = self._checked_value(instance, value) self._register(instance, self) # We mutate the instance dict, which is only OK if used in the conventional idiom of setting # the value via this data descriptor in the instance's constructor. instance_dict[self._name] = value
def _checked_value(self, instance, value): # We allow five forms of value: # 0. None. # 1. An opaque (to us) address pointing to a value that can be resolved by external # means. # 2. A `Resolvable` value that we can lazily resolve and type-check in `__get__`. # 3. A concrete instance that meets our type constraint. # 4. A dict when our type constraint has exactly one Serializable subject type - we convert the # dict into an instance of that type. if value is None: return None if isinstance(value, (six.string_types, Address, Resolvable)): return value # Support untyped dicts that we deserialize on-demand here into the required type. # This feature allows for more brevity in the JSON form (local type inference) and an alternate # construction style in the python forms. type_constraint = self._get_type_constraint(instance) if ( isinstance(value, dict) and len(type_constraint.types) == 1 and Serializable.is_serializable_type(type_constraint.types[0]) ): if not value: # TODO(John Sirois): Is this the right thing to do? Or should an empty serializable_type # be constructed? return None # {} -> None. else: serializable_type = type_constraint.types[0] return serializable_type(**value) if not type_constraint.satisfied_by(value): raise TypeConstraintError( "Got {} of type {} for {} attribute of {} but expected {!r}".format( value, type(value).__name__, self._name, instance, type_constraint ) ) return value
def __set__(self, instance, value): if not Serializable.is_serializable(instance): raise NotSerializableError( 'The addressable descriptor {} can only be applied to methods or ' 'properties of Serializable objects, applied to method {} of ' 'type {}'.format( type(self).__name__, self._name, type(instance).__name__)) instance_dict = instance._asdict() if self._name in instance_dict: raise MutationError( 'Attribute {} of {} has already been set to {}, rejecting attempt to ' 're-set with {}'.format(self._name, instance, instance_dict[self._name], value)) value = self._checked_value(instance, value) self._register(instance, self) # We mutate the instance dict, which is only OK if used in the conventional idiom of setting # the value via this data descriptor in the instance's constructor. instance_dict[self._name] = value
def __init__(self, type_alias, object_type): self._type_alias = type_alias self._object_type = object_type self._serializable = Serializable.is_serializable_type( self._object_type)
def __init__(self, type_alias, object_type): self._type_alias = type_alias self._object_type = object_type self._serializable = Serializable.is_serializable_type(self._object_type)