def resolve_item(item, addr=None):
if Serializable.is_serializable(item):
hydrated_args = {'address': addr} if addr else {}
# Recurse on the Serializable's values and hydrates any addressables found. This unwinds
# from the leaves thus hydrating item's closure in the inline case.
for key, value in item._asdict().items():
is_addressable = AddressableDescriptor.is_addressable(item, key)
def maybe_addr(x):
return parse_addr(x) if is_addressable and isinstance(x, six.string_types) else x
if isinstance(value, collections.MutableMapping):
container_type = type(value)
container = container_type()
container.update((k, resolve_item(maybe_addr(v))) for k, v in value.items())
hydrated_args[key] = container
elif isinstance(value, collections.MutableSequence):
container_type = type(value)
hydrated_args[key] = container_type(resolve_item(maybe_addr(v)) for v in value)
else:
hydrated_args[key] = resolve_item(maybe_addr(value))
# Re-build the thin Serializable with either fully hydrated objects or Resolvables
# substituted for all Address values; ie: Only ever expose fully resolved or resolvable
# closures for requested addresses.
return self._hydrate(type(item), **hydrated_args)
elif isinstance(item, Address):
if self._inline:
return self._resolve_recursively(item, resolve_path)
else:
# TODO(John Sirois): Implement lazy cycle checks across Resolver chains.
return Resolver(self, address=item)
else:
return item
def parse(cls, path, symbol_table_cls):
parse_globals = cls._get_globals(symbol_table_cls)
python = _read(path)
symbols = {}
six.exec_(python, parse_globals, symbols)
objects = []
for name, obj in symbols.items():
if isinstance(obj, type):
# Allow type imports
continue
if not Serializable.is_serializable(obj):
raise ParseError('Found a non-serializable top-level object: {}'.format(obj))
attributes = obj._asdict()
if 'name' in attributes:
attributes = attributes.copy()
redundant_name = attributes.pop('name', None)
if redundant_name and redundant_name != name:
raise ParseError('The object named {!r} is assigned to a mismatching name {!r}'
.format(redundant_name, name))
obj_type = type(obj)
named_obj = obj_type(name=name, **attributes)
objects.append(named_obj)
return objects
def parse(cls, path, symbol_table_cls, parser_cls):
"""Parses a source for addressable Serializable objects.
No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
objects they point to in other namespaces or even in the same namespace but from a seperate
source are left as unresolved pointers.
:param string path: The path to the byte source containing serialized objects.
:param parser_cls: The parser cls to use.
:type parser_cls: A :class:`pants.engine.exp.parser.Parser`
"""
objects = parser_cls.parse(path, symbol_table_cls)
objects_by_name = {}
for obj in objects:
if not Serializable.is_serializable(obj):
raise UnaddressableObjectError('Parsed a non-serializable object: {!r}'.format(obj))
attributes = obj._asdict()
name = attributes.get('name')
if not name:
raise UnaddressableObjectError('Parsed a non-addressable object: {!r}'.format(obj))
if name in objects_by_name:
raise DuplicateNameError('An object already exists at {!r} with name {!r}: {!r}. Cannot '
'map {!r}'.format(path, name, objects_by_name[name], obj))
objects_by_name[name] = obj
return cls(path, objects_by_name)
def parse(cls, path, symbol_table_cls):
parse_globals = cls._get_globals(symbol_table_cls)
python = _read(path)
symbols = {}
six.exec_(python, parse_globals, symbols)
objects = []
for name, obj in symbols.items():
if isinstance(obj, type):
# Allow type imports
continue
if not Serializable.is_serializable(obj):
raise ParseError(
'Found a non-serializable top-level object: {}'.format(
obj))
attributes = obj._asdict()
if 'name' in attributes:
attributes = attributes.copy()
redundant_name = attributes.pop('name', None)
if redundant_name and redundant_name != name:
raise ParseError(
'The object named {!r} is assigned to a mismatching name {!r}'
.format(redundant_name, name))
obj_type = type(obj)
named_obj = obj_type(name=name, **attributes)
objects.append(named_obj)
return objects
def _checked_value(self, instance, value):
# We allow four forms of value:
# 1. An opaque (to us) string address pointing to a value that can be resolved by external
# means.
# 2. A `Resolvable` value that we can lazily resolve and type-check in `__get__`.
# 3. A concrete instance that meets our type constraint.
# 4. A dict when our type constraint has exactly one Serializable subject type - we convert the
# dict into an instance of that type.
if value is None:
return None
if isinstance(value, (six.string_types, Resolvable)):
return value
# Support untyped dicts that we deserialize on-demand here into the required type.
# This feature allows for more brevity in the JSON form (local type inference) and an alternate
# construction style in the python forms.
type_constraint = self._get_type_constraint(instance)
if (isinstance(value, dict) and len(type_constraint.types) == 1 and
Serializable.is_serializable_type(type_constraint.types[0])):
if not value:
# TODO(John Sirois): Is this the right thing to do? Or should an empty serializable_type
# be constructed?
return None # {} -> None.
else:
serializable_type = type_constraint.types[0]
return serializable_type(**value)
if not type_constraint.satisfied_by(value):
raise TypeConstraintError(
'Got {} of type {} for {} attribute of {} but expected {!r}'.
format(value,
type(value).__name__, self._name, instance,
type_constraint))
return value
def parse_python_assignments(python, symbol_table=None):
"""Parses the given python code into a list of top-level addressable Serializable objects found.
Only Serializable objects assigned to top-level variables will be collected and returned. These
objects will be addressable via their top-level variable names in the parsed namespace.
:param string python: A python build file blob.
:returns: A list of decoded addressable, Serializable objects.
:rtype: list
:raises: :class:`ParseError` if there were any problems encountered parsing the given `python`.
"""
def aliased(type_name, object_type, **kwargs):
return object_type(typename=type_name, **kwargs)
parse_globals = {}
for alias, symbol in (symbol_table or {}).items():
parse_globals[alias] = functools.partial(aliased, alias, symbol)
symbols = {}
six.exec_(python, parse_globals, symbols)
objects = []
for name, obj in symbols.items():
if Serializable.is_serializable(obj):
attributes = obj._asdict()
redundant_name = attributes.pop('name', name)
if redundant_name and redundant_name != name:
raise ParseError(
'The object named {!r} is assigned to a mismatching name {!r}'
.format(redundant_name, name))
obj_type = type(obj)
named_obj = obj_type(name=name, **attributes)
objects.append(named_obj)
return objects
def parse(cls, path, parser=None):
"""Parses a source for addressable Serializable objects.
By default an enhanced JSON parser is used. The parser admits extra blank lines, comment lines
and more than one top-level JSON object. See :`pants.engine.exp.parsers.parse_json` for more
details on the modified JSON format and the schema for Serializable json objects.
No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
objects they point to in other namespaces or even in the same namespace but from a seperate
source are left as unresolved pointers.
:param string path: The path to the byte source containing serialized objects.
:param parser: The parser to use; by default a json parser.
:type parser: :class:`collection.Callable` that accepts a file path and returns a list of all
addressable Serializable objects parsed from it.
"""
parse = parser or parsers.parse_json
objects = parse(path)
objects_by_name = {}
for obj in objects:
if not Serializable.is_serializable(obj):
raise UnaddressableObjectError('Parsed a non-serializable object: {!r}'.format(obj))
attributes = obj._asdict()
name = attributes.get('name')
if not name:
raise UnaddressableObjectError('Parsed a non-addressable object: {!r}'.format(obj))
if name in objects_by_name:
raise DuplicateNameError('An object already exists at {!r} with name {!r}: {!r}. Cannot '
'map {!r}'.format(path, name, objects_by_name[name], obj))
objects_by_name[name] = obj
return cls(path, objects_by_name)
def resolve_item(item, addr=None):
if Serializable.is_serializable(item):
hydrated_args = {'address': addr} if addr else {}
# Recurse on the Serializable's values and hydrates any addressables found. This unwinds
# from the leaves thus hydrating item's closure in the inline case.
for key, value in item._asdict().items():
is_addressable = AddressableDescriptor.is_addressable(item, key)
def maybe_addr(x):
return parse_addr(x) if is_addressable and isinstance(x, six.string_types) else x
if isinstance(value, collections.MutableMapping):
container_type = type(value)
container = container_type()
container.update((k, resolve_item(maybe_addr(v))) for k, v in value.items())
hydrated_args[key] = container
elif isinstance(value, collections.MutableSequence):
container_type = type(value)
hydrated_args[key] = container_type(resolve_item(maybe_addr(v)) for v in value)
else:
hydrated_args[key] = resolve_item(maybe_addr(value))
# Re-build the thin Serializable with either fully hydrated objects or Resolvables
# substituted for all Address values; ie: Only ever expose fully resolved or resolvable
# closures for requested addresses.
return self._hydrate(type(item), **hydrated_args)
elif isinstance(item, Address):
if self._inline:
return self._resolve_recursively(item, resolve_path)
else:
# TODO(John Sirois): Implement lazy cycle checks across Resolver chains.
return Resolver(self, address=item)
else:
return item
def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls):
"""Parses a source for addressable Serializable objects.
No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
objects they point to in other namespaces or even in the same namespace but from a seperate
source are left as unresolved pointers.
:param string path: The path to the byte source containing serialized objects.
:param parser_cls: The parser cls to use.
:type parser_cls: A :class:`pants.engine.exp.parser.Parser`
"""
try:
objects = parser_cls.parse(filepath, filecontent, symbol_table_cls)
except Exception as e:
raise MappingError('Failed to parse {}:\n{}'.format(filepath, e))
objects_by_name = {}
for obj in objects:
if not Serializable.is_serializable(obj):
raise UnaddressableObjectError(
'Parsed a non-serializable object: {!r}'.format(obj))
attributes = obj._asdict()
name = attributes.get('name')
if not name:
raise UnaddressableObjectError(
'Parsed a non-addressable object: {!r}'.format(obj))
if name in objects_by_name:
raise DuplicateNameError(
'An object already exists at {!r} with name {!r}: {!r}. Cannot '
'map {!r}'.format(filepath, name, objects_by_name[name],
obj))
objects_by_name[name] = obj
return cls(filepath, OrderedDict(sorted(objects_by_name.items())))
def parse(cls, path, parse=None):
"""Parses a source for addressable Serializable objects.
By default an enhanced JSON parser is used. The parser admits extra blank lines, comment lines
and more than one top-level JSON object. See :`pants.engine.exp.parsers.parse_json` for more
details on the modified JSON format and the schema for Serializable json objects.
No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
objects they point to in other namespaces or even in the same namespace but from a seperate
source are left as unresolved pointers.
:param string path: The path to the byte source containing serialized objects.
:param parse: The parse function to use; by default a json parser.
:type parse: :class:`collection.Callable` that accepts a byte source and returns a list of all
addressable Serializable objects parsed from it.
"""
parse = parse or parsers.parse_json
with open(path, 'r') as fp:
objects = parse(fp.read())
objects_by_name = {}
for obj in objects:
if not Serializable.is_serializable(
obj) or not obj._asdict().get('name'):
raise UnaddressableObjectError(
'Parsed a non-addressable object: {!r}'.format(obj))
attributes = obj._asdict()
name = attributes['name']
if name in objects_by_name:
raise DuplicateNameError(
'An object already exists at {!r} with name {!r}: {!r}. Cannot '
'map {!r}'.format(path, name, objects_by_name[name],
obj))
objects_by_name[name] = obj
return cls(path, objects_by_name)
def registered(type_name, object_type, name=None, **kwargs):
if name:
obj = object_type(name=name, type_alias=type_name, **kwargs)
if Serializable.is_serializable(obj):
objects.append(obj)
return obj
else:
return object_type(type_alias=type_name, **kwargs)
def registered(type_name, object_type, name=None, **kwargs):
if name:
obj = object_type(name=name, type_alias=type_name, **kwargs)
if Serializable.is_serializable(obj):
objects.append(obj)
return obj
else:
return object_type(type_alias=type_name, **kwargs)
def _object_encoder(o):
if not Serializable.is_serializable(o):
raise ParseError(
'Can only encode Serializable objects in JSON, given {!r} of type {}'
.format(o,
type(o).__name__))
encoded = o._asdict()
if 'typename' not in encoded:
encoded['typename'] = '{}.{}'.format(
inspect.getmodule(o).__name__,
type(o).__name__)
return encoded
def resolve_item(item, addr=None):
if Serializable.is_serializable(item):
hydrated_args = {'address': addr} if addr else {}
# Recurse on the Serializable's values and hydrate any `Addressed` found. This unwinds from
# the leaves thus hydrating item's closure.
for key, value in item._asdict().items():
if isinstance(value, collections.MutableMapping):
container_type = type(value)
container = container_type()
container.update(
(k, resolve_item(v)) for k, v in value.items())
hydrated_args[key] = container
elif isinstance(value,
collections.Iterable) and not isinstance(
value, six.string_types):
container_type = type(value)
hydrated_args[key] = container_type(
resolve_item(v) for v in value)
else:
hydrated_args[key] = resolve_item(value)
# Re-build the thin Serializable with fully hydrated objects substituted for all Addressed
# values; ie: Only ever expose full resolved closures for requested addresses.
item_type = type(item)
hydrated_item = item_type(**hydrated_args)
# Let factories replace the hydrated object.
if isinstance(hydrated_item, SerializableFactory):
hydrated_item = hydrated_item.create()
# Finally make sure objects that can self-validate get a chance to do so before we cache
# them as the pointee of `hydrated_item.address`.
if isinstance(hydrated_item, Validatable):
hydrated_item.validate()
return hydrated_item
elif isinstance(item, Addressed):
referenced_address = Address.parse(
spec=item.address_spec, relative_to=address.spec_path)
referenced_item = self._resolve_recursively(
referenced_address, resolve_path)
if not item.type_constraint.satisfied_by(referenced_item):
raise ResolvedTypeMismatchError(
'Found a {} when resolving {} for {}, expected a {!r}'.
format(
type(referenced_item).__name__, referenced_address,
address, item.type_constraint))
return referenced_item
else:
return item
def _object_encoder(obj, inline):
if isinstance(obj, Resolvable):
return obj.resolve() if inline else obj.address
if isinstance(obj, Address):
return obj.reference()
if not Serializable.is_serializable(obj):
raise ParseError('Can only encode Serializable objects in JSON, given {!r} of type {}'
.format(obj, type(obj).__name__))
encoded = obj._asdict()
if 'type_alias' not in encoded:
encoded = encoded.copy()
encoded['type_alias'] = '{}.{}'.format(inspect.getmodule(obj).__name__, type(obj).__name__)
return {k: v for k, v in encoded.items() if v}
def resolve_item(item, addr=None):
if Serializable.is_serializable(item):
hydrated_args = {'address': addr} if addr else {}
# Recurse on the Serializable's values and hydrate any `Addressed` found. This unwinds from
# the leaves thus hydrating item's closure.
for key, value in item._asdict().items():
if isinstance(value, collections.MutableMapping):
container_type = type(value)
container = container_type()
container.update((k, resolve_item(v)) for k, v in value.items())
hydrated_args[key] = container
elif isinstance(value, collections.Iterable) and not isinstance(value, six.string_types):
container_type = type(value)
hydrated_args[key] = container_type(resolve_item(v) for v in value)
else:
hydrated_args[key] = resolve_item(value)
# Re-build the thin Serializable with fully hydrated objects substituted for all Addressed
# values; ie: Only ever expose full resolved closures for requested addresses.
item_type = type(item)
hydrated_item = item_type(**hydrated_args)
# Let factories replace the hydrated object.
if isinstance(hydrated_item, SerializableFactory):
hydrated_item = hydrated_item.create()
# Finally make sure objects that can self-validate get a chance to do so before we cache
# them as the pointee of `hydrated_item.address`.
if isinstance(hydrated_item, Validatable):
hydrated_item.validate()
return hydrated_item
elif isinstance(item, Addressed):
referenced_address = Address.parse(spec=item.address_spec, relative_to=address.spec_path)
referenced_item = self._resolve_recursively(referenced_address, resolve_path)
if not item.type_constraint.satisfied_by(referenced_item):
raise ResolvedTypeMismatchError('Found a {} when resolving {} for {}, expected a {!r}'
.format(type(referenced_item).__name__,
referenced_address,
address,
item.type_constraint))
return referenced_item
else:
return item
def _object_encoder(obj, inline):
if isinstance(obj, Resolvable):
return obj.resolve() if inline else obj.address
if isinstance(obj, Address):
return obj.reference()
if not Serializable.is_serializable(obj):
raise ParseError(
'Can only encode Serializable objects in JSON, given {!r} of type {}'
.format(obj,
type(obj).__name__))
encoded = obj._asdict()
if 'type_alias' not in encoded:
encoded = encoded.copy()
encoded['type_alias'] = '{}.{}'.format(
inspect.getmodule(obj).__name__,
type(obj).__name__)
return {k: v for k, v in encoded.items() if v}
def __set__(self, instance, value):
if not Serializable.is_serializable(instance):
raise NotSerializableError(
"The addressable descriptor {} can only be applied to methods or "
"properties of Serializable objects, applied to method {} of "
"type {}".format(type(self).__name__, self._name, type(instance).__name__)
)
instance_dict = instance._asdict()
if self._name in instance_dict:
raise MutationError(
"Attribute {} of {} has already been set to {}, rejecting attempt to "
"re-set with {}".format(self._name, instance, instance_dict[self._name], value)
)
value = self._checked_value(instance, value)
self._register(instance, self)
# We mutate the instance dict, which is only OK if used in the conventional idiom of setting
# the value via this data descriptor in the instance's constructor.
instance_dict[self._name] = value
def _checked_value(self, instance, value):
# We allow five forms of value:
# 0. None.
# 1. An opaque (to us) address pointing to a value that can be resolved by external
# means.
# 2. A `Resolvable` value that we can lazily resolve and type-check in `__get__`.
# 3. A concrete instance that meets our type constraint.
# 4. A dict when our type constraint has exactly one Serializable subject type - we convert the
# dict into an instance of that type.
if value is None:
return None
if isinstance(value, (six.string_types, Address, Resolvable)):
return value
# Support untyped dicts that we deserialize on-demand here into the required type.
# This feature allows for more brevity in the JSON form (local type inference) and an alternate
# construction style in the python forms.
type_constraint = self._get_type_constraint(instance)
if (
isinstance(value, dict)
and len(type_constraint.types) == 1
and Serializable.is_serializable_type(type_constraint.types[0])
):
if not value:
# TODO(John Sirois): Is this the right thing to do? Or should an empty serializable_type
# be constructed?
return None # {} -> None.
else:
serializable_type = type_constraint.types[0]
return serializable_type(**value)
if not type_constraint.satisfied_by(value):
raise TypeConstraintError(
"Got {} of type {} for {} attribute of {} but expected {!r}".format(
value, type(value).__name__, self._name, instance, type_constraint
)
)
return value
def __set__(self, instance, value):
if not Serializable.is_serializable(instance):
raise NotSerializableError(
'The addressable descriptor {} can only be applied to methods or '
'properties of Serializable objects, applied to method {} of '
'type {}'.format(
type(self).__name__, self._name,
type(instance).__name__))
instance_dict = instance._asdict()
if self._name in instance_dict:
raise MutationError(
'Attribute {} of {} has already been set to {}, rejecting attempt to '
're-set with {}'.format(self._name, instance,
instance_dict[self._name], value))
value = self._checked_value(instance, value)
self._register(instance, self)
# We mutate the instance dict, which is only OK if used in the conventional idiom of setting
# the value via this data descriptor in the instance's constructor.
instance_dict[self._name] = value
def __init__(self, type_alias, object_type):
self._type_alias = type_alias
self._object_type = object_type
self._serializable = Serializable.is_serializable_type(
self._object_type)
def __init__(self, type_alias, object_type): self._type_alias = type_alias self._object_type = object_type self._serializable = Serializable.is_serializable_type(self._object_type)
