def description(self) -> Description:
decoded_record = json.loads(self._serialized_record)
resource_type = TypeIdentifier(tuple(decoded_record['type']))
# TODO: Handle multidimensional references.
shape = (1, )
value = Description(resource_type=resource_type, shape=shape)
return value
def __init__(self, resource_type: TypeIdentifier, *, shape: tuple = (1,)):
type_id = TypeIdentifier.copy_from(resource_type)
if type_id is None:
raise ValueError(f'Could not create a TypeIdentifier for {repr(resource_type)}')
# TODO: Further input validation
self._shape = shape
self._type = type_id
def test_encoder_registration():
# Test low-level registration details for object representation round trip.
# There were some to-dos of things we should check...
...
# Test framework for type creation and automatic registration.
class SpamInstance(BasicSerializable, base_type=('test', 'Spam')):
...
instance = SpamInstance(label='my_spam',
identity=uuid.uuid4().hex,
dtype=['test', 'Spam'],
shape=(1, ),
data=['spam', 'eggs', 'spam', 'spam'])
assert not type(instance) is BasicSerializable
encoded = encode(instance)
decoded = decode(encoded)
assert not type(decoded) is BasicSerializable
assert isinstance(decoded, SpamInstance)
del instance
del decoded
del SpamInstance
import gc
gc.collect()
with pytest.raises(ProtocolError):
decode(encoded)
decode.unregister(TypeIdentifier(('test', 'Spam')))
decoded = decode(encoded)
assert type(decoded) is BasicSerializable
def __init_subclass__(cls, **kwargs):
assert cls is not BasicSerializable
# Handle SCALE-MS Type registration.
base = kwargs.pop('base_type', None)
if base is not None:
typeid = TypeIdentifier.copy_from(base)
else:
typeid = [str(cls.__module__)] + cls.__qualname__.split('.')
registry = BasicSerializable._dtype.base
if cls in registry and registry[cls] is not None:
# This may be a customization or extension point in the future, but not today...
raise ProtocolError(
'Subclassing BasicSerializable for a Type that is already registered.'
)
BasicSerializable._dtype.base[cls] = typeid
# Register encoder for all subclasses. Register the default encoder if not overridden.
# Note: This does not allow us to retain the identity of *cls* for when we call the helpers.
# We may require such information for encoder functions to know why they are being called.
encoder = getattr(cls, 'encode', BasicSerializable.encode)
PythonEncoder.register(cls, encoder)
# Optionally, register a new decoder.
# If no decoder is provided, use the basic decoder.
if hasattr(cls, 'decode') and callable(cls.decode):
_decoder = weakref.WeakMethod(cls.decode)
# Note that we do not require that the decoded object is actually
# an instance of cls.
def _decode(encoded: dict):
decoder = _decoder()
if decoder is None:
raise ProtocolError(
'Decoding a type that has already been de-registered.')
return decoder(encoded)
PythonDecoder.register(cls._dtype, _decode)
# TODO: Register optional instance initializer / input processor.
# Allow instances to be created with something other than a single-argument
# of the registered Input type.
# TODO: Register/generate UI helper.
# From the user's perspective, an importable module function interacts
# with the WorkflowManager to add workflow items and return a handle.
# Do we want to somehow generate an entry-point command
# TODO: Register result dispatcher(s).
# An AbstractDataSource must register a dispatcher to an implementation
# that produces a ConcreteDataSource that provides the registered Result type.
# A ConcreteDataSource must provide support for checksum calculation and verification.
# Optionally, ConcreteDataSource may provide facilities to convert to/from
# native Python objects or other types (such as .npz files).
# Proceed dispatching along the MRO, per documented Python data model.
super().__init_subclass__(**kwargs)
def get_decoder(cls, typeid) -> typing.Union[None, typing.Callable]:
# Normalize the type identifier.
try:
identifier = TypeIdentifier.copy_from(typeid)
typename = identifier.name()
except TypeError:
try:
typename = str(typeid)
except TypeError:
typename = repr(typeid)
identifier = None
# Use the (hashable) normalized form to look up a decoder for dispatching.
if identifier is None or identifier not in cls._dispatchers:
raise TypeError('No decoder registered for {}'.format(typename))
return cls._dispatchers[identifier]
def __init__(self, data, *, dtype, shape=(1, ), label=None, identity=None):
if identity is None:
# TODO: Calculate an appropriate identifier
self.__identity = EphemeralIdentifier()
else:
# TODO: Validate identity
self.__identity = identity
self.__label = str(label)
attrname = BasicSerializable._dtype.attr_name
setattr(self, attrname, TypeIdentifier.copy_from(dtype))
self._shape = Shape(shape)
# TODO: validate data dtype and shape.
# TODO: Ensure that we retain a reference to read-only data.
# TODO: Allow a secondary localized / optimized / implementation-specific version of data.
self.data = data
def decode(cls: typing.Type[ST], encoded: dict) -> ST:
if not isinstance(encoded,
collections.abc.Mapping) or 'type' not in encoded:
raise TypeError(
'Expected a dictionary with a *type* specification for decoding.'
)
dtype = TypeIdentifier.copy_from(encoded['type'])
label = encoded.get('label', None)
identity = encoded.get(
'identity') # TODO: verify and use type schema to decode.
shape = Shape(encoded['shape'])
data = encoded[
'data'] # TODO: use type schema / self._data_decoder to decode.
logger.debug('Decoding {identity} as BasicSerializable.')
return cls(label=label,
identity=identity,
dtype=dtype,
shape=shape,
data=data)
def test_basic_decoding():
# Let the basic encoder/decoder handle things that look like SCALE-MS objects.
encoded = {
'label': None,
'identity': uuid.uuid4().hex,
'type': ['test', 'Spam'],
'shape': [1],
'data': ['spam', 'eggs', 'spam', 'spam']
}
instance = decode(encoded)
assert type(instance) is BasicSerializable
shape_ref = Shape((1, ))
assert instance.shape() == shape_ref
type_ref = TypeIdentifier(('test', 'Spam'))
instance_type = instance.dtype()
assert instance_type == type_ref
# Test basic encoding, too.
assert tuple(instance.encode()['data']) == tuple(encoded['data'])
assert instance.encode() == decode(instance.encode()).encode()
class BasicSerializable(UnboundObject):
__label: typing.Optional[str] = None
__identity: Identifier
_shape: Shape
data: collections.abc.Container
_data_encoder: typing.Callable
_data_decoder: typing.Callable
_dtype = TypeDataDescriptor(
base_type=TypeIdentifier(('scalems', 'BasicSerializable')))
def dtype(self) -> TypeIdentifier:
# Part of the decision of whether to use a property or a method
# is whether we want to normalize on dtype as an instance or class characteristic.
# Initially, we are using inheritance to get registration behavior through metaprogramming.
# In other words, the real question may be how we want to handle registration.
return self._dtype
def __init__(self, data, *, dtype, shape=(1, ), label=None, identity=None):
if identity is None:
# TODO: Calculate an appropriate identifier
self.__identity = EphemeralIdentifier()
else:
# TODO: Validate identity
self.__identity = identity
self.__label = str(label)
attrname = BasicSerializable._dtype.attr_name
setattr(self, attrname, TypeIdentifier.copy_from(dtype))
self._shape = Shape(shape)
# TODO: validate data dtype and shape.
# TODO: Ensure that we retain a reference to read-only data.
# TODO: Allow a secondary localized / optimized / implementation-specific version of data.
self.data = data
def identity(self):
return self.__identity
def label(self):
return str(self.__label)
def shape(self):
return Shape(self._shape)
def encode(self) -> dict:
representation = {
'label': self.label(),
'identity': str(self.identity()),
'type': self.dtype().encode(),
'shape': tuple(self.shape()),
'data': self.data # TODO: use self._data_encoder()
}
return representation
@classmethod
def decode(cls: typing.Type[ST], encoded: dict) -> ST:
if not isinstance(encoded,
collections.abc.Mapping) or 'type' not in encoded:
raise TypeError(
'Expected a dictionary with a *type* specification for decoding.'
)
dtype = TypeIdentifier.copy_from(encoded['type'])
label = encoded.get('label', None)
identity = encoded.get(
'identity') # TODO: verify and use type schema to decode.
shape = Shape(encoded['shape'])
data = encoded[
'data'] # TODO: use type schema / self._data_decoder to decode.
logger.debug('Decoding {identity} as BasicSerializable.')
return cls(label=label,
identity=identity,
dtype=dtype,
shape=shape,
data=data)
def __init_subclass__(cls, **kwargs):
assert cls is not BasicSerializable
# Handle SCALE-MS Type registration.
base = kwargs.pop('base_type', None)
if base is not None:
typeid = TypeIdentifier.copy_from(base)
else:
typeid = [str(cls.__module__)] + cls.__qualname__.split('.')
registry = BasicSerializable._dtype.base
if cls in registry and registry[cls] is not None:
# This may be a customization or extension point in the future, but not today...
raise ProtocolError(
'Subclassing BasicSerializable for a Type that is already registered.'
)
BasicSerializable._dtype.base[cls] = typeid
# Register encoder for all subclasses. Register the default encoder if not overridden.
# Note: This does not allow us to retain the identity of *cls* for when we call the helpers.
# We may require such information for encoder functions to know why they are being called.
encoder = getattr(cls, 'encode', BasicSerializable.encode)
PythonEncoder.register(cls, encoder)
# Optionally, register a new decoder.
# If no decoder is provided, use the basic decoder.
if hasattr(cls, 'decode') and callable(cls.decode):
_decoder = weakref.WeakMethod(cls.decode)
# Note that we do not require that the decoded object is actually
# an instance of cls.
def _decode(encoded: dict):
decoder = _decoder()
if decoder is None:
raise ProtocolError(
'Decoding a type that has already been de-registered.')
return decoder(encoded)
PythonDecoder.register(cls._dtype, _decode)
# TODO: Register optional instance initializer / input processor.
# Allow instances to be created with something other than a single-argument
# of the registered Input type.
# TODO: Register/generate UI helper.
# From the user's perspective, an importable module function interacts
# with the WorkflowManager to add workflow items and return a handle.
# Do we want to somehow generate an entry-point command
# TODO: Register result dispatcher(s).
# An AbstractDataSource must register a dispatcher to an implementation
# that produces a ConcreteDataSource that provides the registered Result type.
# A ConcreteDataSource must provide support for checksum calculation and verification.
# Optionally, ConcreteDataSource may provide facilities to convert to/from
# native Python objects or other types (such as .npz files).
# Proceed dispatching along the MRO, per documented Python data model.
super().__init_subclass__(**kwargs)
def register(cls, typeid: TypeIdentifier, handler: typing.Callable):
# Normalize typeid
typeid = TypeIdentifier.copy_from(typeid)
if typeid in cls._dispatchers:
raise ProtocolError('Type appears to be registered already.')
cls._dispatchers[typeid] = handler
def test_resource_type():
scoped_name = ['scalems', 'subprocess', 'SubprocessTask']
description = scalems.workflow.Description(resource_type=TypeIdentifier(
tuple(scoped_name)),
shape=(1, ))
assert description.type() == TypeIdentifier(tuple(scoped_name))