def encode_np_dtype(v: np.dtype) -> Any:
"""
Specializes :func:`encode` for invocations where ``v`` is an instance of
the :class:`~mxnet.Context` class.
"""
return {
"__kind__": kind_inst,
"class": fqname_for(v.__class__),
"args": encode([v.name]),
}
def encode_pydantic_model(v: BaseModel) -> Any:
"""
Specializes :func:`encode` for invocations where ``v`` is an instance of
the :class:`~BaseModel` class.
"""
return {
"__kind__": kind_inst,
"class": fqname_for(v.__class__),
"kwargs": encode(v.__dict__),
}
def encode_path(v: PurePath) -> Any:
"""
Specializes :func:`encode` for invocations where ``v`` is an instance of
the :class:`~PurePath` class.
"""
return {
"__kind__": kind_inst,
"class": fqname_for(v.__class__),
"args": encode([str(v)]),
}
def assert_serializable(x: transform.Transformation):
t = fqname_for(x.__class__)
y = load_json(dump_json(x))
z = load_code(dump_code(x))
assert dump_json(x) == dump_json(
y
), f"Code serialization for transformer {t} does not work"
assert dump_code(x) == dump_code(
z
), f"JSON serialization for transformer {t} does not work"
def encode_mx_context(v: mx.Context) -> Any:
"""
Specializes :func:`encode` for invocations where ``v`` is an instance of
the :class:`~mxnet.Context` class.
"""
return {
"__kind__": kind_inst,
"class": fqname_for(v.__class__),
"args": encode([v.device_type, v.device_id]),
}
def serialize(self, path: Path) -> None:
with (path / "type.txt").open("w") as fp:
fp.write(fqname_for(self.__class__))
with (path / "version.json").open("w") as fp:
json.dump(
{
"model": self.__version__,
"gluonts": gluonts.__version__
}, fp)
with (path / "predictor.json").open("w") as fp:
print(dump_json(self), file=fp)
def run_train_and_test(
env: TrainEnv, forecaster_type: Type[Union[Estimator, Predictor]]
) -> None:
# train_stats = calculate_dataset_statistics(env.datasets["train"])
# log_metric("train_dataset_stats", train_stats)
forecaster_fq_name = fqname_for(forecaster_type)
forecaster_version = forecaster_type.__version__
logger.info(f"Using gluonts v{gluonts.__version__}")
logger.info(f"Using forecaster {forecaster_fq_name} v{forecaster_version}")
forecaster = forecaster_type.from_inputs(
env.datasets["train"], **env.hyperparameters
)
logger.info(
f"The forecaster can be reconstructed with the following expression: "
f"{dump_code(forecaster)}"
)
logger.info(
"Using the following data channels: "
f"{', '.join(name for name in ['train', 'validation', 'test'] if name in env.datasets)}"
)
if isinstance(forecaster, Predictor):
predictor = forecaster
else:
predictor = run_train(
forecaster=forecaster,
train_dataset=env.datasets["train"],
validation_dataset=env.datasets.get("validation"),
hyperparameters=env.hyperparameters,
)
predictor.serialize(env.path.model)
if "test" in env.datasets:
run_test(env, predictor, env.datasets["test"])
def encode(v: Any) -> Any:
"""
Transforms a value `v` as a serializable intermediate representation (for
example, named tuples are encoded as dictionaries). The intermediate
representation is then recursively traversed and serialized either as
Python code or as JSON string.
This function is decorated with :func:`~functools.singledispatch` and can
be specialized by clients for families of types that are not supported by
the basic implementation (explained below).
Examples
--------
The conversion logic implemented by the basic implementation is used
as a fallback and is best explained by a series of examples.
Lists (as lists).
>>> encode([1, 2.0, '3'])
[1, 2.0, '3']
Tuples (as lists).
>>> encode((1, 2.0, '3'))
[1, 2.0, '3']
Dictionaries (as dictionaries).
>>> encode({'a': 1, 'b': 2.0, 'c': '3'})
{'a': 1, 'b': 2.0, 'c': '3'}
Named tuples (as dictionaries with a ``'__kind__': 'instance'`` member).
>>> from pprint import pprint
>>> from typing import NamedTuple
>>> class ComplexNumber(NamedTuple):
... x: float = 0.0
... y: float = 0.0
>>> pprint(encode(ComplexNumber(4.0, 2.0)))
{'__kind__': 'instance',
'class': 'gluonts.core.serde.ComplexNumber',
'kwargs': {'x': 4.0, 'y': 2.0}}
Classes with a :func:`~gluonts.core.component.validated` initializer (as
dictionaries with a ``'__kind__': 'instance'`` member).
>>> from gluonts.core.component import validated
>>> class ComplexNumber:
... @validated()
... def __init__(self, x: float = 0.0, y: float = 0.0) -> None:
... self.x = x
... self.y = y
>>> pprint(encode(ComplexNumber(4.0, 2.0)))
{'__kind__': 'instance',
'args': [],
'class': 'gluonts.core.serde.ComplexNumber',
'kwargs': {'x': 4.0, 'y': 2.0}}
Classes with a ``__getnewargs_ex__`` magic method (as dictionaries with a
``'__kind__': 'instance'`` member).
>>> from gluonts.core.component import validated
>>> class ComplexNumber:
... def __init__(self, x: float = 0.0, y: float = 0.0) -> None:
... self.x = x
... self.y = y
... def __getnewargs_ex__(self):
... return [], {'x': self.x, 'y': self.y}
>>> pprint(encode(ComplexNumber(4.0, 2.0)))
{'__kind__': 'instance',
'args': [],
'class': 'gluonts.core.serde.ComplexNumber',
'kwargs': {'x': 4.0, 'y': 2.0}}
Types (as dictionaries with a ``'__kind__': 'type' member``).
>>> encode(ComplexNumber)
{'__kind__': 'type', 'class': 'gluonts.core.serde.ComplexNumber'}
Parameters
----------
v
The value to be encoded.
Returns
-------
Any
An encoding of ``v`` that can be serialized to Python code or
JSON string.
See Also
--------
decode
Inverse function.
dump_json
Serializes an object to a JSON string.
dump_code
Serializes an object to a Python code string.
"""
if isinstance(v, type(None)):
return None
if isinstance(v, (float, int, str)):
return v
if np.issubdtype(type(v), np.inexact):
return float(v)
if np.issubdtype(type(v), np.integer):
return int(v)
# we have to check for namedtuples first, to encode them not as plain
# tuples (which would become lists)
if isinstance(v, tuple) and hasattr(v, "_asdict"):
v = cast(NamedTuple, v)
return {
"__kind__": kind_inst,
"class": fqname_for(v.__class__),
"kwargs": encode(v._asdict()),
}
if isinstance(v, (list, set, tuple)):
return list(map(encode, v))
if isinstance(v, dict):
return {k: encode(v) for k, v in v.items()}
if isinstance(v, type):
return {"__kind__": kind_type, "class": fqname_for(v)}
if hasattr(v, "__getnewargs_ex__"):
args, kwargs = v.__getnewargs_ex__() # mypy: ignore
return {
"__kind__": kind_inst,
"class": fqname_for(v.__class__),
"args": encode(args),
"kwargs": encode(kwargs),
}
raise RuntimeError(bad_type_msg.format(fqname_for(v.__class__)))
def log_version(forecaster_type):
name = fqname_for(forecaster_type)
version = forecaster_type.__version__
logger.info(f"Using gluonts v{gluonts.__version__}")
logger.info(f"Using forecaster {name} v{version}")
def encode_from_state(v: Stateful) -> Any:
return {
"__kind__": Kind.Stateful,
"class": fqname_for(v.__class__),
"kwargs": encode(v.__dict__),
}
def encode(v: Any) -> Any:
"""
Transforms a value `v` as a serializable intermediate representation (for
example, named tuples are encoded as dictionaries). The intermediate
representation is then recursively traversed and serialized either as
Python code or as JSON string.
This function is decorated with :func:`~functools.singledispatch` and can
be specialized by clients for families of types that are not supported by
the basic implementation (explained below).
Examples
--------
The conversion logic implemented by the basic implementation is used
as a fallback and is best explained by a series of examples.
Lists (as lists).
>>> encode([1, 2.0, '3'])
[1, 2.0, '3']
Dictionaries (as dictionaries).
>>> encode({'a': 1, 'b': 2.0, 'c': '3'})
{'a': 1, 'b': 2.0, 'c': '3'}
Named tuples (as dictionaries with a
``'__kind__': <Kind.Instance: 'instance'>`` member).
>>> from pprint import pprint
>>> from typing import NamedTuple
>>> class ComplexNumber(NamedTuple):
... x: float = 0.0
... y: float = 0.0
>>> pprint(encode(ComplexNumber(4.0, 2.0)))
{'__kind__': <Kind.Instance: 'instance'>,
'class': 'gluonts.core.serde._base.ComplexNumber',
'kwargs': {'x': 4.0, 'y': 2.0}}
Classes with a :func:`~gluonts.core.component.validated` initializer (as
dictionaries with a ``'__kind__': <Kind.Instance: 'instance'>`` member).
>>> from gluonts.core.component import validated
>>> class ComplexNumber:
... @validated()
... def __init__(self, x: float = 0.0, y: float = 0.0) -> None:
... self.x = x
... self.y = y
>>> pprint(encode(ComplexNumber(4.0, 2.0)))
{'__kind__': <Kind.Instance: 'instance'>,
'args': [],
'class': 'gluonts.core.serde._base.ComplexNumber',
'kwargs': {'x': 4.0, 'y': 2.0}}
Classes with a ``__getnewargs_ex__`` magic method (as dictionaries with a
``'__kind__': <Kind.Instance: 'instance'>`` member).
>>> from gluonts.core.component import validated
>>> class ComplexNumber:
... def __init__(self, x: float = 0.0, y: float = 0.0) -> None:
... self.x = x
... self.y = y
... def __getnewargs_ex__(self):
... return [], {'x': self.x, 'y': self.y}
>>> pprint(encode(ComplexNumber(4.0, 2.0)))
{'__kind__': <Kind.Instance: 'instance'>,
'args': [],
'class': 'gluonts.core.serde._base.ComplexNumber',
'kwargs': {'x': 4.0, 'y': 2.0}}
Types (as dictionaries with a ``'__kind__': <Kind.Type: 'type'> member``).
>>> encode(ComplexNumber)
{'__kind__': <Kind.Type: 'type'>,
'class': 'gluonts.core.serde._base.ComplexNumber'}
Parameters
----------
v
The value to be encoded.
Returns
-------
Any
An encoding of ``v`` that can be serialized to Python code or
JSON string.
See Also
--------
decode
Inverse function.
dump_json
Serializes an object to a JSON string.
dump_code
Serializes an object to a Python code string.
"""
if v is None:
return None
if isinstance(v, (float, int, str)):
return v
# check for namedtuples first, to encode them not as plain tuples
if isinstance(v, tuple) and hasattr(v, "_asdict"):
v = cast(NamedTuple, v)
return {
"__kind__": Kind.Instance,
"class": fqname_for(v.__class__),
"kwargs": encode(v._asdict()),
}
if isinstance(v, (tuple, set)):
return {
"__kind__": Kind.Instance,
"class": fqname_for(type(v)),
"args": [list(map(encode, v))],
}
if isinstance(v, list):
return list(map(encode, v))
if isinstance(v, dict):
return valmap(encode, v)
if isinstance(v, type):
return {"__kind__": Kind.Type, "class": fqname_for(v)}
if hasattr(v, "__getnewargs_ex__"):
args, kwargs = v.__getnewargs_ex__() # mypy: ignore
return {
"__kind__": Kind.Instance,
"class": fqname_for(v.__class__),
# args need to be a list, since we encode tuples explicitly
"args": encode(list(args)),
"kwargs": encode(kwargs),
}
try:
# as fallback, we try to just take the path of the value
fqname = fqname_for(v)
assert ("<lambda>"
not in fqname), f"Can't serialize lambda function {fqname}"
if hasattr(v, "__self__") and hasattr(v, "__func__"):
# v is a method
# to model`obj.method`, we encode `getattr(obj, "method")`
return {
"__kind__": Kind.Instance,
"class": fqname_for(getattr),
"args": encode((v.__self__, v.__func__.__name__)),
}
return {"__kind__": Kind.Type, "class": fqname_for(v)}
except AttributeError:
pass
raise RuntimeError(bad_type_msg.format(fqname_for(v.__class__)))
