def encode_np_dtype(v: np.dtype) -> Any: """ Specializes :func:`encode` for invocations where ``v`` is an instance of the :class:`~mxnet.Context` class. """ return { "__kind__": kind_inst, "class": fqname_for(v.__class__), "args": encode([v.name]), }
def encode_pydantic_model(v: BaseModel) -> Any: """ Specializes :func:`encode` for invocations where ``v`` is an instance of the :class:`~BaseModel` class. """ return { "__kind__": kind_inst, "class": fqname_for(v.__class__), "kwargs": encode(v.__dict__), }
def encode_path(v: PurePath) -> Any: """ Specializes :func:`encode` for invocations where ``v`` is an instance of the :class:`~PurePath` class. """ return { "__kind__": kind_inst, "class": fqname_for(v.__class__), "args": encode([str(v)]), }
def assert_serializable(x: transform.Transformation): t = fqname_for(x.__class__) y = load_json(dump_json(x)) z = load_code(dump_code(x)) assert dump_json(x) == dump_json( y ), f"Code serialization for transformer {t} does not work" assert dump_code(x) == dump_code( z ), f"JSON serialization for transformer {t} does not work"
def encode_mx_context(v: mx.Context) -> Any: """ Specializes :func:`encode` for invocations where ``v`` is an instance of the :class:`~mxnet.Context` class. """ return { "__kind__": kind_inst, "class": fqname_for(v.__class__), "args": encode([v.device_type, v.device_id]), }
def serialize(self, path: Path) -> None: with (path / "type.txt").open("w") as fp: fp.write(fqname_for(self.__class__)) with (path / "version.json").open("w") as fp: json.dump( { "model": self.__version__, "gluonts": gluonts.__version__ }, fp) with (path / "predictor.json").open("w") as fp: print(dump_json(self), file=fp)
def run_train_and_test( env: TrainEnv, forecaster_type: Type[Union[Estimator, Predictor]] ) -> None: # train_stats = calculate_dataset_statistics(env.datasets["train"]) # log_metric("train_dataset_stats", train_stats) forecaster_fq_name = fqname_for(forecaster_type) forecaster_version = forecaster_type.__version__ logger.info(f"Using gluonts v{gluonts.__version__}") logger.info(f"Using forecaster {forecaster_fq_name} v{forecaster_version}") forecaster = forecaster_type.from_inputs( env.datasets["train"], **env.hyperparameters ) logger.info( f"The forecaster can be reconstructed with the following expression: " f"{dump_code(forecaster)}" ) logger.info( "Using the following data channels: " f"{', '.join(name for name in ['train', 'validation', 'test'] if name in env.datasets)}" ) if isinstance(forecaster, Predictor): predictor = forecaster else: predictor = run_train( forecaster=forecaster, train_dataset=env.datasets["train"], validation_dataset=env.datasets.get("validation"), hyperparameters=env.hyperparameters, ) predictor.serialize(env.path.model) if "test" in env.datasets: run_test(env, predictor, env.datasets["test"])
def encode(v: Any) -> Any: """ Transforms a value `v` as a serializable intermediate representation (for example, named tuples are encoded as dictionaries). The intermediate representation is then recursively traversed and serialized either as Python code or as JSON string. This function is decorated with :func:`~functools.singledispatch` and can be specialized by clients for families of types that are not supported by the basic implementation (explained below). Examples -------- The conversion logic implemented by the basic implementation is used as a fallback and is best explained by a series of examples. Lists (as lists). >>> encode([1, 2.0, '3']) [1, 2.0, '3'] Tuples (as lists). >>> encode((1, 2.0, '3')) [1, 2.0, '3'] Dictionaries (as dictionaries). >>> encode({'a': 1, 'b': 2.0, 'c': '3'}) {'a': 1, 'b': 2.0, 'c': '3'} Named tuples (as dictionaries with a ``'__kind__': 'instance'`` member). >>> from pprint import pprint >>> from typing import NamedTuple >>> class ComplexNumber(NamedTuple): ... x: float = 0.0 ... y: float = 0.0 >>> pprint(encode(ComplexNumber(4.0, 2.0))) {'__kind__': 'instance', 'class': 'gluonts.core.serde.ComplexNumber', 'kwargs': {'x': 4.0, 'y': 2.0}} Classes with a :func:`~gluonts.core.component.validated` initializer (as dictionaries with a ``'__kind__': 'instance'`` member). >>> from gluonts.core.component import validated >>> class ComplexNumber: ... @validated() ... def __init__(self, x: float = 0.0, y: float = 0.0) -> None: ... self.x = x ... self.y = y >>> pprint(encode(ComplexNumber(4.0, 2.0))) {'__kind__': 'instance', 'args': [], 'class': 'gluonts.core.serde.ComplexNumber', 'kwargs': {'x': 4.0, 'y': 2.0}} Classes with a ``__getnewargs_ex__`` magic method (as dictionaries with a ``'__kind__': 'instance'`` member). >>> from gluonts.core.component import validated >>> class ComplexNumber: ... def __init__(self, x: float = 0.0, y: float = 0.0) -> None: ... self.x = x ... self.y = y ... def __getnewargs_ex__(self): ... return [], {'x': self.x, 'y': self.y} >>> pprint(encode(ComplexNumber(4.0, 2.0))) {'__kind__': 'instance', 'args': [], 'class': 'gluonts.core.serde.ComplexNumber', 'kwargs': {'x': 4.0, 'y': 2.0}} Types (as dictionaries with a ``'__kind__': 'type' member``). >>> encode(ComplexNumber) {'__kind__': 'type', 'class': 'gluonts.core.serde.ComplexNumber'} Parameters ---------- v The value to be encoded. Returns ------- Any An encoding of ``v`` that can be serialized to Python code or JSON string. See Also -------- decode Inverse function. dump_json Serializes an object to a JSON string. dump_code Serializes an object to a Python code string. """ if isinstance(v, type(None)): return None if isinstance(v, (float, int, str)): return v if np.issubdtype(type(v), np.inexact): return float(v) if np.issubdtype(type(v), np.integer): return int(v) # we have to check for namedtuples first, to encode them not as plain # tuples (which would become lists) if isinstance(v, tuple) and hasattr(v, "_asdict"): v = cast(NamedTuple, v) return { "__kind__": kind_inst, "class": fqname_for(v.__class__), "kwargs": encode(v._asdict()), } if isinstance(v, (list, set, tuple)): return list(map(encode, v)) if isinstance(v, dict): return {k: encode(v) for k, v in v.items()} if isinstance(v, type): return {"__kind__": kind_type, "class": fqname_for(v)} if hasattr(v, "__getnewargs_ex__"): args, kwargs = v.__getnewargs_ex__() # mypy: ignore return { "__kind__": kind_inst, "class": fqname_for(v.__class__), "args": encode(args), "kwargs": encode(kwargs), } raise RuntimeError(bad_type_msg.format(fqname_for(v.__class__)))
def log_version(forecaster_type): name = fqname_for(forecaster_type) version = forecaster_type.__version__ logger.info(f"Using gluonts v{gluonts.__version__}") logger.info(f"Using forecaster {name} v{version}")
def encode_from_state(v: Stateful) -> Any: return { "__kind__": Kind.Stateful, "class": fqname_for(v.__class__), "kwargs": encode(v.__dict__), }
def encode(v: Any) -> Any: """ Transforms a value `v` as a serializable intermediate representation (for example, named tuples are encoded as dictionaries). The intermediate representation is then recursively traversed and serialized either as Python code or as JSON string. This function is decorated with :func:`~functools.singledispatch` and can be specialized by clients for families of types that are not supported by the basic implementation (explained below). Examples -------- The conversion logic implemented by the basic implementation is used as a fallback and is best explained by a series of examples. Lists (as lists). >>> encode([1, 2.0, '3']) [1, 2.0, '3'] Dictionaries (as dictionaries). >>> encode({'a': 1, 'b': 2.0, 'c': '3'}) {'a': 1, 'b': 2.0, 'c': '3'} Named tuples (as dictionaries with a ``'__kind__': <Kind.Instance: 'instance'>`` member). >>> from pprint import pprint >>> from typing import NamedTuple >>> class ComplexNumber(NamedTuple): ... x: float = 0.0 ... y: float = 0.0 >>> pprint(encode(ComplexNumber(4.0, 2.0))) {'__kind__': <Kind.Instance: 'instance'>, 'class': 'gluonts.core.serde._base.ComplexNumber', 'kwargs': {'x': 4.0, 'y': 2.0}} Classes with a :func:`~gluonts.core.component.validated` initializer (as dictionaries with a ``'__kind__': <Kind.Instance: 'instance'>`` member). >>> from gluonts.core.component import validated >>> class ComplexNumber: ... @validated() ... def __init__(self, x: float = 0.0, y: float = 0.0) -> None: ... self.x = x ... self.y = y >>> pprint(encode(ComplexNumber(4.0, 2.0))) {'__kind__': <Kind.Instance: 'instance'>, 'args': [], 'class': 'gluonts.core.serde._base.ComplexNumber', 'kwargs': {'x': 4.0, 'y': 2.0}} Classes with a ``__getnewargs_ex__`` magic method (as dictionaries with a ``'__kind__': <Kind.Instance: 'instance'>`` member). >>> from gluonts.core.component import validated >>> class ComplexNumber: ... def __init__(self, x: float = 0.0, y: float = 0.0) -> None: ... self.x = x ... self.y = y ... def __getnewargs_ex__(self): ... return [], {'x': self.x, 'y': self.y} >>> pprint(encode(ComplexNumber(4.0, 2.0))) {'__kind__': <Kind.Instance: 'instance'>, 'args': [], 'class': 'gluonts.core.serde._base.ComplexNumber', 'kwargs': {'x': 4.0, 'y': 2.0}} Types (as dictionaries with a ``'__kind__': <Kind.Type: 'type'> member``). >>> encode(ComplexNumber) {'__kind__': <Kind.Type: 'type'>, 'class': 'gluonts.core.serde._base.ComplexNumber'} Parameters ---------- v The value to be encoded. Returns ------- Any An encoding of ``v`` that can be serialized to Python code or JSON string. See Also -------- decode Inverse function. dump_json Serializes an object to a JSON string. dump_code Serializes an object to a Python code string. """ if v is None: return None if isinstance(v, (float, int, str)): return v # check for namedtuples first, to encode them not as plain tuples if isinstance(v, tuple) and hasattr(v, "_asdict"): v = cast(NamedTuple, v) return { "__kind__": Kind.Instance, "class": fqname_for(v.__class__), "kwargs": encode(v._asdict()), } if isinstance(v, (tuple, set)): return { "__kind__": Kind.Instance, "class": fqname_for(type(v)), "args": [list(map(encode, v))], } if isinstance(v, list): return list(map(encode, v)) if isinstance(v, dict): return valmap(encode, v) if isinstance(v, type): return {"__kind__": Kind.Type, "class": fqname_for(v)} if hasattr(v, "__getnewargs_ex__"): args, kwargs = v.__getnewargs_ex__() # mypy: ignore return { "__kind__": Kind.Instance, "class": fqname_for(v.__class__), # args need to be a list, since we encode tuples explicitly "args": encode(list(args)), "kwargs": encode(kwargs), } try: # as fallback, we try to just take the path of the value fqname = fqname_for(v) assert ("<lambda>" not in fqname), f"Can't serialize lambda function {fqname}" if hasattr(v, "__self__") and hasattr(v, "__func__"): # v is a method # to model`obj.method`, we encode `getattr(obj, "method")` return { "__kind__": Kind.Instance, "class": fqname_for(getattr), "args": encode((v.__self__, v.__func__.__name__)), } return {"__kind__": Kind.Type, "class": fqname_for(v)} except AttributeError: pass raise RuntimeError(bad_type_msg.format(fqname_for(v.__class__)))