def serialize(self, path: Path) -> None: # call Predictor.serialize() in order to serialize the class name super().serialize(path) # serialize every GluonPredictor-specific parameters try: # serialize the prediction network self.serialize_prediction_net(path) # serialize transformation chain with (path / "input_transform.json").open("w") as fp: print(dump_json(self.input_transform), file=fp) # FIXME: also needs to serialize the output_transform # serialize all remaining constructor parameters with (path / "parameters.json").open("w") as fp: parameters = dict( batch_size=self.batch_size, prediction_length=self.prediction_length, freq=self.freq, ctx=self.ctx, float_type=self.float_type, forecast_cls_name=self.forecast_cls_name, forecast_kwargs=self.forecast_kwargs, input_names=self.input_names, ) print(dump_json(parameters), file=fp) except Exception as e: raise IOError( f"Cannot serialize {fqname_for(self.__class__)}") from e
def serialize(self, path: Path) -> None: super().serialize(path) # serialize network with (path / f"prediction_net.json").open("w") as fp: print(dump_json(self.prediction_net), file=fp) torch.save(self.prediction_net.state_dict(), path / "prediction_net_state") # serialize transformation chain with (path / "input_transform.json").open("w") as fp: print(dump_json(self.input_transform), file=fp) # FIXME: also needs to serialize the output_transform # serialize all remaining constructor parameters with (path / "parameters.json").open("w") as fp: parameters = dict( batch_size=self.batch_size, prediction_length=self.prediction_length, freq=self.freq, forecast_generator=self.forecast_generator, input_names=self.input_names, ) print(dump_json(parameters), file=fp)
def serialize(self, path: Path) -> None: # call Predictor.serialize() in order to serialize the class name super().serialize(path) # serialize every GluonPredictor-specific parameters # serialize the prediction network self.serialize_prediction_net(path) # serialize transformation chain with (path / "input_transform.json").open("w") as fp: print(dump_json(self.input_transform), file=fp) # FIXME: also needs to serialize the output_transform # serialize all remaining constructor parameters with (path / "parameters.json").open("w") as fp: parameters = dict( batch_size=self.batch_size, prediction_length=self.prediction_length, freq=self.freq, ctx=self.ctx, dtype=self.dtype, forecast_generator=self.forecast_generator, input_names=self.input_names, ) print(dump_json(parameters), file=fp)
def assert_serializable(x: transform.Transformation): t = fqname_for(x.__class__) y = load_json(dump_json(x)) z = load_code(dump_code(x)) assert dump_json(x) == dump_json( y), f"Code serialization for transformer {t} does not work" assert dump_code(x) == dump_code( z), f"JSON serialization for transformer {t} does not work"
def test_component_ctor(): random.seed(5_432_671_244) A = 100 B = 200 C = 300 x_list = [ Foo( str(random.randint(0, A)), Complex(x=random.uniform(0, C), y=str(random.uniform(0, C))), b=random.uniform(0, B), ) for i in range(4) ] fields = [ Foo( a=str(random.randint(0, A)), b=random.uniform(0, B), c=Complex(x=str(random.uniform(0, C)), y=random.uniform(0, C)), ) for i in range(5) ] x_dict = { i: Foo( b=random.uniform(0, B), a=str(random.randint(0, A)), c=Complex(x=str(random.uniform(0, C)), y=str(random.uniform(0, C))), ) for i in range(6) } bar01 = Bar(x_list, fields=fields, x_dict=x_dict) bar02 = load_code(dump_code(bar01)) bar03 = load_json(dump_json(bar02)) def compare_tpes(x, y, z, tpe): assert tpe == type(x) == type(y) == type(z) def compare_vals(x, y, z): assert x == y == z compare_tpes(bar02.x_list, bar02.x_list, bar03.x_list, tpe=list) compare_tpes(bar02.x_dict, bar02.x_dict, bar03.x_dict, tpe=dict) compare_tpes(bar02.fields, bar02.fields, bar03.fields, tpe=list) compare_vals(len(bar02.x_list), len(bar02.x_list), len(bar03.x_list)) compare_vals(len(bar02.x_dict), len(bar02.x_dict), len(bar03.x_dict)) compare_vals(len(bar02.fields), len(bar02.fields), len(bar03.fields)) compare_vals(bar02.x_list, bar02.x_list, bar03.x_list) compare_vals(bar02.x_dict, bar02.x_dict, bar03.x_dict) compare_vals(bar02.fields, bar02.fields, bar03.fields) baz01 = Baz(a="0", b="9", c=Complex(x="1", y="2"), d="42") baz02 = load_json(dump_json(baz01)) assert type(baz01) == type(baz02) assert baz01 == baz02
def test_train_script(dataset_name, custom_dataset): # we need to write some data for this test, so we use a temporary directory with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir) dataset = get_dataset( dataset_name, path=temp_dir_path, regenerate=True ) # exchange_rate, m4_yearly # either use provided dataset, in which case it must be present in the directory, or a built in one # for testing we will provide a built in dataset as a custom one too if custom_dataset: args = create_arguments( str(temp_dir_path), dataset_name, s3_dataset_path=str(temp_dir_path / dataset_name), ) else: args = create_arguments(str(temp_dir_path), dataset_name) # the test requires using a deserialized estimator, which we first need to create estimator_cls, hyperparameters = simple_feedforward_estimator() estimator = estimator_cls.from_hyperparameters( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.freq, **hyperparameters ) serialized = serde.dump_json(estimator) with open(temp_dir_path / "estimator.json", "w") as estimator_file: estimator_file.write(serialized) # No assert necessary, the idea is just that the code needs to run through train(args)
def export_symb_block( hb: mx.gluon.HybridBlock, model_dir: Path, model_name: str, epoch: int = 0 ) -> None: """ Serializes a hybridized Gluon `HybridBlock`. Parameters ---------- hb The block to export. model_dir The path where the model will be saved. model_name The name identifying the model. epoch The epoch number, which together with the `model_name` identifies the model parameters. """ hb.export(path=str(model_dir / model_name), epoch=epoch) # FIXME: we persist input/output formats of hybrid blocks as mxnet does not # FIXME: https://github.com/apache/incubator-mxnet/issues/17488 with (model_dir / f"{model_name}-in_out_format.json").open("w") as fp: in_out_format = dict( in_format=hb._in_format, out_format=hb._out_format ) print(dump_json(in_out_format), file=fp)
def serialize(self, path: Path) -> None: # call Predictor.serialize() in order to serialize the class name super().serialize(path) # serialize self.ag_model # move autogluon model to where we want to do the serialization ag_path = self.ag_model.path shutil.move(ag_path, path) ag_path = Path(ag_path) print(f"Autogluon files moved from {ag_path} to {path}.") # reset the path stored in tabular model. AutogluonTabularPredictor.load(path / Path(ag_path.name)) # serialize all remaining constructor parameters with (path / "parameters.json").open("w") as fp: parameters = dict( batch_size=self.batch_size, prediction_length=self.prediction_length, freq=self.freq, dtype=self.dtype, time_features=self.time_features, lag_indices=self.lag_indices, ag_path=path / Path(ag_path.name), ) print(dump_json(parameters), file=fp)
def _upload_estimator(self, locations, estimator): logger.info("Uploading estimator config to s3.") serialized = serde.dump_json(estimator) with self._s3fs.open(locations.estimator_path, "w") as estimator_file: estimator_file.write(serialized)
def encode_sagemaker_parameter(value: Any) -> str: """ All values passed through the SageMaker API must be encoded as strings. """ if not isinstance(value, str): return dump_json(value) else: return value
def serialize(self, path: Path) -> None: # call Predictor.serialize() in order to serialize the class name super().serialize(path) try: with (path / "predictor.json").open("w") as fp: print(dump_json(self), file=fp) except Exception as e: raise IOError( f"Cannot serialize {fqname_for(self.__class__)}") from e
def serialize(self, path: Path) -> None: with (path / "type.txt").open("w") as fp: fp.write(fqname_for(self.__class__)) with (path / "version.json").open("w") as fp: json.dump( {"model": self.__version__, "gluonts": gluonts.__version__}, fp ) with (path / "predictor.json").open("w") as fp: print(dump_json(self), file=fp)
def serialize(self, path: Path) -> None: # serialize some metadata super().serialize(path) # basically save each predictor in its own sub-folder num_digits = len(str(len(self.predictors))) for index, predictor in enumerate(self.predictors): composite_path = path / f"predictor_{str(index).zfill(num_digits)}" os.makedirs(str(composite_path)) predictor.serialize(composite_path) # serialize all remaining constructor parameters with (path / "parameters.json").open("w") as fp: parameters = dict( prediction_length=self.prediction_length, freq=self.freq, aggregation_method=self.aggregation_method, num_predictors=len(self.predictors), ) print(dump_json(parameters), file=fp)
def export_repr_block( rb: mx.gluon.HybridBlock, model_dir: Path, model_name: str, epoch: int = 0 ) -> None: """ Serializes a representable Gluon block. Parameters ---------- rb The block to export. model_dir The path where the model will be saved. model_name The name identifying the model. epoch The epoch number, which together with the `model_name` identifies the model parameters. """ with (model_dir / f"{model_name}-network.json").open("w") as fp: print(dump_json(rb), file=fp) rb.save_parameters(str(model_dir / f"{model_name}-{epoch:04}.params"))
def export_symb_block(hb: mx.gluon.HybridBlock, model_dir: Path, model_name: str, epoch: int = 0) -> None: """ Serializes a hybridized Gluon `HybridBlock`. Parameters ---------- hb The block to export. model_dir The path where the model will be saved. model_name The name identifying the model. epoch The epoch number, which together with the `model_name` identifies the model parameters. """ hb.export(path=str(model_dir / model_name), epoch=epoch) with (model_dir / f"{model_name}-in_out_format.json").open("w") as fp: in_out_format = dict(in_format=hb._in_format, out_format=hb._out_format) print(dump_json(in_out_format), file=fp)
def serialize(self, path: Path) -> None: # call Predictor.serialize() in order to serialize the class name super().serialize(path) with (path / "predictor.json").open("w") as fp: print(dump_json(self), file=fp)
from gluonts.model.common import Tensor, NPArrayLike from gluonts.mx.distribution.distribution import Distribution from gluonts.mx.distribution import ( Gaussian, StudentT, MixtureDistribution, GaussianOutput, StudentTOutput, LaplaceOutput, MultivariateGaussianOutput, MixtureDistributionOutput, ) from gluonts.testutil import empirical_cdf from gluonts.core.serde import dump_json, load_json serialize_fn_list = [lambda x: x, lambda x: load_json(dump_json(x))] def plot_samples(s: Tensor, bins: int = 100) -> None: from matplotlib import pyplot as plt s = s.asnumpy() plt.hist(s, bins=bins) plt.show() BINS = np.linspace(-5, 5, 100) def histogram(samples: NPArrayLike) -> np.ndarray: h, _ = np.histogram(samples, bins=BINS, density=True)
def test_json_serialization(e) -> None: expected, actual = e, serde.load_json(serde.dump_json(e)) assert check_equality(expected, actual)
@pytest.mark.parametrize( "a", [ mx.nd.random.uniform(shape=(3, 5, 2), dtype="float16"), mx.nd.random.uniform(shape=(3, 5, 2), dtype="float32"), mx.nd.random.uniform(shape=(3, 5, 2), dtype="float64"), mx.nd.array([[1, 2, 3], [-1, -2, 0]], dtype=np.uint8), mx.nd.array([[1, 2, 3], [-1, -2, 0]], dtype=np.int32), mx.nd.array([[1, 2, 3], [-1, -2, 0]], dtype=np.int64), mx.nd.array([[1, 2, 3], [1, 2, 0]], dtype=np.uint8), ], ) @pytest.mark.parametrize( "serialize_fn", [ lambda x: serde.load_json(serde.dump_json(x)), lambda x: serde.load_binary(serde.dump_binary(x)), lambda x: serde.load_code(serde.dump_code(x)), ], ) def test_ndarray_serialization(a, serialize_fn) -> None: b = serialize_fn(a) assert type(a) == type(b) assert a.dtype == b.dtype assert a.shape == b.shape assert np.all((a == b).asnumpy()) def test_timestamp_encode_decode() -> None: now = pd.Timestamp.now() assert now == serde.decode(serde.encode(now))
def test_json_serialization(e) -> None: assert equals(e, serde.load_json(serde.dump_json(e)))
def test_json_serialization(e) -> None: assert e == serde.load_json(serde.dump_json(e))
def test_distribution_output_serde(distr_output: DistributionOutput): distr_output_copy = decode(encode(distr_output)) assert isinstance(distr_output_copy, type(distr_output)) assert dump_json(distr_output_copy) == dump_json(distr_output)