def serialize(self, path: Path) -> None:
# call Predictor.serialize() in order to serialize the class name
super().serialize(path)
# serialize every GluonPredictor-specific parameters
try:
# serialize the prediction network
self.serialize_prediction_net(path)
# serialize transformation chain
with (path / "input_transform.json").open("w") as fp:
print(dump_json(self.input_transform), file=fp)
# FIXME: also needs to serialize the output_transform
# serialize all remaining constructor parameters
with (path / "parameters.json").open("w") as fp:
parameters = dict(
batch_size=self.batch_size,
prediction_length=self.prediction_length,
freq=self.freq,
ctx=self.ctx,
float_type=self.float_type,
forecast_cls_name=self.forecast_cls_name,
forecast_kwargs=self.forecast_kwargs,
input_names=self.input_names,
)
print(dump_json(parameters), file=fp)
except Exception as e:
raise IOError(
f"Cannot serialize {fqname_for(self.__class__)}") from e
def serialize(self, path: Path) -> None:
super().serialize(path)
# serialize network
with (path / f"prediction_net.json").open("w") as fp:
print(dump_json(self.prediction_net), file=fp)
torch.save(self.prediction_net.state_dict(),
path / "prediction_net_state")
# serialize transformation chain
with (path / "input_transform.json").open("w") as fp:
print(dump_json(self.input_transform), file=fp)
# FIXME: also needs to serialize the output_transform
# serialize all remaining constructor parameters
with (path / "parameters.json").open("w") as fp:
parameters = dict(
batch_size=self.batch_size,
prediction_length=self.prediction_length,
freq=self.freq,
forecast_generator=self.forecast_generator,
input_names=self.input_names,
)
print(dump_json(parameters), file=fp)
def serialize(self, path: Path) -> None:
# call Predictor.serialize() in order to serialize the class name
super().serialize(path)
# serialize every GluonPredictor-specific parameters
# serialize the prediction network
self.serialize_prediction_net(path)
# serialize transformation chain
with (path / "input_transform.json").open("w") as fp:
print(dump_json(self.input_transform), file=fp)
# FIXME: also needs to serialize the output_transform
# serialize all remaining constructor parameters
with (path / "parameters.json").open("w") as fp:
parameters = dict(
batch_size=self.batch_size,
prediction_length=self.prediction_length,
freq=self.freq,
ctx=self.ctx,
dtype=self.dtype,
forecast_generator=self.forecast_generator,
input_names=self.input_names,
)
print(dump_json(parameters), file=fp)
def assert_serializable(x: transform.Transformation):
t = fqname_for(x.__class__)
y = load_json(dump_json(x))
z = load_code(dump_code(x))
assert dump_json(x) == dump_json(
y), f"Code serialization for transformer {t} does not work"
assert dump_code(x) == dump_code(
z), f"JSON serialization for transformer {t} does not work"
def test_component_ctor():
random.seed(5_432_671_244)
A = 100
B = 200
C = 300
x_list = [
Foo(
str(random.randint(0, A)),
Complex(x=random.uniform(0, C), y=str(random.uniform(0, C))),
b=random.uniform(0, B),
) for i in range(4)
]
fields = [
Foo(
a=str(random.randint(0, A)),
b=random.uniform(0, B),
c=Complex(x=str(random.uniform(0, C)), y=random.uniform(0, C)),
) for i in range(5)
]
x_dict = {
i: Foo(
b=random.uniform(0, B),
a=str(random.randint(0, A)),
c=Complex(x=str(random.uniform(0, C)), y=str(random.uniform(0,
C))),
)
for i in range(6)
}
bar01 = Bar(x_list, fields=fields, x_dict=x_dict)
bar02 = load_code(dump_code(bar01))
bar03 = load_json(dump_json(bar02))
def compare_tpes(x, y, z, tpe):
assert tpe == type(x) == type(y) == type(z)
def compare_vals(x, y, z):
assert x == y == z
compare_tpes(bar02.x_list, bar02.x_list, bar03.x_list, tpe=list)
compare_tpes(bar02.x_dict, bar02.x_dict, bar03.x_dict, tpe=dict)
compare_tpes(bar02.fields, bar02.fields, bar03.fields, tpe=list)
compare_vals(len(bar02.x_list), len(bar02.x_list), len(bar03.x_list))
compare_vals(len(bar02.x_dict), len(bar02.x_dict), len(bar03.x_dict))
compare_vals(len(bar02.fields), len(bar02.fields), len(bar03.fields))
compare_vals(bar02.x_list, bar02.x_list, bar03.x_list)
compare_vals(bar02.x_dict, bar02.x_dict, bar03.x_dict)
compare_vals(bar02.fields, bar02.fields, bar03.fields)
baz01 = Baz(a="0", b="9", c=Complex(x="1", y="2"), d="42")
baz02 = load_json(dump_json(baz01))
assert type(baz01) == type(baz02)
assert baz01 == baz02
def test_train_script(dataset_name, custom_dataset):
# we need to write some data for this test, so we use a temporary directory
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir)
dataset = get_dataset(
dataset_name, path=temp_dir_path, regenerate=True
) # exchange_rate, m4_yearly
# either use provided dataset, in which case it must be present in the directory, or a built in one
# for testing we will provide a built in dataset as a custom one too
if custom_dataset:
args = create_arguments(
str(temp_dir_path),
dataset_name,
s3_dataset_path=str(temp_dir_path / dataset_name),
)
else:
args = create_arguments(str(temp_dir_path), dataset_name)
# the test requires using a deserialized estimator, which we first need to create
estimator_cls, hyperparameters = simple_feedforward_estimator()
estimator = estimator_cls.from_hyperparameters(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
**hyperparameters
)
serialized = serde.dump_json(estimator)
with open(temp_dir_path / "estimator.json", "w") as estimator_file:
estimator_file.write(serialized)
# No assert necessary, the idea is just that the code needs to run through
train(args)
def export_symb_block(
hb: mx.gluon.HybridBlock, model_dir: Path, model_name: str, epoch: int = 0
) -> None:
"""
Serializes a hybridized Gluon `HybridBlock`.
Parameters
----------
hb
The block to export.
model_dir
The path where the model will be saved.
model_name
The name identifying the model.
epoch
The epoch number, which together with the `model_name` identifies the
model parameters.
"""
hb.export(path=str(model_dir / model_name), epoch=epoch)
# FIXME: we persist input/output formats of hybrid blocks as mxnet does not
# FIXME: https://github.com/apache/incubator-mxnet/issues/17488
with (model_dir / f"{model_name}-in_out_format.json").open("w") as fp:
in_out_format = dict(
in_format=hb._in_format, out_format=hb._out_format
)
print(dump_json(in_out_format), file=fp)
def serialize(self, path: Path) -> None:
# call Predictor.serialize() in order to serialize the class name
super().serialize(path)
# serialize self.ag_model
# move autogluon model to where we want to do the serialization
ag_path = self.ag_model.path
shutil.move(ag_path, path)
ag_path = Path(ag_path)
print(f"Autogluon files moved from {ag_path} to {path}.")
# reset the path stored in tabular model.
AutogluonTabularPredictor.load(path / Path(ag_path.name))
# serialize all remaining constructor parameters
with (path / "parameters.json").open("w") as fp:
parameters = dict(
batch_size=self.batch_size,
prediction_length=self.prediction_length,
freq=self.freq,
dtype=self.dtype,
time_features=self.time_features,
lag_indices=self.lag_indices,
ag_path=path / Path(ag_path.name),
)
print(dump_json(parameters), file=fp)
def _upload_estimator(self, locations, estimator):
logger.info("Uploading estimator config to s3.")
serialized = serde.dump_json(estimator)
with self._s3fs.open(locations.estimator_path, "w") as estimator_file:
estimator_file.write(serialized)
def encode_sagemaker_parameter(value: Any) -> str:
"""
All values passed through the SageMaker API must be encoded as strings.
"""
if not isinstance(value, str):
return dump_json(value)
else:
return value
def serialize(self, path: Path) -> None:
# call Predictor.serialize() in order to serialize the class name
super().serialize(path)
try:
with (path / "predictor.json").open("w") as fp:
print(dump_json(self), file=fp)
except Exception as e:
raise IOError(
f"Cannot serialize {fqname_for(self.__class__)}") from e
def serialize(self, path: Path) -> None:
with (path / "type.txt").open("w") as fp:
fp.write(fqname_for(self.__class__))
with (path / "version.json").open("w") as fp:
json.dump(
{"model": self.__version__, "gluonts": gluonts.__version__}, fp
)
with (path / "predictor.json").open("w") as fp:
print(dump_json(self), file=fp)
def serialize(self, path: Path) -> None:
# serialize some metadata
super().serialize(path)
# basically save each predictor in its own sub-folder
num_digits = len(str(len(self.predictors)))
for index, predictor in enumerate(self.predictors):
composite_path = path / f"predictor_{str(index).zfill(num_digits)}"
os.makedirs(str(composite_path))
predictor.serialize(composite_path)
# serialize all remaining constructor parameters
with (path / "parameters.json").open("w") as fp:
parameters = dict(
prediction_length=self.prediction_length,
freq=self.freq,
aggregation_method=self.aggregation_method,
num_predictors=len(self.predictors),
)
print(dump_json(parameters), file=fp)
def export_repr_block(
rb: mx.gluon.HybridBlock, model_dir: Path, model_name: str, epoch: int = 0
) -> None:
"""
Serializes a representable Gluon block.
Parameters
----------
rb
The block to export.
model_dir
The path where the model will be saved.
model_name
The name identifying the model.
epoch
The epoch number, which together with the `model_name` identifies the
model parameters.
"""
with (model_dir / f"{model_name}-network.json").open("w") as fp:
print(dump_json(rb), file=fp)
rb.save_parameters(str(model_dir / f"{model_name}-{epoch:04}.params"))
def export_symb_block(hb: mx.gluon.HybridBlock,
model_dir: Path,
model_name: str,
epoch: int = 0) -> None:
"""
Serializes a hybridized Gluon `HybridBlock`.
Parameters
----------
hb
The block to export.
model_dir
The path where the model will be saved.
model_name
The name identifying the model.
epoch
The epoch number, which together with the `model_name` identifies the
model parameters.
"""
hb.export(path=str(model_dir / model_name), epoch=epoch)
with (model_dir / f"{model_name}-in_out_format.json").open("w") as fp:
in_out_format = dict(in_format=hb._in_format,
out_format=hb._out_format)
print(dump_json(in_out_format), file=fp)
def serialize(self, path: Path) -> None:
# call Predictor.serialize() in order to serialize the class name
super().serialize(path)
with (path / "predictor.json").open("w") as fp:
print(dump_json(self), file=fp)
from gluonts.model.common import Tensor, NPArrayLike
from gluonts.mx.distribution.distribution import Distribution
from gluonts.mx.distribution import (
Gaussian,
StudentT,
MixtureDistribution,
GaussianOutput,
StudentTOutput,
LaplaceOutput,
MultivariateGaussianOutput,
MixtureDistributionOutput,
)
from gluonts.testutil import empirical_cdf
from gluonts.core.serde import dump_json, load_json
serialize_fn_list = [lambda x: x, lambda x: load_json(dump_json(x))]
def plot_samples(s: Tensor, bins: int = 100) -> None:
from matplotlib import pyplot as plt
s = s.asnumpy()
plt.hist(s, bins=bins)
plt.show()
BINS = np.linspace(-5, 5, 100)
def histogram(samples: NPArrayLike) -> np.ndarray:
h, _ = np.histogram(samples, bins=BINS, density=True)
def test_json_serialization(e) -> None:
expected, actual = e, serde.load_json(serde.dump_json(e))
assert check_equality(expected, actual)
@pytest.mark.parametrize(
"a",
[
mx.nd.random.uniform(shape=(3, 5, 2), dtype="float16"),
mx.nd.random.uniform(shape=(3, 5, 2), dtype="float32"),
mx.nd.random.uniform(shape=(3, 5, 2), dtype="float64"),
mx.nd.array([[1, 2, 3], [-1, -2, 0]], dtype=np.uint8),
mx.nd.array([[1, 2, 3], [-1, -2, 0]], dtype=np.int32),
mx.nd.array([[1, 2, 3], [-1, -2, 0]], dtype=np.int64),
mx.nd.array([[1, 2, 3], [1, 2, 0]], dtype=np.uint8),
],
)
@pytest.mark.parametrize(
"serialize_fn",
[
lambda x: serde.load_json(serde.dump_json(x)),
lambda x: serde.load_binary(serde.dump_binary(x)),
lambda x: serde.load_code(serde.dump_code(x)),
],
)
def test_ndarray_serialization(a, serialize_fn) -> None:
b = serialize_fn(a)
assert type(a) == type(b)
assert a.dtype == b.dtype
assert a.shape == b.shape
assert np.all((a == b).asnumpy())
def test_timestamp_encode_decode() -> None:
now = pd.Timestamp.now()
assert now == serde.decode(serde.encode(now))
def test_json_serialization(e) -> None:
assert equals(e, serde.load_json(serde.dump_json(e)))
def test_json_serialization(e) -> None:
assert e == serde.load_json(serde.dump_json(e))
def test_distribution_output_serde(distr_output: DistributionOutput):
distr_output_copy = decode(encode(distr_output))
assert isinstance(distr_output_copy, type(distr_output))
assert dump_json(distr_output_copy) == dump_json(distr_output)