def run_train_and_test(
env: TrainEnv, forecaster_type: Type[Union[Estimator,
Predictor]]) -> None:
check_gpu_support()
forecaster_fq_name = fqname_for(forecaster_type)
forecaster_version = forecaster_type.__version__
logger.info(f"Using gluonts v{gluonts.__version__}")
logger.info(f"Using forecaster {forecaster_fq_name} v{forecaster_version}")
forecaster = forecaster_type.from_hyperparameters(**env.hyperparameters)
logger.info(
f"The forecaster can be reconstructed with the following expression: "
f"{dump_code(forecaster)}")
if isinstance(forecaster, Predictor):
predictor = forecaster
else:
predictor = run_train(forecaster, env.datasets["train"])
predictor.serialize(env.path.model)
if "test" in env.datasets:
run_test(env, predictor, env.datasets["test"])
def make_gunicorn_app(
env: ServeEnv,
forecaster_type: Optional[Type[Union[Estimator, Predictor]]],
settings: Settings,
) -> Application:
check_gpu_support()
if forecaster_type is not None:
logger.info(f"Using dynamic predictor factory")
ctor = forecaster_type.from_hyperparameters
forecaster_fq_name = fqname_for(forecaster_type)
forecaster_version = forecaster_type.__version__
def predictor_factory(request) -> Predictor:
return ctor(**request["configuration"])
else:
logger.info(f"Using static predictor factory")
assert env is not None
predictor = Predictor.deserialize(env.path.model)
forecaster_fq_name = fqname_for(type(predictor))
forecaster_version = predictor.__version__
def predictor_factory(request) -> Predictor:
return predictor
logger.info(f"Using gluonts v{gluonts.__version__}")
logger.info(f"Using forecaster {forecaster_fq_name} v{forecaster_version}")
execution_params = {
"MaxConcurrentTransforms": settings.number_of_workers,
"BatchStrategy": settings.sagemaker_batch_strategy,
"MaxPayloadInMB": settings.sagemaker_max_payload_in_mb,
}
flask_app = make_app(
predictor_factory,
execution_params,
batch_transform_config=env.batch_config,
settings=settings,
)
gunicorn_app = Application(
app=flask_app,
config={
"bind": settings.sagemaker_server_bind,
"workers": settings.number_of_workers,
"timeout": settings.sagemaker_server_timeout,
},
)
return gunicorn_app
def run(env, forecaster):
check_gpu_support()
if isinstance(forecaster, Predictor):
predictor = forecaster
else:
predictor = run_train(env, forecaster, env.datasets["train"])
predictor.serialize(env.path.model)
if "test" in env.datasets:
test_dataset = prepare_test_dataset(
env.datasets["test"],
prediction_length=forecaster.prediction_length,
)
run_test(predictor, test_dataset)
def run(self):
try:
check_gpu_support()
datasets = get_channels(self.paths).get_datasets()
self.hyperparameters['freq'] = datasets.metadata.time_granularity
estimator, predictor = self.run_train(datasets.train)
if datasets.test is not None:
self.run_test(datasets.test, estimator, predictor)
predictor.serialize(self.paths.model)
# TODO: do we want to handle GluonTS exceptions differently?
except Exception as e:
# logger.error(e)
raise e
def run_train_and_test(
env: TrainEnv, forecaster_type: Type[Union[Estimator,
Predictor]]) -> None:
check_gpu_support()
forecaster = forecaster_type.from_hyperparameters(**env.hyperparameters)
if isinstance(forecaster, Predictor):
predictor = forecaster
else:
predictor = run_train(forecaster, env.datasets["train"])
predictor.serialize(env.path.model)
if "test" in env.datasets:
test_dataset = prepare_test_dataset(
env.datasets["test"],
prediction_length=forecaster.prediction_length,
)
run_test(env, predictor, test_dataset)
def run_train_and_test(
env: TrainEnv, forecaster_type: Type[Union[Estimator,
Predictor]]) -> None:
check_gpu_support()
# train_stats = calculate_dataset_statistics(env.datasets["train"])
# log_metric("train_dataset_stats", train_stats)
forecaster_fq_name = fqname_for(forecaster_type)
forecaster_version = forecaster_type.__version__
logger.info(f"Using gluonts v{gluonts.__version__}")
logger.info(f"Using forecaster {forecaster_fq_name} v{forecaster_version}")
forecaster = forecaster_type.from_inputs(env.datasets["train"],
**env.hyperparameters)
logger.info(
f"The forecaster can be reconstructed with the following expression: "
f"{dump_code(forecaster)}")
logger.info(
"Using the following data channels: "
f"{', '.join(name for name in ['train', 'validation', 'test'] if name in env.datasets)}"
)
if isinstance(forecaster, Predictor):
predictor = forecaster
else:
predictor = run_train(
forecaster=forecaster,
train_dataset=env.datasets["train"],
validation_dataset=env.datasets.get("validation"),
hyperparameters=env.hyperparameters,
)
predictor.serialize(env.path.model)
if "test" in env.datasets:
run_test(env, predictor, env.datasets["test"])
def test_jitter_synthetic_gp(jitter_method, float_type, ctx) -> None:
# TODO: Enable GPU tests on Jenkins
if ctx == mx.Context("gpu") and not check_gpu_support():
return
# Initialize problem parameters
batch_size = 1
prediction_length = 50
context_length = 5
num_samples = 3
# Initialize test data to generate Gaussian Process from
lb = -5
ub = 5
dx = (ub - lb) / (prediction_length - 1)
x_test = nd.arange(lb, ub + dx, dx, ctx=ctx,
dtype=float_type).reshape(-1, 1)
x_test = nd.tile(x_test, reps=(batch_size, 1, 1))
# Define the GP hyper parameters
amplitude = nd.ones((batch_size, 1, 1), ctx=ctx, dtype=float_type)
length_scale = math.sqrt(0.4) * nd.ones_like(amplitude)
sigma = math.sqrt(1e-5) * nd.ones_like(amplitude)
# Instantiate desired kernel object and compute kernel matrix
rbf_kernel = RBFKernel(amplitude, length_scale)
# Generate samples from 0 mean Gaussian process with RBF Kernel and plot it
gp = GaussianProcess(
sigma=sigma,
kernel=rbf_kernel,
prediction_length=prediction_length,
context_length=context_length,
num_samples=num_samples,
ctx=ctx,
float_type=float_type,
jitter_method=jitter_method,
sample_noise=False, # Returns sample without noise
)
# Generate training set on subset of interval using the sine function
x_train = nd.array([-4, -3, -2, -1, 1], ctx=ctx,
dtype=float_type).reshape(context_length, 1)
x_train = nd.tile(x_train, reps=(batch_size, 1, 1))
y_train = nd.sin(x_train.squeeze(axis=2))
# Predict exact GP using the GP predictive mean and covariance using the same fixed hyper-parameters
samples, predictive_mean, predictive_std = gp.exact_inference(
x_train, y_train, x_test)
assert (np.sum(np.isnan(
samples.asnumpy())) == 0), "NaNs in predictive samples!"
def test_jitter_unit(jitter_method, float_type, ctx) -> None:
# TODO: Enable GPU tests on Jenkins
if ctx == mx.Context("gpu") and not check_gpu_support():
return
matrix = nd.array([[[1, 2], [3, 4]], [[10, 100], [-21.5, 41]]],
ctx=ctx,
dtype=float_type)
F = mx.nd
num_data_points = matrix.shape[1]
if jitter_method == "eig":
L = jitter_cholesky_eig(F, matrix, num_data_points, ctx, float_type)
elif jitter_method == "iter":
L = jitter_cholesky(F, matrix, num_data_points, ctx, float_type)
assert np.sum(np.isnan(L.asnumpy())) == 0, "NaNs in Cholesky factor!"
def __init__(
self,
paths: PathsEnvironment = PathsEnvironment(),
port: Optional[int] = None,
workers: Optional[int] = None,
) -> None:
app = flask.Flask('GluonTS scoring service')
port = port if port else self.DEFAULT_PORT
options = {
"bind": f"0.0.0.0:{port}",
"workers": workers if workers else number_of_workers(app),
# "post_worker_init": ScoringService.post_worker_init,
"timeout": 100,
}
check_gpu_support()
predictor = Predictor.deserialize(paths.model)
@app.route('/ping')
def ping():
return ''
@app.route("/execution-parameters")
def execution_parameters():
return flask.jsonify({
'MaxConcurrentTransforms': options['workers'],
'BatchStrategy': 'SINGLE_RECORD',
'MaxPayloadInMB': SAGEMAKER_MAX_PAYLOAD_IN_MB,
})
@app.route('/invocations', methods=['POST'])
def invocations() -> Any:
try:
payload = flask.request.json
configuration = payload['configuration']
if 'num_samples' in configuration:
configuration['num_eval_samples'] = configuration[
'num_samples']
config = ForecastConfig.parse_obj(configuration)
def process(forecast: Forecast) -> dict:
prediction = {}
if 'samples' in config.output_types:
if isinstance(forecast, SampleForecast):
prediction['samples'] = forecast.samples.tolist()
else:
prediction['samples'] = []
if 'mean' in config.output_types:
prediction['mean'] = forecast.mean.tolist()
if 'quantiles' in config.output_types:
prediction['quantiles'] = {
q: forecast.quantile(q).tolist()
for q in config.quantiles
}
return prediction
dataset = ListDataset(payload['instances'], predictor.freq)
predictions = list(
map(
process,
predictor.predict(dataset,
no_samples=config.num_eval_samples),
))
return flask.jsonify(predictions=predictions)
except Exception as error:
return flask.jsonify(error=traceback.format_exc()), 500
# NOTE: Stop Flask application when SIGTERM is received as a result
# of "docker stop" command.
signal.signal(signal.SIGTERM, self.stop)
self.options = options
self.application = app
super(DefaultShell, self).__init__()
