def test_dynamic_integration(
train_length: int,
test_length: int,
prediction_length: int,
target_start: str,
rolling_start: str,
num_dynamic_feat: int,
):
"""
Trains an estimator on a rolled dataset with dynamic features.
Tests https://github.com/awslabs/gluon-ts/issues/1390
"""
train_ds = create_dynamic_dataset(target_start, train_length,
num_dynamic_feat)
rolled_ds = generate_rolling_dataset(
dataset=create_dynamic_dataset(target_start, test_length,
num_dynamic_feat),
strategy=StepStrategy(prediction_length=prediction_length),
start_time=pd.Timestamp(rolling_start),
)
estimator = DeepAREstimator(
freq="D",
prediction_length=prediction_length,
context_length=2 * prediction_length,
use_feat_dynamic_real=True,
trainer=Trainer(epochs=1),
)
predictor = estimator.train(training_data=train_ds)
forecast_it, ts_it = make_evaluation_predictions(rolled_ds,
predictor=predictor,
num_samples=100)
training_agg_metrics, _ = Evaluator(num_workers=0)(ts_it, forecast_it)
# it should have failed by this point if the dynamic features were wrong
assert training_agg_metrics
def train_predictor(df_dict, end_train_date, regions_list, target_col, feat_dynamic_cols=None):
estimator = DeepAREstimator(freq=data_freq,
prediction_length=nb_hours_pred,
trainer=Trainer(epochs=max_epochs, learning_rate = learning_rate,
learning_rate_decay_factor=0.01, patience=patience),
use_feat_dynamic_real=feat_dynamic_cols is not None)
if feat_dynamic_cols is not None:
training_data = ListDataset(
[{"item_id": region,
"start": df_dict[region].index[0],
"target": df_dict[region][target_col][:end_train_date],
"feat_dynamic_real": [df_dict[region][feat_dynamic_col][:end_train_date]
for feat_dynamic_col in feat_dynamic_cols]
}
for region in regions_list],
freq = data_freq
)
else:
training_data = ListDataset(
[{"item_id": region,
"start": df_dict[region].index[0],
"target": df_dict[region][target_col][:end_train_date]
}
for region in regions_list],
freq = data_freq
)
predictor = estimator.train(training_data=training_data)
return predictor
def run_model(data_train,
data_meta,
save_path,
num_epochs=50,
lr=1e-3,
batch_size=64,
scaling=False,
context_length=3,
num_layers=3,
embedding_dimension=16,
context='gpu'):
estimator = DeepAREstimator(freq=data_meta['freq'],
prediction_length=82,
scaling=scaling,
context_length=context_length,
num_layers=num_layers,
embedding_dimension=embedding_dimension,
trainer=Trainer(batch_size=batch_size,
epochs=num_epochs,
learning_rate=lr,
ctx=context,
hybridize=False))
predictor = estimator.train(data_train)
predictor.serialize(Path(save_path))
return predictor
def GlounTS():
#from pts.dataset import ListDataset
#from pts.model.deepar import DeepAREstimator
#from pts import Trainer
#from pts.dataset import to_pandas
# gluonts crash in my sistem.
from gluonts.dataset.common import ListDataset
from gluonts.model.deepar import DeepAREstimator
from gluonts.trainer import Trainer
training_data = ListDataset([{
"start": df.index[0],
"target": df.value[:"2015-03-08 23:22:53"]
}],
freq="5min")
#estimator = DeepAREstimator(freq="5min",input_size = 43, prediction_length=forecast_size, trainer=Trainer(epochs=20))
estimator = DeepAREstimator(freq="5min",
prediction_length=forecast_size,
trainer=Trainer(epochs=20))
predictor = estimator.train(training_data=training_data)
test_data = ListDataset([{
"start": df.index[0],
"target": df.value[:"2015-03-08 23:22:53"]
}],
freq="5min")
GluonTS_prediction = next(predictor.predict(test_data))
GluonTS_mean_yhat = GluonTS_prediction.mean
GluonTS_median_yhat = GluonTS_prediction.median
return GluonTS_mean_yhat.tolist(), GluonTS_median_yhat.tolist(
), GluonTS_prediction
def test_listing_1():
"""
Test GluonTS paper examples from arxiv paper:
https://arxiv.org/abs/1906.05264
Listing 1
"""
from gluonts.dataset.repository.datasets import get_dataset
from gluonts.model.deepar import DeepAREstimator
from gluonts.trainer import Trainer
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import backtest_metrics
# We use electricity in the paper but that would take too long to run in
# the unit test
dataset_info, train_ds, test_ds = constant_dataset()
meta = dataset_info.metadata
estimator = DeepAREstimator(
freq=meta.time_granularity,
prediction_length=1,
trainer=Trainer(epochs=1, batch_size=32),
)
predictor = estimator.train(train_ds)
evaluator = Evaluator(quantiles=(0.1, 0.5, 0.9))
agg_metrics, item_metrics = backtest_metrics(
train_dataset=train_ds,
test_dataset=test_ds,
forecaster=predictor,
evaluator=evaluator,
)
def train(epochs, prediction_length, num_layers, dropout_rate):
#create train dataset
df = pd.read_csv(filepath_or_buffer=os.environ['SM_CHANNEL_TRAIN'] +
"/train.csv",
header=0,
index_col=0)
training_data = ListDataset([{
"start": df.index[0],
"target": df.value[:]
}],
freq="5min")
#define DeepAR estimator
deepar_estimator = DeepAREstimator(freq="5min",
prediction_length=prediction_length,
dropout_rate=dropout_rate,
num_layers=num_layers,
trainer=Trainer(epochs=epochs))
#train the model
deepar_predictor = deepar_estimator.train(training_data=training_data)
#create test dataset
df = pd.read_csv(filepath_or_buffer=os.environ['SM_CHANNEL_TEST'] +
"/test.csv",
header=0,
index_col=0)
test_data = ListDataset([{
"start": df.index[0],
"target": df.value[:]
}],
freq="5min")
#evaluate trained model on test data
forecast_it, ts_it = make_evaluation_predictions(test_data,
deepar_predictor,
num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(test_data))
print("MSE:", agg_metrics["MSE"])
#save the model
deepar_predictor.serialize(pathlib.Path(os.environ['SM_MODEL_DIR']))
return deepar_predictor
def fit(self,
*,
timeout: float = None,
iterations: int = None) -> CallResult[None]:
""" Fits DeepAR model using training data from set_training_data and hyperparameters
Keyword Arguments:
timeout {float} -- timeout, considered (default: {None})
iterations {int} -- iterations, considered (default: {None})
Returns:
CallResult[None]
"""
if iterations is None:
iterations = self.hyperparams["epochs"]
has_finished = True
else:
has_finished = False
estimator = DeepAREstimator(
freq=self._freq,
prediction_length=self.hyperparams['prediction_length'],
context_length=self.hyperparams['context_length'],
use_feat_static_cat=self._deepar_dataset.has_cat_cols()
or self._deepar_dataset.has_group_cols(),
use_feat_dynamic_real=self._deepar_dataset.has_real_cols(),
cardinality=self._deepar_dataset.get_cardinality(),
distr_output=self._deepar_dataset.get_distribution_type(),
dropout_rate=self.hyperparams['dropout_rate'],
trainer=Trainer(
epochs=iterations,
learning_rate=self.hyperparams['learning_rate'],
batch_size=self.hyperparams['training_batch_size'],
num_batches_per_epoch=self.hyperparams['steps_per_epoch']))
logger.info(f"Fitting for {iterations} iterations")
start_time = time.time()
predictor = estimator.train(self._train_data)
predictor.batch_size = self.hyperparams['inference_batch_size']
self._is_fit = True
logger.info(
f"Fit for {iterations} epochs, took {time.time() - start_time}s")
if not os.path.isdir(self.hyperparams['weights_dir']):
os.mkdir(self.hyperparams['weights_dir'])
predictor.serialize(Path(self.hyperparams['weights_dir']))
return CallResult(None, has_finished=has_finished)
def model_eval(estimator=None,
TD=None,
cardinalities=None,
istrain=True,
ismetric=True,
isplot=True,
pars=None):
from gluonts.model.deepar import DeepAREstimator
from gluonts.trainer import Trainer
p = pars
if estimator is None:
estimator = DeepAREstimator(
prediction_length=p.get("single_pred_length", 28),
freq="D",
distr_output=p.get("distr_output", None),
use_feat_static_cat=True,
use_feat_dynamic_real=True,
cardinality=p.get("cardinality", None),
trainer=Trainer(
learning_rate=p.get("lr", 1e-4), # 1e-4, #1e-3,
epochs=p.get("epoch", None),
num_batches_per_epoch=p.get("num_batches_per_epoch", 10),
batch_size=p.get("batch_size", 8),
))
if istrain: estimator = estimator.train(TD.train)
#### Evaluate ########################################################################
from gluonts.evaluation.backtest import make_evaluation_predictions
forecast_it, ts_it = make_evaluation_predictions(dataset=TD.test,
predictor=estimator,
num_samples=p.get(
"num_samples", 5))
forecasts, tss = list(forecast_it), list(ts_it)
if isplot:
forecast_graph(forecasts, tss, p.get("ii_series", 0))
####### Metrics ######################################################################
agg_metrics, item_metrics = None, None
if ismetric:
agg_metrics, item_metrics = forecast_metrics(tss,
forecasts,
TD,
quantiles=[0.1, 0.5, 0.9],
show=True,
dir_save=None)
return estimator, forecasts, tss, agg_metrics, item_metrics
def easy_train():
import pandas as pd
df = pd.read_csv("optiver_hacktheburgh/sp.csv",
header=0,
index_col=0,
usecols=[0, 2],
skiprows=lambda x: x % 5 != 0)
# df[:100].plot(linewidth=2)
print("Showing")
# plt.show()
from gluonts.dataset.common import ListDataset
training_data = ListDataset([{
"start": df.index[0],
"target": df.values.flatten()
}],
freq="1s")
#from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator
from gluonts.model.deepar import DeepAREstimator
from gluonts.trainer import Trainer
estimator = DeepAREstimator(freq="1min",
prediction_length=100,
trainer=Trainer(epochs=20))
predictor = estimator.train(training_data=training_data)
test_data = ListDataset([{
"start": df.index[0],
"target": df.values.flatten()[:1000]
}],
freq="10s")
full_test_data = ListDataset([{
"start": df.index[0],
"target": df.values.flatten()
}],
freq="10s")
means = []
for i, (test_entry, forecast) in enumerate(
zip(full_test_data, predictor.predict(test_data))):
# if i > 0:
# break
print(forecast.dim())
plt.plot(test_entry["target"])
#forecast.plot(color='g', prediction_intervals=[], output_file="test.png")
means.extend(list(forecast.mean))
print(forecast.mean)
l = len(test_entry["target"])
plt.axhline(y=means[0], xmin=0, xmax=l, linewidth=2, color='r')
plt.axvline(x=5000, color='b')
plt.grid(which='both')
plt.show()
def test_nested_params():
deepar = DeepAREstimator(prediction_length=7, freq="D")
assert equals(deepar, serde.flat.decode(serde.flat.encode(deepar)))
deepar2 = serde.flat.clone(deepar, {"trainer.epochs": 999})
assert deepar2.trainer.epochs == 999
def prepare(df, P, frac, ep):
rolling_test = []
train_size = int(frac * df.shape[0])
i = 0
delay = 0
train_ds = ListDataset([{
"start":
pd.Timestamp(df.index[0]),
"target":
df.Diff[0:train_size - P],
'feat_dynamic_real': [
df.fear[0:train_size - P], df.anger[0:train_size - P],
df.anticipation[0:train_size - P], df.trust[0:train_size - P],
df.suprise[0:train_size - P], df.positive[0:train_size - P],
df.negative[0:train_size - P], df.sadness[0:train_size - P],
df.disgust[0:train_size - P], df.joy[0:train_size - P],
df.Volume_of_tweets[0:train_size - P],
df.Retweet[0:train_size - P], df.Replies[0:train_size - P],
df.Likes[0:train_size - P]
]
}],
freq='1B')
while train_size + delay < df.shape[0]:
delay = int(P) * i
test_ds = ListDataset([
dict(start=pd.Timestamp(df.index[0]),
target=df.Diff[0:train_size + delay],
feat_dynamic_real=[
df.fear[0:train_size + delay],
df.anger[0:train_size + delay],
df.anticipation[0:train_size + delay],
df.trust[0:train_size + delay],
df.suprise[0:train_size + delay],
df.positive[0:train_size + delay],
df.negative[0:train_size + delay],
df.sadness[0:train_size + delay],
df.disgust[0:train_size + delay],
df.joy[0:train_size + delay],
df.Volume_of_tweets[0:train_size + delay],
df.Retweet[0:train_size + delay],
df.Replies[0:train_size + delay], df.Likes[0:train_size +
delay]
])
],
freq='1B')
i += 1
rolling_test.append(test_ds)
print("We have 1 training set of", train_size, "days and then ",
len(rolling_test), "testing sets of ", delay, " days total")
estimator = DeepAREstimator(prediction_length=P,
context_length=5,
freq='1B',
use_feat_dynamic_real=True,
trainer=Trainer(
ctx="cpu",
epochs=ep,
)) #hybridize=False, ), )
return train_ds, rolling_test, estimator, train_size, i
def run_model(data_train,
data_meta,
save_path,
num_epochs=50,
lr=1e-3,
batch_size=64):
estimator = DeepAREstimator(
freq=data_meta['freq'],
prediction_length=data_meta['prediction_length'],
trainer=Trainer(batch_size=batch_size,
epochs=num_epochs,
learning_rate=lr,
ctx='cpu',
hybridize=False))
predictor = estimator.train(data_train)
predictor.serialize(Path(save_path))
return predictor
def deepar(data="m4_quarterly", seed=42, epochs=100, batches=50):
dataset = get_dataset(data, regenerate=False)
mx.random.seed(seed)
np.random.seed(seed)
trainer = Trainer(
ctx=mx.cpu(0),
epochs=epochs,
num_batches_per_epoch=batches,
learning_rate=1e-3,
)
cardinality = int(dataset.metadata.feat_static_cat[0].cardinality)
estimator = DeepAREstimator(
trainer=trainer,
cardinality=[cardinality],
context_length=dataset.metadata.prediction_length,
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
use_feat_static_cat=True
)
# predictor = estimator.train(training_data=dataset.train,
# validation_data=dataset.test)
predictor = estimator.train(training_data=dataset.train)
forecast_it, ts_it = make_evaluation_predictions(
dataset.test, predictor=predictor, num_samples=100
)
agg_metrics, item_metrics = Evaluator()(
ts_it, forecast_it, num_series=len(dataset.test)
)
metrics = ["MASE", "sMAPE", "MSIS", "wQuantileLoss[0.5]", "wQuantileLoss[0.9]"]
output = {key: round(value, 8) for key, value in agg_metrics.items() if key in metrics}
output["seed"] = seed
output["epochs"] = epochs
output["num_batches"] = batches
df = pd.DataFrame([output])
return df
def deepar_test():
url = "https://raw.githubusercontent.com/numenta/NAB/master/data/realTweets/Twitter_volume_AMZN.csv"
df = pd.read_csv(url, header=0, index_col=0)
data = ListDataset(
[{
"start": df.index[0],
"target": df.value[:"2015-04-05 00:00:00"]
}],
freq="5min"
)
trainer = Trainer(epochs=10)
estimator = DeepAREstimator(freq="5min", prediction_length=12, trainer=trainer)
predictor = estimator.train(training_data=data)
prediction = next(predictor.predict(data))
print(prediction.mean)
prediction.plot(output_file="./graph.png")
def train(self, **kwargs):
epochs = kwargs.get("epochs", 10)
# Adjust class freq.
self.freq = pd.infer_freq(self.train_df.index)
if self.freq == "MS":
self.freq = "M"
estimator = DeepAREstimator(
freq=self.freq,
prediction_length=self.forecast_len,
trainer=Trainer(epochs=epochs,
batch_size=64,
ctx="gpu" if self.GPU else "cpu"),
)
self.model = estimator.train(
training_data=self.format_input(self.train_df, self.freq))
def train(dataset: ListDataset, frequency: str, horizon: int, model_name: str,
num_layers: int, num_cells: int, epochs: int, patience: int,
weight_decay: float, dropout_rate: float, batch_size: int,
snapshot_dir: str, overwrite: bool):
"""
Train a model.
:param dataset:
:param model_name:
:param horizon:
:param frequency:
:param snapshot_dir:
:param epochs:
:param patience:
:param weight_decay:
:param batch_size:
:param dropout_rate:
:param num_layers:
:param num_cells:
:param overwrite:
:return:
"""
model_dir = Path(snapshot_dir)
if not overwrite and os.path.isdir(snapshot_dir):
return Predictor.deserialize(model_dir)
trainer = Trainer(epochs=epochs,
patience=patience,
weight_decay=weight_decay,
batch_size=batch_size)
if model_name == 'deepar':
estimator = DeepAREstimator(freq=frequency,
scaling=False,
dropout_rate=dropout_rate,
num_layers=num_layers,
num_cells=num_cells,
prediction_length=horizon,
trainer=trainer)
else:
raise Exception(f'Unknown model {model_name}')
predictor = estimator.train(training_data=dataset)
model_dir.mkdir(parents=True, exist_ok=overwrite)
predictor.serialize(model_dir)
return predictor
def test_deepar():
from gluonts.model.deepar import DeepAREstimator
config = {}
config['directory'] = 'results/deepar'
model = DeepAREstimator(freq="30min",
prediction_length=48,
trainer=Trainer(epochs=3))
evaluate_model(model, config)
def train_deepar(train_ds,
context_length=10,
prediction_length=20,
period=4320,
epochs=2):
freq = "{}H".format(period / 3600)
estimator = DeepAREstimator(prediction_length=prediction_length,
context_length=context_length,
freq=freq,
num_cells=50,
trainer=Trainer(ctx="gpu",
epochs=epochs,
learning_rate=1e-3,
hybridize=False,
num_batches_per_epoch=100,
batch_size=64),
num_parallel_samples=500)
predictor = estimator.train(train_ds)
return predictor
def train_predictor(region_df_dict, end_train_date, regions_list, max_epochs, learning_rate, target_col,
feat_dynamic_cols=None):
estimator = DeepAREstimator(freq=md.FREQ,
prediction_length=md.NB_HOURS_PRED,
trainer=Trainer(epochs=max_epochs, learning_rate=learning_rate,
learning_rate_decay_factor=md.LR_DECAY_FACTOR),
use_feat_dynamic_real=feat_dynamic_cols is not None)
if feat_dynamic_cols is not None:
training_data = ListDataset(
[{"item_id": region,
"start": region_df_dict[region].index[0],
"target": region_df_dict[region][target_col][:end_train_date],
"feat_dynamic_real": [region_df_dict[region][feat_dynamic_col][:end_train_date]
for feat_dynamic_col in feat_dynamic_cols]
}
for region in regions_list],
freq=md.FREQ
)
else:
training_data = ListDataset(
[{"item_id": region,
"start": region_df_dict[region].index[0],
"target": region_df_dict[region][target_col][:end_train_date]
}
for region in regions_list],
freq=md.FREQ
)
model_path = predictor_path(region_df_dict, regions_list, max_epochs, learning_rate, feat_dynamic_cols)
model_dir, model_name = os.path.split(model_path)
logging.info("Training deepar model {}".format(model_name))
logging.getLogger().setLevel(logging.WARNING)
predictor = estimator.train(training_data=training_data)
logging.getLogger().setLevel(logging.INFO)
logging.info("Saving model with {} epochs and learning rate of {}".format(max_epochs, learning_rate))
with open(model_path, "wb") as file:
pickle.dump(predictor, file)
return predictor
def define_DeepAR_predictor(self, freq, prediction_length, epochs,
num_layers, batch_size):
self.predictor = DeepAREstimator(freq=freq,
prediction_length=prediction_length,
context_length=prediction_length,
trainer=Trainer(
ctx="cpu",
epochs=epochs,
batch_size=batch_size,
num_batches_per_epoch=100),
num_layers=num_layers,
use_feat_dynamic_real=True)
def test_distribution():
"""
Makes sure additional tensors can be accessed and have expected shapes
"""
prediction_length = ds_info.prediction_length
estimator = DeepAREstimator(
freq=freq,
prediction_length=prediction_length,
trainer=Trainer(epochs=2, num_batches_per_epoch=1),
distr_output=StudentTOutput(),
)
train_output = estimator.train_model(train_ds, test_ds)
# todo adapt loader to anomaly detection use-case
batch_size = 2
num_samples = 3
training_data_loader = TrainDataLoader(
dataset=train_ds,
transform=train_output.transformation,
batch_size=batch_size,
num_batches_per_epoch=estimator.trainer.num_batches_per_epoch,
ctx=mx.cpu(),
)
seq_len = 2 * ds_info.prediction_length
for data_entry in islice(training_data_loader, 1):
input_names = get_hybrid_forward_input_names(train_output.trained_net)
distr = train_output.trained_net.distribution(
*[data_entry[k] for k in input_names]
)
assert distr.sample(num_samples).shape == (
num_samples,
batch_size,
seq_len,
)
def test_general_functionality() -> None:
ds_info, train_ds, test_ds = constant_dataset()
freq = ds_info.metadata.freq
prediction_length = ds_info.prediction_length
trainer = Trainer(epochs=3, num_batches_per_epoch=5)
estimator = DeepAREstimator(prediction_length=prediction_length,
freq=freq,
trainer=trainer)
predictor = estimator.train(training_data=train_ds)
agg_metrics, item_metrics = backtest_metrics(
test_dataset=test_ds,
predictor=predictor,
evaluator=Evaluator(calculate_owa=False),
)
# just some sanity check
assert (agg_metrics is not None and item_metrics is not None
), "Metrics should not be None if everything went smooth."
def test_shape():
"""
Makes sure additional tensors can be accessed and have expected shapes
"""
prediction_length = ds_info.prediction_length
estimator = DeepAREstimator(
freq=freq,
prediction_length=prediction_length,
trainer=Trainer(epochs=1, num_batches_per_epoch=1),
distr_output=StudentTOutput(),
)
training_transformation, trained_net = estimator.train_model(train_ds)
# todo adapt loader to anomaly detection use-case
batch_size = 2
training_data_loader = TrainDataLoader(
dataset=train_ds,
transform=training_transformation,
batch_size=batch_size,
num_batches_per_epoch=estimator.trainer.num_batches_per_epoch,
ctx=mx.cpu(),
)
seq_len = 2 * ds_info.prediction_length
for data_entry in islice(training_data_loader, 1):
input_names = get_hybrid_forward_input_names(trained_net)
loss, likelihoods, *distr_args = trained_net(
*[data_entry[k] for k in input_names])
distr = StudentT(*distr_args)
assert likelihoods.shape == (batch_size, seq_len)
assert distr.mu.shape == (batch_size, seq_len)
assert distr.sigma.shape == (batch_size, seq_len)
assert distr.nu.shape == (batch_size, seq_len)
def init_model():
epochs = None
context = 'cpu'
if args.epochs is not None:
epochs = args.epochs
if args.gpu:
context = 'gpu'
predictor = None
if args.train:
my_trainer = Trainer(
ctx=context
) # TODO: Find a way to make it such that we do not set epoch when there is no need to
estimator = DeepAREstimator(freq="5min",
prediction_length=args.prediction,
trainer=my_trainer)
predictor = estimator.train(training_data=training_data)
predictor.serialize(Path("models/"))
else:
# predictor = Predictor.deserialize(Path("models/"))
predictor = RepresentableBlockPredictor.deserialize(Path("models/"))
predictor.ctx = mx.Context('cpu')
return predictor
def evaluate_deepar(config):
""" Pass DeepAR to evaluate_gluon"""
from gluonts.model.deepar import DeepAREstimator
model = DeepAREstimator(
freq=config['freq'],
use_feat_dynamic_real=config['params'].get('use_feat_dynamic_real',
False),
prediction_length=config['prediction_length'],
trainer=Trainer(
epochs=config['params'].get('epochs', 10),
learning_rate=config['params'].get('learning_rate', 1e-3),
num_batches_per_epoch=config['params'].get('num_batches_per_epoch',
100),
))
evaluate_gluon(config, model)
def build_forecaster(self):
if self.type == 'auto_arima':
forecaster = AutoARIMA(sp=7, suppress_warnings=True)
elif self.type == 'prophet':
forecaster = Prophet()
elif self.type == 'deepar':
forecaster = DeepAREstimator(prediction_length=self.horizon,
freq=self.freq,
trainer=Trainer(
ctx="cpu",
epochs=15,
learning_rate=1e-3,
num_batches_per_epoch=100))
else:
raise NotImplementedError(
f'Model {type} is currently not implemented')
return forecaster
def __init__(self,
freq="D",
prediction_length=30,
epochs=50,
batch_size=16,
num_batches_per_epoch=100,
num_layers=4,
list_products=list_products):
self.predictor = DeepAREstimator(
freq=freq,
prediction_length=prediction_length,
trainer=Trainer(ctx="cpu",
epochs=epochs,
batch_size=batch_size,
num_batches_per_epoch=num_batches_per_epoch),
num_layers=num_layers)
self.algorithm = algorithm
self.list_products_names = TransactionsData.get_list_names(
list_products)
def create_estimator(
self,
freq: str,
prediction_length: int,
time_features: bool,
training_time: float,
validation_milestones: List[float],
callbacks: List[Callback],
) -> Estimator:
return DeepAREstimator(
freq=freq,
prediction_length=prediction_length,
num_layers=self.num_layers,
num_cells=self.num_cells,
time_features=None if time_features else [],
trainer=self._create_trainer(
training_time,
validation_milestones,
callbacks, # type: ignore
),
context_length=self.context_length_multiple * prediction_length,
)
# largo de la prediccion para que sean days_of_prediction dias
prediction_lentgh = int(days_of_prediction * 24 * freq_int)
# número de columnas con el que se harán las predicciones (simil: productos)
number_of_products = 15
# fechas de inicio de los conjuntos
start_train = pd.Timestamp(start_train, freq=freq)
start_test = pd.Timestamp(start_test, freq=freq)
# settings del modelo, ver link para conocer los hyperparametros
# https://ts.gluon.ai/api/gluonts/gluonts.model.deepar.html
estimator = DeepAREstimator(freq=freq,
context_length=prediction_lentgh,
prediction_length=prediction_lentgh,
use_feat_static_cat=True,
cardinality=[1],
num_layers=2,
num_cells=128,
cell_type='lstm',
trainer=Trainer(epochs=15))
# reshape de la data de entrenamiento para solo ocupar number_of_products
train_ds = ListDataset([{
FieldName.TARGET: target,
FieldName.START: start_train,
FieldName.FEAT_STATIC_CAT: fsc
} for (target, fsc) in zip(df_train[0:number_of_products],
ts_code[0:number_of_products].reshape(-1, 1))],
freq=freq)
# reshape de la data de test para solo ocupar number_of_products
# Create testing datatset
testing_data = ListDataset(
[{"start": test_data[target_asset].index[0],
"target": train_data[target_asset]}],
freq = "1d"
)
# Create testing datatset
testing_data_plots = ListDataset(
[{"start": test_data[target_asset].index[0],
"target": test_data[target_asset]}],
freq = "1d"
)
#Create the estimator and train
estimator = DeepAREstimator(freq="1d", prediction_length=pred_len, trainer=Trainer(epochs=100))
predictor = estimator.train(training_data=training_data)
### OPTIONAL PLOT PREDICTION RESULTS
#Forecast
for test_entry, forecast in zip(testing_data_plots, predictor.predict(testing_data)):
to_pandas(test_entry)[-60:].plot(linewidth=2)
forecast.plot(color='g', prediction_intervals=[50.0, 90.0])
### GENERATE FORECASTS
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import make_evaluation_predictions
forecast_it, ts_it = make_evaluation_predictions(
dataset=testing_data, # test dataset
predictor=predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
