def test_generic_model(
generic_asset_type_names: List[str],
sensor_name: Optional[str] = None,
from_date: str = "2015-03-10",
period: int = 3,
horizon_hours: int = 1,
training: int = 30,
):
"""Manually test integration of timetomodel for our generic model."""
start = as_server_time(datetime.strptime(from_date, "%Y-%m-%d"))
end = start + timedelta(days=period)
training_and_testing_period = timedelta(days=training)
horizon = timedelta(hours=horizon_hours)
with app.app_context():
sensors = query_sensor_by_name_and_generic_asset_type_name(
sensor_name=sensor_name,
generic_asset_type_names=generic_asset_type_names,
).all()
if len(sensors) == 0:
click.echo(
"No such sensor in db, so I will not add any forecasts.")
return
elif len(sensors) > 1:
click.echo(
"No unique sensor found in db, so I will not add any forecasts."
)
return
linear_model_configurator = lookup_model_specs_configurator("linear")
(
model_specs,
model_identifier,
fallback_model_identifier,
) = linear_model_configurator(
sensor=sensors[0],
forecast_start=start,
forecast_end=end,
forecast_horizon=horizon,
custom_model_params=dict(
training_and_testing_period=training_and_testing_period),
)
# Create and train the model
model = create_fitted_model(model_specs, model_identifier)
print("\n\nparams:\n%s\n\n" % model.params)
evaluate_models(m1=ModelState(model, model_specs), plot_path=None)
return ModelState(model, model_specs)
def update_model(
time_step: datetime,
current_model: MODEL_CLASSES,
specs: ModelSpecs,
feature_frame: pd.DataFrame,
) -> ModelState:
new_model: MODEL_CLASSES = current_model
""" Create model if current one is outdated or not yet created."""
if (current_model is None
or time_step - specs.creation_time >= specs.remodel_frequency):
logger.debug("Fitting new model before predicting %s ..." % time_step)
if current_model is not None:
# move the model's series specs further in time
specs.start_of_training = specs.start_of_training + specs.remodel_frequency
specs.end_of_testing = specs.end_of_testing + specs.remodel_frequency
relevant_time_steps = get_time_steps(time_range="train", specs=specs)
new_model = create_fitted_model(
specs, "", regression_frame=feature_frame.loc[relevant_time_steps])
specs.creation_time = time_step
return ModelState(new_model, specs)
def make_rolling_forecasts(
start: datetime, # Start of forecast period
end: datetime, # End of forecast period
model_specs: ModelSpecs,
) -> Tuple[pd.Series, ModelState]:
"""
Repeatedly call make_forecast - for all time steps the desired time window
(end is excluding).
The time window of the specs (training + test data) is moved forwards also step by step.
Will fail if series specs do not allocate enough data.
May create a model whenever the previous one is outdated.
Return Pandas.Series as result, as well as the last ModelState.
"""
# Prepare time range
for dt in (start, end):
if dt.tzinfo is None:
dt.replace(tzinfo=pytz.utc)
# First, compute one big feature frame, once.
feature_frame: pd.DataFrame = construct_features(
(model_specs.start_of_training, end), model_specs)
pd_frequency = timedelta_to_pandas_freq_str(model_specs.frequency)
values = pd.Series(index=pd.date_range(
start, end, freq=pd_frequency, closed="left", tz=start.tzinfo))
time_step = start
model = None
logger.info("Forecasting from %s to %s" % (start, end))
while time_step < end:
model, specs = update_model(time_step,
model,
model_specs,
feature_frame=feature_frame).split()
features = feature_frame.loc[time_step:time_step].iloc[:, 1:]
values[time_step] = make_forecast_for(model_specs, features, model)
time_step = time_step + model_specs.frequency
return values, ModelState(model, model_specs)
def evaluate_models(m1: ModelState,
m2: Optional[ModelState] = None,
plot_path: str = None):
"""
Run a model or two against test data and plot results.
Useful to judge model performance or compare two models.
Shows RMSE values, plots error distributions and prints the time it took to forecast.
TODO: support testing m2 next to m1
"""
fitted_m1, m1_specs = m1.split()
regression_frame = construct_features(time_range="test", specs=m1_specs)
x_test = regression_frame.iloc[:, 1:]
y_test = np.array(regression_frame.iloc[:, 0])
try:
y_hat_test = fitted_m1.predict(x_test)
except TypeError:
y_hat_test = fitted_m1.predict(start=x_test.index[0],
end=x_test.index[-1],
exog=x_test)
# Back-transform if the data was transformed
if m1_specs.outcome_var.feature_transformation is not None:
y_test = m1_specs.outcome_var.feature_transformation.back_transform_value(
y_test)
y_hat_test = m1_specs.outcome_var.feature_transformation.back_transform_value(
y_hat_test)
print("rmse = %s" % (str(
round(sm.tools.eval_measures.rmse(y_test, y_hat_test, axis=0), 4))))
plot_true_versus_predicted(regression_frame.index, y_test, y_hat_test,
None, None, plot_path)
plot_error_graph(y_test, y_hat_test, plot_path=plot_path)
def test_generic_model(
asset_type: str,
asset: Optional[str] = None,
from_date: str = "2015-03-10",
period: int = 3,
horizon: int = 1,
training: int = 30,
):
"""Manually test integration of timetomodel for our generic model."""
asset_type_name = asset_type
if asset is None:
asset_name = Asset.query.filter_by(
asset_type_name=asset_type_name).first().name
else:
asset_name = asset
start = as_server_time(datetime.strptime(from_date, "%Y-%m-%d"))
end = start + timedelta(days=period)
training_and_testing_period = timedelta(days=training)
horizon = timedelta(hours=horizon)
with app.app_context():
asset = (Asset.query.filter_by(
asset_type_name=asset_type_name).filter_by(
name=asset_name).first())
market = (Market.query.filter_by(
market_type_name=asset_type_name).filter_by(
name=asset_name).first())
sensor = (WeatherSensor.query.filter_by(
weather_sensor_type_name=asset_type_name).filter_by(
name=asset_name).first())
if asset:
generic_asset = asset
elif market:
generic_asset = market
elif sensor:
generic_asset = sensor
else:
click.echo(
"No such assets in db, so I will not add any forecasts.")
return
linear_model_configurator = lookup_model_specs_configurator("linear")
(
model_specs,
model_identifier,
fallback_model_identifier,
) = linear_model_configurator(
generic_asset=generic_asset,
forecast_start=start,
forecast_end=end,
forecast_horizon=horizon,
custom_model_params=dict(
training_and_testing_period=training_and_testing_period),
)
# Create and train the model
model = create_fitted_model(model_specs, model_identifier)
print("\n\nparams:\n%s\n\n" % model.params)
evaluate_models(m1=ModelState(model, model_specs), plot_path=None)
return ModelState(model, model_specs)