def persistence(experiment="one_month_forecast", ):
data_path = get_data_path()
spatial_mask = data_path / "interim/boundaries_preprocessed/kenya_asal_mask.nc"
spatial_mask = None
predictor = Persistence(data_path,
experiment=experiment,
spatial_mask=spatial_mask)
predictor.evaluate(save_preds=True)
def parsimonious(experiment="one_month_forecast", ):
# if the working directory is alread ml_drought don't need ../data
if Path(".").absolute().as_posix().split("/")[-1] == "ml_drought":
data_path = Path("data")
else:
data_path = Path("../data")
predictor = Persistence(data_path, experiment=experiment)
predictor.evaluate(save_preds=True)
def parsimonious(experiment='one_month_forecast', ):
# if the working directory is alread ml_drought don't need ../data
if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
data_path = Path('data')
else:
data_path = Path('../data')
predictor = Persistence(data_path, experiment=experiment)
predictor.evaluate(save_preds=True)
def persistence(experiment="one_month_forecast", ):
predictor = Persistence(get_data_path(), experiment=experiment)
predictor.evaluate(save_preds=True)
true_da = read_true_data(data_dir=data_dir)
assert (persistence_pred_da.time == true_da.time).all()
assert (ealstm_pred_da.time == true_da.time).all()
# --------------------------------
# read in models
# --------------------------------
# READ in model (maybe want to do more predictions on historical data)
from src.models import load_model, Persistence
ealstm_path = data_dir / 'models/one_month_forecast/ealstm/model.pt'
assert ealstm_path.exists(), \
'Expected the unzipped file to have the model.pt file saved'
persistence = Persistence(data_folder=data_dir)
ealstm = load_model(model_path=ealstm_path)
# --------------------------------
# Create comparison over time barchart
# --------------------------------
scores = annual_scores(
data_path=data_dir,
models=['ealstm', 'previous_month'],
metrics=['rmse', 'r2'],
verbose=False,
to_dataframe=True
)
# create long format dataframe
def run_models(data_path, experiment):
# NOTE: this model is the same for all experiments
print("Running persistence model")
predictor = Persistence(data_path, experiment=experiment)
predictor.evaluate(save_preds=True)
# linear regression
print(f"Running Linear Regression model: {experiment}")
predictor = LinearRegression(data_path,
experiment=experiment,
include_pred_month=True,
surrounding_pixels=1)
predictor.train(num_epochs=10, early_stopping=3)
# linear network
print(f"Running Linear Neural Network model: {experiment}")
predictor = LinearNetwork(
data_folder=data_path,
experiment=experiment,
layer_sizes=[100],
include_pred_month=True,
surrounding_pixels=1,
)
predictor.train(num_epochs=10, early_stopping=3)
predictor.evaluate(save_preds=True)
predictor.save_model()
# recurrent network
print(f"Running RNN (LSTM) model: {experiment}")
predictor = RecurrentNetwork(
data_folder=data_path,
hidden_size=128,
experiment=experiment,
include_pred_month=True,
surrounding_pixels=1,
)
predictor.train(num_epochs=10, early_stopping=3)
predictor.evaluate(save_preds=True)
predictor.save_model()
# EA LSTM
print(f"Running Entity Aware LSTM model: {experiment}")
predictor = EARecurrentNetwork(
data_folder=data_path,
hidden_size=128,
experiment=experiment,
include_pred_month=True,
surrounding_pixels=1,
)
predictor.train(num_epochs=10, early_stopping=3)
predictor.evaluate(save_preds=True)
predictor.save_model()
import xarray as xr
from pathlib import Path
from src.models import Persistence, LinearRegression
data_path = Path("data")
# high level api
predictor = Persistence(data_path)
predictor.evaluate(save_preds=True)
predictor = LinearRegression(data_path)
predictor.evaluate(save_preds=True)
# 1 LOAD_TRAIN_DATA get into the guts
x, y = predictor.load_train_arrays()
# 2 ds_folder_to_np (load x/y from the train data)
train_data_path = predictor.data_path / "features/train"
out_x, out_y = [], []
for subtrain in train_data_path.iterdir():
if (subtrain / "x.nc").exists() and (subtrain / "y.nc").exists():
arrays = predictor.ds_folder_to_np(subtrain,
clear_nans=True,
return_latlons=False)
out_x.append(arrays.x)
out_y.append(arrays.y)
# 3
x, y = xr.open_dataset(folder / "x.nc"), xr.open_dataset(folder / "y.nc")
x_np, y_np = x.to_array().values, y.to_array().values
def persistence(experiment="one_month_forecast", include_yearly_aggs=False):
predictor = Persistence(get_data_path(),
experiment=experiment,
include_yearly_aggs=include_yearly_aggs)
predictor.evaluate(save_preds=True)