def models(target_var: str = "VCI1M"):
# NO IGNORE VARS
ignore_vars = None
# drop the target variable from ignore_vars
# ignore_vars = [v for v in ignore_vars if v != target_var]
# assert target_var not in ignore_vars
# -------------
# persistence
# -------------
parsimonious()
# regression(ignore_vars=ignore_vars)
# gbdt(ignore_vars=ignore_vars)
# linear_nn(ignore_vars=ignore_vars)
# -------------
# LSTM
# -------------
rnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50,
early_stopping=5,
hidden_size=256,
include_latlons=True,
)
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
include_latlons=True,
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" /
f"one_month_forecast_BOKU_{target_var}_adede_only_vars",
)
def __init__(
self, train_length: int, train_hilo: str, test_hilo: str, test_length: int = 3
):
self.train_length = train_length
self.train_hilo = train_hilo
self.test_hilo = test_hilo
self.test_length = test_length
assert train_hilo in ["high", "med", "low"]
assert test_hilo in ["high", "med", "low"]
try:
earnn(pretrained=False, ignore_vars=ignore_vars, static=None)
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: EALSTM \n{'*'*10}\n")
def ealstm(ignore_vars, static="features"):
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
surrounding_pixels=None,
pretrained=False,
explain=False,
ignore_vars=ignore_vars,
num_epochs=50,
early_stopping=10,
hidden_size=256,
static_embedding_size=64,
# static data
static=static,
include_pred_month=True,
include_latlons=True,
include_prev_y=True,
)
"tprate_std_3",
"tprate_mean_3",
]
persistence(experiment="nowcast")
# regression(ignore_vars=always_ignore_vars)
# gbdt(ignore_vars=always_ignore_vars)
# linear_nn(ignore_vars=always_ignore_vars)
# rnn(ignore_vars=always_ignore_vars)
earnn(
experiment="nowcast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=1, # 50,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "nowcast",
to_path=data_path / "models" / "nowcast_tommy",
with_datetime=True,
)
def run_all_models_as_experiments(
vars_to_include: List[str],
ignore_vars: List[str],
static: bool,
run_regression: bool = True,
):
print(f"Experiment {vars_to_include} Static: {static}")
# RUN EXPERIMENTS
if run_regression:
regression(ignore_vars=ignore_vars, include_static=static)
if static:
# 'embeddings' or 'features'
try:
linear_nn(ignore_vars=ignore_vars, static="embeddings")
except KeyboardInterrupt:
raise
except Exception as e:
logging.debug(
f"\n{'*'*10}\n FAILED: LinearNN for vars={vars_to_include} static={static}\n{'*'*10}\n"
)
logging.debug(e)
try:
rnn(ignore_vars=ignore_vars, static="embeddings")
except KeyboardInterrupt:
raise
except Exception as e:
logging.debug(
f"\n{'*'*10}\n FAILED: RNN for vars={vars_to_include} static={static}\n{'*'*10}\n"
)
logging.debug(e)
try:
earnn(pretrained=False,
ignore_vars=ignore_vars,
static="embeddings")
except KeyboardInterrupt:
raise
except Exception as e:
logging.debug(
f"\n{'*'*10}\n FAILED: EALSTM for vars={vars_to_include} static={static}\n{'*'*10}\n"
)
logging.debug(e)
else:
try:
linear_nn(ignore_vars=ignore_vars, static=None)
except KeyboardInterrupt:
raise
except Exception as e:
logging.debug(
f"\n{'*'*10}\n FAILED: LinearNN for vars={vars_to_include} static={static}\n{'*'*10}\n"
)
logging.debug(e)
try:
rnn(ignore_vars=ignore_vars, static=None)
except KeyboardInterrupt:
raise
except Exception as e:
logging.debug(
f"\n{'*'*10}\n FAILED: RNN for vars={vars_to_include} static={static}\n{'*'*10}\n"
)
logging.debug(e)
try:
# NO NEED to run the EALSTM without static data because
# just equivalent to the RNN
earnn(pretrained=False, ignore_vars=ignore_vars, static=None)
except KeyboardInterrupt:
raise
except Exception as e:
logging.debug(
f"\n{'*'*10}\n FAILED: EALSTM for vars={vars_to_include} static={static}\n{'*'*10}\n"
)
logging.debug(e)
# RENAME DIRECTORY
data_dir = get_data_path().absolute()
rename_model_experiment_file(data_dir, vars_to_include, static)
print(f"Experiment {vars_to_include} finished")
important_vars = ["VCI", "precip", "t2m", "pev", "E", "SMsurf", "SMroot"]
always_ignore_vars = ["ndvi", "p84.162", "sp", "tp", "Eb"]
# persistence()
# regression(ignore_vars=always_ignore_vars, predict_delta=True)
# gbdt(ignore_vars=always_ignore_vars, predict_delta=True)
# linear_nn(ignore_vars=always_ignore_vars, predict_delta=True)
# rnn(ignore_vars=always_ignore_vars, predict_delta=True)
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=50, # 50
early_stopping=5, # 5
hidden_size=256,
static_embedding_size=64,
predict_delta=True,
spatial_mask="data/interim/boundaries_preprocessed/kenya_asal_mask.nc",
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" / "one_month_forecast_predict_delta_ASAL",
with_datetime=False,
def run_experiments(
train_hilo: str,
test_hilo: str,
train_length: int,
static: bool,
ignore_vars: Optional[List[str]] = None,
run_regression: bool = True,
all_models: bool = False,
):
# run baseline model
print("\n\nBASELINE MODEL:")
persistence()
print("\n\n")
# RUN EXPERIMENTS
if run_regression:
regression(ignore_vars=ignore_vars, include_static=static)
if static:
# 'embeddings' or 'features'
try:
earnn(pretrained=False,
ignore_vars=ignore_vars,
static="embeddings")
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: EALSTM \n{'*'*10}\n")
if all_models: # run all other models ?
try:
linear_nn(ignore_vars=ignore_vars, static="embeddings")
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: LinearNN \n{'*'*10}\n")
try:
rnn(ignore_vars=ignore_vars, static="embeddings")
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: RNN \n{'*'*10}\n")
try:
earnn(pretrained=False, ignore_vars=ignore_vars, static=None)
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: EALSTM \n{'*'*10}\n")
else: # NO STATIC data
try:
rnn(ignore_vars=ignore_vars, static=None)
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: RNN \n{'*'*10}\n")
if all_models: # run all other models ?
try:
linear_nn(ignore_vars=ignore_vars, static=None)
except RuntimeError:
print(f"\n{'*'*10}\n FAILED: LinearNN \n{'*'*10}\n")
# RENAME DIRECTORY
data_dir = get_data_path()
rename_experiment_dir(data_dir,
train_hilo=train_hilo,
test_hilo=test_hilo,
train_length=train_length)
print(
f"\n**Experiment finished**\n",
"train_length: " + str(train_length),
"test_hilo: " + test_hilo,
"train_hilo: " + train_hilo,
"\ntrain_years:\n",
train_years,
"\n",
"test_years:\n",
test_years,
)
clear_nans=True,
weight_observations=False,
)
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=num_epochs,
early_stopping=early_stopping,
hidden_size=hidden_size,
static_embedding_size=static_size,
include_latlons=True,
include_yearly_aggs=False,
clear_nans=True,
weight_observations=False,
pred_month_static=False,
)
# rename the output file
data_path = get_data_path()
# _rename_directory(
# from_path=data_path / "models" / "one_month_forecast",
# to_path=data_path / "models" / "one_month_forecast_BASE_static_vars",
def models(
target_var: str = "boku_VCI",
adede_only=False,
experiment_name=None,
check_inversion=False,
):
if adede_only:
ignore_vars = [
"p84.162",
"sp",
"tp",
"Eb",
"VCI",
"modis_ndvi",
"pev",
"t2m",
"E",
"SMroot",
"SMsurf",
]
else:
ignore_vars = [
"p84.162",
"sp",
"tp",
"Eb",
"VCI",
"modis_ndvi",
"SMroot",
"SMsurf",
]
# drop the target variable from ignore_vars
ignore_vars = [v for v in ignore_vars if v != target_var]
assert target_var not in ignore_vars
# -------------
# persistence
# -------------
parsimonious()
# regression(ignore_vars=ignore_vars)
# gbdt(ignore_vars=ignore_vars)
# linear_nn(ignore_vars=ignore_vars)
# -------------
# LSTM
# -------------
rnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50, # 1, # 50 ,
early_stopping=5,
hidden_size=256,
include_latlons=True,
check_inversion=check_inversion,
)
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50, # 1, # 50 ,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
include_latlons=True,
check_inversion=check_inversion,
)
# rename the output file
data_path = get_data_path()
if experiment_name is None:
experiment_name = (
f"one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}",
)
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" / experiment_name,
)