def run_models(target_var: str):
parsimonious()
# -------
# LSTM
# -------
rnn( # earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
explain=False,
static=None, # "features",
ignore_vars=None,
num_epochs=50, # 50
early_stopping=5, # 5
hidden_size=256,
predict_delta=False,
normalize_y=True,
include_prev_y=False,
include_latlons=False,
)
# -------
# EALSTM
# -------
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" /
f"one_month_forecast_adede_only_target_{target_var}",
with_datetime=False,
)
def check_and_move_already_existing_nc_file(directory: Path) -> None:
"""Check for data.nc in the engineer folder and move it"""
if (directory / "data.nc").exists():
print("Require the engineered folder to be empty. Moving data.nc")
_rename_directory(directory / "data.nc",
directory / "data.nc_",
with_datetime=True)
def models(target_var: str = "VCI1M"):
# NO IGNORE VARS
ignore_vars = None
# drop the target variable from ignore_vars
# ignore_vars = [v for v in ignore_vars if v != target_var]
# assert target_var not in ignore_vars
# -------------
# persistence
# -------------
parsimonious()
# regression(ignore_vars=ignore_vars)
# gbdt(ignore_vars=ignore_vars)
# linear_nn(ignore_vars=ignore_vars)
# -------------
# LSTM
# -------------
rnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50,
early_stopping=5,
hidden_size=256,
include_latlons=True,
)
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
include_latlons=True,
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" /
f"one_month_forecast_BOKU_{target_var}_adede_only_vars",
)
def move_features_dir(target_var):
# rename the features dir
data_path = get_data_path()
_rename_directory(
from_path=data_path / "features" / "one_month_forecast",
to_path=data_path / "features" /
f"one_month_forecast_BOKU_{target_var}_adede_only_vars",
)
def move_features_dir(target_var, adede_only=False, experiment_name=None):
# rename the features dir
data_path = get_data_path()
if experiment_name is None:
experiment_name = f"one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}"
_rename_directory(
from_path=data_path / "features" / "one_month_forecast",
to_path=data_path / "features" / experiment_name,
)
def rename_model_experiment_file(data_dir: Path, vars_: List[str],
static: bool) -> None:
vars_joined = "_".join(vars_)
from_path = data_dir / "models" / "one_month_forecast"
if static:
to_path = data_dir / "models" / f"one_month_forecast_{vars_joined}_YESstatic"
else:
to_path = data_dir / "models" / f"one_month_forecast_{vars_joined}_NOstatic"
# with_datetime ensures that unique
_rename_directory(from_path, to_path, with_datetime=True)
def move_features_dir(data_path):
# FEATURES
if (data_path / "features" / "one_month_forecast").exists():
print(
"Moving data/features/one_month_forecast -> data/features/one_month_forecast_"
)
_rename_directory(
from_path=data_path / "features/one_month_forecast",
to_path=data_path / "features/one_month_forecast_",
with_datetime=False,
)
assert not (data_path / "features" / "one_month_forecast").exists()
def rename_models_dir(data_path: Path):
old_paths = [d for d in data_path.glob("*_models*")]
if old_paths == []:
integer = 0
else:
old_max = max([int(p.name.split("_")[0]) for p in old_paths])
integer = old_max + 1
_rename_directory(
from_path=data_path / "models",
to_path=data_path / f"{integer}_models",
with_datetime=False,
)
def rename_features_dir(data_path: Path):
"""increment the features dir by 1"""
old_paths = [d for d in data_path.glob("*_features*")]
if old_paths == []:
integer = 0
else:
old_max = max([int(p.name.split("_")[0]) for p in old_paths])
integer = old_max + 1
_rename_directory(
from_path=data_path / "features",
to_path=data_path / f"{integer}_features",
with_datetime=False,
)
def revert_interim_dirs():
data_path = get_data_path()
# INTERIM
print("Moving data/interim -> data/interim_adede")
_rename_directory(
from_path=data_path / "interim",
to_path=data_path / f"interim_adede_only",
with_datetime=False,
)
print("Moving data/interim_ -> data/interim")
_rename_directory(
from_path=data_path / "interim_",
to_path=data_path / "interim",
with_datetime=False,
)
def move_features_dir(target_var, adede_only=False, experiment_name=None):
# rename the features dir
data_path = get_data_path()
try:
_rename_directory(
from_path=data_path / "features" / "one_month_forecast",
to_path=data_path / "features" /
f"ICLR_one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}",
)
except Exception as E:
print(E)
date = datetime.datetime.now().strftime("%Y%M%d_%H%M")
_rename_directory(
from_path=data_path / "features" / "one_month_forecast",
to_path=data_path / "features" /
f"ICLR_one_month_forecast_BOKU_{target_var}_our_vars_{date}",
)
def rename_experiment_dir(
data_dir: Path,
train_hilo: str,
test_hilo: str,
train_length: int,
dir_: str = "models",
) -> None:
from_path = data_dir / dir_ / "one_month_forecast"
to_path = (
data_dir / dir_ /
f"one_month_forecast_TR{train_hilo}_TE{test_hilo}_LEN{train_length}")
# with_datetime ensures that unique
_rename_directory(from_path, to_path, with_datetime=True)
# 2. select randomly the TEST years from these groups
test_years = np.random.choice(test_dict[test_hilo],
test_length,
replace=False)
def revert_features_dirs(target_var: str, original_dir: bool = False):
data_path = get_data_path()
# FEATURES
print(
f"Moving data/features/one_month_forecast -> data/features/one_month_forecast_adede_{target_var}"
)
_rename_directory(
from_path=data_path / "features/one_month_forecast",
to_path=data_path / f"features/one_month_forecast_adede_{target_var}",
with_datetime=False,
)
if original_dir:
print(
"Moving data/features/one_month_forecast_ -> data/features/one_month_forecast"
)
_rename_directory(
from_path=data_path / "features/one_month_forecast_",
to_path=data_path / "features/one_month_forecast",
with_datetime=False,
)
def swap_interim_dirs(data_path):
# INTERIM
if (data_path / "interim_adede_only").exists() and (data_path /
"interim").exists():
# move interim -> interim_
# move interim_adede -> interim
print("Moving data/interim -> data/interim_")
_rename_directory(
from_path=data_path / "interim",
to_path=data_path / "interim_",
with_datetime=False,
)
print("Moving data/interim_adede_only -> data/interim")
_rename_directory(
from_path=data_path / "interim_adede_only",
to_path=data_path / "interim",
with_datetime=False,
)
elif (not (data_path / "interim_adede_only").exists()
and (data_path / "interim_").exists()):
# move interim_adede -> interim
print("Moving data/interim_adede_only -> data/interim")
_rename_directory(
from_path=data_path / "interim_adede_only",
to_path=data_path / "interim",
with_datetime=False,
)
# check that correct dirs created
assert not (data_path / "interim_adede_only").exists()
assert (data_path / "interim").exists()
assert (data_path / "interim_").exists()
def rename_dirs():
data_path = get_data_path()
# INTERIM
if (data_path / "interim_adede_only").exists() and (data_path /
"interim").exists():
# move interim -> interim_
# move interim_adede -> interim
print("Moving data/interim -> data/interim_")
_rename_directory(
from_path=data_path / "interim",
to_path=data_path / "interim_",
with_datetime=False,
)
print("Moving data/interim_adede_only -> data/interim")
_rename_directory(
from_path=data_path / "interim_adede_only",
to_path=data_path / "interim",
with_datetime=False,
)
elif (not (data_path / "interim_adede_only").exists()
and (data_path / "interim_").exists()):
# move interim_adede -> interim
print("Moving data/interim_adede_only -> data/interim")
_rename_directory(
from_path=data_path / "interim_adede_only",
to_path=data_path / "interim",
with_datetime=False,
)
# check that correct dirs created
assert not (data_path / "interim_adede_only").exists()
assert (data_path / "interim").exists()
assert (data_path / "interim_").exists()
# FEATURES
if (data_path / "features" / "one_month_forecast").exists():
print(
"Moving data/features/one_month_forecast -> data/features/one_month_forecast_"
)
_rename_directory(
from_path=data_path / "features/one_month_forecast",
to_path=data_path / "features/one_month_forecast_",
with_datetime=False,
)
assert not (data_path / "features" / "one_month_forecast").exists()
# gbdt(ignore_vars=always_ignore_vars, predict_delta=True)
# linear_nn(ignore_vars=always_ignore_vars, predict_delta=True)
# rnn(ignore_vars=always_ignore_vars, predict_delta=True)
rnn( # earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=50, # 50
early_stopping=5, # 5
hidden_size=256,
static_embedding_size=64,
predict_delta=True,
normalize_y=True,
include_prev_y=True,
include_latlons=True,
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" /
"one_month_forecast_RNN_predict_delta_norm_y_pass_prev_latlons",
with_datetime=False,
)
"tprate_mean_3",
]
persistence(experiment="nowcast")
# regression(ignore_vars=always_ignore_vars)
# gbdt(ignore_vars=always_ignore_vars)
# linear_nn(ignore_vars=always_ignore_vars)
# rnn(ignore_vars=always_ignore_vars)
earnn(
experiment="nowcast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=1, # 50,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "nowcast",
to_path=data_path / "models" / "nowcast_tommy",
with_datetime=True,
)
early_stopping=5,
hidden_size=256,
include_latlons=True,
)
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=50,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
include_latlons=True,
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" / "one_month_forecast_BASE_adede_vars",
)
def models(
target_var: str = "boku_VCI",
adede_only=False,
experiment_name=None,
check_inversion=False,
):
if adede_only:
ignore_vars = [
"p84.162",
"sp",
"tp",
"Eb",
"VCI",
"modis_ndvi",
"pev",
"t2m",
"E",
"SMroot",
"SMsurf",
]
else:
ignore_vars = [
"p84.162",
"sp",
"tp",
"Eb",
"VCI",
"modis_ndvi",
"SMroot",
"SMsurf",
]
# drop the target variable from ignore_vars
ignore_vars = [v for v in ignore_vars if v != target_var]
assert target_var not in ignore_vars
# -------------
# persistence
# -------------
parsimonious()
# regression(ignore_vars=ignore_vars)
# gbdt(ignore_vars=ignore_vars)
# linear_nn(ignore_vars=ignore_vars)
# -------------
# LSTM
# -------------
rnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50, # 1, # 50 ,
early_stopping=5,
hidden_size=256,
include_latlons=True,
check_inversion=check_inversion,
)
# -------------
# EALSTM
# -------------
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=ignore_vars,
num_epochs=50, # 1, # 50 ,
early_stopping=5,
hidden_size=256,
static_embedding_size=64,
include_latlons=True,
check_inversion=check_inversion,
)
# rename the output file
data_path = get_data_path()
if experiment_name is None:
experiment_name = (
f"one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}",
)
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" / experiment_name,
)
# persistence()
# regression(ignore_vars=always_ignore_vars, predict_delta=True)
# gbdt(ignore_vars=always_ignore_vars, predict_delta=True)
# linear_nn(ignore_vars=always_ignore_vars, predict_delta=True)
# rnn(ignore_vars=always_ignore_vars, predict_delta=True)
earnn(
experiment="one_month_forecast",
include_pred_month=True,
surrounding_pixels=None,
pretrained=False,
explain=False,
static="features",
ignore_vars=always_ignore_vars,
num_epochs=50, # 50
early_stopping=5, # 5
hidden_size=256,
static_embedding_size=64,
predict_delta=True,
spatial_mask="data/interim/boundaries_preprocessed/kenya_asal_mask.nc",
)
# rename the output file
data_path = get_data_path()
_rename_directory(
from_path=data_path / "models" / "one_month_forecast",
to_path=data_path / "models" / "one_month_forecast_predict_delta_ASAL",
with_datetime=False,
)
