def run_models(target_var: str): parsimonious() # ------- # LSTM # ------- rnn( # earnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, explain=False, static=None, # "features", ignore_vars=None, num_epochs=50, # 50 early_stopping=5, # 5 hidden_size=256, predict_delta=False, normalize_y=True, include_prev_y=False, include_latlons=False, ) # ------- # EALSTM # ------- # rename the output file data_path = get_data_path() _rename_directory( from_path=data_path / "models" / "one_month_forecast", to_path=data_path / "models" / f"one_month_forecast_adede_only_target_{target_var}", with_datetime=False, )
def check_and_move_already_existing_nc_file(directory: Path) -> None: """Check for data.nc in the engineer folder and move it""" if (directory / "data.nc").exists(): print("Require the engineered folder to be empty. Moving data.nc") _rename_directory(directory / "data.nc", directory / "data.nc_", with_datetime=True)
def models(target_var: str = "VCI1M"): # NO IGNORE VARS ignore_vars = None # drop the target variable from ignore_vars # ignore_vars = [v for v in ignore_vars if v != target_var] # assert target_var not in ignore_vars # ------------- # persistence # ------------- parsimonious() # regression(ignore_vars=ignore_vars) # gbdt(ignore_vars=ignore_vars) # linear_nn(ignore_vars=ignore_vars) # ------------- # LSTM # ------------- rnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, explain=False, static="features", ignore_vars=ignore_vars, num_epochs=50, early_stopping=5, hidden_size=256, include_latlons=True, ) # ------------- # EALSTM # ------------- earnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, pretrained=False, explain=False, static="features", ignore_vars=ignore_vars, num_epochs=50, early_stopping=5, hidden_size=256, static_embedding_size=64, include_latlons=True, ) # rename the output file data_path = get_data_path() _rename_directory( from_path=data_path / "models" / "one_month_forecast", to_path=data_path / "models" / f"one_month_forecast_BOKU_{target_var}_adede_only_vars", )
def move_features_dir(target_var): # rename the features dir data_path = get_data_path() _rename_directory( from_path=data_path / "features" / "one_month_forecast", to_path=data_path / "features" / f"one_month_forecast_BOKU_{target_var}_adede_only_vars", )
def move_features_dir(target_var, adede_only=False, experiment_name=None): # rename the features dir data_path = get_data_path() if experiment_name is None: experiment_name = f"one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}" _rename_directory( from_path=data_path / "features" / "one_month_forecast", to_path=data_path / "features" / experiment_name, )
def rename_model_experiment_file(data_dir: Path, vars_: List[str], static: bool) -> None: vars_joined = "_".join(vars_) from_path = data_dir / "models" / "one_month_forecast" if static: to_path = data_dir / "models" / f"one_month_forecast_{vars_joined}_YESstatic" else: to_path = data_dir / "models" / f"one_month_forecast_{vars_joined}_NOstatic" # with_datetime ensures that unique _rename_directory(from_path, to_path, with_datetime=True)
def move_features_dir(data_path): # FEATURES if (data_path / "features" / "one_month_forecast").exists(): print( "Moving data/features/one_month_forecast -> data/features/one_month_forecast_" ) _rename_directory( from_path=data_path / "features/one_month_forecast", to_path=data_path / "features/one_month_forecast_", with_datetime=False, ) assert not (data_path / "features" / "one_month_forecast").exists()
def rename_models_dir(data_path: Path): old_paths = [d for d in data_path.glob("*_models*")] if old_paths == []: integer = 0 else: old_max = max([int(p.name.split("_")[0]) for p in old_paths]) integer = old_max + 1 _rename_directory( from_path=data_path / "models", to_path=data_path / f"{integer}_models", with_datetime=False, )
def rename_features_dir(data_path: Path): """increment the features dir by 1""" old_paths = [d for d in data_path.glob("*_features*")] if old_paths == []: integer = 0 else: old_max = max([int(p.name.split("_")[0]) for p in old_paths]) integer = old_max + 1 _rename_directory( from_path=data_path / "features", to_path=data_path / f"{integer}_features", with_datetime=False, )
def revert_interim_dirs(): data_path = get_data_path() # INTERIM print("Moving data/interim -> data/interim_adede") _rename_directory( from_path=data_path / "interim", to_path=data_path / f"interim_adede_only", with_datetime=False, ) print("Moving data/interim_ -> data/interim") _rename_directory( from_path=data_path / "interim_", to_path=data_path / "interim", with_datetime=False, )
def move_features_dir(target_var, adede_only=False, experiment_name=None): # rename the features dir data_path = get_data_path() try: _rename_directory( from_path=data_path / "features" / "one_month_forecast", to_path=data_path / "features" / f"ICLR_one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}", ) except Exception as E: print(E) date = datetime.datetime.now().strftime("%Y%M%d_%H%M") _rename_directory( from_path=data_path / "features" / "one_month_forecast", to_path=data_path / "features" / f"ICLR_one_month_forecast_BOKU_{target_var}_our_vars_{date}", )
def rename_experiment_dir( data_dir: Path, train_hilo: str, test_hilo: str, train_length: int, dir_: str = "models", ) -> None: from_path = data_dir / dir_ / "one_month_forecast" to_path = ( data_dir / dir_ / f"one_month_forecast_TR{train_hilo}_TE{test_hilo}_LEN{train_length}") # with_datetime ensures that unique _rename_directory(from_path, to_path, with_datetime=True) # 2. select randomly the TEST years from these groups test_years = np.random.choice(test_dict[test_hilo], test_length, replace=False)
def revert_features_dirs(target_var: str, original_dir: bool = False): data_path = get_data_path() # FEATURES print( f"Moving data/features/one_month_forecast -> data/features/one_month_forecast_adede_{target_var}" ) _rename_directory( from_path=data_path / "features/one_month_forecast", to_path=data_path / f"features/one_month_forecast_adede_{target_var}", with_datetime=False, ) if original_dir: print( "Moving data/features/one_month_forecast_ -> data/features/one_month_forecast" ) _rename_directory( from_path=data_path / "features/one_month_forecast_", to_path=data_path / "features/one_month_forecast", with_datetime=False, )
def swap_interim_dirs(data_path): # INTERIM if (data_path / "interim_adede_only").exists() and (data_path / "interim").exists(): # move interim -> interim_ # move interim_adede -> interim print("Moving data/interim -> data/interim_") _rename_directory( from_path=data_path / "interim", to_path=data_path / "interim_", with_datetime=False, ) print("Moving data/interim_adede_only -> data/interim") _rename_directory( from_path=data_path / "interim_adede_only", to_path=data_path / "interim", with_datetime=False, ) elif (not (data_path / "interim_adede_only").exists() and (data_path / "interim_").exists()): # move interim_adede -> interim print("Moving data/interim_adede_only -> data/interim") _rename_directory( from_path=data_path / "interim_adede_only", to_path=data_path / "interim", with_datetime=False, ) # check that correct dirs created assert not (data_path / "interim_adede_only").exists() assert (data_path / "interim").exists() assert (data_path / "interim_").exists()
def rename_dirs(): data_path = get_data_path() # INTERIM if (data_path / "interim_adede_only").exists() and (data_path / "interim").exists(): # move interim -> interim_ # move interim_adede -> interim print("Moving data/interim -> data/interim_") _rename_directory( from_path=data_path / "interim", to_path=data_path / "interim_", with_datetime=False, ) print("Moving data/interim_adede_only -> data/interim") _rename_directory( from_path=data_path / "interim_adede_only", to_path=data_path / "interim", with_datetime=False, ) elif (not (data_path / "interim_adede_only").exists() and (data_path / "interim_").exists()): # move interim_adede -> interim print("Moving data/interim_adede_only -> data/interim") _rename_directory( from_path=data_path / "interim_adede_only", to_path=data_path / "interim", with_datetime=False, ) # check that correct dirs created assert not (data_path / "interim_adede_only").exists() assert (data_path / "interim").exists() assert (data_path / "interim_").exists() # FEATURES if (data_path / "features" / "one_month_forecast").exists(): print( "Moving data/features/one_month_forecast -> data/features/one_month_forecast_" ) _rename_directory( from_path=data_path / "features/one_month_forecast", to_path=data_path / "features/one_month_forecast_", with_datetime=False, ) assert not (data_path / "features" / "one_month_forecast").exists()
# gbdt(ignore_vars=always_ignore_vars, predict_delta=True) # linear_nn(ignore_vars=always_ignore_vars, predict_delta=True) # rnn(ignore_vars=always_ignore_vars, predict_delta=True) rnn( # earnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, pretrained=False, explain=False, static="features", ignore_vars=always_ignore_vars, num_epochs=50, # 50 early_stopping=5, # 5 hidden_size=256, static_embedding_size=64, predict_delta=True, normalize_y=True, include_prev_y=True, include_latlons=True, ) # rename the output file data_path = get_data_path() _rename_directory( from_path=data_path / "models" / "one_month_forecast", to_path=data_path / "models" / "one_month_forecast_RNN_predict_delta_norm_y_pass_prev_latlons", with_datetime=False, )
"tprate_mean_3", ] persistence(experiment="nowcast") # regression(ignore_vars=always_ignore_vars) # gbdt(ignore_vars=always_ignore_vars) # linear_nn(ignore_vars=always_ignore_vars) # rnn(ignore_vars=always_ignore_vars) earnn( experiment="nowcast", include_pred_month=True, surrounding_pixels=None, pretrained=False, explain=False, static="features", ignore_vars=always_ignore_vars, num_epochs=1, # 50, early_stopping=5, hidden_size=256, static_embedding_size=64, ) # rename the output file data_path = get_data_path() _rename_directory( from_path=data_path / "models" / "nowcast", to_path=data_path / "models" / "nowcast_tommy", with_datetime=True, )
early_stopping=5, hidden_size=256, include_latlons=True, ) # ------------- # EALSTM # ------------- earnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, pretrained=False, explain=False, static="features", ignore_vars=always_ignore_vars, num_epochs=50, early_stopping=5, hidden_size=256, static_embedding_size=64, include_latlons=True, ) # rename the output file data_path = get_data_path() _rename_directory( from_path=data_path / "models" / "one_month_forecast", to_path=data_path / "models" / "one_month_forecast_BASE_adede_vars", )
def models( target_var: str = "boku_VCI", adede_only=False, experiment_name=None, check_inversion=False, ): if adede_only: ignore_vars = [ "p84.162", "sp", "tp", "Eb", "VCI", "modis_ndvi", "pev", "t2m", "E", "SMroot", "SMsurf", ] else: ignore_vars = [ "p84.162", "sp", "tp", "Eb", "VCI", "modis_ndvi", "SMroot", "SMsurf", ] # drop the target variable from ignore_vars ignore_vars = [v for v in ignore_vars if v != target_var] assert target_var not in ignore_vars # ------------- # persistence # ------------- parsimonious() # regression(ignore_vars=ignore_vars) # gbdt(ignore_vars=ignore_vars) # linear_nn(ignore_vars=ignore_vars) # ------------- # LSTM # ------------- rnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, explain=False, static="features", ignore_vars=ignore_vars, num_epochs=50, # 1, # 50 , early_stopping=5, hidden_size=256, include_latlons=True, check_inversion=check_inversion, ) # ------------- # EALSTM # ------------- earnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, pretrained=False, explain=False, static="features", ignore_vars=ignore_vars, num_epochs=50, # 1, # 50 , early_stopping=5, hidden_size=256, static_embedding_size=64, include_latlons=True, check_inversion=check_inversion, ) # rename the output file data_path = get_data_path() if experiment_name is None: experiment_name = ( f"one_month_forecast_BOKU_{target_var}_our_vars_{'only_P_VCI' if adede_only else 'ALL'}", ) _rename_directory( from_path=data_path / "models" / "one_month_forecast", to_path=data_path / "models" / experiment_name, )
# persistence() # regression(ignore_vars=always_ignore_vars, predict_delta=True) # gbdt(ignore_vars=always_ignore_vars, predict_delta=True) # linear_nn(ignore_vars=always_ignore_vars, predict_delta=True) # rnn(ignore_vars=always_ignore_vars, predict_delta=True) earnn( experiment="one_month_forecast", include_pred_month=True, surrounding_pixels=None, pretrained=False, explain=False, static="features", ignore_vars=always_ignore_vars, num_epochs=50, # 50 early_stopping=5, # 5 hidden_size=256, static_embedding_size=64, predict_delta=True, spatial_mask="data/interim/boundaries_preprocessed/kenya_asal_mask.nc", ) # rename the output file data_path = get_data_path() _rename_directory( from_path=data_path / "models" / "one_month_forecast", to_path=data_path / "models" / "one_month_forecast_predict_delta_ASAL", with_datetime=False, )