def permutation_test(cfg,
k: int = 2,
epoch: Optional[int] = 30,
split: str = "val") -> Dict:
db_path = str(cfg["run_dir"] / "attributes.db")
if "attribute_selection_file" in cfg.keys():
attribute_selection = np.genfromtxt(cfg["attribute_selection_file"],
dtype="str")
tqdm.write(f"Using {len(attribute_selection)} static features")
else:
attribute_selection = None
i_dict = defaultdict(dict)
basins = get_basin_list(cfg[f"{split}_basin_file"])
features = load_attributes(db_path=db_path,
basins=basins,
keep_features=attribute_selection).columns
for i, feature in enumerate(tqdm(features)):
for k_ in range(k):
nse_values = evaluate(
user_cfg=cfg,
split=split,
epoch=epoch,
store=False,
permutate_feature=feature,
)
i_dict[feature][k_] = nse_values
return i_dict
from pathlib import Path
from camelsml import evaluate, load_config
cfg = load_config("run_config.txt")
cfg["val_basin_file"] = Path(
f"gb_split/split_seed_19970204/basins_validation.txt")
for k in range(2, 5):
runs = list(Path(f"/work/bernharl/train_us_val_gb/{k}").glob("*"))
print(runs)
if len(runs) != 1:
raise RuntimeError(
f"Amount of runs per cross val should be 1, not {len(runs)}")
cfg["run_dir"] = runs[0]
cfg["train_basin_file"] = Path(
f"cv/cross_validation_seed_19970204/{k}/basins_train.txt")
cfg["eval_dir"] = Path(f"/work/bernharl/train_us_val_gb/val_gb/{k}")
for i in range(1, cfg["epochs"] + 1):
evaluate(cfg, split="val", epoch=i)
from pathlib import Path
from camelsml import load_config, evaluate, train
from camelsml.utils import get_basin_list
cfg = load_config(cfg_file="run_config.txt", device="cuda:0", num_workers=30)
cfg["test_basin_file"] = Path("..") / "refit_splits" / "basins_test.txt"
cfg["train_basin_file"] = Path("..") / "refit_splits" / "basins_train.txt"
cfg["evaluate_on_epoch"] = False
cfg["run_dir"] = Path() / "refit"
cfg["epochs"] = 13
#train(cfg)
finished_run = list(Path(cfg["run_dir"]).glob("*"))
if len(finished_run) != 1:
raise RuntimeError(f"Amount of runs in folder needs to be 1, not {len(finished_run)}")
cfg["run_dir"] = finished_run[0]
evaluate(cfg, split="test", epoch=cfg["epochs"])