def setUp(self):
self.test_data_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "test_data")
data_base_params = {
"file_path": os.path.join(self.test_data_path, "keag_small.csv"),
"forecast_history": 20,
"forecast_length": 20,
"relevant_cols": ["cfs", "temp", "precip"],
"target_col": ["cfs"],
"interpolate_param": False,
}
self.train_loader = CSVDataLoader(
os.path.join(self.test_data_path, "keag_small.csv"),
30,
20,
target_col=["cfs"],
relevant_cols=["cfs", "precip", "temp"],
interpolate_param=False,
)
self.test_loader = CSVTestLoader(
os.path.join(self.test_data_path, "keag_small.csv"), 336,
**data_base_params)
self.ae_loader = AEDataloader(
os.path.join(self.test_data_path, "keag_small.csv"),
relevant_cols=["cfs", "temp", "precip"],
)
def make_data_load(
self,
data_path: str,
dataset_params: Dict,
loader_type: str,
the_class="default"):
start_end_params = {}
the_class = dataset_params["class"]
start_end_params = scaling_function(start_end_params, dataset_params)
# TODO clean up else if blocks
if loader_type + "_start" in dataset_params:
start_end_params["start_stamp"] = dataset_params[loader_type + "_start"]
if loader_type + "_end" in dataset_params:
start_end_params["end_stamp"] = dataset_params[loader_type + "_end"]
if "interpolate" in dataset_params:
start_end_params["interpolate_param"] = dataset_params["interpolate"]
if "feature_param" in dataset_params:
start_end_params["feature_params"] = dataset_params["feature_param"]
"Feature param put into stuff"
if "sort_column" in dataset_params:
start_end_params["sort_column"] = dataset_params["sort_column"]
if "scaled_cols" in dataset_params:
start_end_params["scaled_cols"] = dataset_params["scaled_cols"]
if "no_scale" in dataset_params:
start_end_params["no_scale"] = dataset_params["no_scale"]
is_proper_dataloader = loader_type == "test" and the_class == "default"
if is_proper_dataloader and "forecast_test_len" in dataset_params:
loader = CSVDataLoader(
data_path,
dataset_params["forecast_history"],
dataset_params["forecast_test_len"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "default":
loader = CSVDataLoader(
data_path,
dataset_params["forecast_history"],
dataset_params["forecast_length"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "AutoEncoder":
loader = AEDataloader(
data_path,
dataset_params["relevant_cols"],
**start_end_params
)
else:
# TODO support custom DataLoader
loader = None
return loader
def make_data_load(self,
data_path: str,
dataset_params: Dict,
loader_type: str,
the_class="default"):
start_end_params = {}
the_class = dataset_params["class"]
start_end_params = scaling_function(start_end_params, dataset_params)
# TODO clean up else if blocks
if loader_type + "_start" in dataset_params:
start_end_params["start_stamp"] = dataset_params[loader_type +
"_start"]
if loader_type + "_end" in dataset_params:
start_end_params["end_stamp"] = dataset_params[loader_type +
"_end"]
if "interpolate" in dataset_params:
start_end_params["interpolate_param"] = dataset_params[
"interpolate"]
if "feature_param" in dataset_params:
start_end_params["feature_params"] = dataset_params[
"feature_param"]
"Feature param put into stuff"
if "sort_column" in dataset_params:
start_end_params["sort_column"] = dataset_params["sort_column"]
if "scaled_cols" in dataset_params:
start_end_params["scaled_cols"] = dataset_params["scaled_cols"]
if "no_scale" in dataset_params:
start_end_params["no_scale"] = dataset_params["no_scale"]
if "id_series_col" in dataset_params:
start_end_params["id_series_col"] = dataset_params["id_series_col"]
if the_class == "AutoEncoder":
start_end_params["forecast_history"] = dataset_params[
"forecast_history"]
start_end_params["target_col"] = dataset_params["relevant_cols"]
is_proper_dataloader = loader_type == "test" and the_class == "default"
if is_proper_dataloader and "forecast_test_len" in dataset_params:
loader = CSVDataLoader(data_path,
dataset_params["forecast_history"],
dataset_params["forecast_test_len"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "default":
loader = CSVDataLoader(data_path,
dataset_params["forecast_history"],
dataset_params["forecast_length"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "AutoEncoder":
loader = AEDataloader(data_path, dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "TemporalLoader":
start_end_params = self.__re_add_params__(start_end_params,
dataset_params,
data_path)
label_len = 0
if "label_len" in dataset_params:
label_len = dataset_params["label_len"]
loader = TemporalLoader(dataset_params["temporal_feats"],
start_end_params,
label_len=label_len)
elif the_class == "SeriesIDLoader":
start_end_params = self.__re_add_params__(start_end_params,
dataset_params,
data_path)
loader = CSVSeriesIDLoader(dataset_params["series_id_col"],
start_end_params,
dataset_params["return_method"])
elif the_class == "GeneralClassificationLoader":
dataset_params["forecast_length"] = 1
start_end_params = self.__re_add_params__(start_end_params,
dataset_params,
data_path)
start_end_params["sequence_length"] = dataset_params[
"sequence_length"]
loader = GeneralClassificationLoader(start_end_params,
dataset_params["n_classes"])
else:
# TODO support custom DataLoader
loader = None
return loader