def make_data_load(self,
data_path: str,
dataset_params: Dict,
loader_type: str,
the_class="default"):
start_end_params = {}
# TODO clean up else if blocks
if loader_type + "_start" in dataset_params:
start_end_params["start_stamp"] = dataset_params[loader_type +
"_start"]
if loader_type + "_end" in dataset_params:
start_end_params["end_stamp"] = dataset_params[loader_type +
"_end"]
if "scaler" in dataset_params:
start_end_params["scaling"] = scaler_dict[dataset_params["scaler"]]
if "interpolate" in dataset_params:
start_end_params["interpolate_param"] = dataset_params[
"interpolate"]
if loader_type == "test" and "forecast_test_len" in dataset_params:
l = CSVDataLoader(data_path, dataset_params["forecast_history"],
dataset_params["forecast_test_len"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "default":
l = CSVDataLoader(data_path, dataset_params["forecast_history"],
dataset_params["forecast_length"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
else:
# TODO support custom DataLoader
l = None
return l
def setUp(self):
self.test_data_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "test_data")
data_base_params = {
"file_path": os.path.join(self.test_data_path, "keag_small.csv"),
"forecast_history": 20,
"forecast_length": 20,
"relevant_cols": ["cfs", "temp", "precip"],
"target_col": ["cfs"],
"interpolate_param": False,
}
self.train_loader = CSVDataLoader(
os.path.join(self.test_data_path, "keag_small.csv"),
30,
20,
target_col=["cfs"],
relevant_cols=["cfs", "precip", "temp"],
interpolate_param=False,
)
data_base_params["start_stamp"] = 20
self.test_loader = CSVTestLoader(
os.path.join(self.test_data_path, "keag_small.csv"), 336,
**data_base_params)
self.ae_loader = AEDataloader(
os.path.join(self.test_data_path, "keag_small.csv"),
relevant_cols=["cfs", "temp", "precip"],
)
data_base_params["end_stamp"] = 220
self.train_loader2 = CSVDataLoader(**data_base_params)
def make_data_load(
self,
data_path: str,
dataset_params: Dict,
loader_type: str,
the_class="default"):
start_end_params = {}
the_class = dataset_params["class"]
start_end_params = scaling_function(start_end_params, dataset_params)
# TODO clean up else if blocks
if loader_type + "_start" in dataset_params:
start_end_params["start_stamp"] = dataset_params[loader_type + "_start"]
if loader_type + "_end" in dataset_params:
start_end_params["end_stamp"] = dataset_params[loader_type + "_end"]
if "interpolate" in dataset_params:
start_end_params["interpolate_param"] = dataset_params["interpolate"]
if "feature_param" in dataset_params:
start_end_params["feature_params"] = dataset_params["feature_param"]
"Feature param put into stuff"
if "sort_column" in dataset_params:
start_end_params["sort_column"] = dataset_params["sort_column"]
if "scaled_cols" in dataset_params:
start_end_params["scaled_cols"] = dataset_params["scaled_cols"]
if "no_scale" in dataset_params:
start_end_params["no_scale"] = dataset_params["no_scale"]
is_proper_dataloader = loader_type == "test" and the_class == "default"
if is_proper_dataloader and "forecast_test_len" in dataset_params:
loader = CSVDataLoader(
data_path,
dataset_params["forecast_history"],
dataset_params["forecast_test_len"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "default":
loader = CSVDataLoader(
data_path,
dataset_params["forecast_history"],
dataset_params["forecast_length"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "AutoEncoder":
loader = AEDataloader(
data_path,
dataset_params["relevant_cols"],
**start_end_params
)
else:
# TODO support custom DataLoader
loader = None
return loader
def setUp(self):
self.model = SimpleTransformer(3, 30, 20)
self.data_test_path = os.path.join(
os.path.dirname(
os.path.abspath(__file__)),
"test_init",
"chick_final.csv")
self.validation_loader = DataLoader(
CSVDataLoader(
self.data_test_path,
forecast_history=30,
forecast_length=20,
target_col=['cfs'],
relevant_cols=[
'cfs',
'temp',
'precip'],
interpolate_param=False),
shuffle=False,
sampler=None,
batch_sampler=None,
num_workers=0,
collate_fn=None,
pin_memory=False,
drop_last=False,
timeout=0,
worker_init_fn=None)
self.sequence_size = 30
def test_loader_get_gcs_data(self):
test_loader = CSVDataLoader(
file_path="gs://task_ts_data/2020-08-17/Afghanistan____.csv",
forecast_history=14,
forecast_length=14,
target_col=["cases"],
relevant_cols=["cases", "recovered", "active", "deaths"],
sort_column="date",
interpolate_param=False,
gcp_service_key=
None, # for CircleCI tests, local test needs key.json
)
self.assertEqual(
test_loader.local_file_path,
"data/task_ts_data/2020-08-17/Afghanistan____.csv",
)
def make_data_load(self,
data_path: str,
dataset_params: Dict,
loader_type: str,
the_class="default"):
start_end_params = {}
the_class = dataset_params["class"]
start_end_params = scaling_function(start_end_params, dataset_params)
# TODO clean up else if blocks
if loader_type + "_start" in dataset_params:
start_end_params["start_stamp"] = dataset_params[loader_type +
"_start"]
if loader_type + "_end" in dataset_params:
start_end_params["end_stamp"] = dataset_params[loader_type +
"_end"]
if "interpolate" in dataset_params:
start_end_params["interpolate_param"] = dataset_params[
"interpolate"]
if "feature_param" in dataset_params:
start_end_params["feature_params"] = dataset_params[
"feature_param"]
"Feature param put into stuff"
if "sort_column" in dataset_params:
start_end_params["sort_column"] = dataset_params["sort_column"]
if "scaled_cols" in dataset_params:
start_end_params["scaled_cols"] = dataset_params["scaled_cols"]
if "no_scale" in dataset_params:
start_end_params["no_scale"] = dataset_params["no_scale"]
if "id_series_col" in dataset_params:
start_end_params["id_series_col"] = dataset_params["id_series_col"]
if the_class == "AutoEncoder":
start_end_params["forecast_history"] = dataset_params[
"forecast_history"]
start_end_params["target_col"] = dataset_params["relevant_cols"]
is_proper_dataloader = loader_type == "test" and the_class == "default"
if is_proper_dataloader and "forecast_test_len" in dataset_params:
loader = CSVDataLoader(data_path,
dataset_params["forecast_history"],
dataset_params["forecast_test_len"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "default":
loader = CSVDataLoader(data_path,
dataset_params["forecast_history"],
dataset_params["forecast_length"],
dataset_params["target_col"],
dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "AutoEncoder":
loader = AEDataloader(data_path, dataset_params["relevant_cols"],
**start_end_params)
elif the_class == "TemporalLoader":
start_end_params = self.__re_add_params__(start_end_params,
dataset_params,
data_path)
label_len = 0
if "label_len" in dataset_params:
label_len = dataset_params["label_len"]
loader = TemporalLoader(dataset_params["temporal_feats"],
start_end_params,
label_len=label_len)
elif the_class == "SeriesIDLoader":
start_end_params = self.__re_add_params__(start_end_params,
dataset_params,
data_path)
loader = CSVSeriesIDLoader(dataset_params["series_id_col"],
start_end_params,
dataset_params["return_method"])
elif the_class == "GeneralClassificationLoader":
dataset_params["forecast_length"] = 1
start_end_params = self.__re_add_params__(start_end_params,
dataset_params,
data_path)
start_end_params["sequence_length"] = dataset_params[
"sequence_length"]
loader = GeneralClassificationLoader(start_end_params,
dataset_params["n_classes"])
else:
# TODO support custom DataLoader
loader = None
return loader