def create_usgs(meta_data_dir: str, precip_path: str, start: int, end: int):
gage_list = sorted(os.listdir(meta_data_dir))
exceptions = {}
client = get_storage_client()
for i in range(start, end):
try:
file_name = gage_list[i]
gage_id = file_name.split("stations")[0]
with open(os.path.join(meta_data_dir, file_name)) as f:
print(os.path.join(meta_data_dir, file_name))
data = json.load(f)
if len(gage_id) == 7:
gage_id = "0" + gage_id
raw_df = make_usgs_data(
datetime(2014, 1, 1), datetime(2019, 1, 1), gage_id
)
else:
raw_df = make_usgs_data(
datetime(2014, 1, 1), datetime(2019, 1, 1), gage_id
)
df, max_flow, min_flow = process_intermediate_csv(raw_df)
data["time_zone_code"] = df["tz_cd"].iloc[0]
data["max_flow"] = max_flow
data["min_flow"] = min_flow
precip_df = pd.read_csv(
os.path.join(
precip_path, data["stations"][0]["station_id"] + ".csv"
)
)
fixed_df, nan_flow, nan_precip = combine_data(df, precip_df)
data["nan_flow"] = nan_flow
data["nan_precip"] = nan_precip
joined_name = (
str(gage_id) + data["stations"][0]["station_id"] + "_flow.csv"
)
joined_upload = "joined/" + joined_name
meta_path = os.path.join(meta_data_dir, file_name)
data["files"] = [joined_name]
fixed_df.to_csv(joined_name)
with open(meta_path, "w") as f:
json.dump(data, f)
upload_file("predict_cfs", "meta2/" + file_name, meta_path, client)
upload_file("predict_cfs", joined_upload, joined_name, client)
except Exception as e:
exceptions[str(gage_id)] = str(e)
with open("exceptions.json", "w+") as a:
json.dump(exceptions, a)
print("exception")
upload_file(
"predict_cfs",
"meta2/" + "exceptions.json",
"exceptions.json",
client,
)
def __init__(self, model_base: str, training_data: str,
validation_data: str, test_data: str, params: Dict):
self.params = params
if "weight_path" in params:
self.model = self.load_model(model_base, params["model_params"],
params["weight_path"])
else:
self.model = self.load_model(model_base, params["model_params"])
self.training = self.make_data_load(training_data,
params["dataset_params"], "train")
self.validation = self.make_data_load(validation_data,
params["dataset_params"],
"valid")
self.test_data = self.make_data_load(test_data,
params["dataset_params"], "test")
if "GCS" in self.params and self.params["GCS"] != False:
self.gcs_client = get_storage_client()
else:
self.gcs_client = None
self.wandb = self.wandb_init()
def __init__(
self,
model_base: str,
training_data: str,
validation_data: str,
test_data: str,
params: Dict):
self.params = params
if "weight_path" in params:
params["weight_path"] = get_data(params["weight_path"])
self.model = self.load_model(model_base, params["model_params"], params["weight_path"])
else:
self.model = self.load_model(model_base, params["model_params"])
params["dataset_params"]["forecast_test_len"] = params["inference_params"]["hours_to_forecast"]
self.training = self.make_data_load(training_data, params["dataset_params"], "train")
self.validation = self.make_data_load(validation_data, params["dataset_params"], "valid")
self.test_data = self.make_data_load(test_data, params["dataset_params"], "test")
if "GCS" in self.params and self.params["GCS"]:
self.gcs_client = get_storage_client()
else:
self.gcs_client = None
self.wandb = self.wandb_init()
