def sort_var(data,
metadata,
global_data,
output_dir,
key,
complete_data,
time_var='time_yyyymm'):
print(key)
str_template = glob.glob(os.path.join(data[key], '*.tif'))[0]
(year_pos, month_pos
) = find_possible_dates.find_possible_dates_negative(str_template)
if time_var == 'time_yyyymm':
files, dates = becgis.sort_files(data[key],
year_pos,
month_position=month_pos)[0:2]
else:
files, dates = becgis.sort_files(data[key], year_pos)[0:2]
var_name = key.split('_folder')[0]
files = becgis.match_proj_res_ndv(metadata['lu'],
files,
os.path.join(output_dir, 'data',
var_name),
dtype='Float32')
complete_data[var_name] = (files, dates)
return complete_data
def sort_data_short(output_dir, metadata):
data = [
'p', 'et', 'n', 'ndm', 'lai', 'etref', 'etb', 'etg', 'i', 't', 'r',
'bf', 'sr', 'tr', 'perc', 'dperc', 'supply_total', 'dro'
]
complete_data = dict()
for datatype in data:
try:
folder = os.path.join(output_dir, metadata['name'], 'data',
datatype)
for fn in glob.glob(folder + "\\*_km3.tif"):
os.remove(fn)
if datatype in ['ETblueWP', 'ETgreenWP']:
files, dates = becgis.sort_files(folder, [-8, -4])[0:2]
else:
files, dates = becgis.sort_files(folder, [-10, -6],
month_position=[-6, -4])[0:2]
complete_data[datatype] = (files, dates)
except:
#traceback.print_exc()
print(datatype)
continue
data_2dict = {
'supply_sw': 'supply_sw',
'supply_gw': 'supply_gw',
'return_gwsw': 'return_flow_gw_sw',
'return_swsw': 'return_flow_sw_sw',
'return_gwgw': 'return_flow_gw_gw',
'return_swgw': 'return_flow_sw_gw',
r'fractions\fractions': 'fractions'
}
for datatype in list(data_2dict.keys()):
try:
folder = os.path.join(output_dir, metadata['name'], 'data',
datatype)
for fn in glob.glob(folder + "\\*_km3.tif"):
os.remove(fn)
files, dates = becgis.sort_files(folder, [-11, -7],
month_position=[-6, -4])[0:2]
complete_data[data_2dict[datatype]] = (files, dates)
except:
#traceback.print_exc()
print(datatype)
continue
return complete_data
def correct_var(metadata,
complete_data,
output_dir,
formula,
new_var,
slope=False,
bounds=(0, [1.0, 1., 12.])):
var = split_form(formula)[0][-1]
a, x0 = calc_var_correction(metadata,
complete_data,
output_dir,
formula=formula,
slope=slope,
plot=True,
bounds=bounds)
for date, fn in zip(complete_data[var][1], complete_data[var][0]):
geo_info = becgis.get_geoinfo(fn)
data = becgis.open_as_array(fn, nan_values=True)
x = calc_delta_months(x0, date)
fraction = a[0] * (np.cos(
(x - a[2]) * (np.pi / 6)) * 0.5 + 0.5) + (a[1] * (1 - a[0]))
data *= fraction
folder = os.path.join(output_dir, metadata['name'], 'data', new_var)
if not os.path.exists(folder):
os.makedirs(folder)
bla = os.path.split(fn)[1].split('_')[-1]
filen = 'supply_sw_' + bla[0:6] + '.tif'
fn = os.path.join(folder, filen)
becgis.create_geotiff(fn, data, *geo_info)
meta = becgis.sort_files(folder, [-10, -6], month_position=[-6, -4])[0:2]
return a, meta
def prepareSurfWatLoop(data, global_data):
data_needed = ["etref_folder", "et_folder", "p_folder"]
data_dst = {
"etref_folder": r"ETref\Monthly",
"et_folder": r"Evaporation\ETensV1_0",
"p_folder": r"Precipitation\CHIRPS\Monthly"
}
for data_name in data_needed:
print(data_name)
files, dates = becgis.sort_files(data[data_name], [-10, -6],
month_position=[-6, -4])[0:2]
for f, d in zip(files, dates):
fp = os.path.split(f)[1]
dst = os.path.join(
os.environ["WA_HOME"], 'Loop_SW', data_dst[data_name],
fp[:-4] +
"_monthly_{0}.{1}.01.tif".format(d.year,
str(d.month).zfill(2)))
folder = os.path.split(dst)[0]
if not os.path.exists(folder):
os.makedirs(folder)
copyfile(f, dst)
pt = os.path.join(os.environ["WA_HOME"], 'Loop_SW', 'HydroSHED', 'DIR')
if not os.path.exists(pt):
os.makedirs(pt)
copyfile(global_data['dir'], os.path.join(pt, "DIR_HydroShed_-_15s.tif"))
def create_csv_yearly(input_folder,
output_folder,
sheetnb,
start_month,
year_position=[-11, -7],
month_position=[-6, -4],
header_rows=1,
header_columns=1,
minus_header_colums=None):
"""
Calculate yearly csvs from monthly csvs for complete years (i.e. with 12
months of data available).
Parameters
----------
input_folder : str
Folder containing monthly csv-files.
output_folder : str
Folder to store the yearly csv-files.
year_position : list, optional
The indices where the year is positioned in the filenames. Default connects to
the filenames generated by create_sheet4_csv.
month_position : list, optional
The indices where the month is positioned in the filenames. Default connects to
the filenames generated by create_sheet4_csv.
header_rows : int
The number of fixed rows at the top of the csv without any numerical data.
header_columns : int
The number of fixed columns at the left side of the csv without any numerical data.
Returns
-------
output_fhs : ndarray
Array with filehandles pointing to the generated yearly csv-files.
"""
fhs, dates = becgis.sort_files(input_folder,
year_position,
month_position=month_position,
extension='csv')[0:2]
water_dates = np.copy(dates)
for w in water_dates:
if w.month < start_month:
water_dates[water_dates == w] = datetime.date(
w.year - 1, w.month, w.day)
years, years_counts = np.unique([date.year for date in water_dates],
return_counts=True)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
reader = csv.reader(open(fhs[0], 'r'), delimiter=';')
template = np.array(list(reader))
shape = np.shape(template)
output_fhs = np.array([])
data = list()
for date in water_dates:
if date.year in years[years_counts == 12]:
reader = csv.reader(open(fhs[water_dates == date][0], 'r'),
delimiter=';')
data.append(
np.array(
list(reader))[header_rows:,
header_columns:minus_header_colums].astype(
np.float))
if len(data) == 12:
data_stack = np.stack(data)
yearly_data = np.sum(data_stack, axis=0)
data = list()
template[
header_rows:,
header_columns:minus_header_colums] = yearly_data.astype(
np.str)
fh = os.path.join(
output_folder,
'sheet_{1}_{0}.csv'.format(date.year, sheetnb))
csv_file = open(fh, 'w')
writer = csv.writer(csv_file,
delimiter=';',
lineterminator='\n')
for row_index in range(shape[0]):
writer.writerow(template[row_index, :])
output_fhs = np.append(output_fhs, fh)
csv_file.close()
return output_fhs