def lapse_rate(Dir, temperature_map, DEMmap):
"""
This function downscales the GLDAS temperature map by using the DEM map
Keyword arguments:
temperature_map -- 'C:/' path to the temperature map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average altitudes corresponding to T resolution
dest = RC.reproject_dataset_example(DEMmap, temperature_map, method=4)
DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map)
DEM_ave_data = dest.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
dest = None
# determine lapse-rate [degress Celcius per meter]
lapse_rate_number = 0.0065
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method=2)
dem_avg = dest.GetRasterBand(1).ReadAsArray()
dem_avg[dem_avg < 0] = 0
dest = None
# Open the temperature dataset
dest = RC.reproject_dataset_example(temperature_map, DEMmap, method=2)
T = dest.GetRasterBand(1).ReadAsArray()
dest = None
# Open Demmap
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap <= 0] = 0
demmap[demmap == -32768] = np.nan
# calculate first part
T = T + ((dem_avg - demmap) * lapse_rate_number)
return T
def adjust_P(Dir, pressure_map, DEMmap):
"""
This function downscales the GLDAS air pressure map by using the DEM map
Keyword arguments:
pressure_map -- 'C:/' path to the pressure map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average latitudes
destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method=4)
DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map)
DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method=2)
dem_avg = dest.GetRasterBand(1).ReadAsArray()
dest = None
# open maps as numpy arrays
dest = RC.reproject_dataset_example(pressure_map, DEMmap, method=2)
P = dest.GetRasterBand(1).ReadAsArray()
dest = None
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap <= 0] = 0
demmap[demmap == -32768] = np.nan
# calculate second part
P = P + (101.3 * ((293 - 0.0065 * (demmap - dem_avg)) / 293)**5.26 - 101.3)
os.remove(DEM_ave_out_name)
return P
def main(files_DEM_dir, files_DEM, files_Basin, files_Runoff, files_Extraction, startdate, enddate, input_nc, resolution, Format_DEM_dir, Format_DEM, Format_Basin, Format_Runoff, Format_Extraction):
# Define a year to get the epsg and geo
Startdate_timestamp = pd.Timestamp(startdate)
year = Startdate_timestamp.year
############################## Drainage Direction #####################################
# Open Array DEM dir as netCDF
if Format_DEM_dir == "NetCDF":
file_DEM_dir = os.path.join(files_DEM_dir, "%d.nc" %year)
DataCube_DEM_dir = RC.Open_nc_array(file_DEM_dir, "Drainage_Direction")
geo_out_example, epsg_example, size_X_example, size_Y_example, size_Z_example, Time_example = RC.Open_nc_info(files_DEM_dir)
# Create memory file for reprojection
gland = DC.Save_as_MEM(DataCube_DEM_dir, geo_out_example, epsg_example)
dataset_example = file_name_DEM_dir = gland
# Open Array DEM dir as TIFF
if Format_DEM_dir == "TIFF":
file_name_DEM_dir = os.path.join(files_DEM_dir,"DIR_HydroShed_-_%s.tif" %resolution)
DataCube_DEM_dir = RC.Open_tiff_array(file_name_DEM_dir)
geo_out_example, epsg_example, size_X_example, size_Y_example = RC.Open_array_info(file_name_DEM_dir)
dataset_example = file_name_DEM_dir
# Calculate Area per pixel in m2
import watools.Functions.Start.Area_converter as AC
DataCube_Area = AC.Degrees_to_m2(file_name_DEM_dir)
################################## DEM ##########################################
# Open Array DEM as netCDF
if Format_DEM == "NetCDF":
file_DEM = os.path.join(files_DEM, "%d.nc" %year)
DataCube_DEM = RC.Open_nc_array(file_DEM, "Elevation")
# Open Array DEM as TIFF
if Format_DEM == "TIFF":
file_name_DEM = os.path.join(files_DEM,"DEM_HydroShed_m_%s.tif" %resolution)
destDEM = RC.reproject_dataset_example(file_name_DEM, dataset_example, method=1)
DataCube_DEM = destDEM.GetRasterBand(1).ReadAsArray()
################################ Landuse ##########################################
# Open Array Basin as netCDF
if Format_Basin == "NetCDF":
file_Basin = os.path.join(files_Basin, "%d.nc" %year)
DataCube_Basin = RC.Open_nc_array(file_Basin, "Landuse")
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(file_Basin, "Landuse")
dest_basin = DC.Save_as_MEM(DataCube_Basin, geo_out, str(epsg))
destLU = RC.reproject_dataset_example(dest_basin, dataset_example, method=1)
DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray()
DataCube_Basin = np.zeros([size_Y_example, size_X_example])
DataCube_Basin[DataCube_LU_CR > 0] = 1
# Open Array Basin as TIFF
if Format_Basin == "TIFF":
file_name_Basin = files_Basin
destLU = RC.reproject_dataset_example(file_name_Basin, dataset_example, method=1)
DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray()
DataCube_Basin = np.zeros([size_Y_example, size_X_example])
DataCube_Basin[DataCube_LU_CR > 0] = 1
################################ Surface Runoff ##########################################
# Open Array runoff as netCDF
if Format_Runoff == "NetCDF":
DataCube_Runoff = RC.Open_ncs_array(files_Runoff, "Surface_Runoff", startdate, enddate)
size_Z_example = DataCube_Runoff.shape[0]
file_Runoff = os.path.join(files_Runoff, "%d.nc" %year)
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(file_Runoff, "Surface_Runoff")
DataCube_Runoff_CR = np.ones([size_Z_example, size_Y_example, size_X_example]) * np.nan
for i in range(0, size_Z):
DataCube_Runoff_one = DataCube_Runoff[i,:,:]
dest_Runoff_one = DC.Save_as_MEM(DataCube_Runoff_one, geo_out, str(epsg))
dest_Runoff = RC.reproject_dataset_example(dest_Runoff_one, dataset_example, method=4)
DataCube_Runoff_CR[i,:,:] = dest_Runoff.GetRasterBand(1).ReadAsArray()
DataCube_Runoff_CR[:, DataCube_LU_CR == 0] = -9999
DataCube_Runoff_CR[DataCube_Runoff_CR < 0] = -9999
# Open Array runoff as TIFF
if Format_Runoff == "TIFF":
DataCube_Runoff_CR = RC.Get3Darray_time_series_monthly(files_Runoff, startdate, enddate, Example_data = dataset_example)
################################ Surface Withdrawal ##########################################
# Open Array Extraction as netCDF
if Format_Extraction == "NetCDF":
DataCube_Extraction = RC.Open_ncs_array(files_Extraction, "Surface_Withdrawal", startdate, enddate)
size_Z_example = DataCube_Extraction.shape[0]
file_Extraction = os.path.join(files_Extraction, "%d.nc" %year)
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(file_Extraction, "Surface_Withdrawal")
DataCube_Extraction_CR = np.ones([size_Z_example, size_Y_example, size_X_example]) * np.nan
for i in range(0, size_Z):
DataCube_Extraction_one = DataCube_Extraction[i,:,:]
dest_Extraction_one = DC.Save_as_MEM(DataCube_Extraction_one, geo_out, str(epsg))
dest_Extraction = RC.reproject_dataset_example(dest_Extraction_one, dataset_example, method=4)
DataCube_Extraction_CR[i,:,:] = dest_Extraction.GetRasterBand(1).ReadAsArray()
DataCube_Extraction_CR[:, DataCube_LU_CR == 0] = -9999
DataCube_Extraction_CR[DataCube_Extraction_CR < 0] = -9999
# Open Array Extraction as TIFF
if Format_Extraction == "TIFF":
DataCube_Extraction_CR = RC.Get3Darray_time_series_monthly(files_Extraction, startdate, enddate, Example_data = dataset_example)
################################ Create input netcdf ##########################################
# Save data in one NetCDF file
geo_out_example = np.array(geo_out_example)
# Latitude and longitude
lon_ls = np.arange(size_X_example)*geo_out_example[1]+geo_out_example[0] + 0.5 * geo_out_example[1]
lat_ls = np.arange(size_Y_example)*geo_out_example[5]+geo_out_example[3] - 0.5 * geo_out_example[5]
lat_n = len(lat_ls)
lon_n = len(lon_ls)
# Create NetCDF file
nc_file = netCDF4.Dataset(input_nc, 'w')
nc_file.set_fill_on()
# Create dimensions
lat_dim = nc_file.createDimension('latitude', lat_n)
lon_dim = nc_file.createDimension('longitude', lon_n)
# Create NetCDF variables
crso = nc_file.createVariable('crs', 'i4')
crso.long_name = 'Lon/Lat Coords in WGS84'
crso.standard_name = 'crs'
crso.grid_mapping_name = 'latitude_longitude'
crso.projection = epsg_example
crso.longitude_of_prime_meridian = 0.0
crso.semi_major_axis = 6378137.0
crso.inverse_flattening = 298.257223563
crso.geo_reference = geo_out_example
lat_var = nc_file.createVariable('latitude', 'f8', ('latitude',))
lat_var.units = 'degrees_north'
lat_var.standard_name = 'latitude'
lat_var.pixel_size = geo_out_example[5]
lon_var = nc_file.createVariable('longitude', 'f8', ('longitude',))
lon_var.units = 'degrees_east'
lon_var.standard_name = 'longitude'
lon_var.pixel_size = geo_out_example[1]
Dates = pd.date_range(startdate,enddate,freq = 'MS')
time_or=np.zeros(len(Dates))
i = 0
for Date in Dates:
time_or[i] = Date.toordinal()
i += 1
nc_file.createDimension('time', None)
timeo = nc_file.createVariable('time', 'f4', ('time',))
timeo.units = 'Monthly'
timeo.standard_name = 'time'
# Variables
demdir_var = nc_file.createVariable('demdir', 'i',
('latitude', 'longitude'),
fill_value=-9999)
demdir_var.long_name = 'Flow Direction Map'
demdir_var.grid_mapping = 'crs'
dem_var = nc_file.createVariable('dem', 'f8',
('latitude', 'longitude'),
fill_value=-9999)
dem_var.long_name = 'Altitude'
dem_var.units = 'meters'
dem_var.grid_mapping = 'crs'
basin_var = nc_file.createVariable('basin', 'i',
('latitude', 'longitude'),
fill_value=-9999)
basin_var.long_name = 'Altitude'
basin_var.units = 'meters'
basin_var.grid_mapping = 'crs'
area_var = nc_file.createVariable('area', 'f8',
('latitude', 'longitude'),
fill_value=-9999)
area_var.long_name = 'area in squared meters'
area_var.units = 'squared_meters'
area_var.grid_mapping = 'crs'
runoff_var = nc_file.createVariable('Runoff_M', 'f8',
('time', 'latitude', 'longitude'),
fill_value=-9999)
runoff_var.long_name = 'Runoff'
runoff_var.units = 'm3/month'
runoff_var.grid_mapping = 'crs'
extraction_var = nc_file.createVariable('Extraction_M', 'f8',
('time', 'latitude', 'longitude'),
fill_value=-9999)
extraction_var.long_name = 'Surface water Extraction'
extraction_var.units = 'm3/month'
extraction_var.grid_mapping = 'crs'
# Load data
lat_var[:] = lat_ls
lon_var[:] = lon_ls
timeo[:] = time_or
# Static variables
demdir_var[:, :] = DataCube_DEM_dir[:, :]
dem_var[:, :] = DataCube_DEM[:, :]
basin_var[:, :] = DataCube_Basin[:, :]
area_var[:, :] = DataCube_Area[:, :]
for i in range(len(Dates)):
runoff_var[i,:,:] = DataCube_Runoff_CR[i,:,:]
for i in range(len(Dates)):
extraction_var[i,:,:] = DataCube_Extraction_CR[i,:,:]
# Close file
nc_file.close()
return()
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim):
"""
This function collects and clip LANDSAF data
Keyword arguments:
SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID)
Dir -- 'C:/' path to the WA map
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -60 and 60)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Make an array of the days of which the ET is taken
Dates = pd.date_range(Startdate, Enddate, freq='D')
# make directories
SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS')
if os.path.exists(SISdir) is False:
os.makedirs(SISdir)
SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID')
if os.path.exists(SIDdir) is False:
os.makedirs(SIDdir)
ShortwaveBasin(SourceLANDSAF,
Dir,
latlim,
lonlim,
Dates=[Startdate, Enddate])
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str)
# Open DEM map
demmap = RC.Open_tiff_array(DEMmap_str)
demmap[demmap < 0] = 0
# make lat and lon arrays)
dlat = geo_out[5]
dlon = geo_out[1]
lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat
lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon
for date in Dates:
# day of year
day = date.dayofyear
Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence(
demmap, lat, lon, day)
SIDname = os.path.join(
SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
SISname = os.path.join(
SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
#PREPARE SID MAPS
SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3)
SIDdata = SIDdest.GetRasterBand(1).ReadAsArray()
#PREPARE SIS MAPS
SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3)
SISdata = SISdest.GetRasterBand(1).ReadAsArray()
# Calculate ShortWave net
Short_Wave_Net = SIDdata * (Sloping /
Horizontal) + SISdata * 86400 / 1e6
# Calculate ShortWave Clear
Short_Wave = Sloping
Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5)
# make directories
PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky')
if os.path.exists(PathClear) is False:
os.makedirs(PathClear)
PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net')
if os.path.exists(PathNet) is False:
os.makedirs(PathNet)
# name Shortwave Clear and Net
nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameNet = os.path.join(PathNet, nameFileNet)
nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameClear = os.path.join(PathClear, nameFileClear)
# Save net and clear short wave radiation
DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj)
DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj)
return
def Calculate(WA_HOME_folder, Basin, P_Product, ET_Product, LAI_Product,
ETref_Product, Runoff_Product, Startdate, Enddate, Simulation):
"""
This functions is the main framework for calculating sheet 4.
Parameters
----------
Basin : str
Name of the basin
P_Product : str
Name of the rainfall product that will be used
ET_Product : str
Name of the evapotranspiration product that will be used
LAI_Product : str
Name of the LAI product that will be used
Runoff_Product : str
Name of the Runoff product that will be used
Moving_Averiging_Length, int
Defines the length of the moving average
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Simulation : int
Defines the simulation
"""
######################### Import WA modules ###################################
from watools.General import raster_conversions as RC
from watools.General import data_conversions as DC
import watools.Functions.Four as Four
import watools.Functions.Start as Start
import watools.Generator.Sheet4 as Generate
import watools.Functions.Start.Get_Dictionaries as GD
######################### Set General Parameters ##############################
# Get environmental variable for the Home folder
if WA_HOME_folder == '':
WA_env_paths = os.environ["WA_HOME"].split(';')
Dir_Home = WA_env_paths[0]
else:
Dir_Home = WA_HOME_folder
# Create the Basin folder
Dir_Basin = os.path.join(Dir_Home, Basin)
output_dir = os.path.join(Dir_Basin, "Simulations",
"Simulation_%d" % Simulation)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Get the boundaries of the basin based on the shapefile of the watershed
# Boundaries, Shape_file_name_shp = Start.Boundaries.Determine(Basin)
Boundaries, Example_dataset = Start.Boundaries.Determine_LU_Based(
Basin, Dir_Home)
# Find the maximum moving window value
ET_Blue_Green_Classes_dict, Moving_Window_Per_Class_dict = GD.get_bluegreen_classes(
version='1.0')
Additional_Months_tail = np.max(list(
Moving_Window_Per_Class_dict.values()))
############## Cut dates into pieces if it is needed ######################
# Check the years that needs to be calculated
years = list(
range(int(Startdate.split('-')[0]),
int(Enddate.split('-')[0]) + 1))
for year in years:
# Create .nc file if not exists
nc_outname = os.path.join(output_dir, "%d.nc" % year)
if not os.path.exists(nc_outname):
DC.Create_new_NC_file(nc_outname, Example_dataset, Basin)
# Open variables in netcdf
fh = Dataset(nc_outname)
Variables_NC = [var for var in list(fh.variables)]
fh.close()
# Create Start and End date for time chunk
Startdate_part = '%d-01-01' % int(year)
Enddate_part = '%s-12-31' % int(year)
if int(year) == int(years[0]):
Startdate_Moving_Average = pd.Timestamp(Startdate) - pd.DateOffset(
months=Additional_Months_tail)
Startdate_Moving_Average_String = Startdate_Moving_Average.strftime(
'%Y-%m-%d')
else:
Startdate_Moving_Average_String = Startdate_part
############################# Download Data ###################################
# Download data
if not "Precipitation" in Variables_NC:
Data_Path_P_Monthly = Start.Download_Data.Precipitation(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Startdate_part,
Enddate_part, P_Product)
if not "Actual_Evapotranspiration" in Variables_NC:
Data_Path_ET = Start.Download_Data.Evapotranspiration(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Startdate_part,
Enddate_part, ET_Product)
if not "Reference_Evapotranspiration" in Variables_NC:
Data_Path_ETref = Start.Download_Data.ETreference(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']],
Startdate_Moving_Average_String, Enddate_part, ETref_Product)
if not "Grey_Water_Footprint" in Variables_NC:
Data_Path_GWF = Start.Download_Data.GWF(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']])
if not "Theta_Saturated_Topsoil" in Variables_NC:
Data_Path_ThetaSat_topsoil = Start.Download_Data.Soil_Properties(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']],
Para='ThetaSat_TopSoil')
###################### Save Data as netCDF files ##############################
#______________________________Precipitation_______________________________
# 1.) Precipitation data
if not "Precipitation" in Variables_NC:
# Get the data of Precipitation and save as nc
DataCube_Prec = RC.Get3Darray_time_series_monthly(
Data_Path_P_Monthly,
Startdate_part,
Enddate_part,
Example_data=Example_dataset)
DC.Add_NC_Array_Variable(nc_outname, DataCube_Prec,
"Precipitation", "mm/month", 0.01)
del DataCube_Prec
#_______________________Reference Evaporation______________________________
# 2.) Reference Evapotranspiration data
if not "Reference_Evapotranspiration" in Variables_NC:
# Get the data of Precipitation and save as nc
DataCube_ETref = RC.Get3Darray_time_series_monthly(
Data_Path_ETref,
Startdate_part,
Enddate_part,
Example_data=Example_dataset)
DC.Add_NC_Array_Variable(nc_outname, DataCube_ETref,
"Reference_Evapotranspiration",
"mm/month", 0.01)
del DataCube_ETref
#_______________________________Evaporation________________________________
# 3.) Evapotranspiration data
if not "Actual_Evapotranspiration" in Variables_NC:
# Get the data of Evaporation and save as nc
DataCube_ET = RC.Get3Darray_time_series_monthly(
Data_Path_ET,
Startdate_part,
Enddate_part,
Example_data=Example_dataset)
DC.Add_NC_Array_Variable(nc_outname, DataCube_ET,
"Actual_Evapotranspiration", "mm/month",
0.01)
del DataCube_ET
#_____________________________________GWF__________________________________
# 4.) Grey Water Footprint data
if not "Grey_Water_Footprint" in Variables_NC:
# Get the data of grey water footprint and save as nc
GWF_Filepath = os.path.join(Dir_Basin, Data_Path_GWF,
"Gray_Water_Footprint_Fraction.tif")
dest_GWF = RC.reproject_dataset_example(GWF_Filepath,
Example_dataset,
method=1)
DataCube_GWF = dest_GWF.GetRasterBand(1).ReadAsArray()
DC.Add_NC_Array_Static(nc_outname, DataCube_GWF,
"Grey_Water_Footprint", "fraction", 0.0001)
del DataCube_GWF
#________________________Theta_Saturated_Topsoil___________________________
# 5.) Grey Water Footprint data
if not "Theta_Saturated_Topsoil" in Variables_NC:
# Get the data of grey water footprint and save as nc
ThetaSat_Filepath = os.path.join(
Dir_Basin, Data_Path_ThetaSat_topsoil,
"Theta_Saturated_Topsoil_HiHydroSoil.tif")
dest_ThetaSat = RC.reproject_dataset_example(ThetaSat_Filepath,
Example_dataset,
method=1)
DataCube_ThetaSat = dest_ThetaSat.GetRasterBand(1).ReadAsArray()
DC.Add_NC_Array_Static(nc_outname, DataCube_ThetaSat,
"Theta_Saturated_Topsoil", "fraction",
0.0001)
del DataCube_ThetaSat
####################### Calculations Sheet 4 ##############################
############## Cut dates into pieces if it is needed ######################
years = list(
range(int(Startdate.split('-')[0]),
int(Enddate.split('-')[0]) + 1))
for year in years:
if len(years) > 1.0:
if year is years[0]:
Startdate_part = Startdate
Enddate_part = '%s-12-31' % year
if year is years[-1]:
Startdate_part = '%s-01-01' % year
Enddate_part = Enddate
else:
Startdate_part = Startdate
Enddate_part = Enddate
#____________ Evapotranspiration data split in ETblue and ETgreen ____________
if not ("Blue_Evapotranspiration" in Variables_NC
or "Green_Evapotranspiration" in Variables_NC):
# Calculate Blue and Green ET
DataCube_ETblue, DataCube_ETgreen = Four.SplitET.Blue_Green(
Dir_Basin, nc_outname, ETref_Product, P_Product, Startdate,
Enddate)
DC.Add_NC_Array_Variable(nc_outname, DataCube_ETblue,
"Blue_Evapotranspiration", "mm/month",
0.01)
DC.Add_NC_Array_Variable(nc_outname, DataCube_ETgreen,
"Green_Evapotranspiration", "mm/month",
0.01)
del DataCube_ETblue, DataCube_ETgreen
#____________ Calculate non-consumend and Total supply maps by using fractions and consumed maps (blue ET) ____________
if not ("Total_Supply" in Variables_NC
or "Non_Consumed_Water" in Variables_NC):
# Do the calculations
DataCube_Total_Supply, DataCube_Non_Consumed = Four.Total_Supply.Fraction_Based(
nc_outname, Startdate_part, Enddate_part)
# Save the Total Supply and non consumed data as NetCDF files
DC.Add_NC_Array_Variable(nc_outname, DataCube_Total_Supply,
"Total_Supply", "mm/month", 0.01)
DC.Add_NC_Array_Variable(nc_outname, DataCube_Non_Consumed,
"Non_Consumed_Water", "mm/month", 0.01)
del DataCube_Total_Supply, DataCube_Non_Consumed
#____________ Apply fractions over total supply to calculate gw and sw supply ____________
if not ("Total_Supply_Surface_Water" in Variables_NC
or "Total_Supply_Ground_Water" in Variables_NC):
# Do the calculations
DataCube_Total_Supply_SW, DataCube_Total_Supply_GW = Four.SplitGW_SW_Supply.Fraction_Based(
nc_outname, Startdate_part, Enddate_part)
# Save the Total Supply surface water and Total Supply ground water data as NetCDF files
DC.Add_NC_Array_Variable(nc_outname, DataCube_Total_Supply_SW,
"Total_Supply_Surface_Water", "mm/month",
0.01)
DC.Add_NC_Array_Variable(nc_outname, DataCube_Total_Supply_GW,
"Total_Supply_Ground_Water", "mm/month",
0.01)
del DataCube_Total_Supply_SW, DataCube_Total_Supply_GW
#____________ Apply gray water footprint fractions to calculated non recoverable flow based on the non consumed flow ____________
if not ("Non_Recovable_Flow" in Variables_NC
or "Recovable_Flow" in Variables_NC):
# Calculate the non recovable flow and recovable flow by using Grey Water Footprint values
DataCube_NonRecovableFlow, Datacube_RecovableFlow = Four.SplitNonConsumed_NonRecov.GWF_Based(
nc_outname, Startdate_part, Enddate_part)
# Get the data of Evaporation and save as nc
DC.Add_NC_Array_Variable(nc_outname, DataCube_NonRecovableFlow,
"Non_Recovable_Flow", "mm/month", 0.01)
DC.Add_NC_Array_Variable(nc_outname, Datacube_RecovableFlow,
"Recovable_Flow", "mm/month", 0.01)
del DataCube_NonRecovableFlow, Datacube_RecovableFlow
#____________Apply fractions to calculate the non recovarable SW/GW and recovarable SW/GW ____________
# 1. Non recovarable flow
if not ("Non_Recovable_Flow_Ground_Water" in Variables_NC
or "Non_Recovable_Flow_Surface_Water" in Variables_NC):
# Calculate the non recovable return flow to ground and surface water
DataCube_NonRecovableFlow_Return_GW, Datacube_NonRecovableFlow_Return_SW = Four.SplitGW_SW_Return.Fraction_Based(
nc_outname, "Non_Recovable_Flow", Startdate_part, Enddate_part)
# Get the data of Evaporation and save as nc
DC.Add_NC_Array_Variable(nc_outname,
DataCube_NonRecovableFlow_Return_GW,
"Non_Recovable_Flow_Ground_Water",
"mm/month", 0.01)
DC.Add_NC_Array_Variable(nc_outname,
Datacube_NonRecovableFlow_Return_SW,
"Non_Recovable_Flow_Surface_Water",
"mm/month", 0.01)
del DataCube_NonRecovableFlow_Return_GW, Datacube_NonRecovableFlow_Return_SW
# 2. Recovarable flow
if not ("Recovable_Flow_Ground_Water" in Variables_NC
or "Recovable_Flow_Surface_Water" in Variables_NC):
# Calculate the non recovable return flow to ground and surface water
DataCube_RecovableFlow_Return_GW, Datacube_RecovableFlow_Return_SW = Four.SplitGW_SW_Return.Fraction_Based(
nc_outname, "Recovable_Flow", Startdate_part, Enddate_part)
# Get the data of Evaporation and save as nc
DC.Add_NC_Array_Variable(nc_outname,
DataCube_RecovableFlow_Return_GW,
"Recovable_Flow_Ground_Water", "mm/month",
0.01)
DC.Add_NC_Array_Variable(nc_outname,
Datacube_RecovableFlow_Return_SW,
"Recovable_Flow_Surface_Water",
"mm/month", 0.01)
del DataCube_RecovableFlow_Return_GW, Datacube_RecovableFlow_Return_SW
############################ Create CSV 4 #################################
Dir_Basin_CSV, Unit_front = Generate.CSV.Create(
Dir_Basin, Simulation, Basin, Startdate_part, Enddate_part,
nc_outname)
############################ Create Sheet 4 ###############################
Generate.PDF.Create(Dir_Basin, Basin, Simulation, Dir_Basin_CSV,
Unit_front)
return ()
def calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str,
down_short_str, down_long_str, up_long_str, DEMmap_str, DOY):
"""
This function calculates the ETref by using all the input parameters (path)
according to FAO standards
see: http://www.fao.org/docrep/x0490e/x0490e08.htm#TopOfPage
Keyword arguments:
tmin_str -- 'C:/' path to the minimal temperature tiff file [degrees Celcius], e.g. from GLDAS
tmax_str -- 'C:/' path to the maximal temperature tiff file [degrees Celcius], e.g. from GLDAS
humid_str -- 'C:/' path to the humidity tiff file [kg/kg], e.g. from GLDAS
press_str -- 'C:/' path to the air pressure tiff file [kPa], e.g. from GLDAS
wind_str -- 'C:/' path to the wind velocity tiff file [m/s], e.g. from GLDAS
down_short_str -- 'C:/' path to the downward shortwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF
down_long_str -- 'C:/' path to the downward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF
up_long_str -- 'C:/' path to the upward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF
DEMmap_str -- 'C:/' path to the DEM tiff file [m] e.g. from HydroSHED
DOY -- Day of the year
"""
# Get some geo-data to save results
GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap_str)
#NDV, xsize, ysize, GeoT, Projection, DataType = GetGeoInfo(DEMmap_str)
raster_shape = [xsize, ysize]
# Create array to store results
ETref = np.zeros(raster_shape)
# gap fill
tmin_str_GF = RC.gap_filling(tmin_str, -9999)
tmax_str_GF = RC.gap_filling(tmax_str, -9999)
humid_str_GF = RC.gap_filling(humid_str, -9999)
press_str_GF = RC.gap_filling(press_str, -9999)
wind_str_GF = RC.gap_filling(wind_str, -9999)
down_short_str_GF = RC.gap_filling(down_short_str, np.nan)
down_long_str_GF = RC.gap_filling(down_long_str, np.nan)
if up_long_str is not 'not':
up_long_str_GF = RC.gap_filling(up_long_str, np.nan)
else:
up_long_str_GF = 'nan'
#dictionary containing all tthe paths to the input-maps
inputs = dict({
'tmin': tmin_str_GF,
'tmax': tmax_str_GF,
'humid': humid_str_GF,
'press': press_str_GF,
'wind': wind_str_GF,
'down_short': down_short_str_GF,
'down_long': down_long_str_GF,
'up_long': up_long_str_GF
})
#dictionary containing numpy arrays of al initial and intermediate variables
input_array = dict({
'tmin': None,
'tmax': None,
'humid': None,
'press': None,
'wind': None,
'albedo': None,
'down_short': None,
'down_long': None,
'up_short': None,
'up_long': None,
'net_radiation': None,
'ea': None,
'es': None,
'delta': None
})
#APPLY LAPSE RATE CORRECTION ON TEMPERATURE
tmin = lapse_rate(Dir, inputs['tmin'], DEMmap_str)
tmax = lapse_rate(Dir, inputs['tmax'], DEMmap_str)
#PROCESS PRESSURE MAPS
press = adjust_P(Dir, inputs['press'], DEMmap_str)
#PREPARE HUMIDITY MAPS
dest = RC.reproject_dataset_example(inputs['humid'], DEMmap_str, method=2)
humid = dest.GetRasterBand(1).ReadAsArray()
dest = None
#CORRECT WIND MAPS
dest = RC.reproject_dataset_example(inputs['wind'], DEMmap_str, method=2)
wind = dest.GetRasterBand(1).ReadAsArray() * 0.75
dest = None
#PROCESS GLDAS DATA
input_array['ea'], input_array['es'], input_array['delta'] = process_GLDAS(
tmax, tmin, humid, press)
ea = input_array['ea']
es = input_array['es']
delta = input_array['delta']
if up_long_str == 'not':
#CORRECT WIND MAPS
dest = RC.reproject_dataset_example(down_short_str,
DEMmap_str,
method=2)
Short_Net_data = dest.GetRasterBand(1).ReadAsArray() * 0.75
dest = None
dest = RC.reproject_dataset_example(down_long_str,
DEMmap_str,
method=2)
Short_Clear_data = dest.GetRasterBand(1).ReadAsArray() * 0.75
dest = None
# Calculate Long wave Net radiation
Rnl = 4.903e-9 * (
((tmin + 273.16)**4 +
(tmax + 273.16)**4) / 2) * (0.34 - 0.14 * np.sqrt(ea)) * (
1.35 * Short_Net_data / Short_Clear_data - 0.35)
# Calulate Net Radiation and converted to MJ*d-1*m-2
net_radiation = (Short_Net_data * 0.77 + Rnl) * 86400 / 10**6
else:
#OPEN DOWNWARD SHORTWAVE RADIATION
dest = RC.reproject_dataset_example(inputs['down_short'],
DEMmap_str,
method=2)
down_short = dest.GetRasterBand(1).ReadAsArray()
dest = None
down_short, tau, bias = slope_correct(down_short, press, ea,
DEMmap_str, DOY)
#OPEN OTHER RADS
up_short = down_short * 0.23
dest = RC.reproject_dataset_example(inputs['down_long'],
DEMmap_str,
method=2)
down_long = dest.GetRasterBand(1).ReadAsArray()
dest = None
dest = RC.reproject_dataset_example(inputs['up_long'],
DEMmap_str,
method=2)
up_long = dest.GetRasterBand(1).ReadAsArray()
dest = None
#OPEN NET RADIATION AND CONVERT W*m-2 TO MJ*d-1*m-2
net_radiation = ((down_short - up_short) +
(down_long - up_long)) * 86400 / 10**6
#CALCULATE ETref
ETref = (0.408 * delta * net_radiation + 0.665 * 10**-3 * press *
(900 / ((tmax + tmin) / 2 + 273)) * wind *
(es - ea)) / (delta + 0.665 * 10**-3 * press * (1 + 0.34 * wind))
# Set limits ETref
ETref[ETref < 0] = 0
ETref[ETref > 400] = np.nan
#return a reference ET map (numpy array), a dictionary containing all intermediate information and a bias of the slope correction on down_short
return ETref
def Find_Area_Volume_Relation(region, input_JRC, input_nc):
# Find relation between V and A
import numpy as np
import watools.General.raster_conversions as RC
import watools.General.data_conversions as DC
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt
def func(x, a, b):
"""
This function is used for finding relation area and volume
"""
return (a * x**b)
def func3(x, a, b, c, d):
"""
This function is used for finding relation area and volume
"""
return (a * (x - c)**b + d)
#Array, Geo_out = RC.clip_data(input_JRC,latlim=[14.528,14.985],lonlim =[35.810,36.005])
Array, Geo_out = RC.clip_data(
input_JRC,
latlim=[region[2], region[3]],
lonlim=[region[0], region[1]
]) # This reservoir was not filled when SRTM was taken
size_Y = int(np.shape([Array])[-2])
size_X = int(np.shape([Array])[-1])
Water_array = np.zeros(np.shape(Array))
buffer_zone = 4
Array[Array > 0] = 1
for i in range(0, size_Y):
for j in range(0, size_X):
Water_array[i, j] = np.max(Array[
np.maximum(0, i -
buffer_zone):np.minimum(size_Y, i + buffer_zone +
1),
np.maximum(0, j -
buffer_zone):np.minimum(size_X, j + buffer_zone +
1)])
del Array
# Open DEM and reproject
DEM_Array = RC.Open_nc_array(input_nc, "dem")
Geo_out_dem, proj_dem, size_X_dem, size_Y_dem, size_Z_dem, time = RC.Open_nc_info(
input_nc)
# Save Example as memory file
dest_example = DC.Save_as_MEM(Water_array, Geo_out, projection='WGS84')
dest_dem = DC.Save_as_MEM(DEM_Array, Geo_out_dem, projection='WGS84')
# reproject DEM by using example
dest_out = RC.reproject_dataset_example(dest_dem, dest_example, method=2)
DEM = dest_out.GetRasterBand(1).ReadAsArray()
# find DEM water heights
DEM_water = np.zeros(np.shape(Water_array))
DEM_water[Water_array != 1] = np.nan
DEM_water[Water_array == 1.] = DEM[Water_array == 1.]
# Get array with areas
import watools.Functions.Start.Area_converter as Area
dlat, dlon = Area.Calc_dlat_dlon(Geo_out, size_X, size_Y)
area_in_m2 = dlat * dlon
# find volume and Area
min_DEM_water = int(np.round(np.nanmin(DEM_water)))
max_DEM_water = int(np.round(np.nanmax(DEM_water)))
Reservoir_characteristics = np.zeros([1, 5])
i = 0
for height in range(min_DEM_water + 1, max_DEM_water):
DEM_water_below_height = np.zeros(np.shape(DEM_water))
DEM_water[np.isnan(DEM_water)] = 1000000
DEM_water_below_height[DEM_water < height] = 1
pixels = np.sum(DEM_water_below_height)
area = np.sum(DEM_water_below_height * area_in_m2)
if height == min_DEM_water + 1:
volume = 0.5 * area
histogram = pixels
Reservoir_characteristics[:] = [
height, pixels, area, volume, histogram
]
else:
area_previous = Reservoir_characteristics[i, 2]
volume_previous = Reservoir_characteristics[i, 3]
volume = volume_previous + 0.5 * (
area - area_previous) + 1 * area_previous
histogram_previous = Reservoir_characteristics[i, 1]
histogram = pixels - histogram_previous
Reservoir_characteristics_one = [
height, pixels, area, volume, histogram
]
Reservoir_characteristics = np.append(
Reservoir_characteristics, Reservoir_characteristics_one)
i += 1
Reservoir_characteristics = np.resize(Reservoir_characteristics,
(i + 1, 5))
maxi = int(len(Reservoir_characteristics[:, 3]))
# find minimum value for reservoirs height (DEM is same value if reservoir was already filled whe SRTM was created)
Historgram = Reservoir_characteristics[:, 4]
hist_mean = np.mean(Historgram)
hist_std = np.std(Historgram)
mini_tresh = hist_std * 5 + hist_mean
Check_hist = np.zeros([len(Historgram)])
Check_hist[Historgram > mini_tresh] = Historgram[Historgram > mini_tresh]
if np.max(Check_hist) != 0.0:
col = np.argwhere(Historgram == np.max(Check_hist))[0][0]
mini = col + 1
else:
mini = 0
fitted = 0
# find starting point reservoirs
V0 = Reservoir_characteristics[mini, 3]
A0 = Reservoir_characteristics[mini, 2]
# Calculate the best maxi reservoir characteristics, based on the normal V = a*x**b relation
while fitted == 0:
try:
if mini == 0:
popt1, pcov1 = curve_fit(
func, Reservoir_characteristics[mini:maxi, 2],
Reservoir_characteristics[mini:maxi, 3])
else:
popt1, pcov1 = curve_fit(
func, Reservoir_characteristics[mini:maxi, 2] - A0,
Reservoir_characteristics[mini:maxi, 3] - V0)
fitted = 1
except:
maxi -= 1
if maxi < mini:
print('ERROR: was not able to find optimal fit')
fitted = 1
# Remove last couple of pixels of maxi
maxi_end = int(np.round(maxi - 0.2 * (maxi - mini)))
done = 0
times = 0
while done == 0 and times > 20 and maxi_end < mini:
try:
if mini == 0:
popt, pcov = curve_fit(
func, Reservoir_characteristics[mini:maxi_end, 2],
Reservoir_characteristics[mini:maxi_end, 3])
else:
popt, pcov = curve_fit(
func3, Reservoir_characteristics[mini:maxi_end, 2],
Reservoir_characteristics[mini:maxi_end, 3])
except:
maxi_end = int(maxi)
if mini == 0:
popt, pcov = curve_fit(
func, Reservoir_characteristics[mini:maxi_end, 2],
Reservoir_characteristics[mini:maxi_end, 3])
else:
popt, pcov = curve_fit(
func3, Reservoir_characteristics[mini:maxi_end, 2],
Reservoir_characteristics[mini:maxi_end, 3])
if mini == 0:
plt.plot(Reservoir_characteristics[mini:maxi_end, 2],
Reservoir_characteristics[mini:maxi_end, 3], 'ro')
t = np.arange(0., np.max(Reservoir_characteristics[:, 2]), 1000)
plt.plot(t, popt[0] * (t)**popt[1], 'g--')
plt.axis([
0,
np.max(Reservoir_characteristics[mini:maxi_end, 2]), 0,
np.max(Reservoir_characteristics[mini:maxi_end, 3])
])
plt.show()
done = 1
else:
plt.plot(Reservoir_characteristics[mini:maxi_end, 2],
Reservoir_characteristics[mini:maxi_end, 3], 'ro')
t = np.arange(0., np.max(Reservoir_characteristics[:, 2]), 1000)
plt.plot(t, popt[0] * (t - popt[2])**popt[1] + popt[3], 'g--')
plt.axis([
0,
np.max(Reservoir_characteristics[mini:maxi_end, 2]), 0,
np.max(Reservoir_characteristics[mini:maxi_end, 3])
])
plt.show()
Volume_error = popt[3] / V0 * 100 - 100
print('error Volume = %s percent' % Volume_error)
print('error Area = %s percent' % (A0 / popt[2] * 100 - 100))
if Volume_error < 30 and Volume_error > -30:
done = 1
else:
times += 1
maxi_end -= 1
print('Another run is done in order to improve the result')
if done == 0:
popt = np.append(popt1, [A0, V0])
if len(popt) == 2:
popt = np.append(popt, [0, 0])
return (popt)
def Get_Array(nc_filename_waterpix, Var_name, Example_dataset, Startdate,
Enddate):
#import general modules
import numpy as np
import pandas as pd
from netCDF4 import Dataset
import gdal
import osr
#import WA+ modules
import watools.General.raster_conversions as RC
'''
#input files
Name_NC_Runoff_CR = r'F:\\Create_Sheets\\Litani\\Simulations\\Simulation_1\\Sheet_5\\Runoff_CR_Simulation1_monthly_mm_012010_122010.nc'
Example_dataset = r"F:\Create_Sheets\Litani\HydroSHED\DIR\DIR_HydroShed_-_15s.tif"
NC_filename = "F:\Create_Sheets\Litani\WaterPIX\Litani.nc"
Startdate = "2010-01-01"
Enddate = "2010-12-31"
Var = 'SurfaceRunoff_M'
'''
# Define Dates
Dates = pd.date_range(Startdate, Enddate, freq="MS")
# Define end and start date
Start = '%d%02d' % (Dates[0].year, Dates[0].month)
End = '%d%02d' % (Dates[-1].year, Dates[-1].month)
# Open netcdf of WaterPIX
fh = Dataset(nc_filename_waterpix, 'r')
# Get time series of WaterPIX
time = fh.variables['time_yyyymm'][:]
# Define time steps that are needed from WaterPIX
time_yes = np.zeros(len(time))
time_yes[np.logical_and(
np.int_(time) >= int(Start),
np.int_(time) <= int(End))] = 1
time_start = time_yes[1:] - time_yes[:-1]
time_end = time_yes[:-1] - time_yes[1:]
# Set the startpoint
if np.sum(time_start) > 0:
Start_time = np.argwhere(time_start == 1)[0][0] + 1
else:
Start_time = 0
# Set the endpoint
if np.sum(time_end) > 0:
End_time = np.argwhere(time_end == 1)[0][0] + 1
else:
End_time = len(Dates) + Start_time
# Get the wanted variable from WaterPIX
data = fh.variables[Var_name][Start_time:End_time, :, :]
# Fill the WaterPIX veriable
data_filled = np.dstack(np.ma.filled(data, np.nan))
# Get WaterPIX projection
proj = fh.variables['crs'].crs_wkt
lon = fh.variables['longitude'][:]
lat = fh.variables['latitude'][:]
# Find WaterPIX raster parameters
col = int(len(lon))
row = int(len(lat))
y_diff = (lat[0] - lat[-1]) / (row - 1)
x_diff = (lon[0] - lon[-1]) / (col - 1)
geo = tuple([
lon[0] + 0.5 * x_diff, -x_diff, 0.0, lat[0] + 0.5 * y_diff, 0.0,
-y_diff
])
# Find example raster parameters
geo_out, proj, size_X, size_Y = RC.Open_array_info(Example_dataset)
# Create empty raster file
Array_End = np.zeros([int(data_filled.shape[2]), size_Y, size_X])
# Loop over time and add one time period at the time to end array
for i in range(0, int(data_filled.shape[2])):
# Create Memory file containing WaterPIX data
mem_drv = gdal.GetDriverByName('MEM')
dest = mem_drv.Create('', int(data_filled.shape[1]),
int(data_filled.shape[0]),
int(data_filled.shape[2]), gdal.GDT_Float32)
dest.SetGeoTransform(geo)
srse = osr.SpatialReference()
srse.SetWellKnownGeogCS("WGS84")
dest.SetProjection(srse.ExportToWkt())
dest.GetRasterBand(1).WriteArray(data_filled[:, :, i])
dest.GetRasterBand(1).SetNoDataValue(-9999)
# reproject the WaterPIX raster to the example raster
dest_out = RC.reproject_dataset_example(dest, Example_dataset)
# Write the raster array to the end raster
Array_End[i, :, :] = dest_out.GetRasterBand(1).ReadAsArray()
# Set nan value to 0
Array_End[np.isnan(Array_End)] = 0
return (Array_End)