def Convert_dict_to_array(River_dict, Array_dict, Reference_data):
import numpy as np
import os
import watertools.General.raster_conversions as RC
if os.path.splitext(Reference_data)[-1] == '.nc':
# Get raster information
geo_out, proj, size_X, size_Y, size_Z, Time = RC.Open_nc_info(Reference_data)
else:
# Get raster information
geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_data)
# Create ID Matrix
y,x = np.indices((size_Y, size_X))
ID_Matrix = np.int32(np.ravel_multi_index(np.vstack((y.ravel(),x.ravel())),(size_Y,size_X),mode='clip').reshape(x.shape)) + 1
# Get tiff array time dimension:
time_dimension = int(np.shape(Array_dict[0])[0])
# create an empty array
DataCube = np.ones([time_dimension, size_Y, size_X]) * np.nan
for river_part in range(0,len(River_dict)):
for river_pixel in range(1,len(River_dict[river_part])):
river_pixel_ID = River_dict[river_part][river_pixel]
if len(np.argwhere(ID_Matrix == river_pixel_ID))>0:
row, col = np.argwhere(ID_Matrix == river_pixel_ID)[0][:]
DataCube[:,row,col] = Array_dict[river_part][:,river_pixel]
return(DataCube)
def RetrieveData(args):
"""
This function retrieves JRC data for a given date from the
http://storage.googleapis.com/global-surface-water/downloads/ server.
Keyword arguments:
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, Names_to_download, lonlim, latlim] = args
# Collect the data from the JRC webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(Names_to_download, output_folder)
except:
print("Was not able to download the file")
# Clip the data to the users extend
if len(Names_to_download) == 1:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Names_to_download[0])
data_end, geo_end = RC.clip_data(data_in, latlim, lonlim)
else:
data_end = np.zeros([int((latlim[1] - latlim[0])/0.00025), int((lonlim[1] - lonlim[0])/0.00025)])
for Name_to_merge in Names_to_download:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Name_to_merge)
geo_out, proj, size_X, size_Y = RC.Open_array_info(data_in)
lat_min_merge = np.maximum(latlim[0], geo_out[3] + size_Y * geo_out[5])
lat_max_merge = np.minimum(latlim[1], geo_out[3])
lon_min_merge = np.maximum(lonlim[0], geo_out[0])
lon_max_merge = np.minimum(lonlim[1], geo_out[0] + size_X * geo_out[1])
lonmerge = [lon_min_merge, lon_max_merge]
latmerge = [lat_min_merge, lat_max_merge]
data_one, geo_one = RC.clip_data(data_in, latmerge, lonmerge)
Ystart = int((geo_one[3] - latlim[1])/geo_one[5])
Yend = int(Ystart + np.shape(data_one)[0])
Xstart = int((geo_one[0] - lonlim[0])/geo_one[1])
Xend = int(Xstart + np.shape(data_one)[1])
data_end[Ystart:Yend, Xstart:Xend] = data_one
geo_end = tuple([lonlim[0], geo_one[1], 0, latlim[1], 0, geo_one[5]])
# Save results as Gtiff
fileName_out = os.path.join(output_folder, 'JRC_Occurrence_percent.tif')
DC.Save_as_tiff(name=fileName_out, data=data_end, geo=geo_end, projection='WGS84')
shutil.rmtree(trash_folder)
return True
def Calc_Humidity(Temp_format,
P_format,
Hum_format,
output_format,
Startdate,
Enddate,
freq="D"):
folder_dir_out = os.path.dirname(output_format)
if not os.path.exists(folder_dir_out):
os.makedirs(folder_dir_out)
Dates = pd.date_range(Startdate, Enddate, freq="D")
for Date in Dates:
print(Date)
Day = Date.day
Month = Date.month
Year = Date.year
Tempfile_one = Temp_format.format(yyyy=Year, mm=Month, dd=Day)
Presfile_one = P_format.format(yyyy=Year, mm=Month, dd=Day)
Humfile_one = Hum_format.format(yyyy=Year, mm=Month, dd=Day)
out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day)
geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one)
Tdata = RC.Open_tiff_array(Tempfile_one)
if "MERRA_K" in Temp_format:
Tdata = Tdata - 273.15
Tdata[Tdata < -900] = -9999
Pdata = RC.Open_tiff_array(Presfile_one)
Hdata = RC.Open_tiff_array(Humfile_one)
Pdata[Pdata < 0] = -9999
Hdata[Hdata < 0] = -9999
# gapfilling
Tdata = RC.gap_filling(Tdata, -9999)
Pdata = RC.gap_filling(Pdata, -9999)
Hdata = RC.gap_filling(Hdata, -9999)
Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3))
HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100
HumData = HumData.clip(0, 100)
DC.Save_as_tiff(out_folder_one, HumData, geo_out, "WGS84")
return ()
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat.Subsoil(
Dir, latlim, lonlim)
filedir_out_whc = os.path.join(Dir, 'SoilGrids', 'Water_Holding_Capacity')
if not os.path.exists(filedir_out_whc):
os.makedirs(filedir_out_whc)
# Define theta field capacity output
filename_out_whc = os.path.join(
filedir_out_whc,
'Water_Holding_Capacity_%s_SoilGrids_mm-m.tif' % level)
if not os.path.exists(filename_out_whc):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate theta field capacity
theta_whc = np.ones(theta_sat.shape) * -9999
theta_whc = np.where(
theta_sat < 0.301, 80,
450 * np.arccosh(theta_sat + 0.7) - 2 * (theta_sat + 0.7) + 20)
# Save as tiff
DC.Save_as_tiff(filename_out_whc, theta_whc, geo_out, proj)
return
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat2.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat2.Subsoil(
Dir, latlim, lonlim)
filedir_out_n_genuchten = os.path.join(Dir, 'SoilGrids', 'N_van_genuchten')
if not os.path.exists(filedir_out_n_genuchten):
os.makedirs(filedir_out_n_genuchten)
# Define n van genuchten output
filename_out_ngenuchten = os.path.join(
filedir_out_n_genuchten, 'N_genuchten_%s_SoilGrids_-.tif' % level)
if not os.path.exists(filename_out_ngenuchten):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate n van genuchten
n_van_genuchten = np.ones(theta_sat.shape) * -9999
n_van_genuchten = 166.63 * theta_sat**4 - 387.72 * theta_sat**3 + 340.55 * theta_sat**2 - 133.07 * theta_sat + 20.739
# Save as tiff
DC.Save_as_tiff(filename_out_ngenuchten, n_van_genuchten, geo_out,
proj)
return
def Calc_Humidity(Temp_format,
P_format,
Hum_format,
output_format,
Startdate,
Enddate,
freq="D"):
folder_dir_out = os.path.dirname(output_format)
if not os.path.exists(folder_dir_out):
os.makedirs(folder_dir_out)
Dates = pd.date_range(Startdate, Enddate, freq="D")
for Date in Dates:
print(Date)
Day = Date.day
Month = Date.month
Year = Date.year
Windyfile_one = wind_y_format.format(yyyy=Year, mm=Month, dd=Day)
Windxfile_one = wind_x_format.format(yyyy=Year, mm=Month, dd=Day)
out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day)
geo_out, proj, size_X, size_Y = RC.Open_array_info(Windxfile_one)
Windxdata = RC.Open_tiff_array(Windxfile_one)
Windxdata[Windxdata < -900] = -9999
Windydata = RC.Open_tiff_array(Windyfile_one)
Windydata[Windxdata < -900] = -9999
# gapfilling
Windydata = RC.gap_filling(Windydata, -9999)
Windxdata = RC.gap_filling(Windxdata, -9999)
Wind = np.sqrt(Windxdata**2 + Windydata**2)
DC.Save_as_tiff(out_folder_one, Wind, geo_out, "WGS84")
return ()
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat' ,'Theta_Sat2_%s_SoilGrids_kg-kg.tif' %level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat2.Topsoil(Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat2.Subsoil(Dir, latlim, lonlim)
filedir_out_thetares = os.path.join(Dir, 'SoilGrids', 'Theta_Res')
if not os.path.exists(filedir_out_thetares):
os.makedirs(filedir_out_thetares)
# Define theta field capacity output
filename_out_thetares = os.path.join(filedir_out_thetares ,'Theta_Res_%s_SoilGrids_kg-kg.tif' %level)
if not os.path.exists(filename_out_thetares):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate theta field capacity
theta_Res = np.ones(theta_sat.shape) * -9999
#theta_Res = np.where(theta_sat < 0.351, 0.01, 0.4 * np.arccosh(theta_sat + 0.65) - 0.05 * np.power(theta_sat + 0.65, 2.5) + 0.02)
theta_Res = np.where(theta_sat < 0.351, 0.01, 0.271 * np.log(theta_sat) + 0.335)
# Save as tiff
DC.Save_as_tiff(filename_out_thetares, theta_Res, geo_out, proj)
return
def lapse_rate(Dir,temperature_map, DEMmap):
"""
This function downscales the GLDAS temperature map by using the DEM map
Keyword arguments:
temperature_map -- 'C:/' path to the temperature map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average altitudes corresponding to T resolution
dest = RC.reproject_dataset_example(DEMmap, temperature_map,method = 4)
DEM_ave_out_name = os.path.join(Dir,'HydroSHED', 'DEM','DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map)
DEM_ave_data = dest.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
dest = None
# determine lapse-rate [degress Celcius per meter]
lapse_rate_number = 0.0065
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2)
dem_avg=dest.GetRasterBand(1).ReadAsArray()
dem_avg[dem_avg<0]=0
dest = None
# Open the temperature dataset
dest = RC.reproject_dataset_example(temperature_map, DEMmap, method = 2)
T=dest.GetRasterBand(1).ReadAsArray()
dest = None
# Open Demmap
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap<=0]=0
demmap[demmap==-32768]=np.nan
# calculate first part
T = T + ((dem_avg-demmap) * lapse_rate_number)
return T
def Degrees_to_m2(Reference_data):
"""
This functions calculated the area of each pixel in squared meter.
Parameters
----------
Reference_data: str
Path to a tiff file or nc file or memory file of which the pixel area must be defined
Returns
-------
area_in_m2: array
Array containing the area of each pixel in squared meters
"""
try:
# Get the extension of the example data
filename, file_extension = os.path.splitext(Reference_data)
# Get raster information
if str(file_extension) == '.tif':
geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_data)
elif str(file_extension) == '.nc':
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(
Reference_data)
except:
geo_out = Reference_data.GetGeoTransform()
size_X = Reference_data.RasterXSize()
size_Y = Reference_data.RasterYSize()
# Calculate the difference in latitude and longitude in meters
dlat, dlon = Calc_dlat_dlon(geo_out, size_X, size_Y)
# Calculate the area in squared meters
area_in_m2 = dlat * dlon
return (area_in_m2)
def adjust_P(Dir, pressure_map, DEMmap):
"""
This function downscales the GLDAS air pressure map by using the DEM map
Keyword arguments:
pressure_map -- 'C:/' path to the pressure map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average latitudes
destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method = 4)
DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM','DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map)
DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2)
dem_avg=dest.GetRasterBand(1).ReadAsArray()
dest = None
# open maps as numpy arrays
dest = RC.reproject_dataset_example(pressure_map, DEMmap, method = 2)
P=dest.GetRasterBand(1).ReadAsArray()
dest = None
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap<=0]=0
demmap[demmap==-32768]=np.nan
# calculate second part
P = P + (101.3*((293-0.0065*(demmap-dem_avg))/293)**5.26 - 101.3)
os.remove(DEM_ave_out_name)
return P
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None):
"""
This functions calculates monthly tiff files based on the 8 daily tiff files. (will calculate the average)
Parameters
----------
Dir_in : str
Path to the input data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Dir_out : str
Path to the output data, default is same as Dir_in
"""
# import WA+ modules
import watertools.General.data_conversions as DC
import watertools.General.raster_conversions as RC
# Change working directory
os.chdir(Dir_in)
# Find all eight daily files
files = glob.glob('*8-daily*.tif')
# Create array with filename and keys (DOY and year) of all the 8 daily files
i = 0
DOY_Year = np.zeros([len(files), 3])
for File in files:
# Get the time characteristics from the filename
year = File.split('.')[-4][-4:]
month = File.split('.')[-3]
day = File.split('.')[-2]
# Create pandas Timestamp
date_file = '%s-%02s-%02s' % (year, month, day)
Datum = pd.Timestamp(date_file)
# Get day of year
DOY = Datum.strftime('%j')
# Save data in array
DOY_Year[i, 0] = i
DOY_Year[i, 1] = DOY
DOY_Year[i, 2] = year
# Loop over files
i += 1
# Check enddate:
Enddate_split = Enddate.split('-')
month_range = calendar.monthrange(int(Enddate_split[0]),
int(Enddate_split[1]))[1]
Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int(
Enddate_split[1]), month_range)
# Check startdate:
Startdate_split = Startdate.split('-')
Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int(
Startdate_split[1]))
# Define end and start date
Dates = pd.date_range(Startdate, Enddate, freq='MS')
DatesEnd = pd.date_range(Startdate, Enddate, freq='M')
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value
dest = gdal.Open(files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Loop over months and create monthly tiff files
i = 0
for date in Dates:
# Get Start and end DOY of the current month
DOY_month_start = date.strftime('%j')
DOY_month_end = DatesEnd[i].strftime('%j')
# Search for the files that are between those DOYs
year = date.year
DOYs = DOY_Year[DOY_Year[:, 2] == year]
DOYs_oneMonth = DOYs[np.logical_and(
(DOYs[:, 1] + 8) >= int(DOY_month_start),
DOYs[:, 1] <= int(DOY_month_end))]
# Create empty arrays
Monthly = np.zeros([size_Y, size_X])
Weight_tot = np.zeros([size_Y, size_X])
Data_one_month = np.ones([size_Y, size_X]) * np.nan
# Loop over the files that are within the DOYs
for EightDays in DOYs_oneMonth[:, 1]:
# Calculate the amount of days in this month of each file
Weight = np.ones([size_Y, size_X])
# For start of month
if np.min(DOYs_oneMonth[:, 1]) == EightDays:
Weight = Weight * int(EightDays + 8 - int(DOY_month_start))
# For end of month
elif np.max(DOYs_oneMonth[:, 1]) == EightDays:
Weight = Weight * (int(DOY_month_end) - EightDays + 1)
# For the middle of the month
else:
Weight = Weight * 8
row = DOYs_oneMonth[np.argwhere(
DOYs_oneMonth[:, 1] == EightDays)[0][0], :][0]
# Open the array of current file
input_name = os.path.join(Dir_in, files[int(row)])
Data = RC.Open_tiff_array(input_name)
# Remove NDV
Weight[Data == NDV] = 0
Data[Data == NDV] = np.nan
# Multiply weight time data
Data = Data * Weight
# Calculate the total weight and data
Weight_tot += Weight
Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)]
# Go to next month
i += 1
# Calculate the average
Data_one_month[Weight_tot != 0.] = Monthly[
Weight_tot != 0.] / Weight_tot[Weight_tot != 0.]
# Define output directory
if Dir_out == None:
Dir_out = Dir_in
# Define output name
output_name = os.path.join(
Dir_out, files[int(row)].replace('8-daily', 'monthly'))
output_name = output_name[:-9] + '%02d.01.tif' % (date.month)
# Save tiff file
DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj)
return
def calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str,
down_short_str, down_long_str, up_long_str, DEMmap_str, DOY):
"""
This function calculates the ETref by using all the input parameters (path)
according to FAO standards
see: http://www.fao.org/docrep/x0490e/x0490e08.htm#TopOfPage
Keyword arguments:
tmin_str -- 'C:/' path to the minimal temperature tiff file [degrees Celcius], e.g. from GLDAS
tmax_str -- 'C:/' path to the maximal temperature tiff file [degrees Celcius], e.g. from GLDAS
humid_str -- 'C:/' path to the humidity tiff file [kg/kg], e.g. from GLDAS
press_str -- 'C:/' path to the air pressure tiff file [kPa], e.g. from GLDAS
wind_str -- 'C:/' path to the wind velocity tiff file [m/s], e.g. from GLDAS
down_short_str -- 'C:/' path to the downward shortwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF
down_long_str -- 'C:/' path to the downward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF
up_long_str -- 'C:/' path to the upward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF
DEMmap_str -- 'C:/' path to the DEM tiff file [m] e.g. from HydroSHED
DOY -- Day of the year
"""
# Get some geo-data to save results
GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap_str)
#NDV, xsize, ysize, GeoT, Projection, DataType = GetGeoInfo(DEMmap_str)
raster_shape = [xsize, ysize]
# Create array to store results
ETref = np.zeros(raster_shape)
# gap fill
tmin_str_GF = RC.gap_filling(tmin_str, -9999)
tmax_str_GF = RC.gap_filling(tmax_str, -9999)
humid_str_GF = RC.gap_filling(humid_str, -9999)
press_str_GF = RC.gap_filling(press_str, -9999)
wind_str_GF = RC.gap_filling(wind_str, -9999)
down_short_str_GF = RC.gap_filling(down_short_str, np.nan)
down_long_str_GF = RC.gap_filling(down_long_str, np.nan)
if up_long_str is not 'not':
up_long_str_GF = RC.gap_filling(up_long_str, np.nan)
else:
up_long_str_GF = 'nan'
#dictionary containing all tthe paths to the input-maps
inputs = dict({
'tmin': tmin_str_GF,
'tmax': tmax_str_GF,
'humid': humid_str_GF,
'press': press_str_GF,
'wind': wind_str_GF,
'down_short': down_short_str_GF,
'down_long': down_long_str_GF,
'up_long': up_long_str_GF
})
#dictionary containing numpy arrays of al initial and intermediate variables
input_array = dict({
'tmin': None,
'tmax': None,
'humid': None,
'press': None,
'wind': None,
'albedo': None,
'down_short': None,
'down_long': None,
'up_short': None,
'up_long': None,
'net_radiation': None,
'ea': None,
'es': None,
'delta': None
})
#APPLY LAPSE RATE CORRECTION ON TEMPERATURE
tmin = lapse_rate(Dir, inputs['tmin'], DEMmap_str)
tmax = lapse_rate(Dir, inputs['tmax'], DEMmap_str)
#PROCESS PRESSURE MAPS
press = adjust_P(Dir, inputs['press'], DEMmap_str)
#PREPARE HUMIDITY MAPS
dest = RC.reproject_dataset_example(inputs['humid'], DEMmap_str, method=2)
humid = dest.GetRasterBand(1).ReadAsArray()
dest = None
#CORRECT WIND MAPS
dest = RC.reproject_dataset_example(inputs['wind'], DEMmap_str, method=2)
wind = dest.GetRasterBand(1).ReadAsArray() * 0.75
dest = None
#PROCESS GLDAS DATA
input_array['ea'], input_array['es'], input_array['delta'] = process_GLDAS(
tmax, tmin, humid, press)
ea = input_array['ea']
es = input_array['es']
delta = input_array['delta']
if up_long_str == 'not':
#CORRECT WIND MAPS
dest = RC.reproject_dataset_example(down_short_str,
DEMmap_str,
method=2)
Short_Net_data = dest.GetRasterBand(1).ReadAsArray() * 0.75
dest = None
dest = RC.reproject_dataset_example(down_long_str,
DEMmap_str,
method=2)
Short_Clear_data = dest.GetRasterBand(1).ReadAsArray() * 0.75
dest = None
# Calculate Long wave Net radiation
Rnl = 4.903e-9 * (
((tmin + 273.16)**4 +
(tmax + 273.16)**4) / 2) * (0.34 - 0.14 * np.sqrt(ea)) * (
1.35 * Short_Net_data / Short_Clear_data - 0.35)
# Calulate Net Radiation and converted to MJ*d-1*m-2
net_radiation = (Short_Net_data * 0.77 + Rnl) * 86400 / 10**6
else:
#OPEN DOWNWARD SHORTWAVE RADIATION
dest = RC.reproject_dataset_example(inputs['down_short'],
DEMmap_str,
method=2)
down_short = dest.GetRasterBand(1).ReadAsArray()
dest = None
down_short, tau, bias = slope_correct(down_short, press, ea,
DEMmap_str, DOY)
#OPEN OTHER RADS
up_short = down_short * 0.23
dest = RC.reproject_dataset_example(inputs['down_long'],
DEMmap_str,
method=2)
down_long = dest.GetRasterBand(1).ReadAsArray()
dest = None
dest = RC.reproject_dataset_example(inputs['up_long'],
DEMmap_str,
method=2)
up_long = dest.GetRasterBand(1).ReadAsArray()
dest = None
#OPEN NET RADIATION AND CONVERT W*m-2 TO MJ*d-1*m-2
net_radiation = ((down_short - up_short) +
(down_long - up_long)) * 86400 / 10**6
#CALCULATE ETref
ETref = (0.408 * delta * net_radiation + 0.665 * 10**-3 * press *
(900 / ((tmax + tmin) / 2 + 273)) * wind *
(es - ea)) / (delta + 0.665 * 10**-3 * press * (1 + 0.34 * wind))
# Set limits ETref
ETref[ETref < 0] = 0
ETref[ETref > 400] = np.nan
#return a reference ET map (numpy array), a dictionary containing all intermediate information and a bias of the slope correction on down_short
return ETref
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat2.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat2.Subsoil(
Dir, latlim, lonlim)
filename_out_thetares = os.path.join(
Dir, 'SoilGrids', 'Theta_Res',
'Theta_Res_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetares):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Res.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Res.Subsoil(
Dir, latlim, lonlim)
filename_out_n_genuchten = os.path.join(
Dir, 'SoilGrids', 'N_van_genuchten',
'N_genuchten_%s_SoilGrids_-.tif' % level)
if not os.path.exists(filename_out_n_genuchten):
if SL == "sl3":
watertools.Products.SoilGrids.n_van_genuchten.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.n_van_genuchten.Subsoil(
Dir, latlim, lonlim)
filedir_out_thetafc = os.path.join(Dir, 'SoilGrids', 'Theta_FC')
if not os.path.exists(filedir_out_thetafc):
os.makedirs(filedir_out_thetafc)
# Define theta field capacity output
filename_out_thetafc = os.path.join(
filedir_out_thetafc, 'Theta_FC2_%s_SoilGrids_cm3-cm3.tif' % level)
if not os.path.exists(filename_out_thetafc):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
theta_res = RC.Open_tiff_array(filename_out_thetares)
n_genuchten = RC.Open_tiff_array(filename_out_n_genuchten)
# Calculate theta field capacity
theta_FC = np.ones(theta_sat.shape) * -9999
#theta_FC = np.where(theta_sat < 0.301, 0.042, np.arccosh(theta_sat + 0.7) - 0.32 * (theta_sat + 0.7) + 0.2)
#theta_FC = np.where(theta_sat < 0.301, 0.042, -2.95*theta_sat**2+3.96*theta_sat-0.871)
theta_FC = theta_res + (theta_sat - theta_res) / (
1 + (0.02 * 200)**n_genuchten)**(1 - 1 / n_genuchten)
# Save as tiff
DC.Save_as_tiff(filename_out_thetafc, theta_FC, geo_out, proj)
return
os.makedirs(folder_dir_out)
Dates = pd.date_range(Startdate, Enddate, freq="D")
for Date in Dates:
Day = Date.day
Month = Date.month
Year = Date.year
Tempfile_one = Temp_folder.format(yyyy=Year, mm=Month, dd=Day)
Presfile_one = Pres_folder.format(yyyy=Year, mm=Month, dd=Day)
Humfile_one = Hum_folder.format(yyyy=Year, mm=Month, dd=Day)
out_folder_one = out_folder.format(yyyy=Year, mm=Month, dd=Day)
geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one)
Tdata = RC.Open_tiff_array(Tempfile_one)
Tdata[Tdata < -900] = -9999
Pdata = RC.Open_tiff_array(Presfile_one)
Hdata = RC.Open_tiff_array(Humfile_one)
Pdata[Pdata < 0] = -9999
Hdata[Hdata < 0] = -9999
# gapfilling
Tdata = RC.gap_filling(Tdata, -9999)
Pdata = RC.gap_filling(Pdata, -9999)
Hdata = RC.gap_filling(Hdata, -9999)
Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3))
HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100
HumData = HumData.clip(0, 100)
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim):
"""
This function collects and clip LANDSAF data
Keyword arguments:
SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID)
Dir -- 'C:/' path to the WA map
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -60 and 60)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Make an array of the days of which the ET is taken
Dates = pd.date_range(Startdate, Enddate, freq='D')
# make directories
SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS')
if os.path.exists(SISdir) is False:
os.makedirs(SISdir)
SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID')
if os.path.exists(SIDdir) is False:
os.makedirs(SIDdir)
ShortwaveBasin(SourceLANDSAF,
Dir,
latlim,
lonlim,
Dates=[Startdate, Enddate])
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str)
# Open DEM map
demmap = RC.Open_tiff_array(DEMmap_str)
demmap[demmap < 0] = 0
# make lat and lon arrays)
dlat = geo_out[5]
dlon = geo_out[1]
lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat
lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon
for date in Dates:
# day of year
day = date.dayofyear
Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence(
demmap, lat, lon, day)
SIDname = os.path.join(
SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
SISname = os.path.join(
SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
#PREPARE SID MAPS
SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3)
SIDdata = SIDdest.GetRasterBand(1).ReadAsArray()
#PREPARE SIS MAPS
SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3)
SISdata = SISdest.GetRasterBand(1).ReadAsArray()
# Calculate ShortWave net
Short_Wave_Net = SIDdata * (Sloping /
Horizontal) + SISdata * 86400 / 1e6
# Calculate ShortWave Clear
Short_Wave = Sloping
Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5)
# make directories
PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky')
if os.path.exists(PathClear) is False:
os.makedirs(PathClear)
PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net')
if os.path.exists(PathNet) is False:
os.makedirs(PathNet)
# name Shortwave Clear and Net
nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameNet = os.path.join(PathNet, nameFileNet)
nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameClear = os.path.join(PathClear, nameFileClear)
# Save net and clear short wave radiation
DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj)
DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj)
return
def DownloadData(output_folder, latlim, lonlim, parameter, resolution):
"""
This function downloads DEM data from HydroSHED
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
Resample -- 1 = The data will be resampled to 0.001 degree spatial
resolution
-- 0 = The data will have the same pixel size as the data obtained
from the internet
"""
# Define parameter depedent variables
if parameter == "dir_3s":
para_name = "DIR"
unit = "-"
resolution = '3s'
parameter = 'dir'
if parameter == "dem_3s":
para_name = "DEM"
unit = "m"
resolution = '3s'
parameter = 'dem'
if parameter == "dir_15s":
para_name = "DIR"
unit = "-"
resolution = '15s'
parameter = 'dir'
if parameter == "dem_15s":
para_name = "DEM"
unit = "m"
resolution = '15s'
parameter = 'dem'
if parameter == "dir_30s":
para_name = "DIR"
unit = "-"
resolution = '30s'
parameter = 'dir'
if parameter == "dem_30s":
para_name = "DEM"
unit = "m"
resolution = '30s'
parameter = 'dem'
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
if resolution == '3s':
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim,
parameter)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
if resolution == '15s' or resolution == '30s':
name = Find_Document_names_15s_30s(latlim, lonlim, parameter,
resolution)
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash,
parameter, para_name,
resolution)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# Converts the data with a adf extention to a tiff extension.
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif'
file_name_extract = file_name.split('_')[0:3]
if resolution == '3s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1]
if resolution == '15s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1] + '_15s'
if resolution == '30s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1] + '_30s'
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
if (resolution == "15s" or resolution == "3s"):
input_adf = os.path.join(output_folder_trash,
file_name_extract2,
file_name_extract2, 'hdr.adf')
output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff)
# convert data from adf to a tiff file
if resolution == "30s":
input_bil = os.path.join(output_folder_trash,
'%s.bil' % file_name_extract2)
output_tiff = DC.Convert_bil_to_tiff(input_bil, output_tiff)
geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff)
if (resolution == "3s" and
(int(size_X) != int(6000) or int(size_Y) != int(6000))):
data = np.ones((6000, 6000)) * -9999
# Create the latitude bound
Vfile = str(nameFile)[1:3]
SignV = str(nameFile)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(nameFile)[4:7]
SignH = str(nameFile)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
Expected_X_min = Bound1
Expected_Y_max = Bound2 + 5
Xid_start = int(
np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(
np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) /
geo_out[1]))
Yid_start = int(
np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(
np.round((Expected_Y_max - (geo_out[3] +
(size_Y * geo_out[5]))) /
(-geo_out[5])))
data[Yid_start:Yid_end,
Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff)
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
except:
if resolution == '3s':
# If tile not exist create a replacing zero tile (sea tiles)
output = nameFile.split('.')[0] + "_trans_temporary.tif"
output_tiff = os.path.join(output_folder_trash, output)
file_name = nameFile
data = np.ones((6000, 6000)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = str(file_name)[1:3]
SignV = str(file_name)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(file_name)[4:7]
SignH = str(file_name)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
# Geospatial data for the tile
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
if resolution == '15s':
print('no 15s data is in dataset')
if resolution == '3s':
# clip data
Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (nameFile)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd,
data=Data,
geo=Geo_data,
projection="WGS84")
if resolution == '3s':
#size_X_end = int(size_X_tot) #!
#size_Y_end = int(size_Y_tot) #!
size_X_end = int(size_X_tot / len(rangeLat)) + 1 #!
size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 #!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
if resolution == '15s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash,
output_file_merged, latlim,
lonlim, resolution)
if resolution == '30s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash,
output_file_merged, latlim,
lonlim, resolution)
# name of the end result
output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution)
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name,
data=datasetTot,
geo=geo_out,
projection="WGS84")
os.chdir(output_folder)
# Delete the temporary folder
shutil.rmtree(output_folder_trash)
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools.Collect.SoilGrids as SG
# Download needed layers
SG.Clay_Content(Dir, latlim, lonlim, level=SL)
#SG.Organic_Carbon_Content(Dir, latlim, lonlim, level=SL)
SG.Bulk_Density(Dir, latlim, lonlim, level=SL)
# Define path to layers
filename_clay = os.path.join(
Dir, 'SoilGrids', 'Clay_Content',
'ClayContentMassFraction_%s_SoilGrids_percentage.tif' % SL)
#filename_om = os.path.join(Dir, 'SoilGrids', 'Soil_Organic_Carbon_Content' ,'SoilOrganicCarbonContent_%s_SoilGrids_g_kg.tif' %SL)
filename_bulkdensity = os.path.join(
Dir, 'SoilGrids', 'Bulk_Density',
'BulkDensity_%s_SoilGrids_kg-m-3.tif' % SL)
# Define path for output
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
filedir_out_densbulk = os.path.join(Dir, 'SoilGrids', 'Bulk_Density')
if not os.path.exists(filedir_out_densbulk):
os.makedirs(filedir_out_densbulk)
filedir_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat')
if not os.path.exists(filedir_out_thetasat):
os.makedirs(filedir_out_thetasat)
#filename_out_densbulk = os.path.join(filedir_out_densbulk ,'Bulk_Density_%s_SoilGrids_g-cm-3.tif' %level)
filename_out_thetasat = os.path.join(
filedir_out_thetasat, 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level)
#if not (os.path.exists(filename_out_densbulk) and os.path.exists(filename_out_thetasat)):
if not os.path.exists(filename_out_thetasat):
# Open datasets
dest_clay = gdal.Open(filename_clay)
#dest_om = gdal.Open(filename_om)
dest_bulk = gdal.Open(filename_bulkdensity)
# Open Array info
geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_clay)
# Open Arrays
Clay = dest_clay.GetRasterBand(1).ReadAsArray()
#OM = dest_om.GetRasterBand(1).ReadAsArray()
Clay = np.float_(Clay)
Clay[Clay > 100] = np.nan
#OM = np.float_(OM)
#OM[OM<0]=np.nan
#OM = OM/1000
# Calculate bulk density
#bulk_dens = 1/(0.6117 + 0.3601 * Clay/100 + 0.002172 * np.power(OM * 100, 2)+ 0.01715 * np.log(OM * 100))
bulk_dens = dest_bulk.GetRasterBand(1).ReadAsArray()
bulk_dens = bulk_dens / 1000
'''
# Oude methode gebaseerd op Schenost, Sinowski & Priesack (1996)
# Calculate theta saturated
theta_sat = 0.85 * (1- (bulk_dens/2.65)) + 0.13 * Clay/100
'''
# Nieuwe methode gebaseerd op Toth et al (2014)
# Calculate silt fraction based on clay fraction
Silt_fraction = 0.7 * (Clay / 100)**2 + 0.308 * Clay / 100
# Calculate theta sat
theta_sat = 0.8308 - 0.28217 * bulk_dens + 0.02728 * Clay / 100 + 0.0187 * Silt_fraction
# Save data
#DC.Save_as_tiff(filename_out_densbulk, bulk_dens, geo_out, "WGS84")
DC.Save_as_tiff(filename_out_thetasat, theta_sat, geo_out, "WGS84")
return ()
def main(Dir, Startdate = '', Enddate = '',
latlim = [-60, 60], lonlim = [-180, 180], pixel_size = False, cores = False, LANDSAF = 0, SourceLANDSAF= '', Waitbar = 1):
"""
This function downloads TRMM3B43 V7 (monthly) data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine.
It can be 'False' to avoid using parallel computing
routines.
Waitbar -- 1 (Default) will print the waitbar
"""
print('Create monthly Reference ET data for period %s till %s' %(Startdate, Enddate))
# An array of monthly dates which will be calculated
Dates = pd.date_range(Startdate,Enddate,freq = 'MS')
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
# Calculate the ETref day by day for every month
for Date in Dates:
# Collect date data
Y=Date.year
M=Date.month
Mday=calendar.monthrange(Y,M)[1]
Days=pd.date_range(Date,Date+pd.Timedelta(days=Mday),freq='D')
StartTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+ '-01'
EndTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+'-'+str(Mday)
# Get ETref on daily basis
daily(Dir=Dir, Startdate=StartTime,Enddate=EndTime,latlim=latlim, lonlim=lonlim, pixel_size = pixel_size, cores=cores, LANDSAF=LANDSAF, SourceLANDSAF=SourceLANDSAF, Waitbar = 0)
# Load DEM
if not pixel_size:
nameDEM='DEM_HydroShed_m_3s.tif'
DEMmap=os.path.join(Dir,'HydroSHED','DEM',nameDEM )
else:
DEMmap=os.path.join(Dir,'HydroSHED','DEM','DEM_HydroShed_m_reshaped_for_ETref.tif')
# Get some geo-data to save results
geo_ET, proj, size_X, size_Y = RC.Open_array_info(DEMmap)
dataMonth=np.zeros([size_Y,size_X])
for Day in Days[:-1]:
DirDay=os.path.join(Dir,'ETref','Daily','ETref_mm-day-1_daily_' + Day.strftime('%Y.%m.%d') + '.tif')
dataDay=gdal.Open(DirDay)
Dval=dataDay.GetRasterBand(1).ReadAsArray().astype(np.float32)
Dval[Dval<0]=0
dataMonth=dataMonth+Dval
dataDay=None
# make geotiff file
output_folder_month=os.path.join(Dir,'ETref','Monthly')
if os.path.exists(output_folder_month)==False:
os.makedirs(output_folder_month)
DirMonth=os.path.join(output_folder_month,'ETref_mm-month-1_monthly_'+Date.strftime('%Y.%m.%d') + '.tif')
# Create the tiff file
DC.Save_as_tiff(DirMonth,dataMonth, geo_ET, proj)
# Create Waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None):
"""
This functions calculates yearly tiff files based on the monthly tiff files. (will calculate the total sum)
Parameters
----------
Dir_in : str
Path to the input data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Dir_out : str
Path to the output data, default is same as Dir_in
"""
# import WA+ modules
import watertools.General.data_conversions as DC
import watertools.General.raster_conversions as RC
# Change working directory
os.chdir(Dir_in)
# Define end and start date
Dates = pd.date_range(Startdate, Enddate, freq='AS')
# Find all monthly files
files = glob.glob('*monthly*.tif')
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value
dest = gdal.Open(files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
for date in Dates:
Year = date.year
files_one_year = glob.glob('*monthly*%d*.tif' % Year)
# Create empty arrays
Year_data = np.zeros([size_Y, size_X])
if len(files_one_year) is not int(12):
print("One month in year %s is missing!" % Year)
for file_one_year in files_one_year:
file_path = os.path.join(Dir_in, file_one_year)
Month_data = RC.Open_tiff_array(file_path)
Month_data[np.isnan(Month_data)] = 0.0
Month_data[Month_data == -9999] = 0.0
Year_data += Month_data
# Define output directory
if Dir_out == None:
Dir_out = Dir_in
if not os.path.exists(Dir_out):
os.makedirs(Dir_out)
# Define output name
output_name = os.path.join(
Dir_out,
file_one_year.replace('monthly',
'yearly').replace('month', 'year'))
output_name = output_name[:-14] + '%d.01.01.tif' % (date.year)
# Save tiff file
DC.Save_as_tiff(output_name, Year_data, geo_out, proj)
return
def slope_correct(down_short_hor, pressure, ea, DEMmap, DOY):
"""
This function downscales the CFSR solar radiation by using the DEM map
The Slope correction is based on Allen et al. (2006)
'Analytical integrated functions for daily solar radiation on slope'
Keyword arguments:
down_short_hor -- numpy array with the horizontal downwards shortwave radiation
pressure -- numpy array with the air pressure
ea -- numpy array with the actual vapour pressure
DEMmap -- 'C:/' path to the DEM map
DOY -- day of the year
"""
# Get Geo Info
GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap)
minx = GeoT[0]
miny = GeoT[3] + xsize*GeoT[4] + ysize*GeoT[5]
x = np.flipud(np.arange(xsize)*GeoT[1] + minx + GeoT[1]/2)
y = np.flipud(np.arange(ysize)*-GeoT[5] + miny + -GeoT[5]/2)
# Calculate Extraterrestrial Solar Radiation [W m-2]
demmap = RC.Open_tiff_array(DEMmap)
demmap[demmap<0]=0
# apply the slope correction
Ra_hor, Ra_slp, sinb, sinb_hor, fi, slope, ID = SlopeInfluence(demmap,y,x,DOY)
# Calculate atmospheric transmissivity
Rs_hor = down_short_hor
# EQ 39
tau = Rs_hor/Ra_hor
#EQ 41
KB_hor = np.zeros(tau.shape) * np.nan
indice = np.where(tau.flat >= 0.42)
KB_hor.flat[indice] = 1.56*tau.flat[indice] -0.55
indice = np.logical_and(tau.flat > 0.175, tau.flat < 0.42)
KB_hor.flat[indice] = 0.022 - 0.280*tau.flat[indice] + 0.828*tau.flat[indice]**2 + 0.765*tau.flat[indice]**3
indice = np.where(tau.flat <= 0.175)
KB_hor.flat[indice] = 0.016*tau.flat[indice]
# EQ 42
KD_hor = tau - KB_hor
Kt=0.7
#EQ 18
W = 0.14*ea*pressure + 2.1
KB0 = 0.98*np.exp((-0.00146*pressure/Kt/sinb)-0.075*(W/sinb)**0.4)
KB0_hor = 0.98*np.exp((-0.00146*pressure/Kt/sinb_hor)-0.075*(W/sinb_hor)**0.4)
#EQ 34
fB = KB0/KB0_hor * Ra_slp/Ra_hor
fia = (1-KB_hor) * (1 + (KB_hor/(KB_hor+KD_hor))**0.5 * np.sin(slope/2)**3)*fi + fB*KB_hor
Rs = Rs_hor*(fB*(KB_hor/tau) + fia*(KD_hor/tau) + 0.23*(1-fi))
Rs[np.isnan(Rs)] = Rs_hor[np.isnan(Rs)]
Rs_equiv = Rs / np.cos(slope)
bias = np.nansum(Rs_hor)/np.nansum(Rs_equiv)
return Rs_equiv, tau, bias
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools.Collect.SoilGrids as SG
# Download needed layers
SG.Clay_Content(Dir, latlim, lonlim, level=SL)
SG.Organic_Carbon_Content(Dir, latlim, lonlim, level=SL)
#SG.Bulk_Density(Dir, latlim, lonlim, level=SL)
# Define path to layers
filename_clay = os.path.join(Dir, 'SoilGrids', 'Clay_Content' ,'ClayContentMassFraction_%s_SoilGrids_percentage.tif' %SL)
filename_om = os.path.join(Dir, 'SoilGrids', 'Soil_Organic_Carbon_Content' ,'SoilOrganicCarbonContent_%s_SoilGrids_g_kg.tif' %SL)
#filename_bulkdensity = os.path.join(Dir, 'SoilGrids', 'Bulk_density' ,'BulkDensity_%s_SoilGrids_kg-m-3.tif' %SL)
# Define path for output
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
filedir_out_densbulk = os.path.join( Dir, 'SoilGrids', 'Bulk_density')
if not os.path.exists(filedir_out_densbulk):
os.makedirs(filedir_out_densbulk)
filedir_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat')
if not os.path.exists(filedir_out_thetasat):
os.makedirs(filedir_out_thetasat)
filename_out_densbulk = os.path.join(filedir_out_densbulk ,'Bulk_Density_%s_SoilGrids_g-cm-3.tif' %level)
filename_out_thetasat = os.path.join(filedir_out_thetasat,'Theta_Sat_%s_SoilGrids_kg-kg.tif' %level)
if not (os.path.exists(filename_out_densbulk) and os.path.exists(filename_out_thetasat)):
#if not os.path.exists(filename_out_thetasat):
# Open datasets
dest_clay = gdal.Open(filename_clay)
dest_om = gdal.Open(filename_om)
#dest_bulk = gdal.Open(filename_bulkdensity)
# Open Array info
geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_clay)
# Open Arrays
Clay = dest_clay.GetRasterBand(1).ReadAsArray()
OM = dest_om.GetRasterBand(1).ReadAsArray()
Clay = np.float_(Clay)
Clay[Clay>100]=np.nan
OM = np.float_(OM)
OM[OM<0]=np.nan
OM = OM/1000
# Calculate bulk density
bulk_dens = 1/(0.6117 + 0.3601 * Clay/100 + 0.002172 * np.power(OM * 100, 2)+ 0.01715 * np.log(OM * 100))
#bulk_dens = dest_bulk.GetRasterBand(1).ReadAsArray()
# Calculate theta saturated
theta_sat = 0.85 * (1- (bulk_dens/2.65)) + 0.13 * Clay/100
# Save data
DC.Save_as_tiff(filename_out_densbulk, bulk_dens, geo_out, "WGS84")
DC.Save_as_tiff(filename_out_thetasat, theta_sat, geo_out, "WGS84")
return()
def Save_as_NC(namenc, DataCube, Var, Reference_filename, Startdate = '', Enddate = '', Time_steps = '', Scaling_factor = 1):
"""
This function save the array as a netcdf file
Keyword arguments:
namenc -- string, complete path of the output file with .nc extension
DataCube -- [array], dataset of the nc file, can be a 2D or 3D array [time, lat, lon], must be same size as reference data
Var -- string, the name of the variable
Reference_filename -- string, complete path to the reference file name
Startdate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array, defines the Start datum of the dataset
Enddate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array, defines the End datum of the dataset
Time_steps -- 'monthly' or 'daily', needs to be filled when you want to save a 3D array, defines the timestep of the dataset
Scaling_factor -- number, scaling_factor of the dataset, default = 1
"""
# Import modules
import watertools.General.raster_conversions as RC
from netCDF4 import Dataset
if not os.path.exists(namenc):
# Get raster information
geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_filename)
# Create the lat/lon rasters
lon = np.arange(size_X)*geo_out[1]+geo_out[0] - 0.5 * geo_out[1]
lat = np.arange(size_Y)*geo_out[5]+geo_out[3] - 0.5 * geo_out[5]
# Create the nc file
nco = Dataset(namenc, 'w', format='NETCDF4_CLASSIC')
nco.description = '%s data' %Var
# Create dimensions, variables and attributes:
nco.createDimension('longitude', size_X)
nco.createDimension('latitude', size_Y)
# Create time dimension if the parameter is time dependent
if Startdate is not '':
if Time_steps == 'monthly':
Dates = pd.date_range(Startdate,Enddate,freq = 'MS')
if Time_steps == 'daily':
Dates = pd.date_range(Startdate,Enddate,freq = 'D')
time_or=np.zeros(len(Dates))
i = 0
for Date in Dates:
time_or[i] = Date.toordinal()
i += 1
nco.createDimension('time', None)
timeo = nco.createVariable('time', 'f4', ('time',))
timeo.units = '%s' %Time_steps
timeo.standard_name = 'time'
# Create the lon variable
lono = nco.createVariable('longitude', 'f8', ('longitude',))
lono.standard_name = 'longitude'
lono.units = 'degrees_east'
lono.pixel_size = geo_out[1]
# Create the lat variable
lato = nco.createVariable('latitude', 'f8', ('latitude',))
lato.standard_name = 'latitude'
lato.units = 'degrees_north'
lato.pixel_size = geo_out[5]
# Create container variable for CRS: lon/lat WGS84 datum
crso = nco.createVariable('crs', 'i4')
crso.long_name = 'Lon/Lat Coords in WGS84'
crso.grid_mapping_name = 'latitude_longitude'
crso.projection = proj
crso.longitude_of_prime_meridian = 0.0
crso.semi_major_axis = 6378137.0
crso.inverse_flattening = 298.257223563
crso.geo_reference = geo_out
# Create the data variable
if Startdate is not '':
preco = nco.createVariable('%s' %Var, 'f8', ('time', 'latitude', 'longitude'), zlib=True, least_significant_digit=1)
timeo[:]=time_or
else:
preco = nco.createVariable('%s' %Var, 'f8', ('latitude', 'longitude'), zlib=True, least_significant_digit=1)
# Set the data variable information
preco.scale_factor = Scaling_factor
preco.add_offset = 0.00
preco.grid_mapping = 'crs'
preco.set_auto_maskandscale(False)
# Set the lat/lon variable
lono[:] = lon
lato[:] = lat
# Set the data variable
if Startdate is not '':
for i in range(len(Dates)):
preco[i,:,:] = DataCube[i,:,:]*1./np.float(Scaling_factor)
else:
preco[:,:] = DataCube[:,:] * 1./np.float(Scaling_factor)
nco.close()
return()
def ETref(Date, args):
"""
This function starts to calculate ETref (daily) data based on Hydroshed, GLDAS, and (CFSR/LANDSAF) in parallel or single core
Keyword arguments:
Date -- panda timestamp
args -- includes all the parameters that are needed for the ETref
"""
# unpack the arguments
[Dir, lonlim, latlim, pixel_size, LANDSAF] = args
# Set the paths
nameTmin = 'Tair-min_GLDAS-NOAH_C_daily_' + Date.strftime(
'%Y.%m.%d') + ".tif"
tmin_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'tair_f_inst', 'min', nameTmin)
nameTmax = 'Tair-max_GLDAS-NOAH_C_daily_' + Date.strftime(
'%Y.%m.%d') + ".tif"
tmax_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'tair_f_inst', 'max', nameTmax)
nameHumid = 'Hum_GLDAS-NOAH_kg-kg_daily_' + Date.strftime(
'%Y.%m.%d') + ".tif"
humid_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'qair_f_inst', 'mean', nameHumid)
namePress = 'P_GLDAS-NOAH_kpa_daily_' + Date.strftime('%Y.%m.%d') + ".tif"
press_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'psurf_f_inst', 'mean', namePress)
nameWind = 'W_GLDAS-NOAH_m-s-1_daily_' + Date.strftime('%Y.%m.%d') + ".tif"
wind_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'wind_f_inst', 'mean', nameWind)
if LANDSAF == 1:
nameShortClearname = 'ShortWave_Clear_Daily_W-m2_' + Date.strftime(
'%Y-%m-%d') + '.tif'
input2_str = os.path.join(Dir, 'Landsaf_Clipped',
'Shortwave_Clear_Sky', nameShortClearname)
nameShortNetname = 'ShortWave_Net_Daily_W-m2_' + Date.strftime(
'%Y-%m-%d') + '.tif'
input1_str = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net',
nameShortNetname)
input3_str = 'not'
else:
if Date < pd.Timestamp(pd.datetime(2011, 4, 1)):
nameDownLong = 'DLWR_CFSR_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input2_str = os.path.join(Dir, 'Radiation', 'CFSR', nameDownLong)
nameDownShort = 'DSWR_CFSR_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input1_str = os.path.join(Dir, 'Radiation', 'CFSR', nameDownShort)
nameUpLong = 'ULWR_CFSR_W-m2_' + Date.strftime('%Y.%m.%d') + ".tif"
input3_str = os.path.join(Dir, 'Radiation', 'CFSR', nameUpLong)
else:
nameDownLong = 'DLWR_CFSRv2_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input2_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameDownLong)
nameDownShort = 'DSWR_CFSRv2_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input1_str = os.path.join(Dir, 'Radiation', 'CFSRv2',
nameDownShort)
nameUpLong = 'ULWR_CFSRv2_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input3_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameUpLong)
# The day of year
DOY = Date.dayofyear
# Load DEM
if not pixel_size:
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
else:
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
dest, ulx, lry, lrx, uly, epsg_to = RC.reproject_dataset_epsg(
DEMmap_str, pixel_spacing=pixel_size, epsg_to=4326, method=2)
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_reshaped_for_ETref.tif')
DEM_data = dest.GetRasterBand(1).ReadAsArray()
geo_dem = [ulx, pixel_size, 0.0, uly, 0.0, -pixel_size]
DC.Save_as_tiff(name=DEMmap_str,
data=DEM_data,
geo=geo_dem,
projection='4326')
# Calc ETref
ETref = calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str,
input1_str, input2_str, input3_str, DEMmap_str, DOY)
# Make directory for the MODIS ET data
output_folder = os.path.join(Dir, 'ETref', 'Daily')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Create the output names
NameETref = 'ETref_mm-day-1_daily_' + Date.strftime('%Y.%m.%d') + '.tif'
NameEnd = os.path.join(output_folder, NameETref)
# Collect geotiff information
geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str)
# Create daily ETref tiff files
DC.Save_as_tiff(name=NameEnd, data=ETref, geo=geo_out, projection=proj)
def Merge_DEM_15s_30s(output_folder_trash, output_file_merged, latlim, lonlim,
resolution):
os.chdir(output_folder_trash)
tiff_files = glob.glob('*.tif')
resolution_geo = []
lonmin = lonlim[0]
lonmax = lonlim[1]
latmin = latlim[0]
latmax = latlim[1]
if resolution == "15s":
resolution_geo = 0.00416667
if resolution == "30s":
resolution_geo = 0.00416667 * 2
size_x_tot = int(np.round((lonmax - lonmin) / resolution_geo))
size_y_tot = int(np.round((latmax - latmin) / resolution_geo))
data_tot = np.ones([size_y_tot, size_x_tot]) * -9999.
for tiff_file in tiff_files:
inFile = os.path.join(output_folder_trash, tiff_file)
geo, proj, size_X, size_Y = RC.Open_array_info(inFile)
resolution_geo = geo[1]
lonmin_one = geo[0]
lonmax_one = geo[0] + size_X * geo[1]
latmin_one = geo[3] + size_Y * geo[5]
latmax_one = geo[3]
if lonmin_one < lonmin:
lonmin_clip = lonmin
else:
lonmin_clip = lonmin_one
if lonmax_one > lonmax:
lonmax_clip = lonmax
else:
lonmax_clip = lonmax_one
if latmin_one < latmin:
latmin_clip = latmin
else:
latmin_clip = latmin_one
if latmax_one > latmax:
latmax_clip = latmax
else:
latmax_clip = latmax_one
size_x_clip = int(
np.round((lonmax_clip - lonmin_clip) / resolution_geo))
size_y_clip = int(
np.round((latmax_clip - latmin_clip) / resolution_geo))
inFile = os.path.join(output_folder_trash, tiff_file)
geo, proj, size_X, size_Y = RC.Open_array_info(inFile)
Data = RC.Open_tiff_array(inFile)
lonmin_tiff = geo[0]
latmax_tiff = geo[3]
lon_tiff_position = int(
np.round((lonmin_clip - lonmin_tiff) / resolution_geo))
lat_tiff_position = int(
np.round((latmax_tiff - latmax_clip) / resolution_geo))
lon_data_tot_position = int(
np.round((lonmin_clip - lonmin) / resolution_geo))
lat_data_tot_position = int(
np.round((latmax - latmax_clip) / resolution_geo))
Data[Data < -9999.] = -9999.
data_tot[lat_data_tot_position:lat_data_tot_position + size_y_clip,
lon_data_tot_position:lon_data_tot_position +
size_x_clip][data_tot[
lat_data_tot_position:lat_data_tot_position + size_y_clip,
lon_data_tot_position:lon_data_tot_position +
size_x_clip] == -9999] = Data[
lat_tiff_position:lat_tiff_position + size_y_clip,
lon_tiff_position:lon_tiff_position +
size_x_clip][data_tot[
lat_data_tot_position:lat_data_tot_position +
size_y_clip,
lon_data_tot_position:lon_data_tot_position +
size_x_clip] == -9999]
geo_out = [lonmin, resolution_geo, 0.0, latmax, 0.0, -1 * resolution_geo]
geo_out = tuple(geo_out)
data_tot[data_tot < -9999.] = -9999.
return (data_tot, geo_out)
