def Convert_dict_to_array(River_dict, Array_dict, Reference_data): import numpy as np import os import watertools.General.raster_conversions as RC if os.path.splitext(Reference_data)[-1] == '.nc': # Get raster information geo_out, proj, size_X, size_Y, size_Z, Time = RC.Open_nc_info(Reference_data) else: # Get raster information geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_data) # Create ID Matrix y,x = np.indices((size_Y, size_X)) ID_Matrix = np.int32(np.ravel_multi_index(np.vstack((y.ravel(),x.ravel())),(size_Y,size_X),mode='clip').reshape(x.shape)) + 1 # Get tiff array time dimension: time_dimension = int(np.shape(Array_dict[0])[0]) # create an empty array DataCube = np.ones([time_dimension, size_Y, size_X]) * np.nan for river_part in range(0,len(River_dict)): for river_pixel in range(1,len(River_dict[river_part])): river_pixel_ID = River_dict[river_part][river_pixel] if len(np.argwhere(ID_Matrix == river_pixel_ID))>0: row, col = np.argwhere(ID_Matrix == river_pixel_ID)[0][:] DataCube[:,row,col] = Array_dict[river_part][:,river_pixel] return(DataCube)
def RetrieveData(args): """ This function retrieves JRC data for a given date from the http://storage.googleapis.com/global-surface-water/downloads/ server. Keyword arguments: args -- A list of parameters defined in the DownloadData function. """ # Argument [output_folder, Names_to_download, lonlim, latlim] = args # Collect the data from the JRC webpage and returns the data and lat and long in meters of those tiles try: Collect_data(Names_to_download, output_folder) except: print("Was not able to download the file") # Clip the data to the users extend if len(Names_to_download) == 1: trash_folder = os.path.join(output_folder, "Trash") data_in = os.path.join(trash_folder, Names_to_download[0]) data_end, geo_end = RC.clip_data(data_in, latlim, lonlim) else: data_end = np.zeros([int((latlim[1] - latlim[0])/0.00025), int((lonlim[1] - lonlim[0])/0.00025)]) for Name_to_merge in Names_to_download: trash_folder = os.path.join(output_folder, "Trash") data_in = os.path.join(trash_folder, Name_to_merge) geo_out, proj, size_X, size_Y = RC.Open_array_info(data_in) lat_min_merge = np.maximum(latlim[0], geo_out[3] + size_Y * geo_out[5]) lat_max_merge = np.minimum(latlim[1], geo_out[3]) lon_min_merge = np.maximum(lonlim[0], geo_out[0]) lon_max_merge = np.minimum(lonlim[1], geo_out[0] + size_X * geo_out[1]) lonmerge = [lon_min_merge, lon_max_merge] latmerge = [lat_min_merge, lat_max_merge] data_one, geo_one = RC.clip_data(data_in, latmerge, lonmerge) Ystart = int((geo_one[3] - latlim[1])/geo_one[5]) Yend = int(Ystart + np.shape(data_one)[0]) Xstart = int((geo_one[0] - lonlim[0])/geo_one[1]) Xend = int(Xstart + np.shape(data_one)[1]) data_end[Ystart:Yend, Xstart:Xend] = data_one geo_end = tuple([lonlim[0], geo_one[1], 0, latlim[1], 0, geo_one[5]]) # Save results as Gtiff fileName_out = os.path.join(output_folder, 'JRC_Occurrence_percent.tif') DC.Save_as_tiff(name=fileName_out, data=data_end, geo=geo_end, projection='WGS84') shutil.rmtree(trash_folder) return True
def Calc_Humidity(Temp_format, P_format, Hum_format, output_format, Startdate, Enddate, freq="D"): folder_dir_out = os.path.dirname(output_format) if not os.path.exists(folder_dir_out): os.makedirs(folder_dir_out) Dates = pd.date_range(Startdate, Enddate, freq="D") for Date in Dates: print(Date) Day = Date.day Month = Date.month Year = Date.year Tempfile_one = Temp_format.format(yyyy=Year, mm=Month, dd=Day) Presfile_one = P_format.format(yyyy=Year, mm=Month, dd=Day) Humfile_one = Hum_format.format(yyyy=Year, mm=Month, dd=Day) out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day) geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one) Tdata = RC.Open_tiff_array(Tempfile_one) if "MERRA_K" in Temp_format: Tdata = Tdata - 273.15 Tdata[Tdata < -900] = -9999 Pdata = RC.Open_tiff_array(Presfile_one) Hdata = RC.Open_tiff_array(Humfile_one) Pdata[Pdata < 0] = -9999 Hdata[Hdata < 0] = -9999 # gapfilling Tdata = RC.gap_filling(Tdata, -9999) Pdata = RC.gap_filling(Pdata, -9999) Hdata = RC.gap_filling(Hdata, -9999) Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3)) HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100 HumData = HumData.clip(0, 100) DC.Save_as_tiff(out_folder_one, HumData, geo_out, "WGS84") return ()
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join( Dir, 'SoilGrids', 'Theta_Sat', 'Theta_Sat_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat.Subsoil( Dir, latlim, lonlim) filedir_out_whc = os.path.join(Dir, 'SoilGrids', 'Water_Holding_Capacity') if not os.path.exists(filedir_out_whc): os.makedirs(filedir_out_whc) # Define theta field capacity output filename_out_whc = os.path.join( filedir_out_whc, 'Water_Holding_Capacity_%s_SoilGrids_mm-m.tif' % level) if not os.path.exists(filename_out_whc): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info( filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate theta field capacity theta_whc = np.ones(theta_sat.shape) * -9999 theta_whc = np.where( theta_sat < 0.301, 80, 450 * np.arccosh(theta_sat + 0.7) - 2 * (theta_sat + 0.7) + 20) # Save as tiff DC.Save_as_tiff(filename_out_whc, theta_whc, geo_out, proj) return
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join( Dir, 'SoilGrids', 'Theta_Sat', 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat2.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat2.Subsoil( Dir, latlim, lonlim) filedir_out_n_genuchten = os.path.join(Dir, 'SoilGrids', 'N_van_genuchten') if not os.path.exists(filedir_out_n_genuchten): os.makedirs(filedir_out_n_genuchten) # Define n van genuchten output filename_out_ngenuchten = os.path.join( filedir_out_n_genuchten, 'N_genuchten_%s_SoilGrids_-.tif' % level) if not os.path.exists(filename_out_ngenuchten): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info( filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate n van genuchten n_van_genuchten = np.ones(theta_sat.shape) * -9999 n_van_genuchten = 166.63 * theta_sat**4 - 387.72 * theta_sat**3 + 340.55 * theta_sat**2 - 133.07 * theta_sat + 20.739 # Save as tiff DC.Save_as_tiff(filename_out_ngenuchten, n_van_genuchten, geo_out, proj) return
def Calc_Humidity(Temp_format, P_format, Hum_format, output_format, Startdate, Enddate, freq="D"): folder_dir_out = os.path.dirname(output_format) if not os.path.exists(folder_dir_out): os.makedirs(folder_dir_out) Dates = pd.date_range(Startdate, Enddate, freq="D") for Date in Dates: print(Date) Day = Date.day Month = Date.month Year = Date.year Windyfile_one = wind_y_format.format(yyyy=Year, mm=Month, dd=Day) Windxfile_one = wind_x_format.format(yyyy=Year, mm=Month, dd=Day) out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day) geo_out, proj, size_X, size_Y = RC.Open_array_info(Windxfile_one) Windxdata = RC.Open_tiff_array(Windxfile_one) Windxdata[Windxdata < -900] = -9999 Windydata = RC.Open_tiff_array(Windyfile_one) Windydata[Windxdata < -900] = -9999 # gapfilling Windydata = RC.gap_filling(Windydata, -9999) Windxdata = RC.gap_filling(Windxdata, -9999) Wind = np.sqrt(Windxdata**2 + Windydata**2) DC.Save_as_tiff(out_folder_one, Wind, geo_out, "WGS84") return ()
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat' ,'Theta_Sat2_%s_SoilGrids_kg-kg.tif' %level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat2.Topsoil(Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat2.Subsoil(Dir, latlim, lonlim) filedir_out_thetares = os.path.join(Dir, 'SoilGrids', 'Theta_Res') if not os.path.exists(filedir_out_thetares): os.makedirs(filedir_out_thetares) # Define theta field capacity output filename_out_thetares = os.path.join(filedir_out_thetares ,'Theta_Res_%s_SoilGrids_kg-kg.tif' %level) if not os.path.exists(filename_out_thetares): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate theta field capacity theta_Res = np.ones(theta_sat.shape) * -9999 #theta_Res = np.where(theta_sat < 0.351, 0.01, 0.4 * np.arccosh(theta_sat + 0.65) - 0.05 * np.power(theta_sat + 0.65, 2.5) + 0.02) theta_Res = np.where(theta_sat < 0.351, 0.01, 0.271 * np.log(theta_sat) + 0.335) # Save as tiff DC.Save_as_tiff(filename_out_thetares, theta_Res, geo_out, proj) return
def lapse_rate(Dir,temperature_map, DEMmap): """ This function downscales the GLDAS temperature map by using the DEM map Keyword arguments: temperature_map -- 'C:/' path to the temperature map DEMmap -- 'C:/' path to the DEM map """ # calculate average altitudes corresponding to T resolution dest = RC.reproject_dataset_example(DEMmap, temperature_map,method = 4) DEM_ave_out_name = os.path.join(Dir,'HydroSHED', 'DEM','DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map) DEM_ave_data = dest.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) dest = None # determine lapse-rate [degress Celcius per meter] lapse_rate_number = 0.0065 # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2) dem_avg=dest.GetRasterBand(1).ReadAsArray() dem_avg[dem_avg<0]=0 dest = None # Open the temperature dataset dest = RC.reproject_dataset_example(temperature_map, DEMmap, method = 2) T=dest.GetRasterBand(1).ReadAsArray() dest = None # Open Demmap demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap<=0]=0 demmap[demmap==-32768]=np.nan # calculate first part T = T + ((dem_avg-demmap) * lapse_rate_number) return T
def Degrees_to_m2(Reference_data): """ This functions calculated the area of each pixel in squared meter. Parameters ---------- Reference_data: str Path to a tiff file or nc file or memory file of which the pixel area must be defined Returns ------- area_in_m2: array Array containing the area of each pixel in squared meters """ try: # Get the extension of the example data filename, file_extension = os.path.splitext(Reference_data) # Get raster information if str(file_extension) == '.tif': geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_data) elif str(file_extension) == '.nc': geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info( Reference_data) except: geo_out = Reference_data.GetGeoTransform() size_X = Reference_data.RasterXSize() size_Y = Reference_data.RasterYSize() # Calculate the difference in latitude and longitude in meters dlat, dlon = Calc_dlat_dlon(geo_out, size_X, size_Y) # Calculate the area in squared meters area_in_m2 = dlat * dlon return (area_in_m2)
def adjust_P(Dir, pressure_map, DEMmap): """ This function downscales the GLDAS air pressure map by using the DEM map Keyword arguments: pressure_map -- 'C:/' path to the pressure map DEMmap -- 'C:/' path to the DEM map """ # calculate average latitudes destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method = 4) DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM','DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map) DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2) dem_avg=dest.GetRasterBand(1).ReadAsArray() dest = None # open maps as numpy arrays dest = RC.reproject_dataset_example(pressure_map, DEMmap, method = 2) P=dest.GetRasterBand(1).ReadAsArray() dest = None demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap<=0]=0 demmap[demmap==-32768]=np.nan # calculate second part P = P + (101.3*((293-0.0065*(demmap-dem_avg))/293)**5.26 - 101.3) os.remove(DEM_ave_out_name) return P
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None): """ This functions calculates monthly tiff files based on the 8 daily tiff files. (will calculate the average) Parameters ---------- Dir_in : str Path to the input data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Dir_out : str Path to the output data, default is same as Dir_in """ # import WA+ modules import watertools.General.data_conversions as DC import watertools.General.raster_conversions as RC # Change working directory os.chdir(Dir_in) # Find all eight daily files files = glob.glob('*8-daily*.tif') # Create array with filename and keys (DOY and year) of all the 8 daily files i = 0 DOY_Year = np.zeros([len(files), 3]) for File in files: # Get the time characteristics from the filename year = File.split('.')[-4][-4:] month = File.split('.')[-3] day = File.split('.')[-2] # Create pandas Timestamp date_file = '%s-%02s-%02s' % (year, month, day) Datum = pd.Timestamp(date_file) # Get day of year DOY = Datum.strftime('%j') # Save data in array DOY_Year[i, 0] = i DOY_Year[i, 1] = DOY DOY_Year[i, 2] = year # Loop over files i += 1 # Check enddate: Enddate_split = Enddate.split('-') month_range = calendar.monthrange(int(Enddate_split[0]), int(Enddate_split[1]))[1] Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int( Enddate_split[1]), month_range) # Check startdate: Startdate_split = Startdate.split('-') Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int( Startdate_split[1])) # Define end and start date Dates = pd.date_range(Startdate, Enddate, freq='MS') DatesEnd = pd.date_range(Startdate, Enddate, freq='M') # Get array information and define projection geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0]) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Get the No Data Value dest = gdal.Open(files[0]) NDV = dest.GetRasterBand(1).GetNoDataValue() # Loop over months and create monthly tiff files i = 0 for date in Dates: # Get Start and end DOY of the current month DOY_month_start = date.strftime('%j') DOY_month_end = DatesEnd[i].strftime('%j') # Search for the files that are between those DOYs year = date.year DOYs = DOY_Year[DOY_Year[:, 2] == year] DOYs_oneMonth = DOYs[np.logical_and( (DOYs[:, 1] + 8) >= int(DOY_month_start), DOYs[:, 1] <= int(DOY_month_end))] # Create empty arrays Monthly = np.zeros([size_Y, size_X]) Weight_tot = np.zeros([size_Y, size_X]) Data_one_month = np.ones([size_Y, size_X]) * np.nan # Loop over the files that are within the DOYs for EightDays in DOYs_oneMonth[:, 1]: # Calculate the amount of days in this month of each file Weight = np.ones([size_Y, size_X]) # For start of month if np.min(DOYs_oneMonth[:, 1]) == EightDays: Weight = Weight * int(EightDays + 8 - int(DOY_month_start)) # For end of month elif np.max(DOYs_oneMonth[:, 1]) == EightDays: Weight = Weight * (int(DOY_month_end) - EightDays + 1) # For the middle of the month else: Weight = Weight * 8 row = DOYs_oneMonth[np.argwhere( DOYs_oneMonth[:, 1] == EightDays)[0][0], :][0] # Open the array of current file input_name = os.path.join(Dir_in, files[int(row)]) Data = RC.Open_tiff_array(input_name) # Remove NDV Weight[Data == NDV] = 0 Data[Data == NDV] = np.nan # Multiply weight time data Data = Data * Weight # Calculate the total weight and data Weight_tot += Weight Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)] # Go to next month i += 1 # Calculate the average Data_one_month[Weight_tot != 0.] = Monthly[ Weight_tot != 0.] / Weight_tot[Weight_tot != 0.] # Define output directory if Dir_out == None: Dir_out = Dir_in # Define output name output_name = os.path.join( Dir_out, files[int(row)].replace('8-daily', 'monthly')) output_name = output_name[:-9] + '%02d.01.tif' % (date.month) # Save tiff file DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj) return
def calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str, down_short_str, down_long_str, up_long_str, DEMmap_str, DOY): """ This function calculates the ETref by using all the input parameters (path) according to FAO standards see: http://www.fao.org/docrep/x0490e/x0490e08.htm#TopOfPage Keyword arguments: tmin_str -- 'C:/' path to the minimal temperature tiff file [degrees Celcius], e.g. from GLDAS tmax_str -- 'C:/' path to the maximal temperature tiff file [degrees Celcius], e.g. from GLDAS humid_str -- 'C:/' path to the humidity tiff file [kg/kg], e.g. from GLDAS press_str -- 'C:/' path to the air pressure tiff file [kPa], e.g. from GLDAS wind_str -- 'C:/' path to the wind velocity tiff file [m/s], e.g. from GLDAS down_short_str -- 'C:/' path to the downward shortwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF down_long_str -- 'C:/' path to the downward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF up_long_str -- 'C:/' path to the upward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF DEMmap_str -- 'C:/' path to the DEM tiff file [m] e.g. from HydroSHED DOY -- Day of the year """ # Get some geo-data to save results GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap_str) #NDV, xsize, ysize, GeoT, Projection, DataType = GetGeoInfo(DEMmap_str) raster_shape = [xsize, ysize] # Create array to store results ETref = np.zeros(raster_shape) # gap fill tmin_str_GF = RC.gap_filling(tmin_str, -9999) tmax_str_GF = RC.gap_filling(tmax_str, -9999) humid_str_GF = RC.gap_filling(humid_str, -9999) press_str_GF = RC.gap_filling(press_str, -9999) wind_str_GF = RC.gap_filling(wind_str, -9999) down_short_str_GF = RC.gap_filling(down_short_str, np.nan) down_long_str_GF = RC.gap_filling(down_long_str, np.nan) if up_long_str is not 'not': up_long_str_GF = RC.gap_filling(up_long_str, np.nan) else: up_long_str_GF = 'nan' #dictionary containing all tthe paths to the input-maps inputs = dict({ 'tmin': tmin_str_GF, 'tmax': tmax_str_GF, 'humid': humid_str_GF, 'press': press_str_GF, 'wind': wind_str_GF, 'down_short': down_short_str_GF, 'down_long': down_long_str_GF, 'up_long': up_long_str_GF }) #dictionary containing numpy arrays of al initial and intermediate variables input_array = dict({ 'tmin': None, 'tmax': None, 'humid': None, 'press': None, 'wind': None, 'albedo': None, 'down_short': None, 'down_long': None, 'up_short': None, 'up_long': None, 'net_radiation': None, 'ea': None, 'es': None, 'delta': None }) #APPLY LAPSE RATE CORRECTION ON TEMPERATURE tmin = lapse_rate(Dir, inputs['tmin'], DEMmap_str) tmax = lapse_rate(Dir, inputs['tmax'], DEMmap_str) #PROCESS PRESSURE MAPS press = adjust_P(Dir, inputs['press'], DEMmap_str) #PREPARE HUMIDITY MAPS dest = RC.reproject_dataset_example(inputs['humid'], DEMmap_str, method=2) humid = dest.GetRasterBand(1).ReadAsArray() dest = None #CORRECT WIND MAPS dest = RC.reproject_dataset_example(inputs['wind'], DEMmap_str, method=2) wind = dest.GetRasterBand(1).ReadAsArray() * 0.75 dest = None #PROCESS GLDAS DATA input_array['ea'], input_array['es'], input_array['delta'] = process_GLDAS( tmax, tmin, humid, press) ea = input_array['ea'] es = input_array['es'] delta = input_array['delta'] if up_long_str == 'not': #CORRECT WIND MAPS dest = RC.reproject_dataset_example(down_short_str, DEMmap_str, method=2) Short_Net_data = dest.GetRasterBand(1).ReadAsArray() * 0.75 dest = None dest = RC.reproject_dataset_example(down_long_str, DEMmap_str, method=2) Short_Clear_data = dest.GetRasterBand(1).ReadAsArray() * 0.75 dest = None # Calculate Long wave Net radiation Rnl = 4.903e-9 * ( ((tmin + 273.16)**4 + (tmax + 273.16)**4) / 2) * (0.34 - 0.14 * np.sqrt(ea)) * ( 1.35 * Short_Net_data / Short_Clear_data - 0.35) # Calulate Net Radiation and converted to MJ*d-1*m-2 net_radiation = (Short_Net_data * 0.77 + Rnl) * 86400 / 10**6 else: #OPEN DOWNWARD SHORTWAVE RADIATION dest = RC.reproject_dataset_example(inputs['down_short'], DEMmap_str, method=2) down_short = dest.GetRasterBand(1).ReadAsArray() dest = None down_short, tau, bias = slope_correct(down_short, press, ea, DEMmap_str, DOY) #OPEN OTHER RADS up_short = down_short * 0.23 dest = RC.reproject_dataset_example(inputs['down_long'], DEMmap_str, method=2) down_long = dest.GetRasterBand(1).ReadAsArray() dest = None dest = RC.reproject_dataset_example(inputs['up_long'], DEMmap_str, method=2) up_long = dest.GetRasterBand(1).ReadAsArray() dest = None #OPEN NET RADIATION AND CONVERT W*m-2 TO MJ*d-1*m-2 net_radiation = ((down_short - up_short) + (down_long - up_long)) * 86400 / 10**6 #CALCULATE ETref ETref = (0.408 * delta * net_radiation + 0.665 * 10**-3 * press * (900 / ((tmax + tmin) / 2 + 273)) * wind * (es - ea)) / (delta + 0.665 * 10**-3 * press * (1 + 0.34 * wind)) # Set limits ETref ETref[ETref < 0] = 0 ETref[ETref > 400] = np.nan #return a reference ET map (numpy array), a dictionary containing all intermediate information and a bias of the slope correction on down_short return ETref
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join( Dir, 'SoilGrids', 'Theta_Sat', 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat2.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat2.Subsoil( Dir, latlim, lonlim) filename_out_thetares = os.path.join( Dir, 'SoilGrids', 'Theta_Res', 'Theta_Res_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetares): if SL == "sl3": watertools.Products.SoilGrids.Theta_Res.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Res.Subsoil( Dir, latlim, lonlim) filename_out_n_genuchten = os.path.join( Dir, 'SoilGrids', 'N_van_genuchten', 'N_genuchten_%s_SoilGrids_-.tif' % level) if not os.path.exists(filename_out_n_genuchten): if SL == "sl3": watertools.Products.SoilGrids.n_van_genuchten.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.n_van_genuchten.Subsoil( Dir, latlim, lonlim) filedir_out_thetafc = os.path.join(Dir, 'SoilGrids', 'Theta_FC') if not os.path.exists(filedir_out_thetafc): os.makedirs(filedir_out_thetafc) # Define theta field capacity output filename_out_thetafc = os.path.join( filedir_out_thetafc, 'Theta_FC2_%s_SoilGrids_cm3-cm3.tif' % level) if not os.path.exists(filename_out_thetafc): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info( filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) theta_res = RC.Open_tiff_array(filename_out_thetares) n_genuchten = RC.Open_tiff_array(filename_out_n_genuchten) # Calculate theta field capacity theta_FC = np.ones(theta_sat.shape) * -9999 #theta_FC = np.where(theta_sat < 0.301, 0.042, np.arccosh(theta_sat + 0.7) - 0.32 * (theta_sat + 0.7) + 0.2) #theta_FC = np.where(theta_sat < 0.301, 0.042, -2.95*theta_sat**2+3.96*theta_sat-0.871) theta_FC = theta_res + (theta_sat - theta_res) / ( 1 + (0.02 * 200)**n_genuchten)**(1 - 1 / n_genuchten) # Save as tiff DC.Save_as_tiff(filename_out_thetafc, theta_FC, geo_out, proj) return
os.makedirs(folder_dir_out) Dates = pd.date_range(Startdate, Enddate, freq="D") for Date in Dates: Day = Date.day Month = Date.month Year = Date.year Tempfile_one = Temp_folder.format(yyyy=Year, mm=Month, dd=Day) Presfile_one = Pres_folder.format(yyyy=Year, mm=Month, dd=Day) Humfile_one = Hum_folder.format(yyyy=Year, mm=Month, dd=Day) out_folder_one = out_folder.format(yyyy=Year, mm=Month, dd=Day) geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one) Tdata = RC.Open_tiff_array(Tempfile_one) Tdata[Tdata < -900] = -9999 Pdata = RC.Open_tiff_array(Presfile_one) Hdata = RC.Open_tiff_array(Humfile_one) Pdata[Pdata < 0] = -9999 Hdata[Hdata < 0] = -9999 # gapfilling Tdata = RC.gap_filling(Tdata, -9999) Pdata = RC.gap_filling(Pdata, -9999) Hdata = RC.gap_filling(Hdata, -9999) Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3)) HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100 HumData = HumData.clip(0, 100)
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim): """ This function collects and clip LANDSAF data Keyword arguments: SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID) Dir -- 'C:/' path to the WA map Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -60 and 60) lonlim -- [xmin, xmax] (values must be between -180 and 180) """ # Make an array of the days of which the ET is taken Dates = pd.date_range(Startdate, Enddate, freq='D') # make directories SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS') if os.path.exists(SISdir) is False: os.makedirs(SISdir) SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID') if os.path.exists(SIDdir) is False: os.makedirs(SIDdir) ShortwaveBasin(SourceLANDSAF, Dir, latlim, lonlim, Dates=[Startdate, Enddate]) DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_HydroShed_m_3s.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str) # Open DEM map demmap = RC.Open_tiff_array(DEMmap_str) demmap[demmap < 0] = 0 # make lat and lon arrays) dlat = geo_out[5] dlon = geo_out[1] lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon for date in Dates: # day of year day = date.dayofyear Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence( demmap, lat, lon, day) SIDname = os.path.join( SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif') SISname = os.path.join( SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif') #PREPARE SID MAPS SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3) SIDdata = SIDdest.GetRasterBand(1).ReadAsArray() #PREPARE SIS MAPS SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3) SISdata = SISdest.GetRasterBand(1).ReadAsArray() # Calculate ShortWave net Short_Wave_Net = SIDdata * (Sloping / Horizontal) + SISdata * 86400 / 1e6 # Calculate ShortWave Clear Short_Wave = Sloping Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5) # make directories PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky') if os.path.exists(PathClear) is False: os.makedirs(PathClear) PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net') if os.path.exists(PathNet) is False: os.makedirs(PathNet) # name Shortwave Clear and Net nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime( '%Y-%m-%d') + '.tif' nameNet = os.path.join(PathNet, nameFileNet) nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime( '%Y-%m-%d') + '.tif' nameClear = os.path.join(PathClear, nameFileClear) # Save net and clear short wave radiation DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj) DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj) return
def DownloadData(output_folder, latlim, lonlim, parameter, resolution): """ This function downloads DEM data from HydroSHED Keyword arguments: output_folder -- directory of the result latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) Resample -- 1 = The data will be resampled to 0.001 degree spatial resolution -- 0 = The data will have the same pixel size as the data obtained from the internet """ # Define parameter depedent variables if parameter == "dir_3s": para_name = "DIR" unit = "-" resolution = '3s' parameter = 'dir' if parameter == "dem_3s": para_name = "DEM" unit = "m" resolution = '3s' parameter = 'dem' if parameter == "dir_15s": para_name = "DIR" unit = "-" resolution = '15s' parameter = 'dir' if parameter == "dem_15s": para_name = "DEM" unit = "m" resolution = '15s' parameter = 'dem' if parameter == "dir_30s": para_name = "DIR" unit = "-" resolution = '30s' parameter = 'dir' if parameter == "dem_30s": para_name = "DEM" unit = "m" resolution = '30s' parameter = 'dem' # converts the latlim and lonlim into names of the tiles which must be # downloaded if resolution == '3s': name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim, parameter) # Memory for the map x and y shape (starts with zero) size_X_tot = 0 size_Y_tot = 0 if resolution == '15s' or resolution == '30s': name = Find_Document_names_15s_30s(latlim, lonlim, parameter, resolution) nameResults = [] # Create a temporary folder for processing output_folder_trash = os.path.join(output_folder, "Temp") if not os.path.exists(output_folder_trash): os.makedirs(output_folder_trash) # Download, extract, and converts all the files to tiff files for nameFile in name: try: # Download the data from # http://earlywarning.usgs.gov/hydrodata/ output_file, file_name = Download_Data(nameFile, output_folder_trash, parameter, para_name, resolution) # extract zip data DC.Extract_Data(output_file, output_folder_trash) # Converts the data with a adf extention to a tiff extension. # The input is the file name and in which directory the data must be stored file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif' file_name_extract = file_name.split('_')[0:3] if resolution == '3s': file_name_extract2 = file_name_extract[ 0] + '_' + file_name_extract[1] if resolution == '15s': file_name_extract2 = file_name_extract[ 0] + '_' + file_name_extract[1] + '_15s' if resolution == '30s': file_name_extract2 = file_name_extract[ 0] + '_' + file_name_extract[1] + '_30s' output_tiff = os.path.join(output_folder_trash, file_name_tiff) # convert data from adf to a tiff file if (resolution == "15s" or resolution == "3s"): input_adf = os.path.join(output_folder_trash, file_name_extract2, file_name_extract2, 'hdr.adf') output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff) # convert data from adf to a tiff file if resolution == "30s": input_bil = os.path.join(output_folder_trash, '%s.bil' % file_name_extract2) output_tiff = DC.Convert_bil_to_tiff(input_bil, output_tiff) geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff) if (resolution == "3s" and (int(size_X) != int(6000) or int(size_Y) != int(6000))): data = np.ones((6000, 6000)) * -9999 # Create the latitude bound Vfile = str(nameFile)[1:3] SignV = str(nameFile)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer) * int(Vfile) # Create the longitude bound Hfile = str(nameFile)[4:7] SignH = str(nameFile)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) Expected_X_min = Bound1 Expected_Y_max = Bound2 + 5 Xid_start = int( np.round((geo_out[0] - Expected_X_min) / geo_out[1])) Xid_end = int( np.round( ((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) / geo_out[1])) Yid_start = int( np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5]))) Yid_end = int( np.round((Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5]))) / (-geo_out[5]))) data[Yid_start:Yid_end, Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff) if np.max(data) == 255: data[data == 255] = -9999 data[data < -9999] = -9999 geo_in = [ Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333 ] # save chunk as tiff file DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") except: if resolution == '3s': # If tile not exist create a replacing zero tile (sea tiles) output = nameFile.split('.')[0] + "_trans_temporary.tif" output_tiff = os.path.join(output_folder_trash, output) file_name = nameFile data = np.ones((6000, 6000)) * -9999 data = data.astype(np.float32) # Create the latitude bound Vfile = str(file_name)[1:3] SignV = str(file_name)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer) * int(Vfile) # Create the longitude bound Hfile = str(file_name)[4:7] SignH = str(file_name)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) # Geospatial data for the tile geo_in = [ Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333 ] # save chunk as tiff file DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") if resolution == '15s': print('no 15s data is in dataset') if resolution == '3s': # clip data Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim) size_Y_out = int(np.shape(Data)[0]) size_X_out = int(np.shape(Data)[1]) # Total size of the product so far size_Y_tot = int(size_Y_tot + size_Y_out) size_X_tot = int(size_X_tot + size_X_out) if nameFile is name[0]: Geo_x_end = Geo_data[0] Geo_y_end = Geo_data[3] else: Geo_x_end = np.min([Geo_x_end, Geo_data[0]]) Geo_y_end = np.max([Geo_y_end, Geo_data[3]]) # create name for chunk FileNameEnd = "%s_temporary.tif" % (nameFile) nameForEnd = os.path.join(output_folder_trash, FileNameEnd) nameResults.append(str(nameForEnd)) # save chunk as tiff file DC.Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data, projection="WGS84") if resolution == '3s': #size_X_end = int(size_X_tot) #! #size_Y_end = int(size_Y_tot) #! size_X_end = int(size_X_tot / len(rangeLat)) + 1 #! size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 #! # Define the georeference of the end matrix geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]] latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]] lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end] # merge chunk together resulting in 1 tiff map datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end, size_X_end) datasetTot[datasetTot < -9999] = -9999 if resolution == '15s': output_file_merged = os.path.join(output_folder_trash, 'merged.tif') datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash, output_file_merged, latlim, lonlim, resolution) if resolution == '30s': output_file_merged = os.path.join(output_folder_trash, 'merged.tif') datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash, output_file_merged, latlim, lonlim, resolution) # name of the end result output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution) Save_name = os.path.join(output_folder, output_DEM_name) # Make geotiff file DC.Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84") os.chdir(output_folder) # Delete the temporary folder shutil.rmtree(output_folder_trash)
def Calc_Property(Dir, latlim, lonlim, SL): import watertools.Collect.SoilGrids as SG # Download needed layers SG.Clay_Content(Dir, latlim, lonlim, level=SL) #SG.Organic_Carbon_Content(Dir, latlim, lonlim, level=SL) SG.Bulk_Density(Dir, latlim, lonlim, level=SL) # Define path to layers filename_clay = os.path.join( Dir, 'SoilGrids', 'Clay_Content', 'ClayContentMassFraction_%s_SoilGrids_percentage.tif' % SL) #filename_om = os.path.join(Dir, 'SoilGrids', 'Soil_Organic_Carbon_Content' ,'SoilOrganicCarbonContent_%s_SoilGrids_g_kg.tif' %SL) filename_bulkdensity = os.path.join( Dir, 'SoilGrids', 'Bulk_Density', 'BulkDensity_%s_SoilGrids_kg-m-3.tif' % SL) # Define path for output if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" filedir_out_densbulk = os.path.join(Dir, 'SoilGrids', 'Bulk_Density') if not os.path.exists(filedir_out_densbulk): os.makedirs(filedir_out_densbulk) filedir_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat') if not os.path.exists(filedir_out_thetasat): os.makedirs(filedir_out_thetasat) #filename_out_densbulk = os.path.join(filedir_out_densbulk ,'Bulk_Density_%s_SoilGrids_g-cm-3.tif' %level) filename_out_thetasat = os.path.join( filedir_out_thetasat, 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level) #if not (os.path.exists(filename_out_densbulk) and os.path.exists(filename_out_thetasat)): if not os.path.exists(filename_out_thetasat): # Open datasets dest_clay = gdal.Open(filename_clay) #dest_om = gdal.Open(filename_om) dest_bulk = gdal.Open(filename_bulkdensity) # Open Array info geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_clay) # Open Arrays Clay = dest_clay.GetRasterBand(1).ReadAsArray() #OM = dest_om.GetRasterBand(1).ReadAsArray() Clay = np.float_(Clay) Clay[Clay > 100] = np.nan #OM = np.float_(OM) #OM[OM<0]=np.nan #OM = OM/1000 # Calculate bulk density #bulk_dens = 1/(0.6117 + 0.3601 * Clay/100 + 0.002172 * np.power(OM * 100, 2)+ 0.01715 * np.log(OM * 100)) bulk_dens = dest_bulk.GetRasterBand(1).ReadAsArray() bulk_dens = bulk_dens / 1000 ''' # Oude methode gebaseerd op Schenost, Sinowski & Priesack (1996) # Calculate theta saturated theta_sat = 0.85 * (1- (bulk_dens/2.65)) + 0.13 * Clay/100 ''' # Nieuwe methode gebaseerd op Toth et al (2014) # Calculate silt fraction based on clay fraction Silt_fraction = 0.7 * (Clay / 100)**2 + 0.308 * Clay / 100 # Calculate theta sat theta_sat = 0.8308 - 0.28217 * bulk_dens + 0.02728 * Clay / 100 + 0.0187 * Silt_fraction # Save data #DC.Save_as_tiff(filename_out_densbulk, bulk_dens, geo_out, "WGS84") DC.Save_as_tiff(filename_out_thetasat, theta_sat, geo_out, "WGS84") return ()
def main(Dir, Startdate = '', Enddate = '', latlim = [-60, 60], lonlim = [-180, 180], pixel_size = False, cores = False, LANDSAF = 0, SourceLANDSAF= '', Waitbar = 1): """ This function downloads TRMM3B43 V7 (monthly) data Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) cores -- The number of cores used to run the routine. It can be 'False' to avoid using parallel computing routines. Waitbar -- 1 (Default) will print the waitbar """ print('Create monthly Reference ET data for period %s till %s' %(Startdate, Enddate)) # An array of monthly dates which will be calculated Dates = pd.date_range(Startdate,Enddate,freq = 'MS') # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50) # Calculate the ETref day by day for every month for Date in Dates: # Collect date data Y=Date.year M=Date.month Mday=calendar.monthrange(Y,M)[1] Days=pd.date_range(Date,Date+pd.Timedelta(days=Mday),freq='D') StartTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+ '-01' EndTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+'-'+str(Mday) # Get ETref on daily basis daily(Dir=Dir, Startdate=StartTime,Enddate=EndTime,latlim=latlim, lonlim=lonlim, pixel_size = pixel_size, cores=cores, LANDSAF=LANDSAF, SourceLANDSAF=SourceLANDSAF, Waitbar = 0) # Load DEM if not pixel_size: nameDEM='DEM_HydroShed_m_3s.tif' DEMmap=os.path.join(Dir,'HydroSHED','DEM',nameDEM ) else: DEMmap=os.path.join(Dir,'HydroSHED','DEM','DEM_HydroShed_m_reshaped_for_ETref.tif') # Get some geo-data to save results geo_ET, proj, size_X, size_Y = RC.Open_array_info(DEMmap) dataMonth=np.zeros([size_Y,size_X]) for Day in Days[:-1]: DirDay=os.path.join(Dir,'ETref','Daily','ETref_mm-day-1_daily_' + Day.strftime('%Y.%m.%d') + '.tif') dataDay=gdal.Open(DirDay) Dval=dataDay.GetRasterBand(1).ReadAsArray().astype(np.float32) Dval[Dval<0]=0 dataMonth=dataMonth+Dval dataDay=None # make geotiff file output_folder_month=os.path.join(Dir,'ETref','Monthly') if os.path.exists(output_folder_month)==False: os.makedirs(output_folder_month) DirMonth=os.path.join(output_folder_month,'ETref_mm-month-1_monthly_'+Date.strftime('%Y.%m.%d') + '.tif') # Create the tiff file DC.Save_as_tiff(DirMonth,dataMonth, geo_ET, proj) # Create Waitbar if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None): """ This functions calculates yearly tiff files based on the monthly tiff files. (will calculate the total sum) Parameters ---------- Dir_in : str Path to the input data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Dir_out : str Path to the output data, default is same as Dir_in """ # import WA+ modules import watertools.General.data_conversions as DC import watertools.General.raster_conversions as RC # Change working directory os.chdir(Dir_in) # Define end and start date Dates = pd.date_range(Startdate, Enddate, freq='AS') # Find all monthly files files = glob.glob('*monthly*.tif') # Get array information and define projection geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0]) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Get the No Data Value dest = gdal.Open(files[0]) NDV = dest.GetRasterBand(1).GetNoDataValue() for date in Dates: Year = date.year files_one_year = glob.glob('*monthly*%d*.tif' % Year) # Create empty arrays Year_data = np.zeros([size_Y, size_X]) if len(files_one_year) is not int(12): print("One month in year %s is missing!" % Year) for file_one_year in files_one_year: file_path = os.path.join(Dir_in, file_one_year) Month_data = RC.Open_tiff_array(file_path) Month_data[np.isnan(Month_data)] = 0.0 Month_data[Month_data == -9999] = 0.0 Year_data += Month_data # Define output directory if Dir_out == None: Dir_out = Dir_in if not os.path.exists(Dir_out): os.makedirs(Dir_out) # Define output name output_name = os.path.join( Dir_out, file_one_year.replace('monthly', 'yearly').replace('month', 'year')) output_name = output_name[:-14] + '%d.01.01.tif' % (date.year) # Save tiff file DC.Save_as_tiff(output_name, Year_data, geo_out, proj) return
def slope_correct(down_short_hor, pressure, ea, DEMmap, DOY): """ This function downscales the CFSR solar radiation by using the DEM map The Slope correction is based on Allen et al. (2006) 'Analytical integrated functions for daily solar radiation on slope' Keyword arguments: down_short_hor -- numpy array with the horizontal downwards shortwave radiation pressure -- numpy array with the air pressure ea -- numpy array with the actual vapour pressure DEMmap -- 'C:/' path to the DEM map DOY -- day of the year """ # Get Geo Info GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap) minx = GeoT[0] miny = GeoT[3] + xsize*GeoT[4] + ysize*GeoT[5] x = np.flipud(np.arange(xsize)*GeoT[1] + minx + GeoT[1]/2) y = np.flipud(np.arange(ysize)*-GeoT[5] + miny + -GeoT[5]/2) # Calculate Extraterrestrial Solar Radiation [W m-2] demmap = RC.Open_tiff_array(DEMmap) demmap[demmap<0]=0 # apply the slope correction Ra_hor, Ra_slp, sinb, sinb_hor, fi, slope, ID = SlopeInfluence(demmap,y,x,DOY) # Calculate atmospheric transmissivity Rs_hor = down_short_hor # EQ 39 tau = Rs_hor/Ra_hor #EQ 41 KB_hor = np.zeros(tau.shape) * np.nan indice = np.where(tau.flat >= 0.42) KB_hor.flat[indice] = 1.56*tau.flat[indice] -0.55 indice = np.logical_and(tau.flat > 0.175, tau.flat < 0.42) KB_hor.flat[indice] = 0.022 - 0.280*tau.flat[indice] + 0.828*tau.flat[indice]**2 + 0.765*tau.flat[indice]**3 indice = np.where(tau.flat <= 0.175) KB_hor.flat[indice] = 0.016*tau.flat[indice] # EQ 42 KD_hor = tau - KB_hor Kt=0.7 #EQ 18 W = 0.14*ea*pressure + 2.1 KB0 = 0.98*np.exp((-0.00146*pressure/Kt/sinb)-0.075*(W/sinb)**0.4) KB0_hor = 0.98*np.exp((-0.00146*pressure/Kt/sinb_hor)-0.075*(W/sinb_hor)**0.4) #EQ 34 fB = KB0/KB0_hor * Ra_slp/Ra_hor fia = (1-KB_hor) * (1 + (KB_hor/(KB_hor+KD_hor))**0.5 * np.sin(slope/2)**3)*fi + fB*KB_hor Rs = Rs_hor*(fB*(KB_hor/tau) + fia*(KD_hor/tau) + 0.23*(1-fi)) Rs[np.isnan(Rs)] = Rs_hor[np.isnan(Rs)] Rs_equiv = Rs / np.cos(slope) bias = np.nansum(Rs_hor)/np.nansum(Rs_equiv) return Rs_equiv, tau, bias
def Calc_Property(Dir, latlim, lonlim, SL): import watertools.Collect.SoilGrids as SG # Download needed layers SG.Clay_Content(Dir, latlim, lonlim, level=SL) SG.Organic_Carbon_Content(Dir, latlim, lonlim, level=SL) #SG.Bulk_Density(Dir, latlim, lonlim, level=SL) # Define path to layers filename_clay = os.path.join(Dir, 'SoilGrids', 'Clay_Content' ,'ClayContentMassFraction_%s_SoilGrids_percentage.tif' %SL) filename_om = os.path.join(Dir, 'SoilGrids', 'Soil_Organic_Carbon_Content' ,'SoilOrganicCarbonContent_%s_SoilGrids_g_kg.tif' %SL) #filename_bulkdensity = os.path.join(Dir, 'SoilGrids', 'Bulk_density' ,'BulkDensity_%s_SoilGrids_kg-m-3.tif' %SL) # Define path for output if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" filedir_out_densbulk = os.path.join( Dir, 'SoilGrids', 'Bulk_density') if not os.path.exists(filedir_out_densbulk): os.makedirs(filedir_out_densbulk) filedir_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat') if not os.path.exists(filedir_out_thetasat): os.makedirs(filedir_out_thetasat) filename_out_densbulk = os.path.join(filedir_out_densbulk ,'Bulk_Density_%s_SoilGrids_g-cm-3.tif' %level) filename_out_thetasat = os.path.join(filedir_out_thetasat,'Theta_Sat_%s_SoilGrids_kg-kg.tif' %level) if not (os.path.exists(filename_out_densbulk) and os.path.exists(filename_out_thetasat)): #if not os.path.exists(filename_out_thetasat): # Open datasets dest_clay = gdal.Open(filename_clay) dest_om = gdal.Open(filename_om) #dest_bulk = gdal.Open(filename_bulkdensity) # Open Array info geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_clay) # Open Arrays Clay = dest_clay.GetRasterBand(1).ReadAsArray() OM = dest_om.GetRasterBand(1).ReadAsArray() Clay = np.float_(Clay) Clay[Clay>100]=np.nan OM = np.float_(OM) OM[OM<0]=np.nan OM = OM/1000 # Calculate bulk density bulk_dens = 1/(0.6117 + 0.3601 * Clay/100 + 0.002172 * np.power(OM * 100, 2)+ 0.01715 * np.log(OM * 100)) #bulk_dens = dest_bulk.GetRasterBand(1).ReadAsArray() # Calculate theta saturated theta_sat = 0.85 * (1- (bulk_dens/2.65)) + 0.13 * Clay/100 # Save data DC.Save_as_tiff(filename_out_densbulk, bulk_dens, geo_out, "WGS84") DC.Save_as_tiff(filename_out_thetasat, theta_sat, geo_out, "WGS84") return()
def Save_as_NC(namenc, DataCube, Var, Reference_filename, Startdate = '', Enddate = '', Time_steps = '', Scaling_factor = 1): """ This function save the array as a netcdf file Keyword arguments: namenc -- string, complete path of the output file with .nc extension DataCube -- [array], dataset of the nc file, can be a 2D or 3D array [time, lat, lon], must be same size as reference data Var -- string, the name of the variable Reference_filename -- string, complete path to the reference file name Startdate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array, defines the Start datum of the dataset Enddate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array, defines the End datum of the dataset Time_steps -- 'monthly' or 'daily', needs to be filled when you want to save a 3D array, defines the timestep of the dataset Scaling_factor -- number, scaling_factor of the dataset, default = 1 """ # Import modules import watertools.General.raster_conversions as RC from netCDF4 import Dataset if not os.path.exists(namenc): # Get raster information geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_filename) # Create the lat/lon rasters lon = np.arange(size_X)*geo_out[1]+geo_out[0] - 0.5 * geo_out[1] lat = np.arange(size_Y)*geo_out[5]+geo_out[3] - 0.5 * geo_out[5] # Create the nc file nco = Dataset(namenc, 'w', format='NETCDF4_CLASSIC') nco.description = '%s data' %Var # Create dimensions, variables and attributes: nco.createDimension('longitude', size_X) nco.createDimension('latitude', size_Y) # Create time dimension if the parameter is time dependent if Startdate is not '': if Time_steps == 'monthly': Dates = pd.date_range(Startdate,Enddate,freq = 'MS') if Time_steps == 'daily': Dates = pd.date_range(Startdate,Enddate,freq = 'D') time_or=np.zeros(len(Dates)) i = 0 for Date in Dates: time_or[i] = Date.toordinal() i += 1 nco.createDimension('time', None) timeo = nco.createVariable('time', 'f4', ('time',)) timeo.units = '%s' %Time_steps timeo.standard_name = 'time' # Create the lon variable lono = nco.createVariable('longitude', 'f8', ('longitude',)) lono.standard_name = 'longitude' lono.units = 'degrees_east' lono.pixel_size = geo_out[1] # Create the lat variable lato = nco.createVariable('latitude', 'f8', ('latitude',)) lato.standard_name = 'latitude' lato.units = 'degrees_north' lato.pixel_size = geo_out[5] # Create container variable for CRS: lon/lat WGS84 datum crso = nco.createVariable('crs', 'i4') crso.long_name = 'Lon/Lat Coords in WGS84' crso.grid_mapping_name = 'latitude_longitude' crso.projection = proj crso.longitude_of_prime_meridian = 0.0 crso.semi_major_axis = 6378137.0 crso.inverse_flattening = 298.257223563 crso.geo_reference = geo_out # Create the data variable if Startdate is not '': preco = nco.createVariable('%s' %Var, 'f8', ('time', 'latitude', 'longitude'), zlib=True, least_significant_digit=1) timeo[:]=time_or else: preco = nco.createVariable('%s' %Var, 'f8', ('latitude', 'longitude'), zlib=True, least_significant_digit=1) # Set the data variable information preco.scale_factor = Scaling_factor preco.add_offset = 0.00 preco.grid_mapping = 'crs' preco.set_auto_maskandscale(False) # Set the lat/lon variable lono[:] = lon lato[:] = lat # Set the data variable if Startdate is not '': for i in range(len(Dates)): preco[i,:,:] = DataCube[i,:,:]*1./np.float(Scaling_factor) else: preco[:,:] = DataCube[:,:] * 1./np.float(Scaling_factor) nco.close() return()
def ETref(Date, args): """ This function starts to calculate ETref (daily) data based on Hydroshed, GLDAS, and (CFSR/LANDSAF) in parallel or single core Keyword arguments: Date -- panda timestamp args -- includes all the parameters that are needed for the ETref """ # unpack the arguments [Dir, lonlim, latlim, pixel_size, LANDSAF] = args # Set the paths nameTmin = 'Tair-min_GLDAS-NOAH_C_daily_' + Date.strftime( '%Y.%m.%d') + ".tif" tmin_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily', 'tair_f_inst', 'min', nameTmin) nameTmax = 'Tair-max_GLDAS-NOAH_C_daily_' + Date.strftime( '%Y.%m.%d') + ".tif" tmax_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily', 'tair_f_inst', 'max', nameTmax) nameHumid = 'Hum_GLDAS-NOAH_kg-kg_daily_' + Date.strftime( '%Y.%m.%d') + ".tif" humid_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily', 'qair_f_inst', 'mean', nameHumid) namePress = 'P_GLDAS-NOAH_kpa_daily_' + Date.strftime('%Y.%m.%d') + ".tif" press_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily', 'psurf_f_inst', 'mean', namePress) nameWind = 'W_GLDAS-NOAH_m-s-1_daily_' + Date.strftime('%Y.%m.%d') + ".tif" wind_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily', 'wind_f_inst', 'mean', nameWind) if LANDSAF == 1: nameShortClearname = 'ShortWave_Clear_Daily_W-m2_' + Date.strftime( '%Y-%m-%d') + '.tif' input2_str = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky', nameShortClearname) nameShortNetname = 'ShortWave_Net_Daily_W-m2_' + Date.strftime( '%Y-%m-%d') + '.tif' input1_str = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net', nameShortNetname) input3_str = 'not' else: if Date < pd.Timestamp(pd.datetime(2011, 4, 1)): nameDownLong = 'DLWR_CFSR_W-m2_' + Date.strftime( '%Y.%m.%d') + ".tif" input2_str = os.path.join(Dir, 'Radiation', 'CFSR', nameDownLong) nameDownShort = 'DSWR_CFSR_W-m2_' + Date.strftime( '%Y.%m.%d') + ".tif" input1_str = os.path.join(Dir, 'Radiation', 'CFSR', nameDownShort) nameUpLong = 'ULWR_CFSR_W-m2_' + Date.strftime('%Y.%m.%d') + ".tif" input3_str = os.path.join(Dir, 'Radiation', 'CFSR', nameUpLong) else: nameDownLong = 'DLWR_CFSRv2_W-m2_' + Date.strftime( '%Y.%m.%d') + ".tif" input2_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameDownLong) nameDownShort = 'DSWR_CFSRv2_W-m2_' + Date.strftime( '%Y.%m.%d') + ".tif" input1_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameDownShort) nameUpLong = 'ULWR_CFSRv2_W-m2_' + Date.strftime( '%Y.%m.%d') + ".tif" input3_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameUpLong) # The day of year DOY = Date.dayofyear # Load DEM if not pixel_size: DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_HydroShed_m_3s.tif') else: DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_HydroShed_m_3s.tif') dest, ulx, lry, lrx, uly, epsg_to = RC.reproject_dataset_epsg( DEMmap_str, pixel_spacing=pixel_size, epsg_to=4326, method=2) DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_HydroShed_m_reshaped_for_ETref.tif') DEM_data = dest.GetRasterBand(1).ReadAsArray() geo_dem = [ulx, pixel_size, 0.0, uly, 0.0, -pixel_size] DC.Save_as_tiff(name=DEMmap_str, data=DEM_data, geo=geo_dem, projection='4326') # Calc ETref ETref = calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str, input1_str, input2_str, input3_str, DEMmap_str, DOY) # Make directory for the MODIS ET data output_folder = os.path.join(Dir, 'ETref', 'Daily') if not os.path.exists(output_folder): os.makedirs(output_folder) # Create the output names NameETref = 'ETref_mm-day-1_daily_' + Date.strftime('%Y.%m.%d') + '.tif' NameEnd = os.path.join(output_folder, NameETref) # Collect geotiff information geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str) # Create daily ETref tiff files DC.Save_as_tiff(name=NameEnd, data=ETref, geo=geo_out, projection=proj)
def Merge_DEM_15s_30s(output_folder_trash, output_file_merged, latlim, lonlim, resolution): os.chdir(output_folder_trash) tiff_files = glob.glob('*.tif') resolution_geo = [] lonmin = lonlim[0] lonmax = lonlim[1] latmin = latlim[0] latmax = latlim[1] if resolution == "15s": resolution_geo = 0.00416667 if resolution == "30s": resolution_geo = 0.00416667 * 2 size_x_tot = int(np.round((lonmax - lonmin) / resolution_geo)) size_y_tot = int(np.round((latmax - latmin) / resolution_geo)) data_tot = np.ones([size_y_tot, size_x_tot]) * -9999. for tiff_file in tiff_files: inFile = os.path.join(output_folder_trash, tiff_file) geo, proj, size_X, size_Y = RC.Open_array_info(inFile) resolution_geo = geo[1] lonmin_one = geo[0] lonmax_one = geo[0] + size_X * geo[1] latmin_one = geo[3] + size_Y * geo[5] latmax_one = geo[3] if lonmin_one < lonmin: lonmin_clip = lonmin else: lonmin_clip = lonmin_one if lonmax_one > lonmax: lonmax_clip = lonmax else: lonmax_clip = lonmax_one if latmin_one < latmin: latmin_clip = latmin else: latmin_clip = latmin_one if latmax_one > latmax: latmax_clip = latmax else: latmax_clip = latmax_one size_x_clip = int( np.round((lonmax_clip - lonmin_clip) / resolution_geo)) size_y_clip = int( np.round((latmax_clip - latmin_clip) / resolution_geo)) inFile = os.path.join(output_folder_trash, tiff_file) geo, proj, size_X, size_Y = RC.Open_array_info(inFile) Data = RC.Open_tiff_array(inFile) lonmin_tiff = geo[0] latmax_tiff = geo[3] lon_tiff_position = int( np.round((lonmin_clip - lonmin_tiff) / resolution_geo)) lat_tiff_position = int( np.round((latmax_tiff - latmax_clip) / resolution_geo)) lon_data_tot_position = int( np.round((lonmin_clip - lonmin) / resolution_geo)) lat_data_tot_position = int( np.round((latmax - latmax_clip) / resolution_geo)) Data[Data < -9999.] = -9999. data_tot[lat_data_tot_position:lat_data_tot_position + size_y_clip, lon_data_tot_position:lon_data_tot_position + size_x_clip][data_tot[ lat_data_tot_position:lat_data_tot_position + size_y_clip, lon_data_tot_position:lon_data_tot_position + size_x_clip] == -9999] = Data[ lat_tiff_position:lat_tiff_position + size_y_clip, lon_tiff_position:lon_tiff_position + size_x_clip][data_tot[ lat_data_tot_position:lat_data_tot_position + size_y_clip, lon_data_tot_position:lon_data_tot_position + size_x_clip] == -9999] geo_out = [lonmin, resolution_geo, 0.0, latmax, 0.0, -1 * resolution_geo] geo_out = tuple(geo_out) data_tot[data_tot < -9999.] = -9999. return (data_tot, geo_out)