def lu_type_average(data_fh, lu_fh, lu_dict): LULC = RC.Open_tiff_array(lu_fh) in_data = RC.Open_tiff_array(data_fh) out_data = {} for lu_class in list(lu_dict.keys()): mask = [LULC == value for value in lu_dict[lu_class]] mask = (np.sum(mask, axis=0)).astype(bool) out_data[lu_class] = np.nanmean(in_data[mask]) return out_data
def fuel_wood(output_folder, lu_fh, AREA, ndm_fhs, fraction_fhs, ndmdates): """ Calculate natural livestock feed production INPUTS ---------- lu_fh : str filehandle for land use map ndm_fhs: nd array array of filehandles of NDM maps abv_grnd_biomass_ratio: dict dictionnary 'LULC':[above ground biomass] """ Data_Path_Fuel = "Fuel" out_folder = os.path.join(output_folder, Data_Path_Fuel) if not os.path.exists(out_folder): os.mkdir(out_folder) area_ha = AREA * 100 LULC = RC.Open_tiff_array(lu_fh) geo_out, proj, size_X, size_Y = RC.Open_array_info(lu_fh) fuel_classes = [1, 8, 9, 10, 11, 12, 13] fuel_mask = np.zeros(LULC.shape) for fc in fuel_classes: fuel_mask[np.where(LULC == fc)] = 1 fuel_fhs_landscape = [] fuel_fhs_incremental = [] for d in range(len(ndm_fhs)): ndm_fh = ndm_fhs[d] fraction_fh = fraction_fhs[d] yield_fract = RC.Open_tiff_array(fraction_fh) date1 = ndmdates[d] year = '%d' % date1.year month = '%02d' % date1.month # year = ndm_fh[-14:-10] # month = ndm_fh[-9:-7] out_fh_l = out_folder + '\\fuel_prod_landscape_%s_%s.tif' % (year, month) out_fh_i = out_folder + '\\fuel_prod_incremental_%s_%s.tif' % (year, month) NDM = becgis.open_as_array(ndm_fh, nan_values=True) NDM_fuel_incremental = NDM * .05 * fuel_mask * yield_fract * area_ha / 1e6 NDM_fuel_landscape = NDM * .05 * fuel_mask * ( 1 - yield_fract) * area_ha / 1e6 DC.Save_as_tiff(out_fh_i, NDM_fuel_incremental, geo_out) DC.Save_as_tiff(out_fh_l, NDM_fuel_landscape, geo_out) fuel_fhs_landscape.append(out_fh_l) fuel_fhs_incremental.append(out_fh_i) return fuel_fhs_landscape, fuel_fhs_incremental
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim): import watools.General.raster_conversions as RC # Open Dataset HiHydroSoil_Array = RC.Open_tiff_array(local_filename) # Define area XID = [ int(np.floor((180 + lonlim[0]) / 0.00833333)), int(np.ceil((180 + lonlim[1]) / 0.00833333)) ] YID = [ int(np.ceil((90 - latlim[1]) / 0.00833333)), int(np.floor((90 - latlim[0]) / 0.00833333)) ] # Define Georeference geo = tuple([ -180 + 0.00833333 * XID[0], 0.00833333, 0, 90 - 0.00833333 * YID[0], 0, -0.00833333 ]) # Clip Array HiHydroSoil_Array_clipped = HiHydroSoil_Array[YID[0]:YID[1], XID[0]:XID[1]] # Save tiff file DC.Save_as_tiff(Filename_out, HiHydroSoil_Array_clipped, geo, "WGS84")
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim, latlim, yID, xID, TimeStep): """ This function retrieves ALEXI data for a given date from the ftp.wateraccounting.unesco-ihe.org server. Restrictions: The data and this python file may not be distributed to others without permission of the WA+ team due data restriction of the ALEXI developers. Keyword arguments: local_filename -- name of the temporary file which contains global ALEXI data DirFile -- name of the end file with the weekly ALEXI data filename -- name of the end file lonlim -- [ymin, ymax] (values must be between -60 and 70) latlim -- [xmin, xmax] (values must be between -180 and 180) """ # Collect account and FTP information username, password = WebAccounts.Accounts(Type='FTP_WA') ftpserver = "ftp.wateraccounting.unesco-ihe.org" # Download data from FTP ftp = FTP(ftpserver) ftp.login(username, password) if TimeStep is "weekly": directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/" if TimeStep is "daily": directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/" ftp.cwd(directory) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write) lf.close() if TimeStep is "weekly": # Open global ALEXI data dataset = RC.Open_tiff_array(local_filename) # Clip extend out of world data data = dataset[yID[0]:yID[1], xID[0]:xID[1]] data[data < 0] = -9999 if TimeStep is "daily": DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0]) raw_data = np.fromfile(os.path.splitext(local_filename)[0], dtype="<f4") dataset = np.flipud(np.resize(raw_data, [3000, 7200])) data = dataset[ yID[0]:yID[1], xID[0]:xID[1]] / 2.45 # Values are in MJ/m2d so convert to mm/d data[data < 0] = -9999 # make geotiff file geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05] DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84") return
def Calc_Property(Dir, latlim, lonlim, SL): import watools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat' ,'Theta_Sat2_%s_SoilGrids_kg-kg.tif' %level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watools.Products.SoilGrids.Theta_Sat2.Topsoil(Dir, latlim, lonlim) elif SL == "sl6": watools.Products.SoilGrids.Theta_Sat2.Subsoil(Dir, latlim, lonlim) filedir_out_thetafc = os.path.join(Dir, 'SoilGrids', 'Theta_FC') if not os.path.exists(filedir_out_thetafc): os.makedirs(filedir_out_thetafc) # Define theta field capacity output filename_out_thetafc = os.path.join(filedir_out_thetafc, 'Theta_FC2_%s_SoilGrids_cm3-cm3.tif' %level) if not os.path.exists(filename_out_thetafc): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate theta field capacity theta_FC = np.ones(theta_sat.shape) * -9999 theta_FC = np.where(theta_sat < 0.301, 0.042, np.arccosh(theta_sat + 0.7) - 0.32 * (theta_sat + 0.7) + 0.2) # Save as tiff DC.Save_as_tiff(filename_out_thetafc, theta_FC, geo_out, proj) return
def split_yield(output_folder, p_fhs, et_blue_fhs, et_green_fhs, ab=(1.0, 1.0)): Data_Path_split = "split_y" out_folder = os.path.join(output_folder, Data_Path_split) if not os.path.exists(out_folder): os.mkdir(out_folder) sp_yield_fhs = [] geo_out, proj, size_X, size_Y = RC.Open_array_info(p_fhs[0]) for m in range(len(p_fhs)): out_fh = out_folder + '\\split_yield' + et_blue_fhs[m][-12:] P = RC.Open_tiff_array(p_fhs[m]) ETBLUE = RC.Open_tiff_array(et_blue_fhs[m]) ETGREEN = RC.Open_tiff_array(et_green_fhs[m]) etbfraction = ETBLUE / (ETBLUE + ETGREEN) pfraction = P / np.nanmax(P) fraction = sh3.split_Yield(pfraction, etbfraction, ab[0], ab[1]) DC.Save_as_tiff(out_fh, fraction, geo_out) sp_yield_fhs.append(out_fh) return sp_yield_fhs
def lu_type_sum(data_fh, lu_fh, AREA, lu_dict, convert=None): LULC = RC.Open_tiff_array(lu_fh) in_data = becgis.open_as_array(data_fh, nan_values=True) # in_data = RC.Open_tiff_array(data_fh) if convert == 'mm_to_km3': in_data *= AREA / 1e6 out_data = {} for lu_class in list(lu_dict.keys()): mask = [LULC == value for value in lu_dict[lu_class]] mask = (np.sum(mask, axis=0)).astype(bool) out_data[lu_class] = np.nansum(in_data[mask]) return out_data
def lapse_rate(Dir, temperature_map, DEMmap): """ This function downscales the GLDAS temperature map by using the DEM map Keyword arguments: temperature_map -- 'C:/' path to the temperature map DEMmap -- 'C:/' path to the DEM map """ # calculate average altitudes corresponding to T resolution dest = RC.reproject_dataset_example(DEMmap, temperature_map, method=4) DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map) DEM_ave_data = dest.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) dest = None # determine lapse-rate [degress Celcius per meter] lapse_rate_number = 0.0065 # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method=2) dem_avg = dest.GetRasterBand(1).ReadAsArray() dem_avg[dem_avg < 0] = 0 dest = None # Open the temperature dataset dest = RC.reproject_dataset_example(temperature_map, DEMmap, method=2) T = dest.GetRasterBand(1).ReadAsArray() dest = None # Open Demmap demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap <= 0] = 0 demmap[demmap == -32768] = np.nan # calculate first part T = T + ((dem_avg - demmap) * lapse_rate_number) return T
def adjust_P(Dir, pressure_map, DEMmap): """ This function downscales the GLDAS air pressure map by using the DEM map Keyword arguments: pressure_map -- 'C:/' path to the pressure map DEMmap -- 'C:/' path to the DEM map """ # calculate average latitudes destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method=4) DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map) DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method=2) dem_avg = dest.GetRasterBand(1).ReadAsArray() dest = None # open maps as numpy arrays dest = RC.reproject_dataset_example(pressure_map, DEMmap, method=2) P = dest.GetRasterBand(1).ReadAsArray() dest = None demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap <= 0] = 0 demmap[demmap == -32768] = np.nan # calculate second part P = P + (101.3 * ((293 - 0.0065 * (demmap - dem_avg)) / 293)**5.26 - 101.3) os.remove(DEM_ave_out_name) return P
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None): """ This functions calculates monthly tiff files based on the daily tiff files. (will calculate the total sum) Parameters ---------- Dir_in : str Path to the input data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Dir_out : str Path to the output data, default is same as Dir_in """ # import WA+ modules import watools.General.data_conversions as DC import watools.General.raster_conversions as RC # Change working directory os.chdir(Dir_in) # Define end and start date Dates = pd.date_range(Startdate, Enddate, freq='MS') # Find all monthly files files = glob.glob('*daily*.tif') # Get array information and define projection geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0]) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Get the No Data Value dest = gdal.Open(files[0]) NDV = dest.GetRasterBand(1).GetNoDataValue() # Define output directory if Dir_out is None: Dir_out = Dir_in if not os.path.exists(Dir_out): os.makedirs(Dir_out) # loop over the months and sum the days for date in Dates: Year = date.year Month = date.month files_one_year = glob.glob('*daily*%d.%02d*.tif' % (Year, Month)) # Create empty arrays Month_data = np.zeros([size_Y, size_X]) # Get amount of days in month Amount_days_in_month = int(calendar.monthrange(Year, Month)[1]) if len(files_one_year) is not Amount_days_in_month: print("One day is missing!!! month %s year %s" %(Month, Year)) for file_one_year in files_one_year: file_path = os.path.join(Dir_in, file_one_year) Day_data = RC.Open_tiff_array(file_path) Day_data[np.isnan(Day_data)] = 0.0 Day_data[Day_data == -9999] = 0.0 Month_data += Day_data # Define output name output_name = os.path.join(Dir_out, file_one_year .replace('daily', 'monthly') .replace('day', 'month')) output_name = output_name[:-14] + '%d.%02d.01.tif' % (date.year, date.month) # Save tiff file DC.Save_as_tiff(output_name, Month_data, geo_out, proj) return
def Merge_DEM_15s(output_folder_trash, output_file_merged, latlim, lonlim): os.chdir(output_folder_trash) tiff_files = glob.glob('*.tif') resolution_geo = [] lonmin = lonlim[0] lonmax = lonlim[1] latmin = latlim[0] latmax = latlim[1] resolution_geo = 0.00416667 size_x_tot = int(np.round((lonmax - lonmin) / resolution_geo)) size_y_tot = int(np.round((latmax - latmin) / resolution_geo)) data_tot = np.ones([size_y_tot, size_x_tot]) * -9999. for tiff_file in tiff_files: inFile = os.path.join(output_folder_trash, tiff_file) geo, proj, size_X, size_Y = RC.Open_array_info(inFile) resolution_geo = geo[1] lonmin_one = geo[0] lonmax_one = geo[0] + size_X * geo[1] latmin_one = geo[3] + size_Y * geo[5] latmax_one = geo[3] if lonmin_one < lonmin: lonmin_clip = lonmin else: lonmin_clip = lonmin_one if lonmax_one > lonmax: lonmax_clip = lonmax else: lonmax_clip = lonmax_one if latmin_one < latmin: latmin_clip = latmin else: latmin_clip = latmin_one if latmax_one > latmax: latmax_clip = latmax else: latmax_clip = latmax_one size_x_clip = int( np.round((lonmax_clip - lonmin_clip) / resolution_geo)) size_y_clip = int( np.round((latmax_clip - latmin_clip) / resolution_geo)) inFile = os.path.join(output_folder_trash, tiff_file) geo, proj, size_X, size_Y = RC.Open_array_info(inFile) Data = RC.Open_tiff_array(inFile) lonmin_tiff = geo[0] latmax_tiff = geo[3] lon_tiff_position = int( np.round((lonmin_clip - lonmin_tiff) / resolution_geo)) lat_tiff_position = int( np.round((latmax_tiff - latmax_clip) / resolution_geo)) lon_data_tot_position = int( np.round((lonmin_clip - lonmin) / resolution_geo)) lat_data_tot_position = int( np.round((latmax - latmax_clip) / resolution_geo)) Data[Data < -9999.] = -9999. data_tot[lat_data_tot_position:lat_data_tot_position + size_y_clip, lon_data_tot_position:lon_data_tot_position + size_x_clip][data_tot[ lat_data_tot_position:lat_data_tot_position + size_y_clip, lon_data_tot_position:lon_data_tot_position + size_x_clip] == -9999] = Data[ lat_tiff_position:lat_tiff_position + size_y_clip, lon_tiff_position:lon_tiff_position + size_x_clip][data_tot[ lat_data_tot_position:lat_data_tot_position + size_y_clip, lon_data_tot_position:lon_data_tot_position + size_x_clip] == -9999] geo_out = [lonmin, resolution_geo, 0.0, latmax, 0.0, -1 * resolution_geo] geo_out = tuple(geo_out) data_tot[data_tot < -9999.] = -9999. return (data_tot, geo_out)
def DownloadData(output_folder, latlim, lonlim, parameter, resolution): """ This function downloads DEM data from HydroSHED Keyword arguments: output_folder -- directory of the result latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) Resample -- 1 = The data will be resampled to 0.001 degree spatial resolution -- 0 = The data will have the same pixel size as the data obtained from the internet """ # Define parameter depedent variables if parameter == "dir_3s": para_name = "DIR" unit = "-" resolution = '3s' parameter = 'dir' if parameter == "dem_3s": para_name = "DEM" unit = "m" resolution = '3s' parameter = 'dem' if parameter == "dir_15s": para_name = "DIR" unit = "-" resolution = '15s' parameter = 'dir' if parameter == "dem_15s": para_name = "DEM" unit = "m" resolution = '15s' parameter = 'dem' # converts the latlim and lonlim into names of the tiles which must be # downloaded if resolution == '3s': name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim, parameter) # Memory for the map x and y shape (starts with zero) size_X_tot = 0 size_Y_tot = 0 if resolution == '15s': name = Find_Document_names_15s(latlim, lonlim, parameter, resolution) nameResults = [] # Create a temporary folder for processing output_folder_trash = os.path.join(output_folder, "Temp") if not os.path.exists(output_folder_trash): os.makedirs(output_folder_trash) # Download, extract, and converts all the files to tiff files for nameFile in name: try: # Download the data from # http://earlywarning.usgs.gov/hydrodata/ output_file, file_name = Download_Data(nameFile, output_folder_trash, parameter, para_name, resolution) # extract zip data DC.Extract_Data(output_file, output_folder_trash) # Converts the data with a adf extention to a tiff extension. # The input is the file name and in which directory the data must be stored file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif' file_name_extract = file_name.split('_')[0:3] if resolution == '3s': file_name_extract2 = file_name_extract[ 0] + '_' + file_name_extract[1] if resolution == '15s': file_name_extract2 = file_name_extract[ 0] + '_' + file_name_extract[1] + '_15s' input_adf = os.path.join(output_folder_trash, file_name_extract2, file_name_extract2, 'hdr.adf') output_tiff = os.path.join(output_folder_trash, file_name_tiff) # convert data from adf to a tiff file output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff) geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff) if int(size_X) != int(6000) or int(size_Y) != int(6000): data = np.ones((6000, 6000)) * -9999 # Create the latitude bound Vfile = str(nameFile)[1:3] SignV = str(nameFile)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer) * int(Vfile) # Create the longitude bound Hfile = str(nameFile)[4:7] SignH = str(nameFile)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) Expected_X_min = Bound1 Expected_Y_max = Bound2 + 5 Xid_start = int( np.round((geo_out[0] - Expected_X_min) / geo_out[1])) Xid_end = int( np.round( ((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) / geo_out[1])) Yid_start = int( np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5]))) Yid_end = int( np.round((Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5]))) / (-geo_out[5]))) data[Yid_start:Yid_end, Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff) if np.max(data) == 255: data[data == 255] = -9999 data[data < -9999] = -9999 geo_in = [ Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333 ] # save chunk as tiff file DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") except: if resolution == '3s': # If tile not exist create a replacing zero tile (sea tiles) output = nameFile.split('.')[0] + "_trans_temporary.tif" output_tiff = os.path.join(output_folder_trash, output) file_name = nameFile data = np.ones((6000, 6000)) * -9999 data = data.astype(np.float32) # Create the latitude bound Vfile = str(file_name)[1:3] SignV = str(file_name)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer) * int(Vfile) # Create the longitude bound Hfile = str(file_name)[4:7] SignH = str(file_name)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) # Geospatial data for the tile geo_in = [ Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333 ] # save chunk as tiff file DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") if resolution == '15s': print('no 15s data is in dataset') if resolution == '3s': # clip data Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim) size_Y_out = int(np.shape(Data)[0]) size_X_out = int(np.shape(Data)[1]) # Total size of the product so far size_Y_tot = int(size_Y_tot + size_Y_out) size_X_tot = int(size_X_tot + size_X_out) if nameFile is name[0]: Geo_x_end = Geo_data[0] Geo_y_end = Geo_data[3] else: Geo_x_end = np.min([Geo_x_end, Geo_data[0]]) Geo_y_end = np.max([Geo_y_end, Geo_data[3]]) # create name for chunk FileNameEnd = "%s_temporary.tif" % (nameFile) nameForEnd = os.path.join(output_folder_trash, FileNameEnd) nameResults.append(str(nameForEnd)) # save chunk as tiff file DC.Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data, projection="WGS84") if resolution == '3s': #size_X_end = int(size_X_tot) #! #size_Y_end = int(size_Y_tot) #! size_X_end = int(size_X_tot / len(rangeLat)) + 1 #! size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 #! # Define the georeference of the end matrix geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]] latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]] lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end] # merge chunk together resulting in 1 tiff map datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end, size_X_end) datasetTot[datasetTot < -9999] = -9999 if resolution == '15s': output_file_merged = os.path.join(output_folder_trash, 'merged.tif') datasetTot, geo_out = Merge_DEM_15s(output_folder_trash, output_file_merged, latlim, lonlim) # name of the end result output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution) Save_name = os.path.join(output_folder, output_DEM_name) # Make geotiff file DC.Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84") os.chdir(output_folder) # Delete the temporary folder shutil.rmtree(output_folder_trash)
def main(files_DEM_dir, files_DEM, files_Basin, files_Runoff, files_Extraction, startdate, enddate, input_nc, resolution, Format_DEM_dir, Format_DEM, Format_Basin, Format_Runoff, Format_Extraction): # Define a year to get the epsg and geo Startdate_timestamp = pd.Timestamp(startdate) year = Startdate_timestamp.year ############################## Drainage Direction ##################################### # Open Array DEM dir as netCDF if Format_DEM_dir == "NetCDF": file_DEM_dir = os.path.join(files_DEM_dir, "%d.nc" %year) DataCube_DEM_dir = RC.Open_nc_array(file_DEM_dir, "Drainage_Direction") geo_out_example, epsg_example, size_X_example, size_Y_example, size_Z_example, Time_example = RC.Open_nc_info(files_DEM_dir) # Create memory file for reprojection gland = DC.Save_as_MEM(DataCube_DEM_dir, geo_out_example, epsg_example) dataset_example = file_name_DEM_dir = gland # Open Array DEM dir as TIFF if Format_DEM_dir == "TIFF": file_name_DEM_dir = os.path.join(files_DEM_dir,"DIR_HydroShed_-_%s.tif" %resolution) DataCube_DEM_dir = RC.Open_tiff_array(file_name_DEM_dir) geo_out_example, epsg_example, size_X_example, size_Y_example = RC.Open_array_info(file_name_DEM_dir) dataset_example = file_name_DEM_dir # Calculate Area per pixel in m2 import watools.Functions.Start.Area_converter as AC DataCube_Area = AC.Degrees_to_m2(file_name_DEM_dir) ################################## DEM ########################################## # Open Array DEM as netCDF if Format_DEM == "NetCDF": file_DEM = os.path.join(files_DEM, "%d.nc" %year) DataCube_DEM = RC.Open_nc_array(file_DEM, "Elevation") # Open Array DEM as TIFF if Format_DEM == "TIFF": file_name_DEM = os.path.join(files_DEM,"DEM_HydroShed_m_%s.tif" %resolution) destDEM = RC.reproject_dataset_example(file_name_DEM, dataset_example, method=1) DataCube_DEM = destDEM.GetRasterBand(1).ReadAsArray() ################################ Landuse ########################################## # Open Array Basin as netCDF if Format_Basin == "NetCDF": file_Basin = os.path.join(files_Basin, "%d.nc" %year) DataCube_Basin = RC.Open_nc_array(file_Basin, "Landuse") geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(file_Basin, "Landuse") dest_basin = DC.Save_as_MEM(DataCube_Basin, geo_out, str(epsg)) destLU = RC.reproject_dataset_example(dest_basin, dataset_example, method=1) DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray() DataCube_Basin = np.zeros([size_Y_example, size_X_example]) DataCube_Basin[DataCube_LU_CR > 0] = 1 # Open Array Basin as TIFF if Format_Basin == "TIFF": file_name_Basin = files_Basin destLU = RC.reproject_dataset_example(file_name_Basin, dataset_example, method=1) DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray() DataCube_Basin = np.zeros([size_Y_example, size_X_example]) DataCube_Basin[DataCube_LU_CR > 0] = 1 ################################ Surface Runoff ########################################## # Open Array runoff as netCDF if Format_Runoff == "NetCDF": DataCube_Runoff = RC.Open_ncs_array(files_Runoff, "Surface_Runoff", startdate, enddate) size_Z_example = DataCube_Runoff.shape[0] file_Runoff = os.path.join(files_Runoff, "%d.nc" %year) geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(file_Runoff, "Surface_Runoff") DataCube_Runoff_CR = np.ones([size_Z_example, size_Y_example, size_X_example]) * np.nan for i in range(0, size_Z): DataCube_Runoff_one = DataCube_Runoff[i,:,:] dest_Runoff_one = DC.Save_as_MEM(DataCube_Runoff_one, geo_out, str(epsg)) dest_Runoff = RC.reproject_dataset_example(dest_Runoff_one, dataset_example, method=4) DataCube_Runoff_CR[i,:,:] = dest_Runoff.GetRasterBand(1).ReadAsArray() DataCube_Runoff_CR[:, DataCube_LU_CR == 0] = -9999 DataCube_Runoff_CR[DataCube_Runoff_CR < 0] = -9999 # Open Array runoff as TIFF if Format_Runoff == "TIFF": DataCube_Runoff_CR = RC.Get3Darray_time_series_monthly(files_Runoff, startdate, enddate, Example_data = dataset_example) ################################ Surface Withdrawal ########################################## # Open Array Extraction as netCDF if Format_Extraction == "NetCDF": DataCube_Extraction = RC.Open_ncs_array(files_Extraction, "Surface_Withdrawal", startdate, enddate) size_Z_example = DataCube_Extraction.shape[0] file_Extraction = os.path.join(files_Extraction, "%d.nc" %year) geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(file_Extraction, "Surface_Withdrawal") DataCube_Extraction_CR = np.ones([size_Z_example, size_Y_example, size_X_example]) * np.nan for i in range(0, size_Z): DataCube_Extraction_one = DataCube_Extraction[i,:,:] dest_Extraction_one = DC.Save_as_MEM(DataCube_Extraction_one, geo_out, str(epsg)) dest_Extraction = RC.reproject_dataset_example(dest_Extraction_one, dataset_example, method=4) DataCube_Extraction_CR[i,:,:] = dest_Extraction.GetRasterBand(1).ReadAsArray() DataCube_Extraction_CR[:, DataCube_LU_CR == 0] = -9999 DataCube_Extraction_CR[DataCube_Extraction_CR < 0] = -9999 # Open Array Extraction as TIFF if Format_Extraction == "TIFF": DataCube_Extraction_CR = RC.Get3Darray_time_series_monthly(files_Extraction, startdate, enddate, Example_data = dataset_example) ################################ Create input netcdf ########################################## # Save data in one NetCDF file geo_out_example = np.array(geo_out_example) # Latitude and longitude lon_ls = np.arange(size_X_example)*geo_out_example[1]+geo_out_example[0] + 0.5 * geo_out_example[1] lat_ls = np.arange(size_Y_example)*geo_out_example[5]+geo_out_example[3] - 0.5 * geo_out_example[5] lat_n = len(lat_ls) lon_n = len(lon_ls) # Create NetCDF file nc_file = netCDF4.Dataset(input_nc, 'w') nc_file.set_fill_on() # Create dimensions lat_dim = nc_file.createDimension('latitude', lat_n) lon_dim = nc_file.createDimension('longitude', lon_n) # Create NetCDF variables crso = nc_file.createVariable('crs', 'i4') crso.long_name = 'Lon/Lat Coords in WGS84' crso.standard_name = 'crs' crso.grid_mapping_name = 'latitude_longitude' crso.projection = epsg_example crso.longitude_of_prime_meridian = 0.0 crso.semi_major_axis = 6378137.0 crso.inverse_flattening = 298.257223563 crso.geo_reference = geo_out_example lat_var = nc_file.createVariable('latitude', 'f8', ('latitude',)) lat_var.units = 'degrees_north' lat_var.standard_name = 'latitude' lat_var.pixel_size = geo_out_example[5] lon_var = nc_file.createVariable('longitude', 'f8', ('longitude',)) lon_var.units = 'degrees_east' lon_var.standard_name = 'longitude' lon_var.pixel_size = geo_out_example[1] Dates = pd.date_range(startdate,enddate,freq = 'MS') time_or=np.zeros(len(Dates)) i = 0 for Date in Dates: time_or[i] = Date.toordinal() i += 1 nc_file.createDimension('time', None) timeo = nc_file.createVariable('time', 'f4', ('time',)) timeo.units = 'Monthly' timeo.standard_name = 'time' # Variables demdir_var = nc_file.createVariable('demdir', 'i', ('latitude', 'longitude'), fill_value=-9999) demdir_var.long_name = 'Flow Direction Map' demdir_var.grid_mapping = 'crs' dem_var = nc_file.createVariable('dem', 'f8', ('latitude', 'longitude'), fill_value=-9999) dem_var.long_name = 'Altitude' dem_var.units = 'meters' dem_var.grid_mapping = 'crs' basin_var = nc_file.createVariable('basin', 'i', ('latitude', 'longitude'), fill_value=-9999) basin_var.long_name = 'Altitude' basin_var.units = 'meters' basin_var.grid_mapping = 'crs' area_var = nc_file.createVariable('area', 'f8', ('latitude', 'longitude'), fill_value=-9999) area_var.long_name = 'area in squared meters' area_var.units = 'squared_meters' area_var.grid_mapping = 'crs' runoff_var = nc_file.createVariable('Runoff_M', 'f8', ('time', 'latitude', 'longitude'), fill_value=-9999) runoff_var.long_name = 'Runoff' runoff_var.units = 'm3/month' runoff_var.grid_mapping = 'crs' extraction_var = nc_file.createVariable('Extraction_M', 'f8', ('time', 'latitude', 'longitude'), fill_value=-9999) extraction_var.long_name = 'Surface water Extraction' extraction_var.units = 'm3/month' extraction_var.grid_mapping = 'crs' # Load data lat_var[:] = lat_ls lon_var[:] = lon_ls timeo[:] = time_or # Static variables demdir_var[:, :] = DataCube_DEM_dir[:, :] dem_var[:, :] = DataCube_DEM[:, :] basin_var[:, :] = DataCube_Basin[:, :] area_var[:, :] = DataCube_Area[:, :] for i in range(len(Dates)): runoff_var[i,:,:] = DataCube_Runoff_CR[i,:,:] for i in range(len(Dates)): extraction_var[i,:,:] = DataCube_Extraction_CR[i,:,:] # Close file nc_file.close() return()
def main(Dir, Startdate='', Enddate='', latlim=[-50, 50], lonlim=[-180, 180], cores=False, Waitbar=1): """ This function downloads RFE V2.0 (monthly) data Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) cores -- The number of cores used to run the routine. It can be 'False' to avoid using parallel computing routines. Waitbar -- 1 (Default) will print a waitbar """ # Download data print('\nDownload monthly RFE precipitation data for period %s till %s' % (Startdate, Enddate)) # Check variables if not Startdate: Startdate = pd.Timestamp('2001-01-01') if not Enddate: Enddate = pd.Timestamp('Now') Dates = pd.date_range(Startdate, Enddate, freq='MS') # Make directory output_folder = os.path.join(Dir, 'Precipitation', 'RFE', 'Monthly/') if not os.path.exists(output_folder): os.makedirs(output_folder) # Create Waitbar if Waitbar == 1: import watools.Functions.Start.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) for Date in Dates: month = Date.month year = Date.year end_day = calendar.monthrange(year, month)[1] Startdate_one_month = '%s-%02s-01' % (year, month) Enddate_one_month = '%s-%02s-%02s' % (year, month, end_day) DownloadData(Dir, Startdate_one_month, Enddate_one_month, latlim, lonlim, 0, cores) Dates_daily = pd.date_range(Startdate_one_month, Enddate_one_month, freq='D') # Make directory input_folder_daily = os.path.join(Dir, 'Precipitation', 'RFE', 'Daily/') i = 0 for Date_daily in Dates_daily: file_name = 'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' % ( Date_daily.strftime('%Y'), Date_daily.strftime('%m'), Date_daily.strftime('%d')) file_name_daily_path = os.path.join(input_folder_daily, file_name) if os.path.exists(file_name_daily_path): if Date_daily == Dates_daily[i]: Raster_monthly = RC.Open_tiff_array(file_name_daily_path) else: Raster_monthly += RC.Open_tiff_array(file_name_daily_path) else: if Date_daily == Dates_daily[i]: i += 1 geo_out, proj, size_X, size_Y = RC.Open_array_info( file_name_daily_path) file_name = 'P_RFE.v2.0_mm-month-1_monthly_%s.%02s.01.tif' % ( Date.strftime('%Y'), Date.strftime('%m')) file_name_output = os.path.join(output_folder, file_name) DC.Save_as_tiff(file_name_output, Raster_monthly, geo_out, projection="WGS84") if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50)
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim): """ This function collects and clip LANDSAF data Keyword arguments: SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID) Dir -- 'C:/' path to the WA map Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -60 and 60) lonlim -- [xmin, xmax] (values must be between -180 and 180) """ # Make an array of the days of which the ET is taken Dates = pd.date_range(Startdate, Enddate, freq='D') # make directories SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS') if os.path.exists(SISdir) is False: os.makedirs(SISdir) SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID') if os.path.exists(SIDdir) is False: os.makedirs(SIDdir) ShortwaveBasin(SourceLANDSAF, Dir, latlim, lonlim, Dates=[Startdate, Enddate]) DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_HydroShed_m_3s.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str) # Open DEM map demmap = RC.Open_tiff_array(DEMmap_str) demmap[demmap < 0] = 0 # make lat and lon arrays) dlat = geo_out[5] dlon = geo_out[1] lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon for date in Dates: # day of year day = date.dayofyear Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence( demmap, lat, lon, day) SIDname = os.path.join( SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif') SISname = os.path.join( SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif') #PREPARE SID MAPS SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3) SIDdata = SIDdest.GetRasterBand(1).ReadAsArray() #PREPARE SIS MAPS SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3) SISdata = SISdest.GetRasterBand(1).ReadAsArray() # Calculate ShortWave net Short_Wave_Net = SIDdata * (Sloping / Horizontal) + SISdata * 86400 / 1e6 # Calculate ShortWave Clear Short_Wave = Sloping Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5) # make directories PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky') if os.path.exists(PathClear) is False: os.makedirs(PathClear) PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net') if os.path.exists(PathNet) is False: os.makedirs(PathNet) # name Shortwave Clear and Net nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime( '%Y-%m-%d') + '.tif' nameNet = os.path.join(PathNet, nameFileNet) nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime( '%Y-%m-%d') + '.tif' nameClear = os.path.join(PathClear, nameFileClear) # Save net and clear short wave radiation DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj) DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj) return
def Calc_Regions(input_nc, output_nc, input_JRC, Boundaries): import numpy as np import watools.General.raster_conversions as RC sensitivity = 700 # 900 is less sensitive 1 is very sensitive # Get JRC array and information Array_JRC_occ = RC.Open_tiff_array(input_JRC) Geo_out, proj, size_X, size_Y = RC.Open_array_info(input_JRC) # Get Basin boundary based on LU Array_LU = RC.Open_nc_array(input_nc, "basin") LU_array = RC.resize_array_example(Array_LU, Array_JRC_occ) basin_array = np.zeros(np.shape(LU_array)) basin_array[LU_array > 0] = 1 del LU_array # find all pixels with water occurence Array_JRC_occ[basin_array < 1] = 0 Array_JRC_occ[Array_JRC_occ < 30] = 0 Array_JRC_occ[Array_JRC_occ >= 30] = 1 del basin_array # sum larger areas to find lakes x_size = np.round(int(np.shape(Array_JRC_occ)[0]) / 30) y_size = np.round(int(np.shape(Array_JRC_occ)[1]) / 30) sum_array = np.zeros([x_size, y_size]) for i in range(0, len(sum_array)): for j in range(0, len(sum_array[1])): sum_array[i, j] = np.sum(Array_JRC_occ[i * 30:(i + 1) * 30, j * 30:(j + 1) * 30]) del Array_JRC_occ lakes = np.argwhere(sum_array >= sensitivity) lake_info = np.zeros([1, 4]) i = 0 k = 1 # find all neighboring pixels for lake in lakes: added = 0 for j in range(0, k): if (lake[0] >= lake_info[j, 0] and lake[0] <= lake_info[j, 1] and lake[1] >= lake_info[j, 2] and lake[1] <= lake_info[j, 3]): lake_info[j, 0] = np.maximum( np.minimum(lake_info[j, 0], lake[0] - 8), 0) lake_info[j, 1] = np.minimum( np.maximum(lake_info[j, 1], lake[0] + 8), x_size) lake_info[j, 2] = np.maximum( np.minimum(lake_info[j, 2], lake[1] - 8), 0) lake_info[j, 3] = np.minimum( np.maximum(lake_info[j, 3], lake[1] + 8), y_size) added = 1 if added == 0: lake_info_one = np.zeros([4]) lake_info_one[0] = np.maximum(0, lake[0] - 8) lake_info_one[1] = np.minimum(x_size, lake[0] + 8) lake_info_one[2] = np.maximum(0, lake[1] - 8) lake_info_one[3] = np.minimum(y_size, lake[1] + 8) lake_info = np.append(lake_info, lake_info_one) lake_info = np.resize(lake_info, (k + 1, 4)) k += 1 # merge all overlaping regions p = 0 lake_info_end = np.zeros([1, 4]) for i in range(1, k): added = 0 lake_info_one = lake_info[i, :] lake_y_region = list( range(int(lake_info_one[0]), int(lake_info_one[1] + 1))) lake_x_region = list( range(int(lake_info_one[2]), int(lake_info_one[3] + 1))) for j in range(0, p + 1): if len(lake_y_region) + len( list( range(int(lake_info_end[j, 0]), int(lake_info_end[j, 1] + 1))) ) is not len( np.unique( np.append( lake_y_region, list( range(int(lake_info_end[j, 0]), int(lake_info_end[j, 1] + 1))))) ) and len(lake_x_region) + len( list( range(int(lake_info_end[j, 2]), int(lake_info_end[j, 3] + 1)))) is not len( np.unique( np.append( lake_x_region, list( range( int(lake_info_end[j, 2]), int(lake_info_end[j, 3] + 1)))))): lake_info_end[j, 0] = np.min( np.unique( np.append( lake_y_region, list( range(int(lake_info_end[j, 0]), int(lake_info_end[j, 1] + 1)))))) lake_info_end[j, 1] = np.max( np.unique( np.append( lake_y_region, list( range(int(lake_info_end[j, 0]), int(lake_info_end[j, 1] + 1)))))) lake_info_end[j, 2] = np.min( np.unique( np.append( lake_x_region, list( range(int(lake_info_end[j, 2]), int(lake_info_end[j, 3] + 1)))))) lake_info_end[j, 3] = np.max( np.unique( np.append( lake_x_region, list( range(int(lake_info_end[j, 2]), int(lake_info_end[j, 3] + 1)))))) added = 1 if added == 0: lake_info_one = lake_info[i, :] lake_info_end = np.append(lake_info_end, lake_info_one) lake_info_end = np.resize(lake_info_end, (p + 2, 4)) p += 1 # calculate the area Regions = np.zeros([p, 4]) pixel_x_size = Geo_out[1] * 30 pixel_y_size = Geo_out[5] * 30 for region in range(1, p + 1): Regions[region - 1, 0] = Geo_out[0] + pixel_x_size * lake_info_end[region, 2] Regions[region - 1, 1] = Geo_out[0] + pixel_x_size * (lake_info_end[region, 3] + 1) Regions[region - 1, 2] = Geo_out[3] + pixel_y_size * (lake_info_end[region, 1] + 1) Regions[region - 1, 3] = Geo_out[3] + pixel_y_size * lake_info_end[region, 0] return (Regions)
def Calc_Rainy_Days(Dir_Basin, Data_Path_P, Startdate, Enddate): """ This functions calculates the amount of rainy days based on daily precipitation data. Parameters ---------- Dir_Basin : str Path to all the output data of the Basin Data_Path_P : str Path to the daily rainfall data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Returns ------- Data_Path_RD : str Path from the Dir_Basin to the rainy days data """ # import WA+ modules import watools.General.data_conversions as DC import watools.General.raster_conversions as RC # Create an output directory to store the rainy days tiffs Data_Path_RD = os.path.join(Dir_Basin, 'Rainy_Days') if not os.path.exists(Data_Path_RD): os.mkdir(Data_Path_RD) # Define the dates that must be created Dates = pd.date_range(Startdate, Enddate, freq ='MS') # Set working directory to the rainfall folder os.chdir(Data_Path_P) # Open all the daily data and store the data in a 3D array for Date in Dates: # Define the year and month and amount of days in month year = Date.year month = Date.month daysinmonth = calendar.monthrange(year, month)[1] # Set the third (time) dimension of array starting at 0 i = 0 # Find all files of that month files = glob.glob('*daily_%d.%02d.*.tif' %(year, month)) # Check if the amount of files corresponds with the amount of days in month if len(files) is not daysinmonth: print('ERROR: Not all Rainfall days for month %d and year %d are downloaded' %(month, year)) # Loop over the days and store data in raster for File in files: dir_file = os.path.join(Data_Path_P, File) # Get array information and create empty numpy array for daily rainfall when looping the first file if File == files[0]: # Open geolocation info and define projection geo_out, proj, size_X, size_Y = RC.Open_array_info(dir_file) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Create empty array for the whole month P_Daily = np.zeros([daysinmonth,size_Y, size_X]) # Open data and put the data in 3D array Data = RC.Open_tiff_array(dir_file) # Remove the weird numbers Data[Data<0] = 0 # Add the precipitation to the monthly cube P_Daily[i, :, :] = Data i += 1 # Define a rainy day P_Daily[P_Daily > 0.201] = 1 P_Daily[P_Daily != 1] = 0 # Sum the amount of rainy days RD_one_month = np.nansum(P_Daily,0) # Define output name Outname = os.path.join(Data_Path_RD, 'Rainy_Days_NumOfDays_monthly_%d.%02d.01.tif' %(year, month)) # Save tiff file DC.Save_as_tiff(Outname, RD_one_month, geo_out, proj) return(Data_Path_RD)
def NPP_GPP_Based(Dir_Basin, Data_Path_GPP, Data_Path_NPP, Startdate, Enddate): """ This functions calculated monthly NDM based on the yearly NPP and monthly GPP. Parameters ---------- Dir_Basin : str Path to all the output data of the Basin Data_Path_GPP : str Path from the Dir_Basin to the GPP data Data_Path_NPP : str Path from the Dir_Basin to the NPP data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Simulation : int Defines the simulation Returns ------- Data_Path_NDM : str Path from the Dir_Basin to the normalized dry matter data """ # import WA+ modules import watools.General.data_conversions as DC import watools.General.raster_conversions as RC # Define output folder for Normalized Dry Matter Data_Path_NDM = os.path.join(Dir_Basin, "NDM") if not os.path.exists(Data_Path_NDM): os.mkdir(Data_Path_NDM) # Define monthly time steps that will be created Dates = pd.date_range(Startdate, Enddate, freq='MS') # Define the years that will be calculated Year_Start = int(Startdate[0:4]) Year_End = int(Enddate[0:4]) Years = list(range(Year_Start, Year_End + 1)) # Loop over the years for year in Years: # Change working directory to the NPP folder os.chdir(Data_Path_NPP) # Open yearly NPP data yearly_NPP_File = glob.glob('*yearly*%d.01.01.tif' % int(year))[0] Yearly_NPP = RC.Open_tiff_array(yearly_NPP_File) # Get the No Data Value of the NPP file dest = gdal.Open(yearly_NPP_File) NDV = dest.GetRasterBand(1).GetNoDataValue() # Set the No Data Value to Nan Yearly_NPP[Yearly_NPP == NDV] = np.nan # Change working directory to the GPP folder os.chdir(Data_Path_GPP) # Find all the monthly files of that year monthly_GPP_Files = glob.glob('*monthly*%d.*.01.tif' % int(year)) # Check if it are 12 files otherwise something is wrong and send the ERROR if not len(monthly_GPP_Files) == 12: print('ERROR: Some monthly GPP Files are missing') # Get the projection information of the GPP inputs geo_out, proj, size_X, size_Y = RC.Open_array_info( monthly_GPP_Files[0]) geo_out_NPP, proj_NPP, size_X_NPP, size_Y_NPP = RC.Open_array_info( os.path.join(Data_Path_NPP, yearly_NPP_File)) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Get the No Data Value of the GPP files dest = gdal.Open(monthly_GPP_Files[0]) NDV = dest.GetRasterBand(1).GetNoDataValue() # Create a empty numpy array Yearly_GPP = np.zeros([size_Y, size_X]) # Calculte the total yearly GPP for monthly_GPP_File in monthly_GPP_Files: # Open array Data = RC.Open_tiff_array(monthly_GPP_File) # Remove nan values Data[Data == NDV] = np.nan Data[np.isnan(Data)] = 0 # Add data to yearly sum Yearly_GPP += Data # Check if size is the same of NPP and GPP otherwise resize if not (size_X_NPP == size_X and size_Y_NPP == size_Y): Yearly_NPP = RC.resize_array_example(Yearly_NPP, Yearly_GPP) # Loop over the monthly dates for Date in Dates: # If the Date is in the same year as the yearly NPP and GPP if Date.year == year: # Create empty GPP array monthly_GPP = np.ones([size_Y, size_X]) * np.nan # Get current month month = Date.month # Get the GPP file of the current year and month monthly_GPP_File = glob.glob('*monthly_%d.%02d.01.tif' % (int(year), int(month)))[0] monthly_GPP = RC.Open_tiff_array(monthly_GPP_File) monthly_GPP[monthly_GPP == NDV] = np.nan # Calculate the NDM based on the monthly and yearly NPP and GPP (fraction of GPP) Monthly_NDM = Yearly_NPP * monthly_GPP / Yearly_GPP * ( 30. / 12.) * 10000 # kg/ha # Define output name output_name = os.path.join( Data_Path_NDM, 'NDM_MOD17_kg_ha-1_monthly_%d.%02d.01.tif' % (int(year), int(month))) # Save the NDM as tiff file DC.Save_as_tiff(output_name, Monthly_NDM, geo_out, proj) return (Data_Path_NDM)
def RetrieveData(Date, args): """ This function retrieves RFE data for a given date from the ftp://disc2.nascom.nasa.gov server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [output_folder, lonlim, latlim, xID, yID] = args # Create https DirFile = os.path.join( output_folder, 'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' % (Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) if not os.path.isfile(DirFile): # open ftp server ftp = FTP("ftp.cpc.ncep.noaa.gov", "", "") ftp.login() # Define FTP path to directory pathFTP = 'fews/fewsdata/africa/rfe2/geotiff/' # find the document name in this directory ftp.cwd(pathFTP) listing = [] # read all the file names in the directory ftp.retrlines("LIST", listing.append) # create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names filename = 'africa_rfe.%s%02s%02s.tif.zip' % ( Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')) outfilename = os.path.join( output_folder, 'africa_rfe.%s%02s%02s.tif' % (Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) try: local_filename = os.path.join(output_folder, filename) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write) lf.close() # unzip the file zip_filename = os.path.join(output_folder, filename) DC.Extract_Data(zip_filename, output_folder) # open tiff file dataset = RC.Open_tiff_array(outfilename) # clip dataset to the given extent data = dataset[yID[0]:yID[1], xID[0]:xID[1]] data[data < 0] = -9999 # save dataset as geotiff file latlim_adj = 40.05 - 0.1 * yID[0] lonlim_adj = -20.05 + 0.1 * xID[0] geo = [lonlim_adj, 0.1, 0, latlim_adj, 0, -0.1] DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84") # delete old tif file os.remove(outfilename) os.remove(zip_filename) except: print("file not exists") return True
def slope_correct(down_short_hor, pressure, ea, DEMmap, DOY): """ This function downscales the CFSR solar radiation by using the DEM map The Slope correction is based on Allen et al. (2006) 'Analytical integrated functions for daily solar radiation on slope' Keyword arguments: down_short_hor -- numpy array with the horizontal downwards shortwave radiation pressure -- numpy array with the air pressure ea -- numpy array with the actual vapour pressure DEMmap -- 'C:/' path to the DEM map DOY -- day of the year """ # Get Geo Info GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap) minx = GeoT[0] miny = GeoT[3] + xsize * GeoT[4] + ysize * GeoT[5] x = np.flipud(np.arange(xsize) * GeoT[1] + minx + GeoT[1] / 2) y = np.flipud(np.arange(ysize) * -GeoT[5] + miny + -GeoT[5] / 2) # Calculate Extraterrestrial Solar Radiation [W m-2] demmap = RC.Open_tiff_array(DEMmap) demmap[demmap < 0] = 0 # apply the slope correction Ra_hor, Ra_slp, sinb, sinb_hor, fi, slope, ID = SlopeInfluence( demmap, y, x, DOY) # Calculate atmospheric transmissivity Rs_hor = down_short_hor # EQ 39 tau = Rs_hor / Ra_hor #EQ 41 KB_hor = np.zeros(tau.shape) * np.nan indice = np.where(tau.flat >= 0.42) KB_hor.flat[indice] = 1.56 * tau.flat[indice] - 0.55 indice = np.logical_and(tau.flat > 0.175, tau.flat < 0.42) KB_hor.flat[indice] = 0.022 - 0.280 * tau.flat[indice] + 0.828 * tau.flat[ indice]**2 + 0.765 * tau.flat[indice]**3 indice = np.where(tau.flat <= 0.175) KB_hor.flat[indice] = 0.016 * tau.flat[indice] # EQ 42 KD_hor = tau - KB_hor Kt = 0.7 #EQ 18 W = 0.14 * ea * pressure + 2.1 KB0 = 0.98 * np.exp((-0.00146 * pressure / Kt / sinb) - 0.075 * (W / sinb)**0.4) KB0_hor = 0.98 * np.exp((-0.00146 * pressure / Kt / sinb_hor) - 0.075 * (W / sinb_hor)**0.4) #EQ 34 fB = KB0 / KB0_hor * Ra_slp / Ra_hor fia = (1 - KB_hor) * ( 1 + (KB_hor / (KB_hor + KD_hor))**0.5 * np.sin(slope / 2)**3) * fi + fB * KB_hor Rs = Rs_hor * (fB * (KB_hor / tau) + fia * (KD_hor / tau) + 0.23 * (1 - fi)) Rs[np.isnan(Rs)] = Rs_hor[np.isnan(Rs)] Rs_equiv = Rs / np.cos(slope) bias = np.nansum(Rs_hor) / np.nansum(Rs_equiv) return Rs_equiv, tau, bias
def RetrieveData(Date, args): """ This function retrieves CHIRPS data for a given date from the ftp://chg-ftpout.geog.ucsb.edu server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [output_folder, TimeCase, xID, yID, lonlim, latlim] = args # open ftp server ftp = FTP("chg-ftpout.geog.ucsb.edu", "", "") ftp.login() # Define FTP path to directory if TimeCase == 'daily': pathFTP = 'pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/' % Date.strftime( '%Y') elif TimeCase == 'monthly': pathFTP = 'pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/' else: raise KeyError("The input time interval is not supported") # find the document name in this directory ftp.cwd(pathFTP) listing = [] # read all the file names in the directory ftp.retrlines("LIST", listing.append) # create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names if TimeCase == 'daily': filename = 'chirps-v2.0.%s.%02s.%02s.tif.gz' % ( Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')) outfilename = os.path.join( output_folder, 'chirps-v2.0.%s.%02s.%02s.tif' % (Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) DirFileEnd = os.path.join( output_folder, 'P_CHIRPS.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' % (Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) elif TimeCase == 'monthly': filename = 'chirps-v2.0.%s.%02s.tif.gz' % (Date.strftime('%Y'), Date.strftime('%m')) outfilename = os.path.join( output_folder, 'chirps-v2.0.%s.%02s.tif' % (Date.strftime('%Y'), Date.strftime('%m'))) DirFileEnd = os.path.join( output_folder, 'P_CHIRPS.v2.0_mm-month-1_monthly_%s.%02s.%02s.tif' % (Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) else: raise KeyError("The input time interval is not supported") # download the global rainfall file try: local_filename = os.path.join(output_folder, filename) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write, 8192) lf.close() # unzip the file zip_filename = os.path.join(output_folder, filename) DC.Extract_Data_gz(zip_filename, outfilename) # open tiff file dataset = RC.Open_tiff_array(outfilename) # clip dataset to the given extent data = dataset[yID[0]:yID[1], xID[0]:xID[1]] data[data < 0] = -9999 # save dataset as geotiff file geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05] DC.Save_as_tiff(name=DirFileEnd, data=data, geo=geo, projection="WGS84") # delete old tif file os.remove(outfilename) except: print("file not exists") return True
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None): """ This functions calculates monthly tiff files based on the 16 daily tiff files. (will calculate the average) Parameters ---------- Dir_in : str Path to the input data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Dir_out : str Path to the output data, default is same as Dir_in """ # import WA+ modules import watools.General.data_conversions as DC import watools.General.raster_conversions as RC # Change working directory os.chdir(Dir_in) # Find all eight daily files files = glob.glob('*16-daily*.tif') # Create array with filename and keys (DOY and year) of all the 8 daily files i = 0 DOY_Year = np.zeros([len(files), 3]) for File in files: # Get the time characteristics from the filename year = File.split('.')[-4][-4:] month = File.split('.')[-3] day = File.split('.')[-2] # Create pandas Timestamp date_file = '%s-%02s-%02s' % (year, month, day) Datum = pd.Timestamp(date_file) # Get day of year DOY = Datum.strftime('%j') # Save data in array DOY_Year[i, 0] = i DOY_Year[i, 1] = DOY DOY_Year[i, 2] = year # Loop over files i += 1 # Check enddate: Enddate_split = Enddate.split('-') month_range = calendar.monthrange(int(Enddate_split[0]), int(Enddate_split[1]))[1] Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int( Enddate_split[1]), month_range) # Check startdate: Startdate_split = Startdate.split('-') Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int( Startdate_split[1])) # Define end and start date Dates = pd.date_range(Startdate, Enddate, freq='MS') DatesEnd = pd.date_range(Startdate, Enddate, freq='M') # Get array information and define projection geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0]) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Get the No Data Value dest = gdal.Open(files[0]) NDV = dest.GetRasterBand(1).GetNoDataValue() # Loop over months and create monthly tiff files i = 0 for date in Dates: # Get Start and end DOY of the current month DOY_month_start = date.strftime('%j') DOY_month_end = DatesEnd[i].strftime('%j') # Search for the files that are between those DOYs year = date.year DOYs = DOY_Year[DOY_Year[:, 2] == year] DOYs_oneMonth = DOYs[np.logical_and( (DOYs[:, 1] + 16) >= int(DOY_month_start), DOYs[:, 1] <= int(DOY_month_end))] # Create empty arrays Monthly = np.zeros([size_Y, size_X]) Weight_tot = np.zeros([size_Y, size_X]) Data_one_month = np.ones([size_Y, size_X]) * np.nan # Loop over the files that are within the DOYs for EightDays in DOYs_oneMonth[:, 1]: # Calculate the amount of days in this month of each file Weight = np.ones([size_Y, size_X]) # For start of month if np.min(DOYs_oneMonth[:, 1]) == EightDays: Weight = Weight * int(EightDays + 16 - int(DOY_month_start)) # For end of month elif np.max(DOYs_oneMonth[:, 1]) == EightDays: Weight = Weight * (int(DOY_month_end) - EightDays + 1) # For the middle of the month else: Weight = Weight * 16 row = DOYs_oneMonth[np.argwhere( DOYs_oneMonth[:, 1] == EightDays)[0][0], :][0] # Open the array of current file input_name = os.path.join(Dir_in, files[int(row)]) Data = RC.Open_tiff_array(input_name) # Remove NDV Weight[Data == NDV] = 0 Data[Data == NDV] = np.nan # Multiply weight time data (per day) Data = Data * Weight # Calculate the total weight and data Weight_tot += Weight Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)] # Go to next month i += 1 # Calculate the average Data_one_month[Weight_tot != 0.] = Monthly[ Weight_tot != 0.] / Weight_tot[Weight_tot != 0.] # Define output directory if Dir_out == None: Dir_out = Dir_in # Define output name output_name = os.path.join( Dir_out, files[int(row)].replace('16-daily', 'monthly')) output_name = output_name[:-9] + '%02d.01.tif' % (date.month) # Save tiff file DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj) return
def livestock_feed(output_folder, lu_fh, AREA, ndm_fhs, feed_dict, live_feed, cattle_fh, fraction_fhs, ndmdates): """ Calculate natural livestock feed production INPUTS ---------- lu_fh : str filehandle for land use map ndm_fhs: nd array array of filehandles of NDM maps ndm_dates: nd array array of dates for NDM maps feed_dict: dict dictionnary 'pasture class':[list of LULC] feed_pct: dict dictionnary 'pasture class':[percent available as feed] cattle_fh : str filehandle for cattle map """ Data_Path_Feed = "Feed" out_folder = os.path.join(output_folder, Data_Path_Feed) if not os.path.exists(out_folder): os.mkdir(out_folder) area_ha = AREA * 100 LULC = RC.Open_tiff_array(lu_fh) # cattle = RC.Open_tiff_array(cattle_fh) geo_out, proj, size_X, size_Y = RC.Open_array_info(lu_fh) f_pct = np.zeros(LULC.shape) for lu_type in list(feed_dict.keys()): classes = feed_dict[lu_type] mask = np.logical_or.reduce([LULC == value for value in classes]) f_pct[mask] = live_feed[lu_type] feed_fhs_landscape = [] feed_fhs_incremental = [] for d in range(len(ndm_fhs)): ndm_fh = ndm_fhs[d] fraction_fh = fraction_fhs[d] date1 = ndmdates[d] year = '%d' % date1.year month = '%02d' % date1.month yield_fract = RC.Open_tiff_array(fraction_fh) out_fh_l = out_folder + '\\feed_prod_landscape_%s_%s.tif' % (year, month) out_fh_i = out_folder + '\\feed_prod_incremental_%s_%s.tif' % (year, month) # out_fh2 = out_folder+'\\Feed_prod_pH_%s_%s.tif' %(year, month) NDM = becgis.open_as_array(ndm_fh, nan_values=True) NDM_feed = NDM * f_pct NDM_feed_incremental = NDM_feed * yield_fract * area_ha / 1e6 NDM_feed_landscape = (NDM_feed * (1 - yield_fract)) * area_ha / 1e6 DC.Save_as_tiff(out_fh_l, NDM_feed_landscape, geo_out) DC.Save_as_tiff(out_fh_i, NDM_feed_incremental, geo_out) # NDM_feed_perHead = NDM_feed / cattle # DC.Save_as_tiff(out_fh2, NDM_feed, geo_out) feed_fhs_landscape.append(out_fh_l) feed_fhs_incremental.append(out_fh_i) return feed_fhs_landscape, feed_fhs_incremental