def Get_Data(filename_in, Example_Data, reprojection_type): if Example_Data == None: dest = gdal.Open(filename_in) proj = dest.GetProjection() geo = dest.GetGeoTransform() band = dest.GetRasterBand(1) NDV = band.GetNoDataValue() Array = dest.GetRasterBand(1).ReadAsArray() Array = np.float_(Array) Array[Array == NDV] = np.nan else: dest_br = gdal.Open(filename_in) band = dest_br.GetRasterBand(1) NDV = band.GetNoDataValue() Array = band.ReadAsArray() Array = np.float_(Array) Array[Array == NDV] = np.nan Array_NDV = np.where(Array == NDV, 0, 1) proj_br = dest_br.GetProjection() geo_br = dest_br.GetGeoTransform() destNDV = DC.Save_as_MEM(Array_NDV, geo_br, proj_br) destArray = DC.Save_as_MEM(Array, geo_br, proj_br) dest_NDV_rep = RC.reproject_dataset_example(destNDV, Example_Data, 1) dest = RC.reproject_dataset_example(destArray, Example_Data, reprojection_type) proj = dest.GetProjection() geo = dest.GetGeoTransform() Array = dest.GetRasterBand(1).ReadAsArray() ArrayNDV = dest_NDV_rep.GetRasterBand(1).ReadAsArray() Array = np.float_(Array) Array[ArrayNDV == 0] = np.nan return (dest, Array, proj, geo)
def main(output_folder_L1, dest_AOI_MASK, Threshold_Mask): # select input folder input_folder_LCC = os.path.join(output_folder_L1, "L2_LCC_A") # Set input folder as working directory os.chdir(input_folder_LCC) # Find file for extend re = glob.glob("L2_LCC_A_*.tif")[0] # Open file Filename_Example = os.path.join(input_folder_LCC, re) # Open info array dest_shp = RC.reproject_dataset_example(dest_AOI_MASK, Filename_Example, 4) geo = dest_shp.GetGeoTransform() proj = dest_shp.GetProjection() # Array mask Mask_with_Edge = dest_shp.GetRasterBand(1).ReadAsArray() # Create End Mask Mask = np.where(Mask_with_Edge < Threshold_Mask / 100., np.nan, 1) # output file output_file = os.path.join(output_folder_L1, "MASK", "MASK.tif") DC.Save_as_tiff(output_file, Mask, geo, proj) return ()
def Create_LU_MAP(output_folder, Dates_yearly, LU, LUdek, Paths_LU_ESA, Formats_LU_ESA, example_file): # Create output folder LVL2 output_folder_L2 = os.path.join(output_folder, "LEVEL_2") # Open ESACCI input_file_LU_ESACCI = os.path.join(output_folder, Paths_LU_ESA, Formats_LU_ESA) # Converting LU maps into one LU map # open dictionary WAPOR WAPOR_Conversions_dict = WAPOR_Conversions() # open dictionary ESACCI ESACCI_Conversions_dict = ESACCI_Conversions() Phenology_pixels_year = np.ones(LUdek.Size) * np.nan Grassland_pixels_year = np.ones(LUdek.Size) * np.nan # Loop over the years for Year in Dates_yearly: Year_start = int(Dates_yearly[0].year) Year_int = int(Year.year) geo = LU.GeoTransform proj = LU.Projection destLUESACCI = RC.reproject_dataset_example(input_file_LU_ESACCI, example_file) LU_ESACCI = destLUESACCI.GetRasterBand(1).ReadAsArray() # Create LUmap LU_Map_WAPOR = np.ones([LU.Size[1], LU.Size[2]]) * np.nan LU_Map_ESACCI = np.ones([LU.Size[1], LU.Size[2]]) * np.nan for number in WAPOR_Conversions_dict.items(): LU_Map_WAPOR = np.where(LU.Data[int((Year_int - Year_start)),: ,:] == number[0], number[1], LU_Map_WAPOR) for number in ESACCI_Conversions_dict.items(): LU_Map_ESACCI = np.where(LU_ESACCI == number[0], number[1], LU_Map_ESACCI) # Combine LU maps # 1 = rainfed, 2 = irrigated, 3 = Pasture LU_END = np.where(np.logical_and(LU_Map_WAPOR == 1, LU_Map_ESACCI == 1), 1, np.nan) LU_END = np.where(LU_Map_WAPOR > 1, LU_Map_WAPOR, LU_END) # Save LU map DC.Save_as_tiff(os.path.join(output_folder_L2, "LU_END", "LU_%s.tif" %Year_int), LU_END, geo, proj) # find posible Perennial pixels Phenology_pixels_year[int((Year_int - Year_start) * 36):int((Year_int - Year_start) * 36)+36,: ,:] = np.where(np.logical_or(LU_END==1, LU_END==2), 1, np.nan)[None, :, :] Grassland_pixels_year[int((Year_int - Year_start) * 36):int((Year_int - Year_start) * 36)+36,: ,:] = np.where(LU_END==3, 1, np.nan)[None, :, :] return(Phenology_pixels_year, Grassland_pixels_year)
def lapse_rate(Dir,temperature_map, DEMmap): """ This function downscales the GLDAS temperature map by using the DEM map Keyword arguments: temperature_map -- 'C:/' path to the temperature map DEMmap -- 'C:/' path to the DEM map """ # calculate average altitudes corresponding to T resolution dest = RC.reproject_dataset_example(DEMmap, temperature_map,method = 4) DEM_ave_out_name = os.path.join(Dir,'HydroSHED', 'DEM','DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map) DEM_ave_data = dest.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) dest = None # determine lapse-rate [degress Celcius per meter] lapse_rate_number = 0.0065 # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2) dem_avg=dest.GetRasterBand(1).ReadAsArray() dem_avg[dem_avg<0]=0 dest = None # Open the temperature dataset dest = RC.reproject_dataset_example(temperature_map, DEMmap, method = 2) T=dest.GetRasterBand(1).ReadAsArray() dest = None # Open Demmap demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap<=0]=0 demmap[demmap==-32768]=np.nan # calculate first part T = T + ((dem_avg-demmap) * lapse_rate_number) return T
def adjust_P(Dir, pressure_map, DEMmap): """ This function downscales the GLDAS air pressure map by using the DEM map Keyword arguments: pressure_map -- 'C:/' path to the pressure map DEMmap -- 'C:/' path to the DEM map """ # calculate average latitudes destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method = 4) DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM','DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map) DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2) dem_avg=dest.GetRasterBand(1).ReadAsArray() dest = None # open maps as numpy arrays dest = RC.reproject_dataset_example(pressure_map, DEMmap, method = 2) P=dest.GetRasterBand(1).ReadAsArray() dest = None demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap<=0]=0 demmap[demmap==-32768]=np.nan # calculate second part P = P + (101.3*((293-0.0065*(demmap-dem_avg))/293)**5.26 - 101.3) os.remove(DEM_ave_out_name) return P
def __init__(self, input_folder, input_format, Dates=None, Conversion=1, Example_Data=None, Mask_Data=None, gap_filling=None, reprojection_type=2, Variable='', Product='', Description='', Unit='', Dimension_description=''): if Mask_Data != None: dest_MASK = RC.reproject_dataset_example(Mask_Data, Example_Data, 1) MASK = dest_MASK.GetRasterBand(1).ReadAsArray() MASK = np.where(MASK == 1, 1, np.nan) else: MASK = 1 if Dates != None: # Get Ordinal time time_or = np.array(list(map(lambda i: i.toordinal(), Dates))) i = 1 # Define start and enddate string Startdate_str = '%d-%02d-%02d' % (Dates[0].year, Dates[0].month, Dates[0].day) Enddate_str = '%d-%02d-%02d' % (Dates[-1].year, Dates[-1].month, Dates[-1].day) # Define dimensions size_z = len(Dates) for Date in Dates: try: sys.stdout.write("\rLoading Data %s %i/%i (%f %%)" % (Variable, i, len(Dates), i / (len(Dates)) * 100)) sys.stdout.flush() # Create input filename filename_in = os.path.join( input_folder, input_format.format(yyyy=Date.year, mm=Date.month, dd=Date.day)) dest, Array, proj, geo = Get_Data(filename_in, Example_Data, reprojection_type) if Date == Dates[0]: size_x = dest.RasterXSize size_y = dest.RasterYSize proj = dest.GetProjection() geo = dest.GetGeoTransform() Array_end = np.ones([len(Dates), size_y, size_x ]) * np.nan if gap_filling != None and ~np.isnan(np.nanmean(Array)): Array[np.isnan(Array)] = -9999 Array = RC.gap_filling(Array, -9999, gap_filling) Array_end[time_or == Date.toordinal( ), :, :] = Array * Conversion * MASK i += 1 except: print("Was not able to collect %s %s" % (Variable, Date)) shape = [size_z, size_y, size_x] else: sys.stdout.write("\rLoading Constant Data %s" % (Variable)) sys.stdout.flush() # Define start and enddate string Startdate_str = '' Enddate_str = '' # Get constant data filename_in = os.path.join(input_folder, input_format) dest, Array_end, proj, geo = Get_Data(filename_in, Example_Data, reprojection_type) # Get dimension size_x = dest.RasterXSize size_y = dest.RasterYSize shape = [size_y, size_x] time_or = '' # Apply gapfilling if needed if gap_filling != None and ~np.isnan(np.nanmean(Array_end)): Array_end[np.isnan(Array_end)] = -9999 Array_end = RC.gap_filling(Array_end, -9999, gap_filling) Array_end = Array_end * MASK self.Projection = proj self.Size = shape self.GeoTransform = geo self.Data = Array_end.astype(np.float64) self.NoData = np.nan self.Ordinal_time = time_or # Origin Of Data self.Directory = input_folder self.Format = input_format # Describtion of dataset self.Variable = Variable self.Product = Product self.Description = Description self.Unit = Unit self.Dimension_description = Dimension_description # Time Series self.Startdate = Startdate_str self.Enddate = Enddate_str self.Timestep = '' print( " " )
def calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str, down_short_str, down_long_str, up_long_str, DEMmap_str, DOY): """ This function calculates the ETref by using all the input parameters (path) according to FAO standards see: http://www.fao.org/docrep/x0490e/x0490e08.htm#TopOfPage Keyword arguments: tmin_str -- 'C:/' path to the minimal temperature tiff file [degrees Celcius], e.g. from GLDAS tmax_str -- 'C:/' path to the maximal temperature tiff file [degrees Celcius], e.g. from GLDAS humid_str -- 'C:/' path to the humidity tiff file [kg/kg], e.g. from GLDAS press_str -- 'C:/' path to the air pressure tiff file [kPa], e.g. from GLDAS wind_str -- 'C:/' path to the wind velocity tiff file [m/s], e.g. from GLDAS down_short_str -- 'C:/' path to the downward shortwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF down_long_str -- 'C:/' path to the downward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF up_long_str -- 'C:/' path to the upward longwave radiation tiff file [W*m-2], e.g. from CFSR/LANDSAF DEMmap_str -- 'C:/' path to the DEM tiff file [m] e.g. from HydroSHED DOY -- Day of the year """ # Get some geo-data to save results GeoT, Projection, xsize, ysize = RC.Open_array_info(DEMmap_str) #NDV, xsize, ysize, GeoT, Projection, DataType = GetGeoInfo(DEMmap_str) raster_shape = [xsize, ysize] # Create array to store results ETref = np.zeros(raster_shape) # gap fill tmin_str_GF = RC.gap_filling(tmin_str, -9999) tmax_str_GF = RC.gap_filling(tmax_str, -9999) humid_str_GF = RC.gap_filling(humid_str, -9999) press_str_GF = RC.gap_filling(press_str, -9999) wind_str_GF = RC.gap_filling(wind_str, -9999) down_short_str_GF = RC.gap_filling(down_short_str, np.nan) down_long_str_GF = RC.gap_filling(down_long_str, np.nan) if up_long_str is not 'not': up_long_str_GF = RC.gap_filling(up_long_str, np.nan) else: up_long_str_GF = 'nan' #dictionary containing all tthe paths to the input-maps inputs = dict({ 'tmin': tmin_str_GF, 'tmax': tmax_str_GF, 'humid': humid_str_GF, 'press': press_str_GF, 'wind': wind_str_GF, 'down_short': down_short_str_GF, 'down_long': down_long_str_GF, 'up_long': up_long_str_GF }) #dictionary containing numpy arrays of al initial and intermediate variables input_array = dict({ 'tmin': None, 'tmax': None, 'humid': None, 'press': None, 'wind': None, 'albedo': None, 'down_short': None, 'down_long': None, 'up_short': None, 'up_long': None, 'net_radiation': None, 'ea': None, 'es': None, 'delta': None }) #APPLY LAPSE RATE CORRECTION ON TEMPERATURE tmin = lapse_rate(Dir, inputs['tmin'], DEMmap_str) tmax = lapse_rate(Dir, inputs['tmax'], DEMmap_str) #PROCESS PRESSURE MAPS press = adjust_P(Dir, inputs['press'], DEMmap_str) #PREPARE HUMIDITY MAPS dest = RC.reproject_dataset_example(inputs['humid'], DEMmap_str, method=2) humid = dest.GetRasterBand(1).ReadAsArray() dest = None #CORRECT WIND MAPS dest = RC.reproject_dataset_example(inputs['wind'], DEMmap_str, method=2) wind = dest.GetRasterBand(1).ReadAsArray() * 0.75 dest = None #PROCESS GLDAS DATA input_array['ea'], input_array['es'], input_array['delta'] = process_GLDAS( tmax, tmin, humid, press) ea = input_array['ea'] es = input_array['es'] delta = input_array['delta'] if up_long_str == 'not': #CORRECT WIND MAPS dest = RC.reproject_dataset_example(down_short_str, DEMmap_str, method=2) Short_Net_data = dest.GetRasterBand(1).ReadAsArray() * 0.75 dest = None dest = RC.reproject_dataset_example(down_long_str, DEMmap_str, method=2) Short_Clear_data = dest.GetRasterBand(1).ReadAsArray() * 0.75 dest = None # Calculate Long wave Net radiation Rnl = 4.903e-9 * ( ((tmin + 273.16)**4 + (tmax + 273.16)**4) / 2) * (0.34 - 0.14 * np.sqrt(ea)) * ( 1.35 * Short_Net_data / Short_Clear_data - 0.35) # Calulate Net Radiation and converted to MJ*d-1*m-2 net_radiation = (Short_Net_data * 0.77 + Rnl) * 86400 / 10**6 else: #OPEN DOWNWARD SHORTWAVE RADIATION dest = RC.reproject_dataset_example(inputs['down_short'], DEMmap_str, method=2) down_short = dest.GetRasterBand(1).ReadAsArray() dest = None down_short, tau, bias = slope_correct(down_short, press, ea, DEMmap_str, DOY) #OPEN OTHER RADS up_short = down_short * 0.23 dest = RC.reproject_dataset_example(inputs['down_long'], DEMmap_str, method=2) down_long = dest.GetRasterBand(1).ReadAsArray() dest = None dest = RC.reproject_dataset_example(inputs['up_long'], DEMmap_str, method=2) up_long = dest.GetRasterBand(1).ReadAsArray() dest = None #OPEN NET RADIATION AND CONVERT W*m-2 TO MJ*d-1*m-2 net_radiation = ((down_short - up_short) + (down_long - up_long)) * 86400 / 10**6 #CALCULATE ETref ETref = (0.408 * delta * net_radiation + 0.665 * 10**-3 * press * (900 / ((tmax + tmin) / 2 + 273)) * wind * (es - ea)) / (delta + 0.665 * 10**-3 * press * (1 + 0.34 * wind)) # Set limits ETref ETref[ETref < 0] = 0 ETref[ETref > 400] = np.nan #return a reference ET map (numpy array), a dictionary containing all intermediate information and a bias of the slope correction on down_short return ETref
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim): """ This function collects and clip LANDSAF data Keyword arguments: SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID) Dir -- 'C:/' path to the WA map Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -60 and 60) lonlim -- [xmin, xmax] (values must be between -180 and 180) """ # Make an array of the days of which the ET is taken Dates = pd.date_range(Startdate, Enddate, freq='D') # make directories SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS') if os.path.exists(SISdir) is False: os.makedirs(SISdir) SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID') if os.path.exists(SIDdir) is False: os.makedirs(SIDdir) ShortwaveBasin(SourceLANDSAF, Dir, latlim, lonlim, Dates=[Startdate, Enddate]) DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_HydroShed_m_3s.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str) # Open DEM map demmap = RC.Open_tiff_array(DEMmap_str) demmap[demmap < 0] = 0 # make lat and lon arrays) dlat = geo_out[5] dlon = geo_out[1] lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon for date in Dates: # day of year day = date.dayofyear Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence( demmap, lat, lon, day) SIDname = os.path.join( SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif') SISname = os.path.join( SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif') #PREPARE SID MAPS SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3) SIDdata = SIDdest.GetRasterBand(1).ReadAsArray() #PREPARE SIS MAPS SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3) SISdata = SISdest.GetRasterBand(1).ReadAsArray() # Calculate ShortWave net Short_Wave_Net = SIDdata * (Sloping / Horizontal) + SISdata * 86400 / 1e6 # Calculate ShortWave Clear Short_Wave = Sloping Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5) # make directories PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky') if os.path.exists(PathClear) is False: os.makedirs(PathClear) PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net') if os.path.exists(PathNet) is False: os.makedirs(PathNet) # name Shortwave Clear and Net nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime( '%Y-%m-%d') + '.tif' nameNet = os.path.join(PathNet, nameFileNet) nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime( '%Y-%m-%d') + '.tif' nameClear = os.path.join(PathClear, nameFileClear) # Save net and clear short wave radiation DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj) DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj) return
def Get_Dataset_Polygon(input_folder, input_format, Dates, input_shp, Shp_prop='id', Shp_value=1, Stats="mean"): Dataset = np.ones([len(Dates), 2]) * np.nan i = 0 os.chdir(input_folder) filename_test = glob.glob('*.tif')[0] filename_in = os.path.join(input_folder, filename_test) dest = gdal.Open(filename_in) geo = dest.GetGeoTransform() epsg_tiff = RC.Get_epsg(dest) epsg_shp = RC.Get_epsg(input_shp, 'shp') if epsg_tiff != epsg_shp: input_shp = RC.reproject_shapefile(input_shp, epsg_tiff) # Create Mask of polygon mask_tiff = RC.GDAL_rasterize(input_shp, geo[1], Shp_prop) dest = RC.reproject_dataset_example(mask_tiff, filename_in) MASK = dest.GetRasterBand(1).ReadAsArray() MASK[MASK != Shp_value] = np.nan MASK[MASK == Shp_value] = 1 for Date in Dates: Day = Date.day Month = Date.month Year = Date.year DOY = Date.dayofyear filename = glob.glob( input_format.format(yyyy=Year, mm=Month, dd=Day, doy=DOY)) Dataset[i, 0] = Date.toordinal() if len(filename) > 0: filename = filename[0] dest = gdal.Open(filename) Array = dest.GetRasterBand(1).ReadAsArray() Array = np.float_(Array) Array[Array == -9999] = np.nan try: if Stats == "mean": Value = np.nanmean(Array[MASK == 1]) if Stats == "std": Value = np.nanstd(Array[MASK == 1]) except: dest = RC.reproject_dataset_example(mask_tiff, filename) MASK = dest.GetRasterBand(1).ReadAsArray() MASK[MASK > 0] = 1 MASK[MASK <= 0] = np.nan if Stats == "mean": Value = np.nanmean(Array[MASK == 1]) if Stats == "std": Value = np.nanstd(Array[MASK == 1]) Dataset[i, 1] = Value i += 1 return (Dataset)
def Calc_Dekad_Raster_from_Daily(input_folder, input_format, Date, flux_state="flux", example_file=None): # Calculate the start and enddate of the decade Startdate = Date # date in dekad day_dekad = int( "%d" % int(np.minimum(int(("%02d" % int(str(Startdate.day)))[0]), 2))) # Get end date day of dekad if day_dekad == 2: year = Startdate.year month = Startdate.month End_day_dekad = calendar.monthrange(year, month)[1] else: End_day_dekad = int(str(int(day_dekad)) + "1") + 9 # Set Dates Enddate = datetime.datetime(Startdate.year, Startdate.month, End_day_dekad) Dates = pd.date_range(Startdate, Enddate, freq="D") i = 0 for Date_one in Dates: if example_file == None: dest = gdal.Open( os.path.join( input_folder, input_format.format(yyyy=Date_one.year, mm=Date_one.month, dd=Date_one.day))) else: dest = RC.reproject_dataset_example( os.path.join( input_folder, input_format.format(yyyy=Date_one.year, mm=Date_one.month, dd=Date_one.day)), example_file, 2) if Date_one == Dates[0]: size_y = dest.RasterYSize size_x = dest.RasterXSize geo = dest.GetGeoTransform() proj = dest.GetProjection() data = np.ones([len(Dates), size_y, size_x]) array = dest.GetRasterBand(1).ReadAsArray() array = np.float_(array) array[array == -9999] = np.nan data[i, :, :] = array i += 1 if flux_state == "flux": data = np.nansum(data, axis=0) if flux_state == "state": data = np.nanmean(data, axis=0) dest = DC.Save_as_MEM(data, geo, proj) return (dest)