def Save_As_Tiff(self, output_folder): if not os.path.exists(output_folder): os.makedirs(output_folder) Dates = self.Ordinal_time geo = self.GeoTransform proj = self.Projection Data = self.Data Variable = self.Variable Unit = self.Unit print("Save %s as tiff file in %s" % (Variable, output_folder)) if Dates == "Long_Term_Decade": import WaporTranslator.LEVEL_2.Functions as Functions Dates_dek = Functions.Get_Dekads(2009, 2009) i = 0 for Date in Dates_dek: try: output_filename = os.path.join( output_folder, "%s_%s_%02d.%02d.tif" % (Variable.replace( " ", "_"), Unit, Date.month, Date.day)) Data_one = Data[i, :, :] DC.Save_as_tiff(output_filename, Data_one, geo, proj) i += 1 except: print("Was not able to collect %s %s" % (Variable, Date)) elif np.any(Dates != None): for Date in Dates: Date_datetime = datetime.datetime.fromordinal(Date) output_filename = os.path.join( output_folder, "%s_%s_%d.%02d.%02d.tif" % (Variable.replace(" ", "_"), Unit, Date_datetime.year, Date_datetime.month, Date_datetime.day)) Data_one = np.squeeze(Data[Dates == Date, :, :], 0) DC.Save_as_tiff(output_filename, Data_one, geo, proj) else: output_filename = os.path.join( output_folder, "%s_%s.tif" % (Variable.replace(" ", "_"), Unit)) DC.Save_as_tiff(output_filename, Data, geo, proj)
def main(output_folder_L1, dest_AOI_MASK, Threshold_Mask): # select input folder input_folder_LCC = os.path.join(output_folder_L1, "L2_LCC_A") # Set input folder as working directory os.chdir(input_folder_LCC) # Find file for extend re = glob.glob("L2_LCC_A_*.tif")[0] # Open file Filename_Example = os.path.join(input_folder_LCC, re) # Open info array dest_shp = RC.reproject_dataset_example(dest_AOI_MASK, Filename_Example, 4) geo = dest_shp.GetGeoTransform() proj = dest_shp.GetProjection() # Array mask Mask_with_Edge = dest_shp.GetRasterBand(1).ReadAsArray() # Create End Mask Mask = np.where(Mask_with_Edge < Threshold_Mask / 100., np.nan, 1) # output file output_file = os.path.join(output_folder_L1, "MASK", "MASK.tif") DC.Save_as_tiff(output_file, Mask, geo, proj) return ()
def RetrieveData(Date, args): """ This function retrieves TRMM data for a given date from the ftp://disc2.nascom.nasa.gov server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [output_folder, TimeCase, xID, yID, lonlim, latlim] = args year = Date.year month= Date.month day = Date.day from watertools import WebAccounts username, password = WebAccounts.Accounts(Type = 'NASA') # Create https if TimeCase == 'daily': URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/GPM_L3/GPM_3IMERGDF.05/%d/%02d/3B-DAY.MS.MRG.3IMERG.%d%02d%02d-S000000-E235959.V05.nc4.ascii?precipitationCal[%d:1:%d][%d:1:%d]' %(year, month, year, month, day, xID[0], xID[1]-1, yID[0], yID[1]-1) DirFile = os.path.join(output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %(year, month, day)) Scaling = 1 if TimeCase == 'monthly': URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/hyrax/GPM_L3/GPM_3IMERGM.05/%d/3B-MO.MS.MRG.3IMERG.%d%02d01-S000000-E235959.%02d.V05B.HDF5.ascii?precipitation[%d:1:%d][%d:1:%d]' %(year, year, month, month, xID[0], xID[1]-1, yID[0], yID[1]-1) Scaling = calendar.monthrange(year,month)[1] * 24 DirFile = os.path.join(output_folder, "P_GPM.IMERG_mm-month-1_monthly_%d.%02d.01.tif" %(year, month)) if not os.path.isfile(DirFile): dataset = requests.get(URL, allow_redirects=False,stream = True) try: get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True) except: from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) get_dataset = requests.get(dataset.headers['location'], auth = (username, password), verify = False) # download data (first save as text file) pathtext = os.path.join(output_folder,'temp.txt') z = open(pathtext,'wb') z.write(get_dataset.content) z.close() # Open text file and remove header and footer data_start = np.genfromtxt(pathtext,dtype = float,skip_header = 1,delimiter=',') data = data_start[:,1:] * Scaling data[data < 0] = -9999 data = data.transpose() data = np.flipud(data) # Delete .txt file os.remove(pathtext) # Make geotiff file geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1] DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84") return True
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim): import watertools.General.raster_conversions as RC # Open Dataset HiHydroSoil_Array = RC.Open_tiff_array(local_filename) # Define area XID = [ int(np.floor((180 + lonlim[0]) / 0.00833333)), int(np.ceil((180 + lonlim[1]) / 0.00833333)) ] YID = [ int(np.ceil((90 - latlim[1]) / 0.00833333)), int(np.floor((90 - latlim[0]) / 0.00833333)) ] # Define Georeference geo = tuple([ -180 + 0.00833333 * XID[0], 0.00833333, 0, 90 - 0.00833333 * YID[0], 0, -0.00833333 ]) # Clip Array HiHydroSoil_Array_clipped = HiHydroSoil_Array[YID[0]:YID[1], XID[0]:XID[1]] # Save tiff file DC.Save_as_tiff(Filename_out, HiHydroSoil_Array_clipped, geo, "WGS84")
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim, latlim, yID, xID, TimeStep): """ This function retrieves ALEXI data for a given date from the ftp.wateraccounting.unesco-ihe.org server. Restrictions: The data and this python file may not be distributed to others without permission of the WA+ team due data restriction of the ALEXI developers. Keyword arguments: local_filename -- name of the temporary file which contains global ALEXI data DirFile -- name of the end file with the weekly ALEXI data filename -- name of the end file lonlim -- [ymin, ymax] (values must be between -60 and 70) latlim -- [xmin, xmax] (values must be between -180 and 180) """ # Collect account and FTP information username, password = WebAccounts.Accounts(Type='FTP_WA') ftpserver = "ftp.wateraccounting.unesco-ihe.org" # Download data from FTP ftp = FTP(ftpserver) ftp.login(username, password) if TimeStep is "weekly": directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/" if TimeStep is "daily": directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/" ftp.cwd(directory) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write) lf.close() if TimeStep is "weekly": # Open global ALEXI data dataset = RC.Open_tiff_array(local_filename) # Clip extend out of world data data = dataset[yID[0]:yID[1], xID[0]:xID[1]] data[data < 0] = -9999 if TimeStep is "daily": DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0]) raw_data = np.fromfile(os.path.splitext(local_filename)[0], dtype="<f4") dataset = np.flipud(np.resize(raw_data, [3000, 7200])) data = dataset[ yID[0]:yID[1], xID[0]:xID[1]] / 2.45 # Values are in MJ/m2d so convert to mm/d data[data < 0] = -9999 # make geotiff file geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05] DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84") return
def RetrieveData(Date, args): """ This function retrieves MOD16 ET data for a given date from the ftp://ftp.ntsg.umt.edu/ server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [ output_folder, TilesVertical, TilesHorizontal, latlim, lonlim, timestep, hdf_library, Size_pix ] = args if timestep == 'monthly': ETfileName = os.path.join( output_folder, 'ET_MOD16A2_mm-month-1_monthly_' + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.01.tif') elif timestep == '8-daily': ETfileName = os.path.join( output_folder, 'ET_MOD16A2_mm-8days-1_8-daily_' + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif') if not os.path.exists(ETfileName): # Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles try: Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, timestep, hdf_library, Size_pix) except: print("Was not able to download the file") try: # Define the output name of the collect data function name_collect = os.path.join(output_folder, 'Merged.tif') # Reproject the MODIS product to epsg_to epsg_to = '4326' name_reprojected = RC.reproject_MODIS(name_collect, epsg_to) # Clip the data to the users extend data, geo = RC.clip_data(name_reprojected, latlim, lonlim) DC.Save_as_tiff(name=ETfileName, data=data, geo=geo, projection='WGS84') # remove the side products os.remove(os.path.join(output_folder, name_collect)) os.remove(os.path.join(output_folder, name_reprojected)) except: print("Failed for date: %s" % Date) return ()
def gap_filling(dataset, NoDataValue, method=1): """ This function fills the no data gaps in a numpy array Keyword arguments: dataset -- 'C:/' path to the source data (dataset that must be filled) NoDataValue -- Value that must be filled """ import watertools.General.data_conversions as DC try: if dataset.split('.')[-1] == 'tif': # Open the numpy array data = Open_tiff_array(dataset) Save_as_tiff = 1 else: data = dataset Save_as_tiff = 0 except: data = dataset Save_as_tiff = 0 # fill the no data values if NoDataValue is np.nan: mask = ~(np.isnan(data)) else: mask = ~(data == NoDataValue) xx, yy = np.meshgrid(np.arange(data.shape[1]), np.arange(data.shape[0])) xym = np.vstack((np.ravel(xx[mask]), np.ravel(yy[mask]))).T data0 = np.ravel(data[:, :][mask]) if method == 1: interp0 = scipy.interpolate.NearestNDInterpolator(xym, data0) data_end = interp0(np.ravel(xx), np.ravel(yy)).reshape(xx.shape) if method == 2: interp0 = scipy.interpolate.LinearNDInterpolator(xym, data0) data_end = interp0(np.ravel(xx), np.ravel(yy)).reshape(xx.shape) if Save_as_tiff == 1: EndProduct = dataset[:-4] + '_GF.tif' # collect the geoinformation geo_out, proj, size_X, size_Y = Open_array_info(dataset) # Save the filled array as geotiff DC.Save_as_tiff(name=EndProduct, data=data_end, geo=geo_out, projection=proj) else: EndProduct = data_end return (EndProduct)
def RetrieveData(Date, args): """ This function retrieves MOD15 FPAR data for a given date from the http://e4ftl01.cr.usgs.gov/ server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [ output_folder, TilesVertical, TilesHorizontal, lonlim, latlim, unit, dataset, nameDownload, hdf_library ] = args FPARfileName = os.path.join( output_folder, '%s_MOD15_%s_8-daily_' % (dataset, unit) + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif') if not os.path.exists(FPARfileName): # Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles try: Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, nameDownload, hdf_library) except: print("Was not able to download the file") try: # Define the output name of the collect data function name_collect = os.path.join(output_folder, 'Merged.tif') # Reproject the MODIS product to epsg_to epsg_to = '4326' name_reprojected = RC.reproject_MODIS(name_collect, epsg_to) # Clip the data to the users extend data, geo = RC.clip_data(name_reprojected, latlim, lonlim) # Save the file as tiff DC.Save_as_tiff(name=FPARfileName, data=data, geo=geo, projection='WGS84') # remove the side products os.remove(os.path.join(output_folder, name_collect)) os.remove(os.path.join(output_folder, name_reprojected)) except: print("Failed for date: %s" % Date) return True
def Create_LU_MAP(output_folder, Dates_yearly, LU, LUdek, Paths_LU_ESA, Formats_LU_ESA, example_file): # Create output folder LVL2 output_folder_L2 = os.path.join(output_folder, "LEVEL_2") # Open ESACCI input_file_LU_ESACCI = os.path.join(output_folder, Paths_LU_ESA, Formats_LU_ESA) # Converting LU maps into one LU map # open dictionary WAPOR WAPOR_Conversions_dict = WAPOR_Conversions() # open dictionary ESACCI ESACCI_Conversions_dict = ESACCI_Conversions() Phenology_pixels_year = np.ones(LUdek.Size) * np.nan Grassland_pixels_year = np.ones(LUdek.Size) * np.nan # Loop over the years for Year in Dates_yearly: Year_start = int(Dates_yearly[0].year) Year_int = int(Year.year) geo = LU.GeoTransform proj = LU.Projection destLUESACCI = RC.reproject_dataset_example(input_file_LU_ESACCI, example_file) LU_ESACCI = destLUESACCI.GetRasterBand(1).ReadAsArray() # Create LUmap LU_Map_WAPOR = np.ones([LU.Size[1], LU.Size[2]]) * np.nan LU_Map_ESACCI = np.ones([LU.Size[1], LU.Size[2]]) * np.nan for number in WAPOR_Conversions_dict.items(): LU_Map_WAPOR = np.where(LU.Data[int((Year_int - Year_start)),: ,:] == number[0], number[1], LU_Map_WAPOR) for number in ESACCI_Conversions_dict.items(): LU_Map_ESACCI = np.where(LU_ESACCI == number[0], number[1], LU_Map_ESACCI) # Combine LU maps # 1 = rainfed, 2 = irrigated, 3 = Pasture LU_END = np.where(np.logical_and(LU_Map_WAPOR == 1, LU_Map_ESACCI == 1), 1, np.nan) LU_END = np.where(LU_Map_WAPOR > 1, LU_Map_WAPOR, LU_END) # Save LU map DC.Save_as_tiff(os.path.join(output_folder_L2, "LU_END", "LU_%s.tif" %Year_int), LU_END, geo, proj) # find posible Perennial pixels Phenology_pixels_year[int((Year_int - Year_start) * 36):int((Year_int - Year_start) * 36)+36,: ,:] = np.where(np.logical_or(LU_END==1, LU_END==2), 1, np.nan)[None, :, :] Grassland_pixels_year[int((Year_int - Year_start) * 36):int((Year_int - Year_start) * 36)+36,: ,:] = np.where(LU_END==3, 1, np.nan)[None, :, :] return(Phenology_pixels_year, Grassland_pixels_year)
def RetrieveData(args): """ This function retrieves JRC data for a given date from the http://storage.googleapis.com/global-surface-water/downloads/ server. Keyword arguments: args -- A list of parameters defined in the DownloadData function. """ # Argument [output_folder, Names_to_download, lonlim, latlim] = args # Collect the data from the JRC webpage and returns the data and lat and long in meters of those tiles try: Collect_data(Names_to_download, output_folder) except: print("Was not able to download the file") # Clip the data to the users extend if len(Names_to_download) == 1: trash_folder = os.path.join(output_folder, "Trash") data_in = os.path.join(trash_folder, Names_to_download[0]) data_end, geo_end = RC.clip_data(data_in, latlim, lonlim) else: data_end = np.zeros([int((latlim[1] - latlim[0])/0.00025), int((lonlim[1] - lonlim[0])/0.00025)]) for Name_to_merge in Names_to_download: trash_folder = os.path.join(output_folder, "Trash") data_in = os.path.join(trash_folder, Name_to_merge) geo_out, proj, size_X, size_Y = RC.Open_array_info(data_in) lat_min_merge = np.maximum(latlim[0], geo_out[3] + size_Y * geo_out[5]) lat_max_merge = np.minimum(latlim[1], geo_out[3]) lon_min_merge = np.maximum(lonlim[0], geo_out[0]) lon_max_merge = np.minimum(lonlim[1], geo_out[0] + size_X * geo_out[1]) lonmerge = [lon_min_merge, lon_max_merge] latmerge = [lat_min_merge, lat_max_merge] data_one, geo_one = RC.clip_data(data_in, latmerge, lonmerge) Ystart = int((geo_one[3] - latlim[1])/geo_one[5]) Yend = int(Ystart + np.shape(data_one)[0]) Xstart = int((geo_one[0] - lonlim[0])/geo_one[1]) Xend = int(Xstart + np.shape(data_one)[1]) data_end[Ystart:Yend, Xstart:Xend] = data_one geo_end = tuple([lonlim[0], geo_one[1], 0, latlim[1], 0, geo_one[5]]) # Save results as Gtiff fileName_out = os.path.join(output_folder, 'JRC_Occurrence_percent.tif') DC.Save_as_tiff(name=fileName_out, data=data_end, geo=geo_end, projection='WGS84') shutil.rmtree(trash_folder) return True
def RetrieveData(Date, args): """ This function retrieves MOD9 Reflectance data for a given date from the http://e4ftl01.cr.usgs.gov/ server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [ output_folder, TilesVertical, TilesHorizontal, lonlim, latlim, band, resolution, hdf_library ] = args ReffileName = os.path.join( output_folder, 'ReflectanceBand%d_MOD09GQ_-_daily_' % band + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif') if not os.path.exists(ReffileName): # Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles try: Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, band, resolution, hdf_library) except: print("Was not able to download the file") # Define the output name of the collect data function name_collect = os.path.join(output_folder, 'Merged.tif') try: # Reproject the MODIS product to epsg_to epsg_to = '4326' name_reprojected = RC.reproject_MODIS(name_collect, epsg_to) # Clip the data to the users extend data, geo = RC.clip_data(name_reprojected, latlim, lonlim) # Save results as Gtiff DC.Save_as_tiff(name=ReffileName, data=data, geo=geo, projection='WGS84') # remove the side products os.remove(os.path.join(output_folder, name_collect)) os.remove(os.path.join(output_folder, name_reprojected)) except: print('data for %02d-%02d-%d is not available' % (Date.day, Date.month, Date.year)) return True
def Calc_Humidity(Temp_format, P_format, Hum_format, output_format, Startdate, Enddate, freq="D"): folder_dir_out = os.path.dirname(output_format) if not os.path.exists(folder_dir_out): os.makedirs(folder_dir_out) Dates = pd.date_range(Startdate, Enddate, freq="D") for Date in Dates: print(Date) Day = Date.day Month = Date.month Year = Date.year Tempfile_one = Temp_format.format(yyyy=Year, mm=Month, dd=Day) Presfile_one = P_format.format(yyyy=Year, mm=Month, dd=Day) Humfile_one = Hum_format.format(yyyy=Year, mm=Month, dd=Day) out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day) geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one) Tdata = RC.Open_tiff_array(Tempfile_one) if "MERRA_K" in Temp_format: Tdata = Tdata - 273.15 Tdata[Tdata < -900] = -9999 Pdata = RC.Open_tiff_array(Presfile_one) Hdata = RC.Open_tiff_array(Humfile_one) Pdata[Pdata < 0] = -9999 Hdata[Hdata < 0] = -9999 # gapfilling Tdata = RC.gap_filling(Tdata, -9999) Pdata = RC.gap_filling(Pdata, -9999) Hdata = RC.gap_filling(Hdata, -9999) Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3)) HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100 HumData = HumData.clip(0, 100) DC.Save_as_tiff(out_folder_one, HumData, geo_out, "WGS84") return ()
def DownloadData(Dir, latlim, lonlim, Waitbar): """ This function downloads NLDAS Forcing data hourly, daily or monthly data Keyword arguments: Dir -- 'C:/file/to/path/' latlim -- [ymin, ymax] lonlim -- [xmin, xmax] """ # Define the output name output_filename = os.path.join(Dir, 'LU_ESACCI.tif') # Set the url of the server url = r"https://storage.googleapis.com/cci-lc-v207/ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.zip" # Create a Trash folder Dir_trash = os.path.join(Dir, "Trash") if not os.path.exists(Dir_trash): os.makedirs(Dir_trash) # Define location of download filename_out = os.path.join( Dir_trash, "ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.zip") # Download data urllib.request.urlretrieve(url, filename=filename_out) # Extract data DC.Extract_Data(filename_out, Dir_trash) # Define input of the world tiff file filename_world = os.path.join( Dir_trash, "product", "ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.tif") try: # Clip data to user extend data, Geo_out = RC.clip_data(filename_world, latlim, lonlim) # Save data of clipped array DC.Save_as_tiff(output_filename, data, Geo_out, 4326) except: RC.Clip_Dataset_GDAL(RC.clip_data(filename_world, latlim, lonlim)) # Remove trash folder shutil.rmtree(Dir_trash) return ()
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join( Dir, 'SoilGrids', 'Theta_Sat', 'Theta_Sat_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat.Subsoil( Dir, latlim, lonlim) filedir_out_whc = os.path.join(Dir, 'SoilGrids', 'Water_Holding_Capacity') if not os.path.exists(filedir_out_whc): os.makedirs(filedir_out_whc) # Define theta field capacity output filename_out_whc = os.path.join( filedir_out_whc, 'Water_Holding_Capacity_%s_SoilGrids_mm-m.tif' % level) if not os.path.exists(filename_out_whc): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info( filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate theta field capacity theta_whc = np.ones(theta_sat.shape) * -9999 theta_whc = np.where( theta_sat < 0.301, 80, 450 * np.arccosh(theta_sat + 0.7) - 2 * (theta_sat + 0.7) + 20) # Save as tiff DC.Save_as_tiff(filename_out_whc, theta_whc, geo_out, proj) return
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join( Dir, 'SoilGrids', 'Theta_Sat', 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat2.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat2.Subsoil( Dir, latlim, lonlim) filedir_out_n_genuchten = os.path.join(Dir, 'SoilGrids', 'N_van_genuchten') if not os.path.exists(filedir_out_n_genuchten): os.makedirs(filedir_out_n_genuchten) # Define n van genuchten output filename_out_ngenuchten = os.path.join( filedir_out_n_genuchten, 'N_genuchten_%s_SoilGrids_-.tif' % level) if not os.path.exists(filename_out_ngenuchten): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info( filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate n van genuchten n_van_genuchten = np.ones(theta_sat.shape) * -9999 n_van_genuchten = 166.63 * theta_sat**4 - 387.72 * theta_sat**3 + 340.55 * theta_sat**2 - 133.07 * theta_sat + 20.739 # Save as tiff DC.Save_as_tiff(filename_out_ngenuchten, n_van_genuchten, geo_out, proj) return
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim): """ This function retrieves GWF data for a given date from the ftp.wateraccounting.unesco-ihe.org server. Keyword arguments: output_folder -- name of the end file with the weekly ALEXI data End_filename -- name of the end file lonlim -- [ymin, ymax] (values must be between -60 and 70) latlim -- [xmin, xmax] (values must be between -180 and 180) """ try: # Collect account and FTP information username, password = WebAccounts.Accounts(Type='FTP_WA') ftpserver = "ftp.wateraccounting.unesco-ihe.org" # Set the file names and directories filename = "Gray_Water_Footprint.tif" local_filename = os.path.join(output_folder, filename) # Download data from FTP ftp = FTP(ftpserver) ftp.login(username, password) directory = "/WaterAccounting_Guest/Static_WA_Datasets/" ftp.cwd(directory) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write) lf.close() # Clip extend out of world data dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim) # make geotiff file DC.Save_as_tiff(name=filename_Out, data=dataset, geo=Geo_out, projection="WGS84") # delete old tif file os.remove(local_filename) except: print("file not exists") return
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim): # Open Dataset SEBS_Array = spio.loadmat(local_filename)['ETm'] # Define area XID = [int(np.floor((180 + lonlim[0])/0.05)), int(np.ceil((180 + lonlim[1])/0.05))] YID = [int(np.ceil((90 - latlim[1])/0.05)), int(np.floor((90 - latlim[0])/0.05))] # Define Georeference geo = tuple([-180 + 0.05*XID[0],0.05,0,90 - 0.05*YID[0],0,-0.05]) # Clip Array SEBS_Array_clipped = SEBS_Array[YID[0]:YID[1], XID[0]:XID[1]] * 0.1 # Save tiff file DC.Save_as_tiff(Filename_out, SEBS_Array_clipped, geo, "WGS84")
def Calc_Humidity(Temp_format, P_format, Hum_format, output_format, Startdate, Enddate, freq="D"): folder_dir_out = os.path.dirname(output_format) if not os.path.exists(folder_dir_out): os.makedirs(folder_dir_out) Dates = pd.date_range(Startdate, Enddate, freq="D") for Date in Dates: print(Date) Day = Date.day Month = Date.month Year = Date.year Windyfile_one = wind_y_format.format(yyyy=Year, mm=Month, dd=Day) Windxfile_one = wind_x_format.format(yyyy=Year, mm=Month, dd=Day) out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day) geo_out, proj, size_X, size_Y = RC.Open_array_info(Windxfile_one) Windxdata = RC.Open_tiff_array(Windxfile_one) Windxdata[Windxdata < -900] = -9999 Windydata = RC.Open_tiff_array(Windyfile_one) Windydata[Windxdata < -900] = -9999 # gapfilling Windydata = RC.gap_filling(Windydata, -9999) Windxdata = RC.gap_filling(Windxdata, -9999) Wind = np.sqrt(Windxdata**2 + Windydata**2) DC.Save_as_tiff(out_folder_one, Wind, geo_out, "WGS84") return ()
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat' ,'Theta_Sat2_%s_SoilGrids_kg-kg.tif' %level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat2.Topsoil(Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat2.Subsoil(Dir, latlim, lonlim) filedir_out_thetares = os.path.join(Dir, 'SoilGrids', 'Theta_Res') if not os.path.exists(filedir_out_thetares): os.makedirs(filedir_out_thetares) # Define theta field capacity output filename_out_thetares = os.path.join(filedir_out_thetares ,'Theta_Res_%s_SoilGrids_kg-kg.tif' %level) if not os.path.exists(filename_out_thetares): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) # Calculate theta field capacity theta_Res = np.ones(theta_sat.shape) * -9999 #theta_Res = np.where(theta_sat < 0.351, 0.01, 0.4 * np.arccosh(theta_sat + 0.65) - 0.05 * np.power(theta_sat + 0.65, 2.5) + 0.02) theta_Res = np.where(theta_sat < 0.351, 0.01, 0.271 * np.log(theta_sat) + 0.335) # Save as tiff DC.Save_as_tiff(filename_out_thetares, theta_Res, geo_out, proj) return
def lapse_rate(Dir,temperature_map, DEMmap): """ This function downscales the GLDAS temperature map by using the DEM map Keyword arguments: temperature_map -- 'C:/' path to the temperature map DEMmap -- 'C:/' path to the DEM map """ # calculate average altitudes corresponding to T resolution dest = RC.reproject_dataset_example(DEMmap, temperature_map,method = 4) DEM_ave_out_name = os.path.join(Dir,'HydroSHED', 'DEM','DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map) DEM_ave_data = dest.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) dest = None # determine lapse-rate [degress Celcius per meter] lapse_rate_number = 0.0065 # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2) dem_avg=dest.GetRasterBand(1).ReadAsArray() dem_avg[dem_avg<0]=0 dest = None # Open the temperature dataset dest = RC.reproject_dataset_example(temperature_map, DEMmap, method = 2) T=dest.GetRasterBand(1).ReadAsArray() dest = None # Open Demmap demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap<=0]=0 demmap[demmap==-32768]=np.nan # calculate first part T = T + ((dem_avg-demmap) * lapse_rate_number) return T
def Calc_daily_radiation(folder_in, folder_out, Date): filename_trans = "SDS_MSGCPP_W-m-2_15min_%d.%02d.%02d_H{hour}.M{minutes}.tif" % ( Date.year, Date.month, Date.day) filename_trans_out = "SDS_MSGCPP_W-m-2_daily_%d.%02d.%02d.tif" % ( Date.year, Date.month, Date.day) Dir_trans_out = os.path.join(folder_out, filename_trans_out) if not os.path.exists(Dir_trans_out): os.chdir(folder_in) files = glob.glob(filename_trans.format(hour="*", minutes="*")) i = 0 # Open all the 15 minutes files for file in files: file_in = os.path.join(folder_in, file) destswgone = gdal.Open(file_in) try: swgnet_one = destswgone.GetRasterBand(1).ReadAsArray() swgnet_one[swgnet_one < 0] = 0 if not "geo_trans" in locals(): swgnet = np.ones([ destswgone.RasterYSize, destswgone.RasterXSize, len(files) ]) * np.nan geo_trans = destswgone.GetGeoTransform() proj_trans = destswgone.GetProjection() swgnet[:, :, i] = swgnet_one except: pass i += 1 # Calculate the daily mean swgnet_mean = np.nansum(swgnet, 2) / (24 * 4) DC.Save_as_tiff(Dir_trans_out, swgnet_mean, geo_trans, proj_trans) return ()
def adjust_P(Dir, pressure_map, DEMmap): """ This function downscales the GLDAS air pressure map by using the DEM map Keyword arguments: pressure_map -- 'C:/' path to the pressure map DEMmap -- 'C:/' path to the DEM map """ # calculate average latitudes destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method = 4) DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM','DEM_ave.tif') geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map) DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray() DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj) # open maps as numpy arrays dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2) dem_avg=dest.GetRasterBand(1).ReadAsArray() dest = None # open maps as numpy arrays dest = RC.reproject_dataset_example(pressure_map, DEMmap, method = 2) P=dest.GetRasterBand(1).ReadAsArray() dest = None demmap = RC.Open_tiff_array(DEMmap) dem_avg[demmap<=0]=0 demmap[demmap==-32768]=np.nan # calculate second part P = P + (101.3*((293-0.0065*(demmap-dem_avg))/293)**5.26 - 101.3) os.remove(DEM_ave_out_name) return P
def Calc_Humidity(output_folder_L1, output_filename, method): # find date year = int((output_filename.split("_")[-1]).split(".")[0]) month = int((output_filename.split("_")[-1]).split(".")[1]) day = int((output_filename.split("_")[-1]).split(".")[2]) if method == "Daily": # find paths file_t = os.path.join( output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(), "tair_f_inst", "mean", "Tair_GLDAS-NOAH_C_%s_%d.%02d.%02d.tif" % (method.lower(), year, month, day)) file_h = os.path.join( output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(), "qair_f_inst", "mean", "Hum_GLDAS-NOAH_kg-kg_%s_%d.%02d.%02d.tif" % (method.lower(), year, month, day)) file_p = os.path.join( output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(), "psurf_f_inst", "mean", "P_GLDAS-NOAH_kpa_%s_%d.%02d.%02d.tif" % (method.lower(), year, month, day)) if method == "Monthly": # find paths file_t = os.path.join( output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(), "tair_f_inst", "Tair_GLDAS-NOAH_C_%s_%d.%02d.%02d.tif" % (method.lower(), year, month, day)) file_h = os.path.join( output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(), "qair_f_inst", "Hum_GLDAS-NOAH_kg-kg_%s_%d.%02d.%02d.tif" % (method.lower(), year, month, day)) file_p = os.path.join( output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(), "psurf_f_inst", "P_GLDAS-NOAH_kpa_%s_%d.%02d.%02d.tif" % (method.lower(), year, month, day)) # Open paths destt = gdal.Open(file_t) desth = gdal.Open(file_h) destp = gdal.Open(file_p) # Get Geo information geo = destt.GetGeoTransform() proj = destt.GetProjection() # Open Arrays Array_T = destt.GetRasterBand(1).ReadAsArray() Array_H = desth.GetRasterBand(1).ReadAsArray() Array_P = destp.GetRasterBand(1).ReadAsArray() Array_T[Array_T == -9999] = np.nan Array_H[Array_H == -9999] = np.nan Array_P[Array_P == -9999] = np.nan # Calculate relative humidity ESArray = 0.6108 * np.exp((17.27 * Array_T) / (Array_T + 237.3)) RHarray = np.minimum((1.6077717 * Array_H * Array_P / ESArray), 1) * 100 # Save array DC.Save_as_tiff(output_filename, RHarray, geo, proj) return ()
def DownloadData(Dir, Startdate, Enddate, S2_tile): # Import watertools modules import watertools.General.data_conversions as DC # Define the dates Dates = pd.date_range(Startdate, Enddate, freq="D") # Create output folder output_folder_end = os.path.join(Dir, S2_tile) if not os.path.exists(output_folder_end): os.makedirs(output_folder_end) # Loop over the days for Date in Dates: # Get the datum doy = int(Date.dayofyear) year = Date.year try: # Create the right url url = "https://hls.gsfc.nasa.gov/data/v1.4/L30/%s/%s/%s/%s/%s/HLS.L30.T%s.%s%03d.v1.4.hdf" % ( year, S2_tile[0:2], S2_tile[2:3], S2_tile[3:4], S2_tile[4:5], S2_tile, year, doy) filename_out = os.path.join( output_folder_end, "HLS.L30.T%s.%s%03d.v1.4.hdf" % (S2_tile, year, doy)) # Download the data urllib.request.urlretrieve(url, filename=filename_out) # Create a folder for the end results folder_tile = os.path.join( output_folder_end, "HLS_L30_T%s_%s%03d_v1_4" % (S2_tile, year, doy)) if not os.path.exists(folder_tile): os.makedirs(folder_tile) # Write the hdf file in tiff files and store it in the folder dataset = gdal.Open(filename_out) sdsdict = dataset.GetMetadata('SUBDATASETS') for Band in range(1, 12): dest = gdal.Open(sdsdict["SUBDATASET_%d_NAME" % Band]) Array = dest.GetRasterBand(1).ReadAsArray() if Band < 8.: Array = Array * 0.0001 Array[Array == -0.1] = -9999. Band_name = "B%02d" % (Band) if Band == 8.: Band_name = "B%02d" % (Band + 1) Array = Array * 0.0001 Array[Array == -0.1] = -9999. if (Band >= 9. and Band < 11.): Band_name = "B%02d" % (Band + 1) Array = Array * 0.01 Array[Array == -100] = -9999. if Band == 11.: Array[Array == 255] = -9999. Band_name = "QC" meta = dataset.GetMetadata() ulx = int(meta["ULX"]) uly = int(meta["ULY"]) size = int(meta["SPATIAL_RESOLUTION"]) projection_zone = int( meta["HORIZONTAL_CS_NAME"].split(" ")[-1]) projection = dest.GetProjection() time_string = meta["SENSING_TIME"].split(";")[-1][:-9] import datetime Time = datetime.datetime.strptime(time_string.replace(" ", ""), "%Y-%m-%dT%H:%M:%S") hour = int(Time.hour) minute = int(Time.minute) geo = tuple([ulx, size, 0, uly, 0, -size]) DC.Save_as_tiff( os.path.join( folder_tile, "HLS_L30_T%s_%s%03d_H%02dM%02d_%s.tif" % (S2_tile, year, doy, hour, minute, Band_name)), Array, geo, projection) except: pass return ()
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None): """ This functions calculates monthly tiff files based on the 8 daily tiff files. (will calculate the average) Parameters ---------- Dir_in : str Path to the input data Startdate : str Contains the start date of the model 'yyyy-mm-dd' Enddate : str Contains the end date of the model 'yyyy-mm-dd' Dir_out : str Path to the output data, default is same as Dir_in """ # import WA+ modules import watertools.General.data_conversions as DC import watertools.General.raster_conversions as RC # Change working directory os.chdir(Dir_in) # Find all eight daily files files = glob.glob('*8-daily*.tif') # Create array with filename and keys (DOY and year) of all the 8 daily files i = 0 DOY_Year = np.zeros([len(files), 3]) for File in files: # Get the time characteristics from the filename year = File.split('.')[-4][-4:] month = File.split('.')[-3] day = File.split('.')[-2] # Create pandas Timestamp date_file = '%s-%02s-%02s' % (year, month, day) Datum = pd.Timestamp(date_file) # Get day of year DOY = Datum.strftime('%j') # Save data in array DOY_Year[i, 0] = i DOY_Year[i, 1] = DOY DOY_Year[i, 2] = year # Loop over files i += 1 # Check enddate: Enddate_split = Enddate.split('-') month_range = calendar.monthrange(int(Enddate_split[0]), int(Enddate_split[1]))[1] Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int( Enddate_split[1]), month_range) # Check startdate: Startdate_split = Startdate.split('-') Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int( Startdate_split[1])) # Define end and start date Dates = pd.date_range(Startdate, Enddate, freq='MS') DatesEnd = pd.date_range(Startdate, Enddate, freq='M') # Get array information and define projection geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0]) if int(proj.split('"')[-2]) == 4326: proj = "WGS84" # Get the No Data Value dest = gdal.Open(files[0]) NDV = dest.GetRasterBand(1).GetNoDataValue() # Loop over months and create monthly tiff files i = 0 for date in Dates: # Get Start and end DOY of the current month DOY_month_start = date.strftime('%j') DOY_month_end = DatesEnd[i].strftime('%j') # Search for the files that are between those DOYs year = date.year DOYs = DOY_Year[DOY_Year[:, 2] == year] DOYs_oneMonth = DOYs[np.logical_and( (DOYs[:, 1] + 8) >= int(DOY_month_start), DOYs[:, 1] <= int(DOY_month_end))] # Create empty arrays Monthly = np.zeros([size_Y, size_X]) Weight_tot = np.zeros([size_Y, size_X]) Data_one_month = np.ones([size_Y, size_X]) * np.nan # Loop over the files that are within the DOYs for EightDays in DOYs_oneMonth[:, 1]: # Calculate the amount of days in this month of each file Weight = np.ones([size_Y, size_X]) # For start of month if np.min(DOYs_oneMonth[:, 1]) == EightDays: Weight = Weight * int(EightDays + 8 - int(DOY_month_start)) # For end of month elif np.max(DOYs_oneMonth[:, 1]) == EightDays: Weight = Weight * (int(DOY_month_end) - EightDays + 1) # For the middle of the month else: Weight = Weight * 8 row = DOYs_oneMonth[np.argwhere( DOYs_oneMonth[:, 1] == EightDays)[0][0], :][0] # Open the array of current file input_name = os.path.join(Dir_in, files[int(row)]) Data = RC.Open_tiff_array(input_name) # Remove NDV Weight[Data == NDV] = 0 Data[Data == NDV] = np.nan # Multiply weight time data Data = Data * Weight # Calculate the total weight and data Weight_tot += Weight Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)] # Go to next month i += 1 # Calculate the average Data_one_month[Weight_tot != 0.] = Monthly[ Weight_tot != 0.] / Weight_tot[Weight_tot != 0.] # Define output directory if Dir_out == None: Dir_out = Dir_in # Define output name output_name = os.path.join( Dir_out, files[int(row)].replace('8-daily', 'monthly')) output_name = output_name[:-9] + '%02d.01.tif' % (date.month) # Save tiff file DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj) return
for Date in Dates: Day = Date.day Month = Date.month Year = Date.year Tempfile_one = Temp_folder.format(yyyy=Year, mm=Month, dd=Day) Presfile_one = Pres_folder.format(yyyy=Year, mm=Month, dd=Day) Humfile_one = Hum_folder.format(yyyy=Year, mm=Month, dd=Day) out_folder_one = out_folder.format(yyyy=Year, mm=Month, dd=Day) geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one) Tdata = RC.Open_tiff_array(Tempfile_one) Tdata[Tdata < -900] = -9999 Pdata = RC.Open_tiff_array(Presfile_one) Hdata = RC.Open_tiff_array(Humfile_one) Pdata[Pdata < 0] = -9999 Hdata[Hdata < 0] = -9999 # gapfilling Tdata = RC.gap_filling(Tdata, -9999) Pdata = RC.gap_filling(Pdata, -9999) Hdata = RC.gap_filling(Hdata, -9999) Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3)) HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100 HumData = HumData.clip(0, 100) DC.Save_as_tiff(out_folder_one, HumData, geo_out, "WGS84")
def DownloadData(output_folder, latlim, lonlim): """ This function downloads DEM data from SRTM Keyword arguments: output_folder -- directory of the result latlim -- [ymin, ymax] (values must be between -60 and 60) lonlim -- [xmin, xmax] (values must be between -180 and 180) """ # Check the latitude and longitude and otherwise set lat or lon on greatest extent if latlim[0] < -60 or latlim[1] > 60: print( 'Latitude above 60N or below 60S is not possible. Value set to maximum' ) latlim[0] = np.max(latlim[0], -60) latlim[1] = np.min(latlim[1], 60) if lonlim[0] < -180 or lonlim[1] > 180: print( 'Longitude must be between 180E and 180W. Now value is set to maximum' ) lonlim[0] = np.max(lonlim[0], -180) lonlim[1] = np.min(lonlim[1], 180) # converts the latlim and lonlim into names of the tiles which must be # downloaded name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim) # Memory for the map x and y shape (starts with zero) size_X_tot = 0 size_Y_tot = 0 nameResults = [] # Create a temporary folder for processing output_folder_trash = os.path.join(output_folder, "Temp") if not os.path.exists(output_folder_trash): os.makedirs(output_folder_trash) # Download, extract, and converts all the files to tiff files for nameFile in name: try: # Download the data from # http://earlywarning.usgs.gov/hydrodata/ output_file, file_name = Download_Data(nameFile, output_folder_trash) # extract zip data DC.Extract_Data(output_file, output_folder_trash) # The input is the file name and in which directory the data must be stored file_name_tiff = file_name.replace(".zip", ".tif") output_tiff = os.path.join(output_folder_trash, file_name_tiff) # convert data from adf to a tiff file dest_SRTM = gdal.Open(output_tiff) geo_out = dest_SRTM.GetGeoTransform() size_X = dest_SRTM.RasterXSize size_Y = dest_SRTM.RasterYSize if (int(size_X) != int(6001) or int(size_Y) != int(6001)): data = np.ones((6001, 6001)) * -9999 # Create the latitude bound Vfile = nameFile.split("_")[2][0:2] Bound2 = 60 - 5 * (int(Vfile) - 1) # Create the longitude bound Hfile = nameFile.split("_")[1] Bound1 = -180 + 5 * (int(Hfile) - 1) Expected_X_min = Bound1 Expected_Y_max = Bound2 Xid_start = int( np.round((geo_out[0] - Expected_X_min) / geo_out[1])) Xid_end = int( np.round( ((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) / geo_out[1])) Yid_start = int( np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5]))) Yid_end = int( np.round((Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5]))) / (-geo_out[5]))) data_SRTM = dest_SRTM.GetRasterBand(1).ReadAsArray() data[Yid_start:Yid_end, Xid_start:Xid_end] = data_SRTM if np.max(data) == 255: data[data == 255] = -9999 data[data < -9999] = -9999 geo_in = [ Bound1 - 0.5 * 0.00083333333333333, 0.00083333333333333, 0.0, Bound2 + 0.5 * 0.00083333333333333, 0.0, -0.0008333333333333333333 ] # save chunk as tiff file destDEM = DC.Save_as_MEM(data=data, geo=geo_in, projection="WGS84") dest_SRTM = None except: # If tile not exist create a replacing zero tile (sea tiles) file_name_tiff = file_name.replace(".zip", ".tif") output_tiff = os.path.join(output_folder_trash, file_name_tiff) file_name = nameFile data = np.ones((6001, 6001)) * -9999 data = data.astype(np.float32) # Create the latitude bound Vfile = nameFile.split("_")[2][0:2] Bound2 = 60 - 5 * (int(Vfile) - 1) # Create the longitude bound Hfile = nameFile.split("_")[1] Bound1 = -180 + 5 * (int(Hfile) - 1) # Geospatial data for the tile geo_in = [ Bound1 - 0.5 * 0.00083333333333333, 0.00083333333333333, 0.0, Bound2 + 0.5 * 0.00083333333333333, 0.0, -0.0008333333333333333333 ] # save chunk as tiff file destDEM = DC.Save_as_MEM(data=data, geo=geo_in, projection="WGS84") # clip data Data, Geo_data = RC.clip_data(destDEM, latlim, lonlim) size_Y_out = int(np.shape(Data)[0]) size_X_out = int(np.shape(Data)[1]) # Total size of the product so far size_Y_tot = int(size_Y_tot + size_Y_out) size_X_tot = int(size_X_tot + size_X_out) if nameFile is name[0]: Geo_x_end = Geo_data[0] Geo_y_end = Geo_data[3] else: Geo_x_end = np.min([Geo_x_end, Geo_data[0]]) Geo_y_end = np.max([Geo_y_end, Geo_data[3]]) # create name for chunk FileNameEnd = "%s_temporary.tif" % (file_name) nameForEnd = os.path.join(output_folder_trash, FileNameEnd) nameResults.append(str(nameForEnd)) # save chunk as tiff file DC.Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data, projection="WGS84") size_X_end = int(size_X_tot / len(rangeLat)) # + 1 !!! size_Y_end = int(size_Y_tot / len(rangeLon)) # + 1 !!! # Define the georeference of the end matrix geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]] latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]] lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end] # merge chunk together resulting in 1 tiff map datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end, size_X_end) datasetTot[datasetTot < -9999] = -9999 # name of the end result output_DEM_name = "DEM_SRTM_m_3s.tif" Save_name = os.path.join(output_folder, output_DEM_name) # Make geotiff file DC.Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84") os.chdir(output_folder)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar): # Create an array with the dates that will be calculated Dates = pd.date_range(Startdate, Enddate, freq='MS') # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Define the minimum and maximum lat and long ETensemble Tile Min_lat_tile = int(np.floor((100 - latlim[1]) / 10)) Max_lat_tile = int(np.floor((100 - latlim[0] - 0.00125) / 10)) Min_lon_tile = int(np.floor((190 + lonlim[0]) / 10)) Max_lon_tile = int(np.floor((190 + lonlim[1] - 0.00125) / 10)) # Create the Lat and Lon tiles that will be downloaded Lat_tiles = [Min_lat_tile, Max_lat_tile] Lon_tiles = [Min_lon_tile, Max_lon_tile] # Define output folder and create this if it not exists output_folder = os.path.join(Dir, 'Evaporation', 'ETensV1_0') if not os.path.exists(output_folder): os.makedirs(output_folder) # Create Geotransform of the output files GEO_1 = lonlim[0] GEO_2 = 0.0025 GEO_3 = 0.0 GEO_4 = latlim[1] GEO_5 = 0.0 GEO_6 = -0.0025 geo = [GEO_1, GEO_2, GEO_3, GEO_4, GEO_5, GEO_6] geo_new = tuple(geo) # Define the parameter for downloading the data Downloaded = 0 # Calculate the ET data date by date for Date in Dates: # Define the output name and folder file_name = 'ET_ETensemble250m_mm-month-1_monthly_%d.%02d.01.tif' % ( Date.year, Date.month) output_file = os.path.join(output_folder, file_name) # If output file not exists create this if not os.path.exists(output_file): # If not downloaded than download if Downloaded == 0: # Download the ETens data from the FTP server Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles) # Unzip the folder Unzip_ETens_data(output_folder, Lat_tiles, Lon_tiles) Downloaded = 1 # Create the ET data for the area of interest ET_data = Collect_dataset(output_folder, Date, Lat_tiles, Lon_tiles, latlim, lonlim) # Save this array as a tiff file DC.Save_as_tiff(output_file, ET_data, geo_new, projection='WGS84') # Create Waitbar if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) ''' # Remove all the raw dataset for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1): for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1): Tilename = "h%sv%s" %(h_tile, v_tile) filename = os.path.join(output_folder, Tilename) if os.path.exists(filename): shutil.rmtree(filename) # Remove all .zip files for f in os.listdir(output_folder): if re.search(".zip", f): os.remove(os.path.join(output_folder, f)) ''' return ()
def Calc_Property(Dir, latlim, lonlim, SL): import watertools # Define level if SL == "sl3": level = "Topsoil" elif SL == "sl6": level = "Subsoil" # check if you need to download filename_out_thetasat = os.path.join( Dir, 'SoilGrids', 'Theta_Sat', 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetasat): if SL == "sl3": watertools.Products.SoilGrids.Theta_Sat2.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Sat2.Subsoil( Dir, latlim, lonlim) filename_out_thetares = os.path.join( Dir, 'SoilGrids', 'Theta_Res', 'Theta_Res_%s_SoilGrids_kg-kg.tif' % level) if not os.path.exists(filename_out_thetares): if SL == "sl3": watertools.Products.SoilGrids.Theta_Res.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.Theta_Res.Subsoil( Dir, latlim, lonlim) filename_out_n_genuchten = os.path.join( Dir, 'SoilGrids', 'N_van_genuchten', 'N_genuchten_%s_SoilGrids_-.tif' % level) if not os.path.exists(filename_out_n_genuchten): if SL == "sl3": watertools.Products.SoilGrids.n_van_genuchten.Topsoil( Dir, latlim, lonlim) elif SL == "sl6": watertools.Products.SoilGrids.n_van_genuchten.Subsoil( Dir, latlim, lonlim) filedir_out_thetafc = os.path.join(Dir, 'SoilGrids', 'Theta_FC') if not os.path.exists(filedir_out_thetafc): os.makedirs(filedir_out_thetafc) # Define theta field capacity output filename_out_thetafc = os.path.join( filedir_out_thetafc, 'Theta_FC2_%s_SoilGrids_cm3-cm3.tif' % level) if not os.path.exists(filename_out_thetafc): # Get info layer geo_out, proj, size_X, size_Y = RC.Open_array_info( filename_out_thetasat) # Open dataset theta_sat = RC.Open_tiff_array(filename_out_thetasat) theta_res = RC.Open_tiff_array(filename_out_thetares) n_genuchten = RC.Open_tiff_array(filename_out_n_genuchten) # Calculate theta field capacity theta_FC = np.ones(theta_sat.shape) * -9999 #theta_FC = np.where(theta_sat < 0.301, 0.042, np.arccosh(theta_sat + 0.7) - 0.32 * (theta_sat + 0.7) + 0.2) #theta_FC = np.where(theta_sat < 0.301, 0.042, -2.95*theta_sat**2+3.96*theta_sat-0.871) theta_FC = theta_res + (theta_sat - theta_res) / ( 1 + (0.02 * 200)**n_genuchten)**(1 - 1 / n_genuchten) # Save as tiff DC.Save_as_tiff(filename_out_thetafc, theta_FC, geo_out, proj) return
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar): """ This scripts downloads ASCAT SWI data from the VITO server. The output files display the Surface Water Index. Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' lonlim -- [ymin, ymax] latlim -- [xmin, xmax] """ # Check the latitude and longitude and otherwise reset lat and lon. if latlim[0] < -90 or latlim[1] > 90: print('Latitude above 90N or below 90S is not possible.\ Value set to maximum') latlim[0] = np.max(latlim[0], -90) latlim[1] = np.min(latlim[1], 90) if lonlim[0] < -180 or lonlim[1] > 180: print('Longitude must be between 180E and 180W.\ Now value is set to maximum') lonlim[0] = np.max(lonlim[0], -180) lonlim[1] = np.min(lonlim[1], 180) # Check Startdate and Enddate if not Startdate: Startdate = pd.Timestamp('2007-01-01') if not Enddate: Enddate = pd.Timestamp.now() # Make a panda timestamp of the date try: Enddate = pd.Timestamp(Enddate) except: Enddate = Enddate # amount of Dates weekly Dates = pd.date_range(Startdate, Enddate, freq='D') # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Define directory and create it if not exists output_folder = os.path.join(Dir, 'SWI', 'ASCAT', 'Daily') if not os.path.exists(output_folder): os.makedirs(output_folder) output_folder_temp = os.path.join(Dir, 'SWI', 'ASCAT', 'Daily', 'Temp') if not os.path.exists(output_folder_temp): os.makedirs(output_folder_temp) # loop over dates for Date in Dates: # Define end filename End_filename = os.path.join( output_folder, 'SWI_ASCAT_V3_Percentage_daily_%d.%02d.%02d.tif' % (Date.year, Date.month, Date.day)) # Define IDs xID = 1800 + np.int16( np.array([np.ceil((lonlim[0]) * 10), np.floor((lonlim[1]) * 10)])) yID = np.int16( np.array([np.floor((-latlim[1]) * 10), np.ceil((-latlim[0]) * 10)])) + 900 # Download the data from FTP server if the file not exists if not os.path.exists(End_filename): try: data = Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID) # make geotiff file geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1] DC.Save_as_tiff(name=End_filename, data=data, geo=geo, projection="WGS84") except: print("Was not able to download file with date %s" % Date) # Adjust waitbar if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # remove the temporary folder shutil.rmtree(output_folder_temp)