def main(Dir, Startdate = '', Enddate = '', latlim = [-60, 60], lonlim = [-180, 180], pixel_size = False, cores = False, LANDSAF = 0, SourceLANDSAF= '', Waitbar = 1): """ This function downloads TRMM3B43 V7 (monthly) data Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) cores -- The number of cores used to run the routine. It can be 'False' to avoid using parallel computing routines. Waitbar -- 1 (Default) will print the waitbar """ print('Create monthly Reference ET data for period %s till %s' %(Startdate, Enddate)) # An array of monthly dates which will be calculated Dates = pd.date_range(Startdate,Enddate,freq = 'MS') # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50) # Calculate the ETref day by day for every month for Date in Dates: # Collect date data Y=Date.year M=Date.month Mday=calendar.monthrange(Y,M)[1] Days=pd.date_range(Date,Date+pd.Timedelta(days=Mday),freq='D') StartTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+ '-01' EndTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+'-'+str(Mday) # Get ETref on daily basis daily(Dir=Dir, Startdate=StartTime,Enddate=EndTime,latlim=latlim, lonlim=lonlim, pixel_size = pixel_size, cores=cores, LANDSAF=LANDSAF, SourceLANDSAF=SourceLANDSAF, Waitbar = 0) # Load DEM if not pixel_size: nameDEM='DEM_HydroShed_m_3s.tif' DEMmap=os.path.join(Dir,'HydroSHED','DEM',nameDEM ) else: DEMmap=os.path.join(Dir,'HydroSHED','DEM','DEM_HydroShed_m_reshaped_for_ETref.tif') # Get some geo-data to save results geo_ET, proj, size_X, size_Y = RC.Open_array_info(DEMmap) dataMonth=np.zeros([size_Y,size_X]) for Day in Days[:-1]: DirDay=os.path.join(Dir,'ETref','Daily','ETref_mm-day-1_daily_' + Day.strftime('%Y.%m.%d') + '.tif') dataDay=gdal.Open(DirDay) Dval=dataDay.GetRasterBand(1).ReadAsArray().astype(np.float32) Dval[Dval<0]=0 dataMonth=dataMonth+Dval dataDay=None # make geotiff file output_folder_month=os.path.join(Dir,'ETref','Monthly') if os.path.exists(output_folder_month)==False: os.makedirs(output_folder_month) DirMonth=os.path.join(output_folder_month,'ETref_mm-month-1_monthly_'+Date.strftime('%Y.%m.%d') + '.tif') # Create the tiff file DC.Save_as_tiff(DirMonth,dataMonth, geo_ET, proj) # Create Waitbar if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, timestep, Waitbar, cores, hdf_library, remove_hdf): """ This function downloads MOD13 16-daily data Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -90 and 90) lonlim -- [xmin, xmax] (values must be between -180 and 180) cores -- The number of cores used to run the routine. It can be 'False' to avoid using parallel computing routines. Waitbar -- 1 (Default) will print a waitbar """ # Check start and end date and otherwise set the date if not Startdate: Startdate = pd.Timestamp('2000-01-01') if not Enddate: Enddate = pd.Timestamp('2014-12-31') # Make an array of the days of which the ET is taken if timestep == 'monthly': Dates = pd.date_range(Startdate, Enddate, freq='M') TIMESTEP = 'Monthly' Size_pix = 1 elif timestep == '8-daily': Dates = Make_TimeStamps(Startdate, Enddate) TIMESTEP = '8_Daily' Size_pix = 2 # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Make directory for the MODIS ET data output_folder = os.path.join(Dir, 'Evaporation', 'MOD16', TIMESTEP) if not os.path.exists(output_folder): os.makedirs(output_folder) TilesVertical, TilesHorizontal = watertools.Collect.MOD15.DataAccess.Get_tiles_from_txt( output_folder, hdf_library, latlim, lonlim) # Pass variables to parallel function and run args = [ output_folder, TilesVertical, TilesHorizontal, latlim, lonlim, timestep, hdf_library, Size_pix ] if not cores: for Date in Dates: RetrieveData(Date, args) if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) results = True else: results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args) for Date in Dates) if remove_hdf == 1: # Remove all .hdf files os.chdir(output_folder) files = glob.glob("*.hdf") for f in files: os.remove(os.path.join(output_folder, f)) # Remove all .txt files #files = glob.glob("*.txt") #for f in files: # os.remove(os.path.join(output_folder, f)) return (results)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores, hdf_library, remove_hdf): """ This function downloads MOD13 16-daily data Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -90 and 90) lonlim -- [xmin, xmax] (values must be between -180 and 180) cores -- The number of cores used to run the routine. It can be 'False' to avoid using parallel computing routines. Waitbar -- 1 (Default) will print a waitbar """ # Check start and end date and otherwise set the date to max if not Startdate: Startdate = pd.Timestamp('2000-02-18') if not Enddate: Enddate = pd.Timestamp('Now') # Make an array of the days of which the NDVI is taken Dates = Make_TimeStamps(Startdate, Enddate) # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Check the latitude and longitude and otherwise set lat or lon on greatest extent if latlim[0] < -90 or latlim[1] > 90: print( 'Latitude above 90N or below 90S is not possible. Value set to maximum' ) latlim[0] = np.max(latlim[0], -90) latlim[1] = np.min(latlim[1], 90) if lonlim[0] < -180 or lonlim[1] > 180: print( 'Longitude must be between 180E and 180W. Now value is set to maximum' ) lonlim[0] = np.max(lonlim[0], -180) lonlim[1] = np.min(lonlim[1], 180) # Make directory for the MODIS NDVI data Dir = Dir.replace("/", os.sep) output_folder = os.path.join(Dir, 'NDVI', 'MOD13') if not os.path.exists(output_folder): os.makedirs(output_folder) # Define which MODIS tiles are required TilesVertical, TilesHorizontal = watertools.Collect.MOD15.DataAccess.Get_tiles_from_txt( output_folder, hdf_library, latlim, lonlim) # Pass variables to parallel function and run args = [ output_folder, TilesVertical, TilesHorizontal, latlim, lonlim, hdf_library ] if not cores: for Date in Dates: RetrieveData(Date, args) if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) results = True else: results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args) for Date in Dates) # Remove all .hdf files if remove_hdf == 1: os.chdir(output_folder) files = glob.glob("*.hdf") for f in files: os.remove(os.path.join(output_folder, f)) # Remove all .txt files #files = glob.glob("*.txt") #for f in files: # os.remove(os.path.join(output_folder, f)) return (results)
def DownloadData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar, cores, TimeCase, CaseParameters): """ This function downloads NLDAS Forcing data hourly, daily or monthly data Keyword arguments: Dir -- 'C:/file/to/path/' Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys() Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] lonlim -- [xmin, xmax] cores -- 1....8 CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py """ # Load factors / unit / type of variables / accounts VarInfo = VariablesInfo(TimeCase) username, password = WebAccounts.Accounts(Type='NASA') # Set required data for the three hourly option if TimeCase == 'hourly': # Define output folder and create this one if not exists path = os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase, Var) if not os.path.exists(path): os.makedirs(path) # Startdate if not defined sd_date = '1979-01-02' # Define Time frequency TimeFreq = 'D' # Define URL by using personal account #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password) url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_H.002' #%(username,password) # Name the definition that will be used to obtain the data RetrieveData_fcn = RetrieveData_hourly # Set required data for the daily option elif TimeCase == 'daily': # seperate the daily case parameters SumMean, Min, Max = CaseParameters # Define output folder and create this one if not exists path = { 'mean': os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase, Var, 'mean'), 'min': os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase, Var, 'min'), 'max': os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase, Var, 'max') } selected = np.array([SumMean, Min, Max]) types = np.array(('mean', 'min', 'max'))[selected == 1] CaseParameters = [selected, types] for i in range(len(types)): if not os.path.exists(path[types[i]]): os.makedirs(path[types[i]]) # Startdate if not defined sd_date = '1979-01-02' # Define Time frequency TimeFreq = 'D' # Define URL by using personal account #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password) url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_H.002' #%(username,password) # Name the definition that will be used to obtain the data RetrieveData_fcn = RetrieveData_daily # Set required data for the monthly option elif TimeCase == 'monthly': # Define output folder and create this one if not exists path = os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase, Var) if not os.path.exists(path): os.makedirs(path) CaseParameters = [] # Startdate if not defined sd_date = '1979-02-01' # Define Time frequency TimeFreq = 'MS' # Define URL by using personal account #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025_M' %(username,password) url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_M.002' #%(username,password) # Name the definition that will be used to obtain the data RetrieveData_fcn = RetrieveData_monthly # If none of the possible option are chosen else: raise KeyError("The input time interval is not supported") # Define IDs (latitude/longitude) yID = np.int16( np.array( [np.ceil((latlim[0] - 25) * 8), np.floor((latlim[1] - 25) * 8)])) xID = np.int16( np.array( [np.floor((lonlim[0] + 125) * 8), np.ceil((lonlim[1] + 125) * 8)])) # Check dates. If no dates are given, the max number of days is used. if not Startdate: Startdate = pd.Timestamp(sd_date) if not Enddate: Enddate = pd.Timestamp('Now') # Should be much than available # Create all dates that will be calculated Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq) # Create Waitbar if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Define the variable string name VarStr = VarInfo.names[Var] # Create one parameter with all the required arguments args = [ path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim, CaseParameters, username, password ] # Pass variables to parallel function and run if not cores: for Date in Dates: RetrieveData_fcn(Date, args) if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) results = True else: results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args) for Date in Dates) return results
def main(Dir, latlim, lonlim, level='sl1', Waitbar=1): """ Downloads SoilGrids data from ftp://ftp.soilgrids.org/data/recent/ The following keyword arguments are needed: Dir -- 'C:/file/to/path/' latlim -- [ymin, ymax] lonlim -- [xmin, xmax] level -- 'sl1' (Default) 'sl2' 'sl3' 'sl4' 'sl5' 'sl6' 'sl7' Waitbar -- '1' if you want a waitbar (Default = 1) """ # Create directory if not exists for the output output_folder = os.path.join(Dir, 'SoilGrids', 'Coarse_Fragment_Volumetric') if not os.path.exists(output_folder): os.makedirs(output_folder) # Define the output map and create this if not exists nameEnd = os.path.join( output_folder, 'CoarseFragmentVolumetric_%s_SoilGrids_percentage.tif' % level) if not os.path.exists(nameEnd): # Create Waitbar if Waitbar == 1: print( '\nDownload Coarse Fragment Volumetric soil map of %s from SoilGrids.org' % level) import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = 1 amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Download and process the data DownloadData(output_folder, latlim, lonlim, "CRFVOL", level) if Waitbar == 1: amount = 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) else: if Waitbar == 1: print( "\nCoarse Fragment Volumetric soil map of %s from SoilGrids.org already exists in output folder" % level)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores, TimeCase, Product): """ This function downloads GLEAM ET data Keyword arguments: Dir -- 'C:/file/to/path/' Startdate -- 'yyyy-mm-dd' Enddate -- 'yyyy-mm-dd' latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) cores -- The number of cores used to run the routine. It can be 'False' to avoid using parallel computing routines. Waitbar -- 1 (Default) will print a waitbar """ # Check start and end date and otherwise set the date if not Startdate: Startdate = pd.Timestamp('2003-01-01') if not Enddate: Enddate = pd.Timestamp('2015-12-31') # Make an array of the days of which the ET is taken YearsDownloadstart = str(Startdate[0:4]) YearsDownloadend = str(Enddate[0:4]) Years = range(int(YearsDownloadstart), int(YearsDownloadend) + 1) # String Parameters if TimeCase == 'daily': VarCode = '%s_GLEAM.V3.3b_mm-day-1_daily' % Product FTPprefix = 'data/v3.3b/' TimeFreq = 'D' Folder_name = 'Daily' elif TimeCase == 'monthly': VarCode = '%s_GLEAM.V3.3b_mm-month-1_monthly' % Product FTPprefix = 'data/v3.3b/' TimeFreq = 'M' Folder_name = 'Monthly' # Get end of month for Enddate monthDownloadend = str(Enddate[5:7]) End_month = calendar.monthrange(int(YearsDownloadend), int(monthDownloadend))[1] Enddate = '%d-%02d-%d' % (int(YearsDownloadend), int(monthDownloadend), int(End_month)) else: raise KeyError("The input time interval is not supported") Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq) # Make directory for the MODIS ET data if Product == "ET": output_folder = os.path.join(Dir, 'Evaporation', 'GLEAM', Folder_name) if not os.path.exists(output_folder): os.makedirs(output_folder) if Product == "ETpot": output_folder = os.path.join(Dir, 'Potential_Evapotranspiration', 'GLEAM', Folder_name) if not os.path.exists(output_folder): os.makedirs(output_folder) # Check variables if latlim[0] < -50 or latlim[1] > 50: print('Latitude above 50N or below 50S is not possible.' ' Value set to maximum') latlim[0] = np.max(latlim[0], -50) latlim[1] = np.min(lonlim[1], 50) if lonlim[0] < -180 or lonlim[1] > 180: print('Longitude must be between 180E and 180W.' ' Now value is set to maximum') lonlim[0] = np.max(latlim[0], -180) lonlim[1] = np.min(lonlim[1], 180) # Collect the data from the GLEAM webpage and returns the data and lat and long in meters of those tiles try: Collect_data(FTPprefix, Years, output_folder, Waitbar, Product) except: print("Was not able to download the file") # Create Waitbar print('\nProcess the GLEAM data') if Waitbar == 1: import watertools.Functions.Random.WaitbarConsole as WaitbarConsole total_amount = len(Dates) amount = 0 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) # Pass variables to parallel function and run args = [output_folder, latlim, lonlim, VarCode, TimeCase, Product] if not cores: for Date in Dates: RetrieveData(Date, args) if Waitbar == 1: amount += 1 WaitbarConsole.printWaitBar(amount, total_amount, prefix='Progress:', suffix='Complete', length=50) results = True else: results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args) for Date in Dates) # Remove all .hdf files os.chdir(output_folder) files = glob.glob("*.nc") for f in files: os.remove(os.path.join(output_folder, f)) return (results)