def main(Dir, Startdate = '', Enddate = '',
latlim = [-60, 60], lonlim = [-180, 180], pixel_size = False, cores = False, LANDSAF = 0, SourceLANDSAF= '', Waitbar = 1):
"""
This function downloads TRMM3B43 V7 (monthly) data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine.
It can be 'False' to avoid using parallel computing
routines.
Waitbar -- 1 (Default) will print the waitbar
"""
print('Create monthly Reference ET data for period %s till %s' %(Startdate, Enddate))
# An array of monthly dates which will be calculated
Dates = pd.date_range(Startdate,Enddate,freq = 'MS')
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
# Calculate the ETref day by day for every month
for Date in Dates:
# Collect date data
Y=Date.year
M=Date.month
Mday=calendar.monthrange(Y,M)[1]
Days=pd.date_range(Date,Date+pd.Timedelta(days=Mday),freq='D')
StartTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+ '-01'
EndTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+'-'+str(Mday)
# Get ETref on daily basis
daily(Dir=Dir, Startdate=StartTime,Enddate=EndTime,latlim=latlim, lonlim=lonlim, pixel_size = pixel_size, cores=cores, LANDSAF=LANDSAF, SourceLANDSAF=SourceLANDSAF, Waitbar = 0)
# Load DEM
if not pixel_size:
nameDEM='DEM_HydroShed_m_3s.tif'
DEMmap=os.path.join(Dir,'HydroSHED','DEM',nameDEM )
else:
DEMmap=os.path.join(Dir,'HydroSHED','DEM','DEM_HydroShed_m_reshaped_for_ETref.tif')
# Get some geo-data to save results
geo_ET, proj, size_X, size_Y = RC.Open_array_info(DEMmap)
dataMonth=np.zeros([size_Y,size_X])
for Day in Days[:-1]:
DirDay=os.path.join(Dir,'ETref','Daily','ETref_mm-day-1_daily_' + Day.strftime('%Y.%m.%d') + '.tif')
dataDay=gdal.Open(DirDay)
Dval=dataDay.GetRasterBand(1).ReadAsArray().astype(np.float32)
Dval[Dval<0]=0
dataMonth=dataMonth+Dval
dataDay=None
# make geotiff file
output_folder_month=os.path.join(Dir,'ETref','Monthly')
if os.path.exists(output_folder_month)==False:
os.makedirs(output_folder_month)
DirMonth=os.path.join(output_folder_month,'ETref_mm-month-1_monthly_'+Date.strftime('%Y.%m.%d') + '.tif')
# Create the tiff file
DC.Save_as_tiff(DirMonth,dataMonth, geo_ET, proj)
# Create Waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, timestep, Waitbar,
cores, hdf_library, remove_hdf):
"""
This function downloads MOD13 16-daily data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -90 and 90)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine. It can be 'False'
to avoid using parallel computing routines.
Waitbar -- 1 (Default) will print a waitbar
"""
# Check start and end date and otherwise set the date
if not Startdate:
Startdate = pd.Timestamp('2000-01-01')
if not Enddate:
Enddate = pd.Timestamp('2014-12-31')
# Make an array of the days of which the ET is taken
if timestep == 'monthly':
Dates = pd.date_range(Startdate, Enddate, freq='M')
TIMESTEP = 'Monthly'
Size_pix = 1
elif timestep == '8-daily':
Dates = Make_TimeStamps(Startdate, Enddate)
TIMESTEP = '8_Daily'
Size_pix = 2
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Make directory for the MODIS ET data
output_folder = os.path.join(Dir, 'Evaporation', 'MOD16', TIMESTEP)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
TilesVertical, TilesHorizontal = watertools.Collect.MOD15.DataAccess.Get_tiles_from_txt(
output_folder, hdf_library, latlim, lonlim)
# Pass variables to parallel function and run
args = [
output_folder, TilesVertical, TilesHorizontal, latlim, lonlim,
timestep, hdf_library, Size_pix
]
if not cores:
for Date in Dates:
RetrieveData(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args)
for Date in Dates)
if remove_hdf == 1:
# Remove all .hdf files
os.chdir(output_folder)
files = glob.glob("*.hdf")
for f in files:
os.remove(os.path.join(output_folder, f))
# Remove all .txt files
#files = glob.glob("*.txt")
#for f in files:
# os.remove(os.path.join(output_folder, f))
return (results)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
hdf_library, remove_hdf):
"""
This function downloads MOD13 16-daily data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -90 and 90)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine. It can be 'False'
to avoid using parallel computing routines.
Waitbar -- 1 (Default) will print a waitbar
"""
# Check start and end date and otherwise set the date to max
if not Startdate:
Startdate = pd.Timestamp('2000-02-18')
if not Enddate:
Enddate = pd.Timestamp('Now')
# Make an array of the days of which the NDVI is taken
Dates = Make_TimeStamps(Startdate, Enddate)
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -90 or latlim[1] > 90:
print(
'Latitude above 90N or below 90S is not possible. Value set to maximum'
)
latlim[0] = np.max(latlim[0], -90)
latlim[1] = np.min(latlim[1], 90)
if lonlim[0] < -180 or lonlim[1] > 180:
print(
'Longitude must be between 180E and 180W. Now value is set to maximum'
)
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# Make directory for the MODIS NDVI data
Dir = Dir.replace("/", os.sep)
output_folder = os.path.join(Dir, 'NDVI', 'MOD13')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Define which MODIS tiles are required
TilesVertical, TilesHorizontal = watertools.Collect.MOD15.DataAccess.Get_tiles_from_txt(
output_folder, hdf_library, latlim, lonlim)
# Pass variables to parallel function and run
args = [
output_folder, TilesVertical, TilesHorizontal, latlim, lonlim,
hdf_library
]
if not cores:
for Date in Dates:
RetrieveData(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args)
for Date in Dates)
# Remove all .hdf files
if remove_hdf == 1:
os.chdir(output_folder)
files = glob.glob("*.hdf")
for f in files:
os.remove(os.path.join(output_folder, f))
# Remove all .txt files
#files = glob.glob("*.txt")
#for f in files:
# os.remove(os.path.join(output_folder, f))
return (results)
def DownloadData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
TimeCase, CaseParameters):
"""
This function downloads NLDAS Forcing data hourly, daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
cores -- 1....8
CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py
"""
# Load factors / unit / type of variables / accounts
VarInfo = VariablesInfo(TimeCase)
username, password = WebAccounts.Accounts(Type='NASA')
# Set required data for the three hourly option
if TimeCase == 'hourly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA',
TimeCase, Var)
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-02'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_H.002' #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_hourly
# Set required data for the daily option
elif TimeCase == 'daily':
# seperate the daily case parameters
SumMean, Min, Max = CaseParameters
# Define output folder and create this one if not exists
path = {
'mean':
os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase,
Var, 'mean'),
'min':
os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase,
Var, 'min'),
'max':
os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA', TimeCase,
Var, 'max')
}
selected = np.array([SumMean, Min, Max])
types = np.array(('mean', 'min', 'max'))[selected == 1]
CaseParameters = [selected, types]
for i in range(len(types)):
if not os.path.exists(path[types[i]]):
os.makedirs(path[types[i]])
# Startdate if not defined
sd_date = '1979-01-02'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_H.002' #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Weather_Data', 'Model', 'NLDAS_FORA',
TimeCase, Var)
if not os.path.exists(path):
os.makedirs(path)
CaseParameters = []
# Startdate if not defined
sd_date = '1979-02-01'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025_M' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_M.002' #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
# Define IDs (latitude/longitude)
yID = np.int16(
np.array(
[np.ceil((latlim[0] - 25) * 8),
np.floor((latlim[1] - 25) * 8)]))
xID = np.int16(
np.array(
[np.floor((lonlim[0] + 125) * 8),
np.ceil((lonlim[1] + 125) * 8)]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define the variable string name
VarStr = VarInfo.names[Var]
# Create one parameter with all the required arguments
args = [
path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim,
CaseParameters, username, password
]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results
def main(Dir, latlim, lonlim, level='sl1', Waitbar=1):
"""
Downloads SoilGrids data from ftp://ftp.soilgrids.org/data/recent/
The following keyword arguments are needed:
Dir -- 'C:/file/to/path/'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
level -- 'sl1' (Default)
'sl2'
'sl3'
'sl4'
'sl5'
'sl6'
'sl7'
Waitbar -- '1' if you want a waitbar (Default = 1)
"""
# Create directory if not exists for the output
output_folder = os.path.join(Dir, 'SoilGrids',
'Coarse_Fragment_Volumetric')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Define the output map and create this if not exists
nameEnd = os.path.join(
output_folder,
'CoarseFragmentVolumetric_%s_SoilGrids_percentage.tif' % level)
if not os.path.exists(nameEnd):
# Create Waitbar
if Waitbar == 1:
print(
'\nDownload Coarse Fragment Volumetric soil map of %s from SoilGrids.org'
% level)
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = 1
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Download and process the data
DownloadData(output_folder, latlim, lonlim, "CRFVOL", level)
if Waitbar == 1:
amount = 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
else:
if Waitbar == 1:
print(
"\nCoarse Fragment Volumetric soil map of %s from SoilGrids.org already exists in output folder"
% level)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
TimeCase, Product):
"""
This function downloads GLEAM ET data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine. It can be 'False'
to avoid using parallel computing routines.
Waitbar -- 1 (Default) will print a waitbar
"""
# Check start and end date and otherwise set the date
if not Startdate:
Startdate = pd.Timestamp('2003-01-01')
if not Enddate:
Enddate = pd.Timestamp('2015-12-31')
# Make an array of the days of which the ET is taken
YearsDownloadstart = str(Startdate[0:4])
YearsDownloadend = str(Enddate[0:4])
Years = range(int(YearsDownloadstart), int(YearsDownloadend) + 1)
# String Parameters
if TimeCase == 'daily':
VarCode = '%s_GLEAM.V3.3b_mm-day-1_daily' % Product
FTPprefix = 'data/v3.3b/'
TimeFreq = 'D'
Folder_name = 'Daily'
elif TimeCase == 'monthly':
VarCode = '%s_GLEAM.V3.3b_mm-month-1_monthly' % Product
FTPprefix = 'data/v3.3b/'
TimeFreq = 'M'
Folder_name = 'Monthly'
# Get end of month for Enddate
monthDownloadend = str(Enddate[5:7])
End_month = calendar.monthrange(int(YearsDownloadend),
int(monthDownloadend))[1]
Enddate = '%d-%02d-%d' % (int(YearsDownloadend), int(monthDownloadend),
int(End_month))
else:
raise KeyError("The input time interval is not supported")
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Make directory for the MODIS ET data
if Product == "ET":
output_folder = os.path.join(Dir, 'Evaporation', 'GLEAM', Folder_name)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
if Product == "ETpot":
output_folder = os.path.join(Dir, 'Potential_Evapotranspiration',
'GLEAM', Folder_name)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Check variables
if latlim[0] < -50 or latlim[1] > 50:
print('Latitude above 50N or below 50S is not possible.'
' Value set to maximum')
latlim[0] = np.max(latlim[0], -50)
latlim[1] = np.min(lonlim[1], 50)
if lonlim[0] < -180 or lonlim[1] > 180:
print('Longitude must be between 180E and 180W.'
' Now value is set to maximum')
lonlim[0] = np.max(latlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# Collect the data from the GLEAM webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(FTPprefix, Years, output_folder, Waitbar, Product)
except:
print("Was not able to download the file")
# Create Waitbar
print('\nProcess the GLEAM data')
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Pass variables to parallel function and run
args = [output_folder, latlim, lonlim, VarCode, TimeCase, Product]
if not cores:
for Date in Dates:
RetrieveData(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args)
for Date in Dates)
# Remove all .hdf files
os.chdir(output_folder)
files = glob.glob("*.nc")
for f in files:
os.remove(os.path.join(output_folder, f))
return (results)