def Calc_Property(Dir, latlim, lonlim, SL):
import watools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watools.Products.SoilGrids.Theta_Sat.Topsoil(Dir, latlim, lonlim)
elif SL == "sl6":
watools.Products.SoilGrids.Theta_Sat.Subsoil(Dir, latlim, lonlim)
filedir_out_thetares = os.path.join(Dir, 'SoilGrids', 'Theta_Res')
if not os.path.exists(filedir_out_thetares):
os.makedirs(filedir_out_thetares)
# Define theta field capacity output
filename_out_thetares = os.path.join(
filedir_out_thetares, 'Theta_Res_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetares):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate theta field capacity
theta_Res = np.ones(theta_sat.shape) * -9999
theta_Res = np.where(
theta_sat < 0.351, 0.01, 0.4 * np.arccosh(theta_sat + 0.65) -
0.05 * np.power(theta_sat + 0.65, 2.5) + 0.02)
# Save as tiff
DC.Save_as_tiff(filename_out_thetares, theta_Res, geo_out, proj)
return
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
"""
This function retrieves GWF data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Keyword arguments:
output_folder -- name of the end file with the weekly ALEXI data
End_filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Set the file names and directories
filename = "Gray_Water_Footprint.tif"
local_filename = os.path.join(output_folder, filename)
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Clip extend out of world data
dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
# make geotiff file
DC.Save_as_tiff(name=filename_Out,
data=dataset,
geo=Geo_out,
projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print("file not exists")
return
def RetrieveData(Date, args):
"""
This function retrieves MOD15 FPAR data for a given date from the
http://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim,
hdf_library
] = args
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
hdf_library)
except:
print("Was not able to download the file")
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
#epsg_to ='4326'
name_reprojected = RC.reproject_modis_wgs84(name_collect, method=2)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
# Save the file as tiff
FPARfileName = os.path.join(
output_folder,
'SnowFrac_MOD10_unitless_8-daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
DC.Save_as_tiff(name=FPARfileName, data=data, geo=geo, projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
return True
def split_yield(output_folder, p_fhs, et_blue_fhs, et_green_fhs,
ab=(1.0, 1.0)):
Data_Path_split = "split_y"
out_folder = os.path.join(output_folder, Data_Path_split)
if not os.path.exists(out_folder):
os.mkdir(out_folder)
sp_yield_fhs = []
geo_out, proj, size_X, size_Y = RC.Open_array_info(p_fhs[0])
for m in range(len(p_fhs)):
out_fh = out_folder + '\\split_yield' + et_blue_fhs[m][-12:]
P = RC.Open_tiff_array(p_fhs[m])
ETBLUE = RC.Open_tiff_array(et_blue_fhs[m])
ETGREEN = RC.Open_tiff_array(et_green_fhs[m])
etbfraction = ETBLUE / (ETBLUE + ETGREEN)
pfraction = P / np.nanmax(P)
fraction = sh3.split_Yield(pfraction, etbfraction, ab[0], ab[1])
DC.Save_as_tiff(out_fh, fraction, geo_out)
sp_yield_fhs.append(out_fh)
return sp_yield_fhs
def lapse_rate(Dir, temperature_map, DEMmap):
"""
This function downscales the GLDAS temperature map by using the DEM map
Keyword arguments:
temperature_map -- 'C:/' path to the temperature map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average altitudes corresponding to T resolution
dest = RC.reproject_dataset_example(DEMmap, temperature_map, method=4)
DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map)
DEM_ave_data = dest.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
dest = None
# determine lapse-rate [degress Celcius per meter]
lapse_rate_number = 0.0065
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method=2)
dem_avg = dest.GetRasterBand(1).ReadAsArray()
dem_avg[dem_avg < 0] = 0
dest = None
# Open the temperature dataset
dest = RC.reproject_dataset_example(temperature_map, DEMmap, method=2)
T = dest.GetRasterBand(1).ReadAsArray()
dest = None
# Open Demmap
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap <= 0] = 0
demmap[demmap == -32768] = np.nan
# calculate first part
T = T + ((dem_avg - demmap) * lapse_rate_number)
return T
def RetrieveData(Date, args):
"""
This function retrieves MOD16 ET data for a given date from the
ftp://ftp.ntsg.umt.edu/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TilesVertical, TilesHorizontal,latlim, lonlim, timestep, hdf_library, Size_pix] = args
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal,TilesVertical,Date,output_folder, timestep, hdf_library, Size_pix)
except:
print("Was not able to download the file")
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
epsg_to ='4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
if timestep == 'monthly':
ETfileName = os.path.join(output_folder, 'ET_MOD16A2_mm-month-1_monthly_'+Date.strftime('%Y')+'.' + Date.strftime('%m')+'.01.tif')
elif timestep == '8-daily':
ETfileName = os.path.join(output_folder, 'ET_MOD16A2_mm-8days-1_8-daily_'+Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
DC.Save_as_tiff(name=ETfileName, data=data, geo=geo, projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
return()
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim):
# Open Dataset
SEBS_Array = spio.loadmat(local_filename)['ETm']
# Define area
XID = [
int(np.floor((180 + lonlim[0]) / 0.05)),
int(np.ceil((180 + lonlim[1]) / 0.05))
]
YID = [
int(np.ceil((90 - latlim[1]) / 0.05)),
int(np.floor((90 - latlim[0]) / 0.05))
]
# Define Georeference
geo = tuple([-180 + 0.05 * XID[0], 0.05, 0, 90 - 0.05 * YID[0], 0, -0.05])
# Clip Array
SEBS_Array_clipped = SEBS_Array[YID[0]:YID[1], XID[0]:XID[1]] * 0.1
# Save tiff file
DC.Save_as_tiff(Filename_out, SEBS_Array_clipped, geo, "WGS84")
def monthly_to_yearly(state_date, end_date, in_files, out_file):
month_range = pd.date_range(start=state_date, end=end_date,
freq='MS').strftime("%Y.%m").tolist()
#print(month_range)
files_list = []
data = []
files = glob.glob(in_files)
#print(files)
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
for i in month_range:
if 'P_CHIRPS.v2.0_mm-month-1_monthly_' + i + '.01.tif' in files:
files_list.append('P_CHIRPS.v2.0_mm-month-1_monthly_' + i +
'.01.tif')
else:
print("No such file")
for j in files_list:
photo = Image.open(j)
month = np.array(photo)
data.append(month)
#print(data)
arr_year = np.array(data)
#print(year_sum)
year_sum = arr_year.sum(axis=0)
# Save tiff file
DC.Save_as_tiff(out_file, year_sum, geo_out, proj)
def adjust_P(Dir, pressure_map, DEMmap):
"""
This function downscales the GLDAS air pressure map by using the DEM map
Keyword arguments:
pressure_map -- 'C:/' path to the pressure map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average latitudes
destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method=4)
DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM', 'DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map)
DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method=2)
dem_avg = dest.GetRasterBand(1).ReadAsArray()
dest = None
# open maps as numpy arrays
dest = RC.reproject_dataset_example(pressure_map, DEMmap, method=2)
P = dest.GetRasterBand(1).ReadAsArray()
dest = None
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap <= 0] = 0
demmap[demmap == -32768] = np.nan
# calculate second part
P = P + (101.3 * ((293 - 0.0065 * (demmap - dem_avg)) / 293)**5.26 - 101.3)
os.remove(DEM_ave_out_name)
return P
def Calc_Property(Dir, latlim, lonlim, SL):
import watools.Collect.SoilGrids as SG
# Download needed layers
SG.Clay_Content(Dir, latlim, lonlim, level=SL)
#SG.Organic_Carbon_Content(Dir, latlim, lonlim, level=SL)
SG.Bulk_Density(Dir, latlim, lonlim, level=SL)
# Define path to layers
filename_clay = os.path.join(
Dir, 'SoilGrids', 'Clay_Content',
'ClayContentMassFraction_%s_SoilGrids_percentage.tif' % SL)
#filename_om = os.path.join(Dir, 'SoilGrids', 'Soil_Organic_Carbon_Content' ,'SoilOrganicCarbonContent_%s_SoilGrids_g_kg.tif' %SL)
filename_bulkdensity = os.path.join(
Dir, 'SoilGrids', 'Bulk_density',
'BulkDensity_%s_SoilGrids_kg-m-3.tif' % SL)
# Define path for output
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
filedir_out_densbulk = os.path.join(Dir, 'SoilGrids', 'Bulk_density')
if not os.path.exists(filedir_out_densbulk):
os.makedirs(filedir_out_densbulk)
filedir_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat')
if not os.path.exists(filedir_out_thetasat):
os.makedirs(filedir_out_thetasat)
#filename_out_densbulk = os.path.join(filedir_out_densbulk ,'Bulk_Density_%s_SoilGrids_g-cm-3.tif' %level)
filename_out_thetasat = os.path.join(
filedir_out_thetasat, 'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level)
#if not (os.path.exists(filename_out_densbulk) and os.path.exists(filename_out_thetasat)):
if not os.path.exists(filename_out_thetasat):
# Open datasets
dest_clay = gdal.Open(filename_clay)
#dest_om = gdal.Open(filename_om)
dest_bulk = gdal.Open(filename_bulkdensity)
# Open Array info
geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_clay)
# Open Arrays
Clay = dest_clay.GetRasterBand(1).ReadAsArray()
#OM = dest_om.GetRasterBand(1).ReadAsArray()
Clay = np.float_(Clay)
Clay[Clay > 100] = np.nan
#OM = np.float_(OM)
#OM[OM<0]=np.nan
#OM = OM/1000
# Calculate bulk density
#bulk_dens = 1/(0.6117 + 0.3601 * Clay/100 + 0.002172 * np.power(OM * 100, 2)+ 0.01715 * np.log(OM * 100))
bulk_dens = dest_bulk.GetRasterBand(1).ReadAsArray()
bulk_dens = bulk_dens / 1000
# Calculate theta saturated
theta_sat = 0.85 * (1 - (bulk_dens / 2.65)) + 0.13 * Clay / 100
# Save data
#DC.Save_as_tiff(filename_out_densbulk, bulk_dens, geo_out, "WGS84")
DC.Save_as_tiff(filename_out_thetasat, theta_sat, geo_out, "WGS84")
return ()
def RetrieveData(Date, args):
"""
This function retrieves CHIRPS data for a given date from the
ftp://chg-ftpout.geog.ucsb.edu server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
# open ftp server
ftp = FTP("chg-ftpout.geog.ucsb.edu", "", "")
ftp.login()
# Define FTP path to directory
if TimeCase == 'daily':
pathFTP = 'pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/' % Date.strftime(
'%Y')
elif TimeCase == 'monthly':
pathFTP = 'pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/'
else:
raise KeyError("The input time interval is not supported")
# find the document name in this directory
ftp.cwd(pathFTP)
listing = []
# read all the file names in the directory
ftp.retrlines("LIST", listing.append)
# create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names
if TimeCase == 'daily':
filename = 'chirps-v2.0.%s.%02s.%02s.tif.gz' % (
Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))
outfilename = os.path.join(
output_folder, 'chirps-v2.0.%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
DirFileEnd = os.path.join(
output_folder, 'P_CHIRPS.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
elif TimeCase == 'monthly':
filename = 'chirps-v2.0.%s.%02s.tif.gz' % (Date.strftime('%Y'),
Date.strftime('%m'))
outfilename = os.path.join(
output_folder, 'chirps-v2.0.%s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m')))
DirFileEnd = os.path.join(
output_folder,
'P_CHIRPS.v2.0_mm-month-1_monthly_%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
else:
raise KeyError("The input time interval is not supported")
# download the global rainfall file
try:
local_filename = os.path.join(output_folder, filename)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write, 8192)
lf.close()
# unzip the file
zip_filename = os.path.join(output_folder, filename)
DC.Extract_Data_gz(zip_filename, outfilename)
# open tiff file
dataset = RC.Open_tiff_array(outfilename)
# clip dataset to the given extent
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# save dataset as geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFileEnd,
data=data,
geo=geo,
projection="WGS84")
# delete old tif file
os.remove(outfilename)
except:
print("file not exists")
return True
def RetrieveData_monthly(Date, args):
"""
This function retrieves GLDAS CLSM monthly data for a given date.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Open all the parameters
[path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim, CaseParameters, username, password, types] = args
# Reset the begin parameters for downloading
downloaded = 0
N = 0
data_end = []
# Check GLDAS version
version = url[-3:]
# Open all variable info
for T in types:
if T == 'mean':
VarStr = VarInfo.names[Var]
else:
VarStr = VarInfo.names[Var] + '-' + T
# Check whether the file already exist or
# the worldfile is downloaded
BasinDir = os.path.join(path[T], VarStr + '_GLDAS-CLSM_' + VarInfo.units[Var] + '_monthly_' + Date.strftime('%Y.%m.%d') + '.tif')
# Check if the outputfile already excists
if not os.path.isfile(BasinDir):
# Create the time dimension
if version == '2.0':
zID_start = int(((Date - pd.Timestamp("1948-1-1")).days))
Y = int(Date.year)
M = int(Date.month)
Mday = calendar.monthrange(Y,M)[1]
zID_end = zID_start + Mday
if zID_end == 24472:
zID_end = 24470
Mday = Mday - 2
# define total url
url_GLDAS = url + '.ascii?%s[%s:1:%s][%s:1:%s][%s:1:%s]' %(Var,zID_start,zID_end,yID[0],yID[1],xID[0],xID[1])
# if not downloaded try to download file
while downloaded == 0:
try:
# open URL
try:
dataset = requests.get(url_GLDAS, allow_redirects=False,stream = True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
dataset = requests.get(url_GLDAS, allow_redirects=False,stream = True, verify = False)
try:
get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True, verify = False)
# download data (first save as text file)
pathtext = os.path.join(path[T],'temp%s.txt' %str(zID_start))
z = open(pathtext,'w')
z.write(get_dataset.text)
z.close()
# Reshape data
datashape = [Mday,yID[1] - yID[0] + 1,xID[1] - xID[0] + 1]
data_start = np.genfromtxt(pathtext,dtype = float,skip_header = 1,skip_footer = 6,delimiter = ',')
data_list = np.asarray(data_start[:,1:])
data_end = np.resize(data_list,(Mday, datashape[1], datashape[2]))
os.remove(pathtext)
# Add the VarFactor
if VarInfo.factors[Var] < 0:
data_end[data_end != -9999] = data_end[data_end != -9999] + VarInfo.factors[Var]
else:
data_end[data_end != -9999] = data_end[data_end != -9999] * VarInfo.factors[Var]
data_end[data_end < -9999] = -9999
# define geo
lonlimGLDAS = xID[0] * 0.25 - 180
latlimGLDAS = (yID[1] + 1) * 0.25 - 60
# Download was succesfull
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' + Date.strftime('%Y-%m-%d') + ' is not available')
downloaded = 1
try:
# Save to geotiff file
if T == 'mean':
data_end[data_end<-100] = np.nan
data = np.flipud(np.nanmean(data_end, axis=0))
if VarInfo.types[Var] == 'flux':
data = data * Mday
geo = [lonlimGLDAS,0.25,0,latlimGLDAS,0,-0.25]
DC.Save_as_tiff(name=BasinDir, data=data, geo=geo, projection="WGS84")
except:
print('GLDAS map from '+ Date.strftime('%Y-%m-%d') + ' is not created')
return True
def DownloadData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
TimeCase, CaseParameters):
"""
This function downloads ECMWF six-hourly, daily or monthly data
Keyword arguments:
"""
# correct latitude and longitude limits
latlim_corr_one = np.floor(latlim[0] / 0.125) * 0.125
latlim_corr_two = np.ceil(latlim[1] / 0.125) * 0.125
latlim_corr = [latlim_corr_one, latlim_corr_two]
# correct latitude and longitude limits
lonlim_corr_one = np.floor(lonlim[0] / 0.125) * 0.125
lonlim_corr_two = np.ceil(lonlim[1] / 0.125) * 0.125
lonlim_corr = [lonlim_corr_one, lonlim_corr_two]
# Load factors / unit / type of variables / accounts
VarInfo = VariablesInfo(TimeCase)
Varname_dir = VarInfo.file_name[Var]
# Create Out directory
out_dir = os.path.join(Dir, "Weather_Data", "Model", "ECMWF", TimeCase,
Varname_dir, "mean")
if not os.path.exists(out_dir):
os.makedirs(out_dir)
DownloadType = VarInfo.DownloadType[Var]
# Set required data for the three hourly option
if TimeCase == 'six_hourly':
string1 = 'oper'
# Set required data for the daily option
elif TimeCase == 'daily':
Dates = pd.date_range(Startdate, Enddate, freq='D')
elif TimeCase == 'monthly':
Dates = pd.date_range(Startdate, Enddate, freq='MS')
if DownloadType == 1:
string1 = 'oper'
string4 = "0"
string6 = "00:00:00/06:00:00/12:00:00/18:00:00"
string2 = 'sfc'
string8 = 'an'
if DownloadType == 2:
string1 = 'oper'
string4 = "12"
string6 = "00:00:00/12:00:00"
string2 = 'sfc'
string8 = 'fc'
if DownloadType == 3:
string1 = 'oper'
string4 = "0"
string6 = "00:00:00/06:00:00/12:00:00/18:00:00"
string2 = 'pl'
string8 = 'an'
string7 = '%s/to/%s' % (Startdate, Enddate)
parameter_number = VarInfo.number_para[Var]
string3 = '%03d.128' % (parameter_number)
string5 = '0.125/0.125'
string9 = 'ei'
string10 = '%s/%s/%s/%s' % (latlim_corr[1], lonlim_corr[0], latlim_corr[0],
lonlim_corr[1]) #N, W, S, E
# Download data by using the ECMWF API
import watools.Collect.ECMWF.ECMWFdownload as Download
print('Use API ECMWF to collect the data, please wait')
Download.API(Dir, DownloadType, string1, string2, string3, string4,
string5, string6, string7, string8, string9, string10)
# Open the downloaded data
NC_filename = os.path.join(Dir, 'data_interim.nc')
fh = Dataset(NC_filename, mode='r')
# Get the NC variable parameter
parameter_var = VarInfo.var_name[Var]
Var_unit = VarInfo.units[Var]
factors_add = VarInfo.factors_add[Var]
factors_mul = VarInfo.factors_mul[Var]
# Open the NC data
Data = fh.variables[parameter_var][:]
Data_time = fh.variables['time'][:]
lons = fh.variables['longitude'][:]
lats = fh.variables['latitude'][:]
# Define the georeference information
Geo_four = np.nanmax(lats)
Geo_one = np.nanmin(lons)
Geo_out = tuple([Geo_one, 0.125, 0.0, Geo_four, 0.0, -0.125])
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
for date in Dates:
# Define the year, month and day
year = date.year
month = date.month
day = date.day
# Hours since 1900-01-01
start = datetime.datetime(year=1900, month=1, day=1)
end = datetime.datetime(year, month, day)
diff = end - start
hours_from_start_begin = diff.total_seconds() / 60 / 60
Date_good = np.zeros(len(Data_time))
if TimeCase == 'daily':
days_later = 1
if TimeCase == 'monthly':
days_later = calendar.monthrange(year, month)[1]
Date_good[np.logical_and(
Data_time >= hours_from_start_begin, Data_time <
(hours_from_start_begin + 24 * days_later))] = 1
Data_one = np.zeros([
int(np.sum(Date_good)),
int(np.size(Data, 1)),
int(np.size(Data, 2))
])
Data_one = Data[np.int_(Date_good) == 1, :, :]
# Calculate the average temperature in celcius degrees
Data_end = factors_mul * np.nanmean(Data_one, 0) + factors_add
if VarInfo.types[Var] == 'flux':
Data_end = Data_end * days_later
VarOutputname = VarInfo.file_name[Var]
# Define the out name
name_out = os.path.join(
out_dir, "%s_ECMWF_ERA-Interim_%s_%s_%d.%02d.%02d.tif" %
(VarOutputname, Var_unit, TimeCase, year, month, day))
# Create Tiff files
DC.Save_as_tiff(name_out, Data_end, Geo_out, "WGS84")
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
fh.close()
return ()
def RetrieveData(Date, args):
# unpack the arguments
[output_folder, latlim, lonlim, Var, Version] = args
# Name of the model
if Version == 1:
version_name = 'CFSR'
if Version == 2:
version_name = 'CFSRv2'
# Name of the outputfile
if Var == 'dlwsfc':
Outputname = 'DLWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
if Var == 'dswsfc':
Outputname = 'DSWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
if Var == 'ulwsfc':
Outputname = 'ULWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
if Var == 'uswsfc':
Outputname = 'USWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
# Create the total end output name
outputnamePath = os.path.join(output_folder, Outputname)
# If the output name not exists than create this output
if not os.path.exists(outputnamePath):
local_filename = Download_data(Date, Version, output_folder, Var)
# convert grb2 to netcdf (wgrib2 module is needed)
for i in range(0,4):
nameNC = 'Output' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + str(Date.strftime('%d')) + '-' + str(i+1) + '.nc'
# Total path of the output
FileNC6hour = os.path.join(output_folder, nameNC)
# Band number of the grib data which is converted in .nc
band=(int(Date.strftime('%d')) - 1) * 28 + (i + 1) * 7
# Convert the data
DC.Convert_grb2_to_nc(local_filename, FileNC6hour, band)
if Version == 1:
if Date < pd.Timestamp(pd.datetime(2011, 1, 1)):
# Convert the latlim and lonlim into array
Xstart = np.floor((lonlim[0] + 180.1562497) / 0.3125)
Xend = np.ceil((lonlim[1] + 180.1562497) / 0.3125) + 1
Ystart = np.floor((latlim[0] + 89.9171038899) / 0.3122121663)
Yend = np.ceil((latlim[1] + 89.9171038899) / 0.3122121663)
# Create a new dataset
Datatot = np.zeros([576, 1152])
else:
Version = 2
if Version == 2:
# Convert the latlim and lonlim into array
Xstart = np.floor((lonlim[0] + 180.102272725) / 0.204545)
Xend = np.ceil((lonlim[1] + 180.102272725) / 0.204545) + 1
Ystart = np.floor((latlim[0] + 89.9462116040955806) / 0.204423)
Yend = np.ceil((latlim[1] + 89.9462116040955806) / 0.204423)
# Create a new dataset
Datatot = np.zeros([880, 1760])
# Open 4 times 6 hourly dataset
for i in range (0, 4):
nameNC = 'Output' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + str(Date.strftime('%d')) + '-' + str(i + 1) + '.nc'
FileNC6hour = os.path.join(output_folder, nameNC)
f = Dataset(FileNC6hour, mode = 'r')
Data = f.variables['Band1'][0:int(Datatot.shape[0]), 0:int(Datatot.shape[1])]
f.close()
data = np.array(Data)
Datatot = Datatot + data
# Calculate the average in W/m^2 over the day
DatatotDay = Datatot / 4
DatatotDayEnd = np.zeros([int(Datatot.shape[0]), int(Datatot.shape[1])])
DatatotDayEnd[:,0:int(Datatot.shape[0])] = DatatotDay[:, int(Datatot.shape[0]):int(Datatot.shape[1])]
DatatotDayEnd[:,int(Datatot.shape[0]):int(Datatot.shape[1])] = DatatotDay[:, 0:int(Datatot.shape[0])]
# clip the data to the extent difined by the user
DatasetEnd = DatatotDayEnd[int(Ystart):int(Yend), int(Xstart):int(Xend)]
# save file
if Version == 1:
pixel_size = 0.3125
if Version == 2:
pixel_size = 0.204545
geo = [lonlim[0],pixel_size,0,latlim[1],0,-pixel_size]
DC.Save_as_tiff(data = np.flipud(DatasetEnd), name = outputnamePath, geo = geo, projection = "WGS84")
return()
def RetrieveData(Date, args):
"""
This function retrieves GLEAM ET data for a given date from the
www.gleam.eu server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, latlim, lonlim, VarCode, TimeCase, Product] = args
# Adjust latlim to GLEAM dataset
latlim1 = [latlim[1] * -1, latlim[0] * -1]
# select the spatial dataset
Ystart = int(np.floor((latlim1[0] + 90) / 0.25))
Yend = int(np.ceil((latlim1[1] + 90) / 0.25))
Xstart = int(np.floor((lonlim[0] + 180) / 0.25))
Xend = int(np.ceil((lonlim[1] + 180) / 0.25))
Year = Date.year
Month = Date.month
if Product == "ET":
filename = 'E_' + str(Year) + '_GLEAM_v3.2b.nc'
if Product == "ETpot":
filename = 'Ep_' + str(Year) + '_GLEAM_v3.2b.nc'
local_filename = os.path.join(output_folder, filename)
f = Dataset(local_filename, mode='r')
if TimeCase == 'monthly':
# defines the start and end of the month
Datesend1 = str(Date)
Datesend2 = Datesend1.replace(Datesend1[8:10], "01")
Datesend3 = Datesend2[0:10]
Datesend4 = Datesend1[0:10]
Datestart = pd.date_range(Datesend3, Datesend4, freq='MS')
# determine the DOY-1 and DOYend (those are use to define the temporal boundaries of the yearly data)
DOY = int(Datestart[0].strftime('%j'))
DOYend = int(Date.strftime('%j'))
DOYDownload = DOY - 1
Day = 1
if Product == "ET":
Data = f.variables['E'][DOYDownload:DOYend, Xstart:Xend,
Ystart:Yend]
if Product == "ETpot":
Data = f.variables['Ep'][DOYDownload:DOYend, Xstart:Xend,
Ystart:Yend]
data = np.array(Data)
f.close()
# Sum ET data in time and change the no data value into -999
dataSum = sum(data, 1)
dataSum[dataSum < -100] = -999.000
dataCor = np.swapaxes(dataSum, 0, 1)
if TimeCase == 'daily':
Day = Date.day
# Define the DOY, DOY-1 is taken from the yearly dataset
DOY = int(Date.strftime('%j'))
DOYDownload = DOY - 1
if Product == "ET":
Data = f.variables['E'][DOYDownload, Xstart:Xend, Ystart:Yend]
if Product == "ETpot":
Data = f.variables['Ep'][DOYDownload, Xstart:Xend, Ystart:Yend]
data = np.array(Data)
f.close()
data[data < -100] = -999.000
dataCor = np.swapaxes(data, 0, 1)
# The Georeference of the map
geo_in = [lonlim[0], 0.25, 0.0, latlim[1], 0.0, -0.25]
# Name of the map
dataset_name = VarCode + '_' + str(Year) + '.' + str(Month).zfill(
2) + '.' + str(Day).zfill(2) + '.tif'
output_file = os.path.join(output_folder, dataset_name)
# save data as tiff file
DC.Save_as_tiff(name=output_file,
data=dataCor,
geo=geo_in,
projection="WGS84")
return True
def main(Dir,
Startdate='',
Enddate='',
latlim=[-50, 50],
lonlim=[-180, 180],
cores=False,
Waitbar=1):
"""
This function downloads RFE V2.0 (monthly) data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine.
It can be 'False' to avoid using parallel computing
routines.
Waitbar -- 1 (Default) will print a waitbar
"""
# Download data
print('\nDownload monthly RFE precipitation data for period %s till %s' %
(Startdate, Enddate))
# Check variables
if not Startdate:
Startdate = pd.Timestamp('2001-01-01')
if not Enddate:
Enddate = pd.Timestamp('Now')
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Make directory
output_folder = os.path.join(Dir, 'Precipitation', 'RFE', 'Monthly/')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
for Date in Dates:
month = Date.month
year = Date.year
end_day = calendar.monthrange(year, month)[1]
Startdate_one_month = '%s-%02s-01' % (year, month)
Enddate_one_month = '%s-%02s-%02s' % (year, month, end_day)
DownloadData(Dir, Startdate_one_month, Enddate_one_month, latlim,
lonlim, 0, cores)
Dates_daily = pd.date_range(Startdate_one_month,
Enddate_one_month,
freq='D')
# Make directory
input_folder_daily = os.path.join(Dir, 'Precipitation', 'RFE',
'Daily/')
i = 0
for Date_daily in Dates_daily:
file_name = 'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' % (
Date_daily.strftime('%Y'), Date_daily.strftime('%m'),
Date_daily.strftime('%d'))
file_name_daily_path = os.path.join(input_folder_daily, file_name)
if os.path.exists(file_name_daily_path):
if Date_daily == Dates_daily[i]:
Raster_monthly = RC.Open_tiff_array(file_name_daily_path)
else:
Raster_monthly += RC.Open_tiff_array(file_name_daily_path)
else:
if Date_daily == Dates_daily[i]:
i += 1
geo_out, proj, size_X, size_Y = RC.Open_array_info(
file_name_daily_path)
file_name = 'P_RFE.v2.0_mm-month-1_monthly_%s.%02s.01.tif' % (
Date.strftime('%Y'), Date.strftime('%m'))
file_name_output = os.path.join(output_folder, file_name)
DC.Save_as_tiff(file_name_output,
Raster_monthly,
geo_out,
projection="WGS84")
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar):
"""
This scripts downloads ASCAT SWI data from the VITO server.
The output files display the Surface Water Index.
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
lonlim -- [ymin, ymax]
latlim -- [xmin, xmax]
"""
# Check the latitude and longitude and otherwise reset lat and lon.
if latlim[0] < -90 or latlim[1] > 90:
print('Latitude above 90N or below 90S is not possible.\
Value set to maximum')
latlim[0] = np.max(latlim[0], -90)
latlim[1] = np.min(latlim[1], 90)
if lonlim[0] < -180 or lonlim[1] > 180:
print('Longitude must be between 180E and 180W.\
Now value is set to maximum')
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# Check Startdate and Enddate
if not Startdate:
Startdate = pd.Timestamp('2007-01-01')
if not Enddate:
Enddate = pd.Timestamp('2018-12-31')
# Make a panda timestamp of the date
try:
Enddate = pd.Timestamp(Enddate)
except:
Enddate = Enddate
# amount of Dates weekly
Dates = pd.date_range(Startdate, Enddate, freq='D')
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define directory and create it if not exists
output_folder = os.path.join(Dir, 'SWI', 'ASCAT', 'Daily')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
output_folder_temp = os.path.join(Dir, 'SWI', 'ASCAT', 'Daily', 'Temp')
if not os.path.exists(output_folder_temp):
os.makedirs(output_folder_temp)
# loop over dates
for Date in Dates:
# Define end filename
End_filename = os.path.join(
output_folder, 'SWI_ASCAT_V3_Percentage_daily_%d.%02d.%02d.tif' %
(Date.year, Date.month, Date.day))
# Define IDs
xID = 1800 + np.int16(
np.array([np.ceil((lonlim[0]) * 10),
np.floor((lonlim[1]) * 10)]))
yID = np.int16(
np.array([np.floor((-latlim[1]) * 10),
np.ceil((-latlim[0]) * 10)])) + 900
# Download the data from FTP server if the file not exists
if not os.path.exists(End_filename):
try:
data = Download_ASCAT_from_VITO(End_filename,
output_folder_temp, Date, yID,
xID)
# make geotiff file
geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
DC.Save_as_tiff(name=End_filename,
data=data,
geo=geo,
projection="WGS84")
except:
print("Was not able to download file with date %s" % Date)
# Adjust waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# remove the temporary folder
shutil.rmtree(output_folder_temp)
def NPP_GPP_Based(Dir_Basin, Data_Path_GPP, Data_Path_NPP, Startdate, Enddate):
"""
This functions calculated monthly NDM based on the yearly NPP and monthly GPP.
Parameters
----------
Dir_Basin : str
Path to all the output data of the Basin
Data_Path_GPP : str
Path from the Dir_Basin to the GPP data
Data_Path_NPP : str
Path from the Dir_Basin to the NPP data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Simulation : int
Defines the simulation
Returns
-------
Data_Path_NDM : str
Path from the Dir_Basin to the normalized dry matter data
"""
# import WA+ modules
import watools.General.data_conversions as DC
import watools.General.raster_conversions as RC
# Define output folder for Normalized Dry Matter
Data_Path_NDM = os.path.join(Dir_Basin, "NDM")
if not os.path.exists(Data_Path_NDM):
os.mkdir(Data_Path_NDM)
# Define monthly time steps that will be created
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Define the years that will be calculated
Year_Start = int(Startdate[0:4])
Year_End = int(Enddate[0:4])
Years = list(range(Year_Start, Year_End + 1))
# Loop over the years
for year in Years:
# Change working directory to the NPP folder
os.chdir(Data_Path_NPP)
# Open yearly NPP data
yearly_NPP_File = glob.glob('*yearly*%d.01.01.tif' % int(year))[0]
Yearly_NPP = RC.Open_tiff_array(yearly_NPP_File)
# Get the No Data Value of the NPP file
dest = gdal.Open(yearly_NPP_File)
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Set the No Data Value to Nan
Yearly_NPP[Yearly_NPP == NDV] = np.nan
# Change working directory to the GPP folder
os.chdir(Data_Path_GPP)
# Find all the monthly files of that year
monthly_GPP_Files = glob.glob('*monthly*%d.*.01.tif' % int(year))
# Check if it are 12 files otherwise something is wrong and send the ERROR
if not len(monthly_GPP_Files) == 12:
print('ERROR: Some monthly GPP Files are missing')
# Get the projection information of the GPP inputs
geo_out, proj, size_X, size_Y = RC.Open_array_info(
monthly_GPP_Files[0])
geo_out_NPP, proj_NPP, size_X_NPP, size_Y_NPP = RC.Open_array_info(
os.path.join(Data_Path_NPP, yearly_NPP_File))
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value of the GPP files
dest = gdal.Open(monthly_GPP_Files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Create a empty numpy array
Yearly_GPP = np.zeros([size_Y, size_X])
# Calculte the total yearly GPP
for monthly_GPP_File in monthly_GPP_Files:
# Open array
Data = RC.Open_tiff_array(monthly_GPP_File)
# Remove nan values
Data[Data == NDV] = np.nan
Data[np.isnan(Data)] = 0
# Add data to yearly sum
Yearly_GPP += Data
# Check if size is the same of NPP and GPP otherwise resize
if not (size_X_NPP == size_X and size_Y_NPP == size_Y):
Yearly_NPP = RC.resize_array_example(Yearly_NPP, Yearly_GPP)
# Loop over the monthly dates
for Date in Dates:
# If the Date is in the same year as the yearly NPP and GPP
if Date.year == year:
# Create empty GPP array
monthly_GPP = np.ones([size_Y, size_X]) * np.nan
# Get current month
month = Date.month
# Get the GPP file of the current year and month
monthly_GPP_File = glob.glob('*monthly_%d.%02d.01.tif' %
(int(year), int(month)))[0]
monthly_GPP = RC.Open_tiff_array(monthly_GPP_File)
monthly_GPP[monthly_GPP == NDV] = np.nan
# Calculate the NDM based on the monthly and yearly NPP and GPP (fraction of GPP)
Monthly_NDM = Yearly_NPP * monthly_GPP / Yearly_GPP * (
30. / 12.) * 10000 # kg/ha
# Define output name
output_name = os.path.join(
Data_Path_NDM, 'NDM_MOD17_kg_ha-1_monthly_%d.%02d.01.tif' %
(int(year), int(month)))
# Save the NDM as tiff file
DC.Save_as_tiff(output_name, Monthly_NDM, geo_out, proj)
return (Data_Path_NDM)
def DownloadData(output_folder, latlim, lonlim, parameter, resolution):
"""
This function downloads DEM data from HydroSHED
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
Resample -- 1 = The data will be resampled to 0.001 degree spatial
resolution
-- 0 = The data will have the same pixel size as the data obtained
from the internet
"""
# Define parameter depedent variables
if parameter == "dir_3s":
para_name = "DIR"
unit = "-"
resolution = '3s'
parameter = 'dir'
if parameter == "dem_3s":
para_name = "DEM"
unit = "m"
resolution = '3s'
parameter = 'dem'
if parameter == "dir_15s":
para_name = "DIR"
unit = "-"
resolution = '15s'
parameter = 'dir'
if parameter == "dem_15s":
para_name = "DEM"
unit = "m"
resolution = '15s'
parameter = 'dem'
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
if resolution == '3s':
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim,
parameter)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
if resolution == '15s':
name = Find_Document_names_15s(latlim, lonlim, parameter, resolution)
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash,
parameter, para_name,
resolution)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# Converts the data with a adf extention to a tiff extension.
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif'
file_name_extract = file_name.split('_')[0:3]
if resolution == '3s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1]
if resolution == '15s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1] + '_15s'
input_adf = os.path.join(output_folder_trash, file_name_extract2,
file_name_extract2, 'hdr.adf')
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff)
geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff)
if int(size_X) != int(6000) or int(size_Y) != int(6000):
data = np.ones((6000, 6000)) * -9999
# Create the latitude bound
Vfile = str(nameFile)[1:3]
SignV = str(nameFile)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(nameFile)[4:7]
SignH = str(nameFile)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
Expected_X_min = Bound1
Expected_Y_max = Bound2 + 5
Xid_start = int(
np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(
np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) /
geo_out[1]))
Yid_start = int(
np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(
np.round((Expected_Y_max - (geo_out[3] +
(size_Y * geo_out[5]))) /
(-geo_out[5])))
data[Yid_start:Yid_end,
Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff)
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
except:
if resolution == '3s':
# If tile not exist create a replacing zero tile (sea tiles)
output = nameFile.split('.')[0] + "_trans_temporary.tif"
output_tiff = os.path.join(output_folder_trash, output)
file_name = nameFile
data = np.ones((6000, 6000)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = str(file_name)[1:3]
SignV = str(file_name)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(file_name)[4:7]
SignH = str(file_name)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
# Geospatial data for the tile
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
if resolution == '15s':
print('no 15s data is in dataset')
if resolution == '3s':
# clip data
Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (nameFile)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd,
data=Data,
geo=Geo_data,
projection="WGS84")
if resolution == '3s':
#size_X_end = int(size_X_tot) #!
#size_Y_end = int(size_Y_tot) #!
size_X_end = int(size_X_tot / len(rangeLat)) + 1 #!
size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 #!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
if resolution == '15s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s(output_folder_trash,
output_file_merged, latlim, lonlim)
# name of the end result
output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution)
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name,
data=datasetTot,
geo=geo_out,
projection="WGS84")
os.chdir(output_folder)
# Delete the temporary folder
shutil.rmtree(output_folder_trash)
def Calc_Rainy_Days(Dir_Basin, Data_Path_P, Startdate, Enddate):
"""
This functions calculates the amount of rainy days based on daily precipitation data.
Parameters
----------
Dir_Basin : str
Path to all the output data of the Basin
Data_Path_P : str
Path to the daily rainfall data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Returns
-------
Data_Path_RD : str
Path from the Dir_Basin to the rainy days data
"""
# import WA+ modules
import watools.General.data_conversions as DC
import watools.General.raster_conversions as RC
# Create an output directory to store the rainy days tiffs
Data_Path_RD = os.path.join(Dir_Basin, 'Rainy_Days')
if not os.path.exists(Data_Path_RD):
os.mkdir(Data_Path_RD)
# Define the dates that must be created
Dates = pd.date_range(Startdate, Enddate, freq ='MS')
# Set working directory to the rainfall folder
os.chdir(Data_Path_P)
# Open all the daily data and store the data in a 3D array
for Date in Dates:
# Define the year and month and amount of days in month
year = Date.year
month = Date.month
daysinmonth = calendar.monthrange(year, month)[1]
# Set the third (time) dimension of array starting at 0
i = 0
# Find all files of that month
files = glob.glob('*daily_%d.%02d.*.tif' %(year, month))
# Check if the amount of files corresponds with the amount of days in month
if len(files) is not daysinmonth:
print('ERROR: Not all Rainfall days for month %d and year %d are downloaded' %(month, year))
# Loop over the days and store data in raster
for File in files:
dir_file = os.path.join(Data_Path_P, File)
# Get array information and create empty numpy array for daily rainfall when looping the first file
if File == files[0]:
# Open geolocation info and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(dir_file)
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Create empty array for the whole month
P_Daily = np.zeros([daysinmonth,size_Y, size_X])
# Open data and put the data in 3D array
Data = RC.Open_tiff_array(dir_file)
# Remove the weird numbers
Data[Data<0] = 0
# Add the precipitation to the monthly cube
P_Daily[i, :, :] = Data
i += 1
# Define a rainy day
P_Daily[P_Daily > 0.201] = 1
P_Daily[P_Daily != 1] = 0
# Sum the amount of rainy days
RD_one_month = np.nansum(P_Daily,0)
# Define output name
Outname = os.path.join(Data_Path_RD, 'Rainy_Days_NumOfDays_monthly_%d.%02d.01.tif' %(year, month))
# Save tiff file
DC.Save_as_tiff(Outname, RD_one_month, geo_out, proj)
return(Data_Path_RD)
def RetrieveData_three_hourly(Date, args):
"""
This function retrieves GLDAS three-hourly data for a given date.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Open all the parameters
[path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim, CaseParameters, username, password, types] = args
# Open variable info parameters
VarFactor = VarInfo.factors[Var]
# Loop over the periods
for period in CaseParameters:
# Check whether the file already exist or the worldfile is
# downloaded
BasinDir = path + '/' + VarStr + '_GLDAS-CLSM_' + \
VarInfo.units[Var] + '_3hour_' + Date.strftime('%Y.%m.%d') + \
'_'+str(period) + '.tif'
if not os.path.isfile(BasinDir):
# Reset the begin parameters for downloading
downloaded = 0
N=0
while downloaded == 0:
try:
# Define time
zID = int(((Date - pd.Timestamp("1979-1-2")).days) * 8) + (period - 1)
# total URL
url_GLDAS = url + '.ascii?%s[%s][%s:1:%s][%s:1:%s]' %(Var,zID,yID[0],yID[1],xID[0],xID[1])
# open URL
try:
dataset = requests.get(url_GLDAS, allow_redirects=False, stream = True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
dataset = requests.get(url_GLDAS, allow_redirects=False, stream = True, verify = False)
try:
get_dataset = requests.get(dataset.headers['Location'], auth = (username,password), stream = True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True, verify = False)
# download data (first save as text file)
pathtext = os.path.join(path, 'temp%s.txt' % zID)
z = open(pathtext, 'w')
z.write(get_dataset.text)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,dtype = float,skip_header = 1,skip_footer = 6,delimiter=',')
data = data_start[:,1:]
# Add the VarFactor
if VarFactor < 0:
data = data + VarFactor
else:
data = data * VarFactor
if VarInfo.types[Var] == 'flux':
data = data / 8
# Set Nan value for values lower than -9999
data[data < -9999] = -9999
# Say that download was succesfull
downloaded = 1
# If download was not succesfull
except:
data=[]
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' + Date.strftime('%Y-%m-%d') + ' is not available')
downloaded = 1
# define geo
lonlimGLDAS = xID[0] * 1.0 - 180
latlimGLDAS = (yID[1] + 1) * 1.0 - 60
# Save to geotiff file
geo = [lonlimGLDAS,1.0,0,latlimGLDAS,0,-1.0]
DC.Save_as_tiff(name=BasinDir, data=np.flipud(data[:,:]), geo=geo, projection="WGS84")
# Delete data and text file
del data
os.remove(pathtext)
return True
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, version,
Product):
"""
This scripts downloads SSEBop ET data from the UNESCO-IHE ftp server.
The output files display the total ET in mm for a period of one month.
The name of the file corresponds to the first day of the month.
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
lonlim -- [ymin, ymax] (values must be between -90 and 90)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
if version == "FTP":
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -59.2 or latlim[1] > 80:
print(
'Latitude above 80N or below -59.2S is not possible. Value set to maximum'
)
latlim[0] = np.max(latlim[0], -59.2)
latlim[1] = np.min(latlim[1], 80)
if lonlim[0] < -180 or lonlim[1] > 180:
print(
'Longitude must be between 180E and 180W. Now value is set to maximum'
)
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# Check Startdate and Enddate
if not Startdate:
Startdate = pd.Timestamp('2003-01-01')
if not Enddate:
Enddate = pd.Timestamp('2014-10-31')
if version == "V4":
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -60 or latlim[1] > 80.0022588483988670:
print(
'Latitude above 80N or below -59.2S is not possible. Value set to maximum'
)
latlim[0] = np.max(latlim[0], -60)
latlim[1] = np.min(latlim[1], 80.0022588483988670)
if lonlim[0] < -180 or lonlim[1] > 180.0002930387853439:
print(
'Longitude must be between 180E and 180W. Now value is set to maximum'
)
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180.0002930387853439)
# Check Startdate and Enddate
if not Startdate:
Startdate = pd.Timestamp('2003-01-01')
if not Enddate:
import datetime
Enddate = pd.Timestamp(datetime.datetime.now())
# Define directory and create it if not exists
if Product == "ETact":
output_folder = os.path.join(Dir, 'Evaporation', 'SSEBop', 'Monthly')
freq_use = "MS"
if Product == "ETpot":
output_folder = os.path.join(Dir, 'Potential_Evapotranspiration',
'FEWS', 'Daily')
freq_use = "D"
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Creates dates library
Dates = pd.date_range(Startdate, Enddate, freq=freq_use)
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Loop over the dates
for Date in Dates:
# Define year and month
year = Date.year
month = Date.month
day = Date.day
if version == "FTP":
# Date as printed in filename
Filename_out = os.path.join(
output_folder,
'ETa_SSEBop_FTP_mm-month-1_monthly_%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'),
Date.strftime('%d')))
# Define end filename
Filename_dir = os.path.join("%s" % year,
"m%s%02d.tif" % (str(year)[2:], month))
Filename_only = "m%s%02d.tif" % (str(year)[2:], month)
if version == "V4":
# Date as printed in filename
if Product == "ETpot":
Filename_out = os.path.join(
output_folder,
'ETpot_FEWS_mm-day-1_daily_%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'),
Date.strftime('%d')))
# Define the downloaded zip file
Filename_only_zip = 'et%02s%02d%02d.tar.gz' % (str(year)[2:],
month, day)
# The end file name after downloading and unzipping
Filename_only = "et%02s%02d%02d.bil" % (str(year)[2:], month,
day)
# Create bin folder
temp_folder = os.path.join(output_folder, "Temp")
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
local_filename = os.path.join(temp_folder, Filename_only)
if Product == "ETact":
Filename_out = os.path.join(
output_folder,
'ETa_SSEBop_V4_mm-month-1_monthly_%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'),
Date.strftime('%d')))
# Define the downloaded zip file
Filename_only_zip = "m%s%02d.zip" % (str(year), month)
# The end file name after downloading and unzipping
Filename_only = "m%s%02d_modisSSEBopETv4_actual_mm.tif" % (
str(year), month)
# Temporary filename for the downloaded global file
local_filename = os.path.join(output_folder, Filename_only)
# Download the data from FTP server if the file not exists
if not os.path.exists(Filename_out):
try:
if version == "FTP":
Download_SSEBop_from_WA_FTP(local_filename, Filename_dir)
if version == "V4":
if Product == "ETpot":
Download_SSEBop_from_Web(temp_folder,
Filename_only_zip, Product)
if Product == "ETact":
Download_SSEBop_from_Web(output_folder,
Filename_only_zip, Product)
if Product == "ETpot":
Array_ETpot = RC.Open_bil_array(local_filename)
Array_ETpot = Array_ETpot / 100
Geo_out = tuple([-180.5, 1, 0, 90.5, 0, -1])
dest = DC.Save_as_MEM(Array_ETpot, Geo_out, "WGS84")
data, Geo_out = RC.clip_data(dest, latlim, lonlim)
DC.Save_as_tiff(Filename_out, data, Geo_out, "WGS84")
if Product == "ETact":
# Clip dataset
RC.Clip_Dataset_GDAL(local_filename, Filename_out, latlim,
lonlim)
os.remove(local_filename)
except:
print("Was not able to download file with date %s" % Date)
# Adjust waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
if version == "V4":
import glob
os.chdir(output_folder)
if Product == "ETact":
zipfiles = glob.glob("*.zip")
for zipfile in zipfiles:
os.remove(os.path.join(output_folder, zipfile))
xmlfiles = glob.glob("*.xml")
for xmlfile in xmlfiles:
os.remove(os.path.join(output_folder, xmlfile))
if Product == "ETpot":
import shutil
Temp_dir = os.path.join(output_folder, "Temp")
shutil.rmtree(Temp_dir)
return
def RetrieveData_monthly(Date, args):
"""
This function retrieves MSWEP precipitation monthly data for a given date.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[path, url, TimeCase, xID, yID, lonlim, latlim, username, password] = args
# Check whether the file already exist or the worldfile is downloaded
BasinDir = path + '/P_MSWEP_mm-month_monthly_' + Date.strftime(
'%Y.%m.%d') + '.tif'
# Define month and year of current month
Y = Date.year
M = Date.month
# Check if the outputfile already excists
if not os.path.isfile(BasinDir):
# Reset the begin parameters for downloading
downloaded = 0
N = 0
# Create the time dimension
zID = (Y - 1979) * 12 + (M - 1)
# define total url
url_MSWEP = url + '.ascii?precipitation[%s][%s:1:%s][%s:1:%s]' % (
zID, yID[0], yID[1], xID[0], xID[1])
# if not downloaded try to download file
while downloaded == 0:
try:
# open URL
try:
dataset = requests.get(url_MSWEP,
allow_redirects=False,
stream=True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
dataset = requests.get(url_MSWEP,
allow_redirects=False,
stream=True,
verify=False)
# download data (first save as text file)
pathtext = os.path.join(path, 'temp%s.txt' % str(zID))
z = open(pathtext, 'w')
z.write(dataset.content)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,
dtype=float,
skip_header=1,
skip_footer=6,
delimiter=',')
data = data_start[1:, 1:]
# Set Nan value for values lower than -9999
data[data < -9998] = np.nan
# Say that download was succesfull
downloaded = 1
# If download was not succesfull
except:
data = []
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' + Date.strftime('%Y-%m-%d') +
' is not available')
downloaded = 1
# define geo
lonlimMSWEP = xID[0] * 0.10 - 180
latlimMSWEP = yID[1] * 0.10 - 90
# Save to geotiff file
geo = [lonlimMSWEP, 0.1, 0, latlimMSWEP, 0, -0.1]
DC.Save_as_tiff(name=BasinDir,
data=data,
geo=geo,
projection="WGS84")
# Delete data and text file
del data
os.remove(pathtext)
return (True)
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None):
"""
This functions calculates monthly tiff files based on the 16 daily tiff files. (will calculate the average)
Parameters
----------
Dir_in : str
Path to the input data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Dir_out : str
Path to the output data, default is same as Dir_in
"""
# import WA+ modules
import watools.General.data_conversions as DC
import watools.General.raster_conversions as RC
# Change working directory
os.chdir(Dir_in)
# Find all eight daily files
files = glob.glob('*16-daily*.tif')
# Create array with filename and keys (DOY and year) of all the 8 daily files
i = 0
DOY_Year = np.zeros([len(files), 3])
for File in files:
# Get the time characteristics from the filename
year = File.split('.')[-4][-4:]
month = File.split('.')[-3]
day = File.split('.')[-2]
# Create pandas Timestamp
date_file = '%s-%02s-%02s' % (year, month, day)
Datum = pd.Timestamp(date_file)
# Get day of year
DOY = Datum.strftime('%j')
# Save data in array
DOY_Year[i, 0] = i
DOY_Year[i, 1] = DOY
DOY_Year[i, 2] = year
# Loop over files
i += 1
# Check enddate:
Enddate_split = Enddate.split('-')
month_range = calendar.monthrange(int(Enddate_split[0]),
int(Enddate_split[1]))[1]
Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int(
Enddate_split[1]), month_range)
# Check startdate:
Startdate_split = Startdate.split('-')
Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int(
Startdate_split[1]))
# Define end and start date
Dates = pd.date_range(Startdate, Enddate, freq='MS')
DatesEnd = pd.date_range(Startdate, Enddate, freq='M')
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value
dest = gdal.Open(files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Loop over months and create monthly tiff files
i = 0
for date in Dates:
# Get Start and end DOY of the current month
DOY_month_start = date.strftime('%j')
DOY_month_end = DatesEnd[i].strftime('%j')
# Search for the files that are between those DOYs
year = date.year
DOYs = DOY_Year[DOY_Year[:, 2] == year]
DOYs_oneMonth = DOYs[np.logical_and(
(DOYs[:, 1] + 16) >= int(DOY_month_start),
DOYs[:, 1] <= int(DOY_month_end))]
# Create empty arrays
Monthly = np.zeros([size_Y, size_X])
Weight_tot = np.zeros([size_Y, size_X])
Data_one_month = np.ones([size_Y, size_X]) * np.nan
# Loop over the files that are within the DOYs
for EightDays in DOYs_oneMonth[:, 1]:
# Calculate the amount of days in this month of each file
Weight = np.ones([size_Y, size_X])
# For start of month
if np.min(DOYs_oneMonth[:, 1]) == EightDays:
Weight = Weight * int(EightDays + 16 - int(DOY_month_start))
# For end of month
elif np.max(DOYs_oneMonth[:, 1]) == EightDays:
Weight = Weight * (int(DOY_month_end) - EightDays + 1)
# For the middle of the month
else:
Weight = Weight * 16
row = DOYs_oneMonth[np.argwhere(
DOYs_oneMonth[:, 1] == EightDays)[0][0], :][0]
# Open the array of current file
input_name = os.path.join(Dir_in, files[int(row)])
Data = RC.Open_tiff_array(input_name)
# Remove NDV
Weight[Data == NDV] = 0
Data[Data == NDV] = np.nan
# Multiply weight time data (per day)
Data = Data * Weight
# Calculate the total weight and data
Weight_tot += Weight
Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)]
# Go to next month
i += 1
# Calculate the average
Data_one_month[Weight_tot != 0.] = Monthly[
Weight_tot != 0.] / Weight_tot[Weight_tot != 0.]
# Define output directory
if Dir_out == None:
Dir_out = Dir_in
# Define output name
output_name = os.path.join(
Dir_out, files[int(row)].replace('16-daily', 'monthly'))
output_name = output_name[:-9] + '%02d.01.tif' % (date.month)
# Save tiff file
DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj)
return
def ETref(Date, args):
"""
This function starts to calculate ETref (daily) data based on Hydroshed, GLDAS, and (CFSR/LANDSAF) in parallel or single core
Keyword arguments:
Date -- panda timestamp
args -- includes all the parameters that are needed for the ETref
"""
# unpack the arguments
[Dir, lonlim, latlim, pixel_size, LANDSAF] = args
# Set the paths
nameTmin = 'Tair-min_GLDAS-NOAH_C_daily_' + Date.strftime(
'%Y.%m.%d') + ".tif"
tmin_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'tair_f_inst', 'min', nameTmin)
nameTmax = 'Tair-max_GLDAS-NOAH_C_daily_' + Date.strftime(
'%Y.%m.%d') + ".tif"
tmax_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'tair_f_inst', 'max', nameTmax)
nameHumid = 'Hum_GLDAS-NOAH_kg-kg_daily_' + Date.strftime(
'%Y.%m.%d') + ".tif"
humid_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'qair_f_inst', 'mean', nameHumid)
namePress = 'P_GLDAS-NOAH_kpa_daily_' + Date.strftime('%Y.%m.%d') + ".tif"
press_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'psurf_f_inst', 'mean', namePress)
nameWind = 'W_GLDAS-NOAH_m-s-1_daily_' + Date.strftime('%Y.%m.%d') + ".tif"
wind_str = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', 'daily',
'wind_f_inst', 'mean', nameWind)
if LANDSAF == 1:
nameShortClearname = 'ShortWave_Clear_Daily_W-m2_' + Date.strftime(
'%Y-%m-%d') + '.tif'
input2_str = os.path.join(Dir, 'Landsaf_Clipped',
'Shortwave_Clear_Sky', nameShortClearname)
nameShortNetname = 'ShortWave_Net_Daily_W-m2_' + Date.strftime(
'%Y-%m-%d') + '.tif'
input1_str = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net',
nameShortNetname)
input3_str = 'not'
else:
if Date < pd.Timestamp(pd.datetime(2011, 4, 1)):
nameDownLong = 'DLWR_CFSR_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input2_str = os.path.join(Dir, 'Radiation', 'CFSR', nameDownLong)
nameDownShort = 'DSWR_CFSR_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input1_str = os.path.join(Dir, 'Radiation', 'CFSR', nameDownShort)
nameUpLong = 'ULWR_CFSR_W-m2_' + Date.strftime('%Y.%m.%d') + ".tif"
input3_str = os.path.join(Dir, 'Radiation', 'CFSR', nameUpLong)
else:
nameDownLong = 'DLWR_CFSRv2_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input2_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameDownLong)
nameDownShort = 'DSWR_CFSRv2_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input1_str = os.path.join(Dir, 'Radiation', 'CFSRv2',
nameDownShort)
nameUpLong = 'ULWR_CFSRv2_W-m2_' + Date.strftime(
'%Y.%m.%d') + ".tif"
input3_str = os.path.join(Dir, 'Radiation', 'CFSRv2', nameUpLong)
# The day of year
DOY = Date.dayofyear
# Load DEM
if not pixel_size:
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
else:
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
dest, ulx, lry, lrx, uly, epsg_to = RC.reproject_dataset_epsg(
DEMmap_str, pixel_spacing=pixel_size, epsg_to=4326, method=2)
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_reshaped_for_ETref.tif')
DEM_data = dest.GetRasterBand(1).ReadAsArray()
geo_dem = [ulx, pixel_size, 0.0, uly, 0.0, -pixel_size]
DC.Save_as_tiff(name=DEMmap_str,
data=DEM_data,
geo=geo_dem,
projection='4326')
# Calc ETref
ETref = calc_ETref(Dir, tmin_str, tmax_str, humid_str, press_str, wind_str,
input1_str, input2_str, input3_str, DEMmap_str, DOY)
# Make directory for the MODIS ET data
output_folder = os.path.join(Dir, 'ETref', 'Daily')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Create the output names
NameETref = 'ETref_mm-day-1_daily_' + Date.strftime('%Y.%m.%d') + '.tif'
NameEnd = os.path.join(output_folder, NameETref)
# Collect geotiff information
geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str)
# Create daily ETref tiff files
DC.Save_as_tiff(name=NameEnd, data=ETref, geo=geo_out, projection=proj)
def RetrieveData(Date, args):
"""
This function retrieves TRMM data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
year = Date.year
month = Date.month
day = Date.day
from watools import WebAccounts
username, password = WebAccounts.Accounts(Type='NASA')
# Create https
if TimeCase == 'daily':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/GPM_L3/GPM_3IMERGDF.05/%d/%02d/3B-DAY.MS.MRG.3IMERG.%d%02d%02d-S000000-E235959.V05.nc4.ascii?precipitationCal[%d:1:%d][%d:1:%d]' % (
year, month, year, month, day, xID[0], xID[1] - 1, yID[0],
yID[1] - 1)
DirFile = os.path.join(
output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %
(year, month, day))
Scaling = 1
if TimeCase == 'monthly':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/hyrax/GPM_L3/GPM_3IMERGM.05/%d/3B-MO.MS.MRG.3IMERG.%d%02d01-S000000-E235959.%02d.V05B.HDF5.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
year, year, month, month, xID[0], xID[1] - 1, yID[0], yID[1] - 1)
Scaling = calendar.monthrange(year, month)[1] * 24
DirFile = os.path.join(
output_folder,
"P_GPM.IMERG_mm-month-1_monthly_%d.%02d.01.tif" % (year, month))
if not os.path.isfile(DirFile):
dataset = requests.get(URL, allow_redirects=False, stream=True)
try:
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
stream=True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
verify=False)
# download data (first save as text file)
pathtext = os.path.join(output_folder, 'temp.txt')
z = open(pathtext, 'wb')
z.write(get_dataset.content)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,
dtype=float,
skip_header=1,
delimiter=',')
data = data_start[:, 1:] * Scaling
data[data < 0] = -9999
data = data.transpose()
data = np.flipud(data)
# Delete .txt file
os.remove(pathtext)
# Make geotiff file
geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return True
def main(Dir, Startdate = '', Enddate = '',
latlim = [-60, 60], lonlim = [-180, 180], pixel_size = False, cores = False, LANDSAF = 0, SourceLANDSAF= '', Waitbar = 1):
"""
This function downloads TRMM3B43 V7 (monthly) data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine.
It can be 'False' to avoid using parallel computing
routines.
Waitbar -- 1 (Default) will print the waitbar
"""
print('Create monthly Reference ET data for period %s till %s' %(Startdate, Enddate))
# An array of monthly dates which will be calculated
Dates = pd.date_range(Startdate,Enddate,freq = 'MS')
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
# Calculate the ETref day by day for every month
for Date in Dates:
# Collect date data
Y=Date.year
M=Date.month
Mday=calendar.monthrange(Y,M)[1]
Days=pd.date_range(Date,Date+pd.Timedelta(days=Mday),freq='D')
StartTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+ '-01'
EndTime=Date.strftime('%Y')+'-'+Date.strftime('%m')+'-'+str(Mday)
# Get ETref on daily basis
daily(Dir=Dir, Startdate=StartTime,Enddate=EndTime,latlim=latlim, lonlim=lonlim, pixel_size = pixel_size, cores=cores, LANDSAF=LANDSAF, SourceLANDSAF=SourceLANDSAF, Waitbar = 0)
# Load DEM
if not pixel_size:
nameDEM='DEM_HydroShed_m_3s.tif'
DEMmap=os.path.join(Dir,'HydroSHED','DEM',nameDEM )
else:
DEMmap=os.path.join(Dir,'HydroSHED','DEM','DEM_HydroShed_m_reshaped_for_ETref.tif')
# Get some geo-data to save results
geo_ET, proj, size_X, size_Y = RC.Open_array_info(DEMmap)
dataMonth=np.zeros([size_Y,size_X])
for Day in Days[:-1]:
DirDay=os.path.join(Dir,'ETref','Daily','ETref_mm-day-1_daily_' + Day.strftime('%Y.%m.%d') + '.tif')
dataDay=gdal.Open(DirDay)
Dval=dataDay.GetRasterBand(1).ReadAsArray().astype(np.float32)
Dval[Dval<0]=0
dataMonth=dataMonth+Dval
dataDay=None
# make geotiff file
output_folder_month=os.path.join(Dir,'ETref','Monthly')
if os.path.exists(output_folder_month)==False:
os.makedirs(output_folder_month)
DirMonth=os.path.join(output_folder_month,'ETref_mm-month-1_monthly_'+Date.strftime('%Y.%m.%d') + '.tif')
# Create the tiff file
DC.Save_as_tiff(DirMonth,dataMonth, geo_ET, proj)
# Create Waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim):
"""
This function collects and clip LANDSAF data
Keyword arguments:
SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID)
Dir -- 'C:/' path to the WA map
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -60 and 60)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Make an array of the days of which the ET is taken
Dates = pd.date_range(Startdate, Enddate, freq='D')
# make directories
SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS')
if os.path.exists(SISdir) is False:
os.makedirs(SISdir)
SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID')
if os.path.exists(SIDdir) is False:
os.makedirs(SIDdir)
ShortwaveBasin(SourceLANDSAF,
Dir,
latlim,
lonlim,
Dates=[Startdate, Enddate])
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str)
# Open DEM map
demmap = RC.Open_tiff_array(DEMmap_str)
demmap[demmap < 0] = 0
# make lat and lon arrays)
dlat = geo_out[5]
dlon = geo_out[1]
lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat
lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon
for date in Dates:
# day of year
day = date.dayofyear
Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence(
demmap, lat, lon, day)
SIDname = os.path.join(
SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
SISname = os.path.join(
SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
#PREPARE SID MAPS
SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3)
SIDdata = SIDdest.GetRasterBand(1).ReadAsArray()
#PREPARE SIS MAPS
SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3)
SISdata = SISdest.GetRasterBand(1).ReadAsArray()
# Calculate ShortWave net
Short_Wave_Net = SIDdata * (Sloping /
Horizontal) + SISdata * 86400 / 1e6
# Calculate ShortWave Clear
Short_Wave = Sloping
Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5)
# make directories
PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky')
if os.path.exists(PathClear) is False:
os.makedirs(PathClear)
PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net')
if os.path.exists(PathNet) is False:
os.makedirs(PathNet)
# name Shortwave Clear and Net
nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameNet = os.path.join(PathNet, nameFileNet)
nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameClear = os.path.join(PathClear, nameFileClear)
# Save net and clear short wave radiation
DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj)
DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj)
return
def RetrieveData(args):
"""
This function retrieves JRC data for a given date from the
http://storage.googleapis.com/global-surface-water/downloads/ server.
Keyword arguments:
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, Names_to_download, lonlim, latlim] = args
# Collect the data from the JRC webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(Names_to_download, output_folder)
except:
print("Was not able to download the file")
# Clip the data to the users extend
if len(Names_to_download) == 1:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Names_to_download[0])
data_end, geo_end = RC.clip_data(data_in, latlim, lonlim)
else:
data_end = np.zeros([
int((latlim[1] - latlim[0]) / 0.00025),
int((lonlim[1] - lonlim[0]) / 0.00025)
])
for Name_to_merge in Names_to_download:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Name_to_merge)
geo_out, proj, size_X, size_Y = RC.Open_array_info(data_in)
lat_min_merge = np.maximum(latlim[0],
geo_out[3] + size_Y * geo_out[5])
lat_max_merge = np.minimum(latlim[1], geo_out[3])
lon_min_merge = np.maximum(lonlim[0], geo_out[0])
lon_max_merge = np.minimum(lonlim[1],
geo_out[0] + size_X * geo_out[1])
lonmerge = [lon_min_merge, lon_max_merge]
latmerge = [lat_min_merge, lat_max_merge]
data_one, geo_one = RC.clip_data(data_in, latmerge, lonmerge)
Ystart = int((geo_one[3] - latlim[1]) / geo_one[5])
Yend = int(Ystart + np.shape(data_one)[0])
Xstart = int((geo_one[0] - lonlim[0]) / geo_one[1])
Xend = int(Xstart + np.shape(data_one)[1])
data_end[Ystart:Yend, Xstart:Xend] = data_one
geo_end = tuple([lonlim[0], geo_one[1], 0, latlim[1], 0, geo_one[5]])
# Save results as Gtiff
fileName_out = os.path.join(output_folder, 'JRC_Occurrence_percent.tif')
DC.Save_as_tiff(name=fileName_out,
data=data_end,
geo=geo_end,
projection='WGS84')
shutil.rmtree(trash_folder)
return True
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar):
# Create an array with the dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define the minimum and maximum lat and long ETensemble Tile
Min_lat_tile = int(np.floor((100 - latlim[1]) / 10))
Max_lat_tile = int(np.floor((100 - latlim[0] - 0.00125) / 10))
Min_lon_tile = int(np.floor((190 + lonlim[0]) / 10))
Max_lon_tile = int(np.floor((190 + lonlim[1] - 0.00125) / 10))
# Create the Lat and Lon tiles that will be downloaded
Lat_tiles = [Min_lat_tile, Max_lat_tile]
Lon_tiles = [Min_lon_tile, Max_lon_tile]
# Define output folder and create this if it not exists
output_folder = os.path.join(Dir, 'Evaporation', 'ETensV1_0')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Create Geotransform of the output files
GEO_1 = lonlim[0]
GEO_2 = 0.0025
GEO_3 = 0.0
GEO_4 = latlim[1]
GEO_5 = 0.0
GEO_6 = -0.0025
geo = [GEO_1, GEO_2, GEO_3, GEO_4, GEO_5, GEO_6]
geo_new = tuple(geo)
# Define the parameter for downloading the data
Downloaded = 0
# Calculate the ET data date by date
for Date in Dates:
# Define the output name and folder
file_name = 'ET_ETensemble250m_mm-month-1_monthly_%d.%02d.01.tif' % (
Date.year, Date.month)
output_file = os.path.join(output_folder, file_name)
# If output file not exists create this
if not os.path.exists(output_file):
# If not downloaded than download
if Downloaded == 0:
# Download the ETens data from the FTP server
Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles)
# Unzip the folder
Unzip_ETens_data(output_folder, Lat_tiles, Lon_tiles)
Downloaded = 1
# Create the ET data for the area of interest
ET_data = Collect_dataset(output_folder, Date, Lat_tiles,
Lon_tiles, latlim, lonlim)
# Save this array as a tiff file
DC.Save_as_tiff(output_file, ET_data, geo_new, projection='WGS84')
# Create Waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
'''
# Remove all the raw dataset
for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1):
for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1):
Tilename = "h%sv%s" %(h_tile, v_tile)
filename = os.path.join(output_folder, Tilename)
if os.path.exists(filename):
shutil.rmtree(filename)
# Remove all .zip files
for f in os.listdir(output_folder):
if re.search(".zip", f):
os.remove(os.path.join(output_folder, f))
'''
return ()
