def Save_As_Tiff(self, output_folder):
if not os.path.exists(output_folder):
os.makedirs(output_folder)
Dates = self.Ordinal_time
geo = self.GeoTransform
proj = self.Projection
Data = self.Data
Variable = self.Variable
Unit = self.Unit
print("Save %s as tiff file in %s" % (Variable, output_folder))
if Dates == "Long_Term_Decade":
import WaporTranslator.LEVEL_2.Functions as Functions
Dates_dek = Functions.Get_Dekads(2009, 2009)
i = 0
for Date in Dates_dek:
try:
output_filename = os.path.join(
output_folder,
"%s_%s_%02d.%02d.tif" % (Variable.replace(
" ", "_"), Unit, Date.month, Date.day))
Data_one = Data[i, :, :]
DC.Save_as_tiff(output_filename, Data_one, geo, proj)
i += 1
except:
print("Was not able to collect %s %s" % (Variable, Date))
elif np.any(Dates != None):
for Date in Dates:
Date_datetime = datetime.datetime.fromordinal(Date)
output_filename = os.path.join(
output_folder, "%s_%s_%d.%02d.%02d.tif" %
(Variable.replace(" ", "_"), Unit, Date_datetime.year,
Date_datetime.month, Date_datetime.day))
Data_one = np.squeeze(Data[Dates == Date, :, :], 0)
DC.Save_as_tiff(output_filename, Data_one, geo, proj)
else:
output_filename = os.path.join(
output_folder,
"%s_%s.tif" % (Variable.replace(" ", "_"), Unit))
DC.Save_as_tiff(output_filename, Data, geo, proj)
def main(output_folder_L1, dest_AOI_MASK, Threshold_Mask):
# select input folder
input_folder_LCC = os.path.join(output_folder_L1, "L2_LCC_A")
# Set input folder as working directory
os.chdir(input_folder_LCC)
# Find file for extend
re = glob.glob("L2_LCC_A_*.tif")[0]
# Open file
Filename_Example = os.path.join(input_folder_LCC, re)
# Open info array
dest_shp = RC.reproject_dataset_example(dest_AOI_MASK, Filename_Example, 4)
geo = dest_shp.GetGeoTransform()
proj = dest_shp.GetProjection()
# Array mask
Mask_with_Edge = dest_shp.GetRasterBand(1).ReadAsArray()
# Create End Mask
Mask = np.where(Mask_with_Edge < Threshold_Mask / 100., np.nan, 1)
# output file
output_file = os.path.join(output_folder_L1, "MASK", "MASK.tif")
DC.Save_as_tiff(output_file, Mask, geo, proj)
return ()
def RetrieveData(Date, args):
"""
This function retrieves TRMM data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
year = Date.year
month= Date.month
day = Date.day
from watertools import WebAccounts
username, password = WebAccounts.Accounts(Type = 'NASA')
# Create https
if TimeCase == 'daily':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/GPM_L3/GPM_3IMERGDF.05/%d/%02d/3B-DAY.MS.MRG.3IMERG.%d%02d%02d-S000000-E235959.V05.nc4.ascii?precipitationCal[%d:1:%d][%d:1:%d]' %(year, month, year, month, day, xID[0], xID[1]-1, yID[0], yID[1]-1)
DirFile = os.path.join(output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %(year, month, day))
Scaling = 1
if TimeCase == 'monthly':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/hyrax/GPM_L3/GPM_3IMERGM.05/%d/3B-MO.MS.MRG.3IMERG.%d%02d01-S000000-E235959.%02d.V05B.HDF5.ascii?precipitation[%d:1:%d][%d:1:%d]' %(year, year, month, month, xID[0], xID[1]-1, yID[0], yID[1]-1)
Scaling = calendar.monthrange(year,month)[1] * 24
DirFile = os.path.join(output_folder, "P_GPM.IMERG_mm-month-1_monthly_%d.%02d.01.tif" %(year, month))
if not os.path.isfile(DirFile):
dataset = requests.get(URL, allow_redirects=False,stream = True)
try:
get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'], auth = (username, password), verify = False)
# download data (first save as text file)
pathtext = os.path.join(output_folder,'temp.txt')
z = open(pathtext,'wb')
z.write(get_dataset.content)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,dtype = float,skip_header = 1,delimiter=',')
data = data_start[:,1:] * Scaling
data[data < 0] = -9999
data = data.transpose()
data = np.flipud(data)
# Delete .txt file
os.remove(pathtext)
# Make geotiff file
geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return True
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim):
import watertools.General.raster_conversions as RC
# Open Dataset
HiHydroSoil_Array = RC.Open_tiff_array(local_filename)
# Define area
XID = [
int(np.floor((180 + lonlim[0]) / 0.00833333)),
int(np.ceil((180 + lonlim[1]) / 0.00833333))
]
YID = [
int(np.ceil((90 - latlim[1]) / 0.00833333)),
int(np.floor((90 - latlim[0]) / 0.00833333))
]
# Define Georeference
geo = tuple([
-180 + 0.00833333 * XID[0], 0.00833333, 0, 90 - 0.00833333 * YID[0], 0,
-0.00833333
])
# Clip Array
HiHydroSoil_Array_clipped = HiHydroSoil_Array[YID[0]:YID[1], XID[0]:XID[1]]
# Save tiff file
DC.Save_as_tiff(Filename_out, HiHydroSoil_Array_clipped, geo, "WGS84")
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID, TimeStep):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if TimeStep is "weekly":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
if TimeStep is "daily":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
if TimeStep is "weekly":
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
if TimeStep is "daily":
DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])
raw_data = np.fromfile(os.path.splitext(local_filename)[0],
dtype="<f4")
dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
data = dataset[
yID[0]:yID[1],
xID[0]:xID[1]] / 2.45 # Values are in MJ/m2d so convert to mm/d
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return
def RetrieveData(Date, args):
"""
This function retrieves MOD16 ET data for a given date from the
ftp://ftp.ntsg.umt.edu/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, latlim, lonlim,
timestep, hdf_library, Size_pix
] = args
if timestep == 'monthly':
ETfileName = os.path.join(
output_folder, 'ET_MOD16A2_mm-month-1_monthly_' +
Date.strftime('%Y') + '.' + Date.strftime('%m') + '.01.tif')
elif timestep == '8-daily':
ETfileName = os.path.join(
output_folder,
'ET_MOD16A2_mm-8days-1_8-daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if not os.path.exists(ETfileName):
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
timestep, hdf_library, Size_pix)
except:
print("Was not able to download the file")
try:
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
DC.Save_as_tiff(name=ETfileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print("Failed for date: %s" % Date)
return ()
def gap_filling(dataset, NoDataValue, method=1):
"""
This function fills the no data gaps in a numpy array
Keyword arguments:
dataset -- 'C:/' path to the source data (dataset that must be filled)
NoDataValue -- Value that must be filled
"""
import watertools.General.data_conversions as DC
try:
if dataset.split('.')[-1] == 'tif':
# Open the numpy array
data = Open_tiff_array(dataset)
Save_as_tiff = 1
else:
data = dataset
Save_as_tiff = 0
except:
data = dataset
Save_as_tiff = 0
# fill the no data values
if NoDataValue is np.nan:
mask = ~(np.isnan(data))
else:
mask = ~(data == NoDataValue)
xx, yy = np.meshgrid(np.arange(data.shape[1]), np.arange(data.shape[0]))
xym = np.vstack((np.ravel(xx[mask]), np.ravel(yy[mask]))).T
data0 = np.ravel(data[:, :][mask])
if method == 1:
interp0 = scipy.interpolate.NearestNDInterpolator(xym, data0)
data_end = interp0(np.ravel(xx), np.ravel(yy)).reshape(xx.shape)
if method == 2:
interp0 = scipy.interpolate.LinearNDInterpolator(xym, data0)
data_end = interp0(np.ravel(xx), np.ravel(yy)).reshape(xx.shape)
if Save_as_tiff == 1:
EndProduct = dataset[:-4] + '_GF.tif'
# collect the geoinformation
geo_out, proj, size_X, size_Y = Open_array_info(dataset)
# Save the filled array as geotiff
DC.Save_as_tiff(name=EndProduct,
data=data_end,
geo=geo_out,
projection=proj)
else:
EndProduct = data_end
return (EndProduct)
def RetrieveData(Date, args):
"""
This function retrieves MOD15 FPAR data for a given date from the
http://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim, unit,
dataset, nameDownload, hdf_library
] = args
FPARfileName = os.path.join(
output_folder,
'%s_MOD15_%s_8-daily_' % (dataset, unit) + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if not os.path.exists(FPARfileName):
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
nameDownload, hdf_library)
except:
print("Was not able to download the file")
try:
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
# Save the file as tiff
DC.Save_as_tiff(name=FPARfileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print("Failed for date: %s" % Date)
return True
def Create_LU_MAP(output_folder, Dates_yearly, LU, LUdek, Paths_LU_ESA, Formats_LU_ESA, example_file):
# Create output folder LVL2
output_folder_L2 = os.path.join(output_folder, "LEVEL_2")
# Open ESACCI
input_file_LU_ESACCI = os.path.join(output_folder, Paths_LU_ESA, Formats_LU_ESA)
# Converting LU maps into one LU map
# open dictionary WAPOR
WAPOR_Conversions_dict = WAPOR_Conversions()
# open dictionary ESACCI
ESACCI_Conversions_dict = ESACCI_Conversions()
Phenology_pixels_year = np.ones(LUdek.Size) * np.nan
Grassland_pixels_year = np.ones(LUdek.Size) * np.nan
# Loop over the years
for Year in Dates_yearly:
Year_start = int(Dates_yearly[0].year)
Year_int = int(Year.year)
geo = LU.GeoTransform
proj = LU.Projection
destLUESACCI = RC.reproject_dataset_example(input_file_LU_ESACCI, example_file)
LU_ESACCI = destLUESACCI.GetRasterBand(1).ReadAsArray()
# Create LUmap
LU_Map_WAPOR = np.ones([LU.Size[1], LU.Size[2]]) * np.nan
LU_Map_ESACCI = np.ones([LU.Size[1], LU.Size[2]]) * np.nan
for number in WAPOR_Conversions_dict.items():
LU_Map_WAPOR = np.where(LU.Data[int((Year_int - Year_start)),: ,:] == number[0], number[1], LU_Map_WAPOR)
for number in ESACCI_Conversions_dict.items():
LU_Map_ESACCI = np.where(LU_ESACCI == number[0], number[1], LU_Map_ESACCI)
# Combine LU maps
# 1 = rainfed, 2 = irrigated, 3 = Pasture
LU_END = np.where(np.logical_and(LU_Map_WAPOR == 1, LU_Map_ESACCI == 1), 1, np.nan)
LU_END = np.where(LU_Map_WAPOR > 1, LU_Map_WAPOR, LU_END)
# Save LU map
DC.Save_as_tiff(os.path.join(output_folder_L2, "LU_END", "LU_%s.tif" %Year_int), LU_END, geo, proj)
# find posible Perennial pixels
Phenology_pixels_year[int((Year_int - Year_start) * 36):int((Year_int - Year_start) * 36)+36,: ,:] = np.where(np.logical_or(LU_END==1, LU_END==2), 1, np.nan)[None, :, :]
Grassland_pixels_year[int((Year_int - Year_start) * 36):int((Year_int - Year_start) * 36)+36,: ,:] = np.where(LU_END==3, 1, np.nan)[None, :, :]
return(Phenology_pixels_year, Grassland_pixels_year)
def RetrieveData(args):
"""
This function retrieves JRC data for a given date from the
http://storage.googleapis.com/global-surface-water/downloads/ server.
Keyword arguments:
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, Names_to_download, lonlim, latlim] = args
# Collect the data from the JRC webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(Names_to_download, output_folder)
except:
print("Was not able to download the file")
# Clip the data to the users extend
if len(Names_to_download) == 1:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Names_to_download[0])
data_end, geo_end = RC.clip_data(data_in, latlim, lonlim)
else:
data_end = np.zeros([int((latlim[1] - latlim[0])/0.00025), int((lonlim[1] - lonlim[0])/0.00025)])
for Name_to_merge in Names_to_download:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Name_to_merge)
geo_out, proj, size_X, size_Y = RC.Open_array_info(data_in)
lat_min_merge = np.maximum(latlim[0], geo_out[3] + size_Y * geo_out[5])
lat_max_merge = np.minimum(latlim[1], geo_out[3])
lon_min_merge = np.maximum(lonlim[0], geo_out[0])
lon_max_merge = np.minimum(lonlim[1], geo_out[0] + size_X * geo_out[1])
lonmerge = [lon_min_merge, lon_max_merge]
latmerge = [lat_min_merge, lat_max_merge]
data_one, geo_one = RC.clip_data(data_in, latmerge, lonmerge)
Ystart = int((geo_one[3] - latlim[1])/geo_one[5])
Yend = int(Ystart + np.shape(data_one)[0])
Xstart = int((geo_one[0] - lonlim[0])/geo_one[1])
Xend = int(Xstart + np.shape(data_one)[1])
data_end[Ystart:Yend, Xstart:Xend] = data_one
geo_end = tuple([lonlim[0], geo_one[1], 0, latlim[1], 0, geo_one[5]])
# Save results as Gtiff
fileName_out = os.path.join(output_folder, 'JRC_Occurrence_percent.tif')
DC.Save_as_tiff(name=fileName_out, data=data_end, geo=geo_end, projection='WGS84')
shutil.rmtree(trash_folder)
return True
def RetrieveData(Date, args):
"""
This function retrieves MOD9 Reflectance data for a given date from the
http://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim, band,
resolution, hdf_library
] = args
ReffileName = os.path.join(
output_folder,
'ReflectanceBand%d_MOD09GQ_-_daily_' % band + Date.strftime('%Y') +
'.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if not os.path.exists(ReffileName):
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
band, resolution, hdf_library)
except:
print("Was not able to download the file")
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
try:
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
# Save results as Gtiff
DC.Save_as_tiff(name=ReffileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print('data for %02d-%02d-%d is not available' %
(Date.day, Date.month, Date.year))
return True
def Calc_Humidity(Temp_format,
P_format,
Hum_format,
output_format,
Startdate,
Enddate,
freq="D"):
folder_dir_out = os.path.dirname(output_format)
if not os.path.exists(folder_dir_out):
os.makedirs(folder_dir_out)
Dates = pd.date_range(Startdate, Enddate, freq="D")
for Date in Dates:
print(Date)
Day = Date.day
Month = Date.month
Year = Date.year
Tempfile_one = Temp_format.format(yyyy=Year, mm=Month, dd=Day)
Presfile_one = P_format.format(yyyy=Year, mm=Month, dd=Day)
Humfile_one = Hum_format.format(yyyy=Year, mm=Month, dd=Day)
out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day)
geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one)
Tdata = RC.Open_tiff_array(Tempfile_one)
if "MERRA_K" in Temp_format:
Tdata = Tdata - 273.15
Tdata[Tdata < -900] = -9999
Pdata = RC.Open_tiff_array(Presfile_one)
Hdata = RC.Open_tiff_array(Humfile_one)
Pdata[Pdata < 0] = -9999
Hdata[Hdata < 0] = -9999
# gapfilling
Tdata = RC.gap_filling(Tdata, -9999)
Pdata = RC.gap_filling(Pdata, -9999)
Hdata = RC.gap_filling(Hdata, -9999)
Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3))
HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100
HumData = HumData.clip(0, 100)
DC.Save_as_tiff(out_folder_one, HumData, geo_out, "WGS84")
return ()
def DownloadData(Dir, latlim, lonlim, Waitbar):
"""
This function downloads NLDAS Forcing data hourly, daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
"""
# Define the output name
output_filename = os.path.join(Dir, 'LU_ESACCI.tif')
# Set the url of the server
url = r"https://storage.googleapis.com/cci-lc-v207/ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.zip"
# Create a Trash folder
Dir_trash = os.path.join(Dir, "Trash")
if not os.path.exists(Dir_trash):
os.makedirs(Dir_trash)
# Define location of download
filename_out = os.path.join(
Dir_trash, "ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.zip")
# Download data
urllib.request.urlretrieve(url, filename=filename_out)
# Extract data
DC.Extract_Data(filename_out, Dir_trash)
# Define input of the world tiff file
filename_world = os.path.join(
Dir_trash, "product", "ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.tif")
try:
# Clip data to user extend
data, Geo_out = RC.clip_data(filename_world, latlim, lonlim)
# Save data of clipped array
DC.Save_as_tiff(output_filename, data, Geo_out, 4326)
except:
RC.Clip_Dataset_GDAL(RC.clip_data(filename_world, latlim, lonlim))
# Remove trash folder
shutil.rmtree(Dir_trash)
return ()
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat.Subsoil(
Dir, latlim, lonlim)
filedir_out_whc = os.path.join(Dir, 'SoilGrids', 'Water_Holding_Capacity')
if not os.path.exists(filedir_out_whc):
os.makedirs(filedir_out_whc)
# Define theta field capacity output
filename_out_whc = os.path.join(
filedir_out_whc,
'Water_Holding_Capacity_%s_SoilGrids_mm-m.tif' % level)
if not os.path.exists(filename_out_whc):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate theta field capacity
theta_whc = np.ones(theta_sat.shape) * -9999
theta_whc = np.where(
theta_sat < 0.301, 80,
450 * np.arccosh(theta_sat + 0.7) - 2 * (theta_sat + 0.7) + 20)
# Save as tiff
DC.Save_as_tiff(filename_out_whc, theta_whc, geo_out, proj)
return
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat2.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat2.Subsoil(
Dir, latlim, lonlim)
filedir_out_n_genuchten = os.path.join(Dir, 'SoilGrids', 'N_van_genuchten')
if not os.path.exists(filedir_out_n_genuchten):
os.makedirs(filedir_out_n_genuchten)
# Define n van genuchten output
filename_out_ngenuchten = os.path.join(
filedir_out_n_genuchten, 'N_genuchten_%s_SoilGrids_-.tif' % level)
if not os.path.exists(filename_out_ngenuchten):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate n van genuchten
n_van_genuchten = np.ones(theta_sat.shape) * -9999
n_van_genuchten = 166.63 * theta_sat**4 - 387.72 * theta_sat**3 + 340.55 * theta_sat**2 - 133.07 * theta_sat + 20.739
# Save as tiff
DC.Save_as_tiff(filename_out_ngenuchten, n_van_genuchten, geo_out,
proj)
return
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
"""
This function retrieves GWF data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Keyword arguments:
output_folder -- name of the end file with the weekly ALEXI data
End_filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Set the file names and directories
filename = "Gray_Water_Footprint.tif"
local_filename = os.path.join(output_folder, filename)
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Clip extend out of world data
dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
# make geotiff file
DC.Save_as_tiff(name=filename_Out,
data=dataset,
geo=Geo_out,
projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print("file not exists")
return
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim):
# Open Dataset
SEBS_Array = spio.loadmat(local_filename)['ETm']
# Define area
XID = [int(np.floor((180 + lonlim[0])/0.05)), int(np.ceil((180 + lonlim[1])/0.05))]
YID = [int(np.ceil((90 - latlim[1])/0.05)), int(np.floor((90 - latlim[0])/0.05))]
# Define Georeference
geo = tuple([-180 + 0.05*XID[0],0.05,0,90 - 0.05*YID[0],0,-0.05])
# Clip Array
SEBS_Array_clipped = SEBS_Array[YID[0]:YID[1], XID[0]:XID[1]] * 0.1
# Save tiff file
DC.Save_as_tiff(Filename_out, SEBS_Array_clipped, geo, "WGS84")
def Calc_Humidity(Temp_format,
P_format,
Hum_format,
output_format,
Startdate,
Enddate,
freq="D"):
folder_dir_out = os.path.dirname(output_format)
if not os.path.exists(folder_dir_out):
os.makedirs(folder_dir_out)
Dates = pd.date_range(Startdate, Enddate, freq="D")
for Date in Dates:
print(Date)
Day = Date.day
Month = Date.month
Year = Date.year
Windyfile_one = wind_y_format.format(yyyy=Year, mm=Month, dd=Day)
Windxfile_one = wind_x_format.format(yyyy=Year, mm=Month, dd=Day)
out_folder_one = output_format.format(yyyy=Year, mm=Month, dd=Day)
geo_out, proj, size_X, size_Y = RC.Open_array_info(Windxfile_one)
Windxdata = RC.Open_tiff_array(Windxfile_one)
Windxdata[Windxdata < -900] = -9999
Windydata = RC.Open_tiff_array(Windyfile_one)
Windydata[Windxdata < -900] = -9999
# gapfilling
Windydata = RC.gap_filling(Windydata, -9999)
Windxdata = RC.gap_filling(Windxdata, -9999)
Wind = np.sqrt(Windxdata**2 + Windydata**2)
DC.Save_as_tiff(out_folder_one, Wind, geo_out, "WGS84")
return ()
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(Dir, 'SoilGrids', 'Theta_Sat' ,'Theta_Sat2_%s_SoilGrids_kg-kg.tif' %level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat2.Topsoil(Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat2.Subsoil(Dir, latlim, lonlim)
filedir_out_thetares = os.path.join(Dir, 'SoilGrids', 'Theta_Res')
if not os.path.exists(filedir_out_thetares):
os.makedirs(filedir_out_thetares)
# Define theta field capacity output
filename_out_thetares = os.path.join(filedir_out_thetares ,'Theta_Res_%s_SoilGrids_kg-kg.tif' %level)
if not os.path.exists(filename_out_thetares):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
# Calculate theta field capacity
theta_Res = np.ones(theta_sat.shape) * -9999
#theta_Res = np.where(theta_sat < 0.351, 0.01, 0.4 * np.arccosh(theta_sat + 0.65) - 0.05 * np.power(theta_sat + 0.65, 2.5) + 0.02)
theta_Res = np.where(theta_sat < 0.351, 0.01, 0.271 * np.log(theta_sat) + 0.335)
# Save as tiff
DC.Save_as_tiff(filename_out_thetares, theta_Res, geo_out, proj)
return
def lapse_rate(Dir,temperature_map, DEMmap):
"""
This function downscales the GLDAS temperature map by using the DEM map
Keyword arguments:
temperature_map -- 'C:/' path to the temperature map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average altitudes corresponding to T resolution
dest = RC.reproject_dataset_example(DEMmap, temperature_map,method = 4)
DEM_ave_out_name = os.path.join(Dir,'HydroSHED', 'DEM','DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(temperature_map)
DEM_ave_data = dest.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
dest = None
# determine lapse-rate [degress Celcius per meter]
lapse_rate_number = 0.0065
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2)
dem_avg=dest.GetRasterBand(1).ReadAsArray()
dem_avg[dem_avg<0]=0
dest = None
# Open the temperature dataset
dest = RC.reproject_dataset_example(temperature_map, DEMmap, method = 2)
T=dest.GetRasterBand(1).ReadAsArray()
dest = None
# Open Demmap
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap<=0]=0
demmap[demmap==-32768]=np.nan
# calculate first part
T = T + ((dem_avg-demmap) * lapse_rate_number)
return T
def Calc_daily_radiation(folder_in, folder_out, Date):
filename_trans = "SDS_MSGCPP_W-m-2_15min_%d.%02d.%02d_H{hour}.M{minutes}.tif" % (
Date.year, Date.month, Date.day)
filename_trans_out = "SDS_MSGCPP_W-m-2_daily_%d.%02d.%02d.tif" % (
Date.year, Date.month, Date.day)
Dir_trans_out = os.path.join(folder_out, filename_trans_out)
if not os.path.exists(Dir_trans_out):
os.chdir(folder_in)
files = glob.glob(filename_trans.format(hour="*", minutes="*"))
i = 0
# Open all the 15 minutes files
for file in files:
file_in = os.path.join(folder_in, file)
destswgone = gdal.Open(file_in)
try:
swgnet_one = destswgone.GetRasterBand(1).ReadAsArray()
swgnet_one[swgnet_one < 0] = 0
if not "geo_trans" in locals():
swgnet = np.ones([
destswgone.RasterYSize, destswgone.RasterXSize,
len(files)
]) * np.nan
geo_trans = destswgone.GetGeoTransform()
proj_trans = destswgone.GetProjection()
swgnet[:, :, i] = swgnet_one
except:
pass
i += 1
# Calculate the daily mean
swgnet_mean = np.nansum(swgnet, 2) / (24 * 4)
DC.Save_as_tiff(Dir_trans_out, swgnet_mean, geo_trans, proj_trans)
return ()
def adjust_P(Dir, pressure_map, DEMmap):
"""
This function downscales the GLDAS air pressure map by using the DEM map
Keyword arguments:
pressure_map -- 'C:/' path to the pressure map
DEMmap -- 'C:/' path to the DEM map
"""
# calculate average latitudes
destDEMave = RC.reproject_dataset_example(DEMmap, pressure_map, method = 4)
DEM_ave_out_name = os.path.join(Dir, 'HydroSHED', 'DEM','DEM_ave.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(pressure_map)
DEM_ave_data = destDEMave.GetRasterBand(1).ReadAsArray()
DC.Save_as_tiff(DEM_ave_out_name, DEM_ave_data, geo_out, proj)
# open maps as numpy arrays
dest = RC.reproject_dataset_example(DEM_ave_out_name, DEMmap, method = 2)
dem_avg=dest.GetRasterBand(1).ReadAsArray()
dest = None
# open maps as numpy arrays
dest = RC.reproject_dataset_example(pressure_map, DEMmap, method = 2)
P=dest.GetRasterBand(1).ReadAsArray()
dest = None
demmap = RC.Open_tiff_array(DEMmap)
dem_avg[demmap<=0]=0
demmap[demmap==-32768]=np.nan
# calculate second part
P = P + (101.3*((293-0.0065*(demmap-dem_avg))/293)**5.26 - 101.3)
os.remove(DEM_ave_out_name)
return P
def Calc_Humidity(output_folder_L1, output_filename, method):
# find date
year = int((output_filename.split("_")[-1]).split(".")[0])
month = int((output_filename.split("_")[-1]).split(".")[1])
day = int((output_filename.split("_")[-1]).split(".")[2])
if method == "Daily":
# find paths
file_t = os.path.join(
output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(),
"tair_f_inst", "mean", "Tair_GLDAS-NOAH_C_%s_%d.%02d.%02d.tif" %
(method.lower(), year, month, day))
file_h = os.path.join(
output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(),
"qair_f_inst", "mean", "Hum_GLDAS-NOAH_kg-kg_%s_%d.%02d.%02d.tif" %
(method.lower(), year, month, day))
file_p = os.path.join(
output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(),
"psurf_f_inst", "mean", "P_GLDAS-NOAH_kpa_%s_%d.%02d.%02d.tif" %
(method.lower(), year, month, day))
if method == "Monthly":
# find paths
file_t = os.path.join(
output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(),
"tair_f_inst", "Tair_GLDAS-NOAH_C_%s_%d.%02d.%02d.tif" %
(method.lower(), year, month, day))
file_h = os.path.join(
output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(),
"qair_f_inst", "Hum_GLDAS-NOAH_kg-kg_%s_%d.%02d.%02d.tif" %
(method.lower(), year, month, day))
file_p = os.path.join(
output_folder_L1, "Weather_Data", "Model", "GLDAS", method.lower(),
"psurf_f_inst", "P_GLDAS-NOAH_kpa_%s_%d.%02d.%02d.tif" %
(method.lower(), year, month, day))
# Open paths
destt = gdal.Open(file_t)
desth = gdal.Open(file_h)
destp = gdal.Open(file_p)
# Get Geo information
geo = destt.GetGeoTransform()
proj = destt.GetProjection()
# Open Arrays
Array_T = destt.GetRasterBand(1).ReadAsArray()
Array_H = desth.GetRasterBand(1).ReadAsArray()
Array_P = destp.GetRasterBand(1).ReadAsArray()
Array_T[Array_T == -9999] = np.nan
Array_H[Array_H == -9999] = np.nan
Array_P[Array_P == -9999] = np.nan
# Calculate relative humidity
ESArray = 0.6108 * np.exp((17.27 * Array_T) / (Array_T + 237.3))
RHarray = np.minimum((1.6077717 * Array_H * Array_P / ESArray), 1) * 100
# Save array
DC.Save_as_tiff(output_filename, RHarray, geo, proj)
return ()
def DownloadData(Dir, Startdate, Enddate, S2_tile):
# Import watertools modules
import watertools.General.data_conversions as DC
# Define the dates
Dates = pd.date_range(Startdate, Enddate, freq="D")
# Create output folder
output_folder_end = os.path.join(Dir, S2_tile)
if not os.path.exists(output_folder_end):
os.makedirs(output_folder_end)
# Loop over the days
for Date in Dates:
# Get the datum
doy = int(Date.dayofyear)
year = Date.year
try:
# Create the right url
url = "https://hls.gsfc.nasa.gov/data/v1.4/L30/%s/%s/%s/%s/%s/HLS.L30.T%s.%s%03d.v1.4.hdf" % (
year, S2_tile[0:2], S2_tile[2:3], S2_tile[3:4], S2_tile[4:5],
S2_tile, year, doy)
filename_out = os.path.join(
output_folder_end,
"HLS.L30.T%s.%s%03d.v1.4.hdf" % (S2_tile, year, doy))
# Download the data
urllib.request.urlretrieve(url, filename=filename_out)
# Create a folder for the end results
folder_tile = os.path.join(
output_folder_end,
"HLS_L30_T%s_%s%03d_v1_4" % (S2_tile, year, doy))
if not os.path.exists(folder_tile):
os.makedirs(folder_tile)
# Write the hdf file in tiff files and store it in the folder
dataset = gdal.Open(filename_out)
sdsdict = dataset.GetMetadata('SUBDATASETS')
for Band in range(1, 12):
dest = gdal.Open(sdsdict["SUBDATASET_%d_NAME" % Band])
Array = dest.GetRasterBand(1).ReadAsArray()
if Band < 8.:
Array = Array * 0.0001
Array[Array == -0.1] = -9999.
Band_name = "B%02d" % (Band)
if Band == 8.:
Band_name = "B%02d" % (Band + 1)
Array = Array * 0.0001
Array[Array == -0.1] = -9999.
if (Band >= 9. and Band < 11.):
Band_name = "B%02d" % (Band + 1)
Array = Array * 0.01
Array[Array == -100] = -9999.
if Band == 11.:
Array[Array == 255] = -9999.
Band_name = "QC"
meta = dataset.GetMetadata()
ulx = int(meta["ULX"])
uly = int(meta["ULY"])
size = int(meta["SPATIAL_RESOLUTION"])
projection_zone = int(
meta["HORIZONTAL_CS_NAME"].split(" ")[-1])
projection = dest.GetProjection()
time_string = meta["SENSING_TIME"].split(";")[-1][:-9]
import datetime
Time = datetime.datetime.strptime(time_string.replace(" ", ""),
"%Y-%m-%dT%H:%M:%S")
hour = int(Time.hour)
minute = int(Time.minute)
geo = tuple([ulx, size, 0, uly, 0, -size])
DC.Save_as_tiff(
os.path.join(
folder_tile, "HLS_L30_T%s_%s%03d_H%02dM%02d_%s.tif" %
(S2_tile, year, doy, hour, minute, Band_name)), Array,
geo, projection)
except:
pass
return ()
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None):
"""
This functions calculates monthly tiff files based on the 8 daily tiff files. (will calculate the average)
Parameters
----------
Dir_in : str
Path to the input data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Dir_out : str
Path to the output data, default is same as Dir_in
"""
# import WA+ modules
import watertools.General.data_conversions as DC
import watertools.General.raster_conversions as RC
# Change working directory
os.chdir(Dir_in)
# Find all eight daily files
files = glob.glob('*8-daily*.tif')
# Create array with filename and keys (DOY and year) of all the 8 daily files
i = 0
DOY_Year = np.zeros([len(files), 3])
for File in files:
# Get the time characteristics from the filename
year = File.split('.')[-4][-4:]
month = File.split('.')[-3]
day = File.split('.')[-2]
# Create pandas Timestamp
date_file = '%s-%02s-%02s' % (year, month, day)
Datum = pd.Timestamp(date_file)
# Get day of year
DOY = Datum.strftime('%j')
# Save data in array
DOY_Year[i, 0] = i
DOY_Year[i, 1] = DOY
DOY_Year[i, 2] = year
# Loop over files
i += 1
# Check enddate:
Enddate_split = Enddate.split('-')
month_range = calendar.monthrange(int(Enddate_split[0]),
int(Enddate_split[1]))[1]
Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int(
Enddate_split[1]), month_range)
# Check startdate:
Startdate_split = Startdate.split('-')
Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int(
Startdate_split[1]))
# Define end and start date
Dates = pd.date_range(Startdate, Enddate, freq='MS')
DatesEnd = pd.date_range(Startdate, Enddate, freq='M')
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value
dest = gdal.Open(files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Loop over months and create monthly tiff files
i = 0
for date in Dates:
# Get Start and end DOY of the current month
DOY_month_start = date.strftime('%j')
DOY_month_end = DatesEnd[i].strftime('%j')
# Search for the files that are between those DOYs
year = date.year
DOYs = DOY_Year[DOY_Year[:, 2] == year]
DOYs_oneMonth = DOYs[np.logical_and(
(DOYs[:, 1] + 8) >= int(DOY_month_start),
DOYs[:, 1] <= int(DOY_month_end))]
# Create empty arrays
Monthly = np.zeros([size_Y, size_X])
Weight_tot = np.zeros([size_Y, size_X])
Data_one_month = np.ones([size_Y, size_X]) * np.nan
# Loop over the files that are within the DOYs
for EightDays in DOYs_oneMonth[:, 1]:
# Calculate the amount of days in this month of each file
Weight = np.ones([size_Y, size_X])
# For start of month
if np.min(DOYs_oneMonth[:, 1]) == EightDays:
Weight = Weight * int(EightDays + 8 - int(DOY_month_start))
# For end of month
elif np.max(DOYs_oneMonth[:, 1]) == EightDays:
Weight = Weight * (int(DOY_month_end) - EightDays + 1)
# For the middle of the month
else:
Weight = Weight * 8
row = DOYs_oneMonth[np.argwhere(
DOYs_oneMonth[:, 1] == EightDays)[0][0], :][0]
# Open the array of current file
input_name = os.path.join(Dir_in, files[int(row)])
Data = RC.Open_tiff_array(input_name)
# Remove NDV
Weight[Data == NDV] = 0
Data[Data == NDV] = np.nan
# Multiply weight time data
Data = Data * Weight
# Calculate the total weight and data
Weight_tot += Weight
Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)]
# Go to next month
i += 1
# Calculate the average
Data_one_month[Weight_tot != 0.] = Monthly[
Weight_tot != 0.] / Weight_tot[Weight_tot != 0.]
# Define output directory
if Dir_out == None:
Dir_out = Dir_in
# Define output name
output_name = os.path.join(
Dir_out, files[int(row)].replace('8-daily', 'monthly'))
output_name = output_name[:-9] + '%02d.01.tif' % (date.month)
# Save tiff file
DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj)
return
for Date in Dates:
Day = Date.day
Month = Date.month
Year = Date.year
Tempfile_one = Temp_folder.format(yyyy=Year, mm=Month, dd=Day)
Presfile_one = Pres_folder.format(yyyy=Year, mm=Month, dd=Day)
Humfile_one = Hum_folder.format(yyyy=Year, mm=Month, dd=Day)
out_folder_one = out_folder.format(yyyy=Year, mm=Month, dd=Day)
geo_out, proj, size_X, size_Y = RC.Open_array_info(Tempfile_one)
Tdata = RC.Open_tiff_array(Tempfile_one)
Tdata[Tdata < -900] = -9999
Pdata = RC.Open_tiff_array(Presfile_one)
Hdata = RC.Open_tiff_array(Humfile_one)
Pdata[Pdata < 0] = -9999
Hdata[Hdata < 0] = -9999
# gapfilling
Tdata = RC.gap_filling(Tdata, -9999)
Pdata = RC.gap_filling(Pdata, -9999)
Hdata = RC.gap_filling(Hdata, -9999)
Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3))
HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100
HumData = HumData.clip(0, 100)
DC.Save_as_tiff(out_folder_one, HumData, geo_out, "WGS84")
def DownloadData(output_folder, latlim, lonlim):
"""
This function downloads DEM data from SRTM
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -60 and 60)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -60 or latlim[1] > 60:
print(
'Latitude above 60N or below 60S is not possible. Value set to maximum'
)
latlim[0] = np.max(latlim[0], -60)
latlim[1] = np.min(latlim[1], 60)
if lonlim[0] < -180 or lonlim[1] > 180:
print(
'Longitude must be between 180E and 180W. Now value is set to maximum'
)
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.replace(".zip", ".tif")
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
dest_SRTM = gdal.Open(output_tiff)
geo_out = dest_SRTM.GetGeoTransform()
size_X = dest_SRTM.RasterXSize
size_Y = dest_SRTM.RasterYSize
if (int(size_X) != int(6001) or int(size_Y) != int(6001)):
data = np.ones((6001, 6001)) * -9999
# Create the latitude bound
Vfile = nameFile.split("_")[2][0:2]
Bound2 = 60 - 5 * (int(Vfile) - 1)
# Create the longitude bound
Hfile = nameFile.split("_")[1]
Bound1 = -180 + 5 * (int(Hfile) - 1)
Expected_X_min = Bound1
Expected_Y_max = Bound2
Xid_start = int(
np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(
np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) /
geo_out[1]))
Yid_start = int(
np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(
np.round((Expected_Y_max - (geo_out[3] +
(size_Y * geo_out[5]))) /
(-geo_out[5])))
data_SRTM = dest_SRTM.GetRasterBand(1).ReadAsArray()
data[Yid_start:Yid_end, Xid_start:Xid_end] = data_SRTM
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [
Bound1 - 0.5 * 0.00083333333333333, 0.00083333333333333,
0.0, Bound2 + 0.5 * 0.00083333333333333, 0.0,
-0.0008333333333333333333
]
# save chunk as tiff file
destDEM = DC.Save_as_MEM(data=data,
geo=geo_in,
projection="WGS84")
dest_SRTM = None
except:
# If tile not exist create a replacing zero tile (sea tiles)
file_name_tiff = file_name.replace(".zip", ".tif")
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
file_name = nameFile
data = np.ones((6001, 6001)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = nameFile.split("_")[2][0:2]
Bound2 = 60 - 5 * (int(Vfile) - 1)
# Create the longitude bound
Hfile = nameFile.split("_")[1]
Bound1 = -180 + 5 * (int(Hfile) - 1)
# Geospatial data for the tile
geo_in = [
Bound1 - 0.5 * 0.00083333333333333, 0.00083333333333333, 0.0,
Bound2 + 0.5 * 0.00083333333333333, 0.0,
-0.0008333333333333333333
]
# save chunk as tiff file
destDEM = DC.Save_as_MEM(data=data, geo=geo_in, projection="WGS84")
# clip data
Data, Geo_data = RC.clip_data(destDEM, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (file_name)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd,
data=Data,
geo=Geo_data,
projection="WGS84")
size_X_end = int(size_X_tot / len(rangeLat)) # + 1 !!!
size_Y_end = int(size_Y_tot / len(rangeLon)) # + 1 !!!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
# name of the end result
output_DEM_name = "DEM_SRTM_m_3s.tif"
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name,
data=datasetTot,
geo=geo_out,
projection="WGS84")
os.chdir(output_folder)
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar):
# Create an array with the dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define the minimum and maximum lat and long ETensemble Tile
Min_lat_tile = int(np.floor((100 - latlim[1]) / 10))
Max_lat_tile = int(np.floor((100 - latlim[0] - 0.00125) / 10))
Min_lon_tile = int(np.floor((190 + lonlim[0]) / 10))
Max_lon_tile = int(np.floor((190 + lonlim[1] - 0.00125) / 10))
# Create the Lat and Lon tiles that will be downloaded
Lat_tiles = [Min_lat_tile, Max_lat_tile]
Lon_tiles = [Min_lon_tile, Max_lon_tile]
# Define output folder and create this if it not exists
output_folder = os.path.join(Dir, 'Evaporation', 'ETensV1_0')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Create Geotransform of the output files
GEO_1 = lonlim[0]
GEO_2 = 0.0025
GEO_3 = 0.0
GEO_4 = latlim[1]
GEO_5 = 0.0
GEO_6 = -0.0025
geo = [GEO_1, GEO_2, GEO_3, GEO_4, GEO_5, GEO_6]
geo_new = tuple(geo)
# Define the parameter for downloading the data
Downloaded = 0
# Calculate the ET data date by date
for Date in Dates:
# Define the output name and folder
file_name = 'ET_ETensemble250m_mm-month-1_monthly_%d.%02d.01.tif' % (
Date.year, Date.month)
output_file = os.path.join(output_folder, file_name)
# If output file not exists create this
if not os.path.exists(output_file):
# If not downloaded than download
if Downloaded == 0:
# Download the ETens data from the FTP server
Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles)
# Unzip the folder
Unzip_ETens_data(output_folder, Lat_tiles, Lon_tiles)
Downloaded = 1
# Create the ET data for the area of interest
ET_data = Collect_dataset(output_folder, Date, Lat_tiles,
Lon_tiles, latlim, lonlim)
# Save this array as a tiff file
DC.Save_as_tiff(output_file, ET_data, geo_new, projection='WGS84')
# Create Waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
'''
# Remove all the raw dataset
for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1):
for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1):
Tilename = "h%sv%s" %(h_tile, v_tile)
filename = os.path.join(output_folder, Tilename)
if os.path.exists(filename):
shutil.rmtree(filename)
# Remove all .zip files
for f in os.listdir(output_folder):
if re.search(".zip", f):
os.remove(os.path.join(output_folder, f))
'''
return ()
def Calc_Property(Dir, latlim, lonlim, SL):
import watertools
# Define level
if SL == "sl3":
level = "Topsoil"
elif SL == "sl6":
level = "Subsoil"
# check if you need to download
filename_out_thetasat = os.path.join(
Dir, 'SoilGrids', 'Theta_Sat',
'Theta_Sat2_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetasat):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Sat2.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Sat2.Subsoil(
Dir, latlim, lonlim)
filename_out_thetares = os.path.join(
Dir, 'SoilGrids', 'Theta_Res',
'Theta_Res_%s_SoilGrids_kg-kg.tif' % level)
if not os.path.exists(filename_out_thetares):
if SL == "sl3":
watertools.Products.SoilGrids.Theta_Res.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.Theta_Res.Subsoil(
Dir, latlim, lonlim)
filename_out_n_genuchten = os.path.join(
Dir, 'SoilGrids', 'N_van_genuchten',
'N_genuchten_%s_SoilGrids_-.tif' % level)
if not os.path.exists(filename_out_n_genuchten):
if SL == "sl3":
watertools.Products.SoilGrids.n_van_genuchten.Topsoil(
Dir, latlim, lonlim)
elif SL == "sl6":
watertools.Products.SoilGrids.n_van_genuchten.Subsoil(
Dir, latlim, lonlim)
filedir_out_thetafc = os.path.join(Dir, 'SoilGrids', 'Theta_FC')
if not os.path.exists(filedir_out_thetafc):
os.makedirs(filedir_out_thetafc)
# Define theta field capacity output
filename_out_thetafc = os.path.join(
filedir_out_thetafc, 'Theta_FC2_%s_SoilGrids_cm3-cm3.tif' % level)
if not os.path.exists(filename_out_thetafc):
# Get info layer
geo_out, proj, size_X, size_Y = RC.Open_array_info(
filename_out_thetasat)
# Open dataset
theta_sat = RC.Open_tiff_array(filename_out_thetasat)
theta_res = RC.Open_tiff_array(filename_out_thetares)
n_genuchten = RC.Open_tiff_array(filename_out_n_genuchten)
# Calculate theta field capacity
theta_FC = np.ones(theta_sat.shape) * -9999
#theta_FC = np.where(theta_sat < 0.301, 0.042, np.arccosh(theta_sat + 0.7) - 0.32 * (theta_sat + 0.7) + 0.2)
#theta_FC = np.where(theta_sat < 0.301, 0.042, -2.95*theta_sat**2+3.96*theta_sat-0.871)
theta_FC = theta_res + (theta_sat - theta_res) / (
1 + (0.02 * 200)**n_genuchten)**(1 - 1 / n_genuchten)
# Save as tiff
DC.Save_as_tiff(filename_out_thetafc, theta_FC, geo_out, proj)
return
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar):
"""
This scripts downloads ASCAT SWI data from the VITO server.
The output files display the Surface Water Index.
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
lonlim -- [ymin, ymax]
latlim -- [xmin, xmax]
"""
# Check the latitude and longitude and otherwise reset lat and lon.
if latlim[0] < -90 or latlim[1] > 90:
print('Latitude above 90N or below 90S is not possible.\
Value set to maximum')
latlim[0] = np.max(latlim[0], -90)
latlim[1] = np.min(latlim[1], 90)
if lonlim[0] < -180 or lonlim[1] > 180:
print('Longitude must be between 180E and 180W.\
Now value is set to maximum')
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# Check Startdate and Enddate
if not Startdate:
Startdate = pd.Timestamp('2007-01-01')
if not Enddate:
Enddate = pd.Timestamp.now()
# Make a panda timestamp of the date
try:
Enddate = pd.Timestamp(Enddate)
except:
Enddate = Enddate
# amount of Dates weekly
Dates = pd.date_range(Startdate, Enddate, freq='D')
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define directory and create it if not exists
output_folder = os.path.join(Dir, 'SWI', 'ASCAT', 'Daily')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
output_folder_temp = os.path.join(Dir, 'SWI', 'ASCAT', 'Daily', 'Temp')
if not os.path.exists(output_folder_temp):
os.makedirs(output_folder_temp)
# loop over dates
for Date in Dates:
# Define end filename
End_filename = os.path.join(
output_folder, 'SWI_ASCAT_V3_Percentage_daily_%d.%02d.%02d.tif' %
(Date.year, Date.month, Date.day))
# Define IDs
xID = 1800 + np.int16(
np.array([np.ceil((lonlim[0]) * 10),
np.floor((lonlim[1]) * 10)]))
yID = np.int16(
np.array([np.floor((-latlim[1]) * 10),
np.ceil((-latlim[0]) * 10)])) + 900
# Download the data from FTP server if the file not exists
if not os.path.exists(End_filename):
try:
data = Download_ASCAT_from_VITO(End_filename,
output_folder_temp, Date, yID,
xID)
# make geotiff file
geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
DC.Save_as_tiff(name=End_filename,
data=data,
geo=geo,
projection="WGS84")
except:
print("Was not able to download file with date %s" % Date)
# Adjust waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# remove the temporary folder
shutil.rmtree(output_folder_temp)
