def livestock_feed(output_folder, lu_fh, ndm_fhs, feed_dict, live_feed, cattle_fh, fraction_fhs, ndmdates):
"""
Calculate natural livestock feed production
INPUTS
----------
lu_fh : str
filehandle for land use map
ndm_fhs: nd array
array of filehandles of NDM maps
ndm_dates: nd array
array of dates for NDM maps
feed_dict: dict
dictionnary 'pasture class':[list of LULC]
feed_pct: dict
dictionnary 'pasture class':[percent available as feed]
cattle_fh : str
filehandle for cattle map
"""
Data_Path_Feed = "Feed"
out_folder = os.path.join(output_folder, Data_Path_Feed)
if not os.path.exists(out_folder):
os.mkdir(out_folder)
area_ha = becgis.MapPixelAreakm(lu_fh) * 100
LULC = RC.Open_tiff_array(lu_fh)
# cattle = RC.Open_tiff_array(cattle_fh)
geo_out, proj, size_X, size_Y = RC.Open_array_info(lu_fh)
f_pct = np.zeros(LULC.shape)
for lu_type in feed_dict.keys():
classes = feed_dict[lu_type]
mask = np.logical_or.reduce([LULC == value for value in classes])
f_pct[mask] = live_feed[lu_type]
feed_fhs_landscape = []
feed_fhs_incremental = []
for d in range(len(ndm_fhs)):
ndm_fh = ndm_fhs[d]
fraction_fh = fraction_fhs[d]
date1 = ndmdates[d]
year = '%d' %date1.year
month = '%02d' %date1.month
yield_fract = RC.Open_tiff_array(fraction_fh)
out_fh_l = out_folder+'\\feed_prod_landscape_%s_%s.tif' %(year, month)
out_fh_i = out_folder+'\\feed_prod_incremental_%s_%s.tif' %(year, month)
# out_fh2 = out_folder+'\\Feed_prod_pH_%s_%s.tif' %(year, month)
NDM = becgis.OpenAsArray(ndm_fh, nan_values=True)
NDM_feed = NDM * f_pct
NDM_feed_incremental = NDM_feed * yield_fract * area_ha/1e6
NDM_feed_landscape = (NDM_feed *(1-yield_fract)) * area_ha/1e6
DC.Save_as_tiff(out_fh_l, NDM_feed_landscape, geo_out)
DC.Save_as_tiff(out_fh_i, NDM_feed_incremental, geo_out)
# NDM_feed_perHead = NDM_feed / cattle
# DC.Save_as_tiff(out_fh2, NDM_feed, geo_out)
feed_fhs_landscape.append(out_fh_l)
feed_fhs_incremental.append(out_fh_i)
return feed_fhs_landscape, feed_fhs_incremental
def lu_type_average(data_fh, lu_fh, lu_dict):
LULC = RC.Open_tiff_array(lu_fh)
in_data = RC.Open_tiff_array(data_fh)
out_data = {}
for lu_class in lu_dict.keys():
mask = [LULC == value for value in lu_dict[lu_class]]
mask = (np.sum(mask, axis=0)).astype(bool)
out_data[lu_class] = np.nanmean(in_data[mask])
return out_data
def fuel_wood(output_folder, lu_fh, ndm_fhs, fraction_fhs, ndmdates):
"""
Calculate natural livestock feed production
INPUTS
----------
lu_fh : str
filehandle for land use map
ndm_fhs: nd array
array of filehandles of NDM maps
abv_grnd_biomass_ratio: dict
dictionnary 'LULC':[above ground biomass]
"""
Data_Path_Fuel = "Fuel"
out_folder = os.path.join(output_folder, Data_Path_Fuel)
if not os.path.exists(out_folder):
os.mkdir(out_folder)
area_ha = becgis.MapPixelAreakm(lu_fh) * 100
LULC = RC.Open_tiff_array(lu_fh)
geo_out, proj, size_X, size_Y = RC.Open_array_info(lu_fh)
fuel_classes = [1, 8, 9, 10, 11, 12, 13]
fuel_mask = np.zeros(LULC.shape)
for fc in fuel_classes:
fuel_mask[np.where(LULC == fc)] = 1
fuel_fhs_landscape = []
fuel_fhs_incremental = []
for d in range(len(ndm_fhs)):
ndm_fh = ndm_fhs[d]
fraction_fh = fraction_fhs[d]
yield_fract = RC.Open_tiff_array(fraction_fh)
date1 = ndmdates[d]
year = '%d' %date1.year
month = '%02d' %date1.month
# year = ndm_fh[-14:-10]
# month = ndm_fh[-9:-7]
out_fh_l = out_folder+'\\fuel_prod_landscape_%s_%s.tif' %(year, month)
out_fh_i = out_folder+'\\fuel_prod_incremental_%s_%s.tif' %(year, month)
NDM = becgis.OpenAsArray(ndm_fh, nan_values=True)
NDM_fuel_incremental = NDM * .05 * fuel_mask * yield_fract * area_ha/1e6
NDM_fuel_landscape = NDM * .05 * fuel_mask *(1-yield_fract) * area_ha/1e6
DC.Save_as_tiff(out_fh_i, NDM_fuel_incremental, geo_out)
DC.Save_as_tiff(out_fh_l, NDM_fuel_landscape, geo_out)
fuel_fhs_landscape.append(out_fh_l)
fuel_fhs_incremental.append(out_fh_i)
return fuel_fhs_landscape, fuel_fhs_incremental
def Clip_Dataset(local_filename, Filename_out, latlim, lonlim):
import wa.General.raster_conversions as RC
# Open Dataset
HiHydroSoil_Array = RC.Open_tiff_array(local_filename)
# Define area
XID = [
int(np.floor((180 + lonlim[0]) / 0.00833333)),
int(np.ceil((180 + lonlim[1]) / 0.00833333))
]
YID = [
int(np.ceil((90 - latlim[1]) / 0.00833333)),
int(np.floor((90 - latlim[0]) / 0.00833333))
]
# Define Georeference
geo = tuple([
-180 + 0.00833333 * XID[0], 0.00833333, 0, 90 - 0.00833333 * YID[0], 0,
-0.00833333
])
# Clip Array
HiHydroSoil_Array_clipped = HiHydroSoil_Array[YID[0]:YID[1], XID[0]:XID[1]]
# Save tiff file
DC.Save_as_tiff(Filename_out, HiHydroSoil_Array_clipped, geo, "WGS84")
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID, TimeStep):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if TimeStep is "weekly":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
if TimeStep is "daily":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
if TimeStep is "weekly":
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
if TimeStep is "daily":
DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])
raw_data = np.fromfile(os.path.splitext(local_filename)[0],
dtype="<f4")
dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return
def split_yield(output_folder, p_fhs, et_blue_fhs, et_green_fhs, ab=(1.0, 1.0)):
Data_Path_split = "split_y"
out_folder = os.path.join(output_folder, Data_Path_split)
if not os.path.exists(out_folder):
os.mkdir(out_folder)
sp_yield_fhs = []
geo_out, proj, size_X, size_Y = RC.Open_array_info(p_fhs[0])
for m in range(len(p_fhs)):
out_fh = out_folder+'\\split_yield'+et_blue_fhs[m][-12:]
P = RC.Open_tiff_array(p_fhs[m])
ETBLUE = RC.Open_tiff_array(et_blue_fhs[m])
ETGREEN = RC.Open_tiff_array(et_green_fhs[m])
etbfraction = ETBLUE / (ETBLUE + ETGREEN)
pfraction = P / np.nanmax(P)
fraction = sh3.split_Yield(pfraction, etbfraction, ab[0], ab[1])
DC.Save_as_tiff(out_fh, fraction, geo_out)
sp_yield_fhs.append(out_fh)
return sp_yield_fhs
def lu_type_sum(data_fh, lu_fh, lu_dict, convert=None):
LULC = RC.Open_tiff_array(lu_fh)
in_data = becgis.OpenAsArray(data_fh, nan_values=True)
# in_data = RC.Open_tiff_array(data_fh)
if convert == 'mm_to_km3':
AREA = becgis.MapPixelAreakm(data_fh)
in_data *= AREA / 1e6
out_data = {}
for lu_class in lu_dict.keys():
mask = [LULC == value for value in lu_dict[lu_class]]
mask = (np.sum(mask, axis=0)).astype(bool)
out_data[lu_class] = np.nansum(in_data[mask])
return out_data
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print "file not exists"
return
def DownloadData(output_folder, latlim, lonlim, parameter, resolution):
"""
This function downloads DEM data from HydroSHED
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
Resample -- 1 = The data will be resampled to 0.001 degree spatial
resolution
-- 0 = The data will have the same pixel size as the data obtained
from the internet
"""
# Define parameter depedent variables
if parameter == "dir_3s":
para_name = "DIR"
unit = "-"
resolution = '3s'
parameter = 'dir'
if parameter == "dem_3s":
para_name = "DEM"
unit = "m"
resolution = '3s'
parameter = 'dem'
if parameter == "dir_15s":
para_name = "DIR"
unit = "-"
resolution = '15s'
parameter = 'dir'
if parameter == "dem_15s":
para_name = "DEM"
unit = "m"
resolution = '15s'
parameter = 'dem'
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
if resolution == '3s':
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim, parameter)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
if resolution == '15s':
name = Find_Document_names_15s(latlim, lonlim, parameter, resolution)
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash, parameter, para_name,resolution)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# Converts the data with a adf extention to a tiff extension.
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif'
file_name_extract = file_name.split('_')[0:3]
if resolution == '3s':
file_name_extract2 = file_name_extract[0]+'_'+file_name_extract[1]
if resolution == '15s':
file_name_extract2 = file_name_extract[0]+'_'+file_name_extract[1]+'_15s'
input_adf = os.path.join(output_folder_trash, file_name_extract2,
file_name_extract2, 'hdr.adf')
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff)
geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff)
if int(size_X) != int(6000) or int(size_Y) != int(6000):
data = np.ones((6000, 6000)) * -9999
# Create the latitude bound
Vfile = str(nameFile)[1:3]
SignV = str(nameFile)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer)*int(Vfile)
# Create the longitude bound
Hfile = str(nameFile)[4:7]
SignH = str(nameFile)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
Expected_X_min = Bound1
Expected_Y_max = Bound2 + 5
Xid_start = int(np.round((geo_out[0] - Expected_X_min)/geo_out[1]))
Xid_end = int(np.round(((geo_out[0] + size_X * geo_out[1]) - Expected_X_min)/geo_out[1]))
Yid_start = int(np.round((Expected_Y_max - geo_out[3])/(-geo_out[5])))
Yid_end = int(np.round((Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5])))/(-geo_out[5])))
data[Yid_start:Yid_end,Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff)
if np.max(data)==255:
data[data==255] = -9999
data[data<-9999] = -9999
geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5),
0.0, -0.0008333333333333333333]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in,
projection="WGS84")
except:
if resolution == '3s':
# If tile not exist create a replacing zero tile (sea tiles)
output = nameFile.split('.')[0] + "_trans_temporary.tif"
output_tiff = os.path.join(output_folder_trash, output)
file_name = nameFile
data = np.ones((6000, 6000)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = str(file_name)[1:3]
SignV = str(file_name)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer)*int(Vfile)
# Create the longitude bound
Hfile = str(file_name)[4:7]
SignH = str(file_name)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
# Geospatial data for the tile
geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5),
0.0, -0.0008333333333333333333]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in,
projection="WGS84")
if resolution == '15s':
print 'no 15s data is in dataset'
if resolution =='3s':
# clip data
Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end,Geo_data[0]])
Geo_y_end = np.max([Geo_y_end,Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (nameFile)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data,
projection="WGS84")
if resolution =='3s':
#size_X_end = int(size_X_tot) #!
#size_Y_end = int(size_Y_tot) #!
size_X_end = int(size_X_tot/len(rangeLat)) + 1 #!
size_Y_end = int(size_Y_tot/len(rangeLon)) + 1 #!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot<-9999] = -9999
if resolution =='15s':
output_file_merged = os.path.join(output_folder_trash,'merged.tif')
datasetTot, geo_out = Merge_DEM_15s(output_folder_trash, output_file_merged,latlim, lonlim)
# name of the end result
output_DEM_name = "%s_HydroShed_%s_%s.tif" %(para_name,unit,resolution)
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84")
os.chdir(output_folder)
# Delete the temporary folder
shutil.rmtree(output_folder_trash)
# -*- coding: utf-8 -*- """ Created on Mon Jun 19 10:09:38 2017 @author: tih """ Tfile = r"J:\Tyler\Input\Meteo\daily\avgsurft_inst\mean\T_GLDAS-NOAH_C_daily_2016.06.15.tif" Pfile = r"J:\Tyler\Input\Meteo\daily\psurf_f_inst\mean\P_GLDAS-NOAH_kpa_daily_2016.06.15.tif" Hfile = r"J:\Tyler\Input\Meteo\daily\qair_f_inst\mean\Hum_GLDAS-NOAH_kg-kg_daily_2016.06.15.tif" Outfilename = r"J:\Tyler\Input\Meteo\daily\Hum_Calculated\Humidity_percentage_Calculated_daily.tif" import gdal import os import wa.General.raster_conversions as RC import wa.General.data_conversions as DC import numpy as np geo_out, proj, size_X, size_Y = RC.Open_array_info(Tfile) Tdata = RC.Open_tiff_array(Tfile) Tdata[Tdata < -900] = np.nan Pdata = RC.Open_tiff_array(Pfile) Hdata = RC.Open_tiff_array(Hfile) Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3)) HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100 DC.Save_as_tiff(Outfilename, HumData, geo_out, "WGS84")
def CollectLANDSAF(SourceLANDSAF, Dir, Startdate, Enddate, latlim, lonlim):
"""
This function collects and clip LANDSAF data
Keyword arguments:
SourceLANDSAF -- 'C:/' path to the LANDSAF source data (The directory includes SIS and SID)
Dir -- 'C:/' path to the WA map
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -60 and 60)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Make an array of the days of which the ET is taken
Dates = pd.date_range(Startdate, Enddate, freq='D')
# make directories
SISdir = os.path.join(Dir, 'Landsaf_Clipped', 'SIS')
if os.path.exists(SISdir) is False:
os.makedirs(SISdir)
SIDdir = os.path.join(Dir, 'Landsaf_Clipped', 'SID')
if os.path.exists(SIDdir) is False:
os.makedirs(SIDdir)
ShortwaveBasin(SourceLANDSAF,
Dir,
latlim,
lonlim,
Dates=[Startdate, Enddate])
DEMmap_str = os.path.join(Dir, 'HydroSHED', 'DEM',
'DEM_HydroShed_m_3s.tif')
geo_out, proj, size_X, size_Y = RC.Open_array_info(DEMmap_str)
# Open DEM map
demmap = RC.Open_tiff_array(DEMmap_str)
demmap[demmap < 0] = 0
# make lat and lon arrays)
dlat = geo_out[5]
dlon = geo_out[1]
lat = geo_out[3] + (np.arange(size_Y) + 0.5) * dlat
lon = geo_out[0] + (np.arange(size_X) + 0.5) * dlon
for date in Dates:
# day of year
day = date.dayofyear
Horizontal, Sloping, sinb, sinb_hor, fi, slope, ID = SlopeInfluence(
demmap, lat, lon, day)
SIDname = os.path.join(
SIDdir, 'SAF_SID_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
SISname = os.path.join(
SISdir, 'SAF_SIS_Daily_W-m2_' + date.strftime('%Y-%m-%d') + '.tif')
#PREPARE SID MAPS
SIDdest = RC.reproject_dataset_example(SIDname, DEMmap_str, method=3)
SIDdata = SIDdest.GetRasterBand(1).ReadAsArray()
#PREPARE SIS MAPS
SISdest = RC.reproject_dataset_example(SISname, DEMmap_str, method=3)
SISdata = SISdest.GetRasterBand(1).ReadAsArray()
# Calculate ShortWave net
Short_Wave_Net = SIDdata * (Sloping /
Horizontal) + SISdata * 86400 / 1e6
# Calculate ShortWave Clear
Short_Wave = Sloping
Short_Wave_Clear = Short_Wave * (0.75 + demmap * 2 * 10**-5)
# make directories
PathClear = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Clear_Sky')
if os.path.exists(PathClear) is False:
os.makedirs(PathClear)
PathNet = os.path.join(Dir, 'Landsaf_Clipped', 'Shortwave_Net')
if os.path.exists(PathNet) is False:
os.makedirs(PathNet)
# name Shortwave Clear and Net
nameFileNet = 'ShortWave_Net_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameNet = os.path.join(PathNet, nameFileNet)
nameFileClear = 'ShortWave_Clear_Daily_W-m2_' + date.strftime(
'%Y-%m-%d') + '.tif'
nameClear = os.path.join(PathClear, nameFileClear)
# Save net and clear short wave radiation
DC.Save_as_tiff(nameNet, Short_Wave_Net, geo_out, proj)
DC.Save_as_tiff(nameClear, Short_Wave_Clear, geo_out, proj)
return
def Calc_Rainy_Days(Dir_Basin, Data_Path_P, Startdate, Enddate):
"""
This functions calculates the amount of rainy days based on daily precipitation data.
Parameters
----------
Dir_Basin : str
Path to all the output data of the Basin
Data_Path_P : str
Path from the Dir_Basin to the daily rainfall data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Returns
-------
Data_Path_RD : str
Path from the Dir_Basin to the rainy days data
"""
# import WA+ modules
import wa.General.data_conversions as DC
import wa.General.raster_conversions as RC
# Create an output directory to store the rainy days tiffs
Data_Path_RD = 'Rainy_Days'
Dir_RD = os.path.join(Dir_Basin, Data_Path_RD)
if not os.path.exists(Dir_RD):
os.mkdir(Dir_RD)
# Define the dates that must be created
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Set working directory to the rainfall folder
Dir_path_Prec = os.path.join(Dir_Basin, Data_Path_P)
os.chdir(Dir_path_Prec)
# Open all the daily data and store the data in a 3D array
for Date in Dates:
# Define the year and month and amount of days in month
year = Date.year
month = Date.month
daysinmonth = calendar.monthrange(year, month)[1]
# Set the third (time) dimension of array starting at 0
i = 0
# Find all files of that month
files = glob.glob('*daily_%d.%02d.*.tif' % (year, month))
# Check if the amount of files corresponds with the amount of days in month
if len(files) is not daysinmonth:
print 'ERROR: Not all Rainfall days for month %d and year %d are downloaded' % (
month, year)
# Loop over the days and store data in raster
for File in files:
dir_file = os.path.join(Dir_path_Prec, File)
# Get array information and create empty numpy array for daily rainfall when looping the first file
if File == files[0]:
# Open geolocation info and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(dir_file)
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Create empty array for the whole month
P_Daily = np.zeros([daysinmonth, size_Y, size_X])
# Open data and put the data in 3D array
Data = RC.Open_tiff_array(dir_file)
# Remove the weird numbers
Data[Data < 0] = 0
# Add the precipitation to the monthly cube
P_Daily[i, :, :] = Data
i += 1
# Define a rainy day
P_Daily[P_Daily > 0.201] = 1
P_Daily[P_Daily != 1] = 0
# Sum the amount of rainy days
RD_one_month = np.nansum(P_Daily, 0)
# Define output name
Outname = os.path.join(
Dir_RD,
'Rainy_Days_NumOfDays_monthly_%d.%02d.01.tif' % (year, month))
# Save tiff file
DC.Save_as_tiff(Outname, RD_one_month, geo_out, proj)
return (Data_Path_RD)
def main(files_DEM_dir, files_DEM, files_Basin, files_Runoff, files_Extraction,
startdate, enddate, input_nc, resolution, Format_DEM_dir, Format_DEM,
Format_Basin, Format_Runoff, Format_Extraction):
# Define a year to get the epsg and geo
Startdate_timestamp = pd.Timestamp(startdate)
year = Startdate_timestamp.year
############################## Drainage Direction #####################################
# Open Array DEM dir as netCDF
if Format_DEM_dir == "NetCDF":
file_DEM_dir = os.path.join(files_DEM_dir, "%d.nc" % year)
DataCube_DEM_dir = RC.Open_nc_array(file_DEM_dir, "Drainage_Direction")
geo_out_example, epsg_example, size_X_example, size_Y_example, size_Z_example, Time_example = RC.Open_nc_info(
files_DEM_dir)
# Create memory file for reprojection
gland = DC.Save_as_MEM(DataCube_DEM_dir, geo_out_example, epsg_example)
dataset_example = file_name_DEM_dir = gland
# Open Array DEM dir as TIFF
if Format_DEM_dir == "TIFF":
file_name_DEM_dir = os.path.join(files_DEM_dir,
"DIR_HydroShed_-_%s.tif" % resolution)
DataCube_DEM_dir = RC.Open_tiff_array(file_name_DEM_dir)
geo_out_example, epsg_example, size_X_example, size_Y_example = RC.Open_array_info(
file_name_DEM_dir)
dataset_example = file_name_DEM_dir
# Calculate Area per pixel in m2
import wa.Functions.Start.Area_converter as AC
DataCube_Area = AC.Degrees_to_m2(file_name_DEM_dir)
################################## DEM ##########################################
# Open Array DEM as netCDF
if Format_DEM == "NetCDF":
file_DEM = os.path.join(files_DEM, "%d.nc" % year)
DataCube_DEM = RC.Open_nc_array(file_DEM, "Elevation")
# Open Array DEM as TIFF
if Format_DEM == "TIFF":
file_name_DEM = os.path.join(files_DEM,
"DEM_HydroShed_m_%s.tif" % resolution)
DataCube_DEM = RC.Open_tiff_array(file_name_DEM)
################################ Landuse ##########################################
# Open Array Basin as netCDF
if Format_Basin == "NetCDF":
file_Basin = os.path.join(files_Basin, "%d.nc" % year)
DataCube_Basin = RC.Open_nc_array(file_Basin, "Landuse")
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(
file_Basin, "Landuse")
dest_basin = DC.Save_as_MEM(DataCube_Basin, geo_out, str(epsg))
destLU = RC.reproject_dataset_example(dest_basin,
dataset_example,
method=1)
DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray()
DataCube_Basin = np.zeros([size_Y_example, size_X_example])
DataCube_Basin[DataCube_LU_CR > 0] = 1
# Open Array Basin as TIFF
if Format_Basin == "TIFF":
file_name_Basin = files_Basin
destLU = RC.reproject_dataset_example(file_name_Basin,
dataset_example,
method=1)
DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray()
DataCube_Basin = np.zeros([size_Y_example, size_X_example])
DataCube_Basin[DataCube_LU_CR > 0] = 1
################################ Surface Runoff ##########################################
# Open Array runoff as netCDF
if Format_Runoff == "NetCDF":
DataCube_Runoff = RC.Open_ncs_array(files_Runoff, "Surface_Runoff",
startdate, enddate)
size_Z_example = DataCube_Runoff.shape[0]
file_Runoff = os.path.join(files_Runoff, "%d.nc" % year)
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(
file_Runoff, "Surface_Runoff")
DataCube_Runoff_CR = np.ones(
[size_Z_example, size_Y_example, size_X_example]) * np.nan
for i in range(0, size_Z):
DataCube_Runoff_one = DataCube_Runoff[i, :, :]
dest_Runoff_one = DC.Save_as_MEM(DataCube_Runoff_one, geo_out,
str(epsg))
dest_Runoff = RC.reproject_dataset_example(dest_Runoff_one,
dataset_example,
method=4)
DataCube_Runoff_CR[i, :, :] = dest_Runoff.GetRasterBand(
1).ReadAsArray()
DataCube_Runoff_CR[:, DataCube_LU_CR == 0] = -9999
DataCube_Runoff_CR[DataCube_Runoff_CR < 0] = -9999
# Open Array runoff as TIFF
if Format_Runoff == "TIFF":
Data_Path = ''
DataCube_Runoff = RC.Get3Darray_time_series_monthly(
files_Runoff,
Data_Path,
startdate,
enddate,
Example_data=dataset_example)
################################ Surface Withdrawal ##########################################
# Open Array Extraction as netCDF
if Format_Extraction == "NetCDF":
DataCube_Extraction = RC.Open_ncs_array(files_Extraction,
"Surface_Withdrawal",
startdate, enddate)
size_Z_example = DataCube_Extraction.shape[0]
file_Extraction = os.path.join(files_Extraction, "%d.nc" % year)
geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(
file_Extraction, "Surface_Withdrawal")
DataCube_Extraction_CR = np.ones(
[size_Z_example, size_Y_example, size_X_example]) * np.nan
for i in range(0, size_Z):
DataCube_Extraction_one = DataCube_Extraction[i, :, :]
dest_Extraction_one = DC.Save_as_MEM(DataCube_Extraction_one,
geo_out, str(epsg))
dest_Extraction = RC.reproject_dataset_example(dest_Extraction_one,
dataset_example,
method=4)
DataCube_Extraction_CR[i, :, :] = dest_Extraction.GetRasterBand(
1).ReadAsArray()
DataCube_Extraction_CR[:, DataCube_LU_CR == 0] = -9999
DataCube_Extraction_CR[DataCube_Extraction_CR < 0] = -9999
# Open Array Extraction as TIFF
if Format_Extraction == "TIFF":
Data_Path = ''
DataCube_Extraction = RC.Get3Darray_time_series_monthly(
files_Extraction,
Data_Path,
startdate,
enddate,
Example_data=dataset_example)
################################ Create input netcdf ##########################################
# Save data in one NetCDF file
geo_out_example = np.array(geo_out_example)
# Latitude and longitude
lon_ls = np.arange(size_X_example) * geo_out_example[1] + geo_out_example[
0] + 0.5 * geo_out_example[1]
lat_ls = np.arange(size_Y_example) * geo_out_example[5] + geo_out_example[
3] - 0.5 * geo_out_example[5]
lat_n = len(lat_ls)
lon_n = len(lon_ls)
# Create NetCDF file
nc_file = netCDF4.Dataset(input_nc, 'w')
nc_file.set_fill_on()
# Create dimensions
lat_dim = nc_file.createDimension('latitude', lat_n)
lon_dim = nc_file.createDimension('longitude', lon_n)
# Create NetCDF variables
crso = nc_file.createVariable('crs', 'i4')
crso.long_name = 'Lon/Lat Coords in WGS84'
crso.standard_name = 'crs'
crso.grid_mapping_name = 'latitude_longitude'
crso.projection = epsg_example
crso.longitude_of_prime_meridian = 0.0
crso.semi_major_axis = 6378137.0
crso.inverse_flattening = 298.257223563
crso.geo_reference = geo_out_example
lat_var = nc_file.createVariable('latitude', 'f8', ('latitude', ))
lat_var.units = 'degrees_north'
lat_var.standard_name = 'latitude'
lat_var.pixel_size = geo_out_example[5]
lon_var = nc_file.createVariable('longitude', 'f8', ('longitude', ))
lon_var.units = 'degrees_east'
lon_var.standard_name = 'longitude'
lon_var.pixel_size = geo_out_example[1]
Dates = pd.date_range(startdate, enddate, freq='MS')
time_or = np.zeros(len(Dates))
i = 0
for Date in Dates:
time_or[i] = Date.toordinal()
i += 1
nc_file.createDimension('time', None)
timeo = nc_file.createVariable('time', 'f4', ('time', ))
timeo.units = 'Monthly'
timeo.standard_name = 'time'
# Variables
demdir_var = nc_file.createVariable('demdir',
'i', ('latitude', 'longitude'),
fill_value=-9999)
demdir_var.long_name = 'Flow Direction Map'
demdir_var.grid_mapping = 'crs'
dem_var = nc_file.createVariable('dem',
'f8', ('latitude', 'longitude'),
fill_value=-9999)
dem_var.long_name = 'Altitude'
dem_var.units = 'meters'
dem_var.grid_mapping = 'crs'
basin_var = nc_file.createVariable('basin',
'i', ('latitude', 'longitude'),
fill_value=-9999)
basin_var.long_name = 'Altitude'
basin_var.units = 'meters'
basin_var.grid_mapping = 'crs'
area_var = nc_file.createVariable('area',
'f8', ('latitude', 'longitude'),
fill_value=-9999)
area_var.long_name = 'area in squared meters'
area_var.units = 'squared_meters'
area_var.grid_mapping = 'crs'
runoff_var = nc_file.createVariable('Runoff_M',
'f8',
('time', 'latitude', 'longitude'),
fill_value=-9999)
runoff_var.long_name = 'Runoff'
runoff_var.units = 'm3/month'
runoff_var.grid_mapping = 'crs'
extraction_var = nc_file.createVariable('Extraction_M',
'f8',
('time', 'latitude', 'longitude'),
fill_value=-9999)
extraction_var.long_name = 'Surface water Extraction'
extraction_var.units = 'm3/month'
extraction_var.grid_mapping = 'crs'
# Load data
lat_var[:] = lat_ls
lon_var[:] = lon_ls
timeo[:] = time_or
# Static variables
demdir_var[:, :] = DataCube_DEM_dir[:, :]
dem_var[:, :] = DataCube_DEM[:, :]
basin_var[:, :] = DataCube_Basin[:, :]
area_var[:, :] = DataCube_Area[:, :]
for i in range(len(Dates)):
runoff_var[i, :, :] = DataCube_Runoff_CR[i, :, :]
for i in range(len(Dates)):
extraction_var[i, :, :] = DataCube_Extraction_CR[i, :, :]
# Close file
nc_file.close()
return ()
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None):
"""
This functions calculates monthly tiff files based on the 8 daily tiff files. (will calculate the average)
Parameters
----------
Dir_in : str
Path to the input data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Dir_out : str
Path to the output data, default is same as Dir_in
"""
# import WA+ modules
import wa.General.data_conversions as DC
import wa.General.raster_conversions as RC
# Change working directory
os.chdir(Dir_in)
# Find all eight daily files
files = glob.glob('*8-daily*.tif')
# Create array with filename and keys (DOY and year) of all the 8 daily files
i = 0
DOY_Year = np.zeros([len(files), 3])
for File in files:
# Get the time characteristics from the filename
year = File.split('.')[-4][-4:]
month = File.split('.')[-3]
day = File.split('.')[-2]
# Create pandas Timestamp
date_file = '%s-%02s-%02s' % (year, month, day)
Datum = pd.Timestamp(date_file)
# Get day of year
DOY = Datum.strftime('%j')
# Save data in array
DOY_Year[i, 0] = i
DOY_Year[i, 1] = DOY
DOY_Year[i, 2] = year
# Loop over files
i += 1
# Check enddate:
Enddate_split = Enddate.split('-')
month_range = calendar.monthrange(int(Enddate_split[0]),
int(Enddate_split[1]))[1]
Enddate = '%d-%02d-%02d' % (int(Enddate_split[0]), int(
Enddate_split[1]), month_range)
# Check startdate:
Startdate_split = Startdate.split('-')
Startdate = '%d-%02d-01' % (int(Startdate_split[0]), int(
Startdate_split[1]))
# Define end and start date
Dates = pd.date_range(Startdate, Enddate, freq='MS')
DatesEnd = pd.date_range(Startdate, Enddate, freq='M')
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value
dest = gdal.Open(files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Loop over months and create monthly tiff files
i = 0
for date in Dates:
# Get Start and end DOY of the current month
DOY_month_start = date.strftime('%j')
DOY_month_end = DatesEnd[i].strftime('%j')
# Search for the files that are between those DOYs
year = date.year
DOYs = DOY_Year[DOY_Year[:, 2] == year]
DOYs_oneMonth = DOYs[np.logical_and(
(DOYs[:, 1] + 8) >= int(DOY_month_start),
DOYs[:, 1] <= int(DOY_month_end))]
# Create empty arrays
Monthly = np.zeros([size_Y, size_X])
Weight_tot = np.zeros([size_Y, size_X])
Data_one_month = np.ones([size_Y, size_X]) * np.nan
# Loop over the files that are within the DOYs
for EightDays in DOYs_oneMonth[:, 0]:
# Calculate the amount of days in this month of each file
Weight = np.ones([size_Y, size_X])
# For start of month
if EightDays == DOYs_oneMonth[:, 0][0]:
Weight = Weight * int(DOYs_oneMonth[:, 1][0] + 8 -
int(DOY_month_start))
# For end of month
elif EightDays == DOYs_oneMonth[:, 0][-1]:
Weight = Weight * (int(DOY_month_end) -
DOYs_oneMonth[:, 1][-1] + 1)
# For the middle of the month
else:
Weight = Weight * 8
# Open the array of current file
input_name = os.path.join(Dir_in, files[int(EightDays)])
Data = RC.Open_tiff_array(input_name)
# Remove NDV
Weight[Data == NDV] = 0
Data[Data == NDV] = np.nan
# Multiply weight time data
Data = Data * Weight
# Calculate the total weight and data
Weight_tot += Weight
Monthly[~np.isnan(Data)] += Data[~np.isnan(Data)]
# Go to next month
i += 1
# Calculate the average
Data_one_month[Weight_tot != 0.] = Monthly[
Weight_tot != 0.] / Weight_tot[Weight_tot != 0.]
# Define output directory
if Dir_out == None:
Dir_out = Dir_in
# Define output name
output_name = os.path.join(
Dir_out, files[int(EightDays)].replace('8-daily', 'monthly'))
output_name = output_name[:-6] + '01.tif'
# Save tiff file
DC.Save_as_tiff(output_name, Data_one_month, geo_out, proj)
return
def NPP_GPP_Based(Dir_Basin, Data_Path_GPP, Data_Path_NPP, Startdate, Enddate):
"""
This functions calculated monthly NDM based on the yearly NPP and monthly GPP.
Parameters
----------
Dir_Basin : str
Path to all the output data of the Basin
Data_Path_GPP : str
Path from the Dir_Basin to the GPP data
Data_Path_NPP : str
Path from the Dir_Basin to the NPP data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Simulation : int
Defines the simulation
Returns
-------
Data_Path_NDM : str
Path from the Dir_Basin to the normalized dry matter data
"""
# import WA+ modules
import wa.General.data_conversions as DC
import wa.General.raster_conversions as RC
# Define output folder for Normalized Dry Matter
Data_Path_NDM = os.path.join(Dir_Basin, "NDM")
if not os.path.exists(Data_Path_NDM):
os.mkdir(Data_Path_NDM)
# Define monthly time steps that will be created
Dates = pd.date_range(Startdate, Enddate, freq = 'MS')
# Define the years that will be calculated
Year_Start = int(Startdate[0:4])
Year_End = int(Enddate[0:4])
Years = range(Year_Start, Year_End+1)
# Loop over the years
for year in Years:
# Change working directory to the NPP folder
os.chdir(Data_Path_NPP)
# Open yearly NPP data
yearly_NPP_File = glob.glob('*yearly*%d.01.01.tif' %int(year))[0]
Yearly_NPP = RC.Open_tiff_array(yearly_NPP_File)
# Get the No Data Value of the NPP file
dest = gdal.Open(yearly_NPP_File)
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Set the No Data Value to Nan
Yearly_NPP[Yearly_NPP == NDV] = np.nan
# Change working directory to the GPP folder
os.chdir(Data_Path_GPP)
# Find all the monthly files of that year
monthly_GPP_Files = glob.glob('*monthly*%d.*.01.tif' %int(year))
# Check if it are 12 files otherwise something is wrong and send the ERROR
if not len(monthly_GPP_Files) == 12:
print 'ERROR: Some monthly GPP Files are missing'
# Get the projection information of the GPP inputs
geo_out, proj, size_X, size_Y = RC.Open_array_info(monthly_GPP_Files[0])
geo_out_NPP, proj_NPP, size_X_NPP, size_Y_NPP = RC.Open_array_info(os.path.join(Data_Path_NPP,yearly_NPP_File))
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value of the GPP files
dest = gdal.Open(monthly_GPP_Files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
# Create a empty numpy array
Yearly_GPP = np.zeros([size_Y, size_X])
# Calculte the total yearly GPP
for monthly_GPP_File in monthly_GPP_Files:
# Open array
Data = RC.Open_tiff_array(monthly_GPP_File)
# Remove nan values
Data[Data == NDV] = np.nan
# Add data to yearly sum
Yearly_GPP += Data
# Check if size is the same of NPP and GPP otherwise resize
if not (size_X_NPP is size_X or size_Y_NPP is size_Y):
Yearly_NPP = RC.resize_array_example(Yearly_NPP, Yearly_GPP)
# Loop over the monthly dates
for Date in Dates:
# If the Date is in the same year as the yearly NPP and GPP
if Date.year == year:
# Create empty GPP array
monthly_GPP = np.ones([size_Y, size_X]) * np.nan
# Get current month
month = Date.month
# Get the GPP file of the current year and month
monthly_GPP_File = glob.glob('*monthly_%d.%02d.01.tif' %(int(year), int(month)))[0]
monthly_GPP = RC.Open_tiff_array(monthly_GPP_File)
monthly_GPP[monthly_GPP == NDV] = np.nan
# Calculate the NDM based on the monthly and yearly NPP and GPP (fraction of GPP)
Monthly_NDM = Yearly_NPP * monthly_GPP / Yearly_GPP * (30./12.) *10000 # kg/ha
# Define output name
output_name = os.path.join(Data_Path_NDM, 'NDM_MOD17_kg_ha-1_monthly_%d.%02d.01.tif' %(int(year), int(month)))
# Save the NDM as tiff file
DC.Save_as_tiff(output_name, Monthly_NDM, geo_out, proj)
return(Data_Path_NDM)
def Merge_DEM_15s(output_folder_trash,output_file_merged,latlim, lonlim):
os.chdir(output_folder_trash)
tiff_files = glob.glob('*.tif')
resolution_geo = []
lonmin = lonlim[0]
lonmax = lonlim[1]
latmin = latlim[0]
latmax = latlim[1]
resolution_geo = 0.00416667
size_x_tot = int(np.round((lonmax-lonmin) / resolution_geo))
size_y_tot = int(np.round((latmax-latmin) / resolution_geo))
data_tot = np.ones([size_y_tot,size_x_tot]) * -9999.
for tiff_file in tiff_files:
inFile=os.path.join(output_folder_trash,tiff_file)
geo, proj, size_X, size_Y = RC.Open_array_info(inFile)
resolution_geo = geo[1]
lonmin_one = geo[0]
lonmax_one = geo[0] + size_X * geo[1]
latmin_one = geo[3] + size_Y * geo[5]
latmax_one = geo[3]
if lonmin_one < lonmin:
lonmin_clip = lonmin
else:
lonmin_clip = lonmin_one
if lonmax_one > lonmax:
lonmax_clip = lonmax
else:
lonmax_clip = lonmax_one
if latmin_one < latmin:
latmin_clip = latmin
else:
latmin_clip = latmin_one
if latmax_one > latmax:
latmax_clip = latmax
else:
latmax_clip = latmax_one
size_x_clip = int(np.round((lonmax_clip-lonmin_clip) / resolution_geo))
size_y_clip = int(np.round((latmax_clip-latmin_clip) / resolution_geo))
inFile=os.path.join(output_folder_trash,tiff_file)
geo, proj, size_X, size_Y = RC.Open_array_info(inFile)
Data = RC.Open_tiff_array(inFile)
lonmin_tiff = geo[0]
latmax_tiff = geo[3]
lon_tiff_position = int(np.round((lonmin_clip - lonmin_tiff)/ resolution_geo))
lat_tiff_position = int(np.round((latmax_tiff - latmax_clip)/ resolution_geo))
lon_data_tot_position = int(np.round((lonmin_clip - lonmin)/ resolution_geo))
lat_data_tot_position = int(np.round((latmax - latmax_clip)/ resolution_geo))
Data[Data<-9999.] = -9999.
data_tot[lat_data_tot_position:lat_data_tot_position+size_y_clip,lon_data_tot_position:lon_data_tot_position+size_x_clip][data_tot[lat_data_tot_position:lat_data_tot_position+size_y_clip,lon_data_tot_position:lon_data_tot_position+size_x_clip]==-9999]= Data[lat_tiff_position:lat_tiff_position+size_y_clip,lon_tiff_position:lon_tiff_position+size_x_clip][data_tot[lat_data_tot_position:lat_data_tot_position+size_y_clip,lon_data_tot_position:lon_data_tot_position+size_x_clip]==-9999]
geo_out = [lonmin, resolution_geo, 0.0, latmax, 0.0, -1 * resolution_geo]
geo_out = tuple(geo_out)
data_tot[data_tot<-9999.] = -9999.
return(data_tot, geo_out)
def Calc_Regions(Name_NC_Basin_CR, input_JRC, sensitivity, Boundaries):
import numpy as np
import wa.General.raster_conversions as RC
# Get JRC array and information
Array = RC.Open_tiff_array(input_JRC)
Geo_out, proj, size_X, size_Y = RC.Open_array_info(input_JRC)
# Get Basin boundary based on LU
Array_LU = RC.Open_nc_array(Name_NC_Basin_CR)
LU_array = RC.resize_array_example(Array_LU, Array)
basin_array = np.zeros(np.shape(LU_array))
basin_array[LU_array > 0] = 1
del LU_array
# find all pixels with water occurence
Array[basin_array < 1] = 0
Array[Array < 30] = 0
Array[Array >= 30] = 1
del basin_array
# sum larger areas to find lakes
x_size = np.round(int(np.shape(Array)[0]) / 30)
y_size = np.round(int(np.shape(Array)[1]) / 30)
sum_array = np.zeros([x_size, y_size])
for i in range(0, len(sum_array)):
for j in range(0, len(sum_array[1])):
sum_array[i, j] = np.sum(Array[i * 30:(i + 1) * 30,
j * 30:(j + 1) * 30])
del Array
lakes = np.argwhere(sum_array >= sensitivity)
lake_info = np.zeros([1, 4])
i = 0
k = 1
# find all neighboring pixels
for lake in lakes:
added = 0
for j in range(0, k):
if (lake[0] >= lake_info[j, 0] and lake[0] <= lake_info[j, 1]
and lake[1] >= lake_info[j, 2]
and lake[1] <= lake_info[j, 3]):
lake_info[j, 0] = np.maximum(
np.minimum(lake_info[j, 0], lake[0] - 8), 0)
lake_info[j, 1] = np.minimum(
np.maximum(lake_info[j, 1], lake[0] + 8), x_size)
lake_info[j, 2] = np.maximum(
np.minimum(lake_info[j, 2], lake[1] - 8), 0)
lake_info[j, 3] = np.minimum(
np.maximum(lake_info[j, 3], lake[1] + 8), y_size)
added = 1
if added == 0:
lake_info_one = np.zeros([4])
lake_info_one[0] = np.maximum(0, lake[0] - 8)
lake_info_one[1] = np.minimum(x_size, lake[0] + 8)
lake_info_one[2] = np.maximum(0, lake[1] - 8)
lake_info_one[3] = np.minimum(y_size, lake[1] + 8)
lake_info = np.append(lake_info, lake_info_one)
lake_info = np.resize(lake_info, (k + 1, 4))
k += 1
# merge all overlaping regions
p = 0
lake_info_end = np.zeros([1, 4])
for i in range(1, k):
added = 0
lake_info_one = lake_info[i, :]
lake_y_region = range(int(lake_info_one[0]), int(lake_info_one[1] + 1))
lake_x_region = range(int(lake_info_one[2]), int(lake_info_one[3] + 1))
for j in range(0, p + 1):
if len(lake_y_region) + len(
range(int(lake_info_end[j, 0]),
int(lake_info_end[j, 1] + 1))) is not len(
np.unique(
np.append(
lake_y_region,
range(int(lake_info_end[j, 0]),
int(lake_info_end[j, 1] + 1))))
) and len(lake_x_region) + len(
range(int(lake_info_end[j, 2]),
int(lake_info_end[j, 3] + 1))) is not len(
np.unique(
np.append(
lake_x_region,
range(
int(lake_info_end[j, 2]),
int(lake_info_end[j, 3] +
1))))):
lake_info_end[j, 0] = np.min(
np.unique(
np.append(
lake_y_region,
range(int(lake_info_end[j, 0]),
int(lake_info_end[j, 1] + 1)))))
lake_info_end[j, 1] = np.max(
np.unique(
np.append(
lake_y_region,
range(int(lake_info_end[j, 0]),
int(lake_info_end[j, 1] + 1)))))
lake_info_end[j, 2] = np.min(
np.unique(
np.append(
lake_x_region,
range(int(lake_info_end[j, 2]),
int(lake_info_end[j, 3] + 1)))))
lake_info_end[j, 3] = np.max(
np.unique(
np.append(
lake_x_region,
range(int(lake_info_end[j, 2]),
int(lake_info_end[j, 3] + 1)))))
added = 1
if added == 0:
lake_info_one = lake_info[i, :]
lake_info_end = np.append(lake_info_end, lake_info_one)
lake_info_end = np.resize(lake_info_end, (p + 2, 4))
p += 1
# calculate the area
Regions = np.zeros([p, 4])
pixel_x_size = Geo_out[1] * 30
pixel_y_size = Geo_out[5] * 30
for region in range(1, p + 1):
Regions[region - 1,
0] = Geo_out[0] + pixel_x_size * lake_info_end[region, 2]
Regions[region - 1,
1] = Geo_out[0] + pixel_x_size * (lake_info_end[region, 3] + 1)
Regions[region - 1,
2] = Geo_out[3] + pixel_y_size * (lake_info_end[region, 1] + 1)
Regions[region - 1,
3] = Geo_out[3] + pixel_y_size * lake_info_end[region, 0]
return (Regions)
def RetrieveData(Date, args):
"""
This function retrieves RFE data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, lonlim, latlim, xID, yID] = args
# Create https
DirFile = os.path.join(output_folder,'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
if not os.path.isfile(DirFile):
# open ftp server
ftp = FTP("ftp.cpc.ncep.noaa.gov", "", "")
ftp.login()
# Define FTP path to directory
pathFTP = 'fews/fewsdata/africa/rfe2/geotiff/'
# find the document name in this directory
ftp.cwd(pathFTP)
listing = []
# read all the file names in the directory
ftp.retrlines("LIST", listing.append)
# create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names
filename = 'africa_rfe.%s%02s%02s.tif.zip' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))
outfilename = os.path.join(output_folder,'africa_rfe.%s%02s%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
try:
local_filename = os.path.join(output_folder, filename)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# unzip the file
zip_filename = os.path.join(output_folder, filename)
DC.Extract_Data(zip_filename, output_folder)
# open tiff file
dataset = RC.Open_tiff_array(outfilename)
# clip dataset to the given extent
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# save dataset as geotiff file
latlim_adj = 40.05 - 0.1 * yID[0]
lonlim_adj = -20.05 + 0.1 * xID[0]
geo = [lonlim_adj, 0.1, 0, latlim_adj, 0, -0.1]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
# delete old tif file
os.remove(outfilename)
os.remove(zip_filename)
except:
print "file not exists"
return True
def main(Dir,
Startdate='',
Enddate='',
latlim=[-50, 50],
lonlim=[-180, 180],
cores=False,
Waitbar=1):
"""
This function downloads RFE V2.0 (monthly) data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
cores -- The number of cores used to run the routine.
It can be 'False' to avoid using parallel computing
routines.
Waitbar -- 1 (Default) will print a waitbar
"""
# Download data
print '\nDownload monthly RFE precipitation data for period %s till %s' % (
Startdate, Enddate)
# Check variables
if not Startdate:
Startdate = pd.Timestamp('2001-01-01')
if not Enddate:
Enddate = pd.Timestamp('Now')
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Make directory
output_folder = os.path.join(Dir, 'Precipitation', 'RFE', 'Monthly/')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Create Waitbar
if Waitbar == 1:
import wa.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
for Date in Dates:
month = Date.month
year = Date.year
end_day = calendar.monthrange(year, month)[1]
Startdate_one_month = '%s-%02s-01' % (year, month)
Enddate_one_month = '%s-%02s-%02s' % (year, month, end_day)
DownloadData(Dir, Startdate_one_month, Enddate_one_month, latlim,
lonlim, 0, cores)
Dates_daily = pd.date_range(Startdate_one_month,
Enddate_one_month,
freq='D')
# Make directory
input_folder_daily = os.path.join(Dir, 'Precipitation', 'RFE',
'Daily/')
i = 0
for Date_daily in Dates_daily:
file_name = 'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' % (
Date_daily.strftime('%Y'), Date_daily.strftime('%m'),
Date_daily.strftime('%d'))
file_name_daily_path = os.path.join(input_folder_daily, file_name)
if os.path.exists(file_name_daily_path):
if Date_daily == Dates_daily[i]:
Raster_monthly = RC.Open_tiff_array(file_name_daily_path)
else:
Raster_monthly += RC.Open_tiff_array(file_name_daily_path)
else:
if Date_daily == Dates_daily[i]:
i += 1
geo_out, proj, size_X, size_Y = RC.Open_array_info(
file_name_daily_path)
file_name = 'P_RFE.v2.0_mm-month-1_monthly_%s.%02s.01.tif' % (
Date.strftime('%Y'), Date.strftime('%m'))
file_name_output = os.path.join(output_folder, file_name)
DC.Save_as_tiff(file_name_output,
Raster_monthly,
geo_out,
projection="WGS84")
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
def Nearest_Interpolate(Dir_in, Startdate, Enddate, Dir_out=None):
"""
This functions calculates monthly tiff files based on the daily tiff files.
(will calculate the total sum)
Parameters
----------
Dir_in : str
Path to the input data
Startdate : str
Contains the start date of the model 'yyyy-mm-dd'
Enddate : str
Contains the end date of the model 'yyyy-mm-dd'
Dir_out : str
Path to the output data, default is same as Dir_in
"""
# import WA+ modules
import wa.General.data_conversions as DC
import wa.General.raster_conversions as RC
# Change working directory
os.chdir(Dir_in)
# Define end and start date
Dates = pd.date_range(Startdate, Enddate, freq='MS')
# Find all monthly files
files = glob.glob('*daily*.tif')
# Get array information and define projection
geo_out, proj, size_X, size_Y = RC.Open_array_info(files[0])
if int(proj.split('"')[-2]) == 4326:
proj = "WGS84"
# Get the No Data Value
dest = gdal.Open(files[0])
NDV = dest.GetRasterBand(1).GetNoDataValue()
for date in Dates:
Year = date.year
Month = date.month
files_one_year = glob.glob('*daily*%d.%02d*.tif' % (Year, Month))
# Create empty arrays
Month_data = np.zeros([size_Y, size_X])
# Get amount of days in month
Amount_days_in_month = int(calendar.monthrange(Year, Month)[1])
if len(files_one_year) is not Amount_days_in_month:
print("One day is missing!!!")
for file_one_year in files_one_year:
file_path = os.path.join(Dir_in, file_one_year)
Day_data = RC.Open_tiff_array(file_path)
Day_data[np.isnan(Day_data)] = 0.0
Day_data[Day_data == -9999] = 0.0
Month_data += Day_data
# Define output directory
if Dir_out is None:
Dir_out = Dir_in
# Define output name
output_name = os.path.join(Dir_out, file_one_year
.replace('daily', 'monthly')
.replace('day', 'month'))
output_name = output_name[:-14] + '%d.%02d.01.tif' % (date.year, date.month)
# Save tiff file
DC.Save_as_tiff(output_name, Month_data, geo_out, proj)
return
def Calculate(Basin, P_Product, ET_Product, Inflow_Text_Files,
Reservoirs_Lakes_Calculations, Startdate, Enddate, Simulation):
'''
This functions consists of the following sections:
1. Set General Parameters
2. Download Data
3. Convert the RAW data to NETCDF files
4. Create Mask based on LU map
5. Calculate Runoff based on Budyko
6. Add inflow in Runoff
7. Calculate River flow
7.1 Route Runoff
7.2 Add Reservoirs
7.3 Add surface water withdrawals
'''
# import General modules
import os
import gdal
import numpy as np
import pandas as pd
import copy
# import WA plus modules
from wa.General import raster_conversions as RC
from wa.General import data_conversions as DC
import wa.Functions.Five as Five
import wa.Functions.Start as Start
######################### 1. Set General Parameters ##############################
# Get environmental variable for the Home folder
WA_env_paths = os.environ["WA_HOME"].split(';')
Dir_Home = WA_env_paths[0]
# Create the Basin folder
Dir_Basin = os.path.join(Dir_Home, Basin)
if not os.path.exists(Dir_Basin):
os.makedirs(Dir_Basin)
# Get the boundaries of the basin based on the shapefile of the watershed
# Boundaries, Shape_file_name_shp = Start.Boundaries.Determine(Basin)
Boundaries, LU_dataset = Start.Boundaries.Determine_LU_Based(Basin)
LU_data = RC.Open_tiff_array(LU_dataset)
geo_out_LU, proj_LU, size_X_LU, size_Y_LU = RC.Open_array_info(LU_dataset)
# Define resolution of SRTM
Resolution = '15s'
# Get the amount of months
Amount_months = len(pd.date_range(Startdate, Enddate, freq='MS'))
Amount_months_reservoirs = Amount_months + 1
# Startdate for moving window Budyko
Startdate_2months_Timestamp = pd.Timestamp(Startdate) - pd.DateOffset(
months=2)
Startdate_2months = Startdate_2months_Timestamp.strftime('%Y-%m-%d')
############################# 2. Download Data ###################################
# Download data
Data_Path_P = Start.Download_Data.Precipitation(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Startdate_2months,
Enddate, P_Product)
Data_Path_ET = Start.Download_Data.Evapotranspiration(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Startdate_2months,
Enddate, ET_Product)
Data_Path_DEM = Start.Download_Data.DEM(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Resolution)
if Resolution is not '3s':
Data_Path_DEM = Start.Download_Data.DEM(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Resolution)
Data_Path_DEM_Dir = Start.Download_Data.DEM_Dir(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Resolution)
Data_Path_ETref = Start.Download_Data.ETreference(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']], Startdate_2months,
Enddate)
Data_Path_JRC_occurrence = Start.Download_Data.JRC_occurrence(
Dir_Basin, [Boundaries['Latmin'], Boundaries['Latmax']],
[Boundaries['Lonmin'], Boundaries['Lonmax']])
Data_Path_P_Monthly = os.path.join(Data_Path_P, 'Monthly')
###################### 3. Convert the RAW data to NETCDF files ##############################
# The sequence of converting the data is:
# DEM
# DEM flow directions
# Precipitation
# Evapotranspiration
# Reference Evapotranspiration
#_____________________________________DEM__________________________________
# Get the data of DEM and save as nc, This dataset is also used as reference for others
Example_dataset = os.path.join(Dir_Basin, Data_Path_DEM,
'DEM_HydroShed_m_%s.tif' % Resolution)
DEMdest = gdal.Open(Example_dataset)
Xsize_CR = int(DEMdest.RasterXSize)
Ysize_CR = int(DEMdest.RasterYSize)
DataCube_DEM_CR = DEMdest.GetRasterBand(1).ReadAsArray()
Name_NC_DEM_CR = DC.Create_NC_name('DEM_CR', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_DEM_CR):
DC.Save_as_NC(Name_NC_DEM_CR, DataCube_DEM_CR, 'DEM_CR',
Example_dataset)
DEMdest = None
#___________________________________DEM Dir________________________________
# Get the data of flow direction and save as nc.
Dir_dataset = os.path.join(Dir_Basin, Data_Path_DEM_Dir,
'DIR_HydroShed_-_%s.tif' % Resolution)
DEMDirdest = gdal.Open(Dir_dataset)
DataCube_DEM_Dir_CR = DEMDirdest.GetRasterBand(1).ReadAsArray()
Name_NC_DEM_Dir_CR = DC.Create_NC_name('DEM_Dir_CR', Simulation, Dir_Basin,
5)
if not os.path.exists(Name_NC_DEM_Dir_CR):
DC.Save_as_NC(Name_NC_DEM_Dir_CR, DataCube_DEM_Dir_CR, 'DEM_Dir_CR',
Example_dataset)
DEMDirdest = None
del DataCube_DEM_Dir_CR
#______________________________ Precipitation______________________________
# Define info for the nc files
info = [
'monthly', 'mm',
''.join([Startdate_2months[5:7], Startdate_2months[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
# Precipitation data
Name_NC_Prec_CR = DC.Create_NC_name('Prec_CR', Simulation, Dir_Basin, 5,
info)
if not os.path.exists(Name_NC_Prec_CR):
# Get the data of Precipitation and save as nc
DataCube_Prec_CR = RC.Get3Darray_time_series_monthly(
Dir_Basin,
Data_Path_P_Monthly,
Startdate_2months,
Enddate,
Example_data=Example_dataset)
DC.Save_as_NC(Name_NC_Prec_CR, DataCube_Prec_CR, 'Prec_CR',
Example_dataset, Startdate_2months, Enddate, 'monthly',
0.01)
del DataCube_Prec_CR
#____________________________ Evapotranspiration___________________________
# Evapotranspiration data
info = [
'monthly', 'mm',
''.join([Startdate_2months[5:7], Startdate_2months[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_ET_CR = DC.Create_NC_name('ET_CR', Simulation, Dir_Basin, 5, info)
if not os.path.exists(Name_NC_ET_CR):
# Get the data of Evaporation and save as nc
DataCube_ET_CR = RC.Get3Darray_time_series_monthly(
Dir_Basin,
Data_Path_ET,
Startdate_2months,
Enddate,
Example_data=Example_dataset)
DC.Save_as_NC(Name_NC_ET_CR, DataCube_ET_CR, 'ET_CR', Example_dataset,
Startdate_2months, Enddate, 'monthly', 0.01)
del DataCube_ET_CR
#_______________________Reference Evapotranspiration_______________________
# Reference Evapotranspiration data
Name_NC_ETref_CR = DC.Create_NC_name('ETref_CR', Simulation, Dir_Basin, 5,
info)
if not os.path.exists(Name_NC_ETref_CR):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_ETref_CR = RC.Get3Darray_time_series_monthly(
Dir_Basin,
Data_Path_ETref,
Startdate_2months,
Enddate,
Example_data=Example_dataset)
DC.Save_as_NC(Name_NC_ETref_CR, DataCube_ETref_CR, 'ETref_CR',
Example_dataset, Startdate_2months, Enddate, 'monthly',
0.01)
del DataCube_ETref_CR
#_______________________fraction surface water _______________________
Name_NC_frac_sw_CR = DC.Create_NC_name('Fraction_SW_CR', Simulation,
Dir_Basin, 5)
if not os.path.exists(Name_NC_frac_sw_CR):
DataCube_frac_sw = np.ones_like(LU_data) * np.nan
import wa.Functions.Start.Get_Dictionaries as GD
# Get dictionaries and keys
lulc = GD.get_sheet5_classes()
lulc_dict = GD.get_sheet5_classes().keys()
consumed_frac_dict = GD.sw_supply_fractions_sheet5()
for key in lulc_dict:
Numbers = lulc[key]
for LU_nmbr in Numbers:
Mask = np.zeros_like(LU_data)
Mask[LU_data == LU_nmbr] = 1
DataCube_frac_sw[Mask == 1] = consumed_frac_dict[key]
dest_frac_sw = DC.Save_as_MEM(DataCube_frac_sw, geo_out_LU, proj_LU)
dest_frac_sw_CR = RC.reproject_dataset_example(dest_frac_sw,
Example_dataset)
DataCube_frac_sw_CR = dest_frac_sw_CR.ReadAsArray()
DataCube_frac_sw_CR[DataCube_frac_sw_CR == 0] = np.nan
DC.Save_as_NC(Name_NC_frac_sw_CR,
DataCube_frac_sw_CR,
'Fraction_SW_CR',
Example_dataset,
Scaling_factor=0.01)
del DataCube_frac_sw_CR
del DataCube_DEM_CR
##################### 4. Create Mask based on LU map ###########################
# Now a mask will be created to define the area of interest (pixels where there is a landuse defined)
#_____________________________________LU___________________________________
destLU = RC.reproject_dataset_example(LU_dataset,
Example_dataset,
method=1)
DataCube_LU_CR = destLU.GetRasterBand(1).ReadAsArray()
Raster_Basin_CR = np.zeros([Ysize_CR, Xsize_CR])
Raster_Basin_CR[DataCube_LU_CR > 0] = 1
Name_NC_Basin_CR = DC.Create_NC_name('Basin_CR', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_Basin_CR):
DC.Save_as_NC(Name_NC_Basin_CR, Raster_Basin_CR, 'Basin_CR',
Example_dataset)
#del Raster_Basin
'''
Name_NC_Basin = DC.Create_NC_name('Basin_CR', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_Basin):
Raster_Basin = RC.Vector_to_Raster(Dir_Basin, Shape_file_name_shp, Example_dataset)
Raster_Basin = np.clip(Raster_Basin, 0, 1)
DC.Save_as_NC(Name_NC_Basin, Raster_Basin, 'Basin_CR', Example_dataset)
#del Raster_Basin
'''
###################### 5. Calculate Runoff based on Budyko ###########################
# Define info for the nc files
info = [
'monthly', 'mm', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
# Define the output names of section 5 and 6
Name_NC_Runoff_CR = DC.Create_NC_name('Runoff_CR', Simulation, Dir_Basin,
5, info)
Name_NC_Runoff_for_Routing_CR = Name_NC_Runoff_CR
if not os.path.exists(Name_NC_Runoff_CR):
# Calculate runoff based on Budyko
DataCube_Runoff_CR = Five.Budyko.Calc_runoff(Name_NC_ETref_CR,
Name_NC_Prec_CR)
# Save the runoff as netcdf
DC.Save_as_NC(Name_NC_Runoff_CR, DataCube_Runoff_CR, 'Runoff_CR',
Example_dataset, Startdate, Enddate, 'monthly', 0.01)
del DataCube_Runoff_CR
'''
###################### Calculate Runoff with P min ET ###########################
Name_NC_Runoff_CR = DC.Create_NC_name('Runoff_CR', Simulation, Dir_Basin, 5, info)
if not os.path.exists(Name_NC_Runoff_CR):
ET = RC.Open_nc_array(Name_NC_ET_CR)
P = RC.Open_nc_array(Name_NC_Prec_CR)
DataCube_Runoff_CR = P - ET
DataCube_Runoff_CR[:,:,:][DataCube_Runoff_CR<=0.1] = 0
DataCube_Runoff_CR[:,:,:][np.isnan(DataCube_Runoff_CR)] = 0
DC.Save_as_NC(Name_NC_Runoff_CR, DataCube_Runoff_CR, 'Runoff_CR', Example_dataset, Startdate, Enddate, 'monthly')
del DataCube_Runoff_CR
'''
############### 6. Add inflow in basin by using textfile #########################
# add inlets if there are textfiles defined
if len(Inflow_Text_Files) > 0:
# Create name of the Runoff with inlets
Name_NC_Runoff_with_Inlets_CR = DC.Create_NC_name(
'Runoff_with_Inlets_CR', Simulation, Dir_Basin, 5, info)
# Use this runoff name for the routing (it will overwrite the runoff without inlets)
Name_NC_Runoff_for_Routing_CR = Name_NC_Runoff_with_Inlets_CR
# Create the file if it not exists
if not os.path.exists(Name_NC_Runoff_with_Inlets_CR):
# Calculate the runoff that will be routed by including the inlets
DataCube_Runoff_with_Inlets_CR = Five.Inlets.Add_Inlets(
Name_NC_Runoff_CR, Inflow_Text_Files)
# Save this runoff as netcdf
DC.Save_as_NC(Name_NC_Runoff_with_Inlets_CR,
DataCube_Runoff_with_Inlets_CR,
'Runoff_with_Inlets_CR', Example_dataset, Startdate,
Enddate, 'monthly', 0.01)
del DataCube_Runoff_with_Inlets_CR
######################### 7. Now the surface water is calculated #################
# Names for dicionaries and nc files
# CR1 = Natural_flow with only green water
# CR2 = Natural_flow with only green water and reservoirs
# CR3 = Flow with green, blue and reservoirs
######################### 7.1 Apply Channel Routing ###############################
# Create the name for the netcdf outputs for section 7.1
info = [
'monthly', 'pixels', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_Acc_Pixels_CR = DC.Create_NC_name('Acc_Pixels_CR', Simulation,
Dir_Basin, 5)
info = [
'monthly', 'm3', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_Discharge_CR1 = DC.Create_NC_name('Discharge_CR1', Simulation,
Dir_Basin, 5, info)
# If one of the outputs does not exists, run this part
if not (os.path.exists(Name_NC_Acc_Pixels_CR)
and os.path.exists(Name_NC_Discharge_CR1)):
Accumulated_Pixels_CR, Discharge_CR1 = Five.Channel_Routing.Channel_Routing(
Name_NC_DEM_Dir_CR,
Name_NC_Runoff_for_Routing_CR,
Name_NC_Basin_CR,
Example_dataset,
Degrees=1)
# Save Results
DC.Save_as_NC(Name_NC_Acc_Pixels_CR, Accumulated_Pixels_CR,
'Acc_Pixels_CR', Example_dataset)
DC.Save_as_NC(Name_NC_Discharge_CR1, Discharge_CR1, 'Discharge_CR1',
Example_dataset, Startdate, Enddate, 'monthly')
################# Calculate the natural river and river zones #################
Name_NC_Rivers_CR = DC.Create_NC_name('Rivers_CR', Simulation, Dir_Basin,
5, info)
if not os.path.exists(Name_NC_Rivers_CR):
# Open routed discharge array
Discharge_CR1 = RC.Open_nc_array(Name_NC_Discharge_CR1)
Raster_Basin = RC.Open_nc_array(Name_NC_Basin_CR)
# Calculate mean average over the period
if len(np.shape(Discharge_CR1)) > 2:
Routed_Discharge_Ave = np.nanmean(Discharge_CR1, axis=0)
else:
Routed_Discharge_Ave = Discharge_CR1
# Define the 2% highest pixels as rivers
Rivers = np.zeros([
np.size(Routed_Discharge_Ave, 0),
np.size(Routed_Discharge_Ave, 1)
])
Routed_Discharge_Ave[Raster_Basin != 1] = np.nan
Routed_Discharge_Ave_number = np.nanpercentile(Routed_Discharge_Ave,
98)
Rivers[
Routed_Discharge_Ave >
Routed_Discharge_Ave_number] = 1 # if yearly average is larger than 5000km3/month that it is a river
# Save the river file as netcdf file
DC.Save_as_NC(Name_NC_Rivers_CR, Rivers, 'Rivers_CR', Example_dataset)
########################## Create river directories ###########################
Name_py_River_dict_CR1 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'River_dict_CR1_simulation%d.npy' % (Simulation))
Name_py_DEM_dict_CR1 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'DEM_dict_CR1_simulation%d.npy' % (Simulation))
Name_py_Distance_dict_CR1 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'Distance_dict_CR1_simulation%d.npy' % (Simulation))
if not (os.path.exists(Name_py_River_dict_CR1)
and os.path.exists(Name_py_DEM_dict_CR1)
and os.path.exists(Name_py_Distance_dict_CR1)):
# Get river and DEM dict
River_dict_CR1, DEM_dict_CR1, Distance_dict_CR1 = Five.Create_Dict.Rivers_General(
Name_NC_DEM_CR, Name_NC_DEM_Dir_CR, Name_NC_Acc_Pixels_CR,
Name_NC_Rivers_CR, Example_dataset)
np.save(Name_py_River_dict_CR1, River_dict_CR1)
np.save(Name_py_DEM_dict_CR1, DEM_dict_CR1)
np.save(Name_py_Distance_dict_CR1, Distance_dict_CR1)
else:
# Load
River_dict_CR1 = np.load(Name_py_River_dict_CR1).item()
DEM_dict_CR1 = np.load(Name_py_DEM_dict_CR1).item()
Distance_dict_CR1 = np.load(Name_py_Distance_dict_CR1).item()
Name_py_Discharge_dict_CR1 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'Discharge_dict_CR1_simulation%d.npy' % (Simulation))
if not os.path.exists(Name_py_Discharge_dict_CR1):
# Get discharge dict
Discharge_dict_CR1 = Five.Create_Dict.Discharge(
Name_NC_Discharge_CR1, River_dict_CR1, Amount_months,
Example_dataset)
np.save(Name_py_Discharge_dict_CR1, Discharge_dict_CR1)
else:
# Load
Discharge_dict_CR1 = np.load(Name_py_Discharge_dict_CR1).item()
###################### 7.2 Calculate surface water storage characteristics ######################
Name_py_Discharge_dict_CR2 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'Discharge_dict_CR2_simulation%d.npy' % (Simulation))
Name_py_River_dict_CR2 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'River_dict_CR2_simulation%d.npy' % (Simulation))
Name_py_DEM_dict_CR2 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'DEM_dict_CR2_simulation%d.npy' % (Simulation))
Name_py_Distance_dict_CR2 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'Distance_dict_CR2_simulation%d.npy' % (Simulation))
Name_py_Diff_Water_Volume = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'Diff_Water_Volume_CR2_simulation%d.npy' % (Simulation))
Name_py_Regions = os.path.join(Dir_Basin, 'Simulations',
'Simulation_%d' % Simulation, 'Sheet_5',
'Regions_simulation%d.npy' % (Simulation))
if not (os.path.exists(Name_py_Discharge_dict_CR2)
and os.path.exists(Name_py_River_dict_CR2)
and os.path.exists(Name_py_DEM_dict_CR2)
and os.path.exists(Name_py_Distance_dict_CR2)):
# Copy dicts as starting adding reservoir
Discharge_dict_CR2 = copy.deepcopy(Discharge_dict_CR1)
River_dict_CR2 = copy.deepcopy(River_dict_CR1)
DEM_dict_CR2 = copy.deepcopy(DEM_dict_CR1)
Distance_dict_CR2 = copy.deepcopy(Distance_dict_CR1)
if Reservoirs_Lakes_Calculations == 1:
# define input tiffs for surface water calculations
input_JRC = os.path.join(Dir_Basin, Data_Path_JRC_occurrence,
'JRC_Occurrence_percent.tif')
DEM_dataset = os.path.join(Dir_Basin, Data_Path_DEM,
'DEM_HydroShed_m_3s.tif')
sensitivity = 700 # 900 is less sensitive 1 is very sensitive
Regions = Five.Reservoirs.Calc_Regions(Name_NC_Basin_CR, input_JRC,
sensitivity, Boundaries)
Diff_Water_Volume = np.zeros(
[len(Regions), Amount_months_reservoirs - 1, 3])
reservoir = 0
for region in Regions:
popt = Five.Reservoirs.Find_Area_Volume_Relation(
region, input_JRC, DEM_dataset)
Area_Reservoir_Values = Five.Reservoirs.GEE_calc_reservoir_area(
region, Startdate, Enddate)
Diff_Water_Volume[
reservoir, :, :] = Five.Reservoirs.Calc_Diff_Storage(
Area_Reservoir_Values, popt)
reservoir += 1
################# 7.3 Add storage reservoirs and change outflows ##################
Discharge_dict_CR2, River_dict_CR2, DEM_dict_CR2, Distance_dict_CR2 = Five.Reservoirs.Add_Reservoirs(
Name_NC_Rivers_CR, Name_NC_Acc_Pixels_CR, Diff_Water_Volume,
River_dict_CR2, Discharge_dict_CR2, DEM_dict_CR2,
Distance_dict_CR2, Regions, Example_dataset)
np.save(Name_py_Regions, Regions)
np.save(Name_py_Diff_Water_Volume, Diff_Water_Volume)
np.save(Name_py_Discharge_dict_CR2, Discharge_dict_CR2)
np.save(Name_py_River_dict_CR2, River_dict_CR2)
np.save(Name_py_DEM_dict_CR2, DEM_dict_CR2)
np.save(Name_py_Distance_dict_CR2, Distance_dict_CR2)
else:
# Load
Discharge_dict_CR2 = np.load(Name_py_Discharge_dict_CR2).item()
River_dict_CR2 = np.load(Name_py_River_dict_CR2).item()
DEM_dict_CR2 = np.load(Name_py_DEM_dict_CR2).item()
Distance_dict_CR2 = np.load(Name_py_Distance_dict_CR2).item()
####################### 7.3 Add surface water withdrawals #############################
Name_py_Discharge_dict_CR3 = os.path.join(
Dir_Basin, 'Simulations', 'Simulation_%d' % Simulation, 'Sheet_5',
'Discharge_dict_CR3_simulation%d.npy' % (Simulation))
if not os.path.exists(Name_py_Discharge_dict_CR3):
Discharge_dict_CR3, DataCube_ETblue_m3 = Five.Irrigation.Add_irrigation(
Discharge_dict_CR2, River_dict_CR2, Name_NC_Rivers_CR,
Name_NC_ET_CR, Name_NC_ETref_CR, Name_NC_Prec_CR, Name_NC_Basin_CR,
Name_NC_frac_sw_CR, Startdate, Enddate, Example_dataset)
np.save(Name_py_Discharge_dict_CR3, Discharge_dict_CR3)
# save ETblue as nc
info = [
'monthly', 'm3-month-1', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_ETblue = DC.Create_NC_name('ETblue', Simulation, Dir_Basin, 5,
info)
DC.Save_as_NC(Name_NC_ETblue, DataCube_ETblue_m3, 'ETblue',
Example_dataset, Startdate, Enddate, 'monthly')
else:
Discharge_dict_CR3 = np.load(Name_py_Discharge_dict_CR3).item()
################################# Plot graph ##################################
# Draw graph
Five.Channel_Routing.Graph_DEM_Distance_Discharge(
Discharge_dict_CR3, Distance_dict_CR2, DEM_dict_CR2, River_dict_CR2,
Startdate, Enddate, Example_dataset)
######################## Change data to fit the LU data #######################
# Discharge
# Define info for the nc files
info = [
'monthly', 'm3-month-1', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_Discharge = DC.Create_NC_name('Discharge', Simulation, Dir_Basin,
5, info)
if not os.path.exists(Name_NC_Discharge):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_Discharge_CR = DC.Convert_dict_to_array(
River_dict_CR2, Discharge_dict_CR3, Example_dataset)
DC.Save_as_NC(Name_NC_Discharge, DataCube_Discharge_CR, 'Discharge',
Example_dataset, Startdate, Enddate, 'monthly')
del DataCube_Discharge_CR
# DEM
Name_NC_DEM = DC.Create_NC_name('DEM', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_DEM):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_DEM_CR = RC.Open_nc_array(Name_NC_DEM_CR)
DataCube_DEM = RC.resize_array_example(DataCube_DEM_CR,
LU_data,
method=1)
DC.Save_as_NC(Name_NC_DEM, DataCube_DEM, 'DEM', LU_dataset)
del DataCube_DEM
# flow direction
Name_NC_DEM_Dir = DC.Create_NC_name('DEM_Dir', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_DEM_Dir):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_DEM_Dir_CR = RC.Open_nc_array(Name_NC_DEM_Dir_CR)
DataCube_DEM_Dir = RC.resize_array_example(DataCube_DEM_Dir_CR,
LU_data,
method=1)
DC.Save_as_NC(Name_NC_DEM_Dir, DataCube_DEM_Dir, 'DEM_Dir', LU_dataset)
del DataCube_DEM_Dir
# Precipitation
# Define info for the nc files
info = [
'monthly', 'mm', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_Prec = DC.Create_NC_name('Prec', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_Prec):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_Prec = RC.Get3Darray_time_series_monthly(
Dir_Basin, Data_Path_P_Monthly, Startdate, Enddate, LU_dataset)
DC.Save_as_NC(Name_NC_Prec, DataCube_Prec, 'Prec', LU_dataset,
Startdate, Enddate, 'monthly', 0.01)
del DataCube_Prec
# Evapotranspiration
Name_NC_ET = DC.Create_NC_name('ET', Simulation, Dir_Basin, 5)
if not os.path.exists(Name_NC_ET):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_ET = RC.Get3Darray_time_series_monthly(
Dir_Basin, Data_Path_ET, Startdate, Enddate, LU_dataset)
DC.Save_as_NC(Name_NC_ET, DataCube_ET, 'ET', LU_dataset, Startdate,
Enddate, 'monthly', 0.01)
del DataCube_ET
# Reference Evapotranspiration data
Name_NC_ETref = DC.Create_NC_name('ETref', Simulation, Dir_Basin, 5, info)
if not os.path.exists(Name_NC_ETref):
# Get the data of Reference Evapotranspiration and save as nc
DataCube_ETref = RC.Get3Darray_time_series_monthly(
Dir_Basin, Data_Path_ETref, Startdate, Enddate, LU_dataset)
DC.Save_as_NC(Name_NC_ETref, DataCube_ETref, 'ETref', LU_dataset,
Startdate, Enddate, 'monthly', 0.01)
del DataCube_ETref
# Rivers
Name_NC_Rivers = DC.Create_NC_name('Rivers', Simulation, Dir_Basin, 5,
info)
if not os.path.exists(Name_NC_Rivers):
# Get the data of Reference Evapotranspiration and save as nc
Rivers_CR = RC.Open_nc_array(Name_NC_Rivers_CR)
DataCube_Rivers = RC.resize_array_example(Rivers_CR, LU_data)
DC.Save_as_NC(Name_NC_Rivers, DataCube_Rivers, 'Rivers', LU_dataset)
del DataCube_Rivers, Rivers_CR
# Discharge
# Define info for the nc files
info = [
'monthly', 'm3', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_NC_Routed_Discharge = DC.Create_NC_name('Routed_Discharge',
Simulation, Dir_Basin, 5,
info)
if not os.path.exists(Name_NC_Routed_Discharge):
# Get the data of Reference Evapotranspiration and save as nc
Routed_Discharge_CR = RC.Open_nc_array(Name_NC_Discharge)
DataCube_Routed_Discharge = RC.resize_array_example(
Routed_Discharge_CR, LU_data)
DC.Save_as_NC(Name_NC_Routed_Discharge, DataCube_Routed_Discharge,
'Routed_Discharge', LU_dataset, Startdate, Enddate,
'monthly')
del DataCube_Routed_Discharge, Routed_Discharge_CR
# Get raster information
geo_out, proj, size_X, size_Y = RC.Open_array_info(Example_dataset)
Rivers = RC.Open_nc_array(Name_NC_Rivers_CR)
# Create ID Matrix
y, x = np.indices((size_Y, size_X))
ID_Matrix = np.int32(
np.ravel_multi_index(np.vstack((y.ravel(), x.ravel())),
(size_Y, size_X),
mode='clip').reshape(x.shape)) + 1
# Get tiff array time dimension:
time_dimension = int(np.shape(Discharge_dict_CR3[0])[0])
# create an empty array
Result = np.zeros([time_dimension, size_Y, size_X])
for river_part in range(0, len(River_dict_CR2)):
for river_pixel in range(1, len(River_dict_CR2[river_part])):
river_pixel_ID = River_dict_CR2[river_part][river_pixel]
if len(np.argwhere(ID_Matrix == river_pixel_ID)) > 0:
row, col = np.argwhere(ID_Matrix == river_pixel_ID)[0][:]
Result[:, row,
col] = Discharge_dict_CR3[river_part][:, river_pixel]
print(river_part)
Outflow = Discharge_dict_CR3[0][:, 1]
for i in range(0, time_dimension):
output_name = r'C:/testmap/rtest_%s.tif' % i
Result_one = Result[i, :, :]
DC.Save_as_tiff(output_name, Result_one, geo_out, "WGS84")
import os
# Get environmental variable for the Home folder
WA_env_paths = os.environ["WA_HOME"].split(';')
Dir_Home = WA_env_paths[0]
# Create the Basin folder
Dir_Basin = os.path.join(Dir_Home, Basin)
info = [
'monthly', 'm3-month-1', ''.join([Startdate[5:7], Startdate[0:4]]),
''.join([Enddate[5:7], Enddate[0:4]])
]
Name_Result = DC.Create_NC_name('DischargeEnd', Simulation, Dir_Basin, 5,
info)
Result[np.logical_and(Result == 0.0, Rivers == 0.0)] = np.nan
DC.Save_as_NC(Name_Result, Result, 'DischargeEnd', Example_dataset,
Startdate, Enddate, 'monthly')
return ()
humid = r"D:\Weather\Lebanon\Bekaa-ETref\2017\Weather_Data\Model\GLDAS\daily\qair_f_inst\mean/"
humids = [
os.path.join(humid, fn) for fn in next(os.walk(humid))[2]
if fn[-4:] == '.tif'
]
press = r"D:\Weather\Lebanon\Bekaa-ETref\2017\Weather_Data\Model\GLDAS\daily\psurf_f_inst\mean/"
presss = [
os.path.join(press, fn) for fn in next(os.walk(press))[2]
if fn[-4:] == '.tif'
]
Outfilepath = r"D:\Weather\Lebanon\Bekaa-ETref\2017\Weather_Data\Model\GLDAS\daily\humid/"
for i in range(0, len(tmeans)):
geo_out, proj, size_X, size_Y = RC.Open_array_info(tmeans[i])
Tdata = RC.Open_tiff_array(tmeans[i])
Tdata[Tdata < -900] = np.nan
Pdata = RC.Open_tiff_array(presss[i])
Hdata = RC.Open_tiff_array(humids[i])
Esdata = 0.6108 * np.exp((17.27 * Tdata) / (Tdata + 237.3))
HumData = np.minimum((1.6077717 * Hdata * Pdata / Esdata), 1) * 100
datestamp = tmeans[i][-14:-4]
Outfilename = os.path.join(Outfilepath, datestamp + '.tif')
DC.Save_as_tiff(Outfilename, HumData, geo_out, "WGS84")
