def RetrieveData(args):
"""
This function retrieves JRC data for a given date from the
http://storage.googleapis.com/global-surface-water/downloads/ server.
Keyword arguments:
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, Names_to_download, lonlim, latlim] = args
# Collect the data from the JRC webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(Names_to_download, output_folder)
except:
print("Was not able to download the file")
# Clip the data to the users extend
if len(Names_to_download) == 1:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Names_to_download[0])
data_end, geo_end = RC.clip_data(data_in, latlim, lonlim)
else:
data_end = np.zeros([int((latlim[1] - latlim[0])/0.00025), int((lonlim[1] - lonlim[0])/0.00025)])
for Name_to_merge in Names_to_download:
trash_folder = os.path.join(output_folder, "Trash")
data_in = os.path.join(trash_folder, Name_to_merge)
geo_out, proj, size_X, size_Y = RC.Open_array_info(data_in)
lat_min_merge = np.maximum(latlim[0], geo_out[3] + size_Y * geo_out[5])
lat_max_merge = np.minimum(latlim[1], geo_out[3])
lon_min_merge = np.maximum(lonlim[0], geo_out[0])
lon_max_merge = np.minimum(lonlim[1], geo_out[0] + size_X * geo_out[1])
lonmerge = [lon_min_merge, lon_max_merge]
latmerge = [lat_min_merge, lat_max_merge]
data_one, geo_one = RC.clip_data(data_in, latmerge, lonmerge)
Ystart = int((geo_one[3] - latlim[1])/geo_one[5])
Yend = int(Ystart + np.shape(data_one)[0])
Xstart = int((geo_one[0] - lonlim[0])/geo_one[1])
Xend = int(Xstart + np.shape(data_one)[1])
data_end[Ystart:Yend, Xstart:Xend] = data_one
geo_end = tuple([lonlim[0], geo_one[1], 0, latlim[1], 0, geo_one[5]])
# Save results as Gtiff
fileName_out = os.path.join(output_folder, 'JRC_Occurrence_percent.tif')
DC.Save_as_tiff(name=fileName_out, data=data_end, geo=geo_end, projection='WGS84')
shutil.rmtree(trash_folder)
return True
def DownloadData(Dir, latlim, lonlim, Waitbar):
"""
This function downloads NLDAS Forcing data hourly, daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
"""
# Define the output name
output_filename = os.path.join(Dir, 'LU_ESACCI.tif')
# Set the url of the server
url = r"https://storage.googleapis.com/cci-lc-v207/ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.zip"
# Create a Trash folder
Dir_trash = os.path.join(Dir, "Trash")
if not os.path.exists(Dir_trash):
os.makedirs(Dir_trash)
# Define location of download
filename_out = os.path.join(
Dir_trash, "ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.zip")
# Download data
urllib.request.urlretrieve(url, filename=filename_out)
# Extract data
DC.Extract_Data(filename_out, Dir_trash)
# Define input of the world tiff file
filename_world = os.path.join(
Dir_trash, "product", "ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7.tif")
try:
# Clip data to user extend
data, Geo_out = RC.clip_data(filename_world, latlim, lonlim)
# Save data of clipped array
DC.Save_as_tiff(output_filename, data, Geo_out, 4326)
except:
RC.Clip_Dataset_GDAL(RC.clip_data(filename_world, latlim, lonlim))
# Remove trash folder
shutil.rmtree(Dir_trash)
return ()
def RetrieveData(Date, args):
"""
This function retrieves MOD16 ET data for a given date from the
ftp://ftp.ntsg.umt.edu/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, latlim, lonlim,
timestep, hdf_library, Size_pix
] = args
if timestep == 'monthly':
ETfileName = os.path.join(
output_folder, 'ET_MOD16A2_mm-month-1_monthly_' +
Date.strftime('%Y') + '.' + Date.strftime('%m') + '.01.tif')
elif timestep == '8-daily':
ETfileName = os.path.join(
output_folder,
'ET_MOD16A2_mm-8days-1_8-daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if not os.path.exists(ETfileName):
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
timestep, hdf_library, Size_pix)
except:
print("Was not able to download the file")
try:
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
DC.Save_as_tiff(name=ETfileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print("Failed for date: %s" % Date)
return ()
def RetrieveData(Date, args):
"""
This function retrieves MOD15 FPAR data for a given date from the
http://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim, unit,
dataset, nameDownload, hdf_library
] = args
FPARfileName = os.path.join(
output_folder,
'%s_MOD15_%s_8-daily_' % (dataset, unit) + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if not os.path.exists(FPARfileName):
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
nameDownload, hdf_library)
except:
print("Was not able to download the file")
try:
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
# Save the file as tiff
DC.Save_as_tiff(name=FPARfileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print("Failed for date: %s" % Date)
return True
def RetrieveData(Date, args):
"""
This function retrieves MOD9 Reflectance data for a given date from the
http://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim, band,
resolution, hdf_library
] = args
ReffileName = os.path.join(
output_folder,
'ReflectanceBand%d_MOD09GQ_-_daily_' % band + Date.strftime('%Y') +
'.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if not os.path.exists(ReffileName):
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
band, resolution, hdf_library)
except:
print("Was not able to download the file")
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
try:
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
# Save results as Gtiff
DC.Save_as_tiff(name=ReffileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print('data for %02d-%02d-%d is not available' %
(Date.day, Date.month, Date.year))
return True
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
"""
This function retrieves GWF data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Keyword arguments:
output_folder -- name of the end file with the weekly ALEXI data
End_filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Set the file names and directories
filename = "Gray_Water_Footprint.tif"
local_filename = os.path.join(output_folder, filename)
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Clip extend out of world data
dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
# make geotiff file
DC.Save_as_tiff(name=filename_Out,
data=dataset,
geo=Geo_out,
projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print("file not exists")
return
def DownloadData(output_folder, latlim, lonlim):
"""
This function downloads DEM data from SRTM
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -60 and 60)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -60 or latlim[1] > 60:
print(
'Latitude above 60N or below 60S is not possible. Value set to maximum'
)
latlim[0] = np.max(latlim[0], -60)
latlim[1] = np.min(latlim[1], 60)
if lonlim[0] < -180 or lonlim[1] > 180:
print(
'Longitude must be between 180E and 180W. Now value is set to maximum'
)
lonlim[0] = np.max(lonlim[0], -180)
lonlim[1] = np.min(lonlim[1], 180)
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.replace(".zip", ".tif")
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
dest_SRTM = gdal.Open(output_tiff)
geo_out = dest_SRTM.GetGeoTransform()
size_X = dest_SRTM.RasterXSize
size_Y = dest_SRTM.RasterYSize
if (int(size_X) != int(6001) or int(size_Y) != int(6001)):
data = np.ones((6001, 6001)) * -9999
# Create the latitude bound
Vfile = nameFile.split("_")[2][0:2]
Bound2 = 60 - 5 * (int(Vfile) - 1)
# Create the longitude bound
Hfile = nameFile.split("_")[1]
Bound1 = -180 + 5 * (int(Hfile) - 1)
Expected_X_min = Bound1
Expected_Y_max = Bound2
Xid_start = int(
np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(
np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) /
geo_out[1]))
Yid_start = int(
np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(
np.round((Expected_Y_max - (geo_out[3] +
(size_Y * geo_out[5]))) /
(-geo_out[5])))
data_SRTM = dest_SRTM.GetRasterBand(1).ReadAsArray()
data[Yid_start:Yid_end, Xid_start:Xid_end] = data_SRTM
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [
Bound1 - 0.5 * 0.00083333333333333, 0.00083333333333333,
0.0, Bound2 + 0.5 * 0.00083333333333333, 0.0,
-0.0008333333333333333333
]
# save chunk as tiff file
destDEM = DC.Save_as_MEM(data=data,
geo=geo_in,
projection="WGS84")
dest_SRTM = None
except:
# If tile not exist create a replacing zero tile (sea tiles)
file_name_tiff = file_name.replace(".zip", ".tif")
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
file_name = nameFile
data = np.ones((6001, 6001)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = nameFile.split("_")[2][0:2]
Bound2 = 60 - 5 * (int(Vfile) - 1)
# Create the longitude bound
Hfile = nameFile.split("_")[1]
Bound1 = -180 + 5 * (int(Hfile) - 1)
# Geospatial data for the tile
geo_in = [
Bound1 - 0.5 * 0.00083333333333333, 0.00083333333333333, 0.0,
Bound2 + 0.5 * 0.00083333333333333, 0.0,
-0.0008333333333333333333
]
# save chunk as tiff file
destDEM = DC.Save_as_MEM(data=data, geo=geo_in, projection="WGS84")
# clip data
Data, Geo_data = RC.clip_data(destDEM, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (file_name)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd,
data=Data,
geo=Geo_data,
projection="WGS84")
size_X_end = int(size_X_tot / len(rangeLat)) # + 1 !!!
size_Y_end = int(size_Y_tot / len(rangeLon)) # + 1 !!!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
# name of the end result
output_DEM_name = "DEM_SRTM_m_3s.tif"
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name,
data=datasetTot,
geo=geo_out,
projection="WGS84")
os.chdir(output_folder)
def DownloadData(output_folder, latlim, lonlim, parameter, resolution):
"""
This function downloads DEM data from HydroSHED
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
Resample -- 1 = The data will be resampled to 0.001 degree spatial
resolution
-- 0 = The data will have the same pixel size as the data obtained
from the internet
"""
# Define parameter depedent variables
if parameter == "dir_3s":
para_name = "DIR"
unit = "-"
resolution = '3s'
parameter = 'dir'
if parameter == "dem_3s":
para_name = "DEM"
unit = "m"
resolution = '3s'
parameter = 'dem'
if parameter == "dir_15s":
para_name = "DIR"
unit = "-"
resolution = '15s'
parameter = 'dir'
if parameter == "dem_15s":
para_name = "DEM"
unit = "m"
resolution = '15s'
parameter = 'dem'
if parameter == "dir_30s":
para_name = "DIR"
unit = "-"
resolution = '30s'
parameter = 'dir'
if parameter == "dem_30s":
para_name = "DEM"
unit = "m"
resolution = '30s'
parameter = 'dem'
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
if resolution == '3s':
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim,
parameter)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
if resolution == '15s' or resolution == '30s':
name = Find_Document_names_15s_30s(latlim, lonlim, parameter,
resolution)
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash,
parameter, para_name,
resolution)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# Converts the data with a adf extention to a tiff extension.
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif'
file_name_extract = file_name.split('_')[0:3]
if resolution == '3s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1]
if resolution == '15s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1] + '_15s'
if resolution == '30s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1] + '_30s'
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
if (resolution == "15s" or resolution == "3s"):
input_adf = os.path.join(output_folder_trash,
file_name_extract2,
file_name_extract2, 'hdr.adf')
output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff)
# convert data from adf to a tiff file
if resolution == "30s":
input_bil = os.path.join(output_folder_trash,
'%s.bil' % file_name_extract2)
output_tiff = DC.Convert_bil_to_tiff(input_bil, output_tiff)
geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff)
if (resolution == "3s" and
(int(size_X) != int(6000) or int(size_Y) != int(6000))):
data = np.ones((6000, 6000)) * -9999
# Create the latitude bound
Vfile = str(nameFile)[1:3]
SignV = str(nameFile)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(nameFile)[4:7]
SignH = str(nameFile)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
Expected_X_min = Bound1
Expected_Y_max = Bound2 + 5
Xid_start = int(
np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(
np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) /
geo_out[1]))
Yid_start = int(
np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(
np.round((Expected_Y_max - (geo_out[3] +
(size_Y * geo_out[5]))) /
(-geo_out[5])))
data[Yid_start:Yid_end,
Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff)
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
except:
if resolution == '3s':
# If tile not exist create a replacing zero tile (sea tiles)
output = nameFile.split('.')[0] + "_trans_temporary.tif"
output_tiff = os.path.join(output_folder_trash, output)
file_name = nameFile
data = np.ones((6000, 6000)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = str(file_name)[1:3]
SignV = str(file_name)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(file_name)[4:7]
SignH = str(file_name)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
# Geospatial data for the tile
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
if resolution == '15s':
print('no 15s data is in dataset')
if resolution == '3s':
# clip data
Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (nameFile)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd,
data=Data,
geo=Geo_data,
projection="WGS84")
if resolution == '3s':
#size_X_end = int(size_X_tot) #!
#size_Y_end = int(size_Y_tot) #!
size_X_end = int(size_X_tot / len(rangeLat)) + 1 #!
size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 #!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
if resolution == '15s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash,
output_file_merged, latlim,
lonlim, resolution)
if resolution == '30s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash,
output_file_merged, latlim,
lonlim, resolution)
# name of the end result
output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution)
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name,
data=datasetTot,
geo=geo_out,
projection="WGS84")
os.chdir(output_folder)
# Delete the temporary folder
shutil.rmtree(output_folder_trash)
def RetrieveData(Date, args):
"""
This function retrieves MOD11 LST data for a given date from the
https://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim,
TimeStep, hdf_library
] = args
if TimeStep == 8:
LSTfileNamePart = os.path.join(
output_folder, 'LST_MOD11A2_K_8-daily_' + Date.strftime('%Y') +
'.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if TimeStep == 1:
LSTfileNamePart = os.path.join(
output_folder, 'LST_MOD11A1_K_daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.*.tif')
filesMOD = glob.glob(LSTfileNamePart)
if not len(filesMOD) == 1:
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
TimeStep, hdf_library)
except:
print("Was not able to download the file")
try:
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
# Save results as Gtiff
if TimeStep == 8:
LSTfileName = os.path.join(
output_folder,
'LST_MOD11A2_K_8-daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '.tif')
if TimeStep == 1:
name_collect_time = os.path.join(output_folder,
'Merged_Time.tif')
name_reprojected_time = RC.reproject_MODIS(
name_collect_time, epsg_to)
data_time, geo = RC.clip_data(name_reprojected_time, latlim,
lonlim)
data_time[data_time == 25.5] = np.nan
data_time_ave = np.nanmean(data_time)
try:
hour_GMT = int(np.floor(data_time_ave))
minutes_GMT = int(
(data_time_ave - np.floor(data_time_ave)) * 60)
except:
hour_GMT = int(12)
minutes_GMT = int(0)
LSTfileName = os.path.join(
output_folder,
'LST_MOD11A1_K_daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') +
'.%02d%02d.tif' % (hour_GMT, minutes_GMT))
os.remove(name_collect_time)
os.remove(name_reprojected_time)
data[data == 0.] = -9999
DC.Save_as_tiff(name=LSTfileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(os.path.join(output_folder, name_collect))
os.remove(os.path.join(output_folder, name_reprojected))
except:
print("Failed for date: %s" % Date)
return True
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, version, Product):
"""
This scripts downloads SSEBop ET data from the UNESCO-IHE ftp server.
The output files display the total ET in mm for a period of one month.
The name of the file corresponds to the first day of the month.
Keyword arguments:
Dir -- 'C:/file/to/path/'
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
lonlim -- [ymin, ymax] (values must be between -90 and 90)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
if version == "FTP":
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -59.2 or latlim[1] > 80:
print('Latitude above 80N or below -59.2S is not possible. Value set to maximum')
latlim[0] = np.max(latlim[0], -59.2)
latlim[1] = np.min(latlim[1], 80)
if lonlim[0] < -180 or lonlim[1] > 180:
print('Longitude must be between 180E and 180W. Now value is set to maximum')
lonlim[0] = np.max(lonlim[0],-180)
lonlim[1] = np.min(lonlim[1],180)
# Check Startdate and Enddate
if not Startdate:
Startdate = pd.Timestamp('2003-01-01')
if not Enddate:
Enddate = pd.Timestamp('2014-10-31')
if version == "V4":
# Check the latitude and longitude and otherwise set lat or lon on greatest extent
if latlim[0] < -60 or latlim[1] > 80.0022588483988670:
print('Latitude above 80N or below -59.2S is not possible. Value set to maximum')
latlim[0] = np.max(latlim[0], -60)
latlim[1] = np.min(latlim[1], 80.0022588483988670)
if lonlim[0] < -180 or lonlim[1] > 180.0002930387853439:
print('Longitude must be between 180E and 180W. Now value is set to maximum')
lonlim[0] = np.max(lonlim[0],-180)
lonlim[1] = np.min(lonlim[1],180.0002930387853439)
# Check Startdate and Enddate
if not Startdate:
Startdate = pd.Timestamp('2003-01-01')
if not Enddate:
import datetime
Enddate = pd.Timestamp(datetime.datetime.now())
# Define directory and create it if not exists
if Product == "ETact":
output_folder = os.path.join(Dir, 'Evaporation', 'SSEBop', 'Monthly')
freq_use = "MS"
if Product == "ETpot":
output_folder = os.path.join(Dir, 'Potential_Evapotranspiration', 'FEWS', 'Daily')
freq_use = "D"
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# Creates dates library
Dates = pd.date_range(Startdate, Enddate, freq = freq_use)
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
# Loop over the dates
for Date in Dates:
# Define year and month
year = Date.year
month = Date.month
day = Date.day
if version == "FTP":
# Date as printed in filename
Filename_out= os.path.join(output_folder,'ETa_SSEBop_FTP_mm-month-1_monthly_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
# Define end filename
Filename_dir = os.path.join("%s" %year, "m%s%02d.tif" %(str(year)[2:], month))
Filename_only = "m%s%02d.tif" %(str(year)[2:], month)
if version == "V4":
# Date as printed in filename
if Product == "ETpot":
Filename_out= os.path.join(output_folder,'ETpot_FEWS_mm-day-1_daily_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
# Define the downloaded zip file
Filename_only_zip = 'et%02s%02d%02d.tar.gz' %(str(year)[2:], month, day)
# The end file name after downloading and unzipping
Filename_only = "et%02s%02d%02d.bil" %(str(year)[2:], month, day)
# Create bin folder
temp_folder = os.path.join(output_folder, "Temp")
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
local_filename = os.path.join(temp_folder, Filename_only)
if Product == "ETact":
Filename_out= os.path.join(output_folder,'ETa_SSEBop_V4_mm-month-1_monthly_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
# Define the downloaded zip file
Filename_only_zip = "m%s%02d.zip" %(str(year), month)
# The end file name after downloading and unzipping
Filename_only = "m%s%02d_modisSSEBopETv4_actual_mm.tif" %(str(year), month)
# Temporary filename for the downloaded global file
local_filename = os.path.join(output_folder, Filename_only)
# Download the data from FTP server if the file not exists
if not os.path.exists(Filename_out):
try:
if version == "FTP":
Download_SSEBop_from_WA_FTP(local_filename, Filename_dir)
if version == "V4":
if Product == "ETpot":
Download_SSEBop_from_Web(temp_folder, Filename_only_zip, Product)
if Product == "ETact":
Download_SSEBop_from_Web(output_folder, Filename_only_zip, Product)
if Product == "ETpot":
Array_ETpot = RC.Open_bil_array(local_filename)
Array_ETpot = Array_ETpot/100
Geo_out = tuple([-180.5, 1, 0, 90.5, 0, -1])
dest = DC.Save_as_MEM(Array_ETpot, Geo_out, "WGS84")
data, Geo_out = RC.clip_data(dest, latlim, lonlim)
DC.Save_as_tiff(Filename_out, data, Geo_out, "WGS84")
if Product == "ETact":
# Clip dataset
data, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
data[data<-9999] = -9999
DC.Save_as_tiff(Filename_out, data, Geo_out, "WGS84")
os.remove(local_filename)
except:
print("Was not able to download file with date %s" %Date)
# Adjust waitbar
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
if version == "V4":
import glob
os.chdir(output_folder)
if Product == "ETact":
zipfiles = glob.glob("*.zip")
for zipfile in zipfiles:
os.remove(os.path.join(output_folder, zipfile))
xmlfiles = glob.glob("*.xml")
for xmlfile in xmlfiles:
os.remove(os.path.join(output_folder, xmlfile))
if Product == "ETpot":
import shutil
Temp_dir = os.path.join(output_folder, "Temp")
shutil.rmtree(Temp_dir)
return
