def Download_SSEBop_from_Web(output_folder, Filename_only_zip, Product):
"""
This function retrieves SSEBop data for a given date from the
https://edcintl.cr.usgs.gov server.
Keyword arguments:
local_filename -- name of the temporary file which contains global SSEBop data
Filename_dir -- name of the end directory to put in the extracted data
"""
if Product == "ETact":
# Create the total url to the webpage
total_URL = "https://edcintl.cr.usgs.gov/downloads/sciweb1/shared/fews/web/global/monthly/eta/downloads/" + str(
Filename_only_zip)
if Product == "ETpot":
total_URL = "https://edcintl.cr.usgs.gov/downloads/sciweb1/shared/fews/web/global/daily/pet/downloads/daily/" + str(
Filename_only_zip)
# Download the data
if sys.version_info[0] == 2:
urllib.urlretrieve(total_URL,
os.path.join(output_folder, Filename_only_zip))
if sys.version_info[0] == 3:
urllib.request.urlretrieve(
total_URL, os.path.join(output_folder, Filename_only_zip))
# unzip the file
if Product == "ETpot":
DC.Extract_Data_tar_gz(os.path.join(output_folder, Filename_only_zip),
output_folder)
if Product == "ETact":
DC.Extract_Data(os.path.join(output_folder, Filename_only_zip),
output_folder)
return
def Unzip_ETens_data(output_folder, Lat_tiles, Lon_tiles):
"""
This function extract the zip files
Keyword Arguments:
output_folder -- Directory of the outputs
Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number
"""
# Unzip the zip files one by one
for v_tile in range(Lat_tiles[0], Lat_tiles[1] + 1):
for h_tile in range(Lon_tiles[0], Lon_tiles[1] + 1):
# Define the file and path to the zip file
Tilename = "h%sv%s.zip" % (h_tile, v_tile)
input_zip_folder = os.path.join(output_folder, Tilename)
if os.path.exists(input_zip_folder):
try:
# Extract data
DC.Extract_Data(input_zip_folder, output_folder)
except:
print(
'Was not able to unzip %s, data will be replaced by NaN values'
% Tilename)
return ()
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
"""
# Define date
year_data = Date.year
month_data = Date.month
day_data = Date.day
# filename of ASCAT data on server
ASCAT_date = "%d%02d%02d0000" % (year_data, month_data, day_data)
ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.0' % ASCAT_date
ASCAT_filename = "g2_BIOPAR_SWI_%s_GLOBE_ASCAT_V3.0.1.zip" % ASCAT_date
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='VITO')
URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
year_data, month_data, day_data, ASCAT_name, ASCAT_filename)
# Output zipfile
output_zipfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)
# Download the ASCAT data
try:
y = requests.get(URL, auth=HTTPBasicAuth(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(URL, auth=(username, password), verify=False)
# Write the file in system
z = open(output_zipfile_ASCAT, 'wb')
z.write(y.content)
z.close()
# Extract the zipfile
DC.Extract_Data(output_zipfile_ASCAT, output_folder_temp)
# Open the file
f = h5py.File(output_zipfile_ASCAT.replace('.zip', '.h5'))
# Open global ASCAT data
dataset = np.array((f['SWI']['SWI_010']).value)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]].astype("float") * 0.5
data[data > 100] = -9999
return (data)
def DownloadData(output_folder, latlim, lonlim, parameter, resolution):
"""
This function downloads DEM data from HydroSHED
Keyword arguments:
output_folder -- directory of the result
latlim -- [ymin, ymax] (values must be between -50 and 50)
lonlim -- [xmin, xmax] (values must be between -180 and 180)
Resample -- 1 = The data will be resampled to 0.001 degree spatial
resolution
-- 0 = The data will have the same pixel size as the data obtained
from the internet
"""
# Define parameter depedent variables
if parameter == "dir_3s":
para_name = "DIR"
unit = "-"
resolution = '3s'
parameter = 'dir'
if parameter == "dem_3s":
para_name = "DEM"
unit = "m"
resolution = '3s'
parameter = 'dem'
if parameter == "dir_15s":
para_name = "DIR"
unit = "-"
resolution = '15s'
parameter = 'dir'
if parameter == "dem_15s":
para_name = "DEM"
unit = "m"
resolution = '15s'
parameter = 'dem'
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
if resolution == '3s':
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim,
parameter)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
if resolution == '15s':
name = Find_Document_names_15s(latlim, lonlim, parameter, resolution)
nameResults = []
# Create a temporary folder for processing
output_folder_trash = os.path.join(output_folder, "Temp")
if not os.path.exists(output_folder_trash):
os.makedirs(output_folder_trash)
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash,
parameter, para_name,
resolution)
# extract zip data
DC.Extract_Data(output_file, output_folder_trash)
# Converts the data with a adf extention to a tiff extension.
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif'
file_name_extract = file_name.split('_')[0:3]
if resolution == '3s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1]
if resolution == '15s':
file_name_extract2 = file_name_extract[
0] + '_' + file_name_extract[1] + '_15s'
input_adf = os.path.join(output_folder_trash, file_name_extract2,
file_name_extract2, 'hdr.adf')
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff)
geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff)
if int(size_X) != int(6000) or int(size_Y) != int(6000):
data = np.ones((6000, 6000)) * -9999
# Create the latitude bound
Vfile = str(nameFile)[1:3]
SignV = str(nameFile)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(nameFile)[4:7]
SignH = str(nameFile)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
Expected_X_min = Bound1
Expected_Y_max = Bound2 + 5
Xid_start = int(
np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(
np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) /
geo_out[1]))
Yid_start = int(
np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(
np.round((Expected_Y_max - (geo_out[3] +
(size_Y * geo_out[5]))) /
(-geo_out[5])))
data[Yid_start:Yid_end,
Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff)
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
except:
if resolution == '3s':
# If tile not exist create a replacing zero tile (sea tiles)
output = nameFile.split('.')[0] + "_trans_temporary.tif"
output_tiff = os.path.join(output_folder_trash, output)
file_name = nameFile
data = np.ones((6000, 6000)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = str(file_name)[1:3]
SignV = str(file_name)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(file_name)[4:7]
SignH = str(file_name)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
# Geospatial data for the tile
geo_in = [
Bound1, 0.00083333333333333, 0.0,
int(Bound2 + 5), 0.0, -0.0008333333333333333333
]
# save chunk as tiff file
DC.Save_as_tiff(name=output_tiff,
data=data,
geo=geo_in,
projection="WGS84")
if resolution == '15s':
print('no 15s data is in dataset')
if resolution == '3s':
# clip data
Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (nameFile)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
DC.Save_as_tiff(name=nameForEnd,
data=Data,
geo=Geo_data,
projection="WGS84")
if resolution == '3s':
#size_X_end = int(size_X_tot) #!
#size_Y_end = int(size_Y_tot) #!
size_X_end = int(size_X_tot / len(rangeLat)) + 1 #!
size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 #!
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
if resolution == '15s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s(output_folder_trash,
output_file_merged, latlim, lonlim)
# name of the end result
output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution)
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
DC.Save_as_tiff(name=Save_name,
data=datasetTot,
geo=geo_out,
projection="WGS84")
os.chdir(output_folder)
# Delete the temporary folder
shutil.rmtree(output_folder_trash)
def RetrieveData(Date, args):
"""
This function retrieves RFE data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, lonlim, latlim, xID, yID] = args
# Create https
DirFile = os.path.join(
output_folder, 'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
if not os.path.isfile(DirFile):
# open ftp server
ftp = FTP("ftp.cpc.ncep.noaa.gov", "", "")
ftp.login()
# Define FTP path to directory
pathFTP = 'fews/fewsdata/africa/rfe2/geotiff/'
# find the document name in this directory
ftp.cwd(pathFTP)
listing = []
# read all the file names in the directory
ftp.retrlines("LIST", listing.append)
# create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names
filename = 'africa_rfe.%s%02s%02s.tif.zip' % (
Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))
outfilename = os.path.join(
output_folder, 'africa_rfe.%s%02s%02s.tif' %
(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
try:
local_filename = os.path.join(output_folder, filename)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# unzip the file
zip_filename = os.path.join(output_folder, filename)
DC.Extract_Data(zip_filename, output_folder)
# open tiff file
dataset = RC.Open_tiff_array(outfilename)
# clip dataset to the given extent
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# save dataset as geotiff file
latlim_adj = 40.05 - 0.1 * yID[0]
lonlim_adj = -20.05 + 0.1 * xID[0]
geo = [lonlim_adj, 0.1, 0, latlim_adj, 0, -0.1]
DC.Save_as_tiff(name=DirFile,
data=data,
geo=geo,
projection="WGS84")
# delete old tif file
os.remove(outfilename)
os.remove(zip_filename)
except:
print("file not exists")
return True