def Download_SSEBop_from_WA_FTP(local_filename, Filename_dir):
"""
This function retrieves SSEBop data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the SSEBop developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global SSEBop data
Filename_dir -- name of the end file with the monthly SSEBop data
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting/Data_Satellite/Evaporation/SSEBop/sourcefiles/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_dir, lf.write)
lf.close()
return
def Download_HiHydroSoil_from_WA_FTP(local_filename, Filename_in):
"""
This function retrieves HiHydroSoil data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the HiHydroSoil developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global HiHydroSoil data
Filename_in -- name of the end file with the HiHydroSoil data
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_in, lf.write)
lf.close()
return
def Download_ETmonitor_from_WA_FTP(local_filename, Filename_in, Type):
"""
This function retrieves ETmonitor data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ETmonitor developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ETmonitor data
Filename_in -- name of the end file with the weekly ETmonitor data
Type = Type of data ("act" or "pot")
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if Type == "pot":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Potential_Evapotranspiration/"
else:
directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Global/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_in, lf.write)
lf.close()
return
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID, TimeStep):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if TimeStep is "weekly":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
if TimeStep is "daily":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
if TimeStep is "weekly":
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
if TimeStep is "daily":
DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])
raw_data = np.fromfile(os.path.splitext(local_filename)[0],
dtype="<f4")
dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
data = dataset[
yID[0]:yID[1],
xID[0]:xID[1]] / 2.45 # Values are in MJ/m2d so convert to mm/d
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
"""
# Define date
year_data = Date.year
month_data = Date.month
day_data = Date.day
# filename of ASCAT data on server
ASCAT_date = "%d%02d%02d0000" % (year_data, month_data, day_data)
ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.0' % ASCAT_date
ASCAT_filename = "g2_BIOPAR_SWI_%s_GLOBE_ASCAT_V3.0.1.zip" % ASCAT_date
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='VITO')
URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
year_data, month_data, day_data, ASCAT_name, ASCAT_filename)
# Output zipfile
output_zipfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)
# Download the ASCAT data
try:
y = requests.get(URL, auth=HTTPBasicAuth(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(URL, auth=(username, password), verify=False)
# Write the file in system
z = open(output_zipfile_ASCAT, 'wb')
z.write(y.content)
z.close()
# Extract the zipfile
DC.Extract_Data(output_zipfile_ASCAT, output_folder_temp)
# Open the file
f = h5py.File(output_zipfile_ASCAT.replace('.zip', '.h5'))
# Open global ASCAT data
dataset = np.array((f['SWI']['SWI_010']).value)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]].astype("float") * 0.5
data[data > 100] = -9999
return (data)
def Collect_data(FTPprefix, Years, output_folder, Waitbar, Product):
'''
This function downloads all the needed GLEAM files from hydras.ugent.be as a nc file.
Keywords arguments:
FTPprefix -- FTP path to the GLEAM data
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# account of the SFTP server (only password is missing)
server = 'hydras.ugent.be'
portnumber = 2225
username, password = WebAccounts.Accounts(Type='GLEAM')
# Create Waitbar
print('\nDownload GLEAM data')
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount2 = len(Years)
amount2 = 0
WaitbarConsole.printWaitBar(amount2,
total_amount2,
prefix='Progress:',
suffix='Complete',
length=50)
for year in Years:
directory = os.path.join(FTPprefix, '%d' % year)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server,
port=portnumber,
username=username,
password=password)
ftp = ssh.open_sftp()
ftp.chdir(directory)
if Product == "ET":
filename = 'E_' + str(year) + '_GLEAM_v3.2b.nc'
if Product == "ETpot":
filename = 'Ep_' + str(year) + '_GLEAM_v3.2b.nc'
local_filename = os.path.join(output_folder, filename)
if not os.path.exists(local_filename):
ftp.get(filename, local_filename)
if Waitbar == 1:
amount2 += 1
WaitbarConsole.printWaitBar(amount2,
total_amount2,
prefix='Progress:',
suffix='Complete',
length=50)
ftp.close()
ssh.close()
return ()
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
"""
# Define date
year_data = Date.year
month_data = Date.month
day_data = Date.day
# filename of ASCAT data on server
ASCAT_date = "%d%02d%02d1200" % (year_data, month_data, day_data)
ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.1.1' % ASCAT_date
ASCAT_filename = "c_gls_SWI_%s_GLOBE_ASCAT_V3.1.1.nc" % ASCAT_date
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='Copernicus')
URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
year_data, month_data, day_data, ASCAT_name, ASCAT_filename)
# Output zipfile
output_ncfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)
# Download the ASCAT data
try:
y = requests.get(URL, auth=HTTPBasicAuth(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(URL, auth=(username, password), verify=False)
# Write the file in system
z = open(output_ncfile_ASCAT, 'wb')
z.write(y.content)
z.close()
# Open nc file
fh = Dataset(output_ncfile_ASCAT)
dataset = fh.variables['SWI_010'][:, yID[0]:yID[1], xID[0]:xID[1]]
data = np.squeeze(dataset.data, axis=0)
data = data * 0.5
data[data > 100.] = -9999
fh.close()
return (data)
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
"""
This function retrieves GWF data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Keyword arguments:
output_folder -- name of the end file with the weekly ALEXI data
End_filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Set the file names and directories
filename = "Gray_Water_Footprint.tif"
local_filename = os.path.join(output_folder, filename)
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Clip extend out of world data
dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
# make geotiff file
DC.Save_as_tiff(name=filename_Out,
data=dataset,
geo=Geo_out,
projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print("file not exists")
return
def Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles):
"""
This function retrieves ETensV1.0 data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team.
Keyword arguments:
output_folder -- Directory of the outputs
Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number
"""
for v_tile in range(Lat_tiles[0], Lat_tiles[1] + 1):
for h_tile in range(Lon_tiles[0], Lon_tiles[1] + 1):
Tilename = "h%sv%s.zip" % (h_tile, v_tile)
if not os.path.exists(os.path.join(output_folder, Tilename)):
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
FTP_name = "ftp://ftp.wateraccounting.unesco-ihe.org//WaterAccounting_Guest/ETensV1.0/%s" % Tilename
local_filename = os.path.join(output_folder, Tilename)
# Download data from FTP
curl = pycurl.Curl()
curl.setopt(pycurl.URL, FTP_name)
curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password))
fp = open(local_filename, "wb")
curl.setopt(pycurl.WRITEDATA, fp)
curl.perform()
curl.close()
fp.close()
except:
print(
"tile %s is not found and will be replaced by NaN values"
% Tilename)
return ()
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, TimeStep,
hdf_library):
'''
This function downloads all the needed MODIS tiles from http://e4ftl01.cr.usgs.gov/MOLT/MOD13Q1.006/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
sizeX = int((TilesHorizontal[1] - TilesHorizontal[0] + 1) * 1200)
sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 1200)
DataTot = np.zeros((sizeY, sizeX))
# Load accounts
username, password = WebAccounts.Accounts(Type='NASA')
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
Distance = 4 * 231.65635826395834 # resolution of a MODIS pixel in meter
countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
TilesVertical[0])
for Horizontal in range(int(TilesHorizontal[0]),
int(TilesHorizontal[1]) + 1):
countX = Horizontal - TilesHorizontal[0] + 1
# Download the MODIS NDVI data
if TimeStep == 8:
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD11A2.006/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
'%d') + '/'
if TimeStep == 1:
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD11A1.006/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
'%d') + '/'
# Reset the begin parameters for downloading
downloaded = 0
N = 0
# Check the library given by user
if hdf_library is not None:
os.chdir(hdf_library)
if TimeStep == 8:
hdf_name = glob.glob(
"MOD11A2.A%s%03s.h%02dv%02d.*" %
(Date.strftime('%Y'), Date.strftime('%j'), Horizontal,
Vertical))
if TimeStep == 1:
hdf_name = glob.glob(
"MOD11A1.A%s%03s.h%02dv%02d.*" %
(Date.strftime('%Y'), Date.strftime('%j'), Horizontal,
Vertical))
if len(hdf_name) == 1:
hdf_file = os.path.join(hdf_library, hdf_name[0])
if os.path.exists(hdf_file):
downloaded = 1
file_name = hdf_file
if not downloaded == 1:
# Get files on FTP server
if sys.version_info[0] == 3:
f = urllib.request.urlopen(url)
if sys.version_info[0] == 2:
f = urllib.request.urlopen(url)
# Sum all the files on the server
soup = BeautifulSoup(f, "lxml")
for i in soup.findAll('a',
attrs={'href':
re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
Vfile = str(i)[30:32]
Hfile = str(i)[27:29]
if int(Vfile) is int(Vertical) and int(Hfile) is int(
Horizontal):
# Define the whole url name
if sys.version_info[0] == 3:
full_url = urllib.parse.urljoin(url, i['href'])
if sys.version_info[0] == 2:
full_url = urllib.parse.urljoin(url, i['href'])
# if not downloaded try to download file
while downloaded == 0:
try: # open http and download whole .hdf
nameDownload = full_url
file_name = os.path.join(
output_folder,
nameDownload.split('/')[-1])
if os.path.isfile(file_name):
print(("file ", file_name,
" already exists"))
downloaded = 1
else:
x = requests.get(nameDownload,
allow_redirects=False)
try:
y = requests.get(x.headers['location'],
auth=(username,
password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
y = requests.get(x.headers['location'],
auth=(username,
password),
verify=False)
z = open(file_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 10000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' + Date.strftime('%Y-%m-%d') +
' is not available')
downloaded = 1
try:
# Open .hdf only band with NDVI and collect all tiles to one array
dataset = gdal.Open(file_name)
sdsdict = dataset.GetMetadata('SUBDATASETS')
sdslist = [
sdsdict[k] for k in list(sdsdict.keys()) if '_1_NAME' in k
]
sds = []
for n in sdslist:
sds.append(gdal.Open(n))
full_layer = [i for i in sdslist if 'LST_Day_1km' in i]
idx = sdslist.index(full_layer[0])
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
geo_t = sds[idx].GetGeoTransform()
# get the projection value
proj = sds[idx].GetProjection()
data = sds[idx].ReadAsArray()
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[int((countYdata - 1) * 1200):int(countYdata *
1200),
int((countX - 1) * 1200):int(countX *
1200)] = data * 0.02
del data
# if the tile not exists or cannot be opened, create a nan array with the right projection
except:
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
x1 = (TilesHorizontal[0] - 19) * 1200 * Distance
x4 = (TilesVertical[0] - 9) * 1200 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data = np.ones((1200, 1200)) * (-9999 / 0.02)
countYdata = (TilesVertical[1] - TilesVertical[0] + 2) - countY
DataTot[(countYdata - 1) * 1200:countYdata * 1200,
(countX - 1) * 1200:countX * 4800] = data * 0.02
DataTot[DataTot < 1] = -9999
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 1200 * Distance
x4 = (TilesVertical[0] - 9) * 1200 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot)
dst_ds = None
sds = None
return ()
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
TimeCase):
"""
This function downloads MSWEP Version 2.1 daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
Waitbar -- 0 or 1 (1 is waitbar on)
cores -- 1....8
"""
# Load factors / unit / type of variables / accounts
username, password = WebAccounts.Accounts(Type='MSWEP')
# Set required data for the daily option
if TimeCase == 'daily':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'daily')
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-01'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
url = 'https://%s:%[email protected]/opendap/MSWEP_V2.1/global_daily_010deg/' % (
username, password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'monthly')
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-01'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
url = 'https://%s:%[email protected]:443/opendap/MSWEP_V2.1/global_monthly_010deg.nc' % (
username, password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
# Define IDs (latitude/longitude)
yID = np.int16(
np.array(
[np.ceil((latlim[0] + 90) * 10),
np.floor((latlim[1] + 90) * 10)]))
xID = np.int16(
np.array([
np.floor((lonlim[0] + 180) * 10),
np.ceil((lonlim[1] + 180) * 10)
]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Create one parameter with all the required arguments
args = [path, url, TimeCase, xID, yID, lonlim, latlim, username, password]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results
def DownloadData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar, CaseParameters, cores, TimeCase):
"""
This function downloads GLDAS CLSM daily data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
cores -- 1....8
CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py
"""
# Load factors / unit / type of variables / accounts
VarInfo = VariablesInfo(TimeCase)
username, password = WebAccounts.Accounts(Type = 'NASA')
# Set required data for the daily option
# Set required data for the three hourly option
if TimeCase == 'three_hourly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS',
TimeCase, Var)
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-02'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLM10SUBP_3H' #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_three_hourly
types = ['mean']
elif TimeCase == 'daily':
types = ['mean']
# Define output folder and create this one if not exists
path = {'mean': os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS_CLSM',
TimeCase, Var, 'mean')}
for i in range(len(types)):
if not os.path.exists(path[types[i]]):
os.makedirs(path[types[i]])
# Startdate if not defined
sd_date = '1948-01-01'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLSM025_D.2.0'
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
types = ['mean']
# Define output folder and create this one if not exists
path = {'mean': os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS_CLSM',
TimeCase, Var, 'mean')}
for i in range(len(types)):
if not os.path.exists(path[types[i]]):
os.makedirs(path[types[i]])
# Startdate if not defined
sd_date = '1979-01-02'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLSM025_D.2.0'
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
if TimeCase == 'three_hourly':
# Define IDs (latitude/longitude)
yID = np.int16(np.array([np.ceil((latlim[0] + 60)),
np.floor((latlim[1] + 60))]))
xID = np.int16(np.array([np.floor((lonlim[0] + 180)),
np.ceil((lonlim[1] + 180))]))
else:
# Define IDs (latitude/longitude)
yID = np.int16(np.array([np.ceil((latlim[0] + 60) * 4),
np.floor((latlim[1] + 60) * 4)]))
xID = np.int16(np.array([np.floor((lonlim[0] + 180) * 4),
np.ceil((lonlim[1] + 180) * 4)]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import watools.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
# Define the variable string name
VarStr = VarInfo.names[Var]
# Create one parameter with all the required arguments
args = [path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim, CaseParameters, username, password, types]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results
def RetrieveData(Date, args):
"""
This function retrieves TRMM data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
year = Date.year
month = Date.month
day = Date.day
from watools import WebAccounts
username, password = WebAccounts.Accounts(Type='NASA')
# Create https
if TimeCase == 'daily':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/GPM_L3/GPM_3IMERGDF.05/%d/%02d/3B-DAY.MS.MRG.3IMERG.%d%02d%02d-S000000-E235959.V05.nc4.ascii?precipitationCal[%d:1:%d][%d:1:%d]' % (
year, month, year, month, day, xID[0], xID[1] - 1, yID[0],
yID[1] - 1)
DirFile = os.path.join(
output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %
(year, month, day))
Scaling = 1
if TimeCase == 'monthly':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/hyrax/GPM_L3/GPM_3IMERGM.05/%d/3B-MO.MS.MRG.3IMERG.%d%02d01-S000000-E235959.%02d.V05B.HDF5.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
year, year, month, month, xID[0], xID[1] - 1, yID[0], yID[1] - 1)
Scaling = calendar.monthrange(year, month)[1] * 24
DirFile = os.path.join(
output_folder,
"P_GPM.IMERG_mm-month-1_monthly_%d.%02d.01.tif" % (year, month))
if not os.path.isfile(DirFile):
dataset = requests.get(URL, allow_redirects=False, stream=True)
try:
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
stream=True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
verify=False)
# download data (first save as text file)
pathtext = os.path.join(output_folder, 'temp.txt')
z = open(pathtext, 'wb')
z.write(get_dataset.content)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,
dtype=float,
skip_header=1,
delimiter=',')
data = data_start[:, 1:] * Scaling
data[data < 0] = -9999
data = data.transpose()
data = np.flipud(data)
# Delete .txt file
os.remove(pathtext)
# Make geotiff file
geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return True
def Collect_data(TilesHorizontal,TilesVertical,Date,output_folder,timestep, hdf_library, Size_pix):
'''
This function downloads all the needed MODIS tiles from ftp.ntsg.umt.edu/pub/MODIS/NTSG_Products/MOD16/MOD16A2_MONTHLY.MERRA_GMAO_1kmALB/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
sizeX=int(TilesHorizontal[1]-TilesHorizontal[0]+1)*1200 * Size_pix
sizeY=int(TilesVertical[1]-TilesVertical[0]+1)*1200 * Size_pix
DataTot=np.ones((sizeY,sizeX))* -9999
# Make a new tile for the lat and long info
LatMet=np.zeros((sizeY))
LongMet=np.zeros((sizeX))
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]),int(TilesVertical[1])+1):
Distance=926.625 / Size_pix # resolution of a MODIS pixel in meter
countY=int((TilesVertical[1]-TilesVertical[0]+1)-(Vertical-TilesVertical[0]))
LatMet[int((countY-1)*1200 * Size_pix):int((countY)*1200 * Size_pix)]=np.linspace(((8-Vertical)*1200 * Size_pix+0.5)*Distance,((8-Vertical)*1200 * Size_pix + 1200 * Size_pix - 0.5)*Distance,1200 * Size_pix)
for Horizontal in range(int(TilesHorizontal[0]),int(TilesHorizontal[1])+1):
countX=int(Horizontal-TilesHorizontal[0]+1)
LongMet[int((countX-1)*1200 * Size_pix):int((countX)*1200 * Size_pix)]=np.linspace(((Horizontal-18)*1200 * Size_pix+0.5)*Distance,((Horizontal-18)*1200+1200 * Size_pix - 0.5)*Distance,1200* Size_pix)
# Set the download to zero again
downloaded = 0
if hdf_library is not None:
os.chdir(hdf_library)
hdf_name = glob.glob("MOD16A2.A%s%03s.h%02dv%02d.*" %(Date.strftime('%Y'), Date.strftime('%j'), Horizontal, Vertical))
if len(hdf_name) == 1:
hdf_file = os.path.join(hdf_library, hdf_name[0])
if os.path.exists(hdf_file):
downloaded = 1
data = Open_mod16_data(hdf_file)
countYdata=(TilesVertical[1]-TilesVertical[0]+2)-countY
DataTot[(int(countYdata)-1)*1200 * Size_pix:int(countYdata)*1200 * Size_pix,(int(countX)-1)*1200* Size_pix:int(countX)*1200 * Size_pix]=data*0.1
while downloaded == 0:
# Download the MODIS FPAR data
if timestep == 'monthly':
url = 'http://files.ntsg.umt.edu/data/NTSG_Products/MOD16/MOD16A2_MONTHLY.MERRA_GMAO_1kmALB/Y%s/M%02s/' %(Date.strftime('%Y'), Date.strftime('%m'))
if timestep == '8-daily':
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD16A2.006/%d.%02d.%02d/' %(Date.year, Date.month, Date.day)
# Get files on FTP server
if sys.version_info[0] == 3:
f = urllib.request.urlopen(url)
if sys.version_info[0] == 2:
f = urllib2.urlopen(url)
# Sum all the files on the server
soup = BeautifulSoup(f, "lxml")
try:
for i in soup.findAll('a', attrs = {'href': re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
nameHDF=str(i)
HDF_name = nameHDF.split('>')[-2][:-3]
Hfile=HDF_name[18:20]
Vfile=HDF_name[21:23]
if int(Vfile) is int(Vertical) and int(Hfile) is int(Horizontal):
HTTP_name = url + HDF_name
output_name = os.path.join(output_folder, HDF_name)
if timestep == 'monthly':
if not os.path.isfile(output_name):
while downloaded == 0:
if sys.version_info[0] == 2:
urllib.urlretrieve(HTTP_name,output_name)
if sys.version_info[0] == 3:
urllib.request.urlretrieve(HTTP_name,output_name)
statinfo = os.stat(output_name)
# Say that download was succesfull
if int(statinfo.st_size) > 1000:
downloaded = 1
if timestep == '8-daily':
# Reset the begin parameters for downloading
N=0
username, password = WebAccounts.Accounts(Type = 'NASA')
# if not downloaded try to download file
while downloaded == 0:
try:# open http and download whole .hdf
file_name = os.path.join(output_folder,HTTP_name.split('/')[-1])
if os.path.isfile(output_name):
downloaded = 1
else:
x = requests.get(HTTP_name, allow_redirects = False)
try:
y = requests.get(x.headers['location'], auth = (username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(x.headers['location'], auth = (username, password), verify = False)
z = open(output_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 1000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' + Date.strftime('%Y-%m-%d') + ' is not available')
downloaded = 1
# Open .hdf only band with ET and collect all tiles to one array
data = Open_mod16_data(output_name)
countYdata=(TilesVertical[1]-TilesVertical[0]+2)-countY
DataTot[(int(countYdata)-1)*1200 * Size_pix:int(countYdata)*1200 * Size_pix,(int(countX)-1)*1200* Size_pix:int(countX)*1200* Size_pix]=data*0.1
DataTot[DataTot>3000]=-9999
downloaded = 1
del data
except:
proj='PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data=np.ones((1200* Size_pix, 1200* Size_pix)) * (-9999)
countYdata=(TilesVertical[1] - TilesVertical[0] + 2) - countY
DataTot[(countYdata - 1) * 1200* Size_pix:countYdata * 1200* Size_pix,(countX - 1) * 1200* Size_pix:countX * 1200* Size_pix] = data * 0.1
downloaded = 1
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1, gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj='PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 1200 * Size_pix * Distance
x4 = (TilesVertical[0] - 9) * 1200 * Size_pix * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot)
dst_ds = None
return(DataTot,LatMet,LongMet)
def Download_PROBAV_from_VITO(End_filename, output_folder_temp, Date, latlim, lonlim, type_PROBAV, type_Band):
"""
This function retrieves PROBAV data for a given date from the
https://www.vito-eodata.be/PDF/datapool/Free_Data server.
Restrictions:
Sign up is needed. The password and username has to be stored within the
WebAccounts.py
Keyword arguments
"""
# Define date
year_data = Date.year
month_data = Date.month
day_data = Date.day
# Define Tiles
X_tile_max = int(np.floor((lonlim[1] + 180) / 10.))
X_tile_min = int(np.floor((lonlim[0] + 180) / 10.))
Y_tile_min = int(14 - np.ceil((latlim[1] + 65) / 10.))
Y_tile_max = int(14 - np.ceil((latlim[0] + 65) / 10.))
X_tiles = list(range(X_tile_min, X_tile_max+1))
Y_tiles = list(range(Y_tile_min, Y_tile_max+1))
output_files_PROBAV = []
# Band numbers PROBA-V
Band_numbers = {'SM':7,'B1':8,'B2':10,'B3':9,'B4':11}
# Get passwords
username, password = WebAccounts.Accounts(Type='VITO')
# Download all the needed tiles
for X_tile in X_tiles:
for Y_tile in Y_tiles:
# filename of ASCAT data on server
PROBAV_date = "%d%02d%02d" % (year_data, month_data, day_data)
URL_name = "https://www.vito-eodata.be/PDF/datapool/Free_Data/PROBA-V_100m/%s_%s_100_m_C1/%s/%s/%s/" %(type_PROBAV.split('_')[-1], type_PROBAV.split('_')[0], year_data, month_data, day_data)
# Get version name
f = requests.get(URL_name,auth = (username, password))
x = f.content
soup = BeautifulSoup(x, "lxml")
i = str(soup.findAll('a', attrs = {'href': re.compile('(?i)(V10\d)')})[0])
match = re.search(r'_V10\w+', i)
PROBAV_Version = match.group()
# Define some output names
PROBAV_filename = "PROBAV_%s_%s_X%02dY%02d_%s_100M%s.HDF5" %(type_PROBAV.split('_')[-1], type_PROBAV.split('_')[0], X_tile, Y_tile, PROBAV_date, PROBAV_Version)
PROBAV_name = 'PV_%s_%s-%s_100M%s' %(type_PROBAV.split('_')[-1], type_PROBAV.split('_')[0], PROBAV_date, PROBAV_Version)
PROBAV_filename_tiff = os.path.join(output_folder_temp, "PROBAV_%s_%s_X%02dY%02d_%s_100M%s_%s.tif" %(type_PROBAV.split('_')[-1], type_PROBAV.split('_')[0], X_tile, Y_tile, PROBAV_date, PROBAV_Version, type_Band))
# Output
output_file_PROBAV = os.path.join(output_folder_temp, PROBAV_filename)
if not os.path.exists(PROBAV_filename_tiff):
if not os.path.exists(output_file_PROBAV):
URL = "https://www.vito-eodata.be/PDF/datapool/Free_Data/PROBA-V_100m/%s_%s_100_m_C1/%s/%s/%s/%s/%s" % (type_PROBAV.split('_')[-1], type_PROBAV.split('_')[0], year_data, month_data, day_data,
PROBAV_name, PROBAV_filename)
# Download the ASCAT data
try:
y = requests.get(URL, auth=HTTPBasicAuth(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(URL, auth=(username, password), verify=False)
# Write the file in system
try:
z = open(output_file_PROBAV, 'wb')
z.write(y.content)
z.close()
except:
print("%s is not available" %output_file_PROBAV)
# Convert hdf5 into tiff
try:
# output PROBA-V tiff
Band_number = Band_numbers[type_Band]
if not type_Band == "SM":
scaling_factor = 0.005
else:
scaling_factor = 1.0
# Define the x and y spacing
g = gdal.Open(output_file_PROBAV, gdal.GA_ReadOnly)
Meta_data = g.GetMetadata()
Lat_Top = float(Meta_data['LEVEL3_GEOMETRY_TOP_RIGHT_LATITUDE'])
Lon_Left = float(Meta_data['LEVEL3_GEOMETRY_BOTTOM_LEFT_LONGITUDE'])
Pixel_size = float((Meta_data['LEVEL3_RADIOMETRY_BLUE_TOC_MAPPING']).split(' ')[-3])
# Define the georeference of the HDF5 file
geo_out = [Lon_Left-0.5*Pixel_size, Pixel_size, 0, Lat_Top+0.5*Pixel_size, 0, -Pixel_size]
# Convert hdf5 to tiff
DC.Convert_hdf5_to_tiff(output_file_PROBAV, PROBAV_filename_tiff, Band_number, scaling_factor, geo_out)
# Define all the good outputs
output_files_PROBAV = np.append(output_files_PROBAV, PROBAV_filename_tiff)
except:
print("%s is not created, and replaced by fake dataset" %PROBAV_filename_tiff)
# Create empty dataset with nan values
Empty_Data = np.ones([10080, 10080]) * np.nan
Size_Pixel_PROBAV = 0.000992063492063
x_min = X_tile * 10 - 0.5 * Size_Pixel_PROBAV - 180
y_max = Y_tile * -10 + 0.5 * Size_Pixel_PROBAV + 75
Geo_out_fake = tuple([x_min, Size_Pixel_PROBAV, 0, y_max, 0, -Size_Pixel_PROBAV])
# Save the fake layer
DC.Save_as_tiff(PROBAV_filename_tiff, Empty_Data, Geo_out_fake, "WGS84")
else:
# Define all the good outputs
output_files_PROBAV = np.append(output_files_PROBAV, PROBAV_filename_tiff)
return(output_files_PROBAV)