def Download_ETmonitor_from_WA_FTP(local_filename, Filename_in, Type):
"""
This function retrieves ETmonitor data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ETmonitor developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ETmonitor data
Filename_in -- name of the end file with the weekly ETmonitor data
Type = Type of data ("act" or "pot")
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if Type == "pot":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Potential_Evapotranspiration/"
else:
directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Global/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_in, lf.write)
lf.close()
return
def Download_SEBS_from_WA_FTP(local_filename, Filename_in):
"""
This function retrieves SEBS data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the SEBS developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global SEBS data
Filename_in -- name of the end file with the monthly SEBS data
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type = 'FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp=FTP(ftpserver)
ftp.login(username,password)
directory="/WaterAccounting_Guest/SEBS/Global_land_ET_V1/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_in, lf.write)
lf.close()
return
def RetrieveData(Date, args):
"""
This function retrieves TRMM data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
year = Date.year
month= Date.month
day = Date.day
from watertools import WebAccounts
username, password = WebAccounts.Accounts(Type = 'NASA')
# Create https
if TimeCase == 'daily':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/GPM_L3/GPM_3IMERGDF.05/%d/%02d/3B-DAY.MS.MRG.3IMERG.%d%02d%02d-S000000-E235959.V05.nc4.ascii?precipitationCal[%d:1:%d][%d:1:%d]' %(year, month, year, month, day, xID[0], xID[1]-1, yID[0], yID[1]-1)
DirFile = os.path.join(output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %(year, month, day))
Scaling = 1
if TimeCase == 'monthly':
URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/hyrax/GPM_L3/GPM_3IMERGM.05/%d/3B-MO.MS.MRG.3IMERG.%d%02d01-S000000-E235959.%02d.V05B.HDF5.ascii?precipitation[%d:1:%d][%d:1:%d]' %(year, year, month, month, xID[0], xID[1]-1, yID[0], yID[1]-1)
Scaling = calendar.monthrange(year,month)[1] * 24
DirFile = os.path.join(output_folder, "P_GPM.IMERG_mm-month-1_monthly_%d.%02d.01.tif" %(year, month))
if not os.path.isfile(DirFile):
dataset = requests.get(URL, allow_redirects=False,stream = True)
try:
get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'], auth = (username, password), verify = False)
# download data (first save as text file)
pathtext = os.path.join(output_folder,'temp.txt')
z = open(pathtext,'wb')
z.write(get_dataset.content)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,dtype = float,skip_header = 1,delimiter=',')
data = data_start[:,1:] * Scaling
data[data < 0] = -9999
data = data.transpose()
data = np.flipud(data)
# Delete .txt file
os.remove(pathtext)
# Make geotiff file
geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return True
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID, TimeStep):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if TimeStep is "weekly":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
if TimeStep is "daily":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
if TimeStep is "weekly":
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
if TimeStep is "daily":
DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])
raw_data = np.fromfile(os.path.splitext(local_filename)[0],
dtype="<f4")
dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
data = dataset[
yID[0]:yID[1],
xID[0]:xID[1]] / 2.45 # Values are in MJ/m2d so convert to mm/d
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return
def Collect_data(FTPprefix, Years, output_folder, Waitbar, Product):
'''
This function downloads all the needed GLEAM files from hydras.ugent.be as a nc file.
Keywords arguments:
FTPprefix -- FTP path to the GLEAM data
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# account of the SFTP server (only password is missing)
server = 'hydras.ugent.be'
portnumber = 2225
username, password = WebAccounts.Accounts(Type='GLEAM')
# Create Waitbar
print('\nDownload GLEAM data')
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount2 = len(Years)
amount2 = 0
WaitbarConsole.printWaitBar(amount2,
total_amount2,
prefix='Progress:',
suffix='Complete',
length=50)
for year in Years:
directory = os.path.join(FTPprefix, '%d' % year)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server,
port=portnumber,
username=username,
password=password)
ftp = ssh.open_sftp()
ftp.chdir(directory)
if Product == "ET":
filename = 'E_' + str(year) + '_GLEAM_v3.3b.nc'
if Product == "ETpot":
filename = 'Ep_' + str(year) + '_GLEAM_v3.3b.nc'
local_filename = os.path.join(output_folder, filename)
if not os.path.exists(local_filename):
ftp.get(filename, local_filename)
if Waitbar == 1:
amount2 += 1
WaitbarConsole.printWaitBar(amount2,
total_amount2,
prefix='Progress:',
suffix='Complete',
length=50)
ftp.close()
ssh.close()
return ()
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
"""
# Define date
year_data = Date.year
month_data = Date.month
day_data = Date.day
# filename of ASCAT data on server
ASCAT_date = "%d%02d%02d1200" % (year_data, month_data, day_data)
ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.1.1' % ASCAT_date
ASCAT_filename = "c_gls_SWI_%s_GLOBE_ASCAT_V3.1.1.nc" % ASCAT_date
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='Copernicus')
URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
year_data, month_data, day_data, ASCAT_name, ASCAT_filename)
# Output zipfile
output_ncfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)
# Download the ASCAT data
try:
y = requests.get(URL, auth=HTTPBasicAuth(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(URL, auth=(username, password), verify=False)
# Write the file in system
z = open(output_ncfile_ASCAT, 'wb')
z.write(y.content)
z.close()
# Open nc file
fh = Dataset(output_ncfile_ASCAT)
dataset = fh.variables['SWI_010'][:, yID[0]:yID[1], xID[0]:xID[1]]
data = np.squeeze(dataset.data, axis=0)
data = data * 0.5
data[data > 100.] = -9999
fh.close()
return (data)
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
"""
This function retrieves GWF data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Keyword arguments:
output_folder -- name of the end file with the weekly ALEXI data
End_filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Set the file names and directories
filename = "Gray_Water_Footprint.tif"
local_filename = os.path.join(output_folder, filename)
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Clip extend out of world data
dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
# make geotiff file
DC.Save_as_tiff(name=filename_Out,
data=dataset,
geo=Geo_out,
projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print("file not exists")
return
def Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles):
"""
This function retrieves ETensV1.0 data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team.
Keyword arguments:
output_folder -- Directory of the outputs
Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number
"""
for v_tile in range(Lat_tiles[0], Lat_tiles[1] + 1):
for h_tile in range(Lon_tiles[0], Lon_tiles[1] + 1):
Tilename = "h%sv%s.zip" % (h_tile, v_tile)
if not os.path.exists(os.path.join(output_folder, Tilename)):
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
FTP_name = "ftp://ftp.wateraccounting.unesco-ihe.org//WaterAccounting_Guest/ETensV1.0/%s" % Tilename
local_filename = os.path.join(output_folder, Tilename)
# Download data from FTP
curl = pycurl.Curl()
curl.setopt(pycurl.URL, FTP_name)
curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password))
fp = open(local_filename, "wb")
curl.setopt(pycurl.WRITEDATA, fp)
curl.perform()
curl.close()
fp.close()
except:
print(
"tile %s is not found and will be replaced by NaN values"
% Tilename)
return ()
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, band,
resolution, hdf_library):
'''
This function downloads all the needed MODIS tiles from http://e4ftl01.cr.usgs.gov/MOLT/MOD13Q1.006/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
if band >= 3 and resolution == "250m":
sys.exit('Bands higher than 3, are only available in 500m resolution')
if resolution == "250m":
size_factor = 1
else:
size_factor = 2
# Make a new tile for the data
sizeX = int(
(TilesHorizontal[1] - TilesHorizontal[0] + 1) * 4800 / size_factor)
sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 4800 / size_factor)
DataTot = np.zeros((sizeY, sizeX))
# Load accounts
username, password = WebAccounts.Accounts(Type='NASA')
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
Distance = 231.65635826395834 * size_factor # resolution of a MODIS pixel in meter
countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
TilesVertical[0])
for Horizontal in range(int(TilesHorizontal[0]),
int(TilesHorizontal[1]) + 1):
countX = Horizontal - TilesHorizontal[0] + 1
# Download the MODIS NDVI data
if resolution == "250m":
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD09GQ.006/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
'%d') + '/'
else:
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD09GA.006/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
'%d') + '/'
# Reset the begin parameters for downloading
downloaded = 0
N = 0
# Check the library given by user
if hdf_library is not None:
os.chdir(hdf_library)
hdf_name = glob.glob("MOD09GA.A%s%03s.h%02dv%02d.*" %
(Date.strftime('%Y'), Date.strftime('%j'),
Horizontal, Vertical))
if len(hdf_name) == 1:
hdf_file = os.path.join(hdf_library, hdf_name[0])
if os.path.exists(hdf_file):
downloaded = 1
file_name = hdf_file
if not downloaded == 1:
try:
# Get files on FTP server
if sys.version_info[0] == 3:
f = urllib.request.urlopen(url)
if sys.version_info[0] == 2:
f = urllib2.urlopen(url)
# Sum all the files on the server
soup = BeautifulSoup(f, "lxml")
for i in soup.findAll(
'a', attrs={'href': re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
Vfile = str(i)[30:32]
Hfile = str(i)[27:29]
if int(Vfile) is int(Vertical) and int(Hfile) is int(
Horizontal):
# Define the whole url name
if sys.version_info[0] == 3:
full_url = urllib.parse.urljoin(url, i['href'])
if sys.version_info[0] == 2:
full_url = urlparse.urljoin(url, i['href'])
# if not downloaded try to download file
while downloaded == 0:
try: # open http and download whole .hdf
nameDownload = full_url
file_name = os.path.join(
output_folder,
nameDownload.split('/')[-1])
if os.path.isfile(file_name):
downloaded = 1
else:
x = requests.get(nameDownload,
allow_redirects=False)
try:
y = requests.get(
x.headers['location'],
auth=(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
y = requests.get(
x.headers['location'],
auth=(username, password),
verify=False)
z = open(file_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 10000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' +
Date.strftime('%Y-%m-%d') +
' is not available')
downloaded = 1
except:
print("Url not found: %s" % url)
try:
# Open .hdf only band with NDVI and collect all tiles to one array
dataset = gdal.Open(file_name)
sdsdict = dataset.GetMetadata('SUBDATASETS')
if resolution == "250m":
sdslist = [
sdsdict[k] for k in sdsdict.keys()
if 'SUBDATASET_%d_NAME' % int(band + 1) in k
]
else:
sdslist = [
sdsdict[k] for k in sdsdict.keys()
if 'SUBDATASET_%d_NAME' % int(band + 11) in k
]
sds = []
for n in sdslist:
sds.append(gdal.Open(n))
full_layer = [
i for i in sdslist if 'sur_refl_b%02d_1' % band in i
]
idx = sdslist.index(full_layer[0])
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
geo_t = sds[idx].GetGeoTransform()
# get the projection value
proj = sds[idx].GetProjection()
data = sds[idx].ReadAsArray()
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[int((countYdata - 1) * 4800 /
size_factor):int(countYdata * 4800 /
size_factor),
int((countX - 1) * 4800 /
size_factor):int(countX * 4800 /
size_factor)] = data
del data
# if the tile not exists or cannot be opened, create a nan array with the right projection
except:
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
x1 = (TilesHorizontal[0] -
19) * 4800 / size_factor * Distance
x4 = (TilesVertical[0] -
9) * 4800 / size_factor * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data = np.ones((int(4800 / size_factor), int(
4800 / size_factor))) * (-9999)
countYdata = (TilesVertical[1] - TilesVertical[0] + 2) - countY
DataTot[int((countYdata - 1) * 4800 /
size_factor):int(countYdata * 4800 / size_factor),
int((countX - 1) * 4800 /
size_factor):int(countX * 4800 /
size_factor)] = data
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 4800 / size_factor * Distance
x4 = (TilesVertical[0] - 9) * 4800 / size_factor * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot * 0.0001)
dst_ds = None
sds = None
return ()
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
TimeCase):
"""
This function downloads MSWEP Version 2.1 daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
Waitbar -- 0 or 1 (1 is waitbar on)
cores -- 1....8
"""
# Load factors / unit / type of variables / accounts
username, password = WebAccounts.Accounts(Type='MSWEP')
# Set required data for the daily option
if TimeCase == 'daily':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'daily')
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-01'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
url = 'https://%s:%[email protected]/opendap/MSWEP_V2.1/global_daily_010deg/' % (
username, password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'monthly')
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-01'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
url = 'https://%s:%[email protected]:443/opendap/MSWEP_V2.1/global_monthly_010deg.nc' % (
username, password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
# Define IDs (latitude/longitude)
yID = np.int16(
np.array(
[np.ceil((latlim[0] + 90) * 10),
np.floor((latlim[1] + 90) * 10)]))
xID = np.int16(
np.array([
np.floor((lonlim[0] + 180) * 10),
np.ceil((lonlim[1] + 180) * 10)
]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Create one parameter with all the required arguments
args = [path, url, TimeCase, xID, yID, lonlim, latlim, username, password]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results
def DownloadData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar,
CaseParameters, cores, TimeCase):
"""
This function downloads GLDAS CLSM daily data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
cores -- 1....8
CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py
"""
# Load factors / unit / type of variables / accounts
VarInfo = VariablesInfo(TimeCase)
username, password = WebAccounts.Accounts(Type='NASA')
# Set required data for the daily option
# Set required data for the three hourly option
if TimeCase == 'three_hourly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase,
Var)
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-02'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLM10SUBP_3H' #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_three_hourly
types = ['mean']
elif TimeCase == 'daily':
types = ['mean']
# Define output folder and create this one if not exists
path = {
'mean':
os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS_CLSM', TimeCase,
Var, 'mean')
}
for i in range(len(types)):
if not os.path.exists(path[types[i]]):
os.makedirs(path[types[i]])
# Startdate if not defined
sd_date = '1948-01-01'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLSM025_D.2.0'
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
types = ['mean']
# Define output folder and create this one if not exists
path = {
'mean':
os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS_CLSM', TimeCase,
Var, 'mean')
}
for i in range(len(types)):
if not os.path.exists(path[types[i]]):
os.makedirs(path[types[i]])
# Startdate if not defined
sd_date = '1979-01-02'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLSM025_D.2.0'
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
if TimeCase == 'three_hourly':
# Define IDs (latitude/longitude)
yID = np.int16(
np.array([np.ceil((latlim[0] + 60)),
np.floor((latlim[1] + 60))]))
xID = np.int16(
np.array([np.floor((lonlim[0] + 180)),
np.ceil((lonlim[1] + 180))]))
else:
# Define IDs (latitude/longitude)
yID = np.int16(
np.array([
np.ceil((latlim[0] + 60) * 4),
np.floor((latlim[1] + 60) * 4)
]))
xID = np.int16(
np.array([
np.floor((lonlim[0] + 180) * 4),
np.ceil((lonlim[1] + 180) * 4)
]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define the variable string name
VarStr = VarInfo.names[Var]
# Create one parameter with all the required arguments
args = [
path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim,
CaseParameters, username, password, types
]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results
def RetrieveData(Date, args):
"""
This function retrieves MYD11 LST data for a given date from the
https://e4ftl01.cr.usgs.gov/ server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# watertools modules
import watertools.General.raster_conversions as RC
import watertools.General.data_conversions as DC
from watertools import WebAccounts
# Argument
[
output_folder, TilesVertical, TilesHorizontal, lonlim, latlim,
hdf_library
] = args
# Define output names
LSTfileNamePart = os.path.join(
output_folder, 'LST_MYD11A1_K_daily_' + Date.strftime('%Y') + '.' +
Date.strftime('%m') + '.' + Date.strftime('%d') + '*.tif')
filesMOD = glob.glob(LSTfileNamePart)
# Load accounts
username, password = WebAccounts.Accounts(Type='NASA')
if not len(filesMOD) == 1:
# Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
try:
Collect_data(TilesHorizontal, TilesVertical, Date, username,
password, output_folder, hdf_library)
except:
print("Was not able to download the file")
try:
# Define the output name of the collect data function
name_collect = os.path.join(output_folder, 'Merged.tif')
name_collect_time = os.path.join(output_folder, 'Merged_Time.tif')
# Reproject the MODIS product to epsg_to
epsg_to = '4326'
name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
name_reprojected_time = RC.reproject_MODIS(name_collect_time,
epsg_to)
# Clip the data to the users extend
data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
data_time, geo = RC.clip_data(name_reprojected_time, latlim,
lonlim)
# remove wrong values
data[data == 0.] = -9999
data_time[data_time == 25.5] = np.nan
data_time_ave = np.nanmean(data_time)
try:
hour_GMT = int(np.floor(data_time_ave))
minutes_GMT = int(
(data_time_ave - np.floor(data_time_ave)) * 60)
except:
hour_GMT = int(12)
minutes_GMT = int(0)
LSTfileName = os.path.join(
output_folder, 'LST_MYD11A1_K_daily_' + Date.strftime('%Y') +
'.' + Date.strftime('%m') + '.' + Date.strftime('%d') +
'.%02d%02d.tif' % (hour_GMT, minutes_GMT))
# Save results as Gtiff
DC.Save_as_tiff(name=LSTfileName,
data=data,
geo=geo,
projection='WGS84')
# remove the side products
os.remove(name_collect)
os.remove(name_reprojected)
os.remove(name_collect_time)
os.remove(name_reprojected_time)
except:
print("Failed for date: %s" % Date)
return True
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, timestep,
hdf_library, Size_pix):
'''
This function downloads all the needed MODIS tiles from ftp.ntsg.umt.edu/pub/MODIS/NTSG_Products/MOD16/MOD16A2_MONTHLY.MERRA_GMAO_1kmALB/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
sizeX = int(TilesHorizontal[1] - TilesHorizontal[0] + 1) * 1200 * Size_pix
sizeY = int(TilesVertical[1] - TilesVertical[0] + 1) * 1200 * Size_pix
DataTot = np.ones((sizeY, sizeX)) * -9999
# Make a new tile for the lat and long info
LatMet = np.zeros((sizeY))
LongMet = np.zeros((sizeX))
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
Distance = 926.625 / Size_pix # resolution of a MODIS pixel in meter
countY = int((TilesVertical[1] - TilesVertical[0] + 1) -
(Vertical - TilesVertical[0]))
LatMet[int((countY - 1) * 1200 *
Size_pix):int((countY) * 1200 * Size_pix)] = np.linspace(
((8 - Vertical) * 1200 * Size_pix + 0.5) * Distance,
((8 - Vertical) * 1200 * Size_pix + 1200 * Size_pix -
0.5) * Distance, 1200 * Size_pix)
for Horizontal in range(int(TilesHorizontal[0]),
int(TilesHorizontal[1]) + 1):
countX = int(Horizontal - TilesHorizontal[0] + 1)
LongMet[int((countX - 1) * 1200 * Size_pix):int(
(countX) * 1200 * Size_pix)] = np.linspace(
((Horizontal - 18) * 1200 * Size_pix + 0.5) * Distance,
((Horizontal - 18) * 1200 + 1200 * Size_pix - 0.5) *
Distance, 1200 * Size_pix)
# Set the download to zero again
downloaded = 0
if hdf_library is not None:
os.chdir(hdf_library)
hdf_name = glob.glob("MOD16A2.A%s%03s.h%02dv%02d.*" %
(Date.strftime('%Y'), Date.strftime('%j'),
Horizontal, Vertical))
if len(hdf_name) == 1:
hdf_file = os.path.join(hdf_library, hdf_name[0])
if os.path.exists(hdf_file):
downloaded = 1
data = Open_mod16_data(hdf_file)
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[(int(countYdata) - 1) * 1200 *
Size_pix:int(countYdata) * 1200 * Size_pix,
(int(countX) - 1) * 1200 *
Size_pix:int(countX) * 1200 *
Size_pix] = data * 0.1
while downloaded == 0:
# Download the MODIS FPAR data
if timestep == 'monthly':
url = 'http://files.ntsg.umt.edu/data/NTSG_Products/MOD16/MOD16A2_MONTHLY.MERRA_GMAO_1kmALB/Y%s/M%02s/' % (
Date.strftime('%Y'), Date.strftime('%m'))
if timestep == '8-daily':
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD16A2.006/%d.%02d.%02d/' % (
Date.year, Date.month, Date.day)
try:
# Get files on FTP server
if sys.version_info[0] == 3:
f = urllib.request.urlopen(url)
if sys.version_info[0] == 2:
f = urllib2.urlopen(url)
# Sum all the files on the server
soup = BeautifulSoup(f, "lxml")
for i in soup.findAll(
'a', attrs={'href': re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
nameHDF = str(i)
HDF_name = nameHDF.split('>')[-2][:-3]
Hfile = HDF_name[18:20]
Vfile = HDF_name[21:23]
if int(Vfile) is int(Vertical) and int(Hfile) is int(
Horizontal):
HTTP_name = url + HDF_name
output_name = os.path.join(output_folder, HDF_name)
if timestep == 'monthly':
if not os.path.isfile(output_name):
while downloaded == 0:
if sys.version_info[0] == 2:
urllib.urlretrieve(
HTTP_name, output_name)
if sys.version_info[0] == 3:
urllib.request.urlretrieve(
HTTP_name, output_name)
statinfo = os.stat(output_name)
# Say that download was succesfull
if int(statinfo.st_size) > 1000:
downloaded = 1
if timestep == '8-daily':
# Reset the begin parameters for downloading
N = 0
username, password = WebAccounts.Accounts(
Type='NASA')
# if not downloaded try to download file
while downloaded == 0:
try: # open http and download whole .hdf
file_name = os.path.join(
output_folder,
HTTP_name.split('/')[-1])
if os.path.isfile(output_name):
downloaded = 1
else:
x = requests.get(
HTTP_name,
allow_redirects=False)
try:
y = requests.get(
x.headers['location'],
auth=(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
y = requests.get(
x.headers['location'],
auth=(username, password),
verify=False)
z = open(output_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 1000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' +
Date.strftime('%Y-%m-%d') +
' is not available')
downloaded = 1
# Open .hdf only band with ET and collect all tiles to one array
data = Open_mod16_data(output_name)
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[(int(countYdata) - 1) * 1200 *
Size_pix:int(countYdata) * 1200 * Size_pix,
(int(countX) - 1) * 1200 *
Size_pix:int(countX) * 1200 *
Size_pix] = data * 0.1
DataTot[DataTot > 3000] = -9999
downloaded = 1
del data
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data = np.ones(
(1200 * Size_pix, 1200 * Size_pix)) * (-9999)
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[(countYdata - 1) * 1200 * Size_pix:countYdata *
1200 * Size_pix, (countX - 1) * 1200 *
Size_pix:countX * 1200 * Size_pix] = data * 0.1
downloaded = 1
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 1200 * Size_pix * Distance
x4 = (TilesVertical[0] - 9) * 1200 * Size_pix * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot)
dst_ds = None
return (DataTot, LatMet, LongMet)
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
hdf_library):
'''
This function downloads all the needed MODIS tiles from https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
sizeX = int((TilesHorizontal[1] - TilesHorizontal[0] + 1) * 2400)
sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 2400)
DataTot = np.zeros((sizeY, sizeX))
# Load accounts
username, password = WebAccounts.Accounts(Type='NASA')
# Download the MODIS FPAR data
url = 'https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'
dataset = requests.get(url, allow_redirects=False, stream=True)
try:
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
stream=True).content
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
verify=False).content
soup = BeautifulSoup(get_dataset, "lxml")
if len(str(soup)) < 300:
print('Download was not succesfull, please check NASA account')
sys.exit(1)
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
Distance = 231.65635826395834 * 2 # resolution of a MODIS pixel in meter
countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
TilesVertical[0])
for Horizontal in range(int(TilesHorizontal[0]),
int(TilesHorizontal[1]) + 1):
countX = Horizontal - TilesHorizontal[0] + 1
for i in soup.findAll('a',
attrs={'href': re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
Vfile = str(i)[30:32]
Hfile = str(i)[27:29]
if int(Vfile) is int(Vertical) and int(Hfile) is int(
Horizontal):
# Define the whole url name
if sys.version_info[0] == 3:
full_url = urllib.parse.urljoin(url, i['href'])
if sys.version_info[0] == 2:
full_url = urlparse.urljoin(url, i['href'])
# Reset the begin parameters for downloading
downloaded = 0
N = 0
# if not downloaded try to download file
while downloaded == 0:
try: # open http and download whole .hdf
nameDownload_url = full_url
file_name = os.path.join(
output_folder,
nameDownload_url.split('/')[-1])
if os.path.isfile(file_name):
downloaded = 1
else:
x = requests.get(nameDownload_url,
allow_redirects=False)
try:
y = requests.get(x.headers['location'],
auth=(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
y = requests.get(x.headers['location'],
auth=(username, password),
verify=False)
z = open(file_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 1000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print('Data from ' + Date.strftime('%Y-%m-%d') +
' is not available')
downloaded = 1
try:
# Open .hdf only band with SnowFrac and collect all tiles to one array
scale_factor = 1
dataset = gdal.Open(file_name)
sdsdict = dataset.GetMetadata('SUBDATASETS')
sdslist = [
sdsdict[k] for k in sdsdict.keys()
if '_1_NAME' in k
]
sds = []
for n in sdslist:
sds.append(gdal.Open(n))
full_layer = [
i for i in sdslist if 'MOD_Grid_Snow_500m' in i
]
idx = sdslist.index(full_layer[0])
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
geo_t = sds[idx].GetGeoTransform()
# get the projection value
proj = sds[idx].GetProjection()
data = sds[idx].ReadAsArray()
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[int((countYdata - 1) *
2400):int(countYdata * 2400),
int((countX - 1) *
2400):int(countX *
2400)] = data * scale_factor
del data
# if the tile not exists or cannot be opened, create a nan array with the right projection
except:
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
x1 = (TilesHorizontal[0] - 19) * 2400 * Distance
x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data = np.ones((2400, 2400)) * (-9999)
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[(countYdata - 1) * 2400:countYdata * 2400,
(countX - 1) * 2400:countX *
2400] = data * 0.01
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 2400 * Distance
x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot)
dst_ds = None
sds = None
return ()