def Download_ETmonitor_from_WA_FTP(local_filename, Filename_in, Type):
"""
This function retrieves ETmonitor data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ETmonitor developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ETmonitor data
Filename_in -- name of the end file with the weekly ETmonitor data
Type = Type of data ("act" or "pot")
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if Type == "act":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Global/"
if Type == "pot":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Potential_Evapotranspiration/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_in, lf.write)
lf.close()
return
def Download_CMRSET_from_WA_FTP(local_filename, Filename_in):
"""
This function retrieves CMRSET data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the CMRSET developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global CMRSET data
Filename_in -- name of the end file with the monthly CMRSET data
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting/Data_Satellite/Evaporation/CMRSET/Global/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + Filename_in, lf.write)
lf.close()
return
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
"""
# Define date
year_data = Date.year
month_data = Date.month
day_data = Date.day
# filename of ASCAT data on server
ASCAT_date = "%d%02d%02d0000" % (year_data, month_data, day_data)
ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.0' % ASCAT_date
ASCAT_filename = "g2_BIOPAR_SWI_%s_GLOBE_ASCAT_V3.0.1.zip" % ASCAT_date
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='VITO')
URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
year_data, month_data, day_data, ASCAT_name, ASCAT_filename)
# Output zipfile
output_zipfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)
# Download the ASCAT data
try:
y = requests.get(URL, auth=HTTPBasicAuth(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(URL, auth=(username, password), verify=False)
# Write the file in system
z = open(output_zipfile_ASCAT, 'wb')
z.write(y.content)
z.close()
# Extract the zipfile
DC.Extract_Data(output_zipfile_ASCAT, output_folder_temp)
# Open the file
f = h5py.File(output_zipfile_ASCAT.replace('.zip', '.h5'))
# Open global ASCAT data
dataset = np.array((f['SWI']['SWI_010']).value)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]].astype("float") * 0.5
data[data > 100] = -9999
return (data)
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID, TimeStep):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
if TimeStep is "weekly":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
if TimeStep is "daily":
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
if TimeStep is "weekly":
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
if TimeStep is "daily":
DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])
raw_data = np.fromfile(os.path.splitext(local_filename)[0],
dtype="<f4")
dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return
def Collect_data(FTPprefix, Years, output_folder, Waitbar):
'''
This function downloads all the needed GLEAM files from hydras.ugent.be as a nc file.
Keywords arguments:
FTPprefix -- FTP path to the GLEAM data
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# account of the SFTP server (only password is missing)
server = 'hydras.ugent.be'
portnumber = 2225
username, password = WebAccounts.Accounts(Type='GLEAM')
# Create Waitbar
print '\nDownload GLEAM data'
if Waitbar == 1:
import wa.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount2 = len(Years)
amount2 = 0
WaitbarConsole.printWaitBar(amount2,
total_amount2,
prefix='Progress:',
suffix='Complete',
length=50)
for year in Years:
directory = os.path.join(FTPprefix, '%d' % year)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server,
port=portnumber,
username=username,
password=password)
ftp = ssh.open_sftp()
ftp.chdir(directory)
filename = 'E_' + str(year) + '_GLEAM_v3.1b.nc'
local_filename = os.path.join(output_folder, filename)
if not os.path.exists(local_filename):
ftp.get(filename, local_filename)
if Waitbar == 1:
amount2 += 1
WaitbarConsole.printWaitBar(amount2,
total_amount2,
prefix='Progress:',
suffix='Complete',
length=50)
ftp.close()
ssh.close()
return ()
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
latlim, yID, xID):
"""
This function retrieves ALEXI data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.
Keyword arguments:
local_filename -- name of the temporary file which contains global ALEXI data
DirFile -- name of the end file with the weekly ALEXI data
filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Open global ALEXI data
dataset = RC.Open_tiff_array(local_filename)
# Clip extend out of world data
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
data[data < 0] = -9999
# make geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print "file not exists"
return
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
"""
This function retrieves GWF data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Keyword arguments:
output_folder -- name of the end file with the weekly ALEXI data
End_filename -- name of the end file
lonlim -- [ymin, ymax] (values must be between -60 and 70)
latlim -- [xmin, xmax] (values must be between -180 and 180)
"""
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type='FTP_WA')
ftpserver = "ftp.wateraccounting.unesco-ihe.org"
# Set the file names and directories
filename = "Gray_Water_Footprint.tif"
local_filename = os.path.join(output_folder, filename)
# Download data from FTP
ftp = FTP(ftpserver)
ftp.login(username, password)
directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
ftp.cwd(directory)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write)
lf.close()
# Clip extend out of world data
dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)
# make geotiff file
DC.Save_as_tiff(name=filename_Out,
data=dataset,
geo=Geo_out,
projection="WGS84")
# delete old tif file
os.remove(local_filename)
except:
print "file not exists"
return
def Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles):
"""
This function retrieves ETensV1.0 data for a given date from the
ftp.wateraccounting.unesco-ihe.org server.
Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team.
Keyword arguments:
output_folder -- Directory of the outputs
Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number
"""
for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1):
for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1):
Tilename = "h%sv%s.zip" %(h_tile, v_tile)
if not os.path.exists(os.path.join(output_folder,Tilename)):
try:
# Collect account and FTP information
username, password = WebAccounts.Accounts(Type = 'FTP_WA')
FTP_name = "ftp://ftp.wateraccounting.unesco-ihe.org/WaterAccounting/Data_Satellite/Evaporation/ETensV1.0/%s" % Tilename
local_filename = os.path.join(output_folder, Tilename)
# Download data from FTP
curl = pycurl.Curl()
curl.setopt(pycurl.URL, FTP_name)
curl.setopt(pycurl.USERPWD, '%s:%s' %(username, password))
fp = open(local_filename, "wb")
curl.setopt(pycurl.WRITEDATA, fp)
curl.perform()
curl.close()
fp.close()
except:
print "tile %s is not found and will be replaced by NaN values" % Tilename
return()
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
hdf_library):
'''
This function downloads all the needed MODIS tiles from https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
sizeX = int((TilesHorizontal[1] - TilesHorizontal[0] + 1) * 2400)
sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 2400)
DataTot = np.zeros((sizeY, sizeX))
# Load accounts
username, password = WebAccounts.Accounts(Type='NASA')
# Download the MODIS FPAR data
url = 'https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'
dataset = requests.get(url, allow_redirects=False, stream=True)
try:
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
stream=True).content
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
verify=False).content
soup = BeautifulSoup(get_dataset, "lxml")
if len(str(soup)) < 300:
print 'Download was not succesfull, please check NASA account'
sys.exit(1)
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
Distance = 231.65635826395834 * 2 # resolution of a MODIS pixel in meter
countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
TilesVertical[0])
for Horizontal in range(int(TilesHorizontal[0]),
int(TilesHorizontal[1]) + 1):
countX = Horizontal - TilesHorizontal[0] + 1
for i in soup.findAll('a',
attrs={'href': re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
Vfile = str(i)[30:32]
Hfile = str(i)[27:29]
if int(Vfile) is int(Vertical) and int(Hfile) is int(
Horizontal):
# Define the whole url name
full_url = urlparse.urljoin(url, i['href'])
# Reset the begin parameters for downloading
downloaded = 0
N = 0
# if not downloaded try to download file
while downloaded == 0:
try: # open http and download whole .hdf
nameDownload_url = full_url
file_name = os.path.join(
output_folder,
nameDownload_url.split('/')[-1])
if os.path.isfile(file_name):
downloaded = 1
else:
x = requests.get(nameDownload_url,
allow_redirects=False)
try:
y = requests.get(x.headers['location'],
auth=(username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
y = requests.get(x.headers['location'],
auth=(username, password),
verify=False)
z = open(file_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 1000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print 'Data from ' + Date.strftime(
'%Y-%m-%d') + ' is not available'
downloaded = 1
try:
# Open .hdf only band with SnowFrac and collect all tiles to one array
scale_factor = 1
dataset = gdal.Open(file_name)
sdsdict = dataset.GetMetadata('SUBDATASETS')
sdslist = [
sdsdict[k] for k in sdsdict.keys()
if '_1_NAME' in k
]
sds = []
for n in sdslist:
sds.append(gdal.Open(n))
full_layer = [
i for i in sdslist if 'MOD_Grid_Snow_500m' in i
]
idx = sdslist.index(full_layer[0])
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
geo_t = sds[idx].GetGeoTransform()
# get the projection value
proj = sds[idx].GetProjection()
data = sds[idx].ReadAsArray()
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[int((countYdata - 1) *
2400):int(countYdata * 2400),
int((countX - 1) *
2400):int(countX *
2400)] = data * scale_factor
del data
# if the tile not exists or cannot be opened, create a nan array with the right projection
except:
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
x1 = (TilesHorizontal[0] - 19) * 2400 * Distance
x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data = np.ones((2400, 2400)) * (-9999)
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[(countYdata - 1) * 2400:countYdata * 2400,
(countX - 1) * 2400:countX *
2400] = data * 0.01
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 2400 * Distance
x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot)
dst_ds = None
sds = None
return ()
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores, TimeCase):
"""
This function downloads MSWEP Version 2.1 daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
Waitbar -- 0 or 1 (1 is waitbar on)
cores -- 1....8
"""
# Load factors / unit / type of variables / accounts
username, password = WebAccounts.Accounts(Type = 'MSWEP')
# Set required data for the daily option
if TimeCase == 'daily':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'daily')
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-01'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
url = 'https://%s:%[email protected]/opendap/MSWEP_V2.1/global_daily_010deg/' %(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'monthly')
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '1979-01-01'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
url = 'https://%s:%[email protected]:443/opendap/MSWEP_V2.1/global_monthly_010deg.nc' %(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
# Define IDs (latitude/longitude)
yID = np.int16(np.array([np.ceil((latlim[0] + 90) * 10),
np.floor((latlim[1] + 90) * 10)]))
xID = np.int16(np.array([np.floor((lonlim[0] + 180) * 10),
np.ceil((lonlim[1] + 180) * 10)]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import wa.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
# Create one parameter with all the required arguments
args = [path, url, TimeCase, xID, yID, lonlim, latlim, username, password]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
hdf_library):
'''
This function downloads all the needed MODIS tiles from http://e4ftl01.cr.usgs.gov/MOLT/MOD17A3H.006/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
# NPP_SIZE = 2
NPP_SIZE = 4
sizeX = int(
(TilesHorizontal[1] - TilesHorizontal[0] + 1) * 4800 / NPP_SIZE)
sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 4800 / NPP_SIZE)
DataTot = np.zeros((sizeY, sizeX))
# Load accounts
username, password = WebAccounts.Accounts(Type='NASA')
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
Distance = 231.65635826395834 * NPP_SIZE # resolution of a MODIS pixel in meter
countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
TilesVertical[0])
for Horizontal in range(int(TilesHorizontal[0]),
int(TilesHorizontal[1]) + 1):
countX = Horizontal - TilesHorizontal[0] + 1
# Download the MODIS NPP data
#url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD17A3H.006/' + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD17A3.055/' + Date.strftime(
'%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
'%d') + '/'
# Reset the begin parameters for downloading
downloaded = 0
N = 0
# Check the library given by user
if hdf_library is not None:
os.chdir(hdf_library)
hdf_name = glob.glob("MOD17A3.A%s%03s.h%02dv%02d.*" %
(Date.strftime('%Y'), Date.strftime('%j'),
Horizontal, Vertical))
if len(hdf_name) == 1:
hdf_file = os.path.join(hdf_library, hdf_name[0])
if os.path.exists(hdf_file):
downloaded = 1
file_name = hdf_file
if not downloaded == 1:
# Get files on FTP server
f = urllib2.urlopen(url)
# Sum all the files on the server
soup = BeautifulSoup(f, "lxml")
for i in soup.findAll('a',
attrs={'href':
re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
#Vfile=str(i)[31:33]
# Hfile=str(i)[28:30]
Vfile = str(i)[30:32]
Hfile = str(i)[27:29]
if int(Vfile) is int(Vertical) and int(Hfile) is int(
Horizontal):
# Define the whole url name
full_url = urlparse.urljoin(url, i['href'])
# if not downloaded try to download file
while downloaded == 0:
try: # open http and download whole .hdf
nameDownload = full_url
file_name = os.path.join(
output_folder,
nameDownload.split('/')[-1])
if os.path.isfile(file_name):
print "file ", file_name, " already exists"
downloaded = 1
else:
x = requests.get(nameDownload,
allow_redirects=False)
try:
y = requests.get(x.headers['location'],
auth=(username,
password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
y = requests.get(x.headers['location'],
auth=(username,
password),
verify=False)
z = open(file_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 10000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print 'Data from ' + Date.strftime(
'%Y-%m-%d') + ' is not available'
downloaded = 1
try:
# Open .hdf only band with NPP and collect all tiles to one array
dataset = gdal.Open(file_name)
sdsdict = dataset.GetMetadata('SUBDATASETS')
#sdslist = [sdsdict[k] for k in sdsdict.keys() if '_2_NAME' in k]
sdslist = [
sdsdict[k] for k in sdsdict.keys() if '_2_NAME' in k
]
sds = []
for n in sdslist:
sds.append(gdal.Open(n))
#yfull_layer = [i for i in sdslist if 'Npp_500m' in i]
full_layer = [i for i in sdslist if 'Npp_1km' in i]
idx = sdslist.index(full_layer[0])
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
geo_t = sds[idx].GetGeoTransform()
# get the projection value
proj = sds[idx].GetProjection()
data = sds[idx].ReadAsArray()
countYdata = (TilesVertical[1] - TilesVertical[0] +
2) - countY
DataTot[int((countYdata - 1) * 4800 /
NPP_SIZE):int(countYdata * 4800 / NPP_SIZE),
int((countX - 1) * 4800 /
NPP_SIZE):int(countX * 4800 /
NPP_SIZE)] = data * 0.0001
del data
# if the tile not exists or cannot be opened, create a nan array with the right projection
except:
if Horizontal == TilesHorizontal[
0] and Vertical == TilesVertical[0]:
x1 = (TilesHorizontal[0] - 19) * 4800 / NPP_SIZE * Distance
x4 = (TilesVertical[0] -
9) * 4800 / NPP_SIZE * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data = np.ones((4800 / NPP_SIZE, 4800 / NPP_SIZE)) * (0)
countYdata = (TilesVertical[1] - TilesVertical[0] + 2) - countY
DataTot[(countYdata - 1) * 4800 / NPP_SIZE:countYdata * 4800 /
NPP_SIZE, (countX - 1) * 4800 / NPP_SIZE:countX *
4800 / NPP_SIZE] = data * 0.0001
# Make geotiff file
DataTot[DataTot > 3.27] = -9999
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 4800 / NPP_SIZE * Distance
x4 = (TilesVertical[0] - 9) * 4800 / NPP_SIZE * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot)
dst_ds = None
sds = None
return ()
def RetrieveData(Date, args):
"""
This function retrieves TRMM data for a given date from the
ftp://disc2.nascom.nasa.gov server.
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
year = Date.year
month = Date.month
day = Date.day
from wa import WebAccounts
username, password = WebAccounts.Accounts(Type='NASA')
# Create https
if TimeCase == 'daily':
URL = 'https://disc2.gesdisc.eosdis.nasa.gov/opendap/TRMM_L3/TRMM_3B42_Daily.7/%d/%02d/3B42_Daily.%d%02d%02d.7.nc4.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
year, month, year, month, day, xID[0], xID[1] - 1, yID[0],
yID[1] - 1)
DirFile = os.path.join(
output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %
(year, month, day))
Scaling = 1
if TimeCase == 'monthly':
if Date >= pd.Timestamp('2010-10-01'):
URL = 'https://disc2.gesdisc.eosdis.nasa.gov/opendap/TRMM_L3/TRMM_3B43.7/%d/3B43.%d%02d01.7.HDF.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
year, year, month, xID[0], xID[1] - 1, yID[0], yID[1] - 1)
else:
URL = 'https://disc2.gesdisc.eosdis.nasa.gov/opendap/TRMM_L3/TRMM_3B43.7/%d/3B43.%d%02d01.7A.HDF.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
year, year, month, xID[0], xID[1] - 1, yID[0], yID[1] - 1)
Scaling = calendar.monthrange(year, month)[1] * 24
DirFile = os.path.join(
output_folder,
"P_TRMM3B43.V7_mm-month-1_monthly_%d.%02d.01.tif" % (year, month))
if not os.path.isfile(DirFile):
dataset = requests.get(URL, allow_redirects=False, stream=True)
try:
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
stream=True)
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
get_dataset = requests.get(dataset.headers['location'],
auth=(username, password),
verify=False)
# download data (first save as text file)
pathtext = os.path.join(output_folder, 'temp.txt')
z = open(pathtext, 'w')
z.write(get_dataset.content)
z.close()
# Open text file and remove header and footer
data_start = np.genfromtxt(pathtext,
dtype=float,
skip_header=1,
delimiter=',')
data = data_start[:, 1:] * Scaling
data[data < 0] = -9999
data = data.transpose()
data = np.flipud(data)
# Delete .txt file
os.remove(pathtext)
# Make geotiff file
geo = [lonlim[0], 0.25, 0, latlim[1], 0, -0.25]
DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
return True
def DownloadData(Dir,
Var,
Startdate,
Enddate,
latlim,
lonlim,
Waitbar,
cores,
TimeCase,
CaseParameters,
gldas_version='2.1'):
"""
This function downloads GLDAS Version 2 three-hourly, daily or monthly data
Keyword arguments:
Dir -- 'C:/file/to/path/'
Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
Startdate -- 'yyyy-mm-dd'
Enddate -- 'yyyy-mm-dd'
latlim -- [ymin, ymax]
lonlim -- [xmin, xmax]
cores -- 1....8
CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py
"""
# Load factors / unit / type of variables / accounts
VarInfo = VariablesInfo(TimeCase)
username, password = WebAccounts.Accounts(Type='NASA')
# Set required data for the three hourly option
if TimeCase == 'three_hourly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase,
Var)
if not os.path.exists(path):
os.makedirs(path)
# Startdate if not defined
sd_date = '2000-02-24'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_NOAH025_3H.{0}'.format(
gldas_version) #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_three_hourly
# Set required data for the daily option
elif TimeCase == 'daily':
# seperate the daily case parameters
SumMean, Min, Max = CaseParameters
# Define output folder and create this one if not exists
path = {
'mean':
os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase, Var,
'mean'),
'min':
os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase, Var,
'min'),
'max':
os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase, Var,
'max')
}
selected = np.array([SumMean, Min, Max])
types = np.array(('mean', 'min', 'max'))[selected == 1]
CaseParameters = [selected, types]
for i in range(len(types)):
if not os.path.exists(path[types[i]]):
os.makedirs(path[types[i]])
# Startdate if not defined
sd_date = '2000-02-24'
# Define Time frequency
TimeFreq = 'D'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_NOAH025_3H.{0}'.format(
gldas_version) #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_daily
# Set required data for the monthly option
elif TimeCase == 'monthly':
# Define output folder and create this one if not exists
path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase,
Var)
if not os.path.exists(path):
os.makedirs(path)
CaseParameters = []
# Startdate if not defined
sd_date = '2000-03-01'
# Define Time frequency
TimeFreq = 'MS'
# Define URL by using personal account
#url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025_M' %(username,password)
url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_NOAH025_M.{0}'.format(
gldas_version) #%(username,password)
# Name the definition that will be used to obtain the data
RetrieveData_fcn = RetrieveData_monthly
# If none of the possible option are chosen
else:
raise KeyError("The input time interval is not supported")
# Define IDs (latitude/longitude)
yID = np.int16(
np.array(
[np.ceil((latlim[0] + 60) * 4),
np.floor((latlim[1] + 60) * 4)]))
xID = np.int16(
np.array(
[np.floor((lonlim[0] + 180) * 4),
np.ceil((lonlim[1] + 180) * 4)]))
# Check dates. If no dates are given, the max number of days is used.
if not Startdate:
Startdate = pd.Timestamp(sd_date)
if not Enddate:
Enddate = pd.Timestamp('Now') # Should be much than available
# Create all dates that will be calculated
Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)
# Create Waitbar
if Waitbar == 1:
import wa.Functions.Start.WaitbarConsole as WaitbarConsole
total_amount = len(Dates)
amount = 0
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
# Define the variable string name
VarStr = VarInfo.names[Var]
# Create one parameter with all the required arguments
args = [
path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim,
CaseParameters, username, password
]
# Pass variables to parallel function and run
if not cores:
for Date in Dates:
RetrieveData_fcn(Date, args)
if Waitbar == 1:
amount += 1
WaitbarConsole.printWaitBar(amount,
total_amount,
prefix='Progress:',
suffix='Complete',
length=50)
results = True
else:
results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
for Date in Dates)
return results