def RetrieveData(Date, args):
"""
This function retrieves CHIRPS data for a given date from the
ftp://chg-ftpout.geog.ucsb.edu server.
https://data.chc.ucsb.edu/
Keyword arguments:
Date -- 'yyyy-mm-dd'
args -- A list of parameters defined in the DownloadData function.
"""
# Argument
[output_folder, TimeCase, xID, yID, lonlim, latlim] = args
# open ftp server
# ftp = FTP("chg-ftpout.geog.ucsb.edu", "", "")
ftp = FTP("data.chc.ucsb.edu")
ftp.login()
# Define FTP path to directory
if TimeCase == 'daily':
pathFTP = 'pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/' %Date.strftime('%Y')
elif TimeCase == 'monthly':
pathFTP = 'pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/'
else:
raise KeyError("The input time interval is not supported")
# find the document name in this directory
ftp.cwd(pathFTP)
listing = []
# read all the file names in the directory
ftp.retrlines("LIST", listing.append)
# create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names
if TimeCase == 'daily':
filename = 'chirps-v2.0.%s.%02s.%02s.tif.gz' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))
outfilename = os.path.join(output_folder,'chirps-v2.0.%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
DirFileEnd = os.path.join(output_folder,'P_CHIRPS.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
elif TimeCase == 'monthly':
filename = 'chirps-v2.0.%s.%02s.tif.gz' %(Date.strftime('%Y'), Date.strftime('%m'))
outfilename = os.path.join(output_folder,'chirps-v2.0.%s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m')))
DirFileEnd = os.path.join(output_folder,'P_CHIRPS.v2.0_mm-month-1_monthly_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')))
else:
raise KeyError("The input time interval is not supported")
# download the global rainfall file
try:
local_filename = os.path.join(output_folder, filename)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write, 8192)
lf.close()
# unzip the file
zip_filename = os.path.join(output_folder, filename)
Raster.ExtractFromGZ(zip_filename, outfilename, delete=True)
# open tiff file
dataset,NoDataValue = Raster.GetRasterData(outfilename)
# clip dataset to the given extent
data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
# replace -ve values with -9999
data[data < 0] = -9999
# save dataset as geotiff file
geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
Raster.CreateRaster(Path=DirFileEnd, data=data, geo=geo, EPSG="WGS84",NoDataValue = NoDataValue)
# delete old tif file
os.remove(outfilename)
except:
print("file not exists")
return True
def DownloadData(self, Var, Waitbar):
"""
This function downloads ECMWF six-hourly, daily or monthly data
Keyword arguments:
"""
# Load factors / unit / type of variables / accounts
VarInfo = Variables(self.Time)
Varname_dir = VarInfo.file_name[Var]
# Create Out directory
out_dir = os.path.join(self.Path, self.Time, Varname_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
DownloadType = VarInfo.DownloadType[Var]
if DownloadType == 1:
string1 = 'oper'
string4 = "0"
string6 = "00:00:00/06:00:00/12:00:00/18:00:00"
string2 = 'sfc'
string8 = 'an'
if DownloadType == 2:
string1 = 'oper'
string4 = "12"
string6 = "00:00:00/12:00:00"
string2 = 'sfc'
string8 = 'fc'
if DownloadType == 3:
string1 = 'oper'
string4 = "0"
string6 = "00:00:00/06:00:00/12:00:00/18:00:00"
string2 = 'pl'
string8 = 'an'
parameter_number = VarInfo.number_para[Var]
string3 = '%03d.128' %(parameter_number)
string5 = '0.125/0.125'
string9 = 'ei'
string10 = '%s/%s/%s/%s' %(self.latlim_corr[1], self.lonlim_corr[0],
self.latlim_corr[0], self.lonlim_corr[1]) #N, W, S, E
# Download data by using the ECMWF API
print('Use API ECMWF to collect the data, please wait')
RemoteSensing.API(self.Path, DownloadType, string1, string2, string3, string4,
string5, string6, self.string7, string8, string9, string10)
# Open the downloaded data
NC_filename = os.path.join(self.Path,'data_interim.nc')
fh = Dataset(NC_filename, mode='r')
# Get the NC variable parameter
parameter_var = VarInfo.var_name[Var]
Var_unit = VarInfo.units[Var]
factors_add = VarInfo.factors_add[Var]
factors_mul = VarInfo.factors_mul[Var]
# Open the NC data
Data = fh.variables[parameter_var][:]
Data_time = fh.variables['time'][:]
lons = fh.variables['longitude'][:]
lats = fh.variables['latitude'][:]
# Define the georeference information
Geo_four = np.nanmax(lats)
Geo_one = np.nanmin(lons)
Geo_out = tuple([Geo_one, 0.125, 0.0, Geo_four, 0.0, -0.125])
# Create Waitbar
if Waitbar == 1:
total_amount = len(self.Dates)
amount = 0
weirdFn.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
for date in self.Dates:
# Define the year, month and day
year = date.year
month = date.month
day = date.day
# Hours since 1900-01-01
start = dt.datetime(year=1900, month=1, day=1)
end = dt.datetime(year, month, day)
diff = end - start
hours_from_start_begin = diff.total_seconds()/60/60
Date_good = np.zeros(len(Data_time))
if self.Time == 'daily':
days_later = 1
if self.Time == 'monthly':
days_later = calendar.monthrange(year,month)[1]
Date_good[np.logical_and(Data_time>=hours_from_start_begin, Data_time<(hours_from_start_begin + 24 * days_later))] = 1
Data_one = np.zeros([int(np.sum(Date_good)),int(np.size(Data,1)),int(np.size(Data,2))])
Data_one = Data[np.int_(Date_good) == 1, :, :]
# Calculate the average temperature in celcius degrees
Data_end = factors_mul * np.nanmean(Data_one,0) + factors_add
if VarInfo.types[Var] == 'flux':
Data_end = Data_end * days_later
VarOutputname = VarInfo.file_name[Var]
# Define the out name
name_out = os.path.join(out_dir, "%s_ECMWF_ERA-Interim_%s_%s_%d.%02d.%02d.tif" %(VarOutputname, Var_unit, self.Time, year,month,day))
# Create Tiff files
# Raster.Save_as_tiff(name_out, Data_end, Geo_out, "WGS84")
Raster.CreateRaster(Path=name_out, data=Data_end, geo=Geo_out, EPSG="WGS84")
if Waitbar == 1:
amount = amount + 1
weirdFn.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
fh.close()
return()