def load_trmm_series():
# read a series of TRMM data files, returns a matrix and several metadata
global nlon, nlat, ntim
global lon0, lat0, lon1, lat1
global time_stamps, trmm_series
# loading the data
time_stamps = []
trmm_series = []
print 'Loading a series of TRMM matrices... please wait.'
for dday in range(1, 32):
for hour in range(0, 24, 3):
filepath = path + '/' + pref + str(dday).zfill(2) + str(
hour).zfill(2) + suff
trmmfile = TRMM3B42RTFile(filepath)
time_string = trmmfile.header()['granule_ID'].split(".")[1]
time_stamps.append(datetime.strptime(time_string, "%Y%m%d%H"))
trmm_series.append(trmmfile.precip())
print '...created "trmm_series" 3D object containing the data: use it.'
# variables used to define grids and axes
nlon = int(trmmfile.header()['number_of_longitude_bins'])
nlat = int(trmmfile.header()['number_of_latitude_bins'])
ntim = len(trmm_series)
# 'first_box_center': '59.875N,0.125E'
# 'last_box_center': '59.875S,359.875E'
lon0 = get_num(trmmfile.header()['first_box_center'].split(",")[1])
lat0 = -get_num(trmmfile.header()['first_box_center'].split(",")[0])
lon1 = get_num(trmmfile.header()['last_box_center'].split(",")[1])
lat1 = get_num(trmmfile.header()['last_box_center'].split(",")[0])
def trmm2npy(path):
"""
Converts a TRMM 3B42RT time sequence in a Numpy data file.
:param path: the path where TRMM data is located; also the path where npy
file will be saved.
:return: a Numpy file containing TRMM data.
"""
from pytrmm import TRMM3B42RTFile
from glob import glob
from numpy import asarray, save
pref = '3B42RT'
suff = '7R2.bin.gz'
#time_stamps = []
trmm_series = []
print 'Loading a series of TRMM matrices... please wait.'
filelist = sorted(glob(path + '/' + pref + '.*.' + suff))
for filepath in filelist:
trmmfile = TRMM3B42RTFile(filepath)
# time_string = trmmfile.header()['granule_ID'].split(".")[1]
# time_stamps.append(datetime.strptime(time_string, "%Y%m%d%H"))
trmm_series.append(trmmfile.precip())
trmmdata = asarray(trmm_series)
save(path + '/' + pref, trmmdata)
def generate_24h_accumulation():
for f in trmm_files:
local_filename = os.path.join(config.data_dir, "trmm", f)
trmm_file = TRMM3B42RTFile(local_filename)
precip = trmm_file.precip()
#print f
#print 'TRMM precip max:', precip.max()
#print 'TRMM precip min:', precip.min()
#print 'TRMM precip mean:', precip.mean()
#print 'TRMM precip std-dev:', precip.std()
if f == file_00:
data = precip
else:
data += precip
print "data:", data.min(), data.mean(), data.max(), data.std()
nrows = precip.shape[0]
ncols = precip.shape[1]
y = 60
x = -180
res = 0.25
if force:
cmd = "rm " + delete_files
if verbose:
print cmd
os.system(cmd)
if verbose:
print "Creating:", output_file
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(output_file, ncols, nrows, 1, gdal.GDT_Float32)
# top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution
dst_ds.SetGeoTransform([x, res, 0, y, 0, -res])
# set the reference info
srs = osr.SpatialReference()
srs.SetWellKnownGeogCS("WGS84")
dst_ds.SetProjection(srs.ExportToWkt())
# write the band
band = dst_ds.GetRasterBand(1)
band.SetNoDataValue(-31999)
band.WriteArray(data)
dst_ds = None
# color it using colormap for testing
cmd = "gdaldem color-relief -q -alpha " + output_file + " " + color_file + " " + rgb_output_file
if verbose:
print cmd
os.system(cmd)
def load_trmm_series_OLD():
trmmvals = empty([nlat, nlon, ntim], dtype='float32')
counter = 0
for dday in range(1, 32):
for hour in range(0, 24, 3):
filepath = path + '/' + pref + str(dday).zfill(2) + str(
hour).zfill(2) + suff
trmmvals[:, :, counter] = TRMM3B42RTFile(filepath).precip()
counter = counter + 1
return trmmvals
def load_trmm_series_OLD():
trmmvals = empty([480, 1440, 248], dtype='float32')
counter = 0
for dday in range(1, 32):
for hour in range(0, 24, 3):
filepath = '/home/santiago/Datasets/TRMM-3B42RT/200401/3B42RT.200401' + str(
dday).zfill(2) + str(hour).zfill(2) + '.7R2.bin.gz'
print str(dday).zfill(2) + str(hour).zfill(2), counter, filepath
trmmvals[:, :, counter] = TRMM3B42RTFile(filepath).precip()
#dates =
counter = counter + 1
return trmmvals
def load_trmm_series(path, long, lati):
"""
Loads TRMM 3B42RT data as a time series for tha given lon, lat location.
:param path: the location of TRMM 3B42RT data.
:param long: longitude in 0:360 range
:param lati: latitude in -90:90 range.
:return: a large matrix of precipitation data.
"""
from pytrmm import TRMM3B42RTFile
from datetime import datetime
from glob import glob
from numpy import linspace
# loading TRMM data
pref = '3B42RT.'
suff = '.7R2.bin.gz'
time_stamps = []
trmm_series = []
print 'Loading a series of TRMM matrices... please wait.'
filelist = sorted(glob(path + '/' + pref + '*' + suff))
for filepath in filelist:
trmmfile = TRMM3B42RTFile(filepath)
time_string = trmmfile.header()['granule_ID'].split(".")[1]
print time_string
time_stamps.append(datetime.strptime(time_string, "%Y%m%d%H"))
trmm_series.append(trmmfile.precip())
# variables used to define grids and axes
nlon = int(trmmfile.header()['number_of_longitude_bins'])
nlat = int(trmmfile.header()['number_of_latitude_bins'])
ntim = len(trmm_series)
# 'first_box_center': '59.875N,0.125E'
# 'last_box_center': '59.875S,359.875E'
lon0 = get_num(trmmfile.header()['first_box_center'].split(",")[1])
lat0 = -get_num(trmmfile.header()['first_box_center'].split(",")[0])
lon1 = get_num(trmmfile.header()['last_box_center'].split(",")[1])
lat1 = get_num(trmmfile.header()['last_box_center'].split(",")[0])
# creates zonal and meridional axes
lons = linspace(lon0, lon1, nlon)
lats = linspace(lat0, lat1, nlat)
lon_idx = find_nearest_idx(lons, long)
lat_idx = find_nearest_idx(lats, lati)
trmm_point = []
for it in range(ntim):
trmm_point.append(trmm_series[it][lat_idx, lon_idx])
del (trmm_series)
return time_stamps, trmm_point
def plot_trmm_OLD(filepath):
trmmdata = TRMM3B42RTFile(filepath)
prec = trmmdata.precip()
prec_units = 'mm h-1'
lons, lats = get_coordinates()
m = Basemap(projection='cyl', llcrnrlat=-60, llcrnrlon=-180, \
urcrnrlat=60, urcrnrlon=180, resolution='c')
lon, lat = meshgrid(lons, lats)
xi, yi = m(lon, lat)
# Plot data
cs = m.pcolor(xi, yi, squeeze(prec))
# Add grid lines
parallels = arange(-60., 61., 20.)
meridians = arange(-180., 181., 20.)
m.drawparallels(parallels, labels=[1, 0, 0, 0], fontsize=10)
m.drawmeridians(meridians, labels=[0, 0, 0, 1], fontsize=10)
# Add coastlines, states, and country boundaries
m.drawcoastlines()
m.drawstates()
m.drawcountries()
# Add colorbar
cbar = m.colorbar(cs, location='bottom', pad="10%")
cbar.set_label(prec_units)
# Add title
plt.title('Precipitacao TRMM')
plt.show()
def process_latest_trmm_file(filename):
arr = filename.split('.')
dt = arr[1]
region = config.regions['global']
tzoom = region['tiles-zoom']
local_filename = os.path.join(config.data_dir, "trmm", filename)
if verbose:
print "processing:", local_filename
farr = filename.split('.')
baseName = "%s_%s" % (farr[0], farr[1])
trmm_file = TRMM3B42RTFile(local_filename)
precip = trmm_file.precip()
print 'TRMM precip max:', precip.max()
print 'TRMM precip min:', precip.min()
print 'TRMM precip mean:', precip.mean()
print 'TRMM precip std-dev:', precip.std()
nrows = precip.shape[0]
ncols = precip.shape[1]
y = 60
x = -180
res = 0.25
# Set file vars
delete_files = os.path.join(config.data_dir, "trmm", "trmm_3B42RT_*")
output_file = os.path.join(config.data_dir, "trmm",
"trmm_3B42RT_%s_4326.tif" % dt)
blended_file = os.path.join(config.data_dir, "trmm",
"trmm_3B42RT_%s_4326_blended.tif" % dt)
blended_rgb_file = os.path.join(config.data_dir, "trmm",
"trmm_3B42RT_%s_4326_blended_rgb.tif" % dt)
mbtiles_dir = os.path.join(config.data_dir, "mbtiles",
"trmm_3B42RT_%s" % dt)
mbtiles_fname = mbtiles_dir + ".mbtiles"
if force:
cmd = "rm " + delete_files
if verbose:
print cmd
os.system(cmd)
if force or not os.path.exists(output_file):
# Create gtif
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(output_file, ncols, nrows, 1, gdal.GDT_Float32)
# top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution
dst_ds.SetGeoTransform([x, res, 0, y, 0, -res])
# set the reference info
srs = osr.SpatialReference()
srs.SetWellKnownGeogCS("WGS84")
dst_ds.SetProjection(srs.ExportToWkt())
# write the band
band = dst_ds.GetRasterBand(1)
band.SetNoDataValue(-31999)
band.WriteArray(precip)
dst_ds = None
# blend it
if force or not os.path.exists(blended_file):
cmd = "gdalwarp -overwrite -q -r cubicspline -co COMPRESS=LZW -cblend 5 " + output_file + " " + blended_file
if verbose:
print cmd
os.system(cmd)
# color it using colormap
if force or not os.path.exists(blended_rgb_file):
cmd = "gdaldem color-relief -q -alpha " + blended_file + " " + color_file + " " + blended_rgb_file
if verbose:
print cmd
os.system(cmd)
return
# Create mbtiles
if force or not os.path.exists(mbtiles_fname):
cmd = "./gdal2tiles.py -z " + tzoom + " " + blended_rgb_file + " " + mbtiles_dir
if verbose:
print cmd
os.system(cmd)
# generate metadata.json
metafile = os.path.join(mbtiles_dir, "metadata.json")
json = "{\n"
json += " \"name\": \"TRMM 24hr Precipitation - " + ym + "\",\n"
json += " \"description\": \"TRMM\",\n"
json += " \"version\": 1\n"
json += "}"
f = open(metafile, "w")
f.write(json)
f.close()
cmd = "./mb-util " + mbtiles_dir + " " + mbtiles_fname
if verbose:
print cmd
os.system(cmd)
# copy mbtiles to S2
bucketName = region['bucket']
# copy mbtiles to S3
bucketName = region['bucket']
cmd = "./aws-copy.py --bucket " + bucketName + " --file " + mbtiles_fname
if verbose:
cmd += " --verbose"
print cmd
os.system(cmd)
cmd = "rm -rf " + mbtiles_dir
if verbose:
print cmd
os.system(cmd)
process_latest_trmm_d02_file(dt)
process_latest_trmm_d03_file(dt)