def main():
print('main')
indices = ps.from_latlon([-45, 45], [45, 45], 1)
print('level zero: ', indices)
latv, lonv, latc, lonc = ps.to_vertices_latlon(indices)
print(' latv: ', latv)
print(' lonv: ', lonv)
print(' latc: ', latc)
print(' lonc: ', lonc)
if True:
proj = ccrs.PlateCarree()
transf = ccrs.Geodetic()
plt.figure()
ax = plt.axes(projection=proj)
ax.set_global()
ax.coastlines()
gd.plot_indices(ps.from_latlon(np.arange(-89, 89, dtype=np.float),
-2 * np.arange(-89, 89, dtype=np.float),
0),
c='g',
transform=transf,
lw=0.25)
# gd.plot_indices(ps.from_latlon([0,0,15],[-15,0,15],1),c='g',transform=transf,lw=0.25)
# gd.plot_indices(ps.from_latlon([0,0,15],[-15,0,15],2),c='r',transform=transf,lw=0.25)
# gd.plot_indices(ps.from_latlon([-45,45],[45,45],0),c='r',transform=transf,lw=0.25)
# gd.plot_indices(ps.from_latlon([-45],[45],0),c='r',transform=transf,lw=0.25)
plt.show()
return
def sids_from_xy(lon, lat, resolution):
"""Takes a list/array of lon and lat and returns a (set of) STARE index/ices
Parameters
-----------
lon: numerical/float
Longitude of point
lat: numerical/float
Latitude of point
resolution: int
STARE spatial resolution
Returns
----------
sids
Array of STARE index values
Examples
----------
>>> import starepandas
>>> x = [10.1, 20.9]
>>> y = [55.3, 60.1]
>>> starepandas.sids_from_xy(x, y, resolution=15)
array([4254264869405326191, 3640541580264132591])
"""
return pystare.from_latlon(lat, lon, resolution)
def test__intersect(self):
indices = pystare.from_latlon(lat, lon, 12)
intersected = numpy.zeros([3], dtype=numpy.int64)
pystare._intersect(indices, indices, intersected)
expected = numpy.array(
[3643626709468577804, 4151504977312874508, 4161865159985332236])
numpy.testing.assert_array_equal(intersected, expected)
def sid_from_point(point, resolution):
"""Takes a shapely Point, Polygon, or Multipolygon and returns the according SID
Parameters
-------------
point: shapely.geometry.Point
Shapely point
resolution: int
STARE resolution to use for lookup
Returns
----------
sid
spatial index value
Examples
---------
>>> import starepandas
>>> import shapely
>>> point = shapely.geometry.Point(10.5, 20)
>>> starepandas.sid_from_point(point, resolution=20)
4598246232954051060
"""
lat = point.y
lon = point.x
index_value = pystare.from_latlon([lat], [lon], resolution)[0]
return index_value
def w_from_latlon(llr):
# print('')
# print('llr: ',llr)
sids = ps.from_latlon(llr[0], llr[1], int(llr[2][0]))
# print('sids: ',sids)
# print('')
return sids
def test_intersect_multires2(self):
indices1 = pystare.from_latlon(lat, lon, 12)
indices2 = numpy.array([indices1[1]], dtype=numpy.int64)
intersected = pystare.intersect(indices1,
indices2,
multiresolution=True)
expected = numpy.array([4161865159985332236])
numpy.testing.assert_array_equal(intersected, expected)
def test_intersect_multires3(self):
indices1 = pystare.from_latlon(lat, lon, 12)
indices2 = numpy.array([0x100000000000000c], dtype=numpy.int64)
intersected = pystare.intersect(indices1,
indices2,
multiresolution=True)
numpy.testing.assert_array_equal(intersected,
numpy.array([], dtype=numpy.int64))
def test_latlonroundtrip(self):
lat1 = numpy.array([30,45,60], dtype=numpy.double)
lon1 = numpy.array([45,60,10], dtype=numpy.double)
level1 = 12
indices = pystare.from_latlon(lat1, lon1, level1)
lat2, lon2, level2 = pystare.to_latlonlevel(indices)
numpy.testing.assert_allclose(lat1, lat2)
numpy.testing.assert_allclose(lon1, lon2)
self.assertEqual(level1, level2[0])
def main():
# proj = ccrs.PlateCarree()
proj = ccrs.Mollweide()
transf = ccrs.Geodetic()
plt.figure()
ax = plt.axes(projection=proj)
# ax.set_xlim(-180,180)
# ax.set_ylim(-90,90)
ax.stock_img()
ax.coastlines()
ax.gridlines()
lats = np.array([-45,-45,-45,-45,-45, -45, 45,45,45, 45, 45, 45],dtype=np.double)
lons = np.array([ 0, 45, 90,135,180, 225, 0,45,90,135,180,225],dtype=np.double)
tmp_cover = ps.from_latlon(lats,lons,0)
print('tmp_cover len: ',len(tmp_cover))
print('tmp_cover: ',tmp_cover)
clons,clats,cintmat = ps.triangulate_indices(tmp_cover)
ctriang = tri.Triangulation(clons,clats,cintmat)
ax.triplot(ctriang,'b-',transform=transf,lw=1.0,markersize=3,alpha=1.0)
zlatv = np.array([
0, 90, 0, 0, 90, 0, 0, 90, 0, 0, 90, 0,
0,-90, 0, 0, -90, 0, 0, -90, 0, 0, -90, 0
],dtype=np.double)
zlonv = np.array([
0,90,90, 90, 90, 180, 180, 270, 270, 270, 360, 360,
0,90,90, 90, 90, 180, 180, 270, 270, 270, 360, 360
],dtype=np.double)
zlons,zlats,zintmat = gd.triangulate1(zlatv,zlonv)
ztriang = tri.Triangulation(zlons,zlats,zintmat)
ax.triplot(ztriang,'r-',transform=transf,lw=1.0,markersize=3,alpha=1.0)
an = ax.annotate(
'Figure: sketchM0\n'
+'Date: %s\n'%datetime.date.today()
+'Version: 2020-0407-1\n'
+'Author: M. Rilee, RSTLLC\n'
+'Email: [email protected]\n'
,xy=(0.7,0.025)
,xycoords='figure fraction'
,family='monospace'
)
plt.show()
return
def sids_from_gdf(gdf, resolution, convex=False, force_ccw=True, n_workers=1):
"""
Takes a GeoDataFrame and returns a corresponding series of sets of trixel indices
Parameters
-----------
gdf: geopandas.GeoDataFrame
A dataframe containing features to look (sets of) STARE indices up for
resolution: int
STARE resolution
convex: bool
Toggle if STARE indices for the convex hull rather than the G-Ring should be looked up
force_ccw: bool
Toggle if a counter clockwise orientation of the geometries should be enforced
n_workers: int
Number of workers used to lookup STARE indices in parallel
Returns
--------
A numpy array of length=len(gdf.index) holding the set of stare indices of each geometry
Examples
---------
>>> import geopandas
>>> import starepandas
>>> world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
>>> italy = world[world.name=='Italy']
>>> starepandas.sids_from_gdf(italy, resolution=3, convex=False, force_ccw=True, n_workers=1)
[array([4269412446747230211, 4548635623644200963, 4566650022153682947,
4548635623644200963, 4548635623644200963])]
"""
if gdf._geometry_column_name in gdf.keys():
pass
else:
print('no geom column set')
if set(gdf.geom_type) == {'Point'}:
# This might be a speedup since we don't need to iterate over python lists
lat = gdf.geometry.y
lon = gdf.geometry.x
return pystare.from_latlon(lat, lon, resolution)
else:
return sids_from_geoseries(gdf.geometry,
resolution=resolution,
convex=convex,
force_ccw=force_ccw,
n_workers=n_workers)
def stare_from_xy_df(df, level=-1, n_cores=1):
""" Takes a dataframe; assumes columns with lat and lon"""
rename_dict = {}
rename_dict['Latitude'] = 'lat'
rename_dict['latitude'] = 'lat'
rename_dict['y'] = 'lat'
rename_dict['Longitude'] = 'lon'
rename_dict['longitude'] = 'lon'
rename_dict['x'] = 'lon'
df = df.rename(columns=rename_dict)
if n_cores > 1:
ddf = dask.dataframe.from_pandas(df, npartitions=n_cores)
return ddf.map_partitions(stare_row, level,
meta=('stare',
'int')).compute(scheduler='processes')
else:
return pystare.from_latlon(df.lat, df.lon, level)
def sids_from_xy_df(df, resolution, n_workers=1):
""" Takes a dataframe and generates an array of STARE index values.
Assumes latitude column name is {'lat', 'Latitude', 'latitude', or 'y'} and
longitude column name is {'lon', 'Longitude', 'longitude', or 'x'}
Parameters
--------------
df: pandas.DataFrame
Dataframe containing x/y coordinates
resolution: int
STARE spatial resolution
n_workers: int
Number of workers used to lookup STARE indices in parallel
Returns
--------
sids
Array of STARE index values
Examples
------------
>>> import starepandas
>>> import pandas
>>> x = [-119.42, 7.51]
>>> y = [34.25, 47.59]
>>> df = pandas.DataFrame({'lat': y, 'lon': x})
>>> starepandas.sids_from_xy_df(df, resolution=20)
array([3331752989521980116, 4271829667422230484])
"""
rename_dict = {
'Latitude': 'lat',
'latitude': 'lat',
'y': 'lat',
'Longitude': 'lon',
'longitude': 'lon',
'x': 'lon'
}
df = df.rename(columns=rename_dict)
if n_workers > 1:
ddf = dask.dataframe.from_pandas(df, npartitions=n_workers)
return ddf.map_partitions(sids_from_latlon_row,
resolution,
meta=('stare',
'int')).compute(scheduler='processes')
else:
return pystare.from_latlon(df.lat, df.lon, resolution)
def stare_from_gdf(gdf, level=-1, nonconvex=True, force_ccw=True):
"""
Takes a GeoDataFrame and returns a corresponding series of sets of trixel indices
"""
if not gdf._geometry_column_name in gdf.keys():
print('no geom column set')
if set(gdf.geom_type) == {'Point'}:
lat = gdf.geometry.y
lon = gdf.geometry.x
return pystare.from_latlon(lat, lon, level)
else:
index_values = []
for geom in gdf.geometry:
if geom.type == 'Polygon':
index_values.append(
from_polygon(geom, level, nonconvex, force_ccw))
elif geom.type == 'MultiPolygon':
index_values.append(
from_multipolygon(geom, level, nonconvex, force_ccw))
elif geom.type == 'Point':
index_values.append(from_point(geom, level))
return index_values
def stare_row(row, level):
""" Dask helper function """
return pystare.from_latlon(row.lat, row.lon, level)
def stare_from_xy(lon, lat, level=-1, n_cores=1):
return pystare.from_latlon(lat, lon, level)
def from_point(point, level=-1):
lat = point.y
lon = point.x
index_value = pystare.from_latlon([lat], [lon], level)[0]
return index_value
return vsum
# y=0*x+coeffs[0]
# nx=100; ny=100
# nx=20; ny=20
# nx=40; ny=10
nx=15; ny=15
# nx=10; ny=10
x=np.linspace(-180,180,nx); y=np.linspace(-90,90,ny)
xg,yg = np.meshgrid(x,y)
xg_flat = xg.flatten(); yg_flat = yg.flatten()
angleScale = 360.0/nx
resolution = ceil(-np.log2(angleScale/90.0))
sid = ps.from_latlon(yg_flat,xg_flat,resolution)
sidg = sid.reshape(xg.shape)
# print('sid'); print(type(sid))
# print(['{:02x}'.format(i) for i in sid[0:10]])
# print(['{:02x}'.format(i) for i in sid])
# t0=track([0,1],rthick=10.0,xmnmx=[-10,30])
# t0=track([5,0.5,0.005,-0.0005,-0.000001],rthick=10.0,dxmnmx=[-30,45])
# t0=track([5,0.5,0.005,-0.0005,-0.000001],rthick=10.0,xyc=[10,20],dxmnmx=[-30,45])
# t0=track([5],rthick=10.0,dxmnmx=[-30,45])
# t0=track([5],rthick=10.0,xyc=[0,45],dxmnmx=[0,90])
# t0=track([5],rthick=10.0,xyc=[0,-45],dxmnmx=[0,90])
# t0=track([5],rthick=10.0,xyc=[-90,-45],dxmnmx=[0,90])
# t0=track([5],rthick=10.0,xyc=[-90,45],dxmnmx=[0,90])
# t0=track([0,1],rthick=10.0,xyc=[-90,45],dxmnmx=[0,90])
def make_sare(self, res_km=1):
self.sare = ps.from_latlon(self.geo_lat.flatten(),
self.geo_lon.flatten(),
int(gd.resolution(res_km)))
return self
### MERRA 2 DATASET
dataPath = "/home/mrilee/data/"
dataFile = m2_base
fqFilename = dataPath + dataFile
m2_dLat = 0.5
m2_dLon = 5.0 / 8.0
m2_dLatkm = m2_dLat * gd.re_km / gd.deg_per_rad
m2_dLonkm = m2_dLon * gd.re_km / gd.deg_per_rad
m2_ds = Dataset(fqFilename)
print('dims lat,lon: ', m2_ds['lat'].shape, m2_ds['lon'].shape)
m2_lat, m2_lon = np.meshgrid(m2_ds['lat'], m2_ds['lon'])
m2_lat = m2_lat.flatten()
m2_lon = m2_lon.flatten()
m2_idx_ij = np.arange(m2_lat.size, dtype=np.int64)
m2_resolution = int(gd.resolution(m2_dLonkm * 2))
m2_indices = ps.from_latlon(m2_lat, m2_lon, m2_resolution)
# m2_indices = ps.from_latlon(m2_lat,m2_lon,int(gd.resolution(m2_dLonkm*0.5)))
m2_tid = gd.merra2_stare_time(m2_ds)
print('type(m2_indices): ', type(m2_indices))
print('shape(m2_indices): ', m2_indices.shape)
print('size(m2_indices): ', m2_indices.size)
print('type(g5 tid): ', type(goes_b5_tid))
print('g5 tid.shape: ', goes_b5_tid.shape)
print('type(m2_tid): ', type(m2_tid))
print('m2_tid.shape: ', m2_tid.shape)
fine_match = ps.cmp_temporal(np.array(goes_b5_tid, dtype=np.int64), m2_tid)
m2_ifm = np.nonzero(fine_match)[0]
print('m2_ifm: ', m2_ifm)
m2_dataDayI = m2_ds['TQI'][m2_ifm, :, :]
def main():
# GOES 10 (Band 5, 4km)
# 0.036 deg
g_lon, g_lat, g_dat = load_goes()
if False:
idx = range(1000)
g_lon = g_lon[idx]
g_lat = g_lat[idx]
g_dat = g_dat[idx]
g_lon_s, g_lat_s, g_dat_s, g_ilex = gd.lexsort_data(g_lon, g_lat, g_dat)
# MODIS Water Vapor NIR 1km
# 1.57e-4 radians -- 0.009 deg
m_lon, m_lat, m_dat = load_modis()
if False:
idx = range(1000)
m_lon = m_lon[idx]
m_lat = m_lat[idx]
m_dat = m_dat[idx]
print('m_lon size: ', m_lon.size)
m_str = np.zeros([m_lon.size], dtype=np.int64)
for irow in range(2030):
i0 = irow * 1354
i1 = (irow + 1) * 1354
m_str[i0:i1] = ps.from_latlon(m_lat[i0:i1], m_lon[i0:i1],
13) # 13 ~ 10km/(2**3)
m_lon_s, m_lat_s, m_dat_s, m_ilex = gd.lexsort_data(m_lon, m_lat, m_dat)
delta = 0.025 # degree
mlon_mnmx = [np.amin(m_lon) - delta, np.amax(m_lon) + delta]
mlat_mnmx = [np.amin(m_lat) - delta, np.amax(m_lat) + delta]
idlon1 = np.where((mlon_mnmx[0] <= g_lon_s) & (g_lon_s <= mlon_mnmx[1]))
g_lon1 = g_lon_s[idlon1]
g_lat1 = g_lat_s[idlon1]
g_dat1 = g_dat_s[idlon1]
idlat2 = np.where((mlat_mnmx[0] <= g_lat1) & (g_lat1 <= mlat_mnmx[1]))
g_lon2 = g_lon1[idlat2]
g_lat2 = g_lat1[idlat2]
g_dat2 = g_dat1[idlat2]
g_lon3, g_lat3, g_dat3, g_ilex3 = gd.lexsort_data(g_lon2, g_lat2, g_dat2)
viz = True
viz_fig1_NadirVsWing = False
if viz:
proj = ccrs.PlateCarree()
transf = ccrs.Geodetic()
nrows = 2
fig, axs = plt.subplots(nrows=nrows,
subplot_kw={
'projection': proj,
'transform': transf
})
# plt.figure()
# plt.subplot(projection=proj,transform=transf)
# ax=axs[0]
# ax.set_global()
# ax.coastlines()
delta = 0.05 # degree
goes_nadir_hdelta = 0.036 / 2.0 # 1/2 width deg for 4km pixel
mod_nadir_hdelta = 0.009 / 2.0 # 1/2 width deg for 1km pixel
for i in range(g_lon3.size):
# if True:
# i = 0
lo = g_lon3[i]
la = g_lat3[i]
da = g_dat3[i]
lo_range = [lo - delta, lo + delta]
la_range = [la - delta, la + delta]
lo_r, la_r, da_r = gd.subset_data_from_lonlatbox(
m_lon, m_lat, m_dat, lo_range, la_range)
if len(da_r) > 4:
idlo_p = np.where(lo_r > lo)[0]
idlo_m = np.where(lo_r < lo)[0]
idlo_skew = (1.0 * (len(idlo_p) - len(idlo_m))) / (
1.0 * (len(idlo_p) + len(idlo_m)))
idla_p = np.where(la_r > la)[0]
idla_m = np.where(la_r < la)[0]
idla_skew = (1.0 * (len(idla_p) - len(idla_m))) / (
1.0 * (len(idla_p) + len(idla_m)))
if viz and abs(idlo_skew) < 0.5 and abs(idla_skew) < 0.5:
print(i, ' i,len(da_r),skew ', len(da_r), len(idlo_p),
len(idlo_m), idlo_skew, idla_skew)
ax = axs[0]
# ax.set_global()
ax.coastlines()
xlim = [lo - 2 * delta, lo + 2 * delta]
ylim = [la - 2 * delta, la + 2 * delta]
ax.set_xlim(xlim[0], xlim[1])
ax.set_ylim(ylim[0], ylim[1])
plot_box(ax, mlon_mnmx, mlat_mnmx)
plot_box(ax, lo_range, la_range)
for j in range(len(da_r)):
xr, yr = make_box(
lo_r[j], la_r[j],
delta=mod_nadir_hdelta) # NADIR resolution for MODIS
plot_box(ax, xr, yr, c='k')
ax.scatter([lo], [la], s=8, c='r', transform=transf)
g_xr, g_yr = make_box(lo, la, goes_nadir_hdelta)
plot_box(ax, g_xr, g_yr, c='r')
if False: # Lexical sorting
ax.scatter(m_lon_s[0:1354],
m_lat_s[0:1354],
s=2.5,
c='k',
transform=transf)
if False: # All
ax.scatter(m_lon_s,
m_lat_s,
s=2.5,
c='k',
transform=transf)
if True:
for irow in range(2030 - 10, 2030):
i0 = irow * 1354
i1 = (irow + 1) * 1354
if True:
for j in range(i0, i1):
xr, yr = make_box(
m_lon[j], m_lat[j], delta=mod_nadir_hdelta
) # NADIR resolution for MODIS
plot_box(ax, xr, yr, c='k')
ax.scatter(m_lon[i0:i1],
m_lat[i0:i1],
s=2.5,
c='k',
transform=transf)
ind = m_str[i0:i1]
inda = ps.adapt_resolution_to_proximity(ind)
plot_sivs(inda, c0='g', c1='g', transf=transf, ax=ax)
iax = 1
ax = axs[iax]
ax.coastlines()
la0 = 18.125
lo0 = -161.267
xlim = [lo0 - 2 * delta, lo0 + 2 * delta]
ylim = [la0 - 2 * delta, la0 + 2 * delta]
print('iax,xlim,ylim: ', iax, xlim, ylim)
ax.set_xlim(xlim[0], xlim[1])
ax.set_ylim(ylim[0], ylim[1])
if True:
for irow in range(2030 - 10, 2030):
i0 = irow * 1354
i1 = (irow + 1) * 1354
if True:
for j in range(i0, i1):
xr, yr = make_box(
m_lon[j], m_lat[j], delta=mod_nadir_hdelta
) # NADIR resolution for MODIS
plot_box(ax, xr, yr, c='k')
ax.scatter(m_lon[i0:i1],
m_lat[i0:i1],
s=2.5,
c='k',
transform=transf)
ind = m_str[i0:i1]
inda = ps.adapt_resolution_to_proximity(ind)
plot_sivs(inda, c0='g', c1='g', transf=transf, ax=ax)
an = ax.annotate(
'Figure: sketchK0: GOES square and MODIS nadir sampling\n'
+ 'Date: %s\n' % datetime.date.today() +
'Version: 2020-0407-1\n' + 'Author: M. Rilee, RSTLLC\n' +
'Email: [email protected]\n',
xy=(0.7, 0.025),
xycoords='figure fraction',
family='monospace')
plt.show()
return
print('done')
return
for itk_1km in range(nAlong):
for isc_1km in range(nAcross):
if int(100 * ktr / ktr_max) % 5 == 0 or int(100 * ktr / ktr_max) < 2:
print('interpolate from 5km to 1km: %2d%%' %
int(100 * ktr / ktr_max),
end='\r',
flush=True)
if False:
lat[itk_1km, isc_1km], lon[itk_1km,
isc_1km] = pascal_modis.get_1km_pix_pos(
itk_1km, isc_1km, lat_5km, lon_5km)
else:
lat[itk_1km, isc_1km], lon[itk_1km, isc_1km] = hdf_geo_lat[
itk_1km, isc_1km], hdf_geo_lon[itk_1km, isc_1km]
stare_spatial[itk_1km, isc_1km] = ps.from_latlon(
np.array([lat[itk_1km, isc_1km]]),
np.array([lon[itk_1km, isc_1km]]), resolution)
stare_temporal[
itk_1km,
isc_1km] = tid # TODO use np.timedelta for more temporal resolution
ktr = ktr + 1
print('interpolate from 5km to 1km: done')
workFile = h5.File(workFileName, 'w')
image_dtype = np.dtype([('stare_spatial', np.int64),
('stare_temporal_start', np.int64),
('src_coord', np.int64), ('Latitude', np.double),
('Longitude', np.double),
('Water_Vapor_Near_Infrared', np.double)])
image_ds = workFile.create_dataset('image', [nAlong * nAcross],
import geodata as gd
import h5py as h5
from netCDF4 import Dataset
import numpy as np
import pystare as ps
goes_b5_dataPath = "/home/mrilee/data/"
goes_b5_dataFile = "goes10.2005.349.003015.BAND_05.nc"
goes_b5_fqFilename = goes_b5_dataPath + goes_b5_dataFile
goes_b5_ds = Dataset(goes_b5_fqFilename)
print('goes temporal id: ', hex(gd.goes10_img_stare_time(goes_b5_ds)[0]))
print([
hex(i) for i in ps.from_latlon(
goes_b5_ds['lat'][0:4, 0:4].flatten(), goes_b5_ds['lon'][
0:4, 0:4].flatten(), int(gd.resolution(goes_b5_ds['elemRes'][0])))
])
print(goes_b5_ds['lat'][0:4, 0:4].flatten())
def sids_from_latlon_row(row, resolution):
""" Dask helper function """
return pystare.from_latlon(row.lat, row.lon, resolution)
m2_tim = m2_ds['time'][m2_tim0:m2_tim1]
m2_dataDayI = m2_ds['TQI'][m2_tim0:m2_tim1, m2_lat0:m2_lat1, m2_lon0:m2_lon1]
m2_dataDayL = m2_ds['TQL'][m2_tim0:m2_tim1, m2_lat0:m2_lat1, m2_lon0:m2_lon1]
m2_dataDayV = m2_ds['TQV'][m2_tim0:m2_tim1, m2_lat0:m2_lat1, m2_lon0:m2_lon1]
m2_dataDay = m2_dataDayI + m2_dataDayL + m2_dataDayV
m2_data = m2_dataDay[0, :, :].T
m2_latg, m2_long = np.meshgrid(m2_lat, m2_lon)
m2_latg_flat = m2_latg.flatten()
m2_long_flat = m2_long.flatten()
m2_data_flat = m2_data.flatten()
print([
type(i)
for i in [m2_latg_flat, m2_long_flat,
gd.resolution(m2_dLonkm), m2_dLonkm]
])
m2_indices = ps.from_latlon(m2_latg_flat, m2_long_flat,
int(gd.resolution(m2_dLonkm)))
print('m2 lat.shape: ', m2_lat.shape)
print('m2 lon.shape: ', m2_lon.shape)
print('m2 tim.shape: ', m2_tim.shape)
print('m2 latg.shape: ', m2_latg.shape)
print('m2 long.shape: ', m2_long.shape)
print('m2 dataDay.shape: ', m2_dataDay.shape)
print('m2 data.shape: ', m2_data.shape)
print('m2 data flat shape:', m2_data_flat.shape)
print('m2 resolution: ', m2_dLonkm, gd.resolution(m2_dLonkm))
# Limit size for testing
cropped = True
if cropped:
# 1 box
def main():
# client = Client(processes = False) # threads ?
client = Client()
size = 10000000
# size = 20
# shards = 20
# shards = 6
# shards = 1
shards = 12
shape = [size]
lat = np.random.rand(size) * 180.0 - 90.0
lon = np.random.rand(size) * 360.0 - 180.0
resolution_ = 8
resolution = np.full(shape, resolution_, dtype=np.int64)
# print('lat shape: ',lat.shape)
print('')
serial_start = timer()
s_sids = ps.from_latlon(lat, lon, resolution_)
s_sidsstr = [hex16(s_sids[i]) for i in range(len(s_sids))]
serial_end = timer()
# print('0 s_sids: ',s_sids)
print('time s_sids: ', serial_end - serial_start)
def w_from_latlon(llr):
# print('')
# print('llr: ',llr)
sids = ps.from_latlon(llr[0], llr[1], int(llr[2][0]))
# print('sids: ',sids)
# print('')
return sids
# def w_from_latlon1(lat,lon,res):
# return ps.from_latlon(np.array([lat],dtype=np.double)\
# ,np.array([lon],dtype=np.double)\
# ,int(res))
# sid = ps.from_latlon(lat,lon,resolution)
# sid = client.map(w_from_latlon1,lat,lon,resolution) # futures
dask_start = timer()
shard_size = int(size / shards)
shard_bins = np.arange(shards + 1) * shard_size
shard_bins[-1] = size
# print('---')
# print('shards: ',shards)
# print('shard_size: ',shard_size)
# print('shard_bins: ',shard_bins)
# print('---')
lat_shards = [lat[shard_bins[i]:shard_bins[i + 1]] for i in range(shards)]
lon_shards = [lon[shard_bins[i]:shard_bins[i + 1]] for i in range(shards)]
res_shards = [
resolution[shard_bins[i]:shard_bins[i + 1]] for i in range(shards)
]
llr_shards = []
for i in range(shards):
llr_shards.append([lat_shards[i], lon_shards[i], res_shards[i]])
# print('llr_shards len: ',len(llr_shards))
# print('llr_shards: ',llr_shards)
## future = client.submit(func, big_data) # bad
##
## big_future = client.scatter(big_data) # good
## future = client.submit(func, big_future) # good
# sid = client.map(w_from_latlon,llr_shards) # futures
big_future = client.scatter(llr_shards)
sid = client.map(w_from_latlon, big_future) # futures
# print('0 sid: ',sid)
# print('9 len(sid): ',len(sid))
# for i in range(shards):
# print(i, ' 10 sid: ',sid[i])
# print(i, ' 11 sid: ',sid[i].result())
# print('15 sid: ',[type(i) for i in sid])
sid_cat = np.concatenate([i.result() for i in sid])
sidsstr = [hex16(sid_cat[i]) for i in range(len(sid_cat))]
dask_end = timer()
# print('2 sids: ',sids)
sids = sid_cat
print('')
# for i in range(size-20,size):
for i in np.array(np.random.rand(20) * size, dtype=np.int64):
print("%09i" % i, sidsstr[i], s_sidsstr[i], ' ', sids[i] - s_sids[i])
print('')
print('dask total threads: ', sum(client.nthreads().values()))
print('size: ', size)
print('shards: ', shards)
print('')
print('time sids: ', dask_end - dask_start)
print('time s_sids: ', serial_end - serial_start)
print('parallel speed up: ',
(serial_end - serial_start) / (dask_end - dask_start))
client.close()
def test_intersect_multires(self):
indices = pystare.from_latlon(lat, lon, 12)
intersected = pystare.intersect(indices, indices, multiresolution=True)
expected = numpy.array(
[3643626709468577804, 4151504977312874508, 4161865159985332236])
numpy.testing.assert_array_equal(intersected, expected)
def join_goes_and_m2_to_h5(goes_datapath, goes_filenames, m2_datapath,
m2_file_name, workFileName):
##### HDF5 Data types for output
image_dtype = np.dtype([('stare_spatial', np.int64),
('stare_temporal', np.int64),
('goes_src_coord', np.int64),
('goes_b3', np.int64), ('goes_b4', np.int64),
('goes_b5', np.int64),
('merra2_src_coord', np.int64),
('merra2_tpw', np.int64)])
image_description_dtype = np.dtype([('nx', np.int), ('ny', np.int)])
m2_description_dtype = np.dtype([('nx', np.int), ('ny', np.int),
('tpw_offset', np.double),
('tpw_scale', np.double)])
goes_bandnames = {
"BAND_03": "goes_b3",
"BAND_04": "goes_b4",
"BAND_05": "goes_b5"
}
goes_filenames_valid = []
for i in goes_filenames:
goes_band = i.split('.')[4]
if goes_band in goes_bandnames.keys():
goes_filenames_valid.append(i)
if len(goes_filenames_valid) == 0:
print("*ERROR* Join and save -- no valid GOES filenames. Returning.")
return
# print('goes_filenames: ',goes_filenames)
# print('goes_filenames_valid: ',goes_filenames_valid)
###########################################################################
##### MERRA-2
m2_dLat = 0.5
m2_dLon = 5.0 / 8.0
m2_dLatkm = m2_dLat * gd.re_km / gd.deg_per_rad
m2_dLonkm = m2_dLon * gd.re_km / gd.deg_per_rad
m2_ds = Dataset(m2_datapath + m2_file_name)
m2_lat, m2_lon = np.meshgrid(m2_ds['lat'], m2_ds['lon'])
m2_lat = m2_lat.flatten()
m2_lon = m2_lon.flatten()
m2_idx_ij = np.arange(m2_lat.size, dtype=np.int64)
m2_resolution = int(gd.resolution(m2_dLonkm * 2))
m2_indices = ps.from_latlon(m2_lat, m2_lon, m2_resolution)
m2_term = gd.spatial_terminator(m2_indices)
m2_tid = gd.merra2_stare_time(m2_ds)
###########################################################################
##### GOES
igoes = 0
goes_band = goes_filenames_valid[igoes].split('.')[4]
goes_bandname = goes_bandnames[goes_band]
goes_ds = Dataset(goes_datapath + goes_filenames_valid[igoes])
goes_tid = gd.goes10_img_stare_time(goes_ds)
##### MERRA-2 at the GOES time
fine_match = ps.cmp_temporal(np.array(goes_tid, dtype=np.int64), m2_tid)
m2_ifm = np.nonzero(fine_match)[0]
m2_dataDayI = m2_ds['TQI'][m2_ifm, :, :]
m2_dataDayL = m2_ds['TQL'][m2_ifm, :, :]
m2_dataDayV = m2_ds['TQV'][m2_ifm, :, :]
m2_dataDay = m2_dataDayI + m2_dataDayL + m2_dataDayV
if m2_ifm.size > 1:
print('multiple hits on 2m')
m2_dataDay = np.mean(m2_dataDay, axis=1)
print('m2_dataDay shape: ', n2_dataDay.shape)
m2_data = m2_dataDay[:, :].T
m2_data_flat = m2_data.flatten()
g_lat = goes_ds['lat'][:, :].flatten()
g_lon = goes_ds['lon'][:, :].flatten()
g_idx_valid = np.where((g_lat >= -90.0) & (g_lat <= 90.0))
g_idx_invalid = np.where(((g_lat < -90.0) | (g_lat > 90.0)))
goes_indices = np.full(g_lat.shape, -1, dtype=np.int64)
goes_indices[g_idx_valid] = ps.from_latlon(
g_lat[g_idx_valid], g_lon[g_idx_valid],
int(gd.resolution(goes_ds['elemRes'][0])))
##### Allocate MERRA-2 arrays co-aligned with GOES
m2_src_coord_h5 = np.full(g_lat.shape, -1, dtype=np.int64)
m2_tpw_h5 = np.full(g_lat.shape, -1, dtype=np.int64)
#####
join_resolution = m2_resolution
join = SortedDict()
ktr = 0
for k in range(len(g_idx_valid[0])):
id = g_idx_valid[0][k]
jk = gd.spatial_clear_to_resolution(
gd.spatial_coerce_resolution(goes_indices[id], join_resolution))
if jk not in join.keys():
join[jk] = join_value()
join[jk].add(goes_bandname, id)
ktr = ktr + 1
# if ktr > 10:
# break
# # exit();
for k in range(len(m2_indices)):
jk = gd.spatial_clear_to_resolution(m2_indices[k])
if jk not in join.keys():
join[jk] = join_value()
join[jk].add('m2', k)
###########################################################################
tpw_scale = 0.001
tpw_offset = 0
###########################################################################
##### JOIN
jkeys = join.keys()
ktr = 0
nktr = len(jkeys) # gd_idx_valid is a tuple with one element
dktr = nktr / 10.0
elements_pushed = 0
print('Push joined m2 data into the dataset n = ', nktr)
for k in range(nktr):
ktr = ktr + 1
if (ktr % int(dktr)) == 0:
print(int((10.0 * ktr) / dktr), '% complete, ', elements_pushed,
' elements pushed.')
sid = jkeys[k]
if join[sid].contains(goes_bandname):
if join[sid].contains('m2'):
m2s = join[sid].get('m2')[0] # Grab the first one
m2_src_coord_h5[join[sid].get(goes_bandname)] = m2s
# m2_tpw_h5[join[sid].get(goes_bandname)] = (m2_data_flat[m2s]-tpw_offset)/tpw_scale
avg = (np.mean(m2_data_flat[join[sid].get('m2')]) -
tpw_offset) / tpw_scale
m2_tpw_h5[join[sid].get(goes_bandname)] = avg
elements_pushed = elements_pushed + len(
join[sid].get(goes_bandname))
###########################################################################
##### HDF5 SAVE DATASET
workFile = h5.File(workFileName, 'w')
image_ds = workFile.create_dataset('image', [goes_ds['data'].size],
dtype=image_dtype)
image_description_ds = workFile.create_dataset(
'image_description', [], dtype=image_description_dtype)
m2_description_ds = workFile.create_dataset('merra2_description', [],
dtype=m2_description_dtype)
workFile['/image']['stare_spatial'] = goes_indices[:]
workFile['/image']['stare_temporal'] = gd.goes10_img_stare_time(goes_ds)[0]
workFile['/image']['goes_src_coord'] = np.arange(g_lat.size,
dtype=np.int64)
workFile['/image']['merra2_src_coord'] = m2_src_coord_h5.flatten()
workFile['/image']['merra2_tpw'] = m2_tpw_h5.flatten()
# oops
# workFile['/image_description']['nx'] = goes_ds['data'].shape[1]
# workFile['/image_description']['ny'] = goes_ds['data'].shape[2]
workFile['/image_description']['nx'] = goes_ds['data'].shape[2]
workFile['/image_description']['ny'] = goes_ds['data'].shape[1]
print('image nx,ny: ', goes_ds['data'].shape[2], goes_ds['data'].shape[1])
workFile['/merra2_description']['nx'] = 576
workFile['/merra2_description']['ny'] = 361
workFile['/merra2_description']['tpw_offset'] = tpw_offset
workFile['/merra2_description']['tpw_scale'] = tpw_scale
# workFile['/image']['goes_b3'] = goes_ds['data'][0,:,:].flatten()
# workFile['/image']['goes_b4'] = goes_ds['data'][0,:,:].flatten()
while igoes < len(goes_filenames_valid):
print(i, ' saving ', goes_bandname, ' from file ',
goes_filenames_valid[igoes])
workFile['/image'][goes_bandname] = goes_ds['data'][0, :, :].flatten()
goes_ds.close()
igoes = igoes + 1
# Assume remaining GOES bands have the same image sizes and locations.
if igoes < len(goes_filenames_valid):
goes_band = goes_filenames_valid[igoes].split('.')[4]
goes_ds = Dataset(goes_datapath + goes_filenames_valid[igoes])
goes_bandname = goes_bandnames[goes_band]
workFile.close()
return
def test_fromlatlon(self):
lat = numpy.array([30,45,60], dtype=numpy.double)
lon = numpy.array([45,60,10], dtype=numpy.double)
indices = pystare.from_latlon(lat, lon, 12)
expected = numpy.array([4151504989081014892, 4161865161846704588, 3643626718498217164])
numpy.testing.assert_array_equal(indices, expected)
def main():
###########################################################################
# Data source
dataPath="/home/mrilee/data/"
###########################################################################
# MODIS
modis_base = "MOD05_L2."
# modis_item = "A2005349.2120.061.2017294065852"
# modis_time_start = "2005-12-15T21:20:00"
modis_item = "A2005349.2125.061.2017294065400"
modis_time_start = "2005-12-15T21:25:00"
modis_suffix = ".hdf"
modis_filename = modis_base+modis_item+modis_suffix
fmt_suffix = ".h5"
workFileName = "sketchG."+modis_base+modis_item+fmt_suffix
print('loading ',workFileName)
workFile = h5.File(workFileName,'r')
sids = workFile['/image']['stare_spatial']
lat = workFile['/image']['Latitude']
lon = workFile['/image']['Longitude']
data = workFile['/image']['Water_Vapor_Near_Infrared']
workFile.close()
modis_min = np.amin(data)
modis_max = np.amax(data)
sids = sids-1
###########################################################################
# MODIS HDF
hdf = SD(dataPath+modis_filename,SDC.READ)
# ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared')
modis_hdf_resolution = 7
ntri_max = 1000
# hull = ps.to_hull_range_from_latlon(gring_lat[gring_seq],gring_lon[gring_seq],resolution,ntri_max)
modis_hdf_hull = gd.modis_cover_from_gring(hdf,modis_hdf_resolution,ntri_max)
hdf.end()
modis_hdf_lons,modis_hdf_lats,modis_hdf_intmat = ps.triangulate_indices(modis_hdf_hull)
modis_hdf_triang = tri.Triangulation(modis_hdf_lons,modis_hdf_lats,modis_hdf_intmat)
###########################################################################
# GOES
goes_file='sketch9.2005.349.213015.h5'
workFileName = goes_file
workFile = h5.File(workFileName,'r')
goes_sids = workFile['/image']['stare_spatial']
goes_data = workFile['/image']['goes_b3']
workFile.close()
print('goes mnmx: ',np.amin(goes_data),np.amax(goes_data))
goes_min = np.amin(goes_data)
goes_max = np.amax(goes_data)
goes_sids = goes_sids-1
goes_sids_005 = np.unique(gd.spatial_clear_to_resolution((goes_sids[np.where(goes_sids>0)] & ~31)+5))
goes_sids_005_geom = sid_geometry(goes_sids_005)
goes_sids_005_triang = goes_sids_005_geom.triang()
###########################################################################
# Plotting an ROI
cover_rads =[2.0,0,0, 0.125,0,0]
circle_color=[ 'Lime' ,'lightgrey' ,'White' ,'navajowhite' ,'khaki' ,'White' ]
###########################################################################
# HI 28.5N 177W
cover_resolution = 11
cover_lat = 19.5-0.375
cover_lon = -155.5+0.375
###########################################################################
# Plotting
nrows = 1
ncols = 1
# proj = ccrs.PlateCarree()
proj = ccrs.PlateCarree(central_longitude=-160.0)
# proj = ccrs.Mollweide()
# proj = ccrs.Mollweide(central_longitude=-160.0)
transf = ccrs.Geodetic()
fig,axs = plt.subplots(nrows=nrows,ncols=ncols,subplot_kw={'projection': proj})
subplot_title = ['GOES and MODIS Granules']
ax = axs
distribute_to_nodes = False
if True:
iter = 0
ax.set_xlim(-160-45,160+45)
if subplot_title[iter] is not None:
ax.set_title(subplot_title[iter])
if False:
ax.set_global()
if True:
ax.coastlines()
if distribute_to_nodes:
vmin=0; vmax=16
# colors=np.arange(goes_sids_005_triang.x.shape[0]/3,dtype=np.int64)%(vmax-vmin)
colors=np.random.uniform(vmin,vmax,size=int(goes_sids_005_triang.x.shape[0]/3)).astype(np.int64)
ax.tripcolor(goes_sids_005_triang
,facecolors=colors
,edgecolors='k',lw=0
,shading='flat'
,vmin=vmin,vmax=vmax,cmap='rainbow',alpha=0.65
,transform=transf)
ax.triplot(goes_sids_005_triang ,'r-',transform=transf,lw=1.0,markersize=3,alpha=0.5)
ax.triplot(modis_hdf_triang ,'b-',transform=transf,lw=1.0,markersize=3,alpha=0.5)
if True:
phi=np.linspace(0,2*np.pi,64)
cover_rad = cover_rads[iter]
rad=cover_rad
# rad=0.125
ax.plot(cover_lon+rad*np.cos(phi),cover_lat+rad*np.sin(phi),transform=transf,color=circle_color[iter],lw=2)
if iter == 0:
x0 = 0.05
y0 = 0.025; dy = 0.025
plt.figtext(x0,y0+0*dy
,"MODIS (blue): "+"sketchG."+modis_base+modis_item+fmt_suffix+', cover from GRING metadata, max resolution = 7'
,fontsize=10)
k=0;
while goes_sids[k]<0:
k=k+1
plt.figtext(x0,y0+1*dy
,"GOES (red): "+goes_file+', Northern Hemisphere, resolution = 5'
,fontsize=10)
plt.figtext(x0,y0+2*dy
,"Region of Interest near Hawaii, ROI (%s): radius = %0.2f degrees, center = 0x%016x"%(circle_color[iter],cover_rads[0],ps.from_latlon(npf64([cover_lat]),npf64([cover_lon]),cover_resolution)[0])
,fontsize=10)
if distribute_to_nodes:
plt.figtext(x0,y0+3*dy
,"Color tiles show distribution across 16 nodes."
,fontsize=10)
plt.show()
def main():
###########################################################################
# Data source
dataPath = "/home/mrilee/data/"
###########################################################################
# MODIS
modis_base = "MOD05_L2."
# modis_item = "A2005349.2120.061.2017294065852"
# modis_time_start = "2005-12-15T21:20:00"
modis_item = "A2005349.2125.061.2017294065400"
modis_time_start = "2005-12-15T21:25:00"
modis_suffix = ".hdf"
modis_filename = modis_base + modis_item + modis_suffix
# hdf = SD(dataPath+modis_filename,SDC.READ)
# ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared')
fmt_suffix = ".h5"
workFileName = "sketchG." + modis_base + modis_item + fmt_suffix
print('loading ', workFileName)
workFile = h5.File(workFileName, 'r')
sids = workFile['/image']['stare_spatial']
lat = workFile['/image']['Latitude']
lon = workFile['/image']['Longitude']
data = workFile['/image']['Water_Vapor_Near_Infrared']
workFile.close()
modis_min = np.amin(data)
modis_max = np.amax(data)
sids = sids - 1
###########################################################################
# GOES
goes_file = 'sketch9.2005.349.213015.h5'
workFileName = goes_file
workFile = h5.File(workFileName, 'r')
goes_sids = workFile['/image']['stare_spatial']
goes_data = workFile['/image']['goes_b3']
workFile.close()
print('goes mnmx: ', np.amin(goes_data), np.amax(goes_data))
goes_min = np.amin(goes_data)
goes_max = np.amax(goes_data)
goes_sids = goes_sids - 1
###########################################################################
# Plotting
nrows = 2
ncols = 3
nrows = 1
ncols = 1
proj = ccrs.PlateCarree()
# proj = ccrs.Mollweide()
# proj = ccrs.Mollweide(central_longitude=-160.0)
transf = ccrs.Geodetic()
# https://stackoverflow.com/questions/33942233/how-do-i-change-matplotlibs-subplot-projection-of-an-existing-axis
# plt.figure()
fig, axs = plt.subplots(nrows=nrows,
ncols=ncols,
subplot_kw={'projection': proj})
# axs.set_facecolor('k')
# axs.patch.set_facecolor('black')
# axs.set_facecolor('black')
if nrows * ncols == 1:
fig = [fig]
axs = [axs]
goes_line = [False, False, False]
modis_line = [False, False, False]
cover_plot = [True, True, True]
goes_plot_1 = [True, False, True]
goes_plot_1_points = [True, False, True]
modis_plot_1 = [False, True, True]
plt_show_1 = [False, False, True]
goes_line = [False, False, False, True, False, True]
modis_line = [False, False, False, False, True, True]
cover_plot = [False, False, False, False, False, False]
goes_plot_1 = [True, False, True, True, False, True]
goes_plot_1_points = [False, False, False, True, False, True]
modis_plot_1 = [False, True, True, False, True, True]
modis_plot_1_points = [False, False, False, False, True, True]
plt_show_1 = [False, False, True, False, False, True]
irow = [0, 0, 0, 1, 1, 1]
icol = [0, 1, 2, 0, 1, 2]
coastline_color = 'black'
coastline_color = 'black'
# blend
blend_tripcolor_1 = True
blend_tripcolor_1_res = 10
# blend_tripcolor_1_res = 9 # FFE
# blend_tripcolor_1_res = 6 # Test
blend_tripcolor_1_cmap = None
blend_tripcolor_1_alpha = 1
blend_tripcolor_1_gamma_g = 0.65
blend_tripcolor_1_gamma_m = 0.65
if blend_tripcolor_1:
goes_plot_1 = [False] * 6
modis_plot_1 = [False] * 6
# coastline_color = 'white'
coastline_color = 'black'
# 2020-0125 pix 1
# goes_plot_1_res = 9
# modis_plot_1_res = 9
#
# goes_plot_1_res = 6
# modis_plot_1_res = 6
#
# plot_1_res = 9 # FFE
plot_1_res = 6
goes_plot_1_res = plot_1_res
modis_plot_1_res = plot_1_res
# Colors
goes_plot_1_tripcolor = 'Reds'
modis_plot_1_tripcolor = 'Blues'
#
common_alpha = 0.7
goes_plot_1_alpha = common_alpha
modis_plot_1_alpha = common_alpha
# recalculate=[True,False,False,True,False,False]
recalculate = [True, False, True, True, False, True]
cover_rads = [2.0, 0, 2, 0.125, 0, 0.125]
# cover_rads =[2.0,0,0, 0.125,0,0]
circle_plot = [False, False, False, False, False, False]
circle_color = [
'White', 'lightgrey', 'White', 'navajowhite', 'khaki', 'White'
]
modis_scatter_color = [
'darkcyan', 'darkcyan', 'darkcyan', 'darkcyan', 'cyan', 'cyan'
]
nodes_cover = [1, 2, 1, 1, 2, 1] # 1 == goes, 2 == modis, 0 == None
# nodes_cover=[0,0,0,0,0,0]
subplot_title = [
"ROI+GOES", "ROI+MODIS", "ROI+GOES+MODIS", None, None, None
]
# for iter in range(6):
# for iter in [2,5]:
if True:
iter = 2
###########################################################################
if recalculate[iter]:
print('recalculating iter = ', iter)
###########################################################################
cover_resolution = 11
# cover_resolution = 12
cover_type = 'circular'
# cover_resolution = 6
#+ cover_resolution = 5
#+ cover_type = 'bounding_box'
if cover_type == 'circular':
###########################################################################
# HI 28.5N 177W
# Near the Big Island
cover_lat = 19.5 - 0.375
cover_lon = -155.5 + 0.375
# Midway Island
# cover_lat = 28.2
# cover_lon = -177.35
# Ni'ihau
# cover_lat = 21.9
# cover_lon = -160.17
cover_rad = cover_rads[iter]
cover = ps.to_circular_cover(cover_lat, cover_lon, cover_rad,
cover_resolution)
# ,range_size_limit=2000)
elif cover_type == 'bounding_box':
# Set cover to "bounding box."
cover_lat = np.array([15, 15, 38, 38], dtype=np.float)
cover_lon = np.array([-174, -145, -145, -174], dtype=np.float)
cover = ps.to_hull_range_from_latlon(cover_lat, cover_lon,
cover_resolution)
cover_cat = catalog(resolution=cover_resolution, sids=cover)
cover_sids_min = np.amin(cover)
cover_sids_max = np.amax(cover) # Need to convert to terminator
cover_sids_max = gd.spatial_terminator(cover_sids_max)
# for k in list(cover_cat.sdict.keys()):
# print('cc: ',hex(k))
###########################################################################
#
gm_cat_resolution = 5
gm_catalog = catalog(resolution=gm_cat_resolution)
k = 0
for i in range(10):
while (goes_sids[k] < 0):
k = k + 1
# print('adding: ','0x%016x'%goes_sids[k],k)
gm_catalog.add('goes', goes_sids[k], goes_data[k])
k = k + 1
for i in range(10):
# print('adding: ','0x%016x'%sids[i])
gm_catalog.add('modis', sids[i], data[i])
k = 0
# for i in range(10):
idx = np.arange(
goes_sids.size)[np.where((goes_sids > cover_sids_min)
& (goes_sids < cover_sids_max))]
for k in range(len(idx)):
# while(goes_sids[k]<0):
# k=k+1
if goes_sids[idx[k]] > 0:
cover_cat.add_to_entry('goes', goes_sids[idx[k]],
goes_data[idx[k]])
# k=k+1
idx = np.arange(sids.size)[np.where((sids > cover_sids_min)
& (sids < cover_sids_max))]
for k in range(len(idx)):
if sids[idx[k]] > 0:
cover_cat.add_to_entry('modis', sids[idx[k]], data[idx[k]])
# print(yaml.dump(gm_catalog))
# exit()
#
###########################################################################
print('plotting iter ', iter)
if nrows * ncols == 1:
ax = axs[0]
else:
ax = axs[irow[iter], icol[iter]]
if subplot_title[iter] is not None:
ax.set_title(subplot_title[iter])
if False:
ax.set_global()
if True:
ax.coastlines(color=coastline_color)
if iter == 0:
x0 = 0.05
y0 = 0.025
dy = 0.025
plt.figtext(x0,
y0 + 0 * dy,
"MODIS: " + "sketchG." + modis_base + modis_item +
fmt_suffix +
', Water_Vapor_Near_Infrared, resolution = %i' %
(sids[10000] & 31),
fontsize=10)
k = 0
while goes_sids[k] < 0:
k = k + 1
plt.figtext(x0,
y0 + 1 * dy,
"GOES: " + goes_file +
' BAND_3 (6.7mu), resolution = %i' %
(goes_sids[k] & 31),
fontsize=10)
if cover_type == 'circular':
plt.figtext(
x0,
y0 + 2 * dy,
"ROI Cover: resolution = %d, radius = %0.2f (upper) %0.3f (lower) degrees, center = 0x%016x"
% (cover_resolution, cover_rads[0], cover_rads[3],
ps.from_latlon(npf64([cover_lat]), npf64([cover_lon]),
cover_resolution)[0]),
fontsize=10)
# plt.show()
# exit()
if False:
lli = ps.triangulate_indices(cover)
ax.triplot(tri.Triangulation(lli[0], lli[1], lli[2]),
'g-',
transform=transf,
lw=1,
markersize=3)
if True:
if goes_plot_1[iter]:
cc_data = cover_cat.get_all_data('goes')
csids, sdat = zip(*[cd.as_tuple() for cd in cc_data])
if goes_plot_1_points[iter]:
glat, glon = ps.to_latlon(csids)
# csids_at_res = list(map(gd.spatial_clear_to_resolution,csids))
# cc_data_accum = dict()
# for cs in csids_at_res:
# cc_data_accum[cs] = []
# for ics in range(len(csids_at_res)):
# cc_data_accum[csids_at_res[ics]].append(sdat[ics])
# for cs in cc_data_accum.keys():
# if len(cc_data_accum[cs]) > 1:
# cc_data_accum[cs] = [sum(cc_data_accum[cs])/(1.0*len(cc_data_accum[cs]))]
# tmp_values = np.array(list(cc_data_accum.values()))
# vmin = np.amin(tmp_values)
# vmax = np.amax(np.array(tmp_values))
cc_data_accum, vmin, vmax = gd.simple_collect(
csids, sdat, force_resolution=goes_plot_1_res)
# print('a100: ',cc_data)
# print('cc_data type: ',type(cc_data))
# print('cc_data[0] type: ',type(cc_data[0]))
for cs in cc_data_accum.keys():
# print('item: ',hex(cs),cc_data_accum[cs])
lli = ps.triangulate_indices([cs])
triang = tri.Triangulation(lli[0], lli[1], lli[2])
cd_plt = np.array(cc_data_accum[cs])
# print('cd_plt type ',type(cd_plt))
# print('cd_plt shape ',cd_plt.shape)
# print('cd_plt type ',type(cd_plt[0]))
if goes_line[iter]:
ax.triplot(triang,
'r-',
transform=transf,
lw=1.5,
markersize=3,
alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Reds',alpha=0.4)
ax.tripcolor(triang,
facecolors=cd_plt,
edgecolors='k',
lw=0,
shading='flat',
vmin=vmin,
vmax=vmax,
cmap=goes_plot_1_tripcolor,
alpha=goes_plot_1_alpha)
# for cd in cc_data:
# lli = ps.triangulate_indices([cd.sid])
# triang = tri.Triangulation(lli[0],lli[1],lli[2])
# cd_plt = np.array([cd.datum])
# if goes_line[iter]:
# ax.triplot(triang,'r-',transform=transf,lw=3,markersize=3,alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Reds',alpha=0.4)
if modis_plot_1[iter]:
cc_data_m = cover_cat.get_all_data('modis')
csids, sdat = zip(*[cd.as_tuple() for cd in cc_data_m])
# mlat,mlon = ps.to_latlon(csids)
cc_data_m_accum, vmin, vmax = gd.simple_collect(
csids, sdat, force_resolution=modis_plot_1_res)
for cs in cc_data_m_accum.keys():
lli = ps.triangulate_indices([cs])
triang = tri.Triangulation(lli[0], lli[1], lli[2])
cd_plt = np.array(cc_data_m_accum[cs])
# print('lli[0] len ',len(lli[0]))
# print('cd_plt len ', len(cd_plt))
# print('cd_plt type ',type(cd_plt))
# print('cd_plt shape ',cd_plt.shape)
# print('cd_plt type ',type(cd_plt[0]))
if modis_line[iter]:
ax.triplot(triang,
'b-',
transform=transf,
lw=1.5,
markersize=3,
alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Blues',alpha=0.4)
ax.tripcolor(triang,
facecolors=cd_plt,
edgecolors='k',
lw=0,
shading='flat',
vmin=vmin,
vmax=vmax,
cmap=modis_plot_1_tripcolor,
alpha=modis_plot_1_alpha)
# for cd in cc_data_m:
# lli = ps.triangulate_indices([cd.sid])
# triang = tri.Triangulation(lli[0],lli[1],lli[2])
# cd_plt = np.array([cd.datum])
# if modis_line[iter]:
# ax.triplot(triang,'b-',transform=transf,lw=1,markersize=3,alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=modis_min,vmax=modis_max,cmap='Blues',alpha=0.4)
if modis_plot_1_points[iter]:
mlat, mlon = ps.to_latlon(csids)
ax.scatter(mlon, mlat, s=8, c=modis_scatter_color[iter])
# ax.scatter(mlon,mlat,s=8,c='cyan')
# ax.scatter(mlon,mlat,s=8,c='darkcyan')
# blend_tripcolor_1 = False
# blend_tripcolor_res_1 = 6
# blend_tripcolor_1_cmap = None
# blend_tripcolor_1_alpha = 1
if blend_tripcolor_1:
cc_data = cover_cat.get_all_data('goes')
csids, sdat = zip(*[cd.as_tuple() for cd in cc_data])
cc_data_accum, vmin, vmax = gd.simple_collect(
csids, sdat, force_resolution=blend_tripcolor_1_res)
cc_data_m = cover_cat.get_all_data('modis')
csids_m, sdat_m = zip(*[cd.as_tuple() for cd in cc_data_m])
cc_data_m_accum, vmin_m, vmax_m = gd.simple_collect(
csids_m, sdat_m, force_resolution=blend_tripcolor_1_res)
data_accum_keys = set()
for cs in cc_data_accum.keys():
data_accum_keys.add(cs)
for cs in cc_data_m_accum.keys():
data_accum_keys.add(cs)
for cs in data_accum_keys:
# print('item: ',hex(cs),cc_data_accum[cs])
lli = ps.triangulate_indices([cs])
triang = tri.Triangulation(lli[0], lli[1], lli[2])
try:
cd_plt_g = (np.array(cc_data_accum[cs]) -
vmin) / (vmax - vmin)
cd_plt_g = cd_plt_g**blend_tripcolor_1_gamma_g
except:
cd_plt_g = np.array([0])
try:
cd_plt_m = (np.array(cc_data_m_accum[cs]) -
vmin_m) / (vmax_m - vmin_m)
cd_plt_m = cd_plt_m**blend_tripcolor_1_gamma_m
except:
cd_plt_m = np.array([0])
######
# blend 1 & 2
# cd_plt = np.array([[cd_plt_g,0,cd_plt_m]])
######
# blend 3
# print('len: ',cd_plt_g.shape,cd_plt_m.shape)
cd_plt = np.array([[
cd_plt_g[0], 0.5 * (cd_plt_g + cd_plt_m)[0],
cd_plt_m[0]
]])
cd_cmp = colors_to_cmap(cd_plt)
zs = np.asarray(range(3), dtype=np.float) / 2.0
ax.tripcolor(
triang,
zs,
cmap=cd_cmp
# ,facecolors=cd_plt
,
edgecolors='k',
lw=0,
shading='gouraud'
# ,shading='flat'
# ,vmin=vmin,vmax=vmax
# ,cmap=blend_tripcolor_1_cmap
,
alpha=blend_tripcolor_1_alpha)
# ,vmin=vmin,vmax=vmax,cmap=blend_tripcolor_1_cmap,alpha=blend_tripcolor_1_alpha)
if goes_plot_1[iter]:
if goes_plot_1_points[iter]:
ax.scatter(glon, glat, s=8, c='black')
if nodes_cover[iter] > 0:
if nodes_cover[iter] == 1:
cc_data_ = cover_cat.get_all_data('goes')
else:
cc_data_ = cover_cat.get_all_data('modis')
sids_, dat_ = zip(*[cd.as_tuple() for cd in cc_data_])
# print('sids_ len: ',len(sids_))
sids_test = gd.spatial_clear_to_resolution(
npi64([
gd.spatial_coerce_resolution(s, gm_cat_resolution)
for s in sids_
]))
# print('sids_tlen: ',len(sids_test))
if cover_type == 'circular':
print('cover: 0x%016x' %
ps.from_latlon(npf64([cover_lat]), npf64(
[cover_lon]), cover_resolution)[0])
geom_test = sid_geometry(sids_test)
for s in geom_test.triangles.keys():
print(iter, ' 0x%016x' % s)
triang_test = geom_test.triang()
# ax.triplot(triang_test,'g-',transform=transf,lw=1.0,markersize=3,alpha=0.75)
ax.triplot(triang_test,
'k-',
transform=transf,
lw=1.0,
markersize=3,
alpha=0.5)
if False:
for i in range(0, 10):
k = cover_cat.sdict.peekitem(i)[0]
triang = cover_cat.sdict[k].geometry.triang()
ax.triplot(triang,
'b-',
transform=transf,
lw=1,
markersize=3,
alpha=0.5)
if cover_plot[iter]:
# lli = ps.triangulate_indices(ps.expand_intervals(cover,9,result_size_limit=2048))
lli = ps.triangulate_indices(cover)
ax.triplot(tri.Triangulation(lli[0], lli[1], lli[2]),
'k-',
transform=transf,
lw=1,
markersize=3,
alpha=0.5)
# ,'g-',transform=transf,lw=1,markersize=3)
if False:
# k = gm_catalog.sdict.keys()[0]
# for k in gm_catalog.sdict.keys():
for i in range(0, 3):
k = gm_catalog.sdict.peekitem(i)[0]
triang = gm_catalog.sdict[k].geometry.triang()
ax.triplot(triang,
'r-',
transform=transf,
lw=1,
markersize=3)
if circle_plot[iter]:
phi = np.linspace(0, 2 * np.pi, 64)
# rad=cover_rad
rad = 0.125
ax.plot(cover_lon + rad * np.cos(phi),
cover_lat + rad * np.sin(phi),
transform=transf,
color=circle_color[iter])
# ax.set_facecolor('k')
if plt_show_1[iter]:
plt.show()
###########################################################################
#
# if False:
# sw_timer.stamp('triangulating')
# print('triangulating')
# client = Client()
# for lli_ in slam(client,ps.triangulate_indices,sids):
# sw_timer.stamp('slam iteration')
# print('lli_ type: ',type(lli_))
# lli = lli_.result()
# sw_timer.stamp('slam result')
# print('lli type: ',type(lli))
# triang = tri.Triangulation(lli[0],lli[1],lli[2])
# sw_timer.stamp('slam triang')
# plt.triplot(triang,'r-',transform=transf,lw=1.5,markersize=3,alpha=0.5)
# sw_timer.stamp('slam triplot')
#
# sw_timer.stamp('plt show')
# # lons,lats,intmat=ps.triangulate_indices(sids)
# # triang = tri.Triangulation(lons,lats,intmat)
# # plt.triplot(triang,'r-',transform=transf,lw=1.5,markersize=3)
#
# plt.show()
# client.close()
print(sw_timer.report_all())
