def get_projection(self):
""" Returns GeographicSystem and MapProjection instances from
lmatools.coordinateSystems corresponding
to the coordinate system specified by the metadata of the
first NetCDF file in self._filenames.
"""
from lmatools.coordinateSystems import GeographicSystem, MapProjection
geosys = GeographicSystem()
f = NetCDFFile(self._filenames[0])
# Surely someone has written an automated library to parse coordinate
# reference data from CF-compliant files.
if 'Lambert_Azimuthal_Equal_Area' in list(f.variables.keys()):
nc_proj = f.variables['Lambert_Azimuthal_Equal_Area']
proj_name = 'laea'
ctrlon, ctrlat = (nc_proj.longitude_of_projection_origin,
nc_proj.latitude_of_projection_origin,)
try:
ctralt = nc_proj.altitude_of_projection_origin
except AttributeError:
print("No altitude attribute in NetCDF projection data, setting to 0.0")
ctralt = 0.0
mapproj = MapProjection(proj_name, ctrLat = ctrlat, ctrLon=ctrlon,
lat_0=ctrlat, lon_0=ctrlon)
# print geosys.fromECEF(*mapproj.toECEF((0,0), (0,0), (0,0)))
return geosys, mapproj
else:
print("projection not found, assuming lat, lon grid")
return geosys, geosys
f.close()
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
"""
There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
"""
logger = logging.getLogger('FlashAutorunLogger')
if 'mask_length' in params:
mask_length = params['mask_length']
else:
mask_length = 4
lma=LMAdataFile(a_file, mask_length = mask_length)
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0
good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1])
if 'alt' in params:
good = good & (lma.alt < params['alt'][1])
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc
print "sorting {0} total points".format(data.shape[0])
D_max, t_max = 3.0e3, 0.15 # m, s
IDs = np.arange(X.shape[0])
X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector))
lma.sort_status = 'in process'
# Maximum 3 s flash length, normalized to the time separation scale
flash_object_maker = create_flash_objs(lma, data)
label_aggregator = aggregate_ids(flash_object_maker)
clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
chunker = chunk(XYZT[:,-1].min(), 3.0/.15, clusterer)
stream(XYZT.astype('float64'), IDs,chunker)
flash_object_maker.close()
# These are handled by target.close in each coroutine's GeneratorExit handler
# clusterer.close()
# label_aggregator.close()
# flash_object_maker.close()
print lma.sort_status
print len(lma.flash_objects)
return lma, lma.flash_objects
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
"""
There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
"""
logger = logging.getLogger('FlashAutorunLogger')
lma=LMAdataFile(a_file)
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0
good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1])
if 'alt' in params:
good = good & (lma.alt < params['alt'][1])
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc
print("sorting {0} total points".format(data.shape[0]))
D_max, t_max = params['distance'], params['thresh_critical_time'] # m, s
duration_max = params['thresh_duration']
IDs = np.arange(X.shape[0])
X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector))
# Maximum 3 s flash length, normalized to the time separation scale
flash_object_maker = create_flash_objs(lma, data)
label_aggregator = aggregate_ids(flash_object_maker)
clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
if XYZT.shape[0] < 1:
# no data, so minimum time is zero. Assume nothing is done with the data,
# so that time doesn't matter. No flashes can result.
chunker = chunk(0, duration_max/t_max, clusterer)
else:
chunker = chunk(XYZT[:,-1].min(), duration_max/t_max, clusterer)
stream(XYZT.astype('float64'), IDs,chunker)
flash_object_maker.close()
# These are handled by target.close in each coroutine's GeneratorExit handler
# clusterer.close()
# label_aggregator.close()
# flash_object_maker.close()
print(len(lma.flash_objects))
return lma, lma.flash_objects
def geo_to_cartesisan(self, lon, lat, alt):
""" Convert lat, lon in degrees and altitude in meters to
Earth-centered, Earth-fixed cartesian coordinates. Translate
to coordinate center location. Returns X,Y,Z in meters.
"""
geoCS = GeographicSystem()
X, Y, Z = geoCS.toECEF(lon, lat, alt)
Xc, Yc, Zc = geoCS.toECEF(self.ctr_lon, self.ctr_lat, self.ctr_alt)
X, Y, Z = X - Xc, Y - Yc, Z - Zc
return (X, Y, Z)
def get_GOESR_coordsys(sat_lon_nadir=-75.0):
"""
Values from the GOES-R PUG Volume 3, L1b data
Returns geofixcs, grs80lla: the fixed grid coordinate system and the
latitude, longitude, altitude coordinate system referenced to the GRS80
ellipsoid used by GOES-R as its earth reference.
"""
goes_sweep = 'x' # Meteosat is 'y'
ellipse = 'GRS80'
datum = 'WGS84'
sat_ecef_height = 35786023.0
geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
ellipse=ellipse,
datum=datum,
sweep_axis=goes_sweep,
sat_ecef_height=sat_ecef_height)
grs80lla = GeographicSystem(ellipse='GRS80', datum='WGS84')
return geofixcs, grs80lla
def test_fixed_grid_GOESR():
""" Tests from the GOES-R PUG Volume 3, L1b data """
sat_lon_nadir = -75.0
goes_sweep = 'x' # Meteosat is 'y'
ellipse = 'GRS80'
datum = 'WGS84'
sat_ecef_height = 35786023.0
geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
ellipse=ellipse,
datum=datum,
sweep_axis=goes_sweep,
sat_ecef_height=sat_ecef_height)
latloncs = GeographicSystem(ellipse=ellipse, datum=datum)
test_lat = 33.846162
test_lon = -84.690932
test_alt = 0.0
test_fixx = -0.024052
test_fixy = 0.095340
test_fixz = 0.0
atol = 1e-6
# Test forward from geodetic
X, Y, Z = latloncs.toECEF(test_lon, test_lat, test_alt)
x, y, z = geofixcs.fromECEF(X, Y, Z)
assert_allclose(test_fixx, x, rtol=atol)
assert_allclose(test_fixy, y, rtol=atol)
assert_allclose(test_fixz, z, rtol=atol)
# print(test_fixx, test_fixy, test_fixz)
# print(x,y,z)
# Test inverse from fixed grid angle
X, Y, Z = geofixcs.toECEF(test_fixx, test_fixy, test_fixz)
x, y, z = latloncs.fromECEF(X, Y, Z)
assert_allclose(test_lon, x, atol=atol)
assert_allclose(test_lat, y, atol=atol)
assert_allclose(test_alt, z, atol=atol)
import netCDF4
import numpy as np
import pandas as pd
import scipy as sci
# from scipy.spatial import *
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from matplotlib.widgets import Widget
from matplotlib.colors import LogNorm, Normalize
from lmatools.coordinateSystems import GeographicSystem, MapProjection
geosys = GeographicSystem()
from ipywidgets import widgets
# from IPython.display import HTML
# from IPython.display import Javascript
# from IPython.display import display
from lmatools.vis import ctables as ct
import logging, json
# from http://stackoverflow.com/questions/3488934/simplejson-and-numpy-array/24375113#24375113
class NumpyAwareJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, np.ndarray): # and obj.ndim == 1:
return obj.tolist()
return json.JSONEncoder.default(self, obj)
except GeneratorExit:
print(total)
#lma=LMAdataFile('/Users/ebruning/Documents/Lightning\ interpretation/Flash-length/Thomas/LYLOUT_120412_01817.exported.dat.gz')
#ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00
lma = LMAdataFile('/data/20040526/LMA/LYLOUT_040526_224000_0600.dat.gz')
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = 35.2791257, -97.9178678, 417.90 # OKLMA
#ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00 # COLMA
good = (lma.stations >= 6) & (lma.chi2 <= 2.0) & (lma.alt < 20e3)
data = lma.data[good]
geoCS = GeographicSystem()
X, Y, Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF(ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X - Xc, Y - Yc, Z - Zc
D_max, t_max = 3.0e3, 0.15 # m, s
X_vector = np.hstack((X[:, None], Y[:, None], Z[:, None])) / D_max
T_vector = data['time'][:, None] / t_max
XYZT = np.hstack((X_vector, T_vector - T_vector.min()))
# Maximum 3 s flash length, normalized to the time separation scale
chunker = chunk(XYZT[:, -1].min(), 3.0 / .15,
cluster_chunk_pairs(cluster_printer()))
stream(XYZT.astype('float32'), chunker)
def dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=4.0e3,
dy=4.0e3,
x_bnd=(-100e3, 100e3),
y_bnd=(-100e3, 100e3),
proj_datum='WGS84',
proj_ellipse='WGS84'):
"""
Utility function useful for producing a regular grid of lat/lon data,
where an approximate spacing (dx, dy) and total span of the grid (x_bnd, y_bnd)
is desired. Units are in meters.
There is guaranteed to be distortion away from the grid center, i.e.,
only the grid cells adjacent to the center location will have area dx * dy.
Likewise, the lat, lon range is calculated naively using dlat, dlon multiplied
by the number of grid cells implied by x_bnd/dx, y_bnd/dy. This is the naive approach,
but probably what's expected when specifying distances in kilometers for
an inherently distorted lat/lon grid.
Returns:
(dlon, dlat, lon_bnd, lat_bnd)
corresponding to
(dx, dy, x_range, y_range)
"""
# Use the Azimuthal equidistant projection as the method for converting to kilometers.
proj_name = 'aeqd'
mapProj = MapProjection(projection=proj_name,
ctrLat=ctr_lat,
ctrLon=ctr_lon,
lat_ts=ctr_lat,
lon_0=ctr_lon,
lat_0=ctr_lat,
lat_1=ctr_lat,
ellipse=proj_ellipse,
datum=proj_datum)
geoProj = GeographicSystem()
# Get dlat
lon_n, lat_n, z_n = geoProj.fromECEF(*mapProj.toECEF(0, dy, 0))
dlat = lat_n - ctr_lat
# Get dlon
lon_e, lat_e, z_e = geoProj.fromECEF(*mapProj.toECEF(dx, 0, 0))
dlon = lon_e - ctr_lon
lon_min = ctr_lon + dlon * (x_bnd[0] / dx)
lon_max = ctr_lon + dlon * (x_bnd[1] / dx)
lat_min = ctr_lat + dlat * (y_bnd[0] / dy)
lat_max = ctr_lat + dlat * (y_bnd[1] / dy)
# Alternate method: lat lon for the actual distance to the NSEW in the projection
#lon_range_n, lat_range_n, z_range_n = geoProj.fromECEF(*mapProj.toECEF(0,y_bnd,0))
#lon_range_e, lat_range_e, z_range_e = geoProj.fromECEF(*mapProj.toECEF(x_bnd,0,0))
return dlon, dlat, (lon_min, lon_max), (lat_min, lat_max)
def __init__(
self,
start_time,
end_time,
do_3d=True,
frame_interval=120.0,
dx=4.0e3,
dy=4.0e3,
dz=1.0e3,
base_date=None,
x_bnd=(-100e3, 100e3),
y_bnd=(-100e3, 100e3),
z_bnd=(0e3, 20e3),
ctr_lat=35.23833,
ctr_lon=-97.46028,
ctr_alt=0.0,
proj_name='aeqd',
proj_datum='WGS84',
proj_ellipse='WGS84',
pixel_coords=None,
flash_count_logfile=None,
energy_grids=None,
event_grid_area_fraction_key=None,
spatial_scale_factor=1.0 / 1000.0,
subdivide=False,
):
""" Class to support gridding of flash and event data.
On init, specify the grid
If proj_name = 'pixel_grid' then pixel_coords must be an
instance of lmatools.coordinateSystems.PixelGrid
If proj_name = 'geos' then pixel_coords must be an instance of
lmatools.coordinateSystems.GeostationaryFixedGridSystem
energy_grids controls which energy grids are saved, default None.
energy_grids may be True, which will calculate all energy grid
types, or it may be one of 'specific_energy', 'total_energy',
or a list of one or more of these.
event_grid_area_fraction_key specifies the name of the variable
in the events array that gives the fraction of each grid cell
covered by each event. Used only for pixel-based event
detectors.
"""
if energy_grids == True:
energy_grids = ('specific_energy', 'total_energy')
self.energy_grids = energy_grids
self.spatial_scale_factor = spatial_scale_factor
self.event_grid_area_fraction_key = event_grid_area_fraction_key
# args, kwargs that are saved for the future
self.do_3d = do_3d
self.start_time = start_time
self.dx, self.dy, self.dz = dx, dy, dz
self.end_time = end_time
self.frame_interval = frame_interval
self.base_date = base_date
self.min_points_per_flash = 1
self.proj_name = proj_name
self.ctr_lat, self.ctr_lon, self.ctr_alt = ctr_lat, ctr_lon, ctr_alt
if flash_count_logfile is None:
flash_count_logfile = log
self.flash_count_logfile = flash_count_logfile
t_edges, duration = time_edges(self.start_time, self.end_time,
self.frame_interval)
# reference time is the date part of the start_time, unless the user provides a different date.
if self.base_date is None:
t_ref, t_edges_seconds = seconds_since_start_of_day(
self.start_time, t_edges)
else:
t_ref, t_edges_seconds = seconds_since_start_of_day(
self.base_date, t_edges)
self.n_frames = len(t_edges) - 1
xedge = np.arange(x_bnd[0], x_bnd[1] + dx, dx)
yedge = np.arange(y_bnd[0], y_bnd[1] + dy, dy)
zedge = np.arange(z_bnd[0], z_bnd[1] + dz, dz)
if self.proj_name == 'latlong':
dx_units = '{0:6.4f}deg'.format(float(dx))
mapProj = GeographicSystem()
elif self.proj_name == 'pixel_grid':
dx_units = 'pixel'
mapProj = pixel_coords
elif self.proj_name == 'geos':
dx_units = '{0:03d}urad'.format(int(dx * 1e6))
mapProj = pixel_coords
else:
dx_units = '{0:5.1f}m'.format(float(dx))
mapProj = MapProjection(projection=self.proj_name,
ctrLat=ctr_lat,
ctrLon=ctr_lon,
lat_ts=ctr_lat,
lon_0=ctr_lon,
lat_0=ctr_lat,
lat_1=ctr_lat,
ellipse=proj_ellipse,
datum=proj_datum)
geoProj = GeographicSystem()
self.t_ref = t_ref
self.t_edges = t_edges
self.t_edges_seconds = t_edges_seconds
self.duration = duration
self.xedge = xedge
self.yedge = yedge
self.zedge = zedge
self.mapProj = mapProj
self.geoProj = geoProj
self.dx_units = dx_units
self.pipeline_setup()
self.output_setup()
def get_geometry_hrrr(glm, nadir, hrrr, ltgellipsever=1):
# We don't use X, Y, Z below.
# x, y are the fixed grid 2D coord arrays (from meshgrid)
# X, Y, Z are the ECEF coords of each pixel intersected at the ltg ellipsoid
# lon_ltg, lat_ltg are the parallax-corrected lon lat at the earth's surface
# below the lightning position on the lightning ellipsoid.
# outside_glm_full_disk is a boolean mask for the positions that GLM can't
# observe.
# All of the above are 2D arrays corrsponding the center positions of the 2
# km fixed grid pixels in the GLM gridded products.
((x, y), (X, Y, Z), (lon_ltg, lat_ltg, alt_ltg),
outside_glm_full_disk) = get_glm_earth_geometry(glm, nadir, ltgellipsever)
### HRRR interpolation ###
# Everything below presumes LCC.
assert hrrr.MAP_PROJ_CHAR == 'Lambert Conformal'
corner_0_lla = (hrrr.XLONG[0, 0, 0].data, hrrr.XLAT[0, 0, 0].data,
np.asarray(0.0, dtype=hrrr.XLAT[0, 0, 0].dtype))
corner_1_lla = (hrrr.XLONG[0, -1, -1].data, hrrr.XLAT[0, -1, -1].data,
np.asarray(0.0, dtype=hrrr.XLAT[0, 1, -1].dtype))
hrrr_dx, hrrr_dy = hrrr.DX, hrrr.DX
hrrr_Nx, hrrr_Ny = hrrr.dims['west_east'], hrrr.dims['south_north']
hrrrproj = {
'lat_0': hrrr.CEN_LAT,
'lon_0': hrrr.CEN_LON + 360.0,
'lat_1': hrrr.TRUELAT1,
'lat_2': hrrr.TRUELAT2,
# 'R':hrrr.LambertConformal_Projection.earth_radius,
# 'a':6371229,
# 'b':6371229,
'R': 6371229,
}
lcc = MapProjection(projection='lcc',
ctrLat=hrrrproj['lat_0'],
ctrLon=hrrrproj['lon_0'],
**hrrrproj)
hrrr_lla = GeographicSystem(r_equator=hrrrproj['R'], r_pole=hrrrproj['R'])
lcc_cornerx_0, lcc_cornery_0, lcc_cornerz_0 = lcc.fromECEF(
*hrrr_lla.toECEF(*corner_0_lla))
lcc_cornerx_1, lcc_cornery_1, lcc_cornerz_1 = lcc.fromECEF(
*hrrr_lla.toECEF(*corner_1_lla))
# def grid_idx(x, y, x0, y0, dx, dy):
# """
# Convert x, y returned by [projection].fromECEF to the grid index in the
# NetCDF file. x0 and y0 are the [projection] coordinates of the center of
# the zero-index position in the NetCDF grid. dx and dy are the grid spacing
# in meters.
#
# returns (xidx, yidx)
# Taking int(xidx) will give the zero-based grid cell index.
# """
# # To get the correct behavior with int(xidx), add a half
# # since x0 is the center.
# xidx = (x-x0)/dx + 0.5
# yidx = (y-y0)/dy + 0.5
# return xidx, yidx
# The 3D position (X,Y,Z) defines an implicit lon, lat, alt with respect to the
# spherical earth we specified for the HRRR and its associated Lambert
# conformal projection. We let proj4 handle the mapping from the ECEF
# coordinates (an absolute position) directly to LCC.
lccx2, lccy2, lccz2 = lcc.fromECEF(
*hrrr_lla.toECEF(lon_ltg, lat_ltg, np.zeros_like(lon_ltg)))
lccx2.shape = x.shape
lccy2.shape = x.shape
lccx = lccx2
lccy = lccy2
# Set up the model grid, since the hrrr file doesn't include those values.
hrrrx_1d = np.arange(hrrr_Nx, dtype='f4') * hrrr_dx + lcc_cornerx_0
hrrry_1d = np.arange(hrrr_Ny, dtype='f4') * hrrr_dy + lcc_cornery_0
hrrrx, hrrry = np.meshgrid(hrrrx_1d, hrrry_1d)
interp_loc = np.vstack((hrrrx.flatten(), hrrry.flatten())).T
# GLM variables are filled with nan everywhere there is no lightning,
# so set those locations corresponding to valid earth locations to zero.
lcc_glm_x_flat = lccx[:, :].flatten()
lcc_glm_y_flat = lccy[:, :].flatten()
good = np.isfinite(lcc_glm_x_flat) & np.isfinite(lcc_glm_y_flat)
good = good & (~outside_glm_full_disk.flatten())
data_loc = np.vstack((lcc_glm_x_flat[good], lcc_glm_y_flat[good])).T
return (data_loc, good, interp_loc, hrrrx.shape)
