def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
"""
There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
"""
logger = logging.getLogger('FlashAutorunLogger')
if 'mask_length' in params:
mask_length = params['mask_length']
else:
mask_length = 4
lma=LMAdataFile(a_file, mask_length = mask_length)
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0
good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1])
if 'alt' in params:
good = good & (lma.alt < params['alt'][1])
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc
print "sorting {0} total points".format(data.shape[0])
D_max, t_max = params['distance'], params['thresh_critical_time'] # m, s
duration_max = params['thresh_duration']
IDs = np.arange(X.shape[0])
X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector))
lma.sort_status = 'in process'
# Maximum 3 s flash length, normalized to the time separation scale
flash_object_maker = create_flash_objs(lma, data)
label_aggregator = aggregate_ids(flash_object_maker)
clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
chunker = chunk(XYZT[:,-1].min(), duration_max/t_max, clusterer)
stream(XYZT.astype('float64'), IDs,chunker)
flash_object_maker.close()
# These are handled by target.close in each coroutine's GeneratorExit handler
# clusterer.close()
# label_aggregator.close()
# flash_object_maker.close()
print lma.sort_status
print len(lma.flash_objects)
return lma, lma.flash_objects
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
"""
There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
"""
logger = logging.getLogger('FlashAutorunLogger')
if 'mask_length' in params:
mask_length = params['mask_length']
else:
mask_length = 4
lma=LMAdataFile(a_file, mask_length = mask_length)
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0
good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1])
if 'alt' in params:
good = good & (lma.alt < params['alt'][1])
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc
print "sorting {0} total points".format(data.shape[0])
D_max, t_max = 3.0e3, 0.15 # m, s
IDs = np.arange(X.shape[0])
X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector))
lma.sort_status = 'in process'
# Maximum 3 s flash length, normalized to the time separation scale
flash_object_maker = create_flash_objs(lma, data)
label_aggregator = aggregate_ids(flash_object_maker)
clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
chunker = chunk(XYZT[:,-1].min(), 3.0/.15, clusterer)
stream(XYZT.astype('float64'), IDs,chunker)
flash_object_maker.close()
# These are handled by target.close in each coroutine's GeneratorExit handler
# clusterer.close()
# label_aggregator.close()
# flash_object_maker.close()
print lma.sort_status
print len(lma.flash_objects)
return lma, lma.flash_objects
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
"""
There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
"""
logger = logging.getLogger('FlashAutorunLogger')
lma=LMAdataFile(a_file)
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0
good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1])
if 'alt' in params:
good = good & (lma.alt < params['alt'][1])
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc
print("sorting {0} total points".format(data.shape[0]))
D_max, t_max = params['distance'], params['thresh_critical_time'] # m, s
duration_max = params['thresh_duration']
IDs = np.arange(X.shape[0])
X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector))
# Maximum 3 s flash length, normalized to the time separation scale
flash_object_maker = create_flash_objs(lma, data)
label_aggregator = aggregate_ids(flash_object_maker)
clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
if XYZT.shape[0] < 1:
# no data, so minimum time is zero. Assume nothing is done with the data,
# so that time doesn't matter. No flashes can result.
chunker = chunk(0, duration_max/t_max, clusterer)
else:
chunker = chunk(XYZT[:,-1].min(), duration_max/t_max, clusterer)
stream(XYZT.astype('float64'), IDs,chunker)
flash_object_maker.close()
# These are handled by target.close in each coroutine's GeneratorExit handler
# clusterer.close()
# label_aggregator.close()
# flash_object_maker.close()
print(len(lma.flash_objects))
return lma, lma.flash_objects
def geo_to_cartesisan(self, lon, lat, alt):
""" Convert lat, lon in degrees and altitude in meters to
Earth-centered, Earth-fixed cartesian coordinates. Translate
to coordinate center location. Returns X,Y,Z in meters.
"""
geoCS = GeographicSystem()
X, Y, Z = geoCS.toECEF(lon, lat, alt)
Xc, Yc, Zc = geoCS.toECEF(self.ctr_lon, self.ctr_lat, self.ctr_alt)
X, Y, Z = X - Xc, Y - Yc, Z - Zc
return (X, Y, Z)
def dlonlat_at_grid_center(ctr_lat, ctr_lon, dx=4.0e3, dy=4.0e3,
x_bnd = (-100e3, 100e3), y_bnd = (-100e3, 100e3),
proj_datum = 'WGS84', proj_ellipse = 'WGS84'):
"""
Utility function useful for producing a regular grid of lat/lon data,
where an approximate spacing (dx, dy) and total span of the grid (x_bnd, y_bnd)
is desired. Units are in meters.
There is guaranteed to be distortion away from the grid center, i.e.,
only the grid cells adjacent to the center location will have area dx * dy.
Likewise, the lat, lon range is calculated naively using dlat, dlon multiplied
by the number of grid cells implied by x_bnd/dx, y_bnd/dy. This is the naive approach,
but probably what's expected when specifying distances in kilometers for
an inherently distorted lat/lon grid.
Returns:
(dlon, dlat, lon_bnd, lat_bnd)
corresponding to
(dx, dy, x_range, y_range)
"""
# Use the Azimuthal equidistant projection as the method for converting to kilometers.
proj_name = 'aeqd'
mapProj = MapProjection(projection=proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat,
lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum)
geoProj = GeographicSystem()
# Get dlat
lon_n, lat_n, z_n = geoProj.fromECEF(*mapProj.toECEF(0,dy,0))
dlat = lat_n - ctr_lat
# Get dlon
lon_e, lat_e, z_e = geoProj.fromECEF(*mapProj.toECEF(dx,0,0))
dlon = lon_e - ctr_lon
lon_min = ctr_lon + dlon * (x_bnd[0]/dx)
lon_max = ctr_lon + dlon * (x_bnd[1]/dx)
lat_min = ctr_lat + dlat * (y_bnd[0]/dy)
lat_max = ctr_lat + dlat * (y_bnd[1]/dy)
# Alternate method: lat lon for the actual distance to the NSEW in the projection
#lon_range_n, lat_range_n, z_range_n = geoProj.fromECEF(*mapProj.toECEF(0,y_bnd,0))
#lon_range_e, lat_range_e, z_range_e = geoProj.fromECEF(*mapProj.toECEF(x_bnd,0,0))
return dlon, dlat, (lon_min, lon_max), (lat_min, lat_max)
def get_projection(self):
""" Returns GeographicSystem and MapProjection instances from
lmatools.coordinateSystems corresponding
to the coordinate system specified by the metadata of the
first NetCDF file in self._filenames.
"""
from lmatools.coordinateSystems import GeographicSystem, MapProjection
geosys = GeographicSystem()
f = NetCDFFile(self._filenames[0])
# Surely someone has written an automated library to parse coordinate
# reference data from CF-compliant files.
if 'Lambert_Azimuthal_Equal_Area' in list(f.variables.keys()):
nc_proj = f.variables['Lambert_Azimuthal_Equal_Area']
proj_name = 'laea'
ctrlon, ctrlat = (nc_proj.longitude_of_projection_origin,
nc_proj.latitude_of_projection_origin,)
try:
ctralt = nc_proj.altitude_of_projection_origin
except AttributeError:
print("No altitude attribute in NetCDF projection data, setting to 0.0")
ctralt = 0.0
mapproj = MapProjection(proj_name, ctrLat = ctrlat, ctrLon=ctrlon,
lat_0=ctrlat, lon_0=ctrlon)
# print geosys.fromECEF(*mapproj.toECEF((0,0), (0,0), (0,0)))
return geosys, mapproj
else:
print("projection not found, assuming lat, lon grid")
return geosys, geosys
f.close()
def test_fixed_grid_GOESR():
""" Tests from the GOES-R PUG Volume 3, L1b data """
sat_lon_nadir = -75.0
goes_sweep = 'x' # Meteosat is 'y'
ellipse = 'GRS80'
datum = 'WGS84'
sat_ecef_height = 35786023.0
geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
ellipse=ellipse,
datum=datum,
sweep_axis=goes_sweep,
sat_ecef_height=sat_ecef_height)
latloncs = GeographicSystem(ellipse=ellipse, datum=datum)
test_lat = 33.846162
test_lon = -84.690932
test_alt = 0.0
test_fixx = -0.024052
test_fixy = 0.095340
test_fixz = 0.0
atol = 1e-6
# Test forward from geodetic
X, Y, Z = latloncs.toECEF(test_lon, test_lat, test_alt)
x, y, z = geofixcs.fromECEF(X, Y, Z)
assert_allclose(test_fixx, x, rtol=atol)
assert_allclose(test_fixy, y, rtol=atol)
assert_allclose(test_fixz, z, rtol=atol)
# print(test_fixx, test_fixy, test_fixz)
# print(x,y,z)
# Test inverse from fixed grid angle
X, Y, Z = geofixcs.toECEF(test_fixx, test_fixy, test_fixz)
x, y, z = latloncs.fromECEF(X, Y, Z)
assert_allclose(test_lon, x, atol=atol)
assert_allclose(test_lat, y, atol=atol)
assert_allclose(test_alt, z, atol=atol)
def test_fixed_grid_GOESR():
""" Tests from the GOES-R PUG Volume 3, L1b data """
sat_lon_nadir = -75.0
goes_sweep = 'x' # Meteosat is 'y'
ellipse = 'GRS80'
datum = 'WGS84'
sat_ecef_height=35786023.0
geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
ellipse=ellipse, datum=datum, sweep_axis=goes_sweep,
sat_ecef_height=sat_ecef_height)
latloncs = GeographicSystem(ellipse=ellipse, datum=datum)
test_lat = 33.846162
test_lon = -84.690932
test_alt = 0.0
test_fixx = -0.024052
test_fixy = 0.095340
test_fixz = 0.0
atol = 1e-6
# Test forward from geodetic
X,Y,Z = latloncs.toECEF(test_lon, test_lat, test_alt)
x, y, z = geofixcs.fromECEF(X, Y, Z)
assert_allclose(test_fixx, x, rtol=atol)
assert_allclose(test_fixy, y, rtol=atol)
assert_allclose(test_fixz, z, rtol=atol)
# print(test_fixx, test_fixy, test_fixz)
# print(x,y,z)
# Test inverse from fixed grid angle
X,Y,Z = geofixcs.toECEF(test_fixx, test_fixy, test_fixz)
x, y, z = latloncs.fromECEF(X, Y, Z)
assert_allclose(test_lon, x, atol=atol)
assert_allclose(test_lat, y, atol=atol)
assert_allclose(test_alt, z, atol=atol)
def get_GOESR_coordsys(sat_lon_nadir=-75.0):
"""
Values from the GOES-R PUG Volume 3, L1b data
Returns geofixcs, grs80lla: the fixed grid coordinate system and the
latitude, longitude, altitude coordinate system referenced to the GRS80
ellipsoid used by GOES-R as its earth reference.
"""
goes_sweep = 'x' # Meteosat is 'y'
ellipse = 'GRS80'
datum = 'WGS84'
sat_ecef_height = 35786023.0
geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
ellipse=ellipse,
datum=datum,
sweep_axis=goes_sweep,
sat_ecef_height=sat_ecef_height)
grs80lla = GeographicSystem(ellipse='GRS80', datum='WGS84')
return geofixcs, grs80lla
import netCDF4
import numpy as np
import pandas as pd
import scipy as sci
# from scipy.spatial import *
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from matplotlib.widgets import Widget
from matplotlib.colors import LogNorm, Normalize
from lmatools.coordinateSystems import GeographicSystem, MapProjection
geosys = GeographicSystem()
from ipywidgets import widgets
# from IPython.display import HTML
# from IPython.display import Javascript
# from IPython.display import display
from lmatools.vis import ctables as ct
import logging, json
# from http://stackoverflow.com/questions/3488934/simplejson-and-numpy-array/24375113#24375113
class NumpyAwareJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, np.ndarray): # and obj.ndim == 1:
return obj.tolist()
return json.JSONEncoder.default(self, obj)
except GeneratorExit:
print(total)
#lma=LMAdataFile('/Users/ebruning/Documents/Lightning\ interpretation/Flash-length/Thomas/LYLOUT_120412_01817.exported.dat.gz')
#ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00
lma = LMAdataFile('/data/20040526/LMA/LYLOUT_040526_224000_0600.dat.gz')
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = 35.2791257, -97.9178678, 417.90 # OKLMA
#ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00 # COLMA
good = (lma.stations >= 6) & (lma.chi2 <= 2.0) & (lma.alt < 20e3)
data = lma.data[good]
geoCS = GeographicSystem()
X, Y, Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF(ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X - Xc, Y - Yc, Z - Zc
D_max, t_max = 3.0e3, 0.15 # m, s
X_vector = np.hstack((X[:, None], Y[:, None], Z[:, None])) / D_max
T_vector = data['time'][:, None] / t_max
XYZT = np.hstack((X_vector, T_vector - T_vector.min()))
# Maximum 3 s flash length, normalized to the time separation scale
chunker = chunk(XYZT[:, -1].min(), 3.0 / .15,
cluster_chunk_pairs(cluster_printer()))
stream(XYZT.astype('float32'), chunker)
def dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=4.0e3,
dy=4.0e3,
x_bnd=(-100e3, 100e3),
y_bnd=(-100e3, 100e3),
proj_datum='WGS84',
proj_ellipse='WGS84'):
"""
Utility function useful for producing a regular grid of lat/lon data,
where an approximate spacing (dx, dy) and total span of the grid (x_bnd, y_bnd)
is desired. Units are in meters.
There is guaranteed to be distortion away from the grid center, i.e.,
only the grid cells adjacent to the center location will have area dx * dy.
Likewise, the lat, lon range is calculated naively using dlat, dlon multiplied
by the number of grid cells implied by x_bnd/dx, y_bnd/dy. This is the naive approach,
but probably what's expected when specifying distances in kilometers for
an inherently distorted lat/lon grid.
Returns:
(dlon, dlat, lon_bnd, lat_bnd)
corresponding to
(dx, dy, x_range, y_range)
"""
# Use the Azimuthal equidistant projection as the method for converting to kilometers.
proj_name = 'aeqd'
mapProj = MapProjection(projection=proj_name,
ctrLat=ctr_lat,
ctrLon=ctr_lon,
lat_ts=ctr_lat,
lon_0=ctr_lon,
lat_0=ctr_lat,
lat_1=ctr_lat,
ellipse=proj_ellipse,
datum=proj_datum)
geoProj = GeographicSystem()
# Get dlat
lon_n, lat_n, z_n = geoProj.fromECEF(*mapProj.toECEF(0, dy, 0))
dlat = lat_n - ctr_lat
# Get dlon
lon_e, lat_e, z_e = geoProj.fromECEF(*mapProj.toECEF(dx, 0, 0))
dlon = lon_e - ctr_lon
lon_min = ctr_lon + dlon * (x_bnd[0] / dx)
lon_max = ctr_lon + dlon * (x_bnd[1] / dx)
lat_min = ctr_lat + dlat * (y_bnd[0] / dy)
lat_max = ctr_lat + dlat * (y_bnd[1] / dy)
# Alternate method: lat lon for the actual distance to the NSEW in the projection
#lon_range_n, lat_range_n, z_range_n = geoProj.fromECEF(*mapProj.toECEF(0,y_bnd,0))
#lon_range_e, lat_range_e, z_range_e = geoProj.fromECEF(*mapProj.toECEF(x_bnd,0,0))
return dlon, dlat, (lon_min, lon_max), (lat_min, lat_max)
def __init__(
self,
start_time,
end_time,
do_3d=True,
frame_interval=120.0,
dx=4.0e3,
dy=4.0e3,
dz=1.0e3,
base_date=None,
x_bnd=(-100e3, 100e3),
y_bnd=(-100e3, 100e3),
z_bnd=(0e3, 20e3),
ctr_lat=35.23833,
ctr_lon=-97.46028,
ctr_alt=0.0,
proj_name='aeqd',
proj_datum='WGS84',
proj_ellipse='WGS84',
pixel_coords=None,
flash_count_logfile=None,
energy_grids=None,
event_grid_area_fraction_key=None,
spatial_scale_factor=1.0 / 1000.0,
subdivide=False,
):
""" Class to support gridding of flash and event data.
On init, specify the grid
If proj_name = 'pixel_grid' then pixel_coords must be an
instance of lmatools.coordinateSystems.PixelGrid
If proj_name = 'geos' then pixel_coords must be an instance of
lmatools.coordinateSystems.GeostationaryFixedGridSystem
energy_grids controls which energy grids are saved, default None.
energy_grids may be True, which will calculate all energy grid
types, or it may be one of 'specific_energy', 'total_energy',
or a list of one or more of these.
event_grid_area_fraction_key specifies the name of the variable
in the events array that gives the fraction of each grid cell
covered by each event. Used only for pixel-based event
detectors.
"""
if energy_grids == True:
energy_grids = ('specific_energy', 'total_energy')
self.energy_grids = energy_grids
self.spatial_scale_factor = spatial_scale_factor
self.event_grid_area_fraction_key = event_grid_area_fraction_key
# args, kwargs that are saved for the future
self.do_3d = do_3d
self.start_time = start_time
self.dx, self.dy, self.dz = dx, dy, dz
self.end_time = end_time
self.frame_interval = frame_interval
self.base_date = base_date
self.min_points_per_flash = 1
self.proj_name = proj_name
self.ctr_lat, self.ctr_lon, self.ctr_alt = ctr_lat, ctr_lon, ctr_alt
if flash_count_logfile is None:
flash_count_logfile = log
self.flash_count_logfile = flash_count_logfile
t_edges, duration = time_edges(self.start_time, self.end_time,
self.frame_interval)
# reference time is the date part of the start_time, unless the user provides a different date.
if self.base_date is None:
t_ref, t_edges_seconds = seconds_since_start_of_day(
self.start_time, t_edges)
else:
t_ref, t_edges_seconds = seconds_since_start_of_day(
self.base_date, t_edges)
self.n_frames = len(t_edges) - 1
xedge = np.arange(x_bnd[0], x_bnd[1] + dx, dx)
yedge = np.arange(y_bnd[0], y_bnd[1] + dy, dy)
zedge = np.arange(z_bnd[0], z_bnd[1] + dz, dz)
if self.proj_name == 'latlong':
dx_units = '{0:6.4f}deg'.format(float(dx))
mapProj = GeographicSystem()
elif self.proj_name == 'pixel_grid':
dx_units = 'pixel'
mapProj = pixel_coords
elif self.proj_name == 'geos':
dx_units = '{0:03d}urad'.format(int(dx * 1e6))
mapProj = pixel_coords
else:
dx_units = '{0:5.1f}m'.format(float(dx))
mapProj = MapProjection(projection=self.proj_name,
ctrLat=ctr_lat,
ctrLon=ctr_lon,
lat_ts=ctr_lat,
lon_0=ctr_lon,
lat_0=ctr_lat,
lat_1=ctr_lat,
ellipse=proj_ellipse,
datum=proj_datum)
geoProj = GeographicSystem()
self.t_ref = t_ref
self.t_edges = t_edges
self.t_edges_seconds = t_edges_seconds
self.duration = duration
self.xedge = xedge
self.yedge = yedge
self.zedge = zedge
self.mapProj = mapProj
self.geoProj = geoProj
self.dx_units = dx_units
self.pipeline_setup()
self.output_setup()
def get_geometry_hrrr(glm, nadir, hrrr, ltgellipsever=1):
# We don't use X, Y, Z below.
# x, y are the fixed grid 2D coord arrays (from meshgrid)
# X, Y, Z are the ECEF coords of each pixel intersected at the ltg ellipsoid
# lon_ltg, lat_ltg are the parallax-corrected lon lat at the earth's surface
# below the lightning position on the lightning ellipsoid.
# outside_glm_full_disk is a boolean mask for the positions that GLM can't
# observe.
# All of the above are 2D arrays corrsponding the center positions of the 2
# km fixed grid pixels in the GLM gridded products.
((x, y), (X, Y, Z), (lon_ltg, lat_ltg, alt_ltg),
outside_glm_full_disk) = get_glm_earth_geometry(glm, nadir, ltgellipsever)
### HRRR interpolation ###
# Everything below presumes LCC.
assert hrrr.MAP_PROJ_CHAR == 'Lambert Conformal'
corner_0_lla = (hrrr.XLONG[0, 0, 0].data, hrrr.XLAT[0, 0, 0].data,
np.asarray(0.0, dtype=hrrr.XLAT[0, 0, 0].dtype))
corner_1_lla = (hrrr.XLONG[0, -1, -1].data, hrrr.XLAT[0, -1, -1].data,
np.asarray(0.0, dtype=hrrr.XLAT[0, 1, -1].dtype))
hrrr_dx, hrrr_dy = hrrr.DX, hrrr.DX
hrrr_Nx, hrrr_Ny = hrrr.dims['west_east'], hrrr.dims['south_north']
hrrrproj = {
'lat_0': hrrr.CEN_LAT,
'lon_0': hrrr.CEN_LON + 360.0,
'lat_1': hrrr.TRUELAT1,
'lat_2': hrrr.TRUELAT2,
# 'R':hrrr.LambertConformal_Projection.earth_radius,
# 'a':6371229,
# 'b':6371229,
'R': 6371229,
}
lcc = MapProjection(projection='lcc',
ctrLat=hrrrproj['lat_0'],
ctrLon=hrrrproj['lon_0'],
**hrrrproj)
hrrr_lla = GeographicSystem(r_equator=hrrrproj['R'], r_pole=hrrrproj['R'])
lcc_cornerx_0, lcc_cornery_0, lcc_cornerz_0 = lcc.fromECEF(
*hrrr_lla.toECEF(*corner_0_lla))
lcc_cornerx_1, lcc_cornery_1, lcc_cornerz_1 = lcc.fromECEF(
*hrrr_lla.toECEF(*corner_1_lla))
# def grid_idx(x, y, x0, y0, dx, dy):
# """
# Convert x, y returned by [projection].fromECEF to the grid index in the
# NetCDF file. x0 and y0 are the [projection] coordinates of the center of
# the zero-index position in the NetCDF grid. dx and dy are the grid spacing
# in meters.
#
# returns (xidx, yidx)
# Taking int(xidx) will give the zero-based grid cell index.
# """
# # To get the correct behavior with int(xidx), add a half
# # since x0 is the center.
# xidx = (x-x0)/dx + 0.5
# yidx = (y-y0)/dy + 0.5
# return xidx, yidx
# The 3D position (X,Y,Z) defines an implicit lon, lat, alt with respect to the
# spherical earth we specified for the HRRR and its associated Lambert
# conformal projection. We let proj4 handle the mapping from the ECEF
# coordinates (an absolute position) directly to LCC.
lccx2, lccy2, lccz2 = lcc.fromECEF(
*hrrr_lla.toECEF(lon_ltg, lat_ltg, np.zeros_like(lon_ltg)))
lccx2.shape = x.shape
lccy2.shape = x.shape
lccx = lccx2
lccy = lccy2
# Set up the model grid, since the hrrr file doesn't include those values.
hrrrx_1d = np.arange(hrrr_Nx, dtype='f4') * hrrr_dx + lcc_cornerx_0
hrrry_1d = np.arange(hrrr_Ny, dtype='f4') * hrrr_dy + lcc_cornery_0
hrrrx, hrrry = np.meshgrid(hrrrx_1d, hrrry_1d)
interp_loc = np.vstack((hrrrx.flatten(), hrrry.flatten())).T
# GLM variables are filled with nan everywhere there is no lightning,
# so set those locations corresponding to valid earth locations to zero.
lcc_glm_x_flat = lccx[:, :].flatten()
lcc_glm_y_flat = lccy[:, :].flatten()
good = np.isfinite(lcc_glm_x_flat) & np.isfinite(lcc_glm_y_flat)
good = good & (~outside_glm_full_disk.flatten())
data_loc = np.vstack((lcc_glm_x_flat[good], lcc_glm_y_flat[good])).T
return (data_loc, good, interp_loc, hrrrx.shape)
#lma=LMAdataFile('/Users/ebruning/Documents/Lightning\ interpretation/Flash-length/Thomas/LYLOUT_120412_01817.exported.dat.gz')
#ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00
lma=LMAdataFile('/data/20040526/LMA/LYLOUT_040526_224000_0600.dat.gz')
# for line in lma.header:
# print line
ctr_lat, ctr_lon, ctr_alt = 35.2791257, -97.9178678, 417.90 # OKLMA
#ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00 # COLMA
good = (lma.stations >= 6) & (lma.chi2 <= 2.0) & (lma.alt < 20e3)
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc
D_max, t_max = 3.0e3, 0.15 # m, s
X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector-T_vector.min()))
# Maximum 3 s flash length, normalized to the time separation scale
chunker = chunk(XYZT[:,-1].min(), 3.0/.15, cluster_chunk_pairs(cluster_printer()))
stream(XYZT.astype('float32'),chunker)
def grid_h5flashfiles(h5_filenames, start_time, end_time,
frame_interval=120.0, dx=4.0e3, dy=4.0e3, dz=1.0e3,
x_bnd = (-100e3, 100e3),
y_bnd = (-100e3, 100e3),
z_bnd = (0e3, 20e3),
ctr_lat = 35.23833, ctr_lon = -97.46028, ctr_alt=0.0,
min_points_per_flash=10,
outpath = '',
flash_count_logfile = None,
proj_name = 'aeqd',
proj_datum = 'WGS84',
proj_ellipse = 'WGS84',
output_writer = write_cf_netcdf,
output_writer_3d = write_cf_netcdf_3d,
output_filename_prefix="LMA",
output_kwargs = {},
spatial_scale_factor = 1.0/1000.0,
):
from math import ceil
"""
Create 2D plan-view density grids for events, flash origins, flash extents, and mean flash footprint
frame_interval: Frame time-step in seconds
dx, dy: horizontal grid size in m (or deg)
{x,y,z}_bnd: horizontal grid edges in m
ctr_lat, ctr_lon: coordinate center
Uses an azimuthal equidistant map projection on the WGS84 ellipsoid.
read_flashes
filter_flash
extract_events
flash_to_frame
frame0_broadcast, frame1_broadcast, ...
each broadcaster above sends events and flashes to:
projection( event_location), projection(flash_init_location), projection(event_location)
which map respectively to:
point_density->accum_on_grid(event density), point_density->accum_on_grid(flash init density), extent_density->accum_on_grid(flash_extent_density)
grids are in an HDF5 file. how to handle flushing?
"""
if flash_count_logfile is None:
flash_count_logfile = sys.stdout
# reference time is the date part of the start_time
t_edges, duration = time_edges(start_time, end_time, frame_interval)
t_ref, t_edges_seconds = seconds_since_start_of_day(start_time, t_edges)
n_frames = len(t_edges)-1
xedge=np.arange(x_bnd[0], x_bnd[1]+dx, dx)
yedge=np.arange(y_bnd[0], y_bnd[1]+dy, dy)
zedge=np.arange(z_bnd[0], z_bnd[1]+dz, dz)
x0 = xedge[0]
y0 = yedge[0]
z0 = zedge[0]
if proj_name == 'latlong':
dx_units = '{0:6.4f}deg'.format(dx)
mapProj = GeographicSystem()
else:
dx_units = '{0:5.1f}m'.format(dx)
mapProj = MapProjection(projection=proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat,
lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum)
geoProj = GeographicSystem()
event_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
init_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
footprint_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
event_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
init_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
extent_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
footprint_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')
all_frames = []
# extent_frames = []
# init_frames = []
# event_frames = []
# extent_frames_3d = []
# init_frames_3d = []
# event_frames_3d = []
for i in range(n_frames):
extent_out = {'name':'extent'}
init_out = {'name':'init'}
event_out = {'name':'event'}
extent_out_3d = {'name':'extent_3d'}
init_out_3d = {'name':'init_3d'}
event_out_3d = {'name':'event_3d'}
accum_event_density = density_to_files.accumulate_points_on_grid(event_density_grid[:,:,i], xedge, yedge, out=event_out, label='event')
accum_init_density = density_to_files.accumulate_points_on_grid(init_density_grid[:,:,i], xedge, yedge, out=init_out, label='init')
accum_extent_density = density_to_files.accumulate_points_on_grid(extent_density_grid[:,:,i], xedge, yedge, out=extent_out,label='extent')
accum_footprint = density_to_files.accumulate_points_on_grid(footprint_grid[:,:,i], xedge, yedge, label='footprint')
accum_event_density_3d = density_to_files.accumulate_points_on_grid_3d(event_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=event_out_3d, label='event_3d')
accum_init_density_3d = density_to_files.accumulate_points_on_grid_3d(init_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=init_out_3d, label='init_3d')
accum_extent_density_3d = density_to_files.accumulate_points_on_grid_3d(extent_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='extent_3d')
accum_footprint_3d = density_to_files.accumulate_points_on_grid_3d(footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, label='footprint_3d')
extent_out['func'] = accum_extent_density
init_out['func'] = accum_init_density
event_out['func'] = accum_event_density
# extent_frames.append(extent_out)
# init_frames.append(init_out)
# event_frames.append(event_out)
extent_out_3d['func'] = accum_extent_density_3d
init_out_3d['func'] = accum_init_density_3d
event_out_3d['func'] = accum_event_density_3d
# extent_frames_3d.append(extent_out_3d)
# init_frames_3d.append(init_out_3d)
# event_frames_3d.append(event_out_3d)
event_density_target = density_to_files.point_density(accum_event_density)
init_density_target = density_to_files.point_density(accum_init_density)
extent_density_target = density_to_files.extent_density(x0, y0, dx, dy, accum_extent_density)
mean_footprint_target = density_to_files.extent_density(x0, y0, dx, dy, accum_footprint, weight_key='area')
event_density_target_3d = density_to_files.point_density_3d(accum_event_density_3d)
init_density_target_3d = density_to_files.point_density_3d(accum_init_density_3d)
extent_density_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_extent_density_3d)
mean_footprint_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area')
spew_to_density_types = broadcast( (
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False),
density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False),
density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False),
) )
all_frames.append( density_to_files.extract_events_for_flashes( spew_to_density_types ) )
frame_count_log = density_to_files.flash_count_log(flash_count_logfile)
framer = density_to_files.flashes_to_frames(t_edges_seconds, all_frames, time_key='start', time_edges_datetime=t_edges, flash_counter=frame_count_log)
read_flashes( h5_filenames, framer, base_date=t_ref, min_points=min_points_per_flash)
# print 'event_density_grid ', id(event_density_grid[:,:,-1])
# print 'extent_density_grid', id(extent_density_grid[:,:,-1])
# print 'init_density_grid ', id(init_density_grid[:,:,-1])
x_coord = (xedge[:-1] + xedge[1:])/2.0
y_coord = (yedge[:-1] + yedge[1:])/2.0
z_coord = (zedge[:-1] + zedge[1:])/2.0
nx = x_coord.shape[0]
ny = y_coord.shape[0]
nz = z_coord.shape[0]
x_all, y_all = (a.T for a in np.meshgrid(x_coord, y_coord))
assert x_all.shape == y_all.shape
assert x_all.shape[0] == nx
assert x_all.shape[1] == ny
z_all = np.zeros_like(x_all)
grid_shape_3d = (nx,ny,nz)
x_ones_3d = np.ones(grid_shape_3d, dtype='f4')
y_ones_3d = np.ones(grid_shape_3d, dtype='f4')
z_ones_3d = np.ones(grid_shape_3d, dtype='f4')
x_all_3d = x_coord[:, None, None]*x_ones_3d
y_all_3d = y_coord[None,:,None]*y_ones_3d
z_all_3d = z_coord[None, None, :]*z_ones_3d
lons, lats, alts = x,y,z = geoProj.fromECEF( *mapProj.toECEF(x_all, y_all, z_all) )
lons.shape=x_all.shape
lats.shape=y_all.shape
lons_3d, lats_3d, alts_3d = x_3d,y_3d,z_3d = geoProj.fromECEF( *mapProj.toECEF(x_all_3d, y_all_3d, z_all_3d) )
lons_3d.shape=x_all_3d.shape
lats_3d.shape=y_all_3d.shape
alts_3d.shape=z_all_3d.shape
outflile_basename = os.path.join(outpath,'%s_%s_%d_%dsrc_%s-dx_' % (output_filename_prefix, start_time.strftime('%Y%m%d_%H%M%S'), to_seconds(duration), min_points_per_flash, dx_units))
outfiles = (outflile_basename+'flash_extent.nc',
outflile_basename+'flash_init.nc',
outflile_basename+'source.nc',
outflile_basename+'footprint.nc',
)
outfiles_3d = (outflile_basename+'flash_extent_3d.nc',
outflile_basename+'flash_init_3d.nc',
outflile_basename+'source_3d.nc',
outflile_basename+'footprint_3d.nc',
)
outgrids = (extent_density_grid,
init_density_grid,
event_density_grid,
footprint_grid,
)
outgrids_3d = (extent_density_grid_3d,
init_density_grid_3d,
event_density_grid_3d,
footprint_grid_3d,)
field_names = ('flash_extent', 'flash_initiation', 'lma_source', 'flash_footprint')
field_descriptions = ('LMA flash extent density',
'LMA flash initiation density',
'LMA source density',
'LMA local mean flash area')
if proj_name=='latlong':
density_units = "grid"
density_units_3d = "grid"
else:
density_units = "{0:5.1f} km^2".format(dx/1000.0 * dy/1000.0).lstrip()
density_units_3d = "{0:5.1f} km^3".format(dx/1000.0 * dy/1000.0 * dz/1000.0).lstrip()
time_units = "{0:5.1f} min".format(frame_interval/60.0).lstrip()
density_label = 'Count per ' + density_units + " pixel per "+ time_units
density_label_3d = 'Count per ' + density_units_3d + " pixel per "+ time_units
field_units = ( density_label,
density_label,
density_label,
"km^2 per flash",
)
field_units_3d = ( density_label_3d,
density_label_3d,
density_label_3d,
"km^2 per flash",
)
output_writer(outfiles[0], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[0], field_names[0], field_descriptions[0],
grid_units=field_units[0],
**output_kwargs)
output_writer(outfiles[1], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[1], field_names[1], field_descriptions[1],
grid_units=field_units[1],
**output_kwargs)
output_writer(outfiles[2], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[2], field_names[2], field_descriptions[2],
grid_units=field_units[2],
**output_kwargs)
output_writer(outfiles[3], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[3], field_names[3], field_descriptions[3], format='f',
grid_units=field_units[3],
**output_kwargs)
output_writer_3d(outfiles_3d[0], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[0], field_names[0], field_descriptions[0],
grid_units=field_units_3d[0],
**output_kwargs)
output_writer_3d(outfiles_3d[1], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[1], field_names[1], field_descriptions[1],
grid_units=field_units_3d[1],
**output_kwargs)
output_writer_3d(outfiles_3d[2], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[2], field_names[2], field_descriptions[2],
grid_units=field_units_3d[2],
**output_kwargs)
output_writer_3d(outfiles_3d[3], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[3], field_names[3], field_descriptions[3], format='f',
grid_units=field_units_3d[3],
**output_kwargs)
print('max extent is', extent_density_grid.max())
return x_coord, y_coord, z_coord, lons, lats, alts, extent_density_grid, outfiles, field_names
