def footprint_stats(h5_filenames, save=False, fig=None, min_points=10,
base_date=None, other_analysis_targets=None, filterer=None):
""" filter should be a non-running coroutine that receives the (events, flashes)
arrays emitted by io.read_flashes and sends filtered (events, flashes) arrays to
a target defined by this function and passed to filter by keyword argument.
"""
#1e-2 to 1e4
min_pwr = -2
max_pwr = 4
delta_pwr = 0.1
powers = np.arange(min_pwr, max_pwr+delta_pwr, delta_pwr)
footprint_bin_edges = 10**powers
plotter = plot_energy_from_area_histogram
histogram_plot = histogram_accumulate_plot(plotter, save=save, fig=fig)
histogrammer=histogram_for_parameter('area', footprint_bin_edges, target=histogram_plot)
ev_fl_rx = events_flashes_receiver(target=histogrammer)
brancher = Branchpoint([ev_fl_rx])
if other_analysis_targets is not None:
for t in other_analysis_targets:
brancher.targets.add(t)
if filterer is not None:
rcvr = filterer(target=brancher.broadcast())
else:
rcvr = brancher.broadcast()
read_flashes(h5_filenames, rcvr, min_points=min_points, base_date=base_date)
def footprint_stats(h5_filenames, save=False, fig=None, min_points=10,
base_date=None, other_analysis_targets=None, filterer=None):
""" filter should be a non-running coroutine that receives the (events, flashes)
arrays emitted by io.read_flashes and sends filtered (events, flashes) arrays to
a target defined by this function and passed to filter by keyword argument.
"""
#1e-2 to 1e4
min_pwr = -2
max_pwr = 4
delta_pwr = 0.1
powers = np.arange(min_pwr, max_pwr+delta_pwr, delta_pwr)
footprint_bin_edges = 10**powers
plotter = plot_energy_from_area_histogram
histogram_plot = histogram_accumulate_plot(plotter, save=save, fig=fig)
histogrammer=histogram_for_parameter('area', footprint_bin_edges, target=histogram_plot)
ev_fl_rx = events_flashes_receiver(target=histogrammer)
brancher = Branchpoint([ev_fl_rx])
if other_analysis_targets is not None:
for t in other_analysis_targets:
brancher.targets.add(t)
if filterer is not None:
rcvr = filterer(target=brancher.broadcast())
else:
rcvr = brancher.broadcast()
read_flashes(h5_filenames, rcvr, min_points=min_points, base_date=base_date)
def grid_h5flashfiles(
h5_filenames,
start_time,
end_time,
frame_interval=120.0,
dx=4.0e3,
dy=4.0e3,
x_bnd=(-100e3, 100e3),
y_bnd=(-100e3, 100e3),
z_bnd=(-20e3, 20e3),
ctr_lat=35.23833,
ctr_lon=-97.46028,
min_points_per_flash=10,
outpath="",
flash_count_logfile=None,
proj_name="aeqd",
proj_datum="WGS84",
proj_ellipse="WGS84",
output_writer=write_cf_netcdf,
output_filename_prefix="LMA",
output_kwargs={},
spatial_scale_factor=1.0 / 1000.0,
):
from math import ceil
"""
Create 2D plan-view density grids for events, flash origins, flash extents, and mean flash footprint
frame_interval: Frame time-step in seconds
dx, dy: horizontal grid size in m (or deg)
{x,y,z}_bnd: horizontal grid edges in m
ctr_lat, ctr_lon: coordinate center
Uses an azimuthal equidistant map projection on the WGS84 ellipsoid.
read_flashes
filter_flash
extract_events
flash_to_frame
frame0_broadcast, frame1_broadcast, ...
each broadcaster above sends events and flashes to:
projection( event_location), projection(flash_init_location), projection(event_location)
which map respectively to:
point_density->accum_on_grid(event density), point_density->accum_on_grid(flash init density), extent_density->accum_on_grid(flash_extent_density)
grids are in an HDF5 file. how to handle flushing?
"""
if flash_count_logfile is None:
flash_count_logfile = sys.stdout
# reference time is the date part of the start_time
t_edges, duration = time_edges(start_time, end_time, frame_interval)
t_ref, t_edges_seconds = seconds_since_start_of_day(start_time, t_edges)
n_frames = len(t_edges) - 1
xedge = np.arange(x_bnd[0], x_bnd[1] + dx, dx)
yedge = np.arange(y_bnd[0], y_bnd[1] + dy, dy)
x0 = xedge[0]
y0 = yedge[0]
if proj_name == "latlong":
dx_units = "{0:6.4f}deg".format(dx)
mapProj = GeographicSystem()
else:
dx_units = "{0:5.1f}m".format(dx)
mapProj = MapProjection(
projection=proj_name,
ctrLat=ctr_lat,
ctrLon=ctr_lon,
lat_ts=ctr_lat,
lon_0=ctr_lon,
lat_0=ctr_lat,
lat_1=ctr_lat,
ellipse=proj_ellipse,
datum=proj_datum,
)
geoProj = GeographicSystem()
event_density_grid = np.zeros((xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype="int32")
init_density_grid = np.zeros((xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype="int32")
extent_density_grid = np.zeros((xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype="int32")
footprint_grid = np.zeros((xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype="float32")
all_frames = []
extent_frames = []
init_frames = []
event_frames = []
for i in range(n_frames):
extent_out = {"name": "extent"}
init_out = {"name": "init"}
event_out = {"name": "event"}
accum_event_density = density_to_files.accumulate_points_on_grid(
event_density_grid[:, :, i], xedge, yedge, out=event_out, label="event"
)
accum_init_density = density_to_files.accumulate_points_on_grid(
init_density_grid[:, :, i], xedge, yedge, out=init_out, label="init"
)
accum_extent_density = density_to_files.accumulate_points_on_grid(
extent_density_grid[:, :, i], xedge, yedge, out=extent_out, label="extent"
)
accum_footprint = density_to_files.accumulate_points_on_grid(
footprint_grid[:, :, i], xedge, yedge, label="footprint"
)
extent_out["func"] = accum_extent_density
init_out["func"] = accum_init_density
event_out["func"] = accum_event_density
extent_frames.append(extent_out)
init_frames.append(init_out)
event_frames.append(event_out)
event_density_target = density_to_files.point_density(accum_event_density)
init_density_target = density_to_files.point_density(accum_init_density)
extent_density_target = density_to_files.extent_density(x0, y0, dx, dy, accum_extent_density)
mean_footprint_target = density_to_files.extent_density(x0, y0, dx, dy, accum_footprint, weight_key="area")
spew_to_density_types = density_to_files.broadcast(
(
density_to_files.project(
"lon", "lat", "alt", mapProj, geoProj, event_density_target, use_flashes=False
),
density_to_files.project(
"init_lon", "init_lat", "init_alt", mapProj, geoProj, init_density_target, use_flashes=True
),
density_to_files.project(
"lon", "lat", "alt", mapProj, geoProj, extent_density_target, use_flashes=False
),
density_to_files.project(
"lon", "lat", "alt", mapProj, geoProj, mean_footprint_target, use_flashes=False
),
)
)
all_frames.append(density_to_files.extract_events_for_flashes(spew_to_density_types))
frame_count_log = density_to_files.flash_count_log(flash_count_logfile)
framer = density_to_files.flashes_to_frames(
t_edges_seconds, all_frames, time_key="start", time_edges_datetime=t_edges, flash_counter=frame_count_log
)
read_flashes(h5_filenames, framer, base_date=t_ref, min_points=min_points_per_flash)
# print 'event_density_grid ', id(event_density_grid[:,:,-1])
# print 'extent_density_grid', id(extent_density_grid[:,:,-1])
# print 'init_density_grid ', id(init_density_grid[:,:,-1])
x_coord = (xedge[:-1] + xedge[1:]) / 2.0
y_coord = (yedge[:-1] + yedge[1:]) / 2.0
nx = x_coord.shape[0]
ny = y_coord.shape[0]
x_all, y_all = (a.T for a in np.meshgrid(x_coord, y_coord))
assert x_all.shape == y_all.shape
assert x_all.shape[0] == nx
assert x_all.shape[1] == ny
z_all = np.zeros_like(x_all)
lons, lats, alts = x, y, z = geoProj.fromECEF(*mapProj.toECEF(x_all, y_all, z_all))
lons.shape = x_all.shape
lats.shape = y_all.shape
outflile_basename = os.path.join(
outpath,
"%s_%s_%d_%dsrc_%s-dx_"
% (
output_filename_prefix,
start_time.strftime("%Y%m%d_%H%M%S"),
to_seconds(duration),
min_points_per_flash,
dx_units,
),
)
outfiles = (
outflile_basename + "flash_extent.nc",
outflile_basename + "flash_init.nc",
outflile_basename + "source.nc",
outflile_basename + "footprint.nc",
)
outgrids = (extent_density_grid, init_density_grid, event_density_grid, footprint_grid)
field_names = ("flash_extent", "flash_initiation", "lma_source", "flash_footprint")
field_descriptions = (
"LMA flash extent density",
"LMA flash initiation density",
"LMA source density",
"LMA local mean flash area",
)
if proj_name == "latlong":
density_units = "grid"
else:
density_units = "{0:5.1f} km^2".format(dx / 1000.0 * dy / 1000.0).lstrip()
time_units = "{0:5.1f} min".format(frame_interval / 60.0).lstrip()
density_label = "Count per " + density_units + " pixel per " + time_units
field_units = (density_label, density_label, density_label, "km^2 per flash")
output_writer(
outfiles[0],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[0],
field_names[0],
field_descriptions[0],
grid_units=field_units[0],
**output_kwargs
)
output_writer(
outfiles[1],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[1],
field_names[1],
field_descriptions[1],
grid_units=field_units[1],
**output_kwargs
)
output_writer(
outfiles[2],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[2],
field_names[2],
field_descriptions[2],
grid_units=field_units[2],
**output_kwargs
)
output_writer(
outfiles[3],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[3],
field_names[3],
field_descriptions[3],
format="f",
grid_units=field_units[3],
**output_kwargs
)
print "max extent is", extent_density_grid.max()
return x_coord, y_coord, lons, lats, extent_density_grid, outfiles, field_names
def grid_h5flashfiles(
h5_filenames,
start_time,
end_time,
frame_interval=120.0,
dx=4.0e3,
dy=4.0e3,
x_bnd=(-100e3, 100e3),
y_bnd=(-100e3, 100e3),
z_bnd=(-20e3, 20e3),
ctr_lat=35.23833,
ctr_lon=-97.46028,
min_points_per_flash=10,
outpath='',
flash_count_logfile=None,
proj_name='aeqd',
proj_datum='WGS84',
proj_ellipse='WGS84',
output_writer=write_cf_netcdf,
output_filename_prefix="LMA",
output_kwargs={},
spatial_scale_factor=1.0 / 1000.0,
):
from math import ceil
"""
Create 2D plan-view density grids for events, flash origins, flash extents, and mean flash footprint
frame_interval: Frame time-step in seconds
dx, dy: horizontal grid size in m (or deg)
{x,y,z}_bnd: horizontal grid edges in m
ctr_lat, ctr_lon: coordinate center
Uses an azimuthal equidistant map projection on the WGS84 ellipsoid.
read_flashes
filter_flash
extract_events
flash_to_frame
frame0_broadcast, frame1_broadcast, ...
each broadcaster above sends events and flashes to:
projection( event_location), projection(flash_init_location), projection(event_location)
which map respectively to:
point_density->accum_on_grid(event density), point_density->accum_on_grid(flash init density), extent_density->accum_on_grid(flash_extent_density)
grids are in an HDF5 file. how to handle flushing?
"""
if flash_count_logfile is None:
flash_count_logfile = sys.stdout
# reference time is the date part of the start_time
t_edges, duration = time_edges(start_time, end_time, frame_interval)
t_ref, t_edges_seconds = seconds_since_start_of_day(start_time, t_edges)
n_frames = len(t_edges) - 1
xedge = np.arange(x_bnd[0], x_bnd[1] + dx, dx)
yedge = np.arange(y_bnd[0], y_bnd[1] + dy, dy)
x0 = xedge[0]
y0 = yedge[0]
if proj_name == 'latlong':
dx_units = '{0:6.4f}deg'.format(dx)
mapProj = GeographicSystem()
else:
dx_units = '{0:5.1f}m'.format(dx)
mapProj = MapProjection(projection=proj_name,
ctrLat=ctr_lat,
ctrLon=ctr_lon,
lat_ts=ctr_lat,
lon_0=ctr_lon,
lat_0=ctr_lat,
lat_1=ctr_lat,
ellipse=proj_ellipse,
datum=proj_datum)
geoProj = GeographicSystem()
event_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32')
init_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32')
extent_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32')
footprint_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='float32')
all_frames = []
extent_frames = []
init_frames = []
event_frames = []
for i in range(n_frames):
extent_out = {'name': 'extent'}
init_out = {'name': 'init'}
event_out = {'name': 'event'}
accum_event_density = density_to_files.accumulate_points_on_grid(
event_density_grid[:, :, i],
xedge,
yedge,
out=event_out,
label='event')
accum_init_density = density_to_files.accumulate_points_on_grid(
init_density_grid[:, :, i],
xedge,
yedge,
out=init_out,
label='init')
accum_extent_density = density_to_files.accumulate_points_on_grid(
extent_density_grid[:, :, i],
xedge,
yedge,
out=extent_out,
label='extent')
accum_footprint = density_to_files.accumulate_points_on_grid(
footprint_grid[:, :, i], xedge, yedge, label='footprint')
extent_out['func'] = accum_extent_density
init_out['func'] = accum_init_density
event_out['func'] = accum_event_density
extent_frames.append(extent_out)
init_frames.append(init_out)
event_frames.append(event_out)
event_density_target = density_to_files.point_density(
accum_event_density)
init_density_target = density_to_files.point_density(
accum_init_density)
extent_density_target = density_to_files.extent_density(
x0, y0, dx, dy, accum_extent_density)
mean_footprint_target = density_to_files.extent_density(
x0, y0, dx, dy, accum_footprint, weight_key='area')
spew_to_density_types = density_to_files.broadcast((
density_to_files.project('lon',
'lat',
'alt',
mapProj,
geoProj,
event_density_target,
use_flashes=False),
density_to_files.project('init_lon',
'init_lat',
'init_alt',
mapProj,
geoProj,
init_density_target,
use_flashes=True),
density_to_files.project('lon',
'lat',
'alt',
mapProj,
geoProj,
extent_density_target,
use_flashes=False),
density_to_files.project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_footprint_target,
use_flashes=False),
))
all_frames.append(
density_to_files.extract_events_for_flashes(spew_to_density_types))
frame_count_log = density_to_files.flash_count_log(flash_count_logfile)
framer = density_to_files.flashes_to_frames(t_edges_seconds,
all_frames,
time_key='start',
time_edges_datetime=t_edges,
flash_counter=frame_count_log)
read_flashes(h5_filenames,
framer,
base_date=t_ref,
min_points=min_points_per_flash)
# print 'event_density_grid ', id(event_density_grid[:,:,-1])
# print 'extent_density_grid', id(extent_density_grid[:,:,-1])
# print 'init_density_grid ', id(init_density_grid[:,:,-1])
x_coord = (xedge[:-1] + xedge[1:]) / 2.0
y_coord = (yedge[:-1] + yedge[1:]) / 2.0
nx = x_coord.shape[0]
ny = y_coord.shape[0]
x_all, y_all = (a.T for a in np.meshgrid(x_coord, y_coord))
assert x_all.shape == y_all.shape
assert x_all.shape[0] == nx
assert x_all.shape[1] == ny
z_all = np.zeros_like(x_all)
lons, lats, alts = x, y, z = geoProj.fromECEF(
*mapProj.toECEF(x_all, y_all, z_all))
lons.shape = x_all.shape
lats.shape = y_all.shape
outflile_basename = os.path.join(
outpath, '%s_%s_%d_%dsrc_%s-dx_' %
(output_filename_prefix, start_time.strftime('%Y%m%d_%H%M%S'),
to_seconds(duration), min_points_per_flash, dx_units))
outfiles = (
outflile_basename + 'flash_extent.nc',
outflile_basename + 'flash_init.nc',
outflile_basename + 'source.nc',
outflile_basename + 'footprint.nc',
)
outgrids = (extent_density_grid, init_density_grid, event_density_grid,
footprint_grid)
field_names = ('flash_extent', 'flash_initiation', 'lma_source',
'flash_footprint')
field_descriptions = ('LMA flash extent density',
'LMA flash initiation density', 'LMA source density',
'LMA local mean flash area')
if proj_name == 'latlong':
density_units = "grid"
else:
density_units = "{0:5.1f} km^2".format(dx / 1000.0 * dy /
1000.0).lstrip()
time_units = "{0:5.1f} min".format(frame_interval / 60.0).lstrip()
density_label = 'Count per ' + density_units + " pixel per " + time_units
field_units = (
density_label,
density_label,
density_label,
"km^2 per flash",
)
output_writer(outfiles[0],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[0],
field_names[0],
field_descriptions[0],
grid_units=field_units[0],
**output_kwargs)
output_writer(outfiles[1],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[1],
field_names[1],
field_descriptions[1],
grid_units=field_units[1],
**output_kwargs)
output_writer(outfiles[2],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[2],
field_names[2],
field_descriptions[2],
grid_units=field_units[2],
**output_kwargs)
output_writer(outfiles[3],
t_ref,
np.asarray(t_edges_seconds[:-1]),
x_coord * spatial_scale_factor,
y_coord * spatial_scale_factor,
lons,
lats,
ctr_lat,
ctr_lon,
outgrids[3],
field_names[3],
field_descriptions[3],
format='f',
grid_units=field_units[3],
**output_kwargs)
print 'max extent is', extent_density_grid.max()
return x_coord, y_coord, lons, lats, extent_density_grid, outfiles, field_names
