def event_pipeline_setup(self):
""" Create target grids and set up the event accumulation pipeline for events
that have been pre-aggregated using an event lookup table.
Arguments: none
Returns:
outgrids -- tuple of grids (numpy arrays) for the target grids
framer -- first stage of the pipeline that routes flashes to the correct
time window for accumulation.
"""
(event_grid_area_fraction_key, energy_grids, n_frames,
xedge, yedge, zedge, dx, dy, dz, x0, y0, z0, mapProj, geoProj) = (
self.gridspec_locals())
grid_shape = (xedge.shape[0]-1, yedge.shape[0]-1, n_frames)
event_density_grid = np.zeros(grid_shape, dtype='float32')
total_energy_grid = np.zeros(grid_shape, dtype='float32')
all_frames = []
for i in range(n_frames):
accum_event_density = accumulate_energy_on_grid(
event_density_grid[:,:,i], xedge, yedge,
label='event extent', grid_frac_weights=False)
event_density_target = point_density(accum_event_density,
weight_key='lutevent_count', weight_flashes=False)
# total_energy is built from split_event_energy, which has
# already divided up the event energy into the sub-event
# corresponding to each pixel. We don't need to weight by the
# grid fractional area. We just need to sum the 'power' variable
# which mimic_lma assigns the values of split_event_energy.
accum_total_energy = accumulate_energy_on_grid(
total_energy_grid[:,:,i], xedge, yedge,
label='total energy', grid_frac_weights=False)
total_energy_target = point_density(accum_total_energy,
weight_key='power', weight_flashes=False)
broadcast_targets = (
project('lon', 'lat', 'alt', mapProj, geoProj,
event_density_target, use_flashes=False),
project('lon', 'lat', 'alt', mapProj, geoProj,
total_energy_target, use_flashes=False),
)
spew_to_density_types = broadcast( broadcast_targets )
all_frames.append(spew_to_density_types)
frame_count_log = flash_count_log(self.flash_count_logfile)
framer = flashes_to_frames(self.t_edges_seconds, all_frames,
time_key='time', time_edges_datetime=self.t_edges,
flash_counter=frame_count_log, do_events='time')
outgrids = (event_density_grid, total_energy_grid)
return outgrids, framer
def flash_pipeline_setup(self):
""" Create target grids and set up the flash accumulation pipeline. Also used
to set up an equivalent group accumulation pipeline.
Arguments: none
Returns:
outgrids -- tuple of grids (numpy arrays) for the target grids
framer -- first stage of the pipeline that routes flashes to the correct
time window for accumulation.
"""
(event_grid_area_fraction_key, energy_grids, n_frames, xedge, yedge,
zedge, dx, dy, dz, x0, y0, z0, mapProj,
geoProj) = (self.gridspec_locals())
grid_shape = (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames)
init_density_grid = np.zeros(grid_shape, dtype='float32')
extent_density_grid = np.zeros(grid_shape, dtype='float32')
footprint_grid = np.zeros(grid_shape, dtype='float32')
# flashsize_std_grid = np.zeros(grid_shape, dtype='float32')
all_frames = []
for i in range(n_frames):
accum_init_density = accumulate_points_on_grid(
init_density_grid[:, :, i], xedge, yedge, label='init')
accum_extent_density = accumulate_points_on_grid(
extent_density_grid[:, :, i],
xedge,
yedge,
label='extent',
grid_frac_weights=True)
accum_footprint = accumulate_points_on_grid(footprint_grid[:, :,
i],
xedge,
yedge,
label='footprint',
grid_frac_weights=True)
# accum_flashstd = accumulate_points_on_grid_sdev(
# flashsize_std_grid[:,:,i], footprint_grid[:,:,i], xedge, yedge,
# label='flashsize_std', grid_frac_weights=True)
init_density_target = point_density(accum_init_density)
extent_density_target = extent_density(
x0,
y0,
dx,
dy,
accum_extent_density,
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_footprint_target = extent_density(
x0,
y0,
dx,
dy,
accum_footprint,
weight_key='area',
event_grid_area_fraction_key=event_grid_area_fraction_key)
# std_flashsize_target = extent_density(x0, y0, dx, dy,
# accum_flashstd, weight_key='area',
# event_grid_area_fraction_key=event_grid_area_fraction_key)
broadcast_targets = (
project('init_lon',
'init_lat',
'init_alt',
mapProj,
geoProj,
init_density_target,
use_flashes=True),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
extent_density_target,
use_flashes=False),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_footprint_target,
use_flashes=False),
# project('lon', 'lat', 'alt', mapProj, geoProj,
# std_flashsize_target, use_flashes=False),
)
spew_to_density_types = broadcast(broadcast_targets)
all_frames.append(
extract_events_for_flashes(spew_to_density_types))
frame_count_log = flash_count_log(self.flash_count_logfile)
framer = flashes_to_frames(self.t_edges_seconds,
all_frames,
time_key='start',
time_edges_datetime=self.t_edges,
flash_counter=frame_count_log)
outgrids = (
init_density_grid,
extent_density_grid,
footprint_grid,
# flashsize_std_grid,
)
return outgrids, framer
def group_pipeline_setup(self):
""" Create target grids and set up the flash accumulation pipeline. Also used
to set up an equivalent group accumulation pipeline. Actually uses event-level
data that have been pre-aggregated using an event lookup table.
Arguments: none
Returns:
outgrids -- tuple of grids (numpy arrays) for the target grids
framer -- first stage of the pipeline that routes flashes to the correct
time window for accumulation.
"""
(event_grid_area_fraction_key, energy_grids, n_frames, xedge, yedge,
zedge, dx, dy, dz, x0, y0, z0, mapProj,
geoProj) = (self.gridspec_locals())
grid_shape = (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames)
init_density_grid = np.zeros(grid_shape, dtype='float32')
extent_density_grid = np.zeros(grid_shape, dtype='float32')
footprint_grid = np.zeros(grid_shape, dtype='float32')
all_frames = []
for i in range(n_frames):
accum_init_density = accumulate_points_on_grid(
init_density_grid[:, :, i], xedge, yedge, label='init')
accum_extent_density = accumulate_var_on_grid_direct_idx(
extent_density_grid[:, :, i], 'lutevent_group_count',
'mesh_xi', 'mesh_yi')
accum_footprint = accumulate_var_on_grid_direct_idx(
footprint_grid[:, :, i], 'lutevent_total_group_area',
'mesh_xi', 'mesh_yi')
init_density_target = point_density(accum_init_density)
broadcast_targets = (
no_projection('ctr_x',
'ctr_y',
'ctr_z',
init_density_target,
use_flashes=True),
select_dataset(accum_extent_density, use_event_data=True),
select_dataset(accum_footprint, use_event_data=True),
)
spew_to_density_types = broadcast(broadcast_targets)
all_frames.append(spew_to_density_types)
frame_count_log = flash_count_log(self.flash_count_logfile)
framer = flashes_to_frames(self.t_edges_seconds,
all_frames,
time_key='start',
time_edges_datetime=self.t_edges,
flash_counter=frame_count_log,
do_events='time')
outgrids = (
init_density_grid,
extent_density_grid,
footprint_grid,
)
return outgrids, framer
def pipeline_setup(self):
"""
Create the grids for each variable of interest.
Set up the data processing pipeline up to the point at which data are
written to grids in memory.
"""
event_grid_area_fraction_key = self.event_grid_area_fraction_key
energy_grids = self.energy_grids
n_frames = self.n_frames
xedge, yedge, zedge = self.xedge, self.yedge, self.zedge
dx, dy, dz = self.dx, self.dy, self.dz
x0 = xedge[0]
y0 = yedge[0]
z0 = zedge[0]
mapProj = self.mapProj
geoProj = self.geoProj
event_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32')
init_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32')
if self.event_grid_area_fraction_key is not None:
extent_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames),
dtype='float32')
else:
extent_density_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames),
dtype='int32')
footprint_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames),
dtype='float32')
specific_energy_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames),
dtype='float32')
total_energy_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames),
dtype='float32')
flashsize_std_grid = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames),
dtype='float32')
if self.do_3d == True:
event_density_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='int32')
init_density_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='int32')
extent_density_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='int32')
footprint_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='float32')
specific_energy_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='float32')
total_energy_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='float32')
flashsize_std_grid_3d = np.zeros(
(xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1,
n_frames),
dtype='float32')
self.outgrids = (
extent_density_grid,
init_density_grid,
event_density_grid,
footprint_grid,
specific_energy_grid,
flashsize_std_grid,
total_energy_grid,
)
if self.do_3d == True:
self.outgrids_3d = (extent_density_grid_3d, init_density_grid_3d,
event_density_grid_3d, footprint_grid_3d,
specific_energy_grid_3d, flashsize_std_grid_3d,
total_energy_grid_3d)
else:
self.outgrids_3d = None
all_frames = []
for i in range(n_frames):
extent_out = {'name': 'extent'}
init_out = {'name': 'init'}
event_out = {'name': 'event'}
std_out = {'name': 'std'}
extent_out_3d = {'name': 'extent_3d'}
init_out_3d = {'name': 'init_3d'}
event_out_3d = {'name': 'event_3d'}
std_out_3d = {'name': 'std_3d'}
accum_event_density = accumulate_points_on_grid(
event_density_grid[:, :, i],
xedge,
yedge,
out=event_out,
label='event')
accum_init_density = accumulate_points_on_grid(
init_density_grid[:, :, i],
xedge,
yedge,
out=init_out,
label='init')
accum_extent_density = accumulate_points_on_grid(
extent_density_grid[:, :, i],
xedge,
yedge,
out=extent_out,
label='extent',
grid_frac_weights=True)
accum_footprint = accumulate_points_on_grid(footprint_grid[:, :,
i],
xedge,
yedge,
label='footprint',
grid_frac_weights=True)
accum_specific_energy = accumulate_energy_on_grid(
specific_energy_grid[:, :, i],
xedge,
yedge,
out=extent_out,
label='specific_energy',
grid_frac_weights=True)
accum_flashstd = accumulate_points_on_grid_sdev(
flashsize_std_grid[:, :, i],
footprint_grid[:, :, i],
xedge,
yedge,
out=extent_out,
label='flashsize_std',
grid_frac_weights=True)
accum_total_energy = accumulate_energy_on_grid(
total_energy_grid[:, :, i],
xedge,
yedge,
out=extent_out,
label='total_energy',
grid_frac_weights=True)
if self.do_3d == True:
accum_event_density_3d = accumulate_points_on_grid_3d(
event_density_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
out=event_out_3d,
label='event_3d')
accum_init_density_3d = accumulate_points_on_grid_3d(
init_density_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
out=init_out_3d,
label='init_3d')
accum_extent_density_3d = accumulate_points_on_grid_3d(
extent_density_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
out=extent_out_3d,
label='extent_3d')
accum_footprint_3d = accumulate_points_on_grid_3d(
footprint_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
label='footprint_3d')
accum_specific_energy_3d = accumulate_energy_on_grid_3d(
specific_energy_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
out=extent_out_3d,
label='specific_energy_3d')
accum_flashstd_3d = accumulate_points_on_grid_sdev_3d(
flashsize_std_grid_3d[:, :, :, i],
footprint_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
out=extent_out_3d,
label='flashsize_std_3d')
accum_total_energy_3d = accumulate_energy_on_grid_3d(
total_energy_grid_3d[:, :, :, i],
xedge,
yedge,
zedge,
out=extent_out_3d,
label='total_energy_3d')
extent_out['func'] = accum_extent_density
init_out['func'] = accum_init_density
event_out['func'] = accum_event_density
if self.do_3d == True:
extent_out_3d['func'] = accum_extent_density_3d
init_out_3d['func'] = accum_init_density_3d
event_out_3d['func'] = accum_event_density_3d
event_density_target = point_density(accum_event_density)
init_density_target = point_density(accum_init_density)
extent_density_target = extent_density(
x0,
y0,
dx,
dy,
accum_extent_density,
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_footprint_target = extent_density(
x0,
y0,
dx,
dy,
accum_footprint,
weight_key='area',
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_energy_target = extent_density(
x0,
y0,
dx,
dy,
accum_specific_energy,
weight_key='specific_energy',
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_total_energy_target = extent_density(
x0,
y0,
dx,
dy,
accum_total_energy,
weight_key='total_energy',
event_grid_area_fraction_key=event_grid_area_fraction_key)
std_flashsize_target = extent_density(
x0,
y0,
dx,
dy,
accum_flashstd,
weight_key='area',
event_grid_area_fraction_key=event_grid_area_fraction_key)
if self.do_3d == True:
event_density_target_3d = point_density_3d(
accum_event_density_3d)
init_density_target_3d = point_density_3d(
accum_init_density_3d)
extent_density_target_3d = extent_density_3d(
x0, y0, z0, dx, dy, dz, accum_extent_density_3d)
mean_footprint_target_3d = extent_density_3d(
x0,
y0,
z0,
dx,
dy,
dz,
accum_footprint_3d,
weight_key='area')
mean_energy_target_3d = extent_density_3d(
x0,
y0,
z0,
dx,
dy,
dz,
accum_specific_energy_3d,
weight_key='specific_energy')
mean_total_energy_target_3d = extent_density_3d(
x0,
y0,
z0,
dx,
dy,
dz,
accum_total_energy_3d,
weight_key='total_energy')
std_flashsize_target_3d = extent_density_3d(x0,
y0,
z0,
dx,
dy,
dz,
accum_flashstd_3d,
weight_key='area')
broadcast_targets = (
project('lon',
'lat',
'alt',
mapProj,
geoProj,
event_density_target,
use_flashes=False),
project('init_lon',
'init_lat',
'init_alt',
mapProj,
geoProj,
init_density_target,
use_flashes=True),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
extent_density_target,
use_flashes=False),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_footprint_target,
use_flashes=False),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
std_flashsize_target,
use_flashes=False),
)
if energy_grids is not None:
if ('specific_energy' == energy_grids) | ('specific_energy'
in energy_grids):
broadcast_targets += (project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_energy_target,
use_flashes=False), )
if ('total_energy' == energy_grids) | ('total_energy'
in energy_grids):
broadcast_targets += (project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_total_energy_target,
use_flashes=False), )
if self.do_3d == True:
broadcast_targets += (
project('lon',
'lat',
'alt',
mapProj,
geoProj,
event_density_target_3d,
use_flashes=False),
project('init_lon',
'init_lat',
'init_alt',
mapProj,
geoProj,
init_density_target_3d,
use_flashes=True),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
extent_density_target_3d,
use_flashes=False),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_footprint_target_3d,
use_flashes=False),
project('lon',
'lat',
'alt',
mapProj,
geoProj,
std_flashsize_target_3d,
use_flashes=False),
)
if energy_grids is not None:
if ('specific_energy' == energy_grids) | ('specific_energy'
in energy_grids):
broadcast_targets += (project('lon',
'lat',
'alt',
mapProj,
geoProj,
mean_energy_target_3d,
use_flashes=False), )
if ('total_energy' == energy_grids) | ('total_energy'
in energy_grids):
broadcast_targets += (project(
'lon',
'lat',
'alt',
mapProj,
geoProj,
mean_total_energy_target_3d,
use_flashes=False), )
spew_to_density_types = broadcast(broadcast_targets)
all_frames.append(
extract_events_for_flashes(spew_to_density_types))
frame_count_log = flash_count_log(self.flash_count_logfile)
framer = flashes_to_frames(self.t_edges_seconds,
all_frames,
time_key='start',
time_edges_datetime=self.t_edges,
flash_counter=frame_count_log)
self.framer = framer
def pipeline_setup(self):
"""
Create the grids for each variable of interest.
Set up the data processing pipeline up to the point at which data are
written to grids in memory.
"""
event_grid_area_fraction_key=self.event_grid_area_fraction_key
energy_grids=self.energy_grids
n_frames = self.n_frames
xedge, yedge, zedge = self.xedge, self.yedge, self.zedge
dx, dy, dz = self.dx, self.dy, self.dz
x0 = xedge[0]
y0 = yedge[0]
z0 = zedge[0]
mapProj = self.mapProj
geoProj = self.geoProj
event_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
init_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
if self.event_grid_area_fraction_key is not None:
extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
else:
extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
footprint_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
specific_energy_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
total_energy_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
flashsize_std_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
if self.do_3d == True:
event_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
init_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
extent_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
footprint_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')
specific_energy_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')
total_energy_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')
flashsize_std_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')
self.outgrids = (extent_density_grid,
init_density_grid,
event_density_grid,
footprint_grid,
specific_energy_grid,
flashsize_std_grid,
total_energy_grid,
)
if self.do_3d == True:
self.outgrids_3d = (extent_density_grid_3d,
init_density_grid_3d,
event_density_grid_3d,
footprint_grid_3d,
specific_energy_grid_3d,
flashsize_std_grid_3d,
total_energy_grid_3d
)
else:
self.outgrids_3d = None
all_frames = []
for i in range(n_frames):
extent_out = {'name':'extent'}
init_out = {'name':'init'}
event_out = {'name':'event'}
std_out = {'name':'std'}
extent_out_3d = {'name':'extent_3d'}
init_out_3d = {'name':'init_3d'}
event_out_3d = {'name':'event_3d'}
std_out_3d = {'name':'std_3d'}
accum_event_density = accumulate_points_on_grid(event_density_grid[:,:,i], xedge, yedge, out=event_out, label='event')
accum_init_density = accumulate_points_on_grid(init_density_grid[:,:,i], xedge, yedge, out=init_out, label='init')
accum_extent_density = accumulate_points_on_grid(extent_density_grid[:,:,i], xedge, yedge, out=extent_out,label='extent', grid_frac_weights=True)
accum_footprint = accumulate_points_on_grid(footprint_grid[:,:,i], xedge, yedge, label='footprint', grid_frac_weights=True)
accum_specific_energy = accumulate_energy_on_grid(specific_energy_grid[:,:,i], xedge, yedge, out=extent_out, label='specific_energy', grid_frac_weights=True)
accum_flashstd = accumulate_points_on_grid_sdev(flashsize_std_grid[:,:,i], footprint_grid[:,:,i], xedge, yedge, out=extent_out, label='flashsize_std', grid_frac_weights=True)
accum_total_energy = accumulate_energy_on_grid(total_energy_grid[:,:,i], xedge, yedge, out=extent_out, label='total_energy', grid_frac_weights=True)
if self.do_3d == True:
accum_event_density_3d = accumulate_points_on_grid_3d(event_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=event_out_3d, label='event_3d')
accum_init_density_3d = accumulate_points_on_grid_3d(init_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=init_out_3d, label='init_3d')
accum_extent_density_3d = accumulate_points_on_grid_3d(extent_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='extent_3d')
accum_footprint_3d = accumulate_points_on_grid_3d(footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, label='footprint_3d')
accum_specific_energy_3d = accumulate_energy_on_grid_3d(specific_energy_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='specific_energy_3d')
accum_flashstd_3d = accumulate_points_on_grid_sdev_3d(flashsize_std_grid_3d[:,:,:,i], footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='flashsize_std_3d')
accum_total_energy_3d = accumulate_energy_on_grid_3d(total_energy_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='total_energy_3d')
extent_out['func'] = accum_extent_density
init_out['func'] = accum_init_density
event_out['func'] = accum_event_density
if self.do_3d == True:
extent_out_3d['func'] = accum_extent_density_3d
init_out_3d['func'] = accum_init_density_3d
event_out_3d['func'] = accum_event_density_3d
event_density_target = point_density(accum_event_density)
init_density_target = point_density(accum_init_density)
extent_density_target = extent_density(x0, y0, dx, dy, accum_extent_density,
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_footprint_target = extent_density(x0, y0, dx, dy, accum_footprint, weight_key='area',
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_energy_target = extent_density(x0, y0, dx, dy, accum_specific_energy, weight_key='specific_energy',
event_grid_area_fraction_key=event_grid_area_fraction_key)
mean_total_energy_target = extent_density(x0, y0, dx, dy, accum_total_energy, weight_key='total_energy',
event_grid_area_fraction_key=event_grid_area_fraction_key)
std_flashsize_target = extent_density(x0, y0, dx, dy, accum_flashstd, weight_key='area',
event_grid_area_fraction_key=event_grid_area_fraction_key)
if self.do_3d == True:
event_density_target_3d = point_density_3d(accum_event_density_3d)
init_density_target_3d = point_density_3d(accum_init_density_3d)
extent_density_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_extent_density_3d)
mean_footprint_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area')
mean_energy_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_specific_energy_3d, weight_key='specific_energy')
mean_total_energy_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_total_energy_3d, weight_key='total_energy')
std_flashsize_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_flashstd_3d, weight_key='area')
broadcast_targets = (
project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False),
project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True),
project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False),
project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False),
project('lon', 'lat', 'alt', mapProj, geoProj, std_flashsize_target, use_flashes=False),
)
if energy_grids is not None:
if ('specific_energy' == energy_grids) | ('specific_energy' in energy_grids):
broadcast_targets += (
project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target, use_flashes=False),
)
if ('total_energy' == energy_grids) | ('total_energy' in energy_grids):
broadcast_targets += (
project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target, use_flashes=False),
)
if self.do_3d == True:
broadcast_targets += (
project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False),
project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True),
project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False),
project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False),
project('lon', 'lat', 'alt', mapProj, geoProj, std_flashsize_target_3d, use_flashes=False),
)
if energy_grids is not None:
if ('specific_energy' == energy_grids) | ('specific_energy' in energy_grids):
broadcast_targets += (
project('lon', 'lat', 'alt', mapProj, geoProj, mean_energy_target_3d, use_flashes=False),
)
if ('total_energy' == energy_grids) | ('total_energy' in energy_grids):
broadcast_targets += (
project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target_3d, use_flashes=False),
)
spew_to_density_types = broadcast( broadcast_targets )
all_frames.append( extract_events_for_flashes( spew_to_density_types ) )
frame_count_log = flash_count_log(self.flash_count_logfile)
framer = flashes_to_frames(self.t_edges_seconds, all_frames,
time_key='start', time_edges_datetime=self.t_edges,
flash_counter=frame_count_log)
self.framer=framer
def grid_h5flashfiles(h5_filenames, start_time, end_time,
frame_interval=120.0, dx=4.0e3, dy=4.0e3, dz=1.0e3,
x_bnd = (-100e3, 100e3),
y_bnd = (-100e3, 100e3),
z_bnd = (0e3, 20e3),
ctr_lat = 35.23833, ctr_lon = -97.46028, ctr_alt=0.0,
min_points_per_flash=10,
outpath = '',
flash_count_logfile = None,
proj_name = 'aeqd',
proj_datum = 'WGS84',
proj_ellipse = 'WGS84',
output_writer = write_cf_netcdf,
output_writer_3d = write_cf_netcdf_3d,
output_filename_prefix="LMA",
output_kwargs = {},
spatial_scale_factor = 1.0/1000.0,
):
from math import ceil
"""
Create 2D plan-view density grids for events, flash origins, flash extents, and mean flash footprint
frame_interval: Frame time-step in seconds
dx, dy: horizontal grid size in m (or deg)
{x,y,z}_bnd: horizontal grid edges in m
ctr_lat, ctr_lon: coordinate center
Uses an azimuthal equidistant map projection on the WGS84 ellipsoid.
read_flashes
filter_flash
extract_events
flash_to_frame
frame0_broadcast, frame1_broadcast, ...
each broadcaster above sends events and flashes to:
projection( event_location), projection(flash_init_location), projection(event_location)
which map respectively to:
point_density->accum_on_grid(event density), point_density->accum_on_grid(flash init density), extent_density->accum_on_grid(flash_extent_density)
grids are in an HDF5 file. how to handle flushing?
"""
if flash_count_logfile is None:
flash_count_logfile = sys.stdout
# reference time is the date part of the start_time
t_edges, duration = time_edges(start_time, end_time, frame_interval)
t_ref, t_edges_seconds = seconds_since_start_of_day(start_time, t_edges)
n_frames = len(t_edges)-1
xedge=np.arange(x_bnd[0], x_bnd[1]+dx, dx)
yedge=np.arange(y_bnd[0], y_bnd[1]+dy, dy)
zedge=np.arange(z_bnd[0], z_bnd[1]+dz, dz)
x0 = xedge[0]
y0 = yedge[0]
z0 = zedge[0]
if proj_name == 'latlong':
dx_units = '{0:6.4f}deg'.format(dx)
mapProj = GeographicSystem()
else:
dx_units = '{0:5.1f}m'.format(dx)
mapProj = MapProjection(projection=proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat,
lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum)
geoProj = GeographicSystem()
event_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
init_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
footprint_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')
event_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
init_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
extent_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
footprint_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')
all_frames = []
# extent_frames = []
# init_frames = []
# event_frames = []
# extent_frames_3d = []
# init_frames_3d = []
# event_frames_3d = []
for i in range(n_frames):
extent_out = {'name':'extent'}
init_out = {'name':'init'}
event_out = {'name':'event'}
extent_out_3d = {'name':'extent_3d'}
init_out_3d = {'name':'init_3d'}
event_out_3d = {'name':'event_3d'}
accum_event_density = density_to_files.accumulate_points_on_grid(event_density_grid[:,:,i], xedge, yedge, out=event_out, label='event')
accum_init_density = density_to_files.accumulate_points_on_grid(init_density_grid[:,:,i], xedge, yedge, out=init_out, label='init')
accum_extent_density = density_to_files.accumulate_points_on_grid(extent_density_grid[:,:,i], xedge, yedge, out=extent_out,label='extent')
accum_footprint = density_to_files.accumulate_points_on_grid(footprint_grid[:,:,i], xedge, yedge, label='footprint')
accum_event_density_3d = density_to_files.accumulate_points_on_grid_3d(event_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=event_out_3d, label='event_3d')
accum_init_density_3d = density_to_files.accumulate_points_on_grid_3d(init_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=init_out_3d, label='init_3d')
accum_extent_density_3d = density_to_files.accumulate_points_on_grid_3d(extent_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='extent_3d')
accum_footprint_3d = density_to_files.accumulate_points_on_grid_3d(footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, label='footprint_3d')
extent_out['func'] = accum_extent_density
init_out['func'] = accum_init_density
event_out['func'] = accum_event_density
# extent_frames.append(extent_out)
# init_frames.append(init_out)
# event_frames.append(event_out)
extent_out_3d['func'] = accum_extent_density_3d
init_out_3d['func'] = accum_init_density_3d
event_out_3d['func'] = accum_event_density_3d
# extent_frames_3d.append(extent_out_3d)
# init_frames_3d.append(init_out_3d)
# event_frames_3d.append(event_out_3d)
event_density_target = density_to_files.point_density(accum_event_density)
init_density_target = density_to_files.point_density(accum_init_density)
extent_density_target = density_to_files.extent_density(x0, y0, dx, dy, accum_extent_density)
mean_footprint_target = density_to_files.extent_density(x0, y0, dx, dy, accum_footprint, weight_key='area')
event_density_target_3d = density_to_files.point_density_3d(accum_event_density_3d)
init_density_target_3d = density_to_files.point_density_3d(accum_init_density_3d)
extent_density_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_extent_density_3d)
mean_footprint_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area')
spew_to_density_types = broadcast( (
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False),
density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False),
density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False),
density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False),
) )
all_frames.append( density_to_files.extract_events_for_flashes( spew_to_density_types ) )
frame_count_log = density_to_files.flash_count_log(flash_count_logfile)
framer = density_to_files.flashes_to_frames(t_edges_seconds, all_frames, time_key='start', time_edges_datetime=t_edges, flash_counter=frame_count_log)
read_flashes( h5_filenames, framer, base_date=t_ref, min_points=min_points_per_flash)
# print 'event_density_grid ', id(event_density_grid[:,:,-1])
# print 'extent_density_grid', id(extent_density_grid[:,:,-1])
# print 'init_density_grid ', id(init_density_grid[:,:,-1])
x_coord = (xedge[:-1] + xedge[1:])/2.0
y_coord = (yedge[:-1] + yedge[1:])/2.0
z_coord = (zedge[:-1] + zedge[1:])/2.0
nx = x_coord.shape[0]
ny = y_coord.shape[0]
nz = z_coord.shape[0]
x_all, y_all = (a.T for a in np.meshgrid(x_coord, y_coord))
assert x_all.shape == y_all.shape
assert x_all.shape[0] == nx
assert x_all.shape[1] == ny
z_all = np.zeros_like(x_all)
grid_shape_3d = (nx,ny,nz)
x_ones_3d = np.ones(grid_shape_3d, dtype='f4')
y_ones_3d = np.ones(grid_shape_3d, dtype='f4')
z_ones_3d = np.ones(grid_shape_3d, dtype='f4')
x_all_3d = x_coord[:, None, None]*x_ones_3d
y_all_3d = y_coord[None,:,None]*y_ones_3d
z_all_3d = z_coord[None, None, :]*z_ones_3d
lons, lats, alts = x,y,z = geoProj.fromECEF( *mapProj.toECEF(x_all, y_all, z_all) )
lons.shape=x_all.shape
lats.shape=y_all.shape
lons_3d, lats_3d, alts_3d = x_3d,y_3d,z_3d = geoProj.fromECEF( *mapProj.toECEF(x_all_3d, y_all_3d, z_all_3d) )
lons_3d.shape=x_all_3d.shape
lats_3d.shape=y_all_3d.shape
alts_3d.shape=z_all_3d.shape
outflile_basename = os.path.join(outpath,'%s_%s_%d_%dsrc_%s-dx_' % (output_filename_prefix, start_time.strftime('%Y%m%d_%H%M%S'), to_seconds(duration), min_points_per_flash, dx_units))
outfiles = (outflile_basename+'flash_extent.nc',
outflile_basename+'flash_init.nc',
outflile_basename+'source.nc',
outflile_basename+'footprint.nc',
)
outfiles_3d = (outflile_basename+'flash_extent_3d.nc',
outflile_basename+'flash_init_3d.nc',
outflile_basename+'source_3d.nc',
outflile_basename+'footprint_3d.nc',
)
outgrids = (extent_density_grid,
init_density_grid,
event_density_grid,
footprint_grid,
)
outgrids_3d = (extent_density_grid_3d,
init_density_grid_3d,
event_density_grid_3d,
footprint_grid_3d,)
field_names = ('flash_extent', 'flash_initiation', 'lma_source', 'flash_footprint')
field_descriptions = ('LMA flash extent density',
'LMA flash initiation density',
'LMA source density',
'LMA local mean flash area')
if proj_name=='latlong':
density_units = "grid"
density_units_3d = "grid"
else:
density_units = "{0:5.1f} km^2".format(dx/1000.0 * dy/1000.0).lstrip()
density_units_3d = "{0:5.1f} km^3".format(dx/1000.0 * dy/1000.0 * dz/1000.0).lstrip()
time_units = "{0:5.1f} min".format(frame_interval/60.0).lstrip()
density_label = 'Count per ' + density_units + " pixel per "+ time_units
density_label_3d = 'Count per ' + density_units_3d + " pixel per "+ time_units
field_units = ( density_label,
density_label,
density_label,
"km^2 per flash",
)
field_units_3d = ( density_label_3d,
density_label_3d,
density_label_3d,
"km^2 per flash",
)
output_writer(outfiles[0], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[0], field_names[0], field_descriptions[0],
grid_units=field_units[0],
**output_kwargs)
output_writer(outfiles[1], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[1], field_names[1], field_descriptions[1],
grid_units=field_units[1],
**output_kwargs)
output_writer(outfiles[2], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[2], field_names[2], field_descriptions[2],
grid_units=field_units[2],
**output_kwargs)
output_writer(outfiles[3], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
lons, lats, ctr_lat, ctr_lon,
outgrids[3], field_names[3], field_descriptions[3], format='f',
grid_units=field_units[3],
**output_kwargs)
output_writer_3d(outfiles_3d[0], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[0], field_names[0], field_descriptions[0],
grid_units=field_units_3d[0],
**output_kwargs)
output_writer_3d(outfiles_3d[1], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[1], field_names[1], field_descriptions[1],
grid_units=field_units_3d[1],
**output_kwargs)
output_writer_3d(outfiles_3d[2], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[2], field_names[2], field_descriptions[2],
grid_units=field_units_3d[2],
**output_kwargs)
output_writer_3d(outfiles_3d[3], t_ref, np.asarray(t_edges_seconds[:-1]),
x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
z_coord*spatial_scale_factor,
lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
outgrids_3d[3], field_names[3], field_descriptions[3], format='f',
grid_units=field_units_3d[3],
**output_kwargs)
print('max extent is', extent_density_grid.max())
return x_coord, y_coord, z_coord, lons, lats, alts, extent_density_grid, outfiles, field_names
