def event_pipeline_setup(self): """ Create target grids and set up the event accumulation pipeline for events that have been pre-aggregated using an event lookup table. Arguments: none Returns: outgrids -- tuple of grids (numpy arrays) for the target grids framer -- first stage of the pipeline that routes flashes to the correct time window for accumulation. """ (event_grid_area_fraction_key, energy_grids, n_frames, xedge, yedge, zedge, dx, dy, dz, x0, y0, z0, mapProj, geoProj) = ( self.gridspec_locals()) grid_shape = (xedge.shape[0]-1, yedge.shape[0]-1, n_frames) event_density_grid = np.zeros(grid_shape, dtype='float32') total_energy_grid = np.zeros(grid_shape, dtype='float32') all_frames = [] for i in range(n_frames): accum_event_density = accumulate_energy_on_grid( event_density_grid[:,:,i], xedge, yedge, label='event extent', grid_frac_weights=False) event_density_target = point_density(accum_event_density, weight_key='lutevent_count', weight_flashes=False) # total_energy is built from split_event_energy, which has # already divided up the event energy into the sub-event # corresponding to each pixel. We don't need to weight by the # grid fractional area. We just need to sum the 'power' variable # which mimic_lma assigns the values of split_event_energy. accum_total_energy = accumulate_energy_on_grid( total_energy_grid[:,:,i], xedge, yedge, label='total energy', grid_frac_weights=False) total_energy_target = point_density(accum_total_energy, weight_key='power', weight_flashes=False) broadcast_targets = ( project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, total_energy_target, use_flashes=False), ) spew_to_density_types = broadcast( broadcast_targets ) all_frames.append(spew_to_density_types) frame_count_log = flash_count_log(self.flash_count_logfile) framer = flashes_to_frames(self.t_edges_seconds, all_frames, time_key='time', time_edges_datetime=self.t_edges, flash_counter=frame_count_log, do_events='time') outgrids = (event_density_grid, total_energy_grid) return outgrids, framer
def flash_pipeline_setup(self): """ Create target grids and set up the flash accumulation pipeline. Also used to set up an equivalent group accumulation pipeline. Arguments: none Returns: outgrids -- tuple of grids (numpy arrays) for the target grids framer -- first stage of the pipeline that routes flashes to the correct time window for accumulation. """ (event_grid_area_fraction_key, energy_grids, n_frames, xedge, yedge, zedge, dx, dy, dz, x0, y0, z0, mapProj, geoProj) = (self.gridspec_locals()) grid_shape = (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames) init_density_grid = np.zeros(grid_shape, dtype='float32') extent_density_grid = np.zeros(grid_shape, dtype='float32') footprint_grid = np.zeros(grid_shape, dtype='float32') # flashsize_std_grid = np.zeros(grid_shape, dtype='float32') all_frames = [] for i in range(n_frames): accum_init_density = accumulate_points_on_grid( init_density_grid[:, :, i], xedge, yedge, label='init') accum_extent_density = accumulate_points_on_grid( extent_density_grid[:, :, i], xedge, yedge, label='extent', grid_frac_weights=True) accum_footprint = accumulate_points_on_grid(footprint_grid[:, :, i], xedge, yedge, label='footprint', grid_frac_weights=True) # accum_flashstd = accumulate_points_on_grid_sdev( # flashsize_std_grid[:,:,i], footprint_grid[:,:,i], xedge, yedge, # label='flashsize_std', grid_frac_weights=True) init_density_target = point_density(accum_init_density) extent_density_target = extent_density( x0, y0, dx, dy, accum_extent_density, event_grid_area_fraction_key=event_grid_area_fraction_key) mean_footprint_target = extent_density( x0, y0, dx, dy, accum_footprint, weight_key='area', event_grid_area_fraction_key=event_grid_area_fraction_key) # std_flashsize_target = extent_density(x0, y0, dx, dy, # accum_flashstd, weight_key='area', # event_grid_area_fraction_key=event_grid_area_fraction_key) broadcast_targets = ( project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True), project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False), # project('lon', 'lat', 'alt', mapProj, geoProj, # std_flashsize_target, use_flashes=False), ) spew_to_density_types = broadcast(broadcast_targets) all_frames.append( extract_events_for_flashes(spew_to_density_types)) frame_count_log = flash_count_log(self.flash_count_logfile) framer = flashes_to_frames(self.t_edges_seconds, all_frames, time_key='start', time_edges_datetime=self.t_edges, flash_counter=frame_count_log) outgrids = ( init_density_grid, extent_density_grid, footprint_grid, # flashsize_std_grid, ) return outgrids, framer
def group_pipeline_setup(self): """ Create target grids and set up the flash accumulation pipeline. Also used to set up an equivalent group accumulation pipeline. Actually uses event-level data that have been pre-aggregated using an event lookup table. Arguments: none Returns: outgrids -- tuple of grids (numpy arrays) for the target grids framer -- first stage of the pipeline that routes flashes to the correct time window for accumulation. """ (event_grid_area_fraction_key, energy_grids, n_frames, xedge, yedge, zedge, dx, dy, dz, x0, y0, z0, mapProj, geoProj) = (self.gridspec_locals()) grid_shape = (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames) init_density_grid = np.zeros(grid_shape, dtype='float32') extent_density_grid = np.zeros(grid_shape, dtype='float32') footprint_grid = np.zeros(grid_shape, dtype='float32') all_frames = [] for i in range(n_frames): accum_init_density = accumulate_points_on_grid( init_density_grid[:, :, i], xedge, yedge, label='init') accum_extent_density = accumulate_var_on_grid_direct_idx( extent_density_grid[:, :, i], 'lutevent_group_count', 'mesh_xi', 'mesh_yi') accum_footprint = accumulate_var_on_grid_direct_idx( footprint_grid[:, :, i], 'lutevent_total_group_area', 'mesh_xi', 'mesh_yi') init_density_target = point_density(accum_init_density) broadcast_targets = ( no_projection('ctr_x', 'ctr_y', 'ctr_z', init_density_target, use_flashes=True), select_dataset(accum_extent_density, use_event_data=True), select_dataset(accum_footprint, use_event_data=True), ) spew_to_density_types = broadcast(broadcast_targets) all_frames.append(spew_to_density_types) frame_count_log = flash_count_log(self.flash_count_logfile) framer = flashes_to_frames(self.t_edges_seconds, all_frames, time_key='start', time_edges_datetime=self.t_edges, flash_counter=frame_count_log, do_events='time') outgrids = ( init_density_grid, extent_density_grid, footprint_grid, ) return outgrids, framer
def pipeline_setup(self): """ Create the grids for each variable of interest. Set up the data processing pipeline up to the point at which data are written to grids in memory. """ event_grid_area_fraction_key = self.event_grid_area_fraction_key energy_grids = self.energy_grids n_frames = self.n_frames xedge, yedge, zedge = self.xedge, self.yedge, self.zedge dx, dy, dz = self.dx, self.dy, self.dz x0 = xedge[0] y0 = yedge[0] z0 = zedge[0] mapProj = self.mapProj geoProj = self.geoProj event_density_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32') init_density_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32') if self.event_grid_area_fraction_key is not None: extent_density_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='float32') else: extent_density_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='int32') footprint_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='float32') specific_energy_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='float32') total_energy_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='float32') flashsize_std_grid = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, n_frames), dtype='float32') if self.do_3d == True: event_density_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='int32') init_density_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='int32') extent_density_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='int32') footprint_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='float32') specific_energy_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='float32') total_energy_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='float32') flashsize_std_grid_3d = np.zeros( (xedge.shape[0] - 1, yedge.shape[0] - 1, zedge.shape[0] - 1, n_frames), dtype='float32') self.outgrids = ( extent_density_grid, init_density_grid, event_density_grid, footprint_grid, specific_energy_grid, flashsize_std_grid, total_energy_grid, ) if self.do_3d == True: self.outgrids_3d = (extent_density_grid_3d, init_density_grid_3d, event_density_grid_3d, footprint_grid_3d, specific_energy_grid_3d, flashsize_std_grid_3d, total_energy_grid_3d) else: self.outgrids_3d = None all_frames = [] for i in range(n_frames): extent_out = {'name': 'extent'} init_out = {'name': 'init'} event_out = {'name': 'event'} std_out = {'name': 'std'} extent_out_3d = {'name': 'extent_3d'} init_out_3d = {'name': 'init_3d'} event_out_3d = {'name': 'event_3d'} std_out_3d = {'name': 'std_3d'} accum_event_density = accumulate_points_on_grid( event_density_grid[:, :, i], xedge, yedge, out=event_out, label='event') accum_init_density = accumulate_points_on_grid( init_density_grid[:, :, i], xedge, yedge, out=init_out, label='init') accum_extent_density = accumulate_points_on_grid( extent_density_grid[:, :, i], xedge, yedge, out=extent_out, label='extent', grid_frac_weights=True) accum_footprint = accumulate_points_on_grid(footprint_grid[:, :, i], xedge, yedge, label='footprint', grid_frac_weights=True) accum_specific_energy = accumulate_energy_on_grid( specific_energy_grid[:, :, i], xedge, yedge, out=extent_out, label='specific_energy', grid_frac_weights=True) accum_flashstd = accumulate_points_on_grid_sdev( flashsize_std_grid[:, :, i], footprint_grid[:, :, i], xedge, yedge, out=extent_out, label='flashsize_std', grid_frac_weights=True) accum_total_energy = accumulate_energy_on_grid( total_energy_grid[:, :, i], xedge, yedge, out=extent_out, label='total_energy', grid_frac_weights=True) if self.do_3d == True: accum_event_density_3d = accumulate_points_on_grid_3d( event_density_grid_3d[:, :, :, i], xedge, yedge, zedge, out=event_out_3d, label='event_3d') accum_init_density_3d = accumulate_points_on_grid_3d( init_density_grid_3d[:, :, :, i], xedge, yedge, zedge, out=init_out_3d, label='init_3d') accum_extent_density_3d = accumulate_points_on_grid_3d( extent_density_grid_3d[:, :, :, i], xedge, yedge, zedge, out=extent_out_3d, label='extent_3d') accum_footprint_3d = accumulate_points_on_grid_3d( footprint_grid_3d[:, :, :, i], xedge, yedge, zedge, label='footprint_3d') accum_specific_energy_3d = accumulate_energy_on_grid_3d( specific_energy_grid_3d[:, :, :, i], xedge, yedge, zedge, out=extent_out_3d, label='specific_energy_3d') accum_flashstd_3d = accumulate_points_on_grid_sdev_3d( flashsize_std_grid_3d[:, :, :, i], footprint_grid_3d[:, :, :, i], xedge, yedge, zedge, out=extent_out_3d, label='flashsize_std_3d') accum_total_energy_3d = accumulate_energy_on_grid_3d( total_energy_grid_3d[:, :, :, i], xedge, yedge, zedge, out=extent_out_3d, label='total_energy_3d') extent_out['func'] = accum_extent_density init_out['func'] = accum_init_density event_out['func'] = accum_event_density if self.do_3d == True: extent_out_3d['func'] = accum_extent_density_3d init_out_3d['func'] = accum_init_density_3d event_out_3d['func'] = accum_event_density_3d event_density_target = point_density(accum_event_density) init_density_target = point_density(accum_init_density) extent_density_target = extent_density( x0, y0, dx, dy, accum_extent_density, event_grid_area_fraction_key=event_grid_area_fraction_key) mean_footprint_target = extent_density( x0, y0, dx, dy, accum_footprint, weight_key='area', event_grid_area_fraction_key=event_grid_area_fraction_key) mean_energy_target = extent_density( x0, y0, dx, dy, accum_specific_energy, weight_key='specific_energy', event_grid_area_fraction_key=event_grid_area_fraction_key) mean_total_energy_target = extent_density( x0, y0, dx, dy, accum_total_energy, weight_key='total_energy', event_grid_area_fraction_key=event_grid_area_fraction_key) std_flashsize_target = extent_density( x0, y0, dx, dy, accum_flashstd, weight_key='area', event_grid_area_fraction_key=event_grid_area_fraction_key) if self.do_3d == True: event_density_target_3d = point_density_3d( accum_event_density_3d) init_density_target_3d = point_density_3d( accum_init_density_3d) extent_density_target_3d = extent_density_3d( x0, y0, z0, dx, dy, dz, accum_extent_density_3d) mean_footprint_target_3d = extent_density_3d( x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area') mean_energy_target_3d = extent_density_3d( x0, y0, z0, dx, dy, dz, accum_specific_energy_3d, weight_key='specific_energy') mean_total_energy_target_3d = extent_density_3d( x0, y0, z0, dx, dy, dz, accum_total_energy_3d, weight_key='total_energy') std_flashsize_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_flashstd_3d, weight_key='area') broadcast_targets = ( project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False), project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True), project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, std_flashsize_target, use_flashes=False), ) if energy_grids is not None: if ('specific_energy' == energy_grids) | ('specific_energy' in energy_grids): broadcast_targets += (project('lon', 'lat', 'alt', mapProj, geoProj, mean_energy_target, use_flashes=False), ) if ('total_energy' == energy_grids) | ('total_energy' in energy_grids): broadcast_targets += (project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target, use_flashes=False), ) if self.do_3d == True: broadcast_targets += ( project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False), project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True), project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, std_flashsize_target_3d, use_flashes=False), ) if energy_grids is not None: if ('specific_energy' == energy_grids) | ('specific_energy' in energy_grids): broadcast_targets += (project('lon', 'lat', 'alt', mapProj, geoProj, mean_energy_target_3d, use_flashes=False), ) if ('total_energy' == energy_grids) | ('total_energy' in energy_grids): broadcast_targets += (project( 'lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target_3d, use_flashes=False), ) spew_to_density_types = broadcast(broadcast_targets) all_frames.append( extract_events_for_flashes(spew_to_density_types)) frame_count_log = flash_count_log(self.flash_count_logfile) framer = flashes_to_frames(self.t_edges_seconds, all_frames, time_key='start', time_edges_datetime=self.t_edges, flash_counter=frame_count_log) self.framer = framer
def pipeline_setup(self): """ Create the grids for each variable of interest. Set up the data processing pipeline up to the point at which data are written to grids in memory. """ event_grid_area_fraction_key=self.event_grid_area_fraction_key energy_grids=self.energy_grids n_frames = self.n_frames xedge, yedge, zedge = self.xedge, self.yedge, self.zedge dx, dy, dz = self.dx, self.dy, self.dz x0 = xedge[0] y0 = yedge[0] z0 = zedge[0] mapProj = self.mapProj geoProj = self.geoProj event_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32') init_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32') if self.event_grid_area_fraction_key is not None: extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32') else: extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32') footprint_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32') specific_energy_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32') total_energy_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32') flashsize_std_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32') if self.do_3d == True: event_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32') init_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32') extent_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32') footprint_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32') specific_energy_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32') total_energy_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32') flashsize_std_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32') self.outgrids = (extent_density_grid, init_density_grid, event_density_grid, footprint_grid, specific_energy_grid, flashsize_std_grid, total_energy_grid, ) if self.do_3d == True: self.outgrids_3d = (extent_density_grid_3d, init_density_grid_3d, event_density_grid_3d, footprint_grid_3d, specific_energy_grid_3d, flashsize_std_grid_3d, total_energy_grid_3d ) else: self.outgrids_3d = None all_frames = [] for i in range(n_frames): extent_out = {'name':'extent'} init_out = {'name':'init'} event_out = {'name':'event'} std_out = {'name':'std'} extent_out_3d = {'name':'extent_3d'} init_out_3d = {'name':'init_3d'} event_out_3d = {'name':'event_3d'} std_out_3d = {'name':'std_3d'} accum_event_density = accumulate_points_on_grid(event_density_grid[:,:,i], xedge, yedge, out=event_out, label='event') accum_init_density = accumulate_points_on_grid(init_density_grid[:,:,i], xedge, yedge, out=init_out, label='init') accum_extent_density = accumulate_points_on_grid(extent_density_grid[:,:,i], xedge, yedge, out=extent_out,label='extent', grid_frac_weights=True) accum_footprint = accumulate_points_on_grid(footprint_grid[:,:,i], xedge, yedge, label='footprint', grid_frac_weights=True) accum_specific_energy = accumulate_energy_on_grid(specific_energy_grid[:,:,i], xedge, yedge, out=extent_out, label='specific_energy', grid_frac_weights=True) accum_flashstd = accumulate_points_on_grid_sdev(flashsize_std_grid[:,:,i], footprint_grid[:,:,i], xedge, yedge, out=extent_out, label='flashsize_std', grid_frac_weights=True) accum_total_energy = accumulate_energy_on_grid(total_energy_grid[:,:,i], xedge, yedge, out=extent_out, label='total_energy', grid_frac_weights=True) if self.do_3d == True: accum_event_density_3d = accumulate_points_on_grid_3d(event_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=event_out_3d, label='event_3d') accum_init_density_3d = accumulate_points_on_grid_3d(init_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=init_out_3d, label='init_3d') accum_extent_density_3d = accumulate_points_on_grid_3d(extent_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='extent_3d') accum_footprint_3d = accumulate_points_on_grid_3d(footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, label='footprint_3d') accum_specific_energy_3d = accumulate_energy_on_grid_3d(specific_energy_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='specific_energy_3d') accum_flashstd_3d = accumulate_points_on_grid_sdev_3d(flashsize_std_grid_3d[:,:,:,i], footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='flashsize_std_3d') accum_total_energy_3d = accumulate_energy_on_grid_3d(total_energy_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='total_energy_3d') extent_out['func'] = accum_extent_density init_out['func'] = accum_init_density event_out['func'] = accum_event_density if self.do_3d == True: extent_out_3d['func'] = accum_extent_density_3d init_out_3d['func'] = accum_init_density_3d event_out_3d['func'] = accum_event_density_3d event_density_target = point_density(accum_event_density) init_density_target = point_density(accum_init_density) extent_density_target = extent_density(x0, y0, dx, dy, accum_extent_density, event_grid_area_fraction_key=event_grid_area_fraction_key) mean_footprint_target = extent_density(x0, y0, dx, dy, accum_footprint, weight_key='area', event_grid_area_fraction_key=event_grid_area_fraction_key) mean_energy_target = extent_density(x0, y0, dx, dy, accum_specific_energy, weight_key='specific_energy', event_grid_area_fraction_key=event_grid_area_fraction_key) mean_total_energy_target = extent_density(x0, y0, dx, dy, accum_total_energy, weight_key='total_energy', event_grid_area_fraction_key=event_grid_area_fraction_key) std_flashsize_target = extent_density(x0, y0, dx, dy, accum_flashstd, weight_key='area', event_grid_area_fraction_key=event_grid_area_fraction_key) if self.do_3d == True: event_density_target_3d = point_density_3d(accum_event_density_3d) init_density_target_3d = point_density_3d(accum_init_density_3d) extent_density_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_extent_density_3d) mean_footprint_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area') mean_energy_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_specific_energy_3d, weight_key='specific_energy') mean_total_energy_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_total_energy_3d, weight_key='total_energy') std_flashsize_target_3d = extent_density_3d(x0, y0, z0, dx, dy, dz, accum_flashstd_3d, weight_key='area') broadcast_targets = ( project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False), project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True), project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, std_flashsize_target, use_flashes=False), ) if energy_grids is not None: if ('specific_energy' == energy_grids) | ('specific_energy' in energy_grids): broadcast_targets += ( project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target, use_flashes=False), ) if ('total_energy' == energy_grids) | ('total_energy' in energy_grids): broadcast_targets += ( project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target, use_flashes=False), ) if self.do_3d == True: broadcast_targets += ( project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False), project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True), project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False), project('lon', 'lat', 'alt', mapProj, geoProj, std_flashsize_target_3d, use_flashes=False), ) if energy_grids is not None: if ('specific_energy' == energy_grids) | ('specific_energy' in energy_grids): broadcast_targets += ( project('lon', 'lat', 'alt', mapProj, geoProj, mean_energy_target_3d, use_flashes=False), ) if ('total_energy' == energy_grids) | ('total_energy' in energy_grids): broadcast_targets += ( project('lon', 'lat', 'alt', mapProj, geoProj, mean_total_energy_target_3d, use_flashes=False), ) spew_to_density_types = broadcast( broadcast_targets ) all_frames.append( extract_events_for_flashes( spew_to_density_types ) ) frame_count_log = flash_count_log(self.flash_count_logfile) framer = flashes_to_frames(self.t_edges_seconds, all_frames, time_key='start', time_edges_datetime=self.t_edges, flash_counter=frame_count_log) self.framer=framer
def grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=120.0, dx=4.0e3, dy=4.0e3, dz=1.0e3, x_bnd = (-100e3, 100e3), y_bnd = (-100e3, 100e3), z_bnd = (0e3, 20e3), ctr_lat = 35.23833, ctr_lon = -97.46028, ctr_alt=0.0, min_points_per_flash=10, outpath = '', flash_count_logfile = None, proj_name = 'aeqd', proj_datum = 'WGS84', proj_ellipse = 'WGS84', output_writer = write_cf_netcdf, output_writer_3d = write_cf_netcdf_3d, output_filename_prefix="LMA", output_kwargs = {}, spatial_scale_factor = 1.0/1000.0, ): from math import ceil """ Create 2D plan-view density grids for events, flash origins, flash extents, and mean flash footprint frame_interval: Frame time-step in seconds dx, dy: horizontal grid size in m (or deg) {x,y,z}_bnd: horizontal grid edges in m ctr_lat, ctr_lon: coordinate center Uses an azimuthal equidistant map projection on the WGS84 ellipsoid. read_flashes filter_flash extract_events flash_to_frame frame0_broadcast, frame1_broadcast, ... each broadcaster above sends events and flashes to: projection( event_location), projection(flash_init_location), projection(event_location) which map respectively to: point_density->accum_on_grid(event density), point_density->accum_on_grid(flash init density), extent_density->accum_on_grid(flash_extent_density) grids are in an HDF5 file. how to handle flushing? """ if flash_count_logfile is None: flash_count_logfile = sys.stdout # reference time is the date part of the start_time t_edges, duration = time_edges(start_time, end_time, frame_interval) t_ref, t_edges_seconds = seconds_since_start_of_day(start_time, t_edges) n_frames = len(t_edges)-1 xedge=np.arange(x_bnd[0], x_bnd[1]+dx, dx) yedge=np.arange(y_bnd[0], y_bnd[1]+dy, dy) zedge=np.arange(z_bnd[0], z_bnd[1]+dz, dz) x0 = xedge[0] y0 = yedge[0] z0 = zedge[0] if proj_name == 'latlong': dx_units = '{0:6.4f}deg'.format(dx) mapProj = GeographicSystem() else: dx_units = '{0:5.1f}m'.format(dx) mapProj = MapProjection(projection=proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat, lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum) geoProj = GeographicSystem() event_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32') init_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32') extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32') footprint_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32') event_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32') init_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32') extent_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32') footprint_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32') all_frames = [] # extent_frames = [] # init_frames = [] # event_frames = [] # extent_frames_3d = [] # init_frames_3d = [] # event_frames_3d = [] for i in range(n_frames): extent_out = {'name':'extent'} init_out = {'name':'init'} event_out = {'name':'event'} extent_out_3d = {'name':'extent_3d'} init_out_3d = {'name':'init_3d'} event_out_3d = {'name':'event_3d'} accum_event_density = density_to_files.accumulate_points_on_grid(event_density_grid[:,:,i], xedge, yedge, out=event_out, label='event') accum_init_density = density_to_files.accumulate_points_on_grid(init_density_grid[:,:,i], xedge, yedge, out=init_out, label='init') accum_extent_density = density_to_files.accumulate_points_on_grid(extent_density_grid[:,:,i], xedge, yedge, out=extent_out,label='extent') accum_footprint = density_to_files.accumulate_points_on_grid(footprint_grid[:,:,i], xedge, yedge, label='footprint') accum_event_density_3d = density_to_files.accumulate_points_on_grid_3d(event_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=event_out_3d, label='event_3d') accum_init_density_3d = density_to_files.accumulate_points_on_grid_3d(init_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=init_out_3d, label='init_3d') accum_extent_density_3d = density_to_files.accumulate_points_on_grid_3d(extent_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='extent_3d') accum_footprint_3d = density_to_files.accumulate_points_on_grid_3d(footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, label='footprint_3d') extent_out['func'] = accum_extent_density init_out['func'] = accum_init_density event_out['func'] = accum_event_density # extent_frames.append(extent_out) # init_frames.append(init_out) # event_frames.append(event_out) extent_out_3d['func'] = accum_extent_density_3d init_out_3d['func'] = accum_init_density_3d event_out_3d['func'] = accum_event_density_3d # extent_frames_3d.append(extent_out_3d) # init_frames_3d.append(init_out_3d) # event_frames_3d.append(event_out_3d) event_density_target = density_to_files.point_density(accum_event_density) init_density_target = density_to_files.point_density(accum_init_density) extent_density_target = density_to_files.extent_density(x0, y0, dx, dy, accum_extent_density) mean_footprint_target = density_to_files.extent_density(x0, y0, dx, dy, accum_footprint, weight_key='area') event_density_target_3d = density_to_files.point_density_3d(accum_event_density_3d) init_density_target_3d = density_to_files.point_density_3d(accum_init_density_3d) extent_density_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_extent_density_3d) mean_footprint_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area') spew_to_density_types = broadcast( ( density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False), density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True), density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False), density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False), density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False), density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True), density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False), density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False), ) ) all_frames.append( density_to_files.extract_events_for_flashes( spew_to_density_types ) ) frame_count_log = density_to_files.flash_count_log(flash_count_logfile) framer = density_to_files.flashes_to_frames(t_edges_seconds, all_frames, time_key='start', time_edges_datetime=t_edges, flash_counter=frame_count_log) read_flashes( h5_filenames, framer, base_date=t_ref, min_points=min_points_per_flash) # print 'event_density_grid ', id(event_density_grid[:,:,-1]) # print 'extent_density_grid', id(extent_density_grid[:,:,-1]) # print 'init_density_grid ', id(init_density_grid[:,:,-1]) x_coord = (xedge[:-1] + xedge[1:])/2.0 y_coord = (yedge[:-1] + yedge[1:])/2.0 z_coord = (zedge[:-1] + zedge[1:])/2.0 nx = x_coord.shape[0] ny = y_coord.shape[0] nz = z_coord.shape[0] x_all, y_all = (a.T for a in np.meshgrid(x_coord, y_coord)) assert x_all.shape == y_all.shape assert x_all.shape[0] == nx assert x_all.shape[1] == ny z_all = np.zeros_like(x_all) grid_shape_3d = (nx,ny,nz) x_ones_3d = np.ones(grid_shape_3d, dtype='f4') y_ones_3d = np.ones(grid_shape_3d, dtype='f4') z_ones_3d = np.ones(grid_shape_3d, dtype='f4') x_all_3d = x_coord[:, None, None]*x_ones_3d y_all_3d = y_coord[None,:,None]*y_ones_3d z_all_3d = z_coord[None, None, :]*z_ones_3d lons, lats, alts = x,y,z = geoProj.fromECEF( *mapProj.toECEF(x_all, y_all, z_all) ) lons.shape=x_all.shape lats.shape=y_all.shape lons_3d, lats_3d, alts_3d = x_3d,y_3d,z_3d = geoProj.fromECEF( *mapProj.toECEF(x_all_3d, y_all_3d, z_all_3d) ) lons_3d.shape=x_all_3d.shape lats_3d.shape=y_all_3d.shape alts_3d.shape=z_all_3d.shape outflile_basename = os.path.join(outpath,'%s_%s_%d_%dsrc_%s-dx_' % (output_filename_prefix, start_time.strftime('%Y%m%d_%H%M%S'), to_seconds(duration), min_points_per_flash, dx_units)) outfiles = (outflile_basename+'flash_extent.nc', outflile_basename+'flash_init.nc', outflile_basename+'source.nc', outflile_basename+'footprint.nc', ) outfiles_3d = (outflile_basename+'flash_extent_3d.nc', outflile_basename+'flash_init_3d.nc', outflile_basename+'source_3d.nc', outflile_basename+'footprint_3d.nc', ) outgrids = (extent_density_grid, init_density_grid, event_density_grid, footprint_grid, ) outgrids_3d = (extent_density_grid_3d, init_density_grid_3d, event_density_grid_3d, footprint_grid_3d,) field_names = ('flash_extent', 'flash_initiation', 'lma_source', 'flash_footprint') field_descriptions = ('LMA flash extent density', 'LMA flash initiation density', 'LMA source density', 'LMA local mean flash area') if proj_name=='latlong': density_units = "grid" density_units_3d = "grid" else: density_units = "{0:5.1f} km^2".format(dx/1000.0 * dy/1000.0).lstrip() density_units_3d = "{0:5.1f} km^3".format(dx/1000.0 * dy/1000.0 * dz/1000.0).lstrip() time_units = "{0:5.1f} min".format(frame_interval/60.0).lstrip() density_label = 'Count per ' + density_units + " pixel per "+ time_units density_label_3d = 'Count per ' + density_units_3d + " pixel per "+ time_units field_units = ( density_label, density_label, density_label, "km^2 per flash", ) field_units_3d = ( density_label_3d, density_label_3d, density_label_3d, "km^2 per flash", ) output_writer(outfiles[0], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, lons, lats, ctr_lat, ctr_lon, outgrids[0], field_names[0], field_descriptions[0], grid_units=field_units[0], **output_kwargs) output_writer(outfiles[1], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, lons, lats, ctr_lat, ctr_lon, outgrids[1], field_names[1], field_descriptions[1], grid_units=field_units[1], **output_kwargs) output_writer(outfiles[2], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, lons, lats, ctr_lat, ctr_lon, outgrids[2], field_names[2], field_descriptions[2], grid_units=field_units[2], **output_kwargs) output_writer(outfiles[3], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, lons, lats, ctr_lat, ctr_lon, outgrids[3], field_names[3], field_descriptions[3], format='f', grid_units=field_units[3], **output_kwargs) output_writer_3d(outfiles_3d[0], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, z_coord*spatial_scale_factor, lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt, outgrids_3d[0], field_names[0], field_descriptions[0], grid_units=field_units_3d[0], **output_kwargs) output_writer_3d(outfiles_3d[1], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, z_coord*spatial_scale_factor, lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt, outgrids_3d[1], field_names[1], field_descriptions[1], grid_units=field_units_3d[1], **output_kwargs) output_writer_3d(outfiles_3d[2], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, z_coord*spatial_scale_factor, lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt, outgrids_3d[2], field_names[2], field_descriptions[2], grid_units=field_units_3d[2], **output_kwargs) output_writer_3d(outfiles_3d[3], t_ref, np.asarray(t_edges_seconds[:-1]), x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, z_coord*spatial_scale_factor, lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt, outgrids_3d[3], field_names[3], field_descriptions[3], format='f', grid_units=field_units_3d[3], **output_kwargs) print('max extent is', extent_density_grid.max()) return x_coord, y_coord, z_coord, lons, lats, alts, extent_density_grid, outfiles, field_names