Exemplo n.º 1
0
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
    """
    There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
    
    """
    logger = logging.getLogger('FlashAutorunLogger')
    
    
    if 'mask_length' in params:
        mask_length = params['mask_length']
    else:
        mask_length = 4
    
    lma=LMAdataFile(a_file, mask_length = mask_length)
    # for line in lma.header:
        # print line

    ctr_lat, ctr_lon, ctr_alt =  params['ctr_lat'], params['ctr_lon'], 0.0

    good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1]) 
    if 'alt' in params:
        good = good & (lma.alt < params['alt'][1])
    
    
    data = lma.data[good]
    geoCS = GeographicSystem()
    X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
    Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
    X, Y, Z = X-Xc, Y-Yc, Z-Zc
    
    print "sorting {0} total points".format(data.shape[0])

    D_max, t_max = params['distance'], params['thresh_critical_time'] # m, s
    duration_max = params['thresh_duration']

    IDs = np.arange(X.shape[0])
    X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
    T_vector = data['time'][:,None] / t_max
    XYZT = np.hstack((X_vector, T_vector))
    
    lma.sort_status = 'in process'
    
    # Maximum 3 s flash length, normalized to the time separation scale

    flash_object_maker = create_flash_objs(lma, data)
    label_aggregator = aggregate_ids(flash_object_maker)
    clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
    chunker = chunk(XYZT[:,-1].min(), duration_max/t_max,  clusterer)
    stream(XYZT.astype('float64'), IDs,chunker)
    flash_object_maker.close()
    
    # These are handled by target.close in each coroutine's GeneratorExit handler
    # clusterer.close()
    # label_aggregator.close()
    # flash_object_maker.close()
    
    print lma.sort_status
    print len(lma.flash_objects)
                    
    return lma, lma.flash_objects
Exemplo n.º 2
0
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
    """
    There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
    
    """
    logger = logging.getLogger('FlashAutorunLogger')
    
    
    if 'mask_length' in params:
        mask_length = params['mask_length']
    else:
        mask_length = 4
    
    lma=LMAdataFile(a_file, mask_length = mask_length)
    # for line in lma.header:
        # print line

    ctr_lat, ctr_lon, ctr_alt =  params['ctr_lat'], params['ctr_lon'], 0.0

    good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1]) 
    if 'alt' in params:
        good = good & (lma.alt < params['alt'][1])
    
    
    data = lma.data[good]
    geoCS = GeographicSystem()
    X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
    Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
    X, Y, Z = X-Xc, Y-Yc, Z-Zc
    
    print "sorting {0} total points".format(data.shape[0])

    D_max, t_max = 3.0e3, 0.15 # m, s

    IDs = np.arange(X.shape[0])
    X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
    T_vector = data['time'][:,None] / t_max
    XYZT = np.hstack((X_vector, T_vector))
    
    lma.sort_status = 'in process'
    
    # Maximum 3 s flash length, normalized to the time separation scale

    flash_object_maker = create_flash_objs(lma, data)
    label_aggregator = aggregate_ids(flash_object_maker)
    clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
    chunker = chunk(XYZT[:,-1].min(), 3.0/.15,  clusterer)
    stream(XYZT.astype('float64'), IDs,chunker)
    flash_object_maker.close()
    
    # These are handled by target.close in each coroutine's GeneratorExit handler
    # clusterer.close()
    # label_aggregator.close()
    # flash_object_maker.close()
    
    print lma.sort_status
    print len(lma.flash_objects)
                    
    return lma, lma.flash_objects
Exemplo n.º 3
0
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs):
    """
    There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects.
    
    """
    logger = logging.getLogger('FlashAutorunLogger')
    
    lma=LMAdataFile(a_file)
    # for line in lma.header:
        # print line

    ctr_lat, ctr_lon, ctr_alt =  params['ctr_lat'], params['ctr_lon'], 0.0

    good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1]) 
    if 'alt' in params:
        good = good & (lma.alt < params['alt'][1])
    
    
    data = lma.data[good]
    geoCS = GeographicSystem()
    X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
    Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
    X, Y, Z = X-Xc, Y-Yc, Z-Zc
    
    print("sorting {0} total points".format(data.shape[0]))

    D_max, t_max = params['distance'], params['thresh_critical_time'] # m, s
    duration_max = params['thresh_duration']

    IDs = np.arange(X.shape[0])
    X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
    T_vector = data['time'][:,None] / t_max
    XYZT = np.hstack((X_vector, T_vector))
    

    
    # Maximum 3 s flash length, normalized to the time separation scale

    flash_object_maker = create_flash_objs(lma, data)
    label_aggregator = aggregate_ids(flash_object_maker)
    clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points)
    if XYZT.shape[0] < 1:
        # no data, so minimum time is zero. Assume nothing is done with the data,
        # so that time doesn't matter. No flashes can result.
        chunker = chunk(0, duration_max/t_max,  clusterer)
    else:
        chunker = chunk(XYZT[:,-1].min(), duration_max/t_max,  clusterer)
    stream(XYZT.astype('float64'), IDs,chunker)
    flash_object_maker.close()
    
    # These are handled by target.close in each coroutine's GeneratorExit handler
    # clusterer.close()
    # label_aggregator.close()
    # flash_object_maker.close()
    
    print(len(lma.flash_objects))
                    
    return lma, lma.flash_objects
Exemplo n.º 4
0
 def geo_to_cartesisan(self, lon, lat, alt):
     """ Convert lat, lon in degrees and altitude in meters to 
         Earth-centered, Earth-fixed cartesian coordinates. Translate
         to coordinate center location. Returns X,Y,Z in meters.
     """
     geoCS = GeographicSystem()
     X, Y, Z = geoCS.toECEF(lon, lat, alt)
     Xc, Yc, Zc = geoCS.toECEF(self.ctr_lon, self.ctr_lat, self.ctr_alt)
     X, Y, Z = X - Xc, Y - Yc, Z - Zc
     return (X, Y, Z)
Exemplo n.º 5
0
def test_fixed_grid_GOESR():
    """ Tests from the GOES-R PUG Volume 3, L1b data """
    sat_lon_nadir = -75.0
    goes_sweep = 'x'  # Meteosat is 'y'
    ellipse = 'GRS80'
    datum = 'WGS84'
    sat_ecef_height = 35786023.0

    geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
                                            ellipse=ellipse,
                                            datum=datum,
                                            sweep_axis=goes_sweep,
                                            sat_ecef_height=sat_ecef_height)
    latloncs = GeographicSystem(ellipse=ellipse, datum=datum)

    test_lat = 33.846162
    test_lon = -84.690932
    test_alt = 0.0

    test_fixx = -0.024052
    test_fixy = 0.095340
    test_fixz = 0.0

    atol = 1e-6

    # Test forward from geodetic
    X, Y, Z = latloncs.toECEF(test_lon, test_lat, test_alt)
    x, y, z = geofixcs.fromECEF(X, Y, Z)
    assert_allclose(test_fixx, x, rtol=atol)
    assert_allclose(test_fixy, y, rtol=atol)
    assert_allclose(test_fixz, z, rtol=atol)

    # print(test_fixx, test_fixy, test_fixz)
    # print(x,y,z)

    # Test inverse from fixed grid angle
    X, Y, Z = geofixcs.toECEF(test_fixx, test_fixy, test_fixz)
    x, y, z = latloncs.fromECEF(X, Y, Z)
    assert_allclose(test_lon, x, atol=atol)
    assert_allclose(test_lat, y, atol=atol)
    assert_allclose(test_alt, z, atol=atol)
Exemplo n.º 6
0
def test_fixed_grid_GOESR():
    """ Tests from the GOES-R PUG Volume 3, L1b data """
    sat_lon_nadir = -75.0
    goes_sweep = 'x' # Meteosat is 'y'
    ellipse = 'GRS80'
    datum = 'WGS84'
    sat_ecef_height=35786023.0
        
    geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir,
                   ellipse=ellipse, datum=datum, sweep_axis=goes_sweep,
                   sat_ecef_height=sat_ecef_height)
    latloncs = GeographicSystem(ellipse=ellipse, datum=datum)
    

    test_lat = 33.846162
    test_lon = -84.690932
    test_alt = 0.0
    
    test_fixx = -0.024052
    test_fixy = 0.095340
    test_fixz = 0.0
    
    atol = 1e-6
    
    # Test forward from geodetic
    X,Y,Z = latloncs.toECEF(test_lon, test_lat, test_alt)
    x, y, z = geofixcs.fromECEF(X, Y, Z)
    assert_allclose(test_fixx, x, rtol=atol)
    assert_allclose(test_fixy, y, rtol=atol)
    assert_allclose(test_fixz, z, rtol=atol)
    
    # print(test_fixx, test_fixy, test_fixz)
    # print(x,y,z)

    # Test inverse from fixed grid angle
    X,Y,Z = geofixcs.toECEF(test_fixx, test_fixy, test_fixz)
    x, y, z = latloncs.fromECEF(X, Y, Z)
    assert_allclose(test_lon, x, atol=atol)
    assert_allclose(test_lat, y, atol=atol)
    assert_allclose(test_alt, z, atol=atol)
Exemplo n.º 7
0
    except GeneratorExit:
        print(total)


#lma=LMAdataFile('/Users/ebruning/Documents/Lightning\ interpretation/Flash-length/Thomas/LYLOUT_120412_01817.exported.dat.gz')
#ctr_lat, ctr_lon, ctr_alt =  40.4463980, -104.6368130, 1000.00

lma = LMAdataFile('/data/20040526/LMA/LYLOUT_040526_224000_0600.dat.gz')
# for line in lma.header:
# print line

ctr_lat, ctr_lon, ctr_alt = 35.2791257, -97.9178678, 417.90  # OKLMA
#ctr_lat, ctr_lon, ctr_alt =  40.4463980, -104.6368130, 1000.00 # COLMA
good = (lma.stations >= 6) & (lma.chi2 <= 2.0) & (lma.alt < 20e3)
data = lma.data[good]
geoCS = GeographicSystem()
X, Y, Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF(ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X - Xc, Y - Yc, Z - Zc

D_max, t_max = 3.0e3, 0.15  # m, s

X_vector = np.hstack((X[:, None], Y[:, None], Z[:, None])) / D_max
T_vector = data['time'][:, None] / t_max
XYZT = np.hstack((X_vector, T_vector - T_vector.min()))

# Maximum 3 s flash length, normalized to the time separation scale
chunker = chunk(XYZT[:, -1].min(), 3.0 / .15,
                cluster_chunk_pairs(cluster_printer()))
stream(XYZT.astype('float32'), chunker)
Exemplo n.º 8
0
def get_geometry_hrrr(glm, nadir, hrrr, ltgellipsever=1):

    # We don't use X, Y, Z below.
    # x, y are the fixed grid 2D coord arrays (from meshgrid)
    # X, Y, Z are the ECEF coords of each pixel intersected at the ltg ellipsoid
    # lon_ltg, lat_ltg are the parallax-corrected lon lat at the earth's surface
    # below the lightning position on the lightning ellipsoid.
    # outside_glm_full_disk is a boolean mask for the positions that GLM can't
    #   observe.
    # All of the above are 2D arrays corrsponding the center positions of the 2
    # km fixed grid pixels in the GLM gridded products.

    ((x, y), (X, Y, Z), (lon_ltg, lat_ltg, alt_ltg),
     outside_glm_full_disk) = get_glm_earth_geometry(glm, nadir, ltgellipsever)

    ### HRRR interpolation ###

    # Everything below presumes LCC.
    assert hrrr.MAP_PROJ_CHAR == 'Lambert Conformal'

    corner_0_lla = (hrrr.XLONG[0, 0, 0].data, hrrr.XLAT[0, 0, 0].data,
                    np.asarray(0.0, dtype=hrrr.XLAT[0, 0, 0].dtype))
    corner_1_lla = (hrrr.XLONG[0, -1, -1].data, hrrr.XLAT[0, -1, -1].data,
                    np.asarray(0.0, dtype=hrrr.XLAT[0, 1, -1].dtype))

    hrrr_dx, hrrr_dy = hrrr.DX, hrrr.DX
    hrrr_Nx, hrrr_Ny = hrrr.dims['west_east'], hrrr.dims['south_north']

    hrrrproj = {
        'lat_0': hrrr.CEN_LAT,
        'lon_0': hrrr.CEN_LON + 360.0,
        'lat_1': hrrr.TRUELAT1,
        'lat_2': hrrr.TRUELAT2,
        # 'R':hrrr.LambertConformal_Projection.earth_radius,
        # 'a':6371229,
        # 'b':6371229,
        'R': 6371229,
    }

    lcc = MapProjection(projection='lcc',
                        ctrLat=hrrrproj['lat_0'],
                        ctrLon=hrrrproj['lon_0'],
                        **hrrrproj)
    hrrr_lla = GeographicSystem(r_equator=hrrrproj['R'], r_pole=hrrrproj['R'])

    lcc_cornerx_0, lcc_cornery_0, lcc_cornerz_0 = lcc.fromECEF(
        *hrrr_lla.toECEF(*corner_0_lla))
    lcc_cornerx_1, lcc_cornery_1, lcc_cornerz_1 = lcc.fromECEF(
        *hrrr_lla.toECEF(*corner_1_lla))

    # def grid_idx(x, y, x0, y0, dx, dy):
    #     """
    #     Convert x, y returned by [projection].fromECEF to the grid index in the
    #     NetCDF file. x0 and y0 are the [projection] coordinates of the center of
    #     the zero-index position in the NetCDF grid. dx and dy are the grid spacing
    #     in meters.
    #
    #     returns (xidx, yidx)
    #     Taking int(xidx) will give the zero-based grid cell index.
    #     """
    #     # To get the correct behavior with int(xidx), add a half
    #     # since x0 is the center.
    #     xidx = (x-x0)/dx + 0.5
    #     yidx = (y-y0)/dy + 0.5
    #     return xidx, yidx

    # The 3D position (X,Y,Z) defines an implicit lon, lat, alt with respect to the
    # spherical earth we specified for the HRRR and its associated Lambert
    # conformal projection. We let proj4 handle the mapping from the ECEF
    # coordinates (an absolute position) directly to LCC.

    lccx2, lccy2, lccz2 = lcc.fromECEF(
        *hrrr_lla.toECEF(lon_ltg, lat_ltg, np.zeros_like(lon_ltg)))
    lccx2.shape = x.shape
    lccy2.shape = x.shape
    lccx = lccx2
    lccy = lccy2

    # Set up the model grid, since the hrrr file doesn't include those values.

    hrrrx_1d = np.arange(hrrr_Nx, dtype='f4') * hrrr_dx + lcc_cornerx_0
    hrrry_1d = np.arange(hrrr_Ny, dtype='f4') * hrrr_dy + lcc_cornery_0
    hrrrx, hrrry = np.meshgrid(hrrrx_1d, hrrry_1d)
    interp_loc = np.vstack((hrrrx.flatten(), hrrry.flatten())).T

    # GLM variables are filled with nan everywhere there is no lightning,
    # so set those locations corresponding to valid earth locations to zero.

    lcc_glm_x_flat = lccx[:, :].flatten()
    lcc_glm_y_flat = lccy[:, :].flatten()

    good = np.isfinite(lcc_glm_x_flat) & np.isfinite(lcc_glm_y_flat)
    good = good & (~outside_glm_full_disk.flatten())
    data_loc = np.vstack((lcc_glm_x_flat[good], lcc_glm_y_flat[good])).T

    return (data_loc, good, interp_loc, hrrrx.shape)
Exemplo n.º 9
0



#lma=LMAdataFile('/Users/ebruning/Documents/Lightning\ interpretation/Flash-length/Thomas/LYLOUT_120412_01817.exported.dat.gz')
#ctr_lat, ctr_lon, ctr_alt =  40.4463980, -104.6368130, 1000.00

lma=LMAdataFile('/data/20040526/LMA/LYLOUT_040526_224000_0600.dat.gz')
# for line in lma.header:
    # print line

ctr_lat, ctr_lon, ctr_alt =  35.2791257, -97.9178678, 417.90 # OKLMA
#ctr_lat, ctr_lon, ctr_alt =  40.4463980, -104.6368130, 1000.00 # COLMA
good = (lma.stations >= 6) & (lma.chi2 <= 2.0) & (lma.alt < 20e3)
data = lma.data[good]
geoCS = GeographicSystem()
X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt'])
Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt)
X, Y, Z = X-Xc, Y-Yc, Z-Zc


D_max, t_max = 3.0e3, 0.15 # m, s

X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max
T_vector = data['time'][:,None] / t_max
XYZT = np.hstack((X_vector, T_vector-T_vector.min()))

# Maximum 3 s flash length, normalized to the time separation scale
chunker = chunk(XYZT[:,-1].min(), 3.0/.15, cluster_chunk_pairs(cluster_printer()))
stream(XYZT.astype('float32'),chunker)
Exemplo n.º 10
0
def grid_h5flashfiles(h5_filenames, start_time, end_time, 
                        frame_interval=120.0, dx=4.0e3, dy=4.0e3, dz=1.0e3,
                        x_bnd = (-100e3, 100e3),
                        y_bnd = (-100e3, 100e3),
                        z_bnd = (0e3, 20e3),
                        ctr_lat = 35.23833, ctr_lon = -97.46028, ctr_alt=0.0,
                        min_points_per_flash=10,
                        outpath = '',
                        flash_count_logfile = None,
                        proj_name = 'aeqd',
                        proj_datum = 'WGS84',
                        proj_ellipse = 'WGS84',
                        output_writer = write_cf_netcdf, 
                        output_writer_3d = write_cf_netcdf_3d,
                        output_filename_prefix="LMA",
                        output_kwargs = {},
                        spatial_scale_factor = 1.0/1000.0,
                        ):
    from math import ceil
    """
    
    Create 2D plan-view density grids for events, flash origins, flash extents, and mean flash footprint
    
    frame_interval: Frame time-step in seconds
    dx, dy: horizontal grid size in m (or deg)
    {x,y,z}_bnd: horizontal grid edges in m
    ctr_lat, ctr_lon: coordinate center
    
    Uses an azimuthal equidistant map projection on the WGS84 ellipsoid.
    
    
    read_flashes
    
    filter_flash
    extract_events
    flash_to_frame
    
    frame0_broadcast, frame1_broadcast, ...
    
    each broadcaster above sends events and flashes to:
    projection( event_location), projection(flash_init_location), projection(event_location)
    which map respectively to:
    point_density->accum_on_grid(event density), point_density->accum_on_grid(flash init density), extent_density->accum_on_grid(flash_extent_density)
    grids are in an HDF5 file. how to handle flushing?
    """
    
    if flash_count_logfile is None:
        flash_count_logfile = sys.stdout
    
    # reference time is the date part of the start_time

    t_edges, duration = time_edges(start_time, end_time, frame_interval)
    t_ref, t_edges_seconds = seconds_since_start_of_day(start_time, t_edges)
    n_frames = len(t_edges)-1
    
    xedge=np.arange(x_bnd[0], x_bnd[1]+dx, dx)
    yedge=np.arange(y_bnd[0], y_bnd[1]+dy, dy)
    zedge=np.arange(z_bnd[0], z_bnd[1]+dz, dz) 
    
    x0 = xedge[0]
    y0 = yedge[0]
    z0 = zedge[0]
    
    if proj_name == 'latlong':
        dx_units = '{0:6.4f}deg'.format(dx)
        mapProj = GeographicSystem()
    else:
        dx_units = '{0:5.1f}m'.format(dx)
        mapProj = MapProjection(projection=proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat, 
                            lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum)
    geoProj = GeographicSystem()
    
    event_density_grid  = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
    init_density_grid   = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
    extent_density_grid = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='int32')
    footprint_grid      = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, n_frames), dtype='float32')

    event_density_grid_3d  = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
    init_density_grid_3d   = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
    extent_density_grid_3d = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='int32')
    footprint_grid_3d      = np.zeros((xedge.shape[0]-1, yedge.shape[0]-1, zedge.shape[0]-1, n_frames), dtype='float32')

        
    all_frames = []
    # extent_frames = []
    # init_frames = []
    # event_frames = []
    # extent_frames_3d = []
    # init_frames_3d = []
    # event_frames_3d = []
    for i in range(n_frames):
        extent_out = {'name':'extent'}
        init_out   = {'name':'init'}
        event_out  = {'name':'event'}
        
        extent_out_3d = {'name':'extent_3d'}
        init_out_3d   = {'name':'init_3d'}
        event_out_3d  = {'name':'event_3d'}
        
        accum_event_density  = density_to_files.accumulate_points_on_grid(event_density_grid[:,:,i], xedge, yedge,  out=event_out, label='event')
        accum_init_density   = density_to_files.accumulate_points_on_grid(init_density_grid[:,:,i], xedge, yedge,   out=init_out,  label='init')
        accum_extent_density = density_to_files.accumulate_points_on_grid(extent_density_grid[:,:,i], xedge, yedge, out=extent_out,label='extent')
        accum_footprint      = density_to_files.accumulate_points_on_grid(footprint_grid[:,:,i], xedge, yedge, label='footprint')

        accum_event_density_3d  = density_to_files.accumulate_points_on_grid_3d(event_density_grid_3d[:,:,:,i], xedge, yedge, zedge,  out=event_out_3d, label='event_3d')
        accum_init_density_3d   = density_to_files.accumulate_points_on_grid_3d(init_density_grid_3d[:,:,:,i], xedge, yedge, zedge,   out=init_out_3d,  label='init_3d')
        accum_extent_density_3d = density_to_files.accumulate_points_on_grid_3d(extent_density_grid_3d[:,:,:,i], xedge, yedge, zedge, out=extent_out_3d,label='extent_3d')
        accum_footprint_3d      = density_to_files.accumulate_points_on_grid_3d(footprint_grid_3d[:,:,:,i], xedge, yedge, zedge, label='footprint_3d')

        extent_out['func'] = accum_extent_density
        init_out['func'] = accum_init_density
        event_out['func'] = accum_event_density
        # extent_frames.append(extent_out)
        # init_frames.append(init_out)
        # event_frames.append(event_out)

        extent_out_3d['func'] = accum_extent_density_3d
        init_out_3d['func'] = accum_init_density_3d
        event_out_3d['func'] = accum_event_density_3d
        # extent_frames_3d.append(extent_out_3d)
        # init_frames_3d.append(init_out_3d)
        # event_frames_3d.append(event_out_3d)
        
        event_density_target  = density_to_files.point_density(accum_event_density)
        init_density_target   = density_to_files.point_density(accum_init_density)
        extent_density_target = density_to_files.extent_density(x0, y0, dx, dy, accum_extent_density)
        mean_footprint_target = density_to_files.extent_density(x0, y0, dx, dy, accum_footprint, weight_key='area')

        event_density_target_3d  = density_to_files.point_density_3d(accum_event_density_3d)
        init_density_target_3d   = density_to_files.point_density_3d(accum_init_density_3d)
        extent_density_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_extent_density_3d)
        mean_footprint_target_3d = density_to_files.extent_density_3d(x0, y0, z0, dx, dy, dz, accum_footprint_3d, weight_key='area')


        spew_to_density_types = broadcast( ( 
                    density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target, use_flashes=False),
                    density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target, use_flashes=True),
                    density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target, use_flashes=False),
                    density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target, use_flashes=False),
                    
                    density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, event_density_target_3d, use_flashes=False),
                    density_to_files.project('init_lon', 'init_lat', 'init_alt', mapProj, geoProj, init_density_target_3d, use_flashes=True),
                    density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, extent_density_target_3d, use_flashes=False),
                    density_to_files.project('lon', 'lat', 'alt', mapProj, geoProj, mean_footprint_target_3d, use_flashes=False),
                    ) )

        all_frames.append( density_to_files.extract_events_for_flashes( spew_to_density_types ) )

    frame_count_log = density_to_files.flash_count_log(flash_count_logfile)
        
    framer = density_to_files.flashes_to_frames(t_edges_seconds, all_frames, time_key='start', time_edges_datetime=t_edges, flash_counter=frame_count_log)
    
    read_flashes( h5_filenames, framer, base_date=t_ref, min_points=min_points_per_flash)
    
    # print 'event_density_grid ', id(event_density_grid[:,:,-1])
    # print 'extent_density_grid', id(extent_density_grid[:,:,-1])
    # print 'init_density_grid  ', id(init_density_grid[:,:,-1])
    
    
    x_coord = (xedge[:-1] + xedge[1:])/2.0
    y_coord = (yedge[:-1] + yedge[1:])/2.0
    z_coord = (zedge[:-1] + zedge[1:])/2.0
    nx = x_coord.shape[0]
    ny = y_coord.shape[0]
    nz = z_coord.shape[0]
    
    x_all, y_all = (a.T for a in np.meshgrid(x_coord, y_coord))
    assert x_all.shape == y_all.shape
    assert x_all.shape[0] == nx
    assert x_all.shape[1] == ny
    z_all = np.zeros_like(x_all)
    
    
    grid_shape_3d = (nx,ny,nz) 
    x_ones_3d = np.ones(grid_shape_3d, dtype='f4')
    y_ones_3d = np.ones(grid_shape_3d, dtype='f4')
    z_ones_3d = np.ones(grid_shape_3d, dtype='f4')
    
    x_all_3d = x_coord[:, None, None]*x_ones_3d
    y_all_3d = y_coord[None,:,None]*y_ones_3d
    z_all_3d = z_coord[None, None, :]*z_ones_3d
    
            
    lons, lats, alts = x,y,z = geoProj.fromECEF( *mapProj.toECEF(x_all, y_all, z_all) )
    lons.shape=x_all.shape
    lats.shape=y_all.shape
    
    lons_3d, lats_3d, alts_3d = x_3d,y_3d,z_3d = geoProj.fromECEF( *mapProj.toECEF(x_all_3d, y_all_3d, z_all_3d) )
    lons_3d.shape=x_all_3d.shape
    lats_3d.shape=y_all_3d.shape
    alts_3d.shape=z_all_3d.shape
    
    
    outflile_basename = os.path.join(outpath,'%s_%s_%d_%dsrc_%s-dx_' % (output_filename_prefix, start_time.strftime('%Y%m%d_%H%M%S'), to_seconds(duration), min_points_per_flash, dx_units))
    
    outfiles = (outflile_basename+'flash_extent.nc',
                outflile_basename+'flash_init.nc',
                outflile_basename+'source.nc',
                outflile_basename+'footprint.nc',
                )
    outfiles_3d = (outflile_basename+'flash_extent_3d.nc',
                outflile_basename+'flash_init_3d.nc',
                outflile_basename+'source_3d.nc',
                outflile_basename+'footprint_3d.nc',
                )
                
    outgrids = (extent_density_grid, 
                init_density_grid,   
                event_density_grid,  
                footprint_grid,
                )
                
    outgrids_3d = (extent_density_grid_3d,
                init_density_grid_3d,
                event_density_grid_3d,
                footprint_grid_3d,)
                
                
    field_names = ('flash_extent', 'flash_initiation', 'lma_source', 'flash_footprint')
    
    field_descriptions = ('LMA flash extent density',
                        'LMA flash initiation density',
                        'LMA source density',
                        'LMA local mean flash area')
    
    if proj_name=='latlong':
        density_units = "grid"
        density_units_3d = "grid"
    else:
        density_units = "{0:5.1f} km^2".format(dx/1000.0 * dy/1000.0).lstrip()
        density_units_3d = "{0:5.1f} km^3".format(dx/1000.0 * dy/1000.0 * dz/1000.0).lstrip()
    time_units = "{0:5.1f} min".format(frame_interval/60.0).lstrip()
    density_label = 'Count per ' + density_units + " pixel per "+ time_units
    density_label_3d = 'Count per ' + density_units_3d + " pixel per "+ time_units
    
    field_units = ( density_label,
                    density_label,
                    density_label,
                    "km^2 per flash",
                     )
    field_units_3d = ( density_label_3d,
                    density_label_3d,
                    density_label_3d,
                    "km^2 per flash",
                     )
    
    output_writer(outfiles[0], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, 
                    lons, lats, ctr_lat, ctr_lon, 
                    outgrids[0], field_names[0], field_descriptions[0], 
                    grid_units=field_units[0],
                    **output_kwargs)
    output_writer(outfiles[1], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, 
                    lons, lats, ctr_lat, ctr_lon, 
                    outgrids[1], field_names[1], field_descriptions[1], 
                    grid_units=field_units[1],
                    **output_kwargs)
    output_writer(outfiles[2], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, 
                    lons, lats, ctr_lat, ctr_lon, 
                    outgrids[2], field_names[2], field_descriptions[2], 
                    grid_units=field_units[2],
                    **output_kwargs)
    output_writer(outfiles[3], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor, 
                    lons, lats, ctr_lat, ctr_lon, 
                    outgrids[3], field_names[3], field_descriptions[3], format='f', 
                    grid_units=field_units[3],
                    **output_kwargs)
                    
    output_writer_3d(outfiles_3d[0], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
                    z_coord*spatial_scale_factor, 
                    lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
                    outgrids_3d[0], field_names[0], field_descriptions[0], 
                    grid_units=field_units_3d[0],
                    **output_kwargs)
    output_writer_3d(outfiles_3d[1], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
                    z_coord*spatial_scale_factor, 
                    lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
                    outgrids_3d[1], field_names[1], field_descriptions[1], 
                    grid_units=field_units_3d[1],
                    **output_kwargs)
    output_writer_3d(outfiles_3d[2], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
                    z_coord*spatial_scale_factor, 
                    lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
                    outgrids_3d[2], field_names[2], field_descriptions[2], 
                    grid_units=field_units_3d[2],
                    **output_kwargs)
    output_writer_3d(outfiles_3d[3], t_ref, np.asarray(t_edges_seconds[:-1]),
                    x_coord*spatial_scale_factor, y_coord*spatial_scale_factor,
                    z_coord*spatial_scale_factor, 
                    lons_3d, lats_3d, alts_3d, ctr_lat, ctr_lon, ctr_alt,
                    outgrids_3d[3], field_names[3], field_descriptions[3], format='f', 
                    grid_units=field_units_3d[3],
                    **output_kwargs)
                        
                    
    print('max extent is', extent_density_grid.max())

    return x_coord, y_coord, z_coord, lons, lats, alts, extent_density_grid, outfiles, field_names