def grid_and_plot(h5_filenames,
base_sort_dir,
dx=1.0e3,
dy=1.0e3,
dz=1.0e3,
frame_interval=60.0,
x_bnd=(-200.0e3, 200.0e3),
y_bnd=(-200.0e3, 200.0e3),
z_bnd=(0.0e3, 20.0e3),
ctr_lat=33.5,
ctr_lon=-101.5,
center_ID='WTLMA',
n_cols=2,
base_date=None):
""" Given a list of HDF5 filenames (sorted by time order) in h5_filenames,
create 2D and 3D NetCDF grids with spacing dx, dy, dz in meters,
frame_interval in seconds, and tuples of grid edges
x_bnd, y_bnd, and z_bnd in meters
The actual grids are in regular lat,lon coordinates, with spacing at the
grid center matched to the dx, dy values given.
n_cols controls how many columns are plotted on each page.
Grids and plots are written to base_sort_dir/grid_files/ and base_sort_dir/plots/
base_date is used to optionally set a common reference time for each of the NetCDF grids.
"""
# not really in km, just a different name to distinguish from similar variables below.
dx_km = dx
dy_km = dy
x_bnd_km = x_bnd
y_bnd_km = y_bnd
z_bnd_km = z_bnd
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y, m, d, H, M, S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y, m, d, H, M, S)
end_time = start_time + timedelta(0, 600)
date = start_time
# print start_time, end_time
outpath = grid_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
grid_dir + '/20%s' % (date.strftime('%y/%b')),
grid_dir + '/20%s' % (date.strftime('%y'))
])
if True:
grid_h5flashfiles(h5_filenames,
start_time,
end_time,
frame_interval=frame_interval,
proj_name='latlong',
base_date=base_date,
energy_grids=True,
dx=dx,
dy=dy,
x_bnd=x_bnd,
y_bnd=y_bnd,
z_bnd=z_bnd_km,
ctr_lon=ctr_lon,
ctr_lat=ctr_lat,
outpath=outpath,
output_writer=write_cf_netcdf_latlon,
output_writer_3d=write_cf_netcdf_3d_latlon,
output_filename_prefix=center_ID,
spatial_scale_factor=1.0)
# Create plots
mapping = {
'source': 'lma_source',
'flash_extent': 'flash_extent',
'flash_init': 'flash_initiation',
'footprint': 'flash_footprint',
'specific_energy': 'specific_energy',
'flashsize_std': 'flashsize_std',
'total_energy': 'total_energy'
}
nc_names = glob.glob(grid_dir + '/20%s/*.nc' % (date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir + '/20%s/*_3d.nc' %
(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
plot_dir + '/20%s' % (date.strftime('%y/%b')),
plot_dir + '/20%s' % (date.strftime('%y'))
])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
make_plot(f,
var,
n_cols=n_cols,
x_name='longitude',
y_name='latitude',
outpath=outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(
f, var, x_name='longitude', y_name='latitude', z_name='altitude')
make_plot_3d(grid,
grid_name,
x,
y,
z,
t,
grid_t_idx,
grid_x_idx,
grid_z_idx,
n_cols=n_cols,
outpath=outpath)
#, grid_range=grid_range)
return nc_names_2d, nc_names_3d
def test_sort_and_grid_and_plot(outpath):
""" Given an output path, run sample data included in lmatools through flash sorting and gridding"""
base_sort_dir = outpath
logger_setup(outpath)
files = get_sample_data_list()
center_ID = 'WTLMA'
ctr_lat, ctr_lon = 33.5, -101.5
params = {'stations':(6,13),
'chi2':(0,1.0),
'ctr_lat':ctr_lat, 'ctr_lon':ctr_lon,
'distance':3000.0, 'thresh_critical_time':0.15,
'thresh_duration':3.0,
'mask_length':6,
}
h5_dir = os.path.join(base_sort_dir, 'h5_files')
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
y,m,d,H,M,S = tfromfile(files[0])
date = datetime(y,m,d, H,M,S)
# Create HDF5 flash files
base_out_dir = (h5_dir+"/20%s" %(date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call(['chmod', 'a+w', base_out_dir, h5_dir+'/20%s' %(date.strftime('%y/%b')), h5_dir+'/20%s' %(date.strftime('%y'))])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
if True:
cluster = DBSCANFlashSorter(params).cluster
sort_files(files, outdir, cluster)
# Figure out which HDF5 files were created
h5_filenames = glob.glob(h5_dir+'/20%s/LYLOUT*.dat.flash.h5' %(date.strftime('%y/%b/%d')))
h5_filenames.sort()
# Create NetCDF gridded data
frame_interval=60.0*2 # seconds
dx_km=3.0e3 # meters
dy_km=3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
z_bnd_km = (0.0e3, 15.0e3)
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
dx=dx_km, dy=dy_km,
x_bnd = x_bnd_km, y_bnd = y_bnd_km )
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
# print start_time, end_time
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
if True:
grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=frame_interval, proj_name='latlong',
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, z_bnd=z_bnd_km,
ctr_lon=ctr_lon, ctr_lat=ctr_lat, outpath = outpath,
output_writer = write_cf_netcdf_latlon, output_writer_3d = write_cf_netcdf_3d_latlon,
output_filename_prefix=center_ID, spatial_scale_factor=1.0
)
# Create plots
n_cols=2
mapping = { 'source':'lma_source',
'flash_extent':'flash_extent',
'flash_init':'flash_initiation',
'footprint':'flash_footprint'}
nc_names = glob.glob(grid_dir+'/20%s/*.nc' %(date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir+'/20%s/*_3d.nc' %(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, plot_dir+'/20%s' %(date.strftime('%y/%b')), plot_dir+'/20%s' %(date.strftime('%y'))])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
make_plot(f, var, n_cols=n_cols, x_name='longitude', y_name='latitude', outpath = outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(f, var, x_name='longitude', y_name='latitude', z_name='altitude')
make_plot_3d(grid, grid_name, x, y, z, t,
grid_t_idx, grid_x_idx, grid_z_idx,
n_cols = n_cols, outpath = outpath)
def test_sort_and_grid_and_plot(outpath):
""" Given an output path, run sample data included in lmatools through flash sorting and gridding"""
base_sort_dir = outpath
logger_setup(outpath)
files = get_sample_data_list()
center_ID = 'WTLMA'
ctr_lat, ctr_lon = 33.5, -101.5
params = {'stations':(6,13),
'chi2':(0,1.0),
'ctr_lat':ctr_lat, 'ctr_lon':ctr_lon,
'distance':3000.0, 'thresh_critical_time':0.15,
'thresh_duration':3.0,
'mask_length':6,
}
h5_dir = os.path.join(base_sort_dir, 'h5_files')
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
y,m,d,H,M,S = tfromfile(files[0])
date = datetime(y,m,d, H,M,S)
# Create HDF5 flash files
base_out_dir = (h5_dir+"/20%s" %(date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call(['chmod', 'a+w', base_out_dir, h5_dir+'/20%s' %(date.strftime('%y/%b')), h5_dir+'/20%s' %(date.strftime('%y'))])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
if True:
cluster = DBSCANFlashSorter(params).cluster
sort_files(files, outdir, cluster)
# Figure out which HDF5 files were created
h5_filenames = glob.glob(h5_dir+'/20%s/LYLOUT*.dat.flash.h5' %(date.strftime('%y/%b/%d')))
h5_filenames.sort()
# Create NetCDF gridded data
frame_interval=60.0*2 # seconds
dx_km=3.0e3 # meters
dy_km=3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
z_bnd_km = (0.0e3, 15.0e3)
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
# dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
# dx=dx_km, dy=dy_km,
# x_bnd = x_bnd_km, y_bnd = y_bnd_km )
dx, dy, x_bnd, y_bnd = dx_km, dy_km, x_bnd_km, y_bnd_km
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
# print start_time, end_time
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
if True:
grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=frame_interval,
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, z_bnd=z_bnd_km,
ctr_lon=ctr_lon, ctr_lat=ctr_lat, outpath = outpath,
proj_name='aeqd',
output_writer = write_cf_netcdf,
output_writer_3d = write_cf_netcdf_3d,
output_filename_prefix=center_ID, spatial_scale_factor=1.0e-3,
# proj_name='latlong',
# output_writer = write_cf_netcdf_latlon,
# output_writer_3d = write_cf_netcdf_3d_latlon,
# output_filename_prefix=center_ID, spatial_scale_factor=1.0,
energy_grids = True
)
# Create plots
n_cols=2
mapping = { 'source':'lma_source',
'flash_extent':'flash_extent',
'flash_init':'flash_initiation',
'footprint':'flash_footprint',
'specific_energy':'specific_energy',
'flashsize_std':'flashsize_std',
'total_energy': 'total_energy'
}
nc_names = glob.glob(grid_dir+'/20%s/*.nc' %(date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir+'/20%s/*_3d.nc' %(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, plot_dir+'/20%s' %(date.strftime('%y/%b')), plot_dir+'/20%s' %(date.strftime('%y'))])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
# make_plot(f, var, n_cols=n_cols, x_name='longitude', y_name='latitude', outpath = outpath)
make_plot(f, var, n_cols=n_cols, x_name='x', y_name='y', outpath = outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
# grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(f, var, x_name='longitude', y_name='latitude', z_name='altitude')
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(f, var, x_name='x', y_name='y', z_name='z')
make_plot_3d(grid, grid_name, x, y, z, t,
grid_t_idx, grid_x_idx, grid_z_idx,
n_cols = n_cols, outpath = outpath)
def grid_and_plot(h5_filenames, base_sort_dir, dx=1.0e3, dy=1.0e3, dz=1.0e3, frame_interval=60.0,
x_bnd=(-200.0e3, 200.0e3), y_bnd=(-200.0e3, 200.0e3), z_bnd=(0.0e3, 20.0e3),
ctr_lat=33.5, ctr_lon=-101.5, center_ID='WTLMA',
n_cols=2, base_date=None
):
""" Given a list of HDF5 filenames (sorted by time order) in h5_filenames,
create 2D and 3D NetCDF grids with spacing dx, dy, dz in meters,
frame_interval in seconds, and tuples of grid edges
x_bnd, y_bnd, and z_bnd in meters
The actual grids are in regular lat,lon coordinates, with spacing at the
grid center matched to the dx, dy values given.
n_cols controls how many columns are plotted on each page.
Grids and plots are written to base_sort_dir/grid_files/ and base_sort_dir/plots/
base_date is used to optionally set a common reference time for each of the NetCDF grids.
"""
# not really in km, just a different name to distinguish from similar variables below.
dx_km=dx
dy_km=dy
x_bnd_km = x_bnd
y_bnd_km = y_bnd
z_bnd_km = z_bnd
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
dx=dx_km, dy=dy_km,
x_bnd = x_bnd_km, y_bnd = y_bnd_km )
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
date = start_time
# print start_time, end_time
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
if True:
grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=frame_interval, proj_name='latlong',
base_date = base_date, energy_grids=True,
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, z_bnd=z_bnd_km,
ctr_lon=ctr_lon, ctr_lat=ctr_lat, outpath = outpath,
output_writer = write_cf_netcdf_latlon, output_writer_3d = write_cf_netcdf_3d_latlon,
output_filename_prefix=center_ID, spatial_scale_factor=1.0
)
# Create plots
mapping = { 'source':'lma_source',
'flash_extent':'flash_extent',
'flash_init':'flash_initiation',
'footprint':'flash_footprint',
'specific_energy':'specific_energy',
'flashsize_std':'flashsize_std',
'total_energy': 'total_energy'
}
nc_names = glob.glob(grid_dir+'/20%s/*.nc' %(date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir+'/20%s/*_3d.nc' %(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, plot_dir+'/20%s' %(date.strftime('%y/%b')), plot_dir+'/20%s' %(date.strftime('%y'))])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
make_plot(f, var, n_cols=n_cols, x_name='longitude', y_name='latitude', outpath = outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(f, var, x_name='longitude', y_name='latitude', z_name='altitude')
make_plot_3d(grid, grid_name, x, y, z, t,
grid_t_idx, grid_x_idx, grid_z_idx,
n_cols = n_cols, outpath = outpath)
#, grid_range=grid_range)
return nc_names_2d, nc_names_3d