def grid_and_plot(h5_filenames,
base_sort_dir,
dx=1.0e3,
dy=1.0e3,
dz=1.0e3,
frame_interval=60.0,
x_bnd=(-200.0e3, 200.0e3),
y_bnd=(-200.0e3, 200.0e3),
z_bnd=(0.0e3, 20.0e3),
ctr_lat=33.5,
ctr_lon=-101.5,
center_ID='WTLMA',
n_cols=2,
base_date=None):
""" Given a list of HDF5 filenames (sorted by time order) in h5_filenames,
create 2D and 3D NetCDF grids with spacing dx, dy, dz in meters,
frame_interval in seconds, and tuples of grid edges
x_bnd, y_bnd, and z_bnd in meters
The actual grids are in regular lat,lon coordinates, with spacing at the
grid center matched to the dx, dy values given.
n_cols controls how many columns are plotted on each page.
Grids and plots are written to base_sort_dir/grid_files/ and base_sort_dir/plots/
base_date is used to optionally set a common reference time for each of the NetCDF grids.
"""
# not really in km, just a different name to distinguish from similar variables below.
dx_km = dx
dy_km = dy
x_bnd_km = x_bnd
y_bnd_km = y_bnd
z_bnd_km = z_bnd
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y, m, d, H, M, S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y, m, d, H, M, S)
end_time = start_time + timedelta(0, 600)
date = start_time
# print start_time, end_time
outpath = grid_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
grid_dir + '/20%s' % (date.strftime('%y/%b')),
grid_dir + '/20%s' % (date.strftime('%y'))
])
if True:
grid_h5flashfiles(h5_filenames,
start_time,
end_time,
frame_interval=frame_interval,
proj_name='latlong',
base_date=base_date,
energy_grids=True,
dx=dx,
dy=dy,
x_bnd=x_bnd,
y_bnd=y_bnd,
z_bnd=z_bnd_km,
ctr_lon=ctr_lon,
ctr_lat=ctr_lat,
outpath=outpath,
output_writer=write_cf_netcdf_latlon,
output_writer_3d=write_cf_netcdf_3d_latlon,
output_filename_prefix=center_ID,
spatial_scale_factor=1.0)
# Create plots
mapping = {
'source': 'lma_source',
'flash_extent': 'flash_extent',
'flash_init': 'flash_initiation',
'footprint': 'flash_footprint',
'specific_energy': 'specific_energy',
'flashsize_std': 'flashsize_std',
'total_energy': 'total_energy'
}
nc_names = glob.glob(grid_dir + '/20%s/*.nc' % (date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir + '/20%s/*_3d.nc' %
(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
plot_dir + '/20%s' % (date.strftime('%y/%b')),
plot_dir + '/20%s' % (date.strftime('%y'))
])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
make_plot(f,
var,
n_cols=n_cols,
x_name='longitude',
y_name='latitude',
outpath=outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(
f, var, x_name='longitude', y_name='latitude', z_name='altitude')
make_plot_3d(grid,
grid_name,
x,
y,
z,
t,
grid_t_idx,
grid_x_idx,
grid_z_idx,
n_cols=n_cols,
outpath=outpath)
#, grid_range=grid_range)
return nc_names_2d, nc_names_3d
os.makedirs(outpath)
# subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
# center_ID='WTLMA'
ctr_lat = float(args.ctr_lat)
ctr_lon = float(args.ctr_lon)
dx_km = float(args.dx) * 1.0e3
dy_km = float(args.dy) * 1.0e3
width, height = 1000.0 * float(args.width), 1000.0 * float(args.height)
x_bnd_km = (-width / 2.0, width / 2.0)
y_bnd_km = (-height / 2.0, height / 2.0)
frame_interval = float(args.dt)
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# lon lat tuples of the corners
corners = np.vstack([(x_bnd[0], y_bnd[0]), (x_bnd[0], y_bnd[1]),
(x_bnd[1], y_bnd[1]), (x_bnd[1], y_bnd[0])])
# print(x_bnd, y_bnd)
# Default
proj_name = 'latlong'
output_writer = write_cf_netcdf_latlon
# Old support for a direct lon/lat to fixed grid lookup table. Unused
# at the moment.
pixel_grid = None
if pixel_grid is not None:
def test_sort_and_grid_and_plot(outpath):
""" Given an output path, run sample data included in lmatools through flash sorting and gridding"""
base_sort_dir = outpath
logger_setup(outpath)
files = get_sample_data_list()
center_ID = 'WTLMA'
ctr_lat, ctr_lon = 33.5, -101.5
params = {
'stations': (6, 13),
'chi2': (0, 1.0),
'ctr_lat': ctr_lat,
'ctr_lon': ctr_lon,
'distance': 3000.0,
'thresh_critical_time': 0.15,
'thresh_duration': 3.0,
'mask_length': 6,
}
h5_dir = os.path.join(base_sort_dir, 'h5_files')
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
y, m, d, H, M, S = tfromfile(files[0])
date = datetime(y, m, d, H, M, S)
# Create HDF5 flash files
base_out_dir = (h5_dir + "/20%s" % (date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call([
'chmod', 'a+w', base_out_dir,
h5_dir + '/20%s' % (date.strftime('%y/%b')),
h5_dir + '/20%s' % (date.strftime('%y'))
])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
run_files_with_params(files,
outdir,
params,
cluster,
retain_ascii_output=False,
cleanup_tmp=True)
# Figure out which HDF5 files were created
h5_filenames = glob.glob(h5_dir + '/20%s/LYLOUT*.dat.flash.h5' %
(date.strftime('%y/%b/%d')))
h5_filenames.sort()
# Create NetCDF gridded data
frame_interval = 60.0 * 2 # seconds
dx_km = 3.0e3 # meters
dy_km = 3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y, m, d, H, M, S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y, m, d, H, M, S)
end_time = start_time + timedelta(0, 600)
# print start_time, end_time
outpath = grid_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
grid_dir + '/20%s' % (date.strftime('%y/%b')),
grid_dir + '/20%s' % (date.strftime('%y'))
])
grid_h5flashfiles(h5_filenames,
start_time,
end_time,
frame_interval=frame_interval,
proj_name='latlong',
dx=dx,
dy=dy,
x_bnd=x_bnd,
y_bnd=y_bnd,
ctr_lon=ctr_lon,
ctr_lat=ctr_lat,
outpath=outpath,
output_writer=write_cf_netcdf_latlon,
output_filename_prefix=center_ID,
spatial_scale_factor=1.0,
energy_grids=True)
# Create plots
n_cols = 2
mapping = {
'source': 'lma_source',
'flash_extent': 'flash_extent',
'flash_init': 'flash_initiation',
'footprint': 'flash_footprint'
}
#nc_names = glob.glob('/home/ebruning/Mar18-19/grids/*.nc')
nc_names = glob.glob(grid_dir + '/20%s/*.nc' % (date.strftime('%y/%b/%d')))
nc_names.sort()
outpath = plot_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
plot_dir + '/20%s' % (date.strftime('%y/%b')),
plot_dir + '/20%s' % (date.strftime('%y'))
])
for f in nc_names:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
# print f
make_plot(f,
var,
n_cols=n_cols,
x_name='longitude',
y_name='latitude',
outpath=outpath)
def test_sort_and_grid_and_plot(outpath):
""" Given an output path, run sample data included in lmatools through flash sorting and gridding"""
base_sort_dir = outpath
logger_setup(outpath)
files = get_sample_data_list()
center_ID = 'WTLMA'
ctr_lat, ctr_lon = 33.5, -101.5
params = {'stations':(6,13),
'chi2':(0,1.0),
'ctr_lat':ctr_lat, 'ctr_lon':ctr_lon,
'distance':3000.0, 'thresh_critical_time':0.15,
'thresh_duration':3.0,
'mask_length':6,
}
h5_dir = os.path.join(base_sort_dir, 'h5_files')
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
y,m,d,H,M,S = tfromfile(files[0])
date = datetime(y,m,d, H,M,S)
# Create HDF5 flash files
base_out_dir = (h5_dir+"/20%s" %(date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call(['chmod', 'a+w', base_out_dir, h5_dir+'/20%s' %(date.strftime('%y/%b')), h5_dir+'/20%s' %(date.strftime('%y'))])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
if True:
cluster = DBSCANFlashSorter(params).cluster
sort_files(files, outdir, cluster)
# Figure out which HDF5 files were created
h5_filenames = glob.glob(h5_dir+'/20%s/LYLOUT*.dat.flash.h5' %(date.strftime('%y/%b/%d')))
h5_filenames.sort()
# Create NetCDF gridded data
frame_interval=60.0*2 # seconds
dx_km=3.0e3 # meters
dy_km=3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
z_bnd_km = (0.0e3, 15.0e3)
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
dx=dx_km, dy=dy_km,
x_bnd = x_bnd_km, y_bnd = y_bnd_km )
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
# print start_time, end_time
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
if True:
grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=frame_interval, proj_name='latlong',
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, z_bnd=z_bnd_km,
ctr_lon=ctr_lon, ctr_lat=ctr_lat, outpath = outpath,
output_writer = write_cf_netcdf_latlon, output_writer_3d = write_cf_netcdf_3d_latlon,
output_filename_prefix=center_ID, spatial_scale_factor=1.0
)
# Create plots
n_cols=2
mapping = { 'source':'lma_source',
'flash_extent':'flash_extent',
'flash_init':'flash_initiation',
'footprint':'flash_footprint'}
nc_names = glob.glob(grid_dir+'/20%s/*.nc' %(date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir+'/20%s/*_3d.nc' %(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, plot_dir+'/20%s' %(date.strftime('%y/%b')), plot_dir+'/20%s' %(date.strftime('%y'))])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
make_plot(f, var, n_cols=n_cols, x_name='longitude', y_name='latitude', outpath = outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(f, var, x_name='longitude', y_name='latitude', z_name='altitude')
make_plot_3d(grid, grid_name, x, y, z, t,
grid_t_idx, grid_x_idx, grid_z_idx,
n_cols = n_cols, outpath = outpath)
def grid_and_plot(h5_filenames, base_sort_dir, dx=1.0e3, dy=1.0e3, dz=1.0e3, frame_interval=60.0,
x_bnd=(-200.0e3, 200.0e3), y_bnd=(-200.0e3, 200.0e3), z_bnd=(0.0e3, 20.0e3),
ctr_lat=33.5, ctr_lon=-101.5, center_ID='WTLMA',
n_cols=2, base_date=None
):
""" Given a list of HDF5 filenames (sorted by time order) in h5_filenames,
create 2D and 3D NetCDF grids with spacing dx, dy, dz in meters,
frame_interval in seconds, and tuples of grid edges
x_bnd, y_bnd, and z_bnd in meters
The actual grids are in regular lat,lon coordinates, with spacing at the
grid center matched to the dx, dy values given.
n_cols controls how many columns are plotted on each page.
Grids and plots are written to base_sort_dir/grid_files/ and base_sort_dir/plots/
base_date is used to optionally set a common reference time for each of the NetCDF grids.
"""
# not really in km, just a different name to distinguish from similar variables below.
dx_km=dx
dy_km=dy
x_bnd_km = x_bnd
y_bnd_km = y_bnd
z_bnd_km = z_bnd
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
dx=dx_km, dy=dy_km,
x_bnd = x_bnd_km, y_bnd = y_bnd_km )
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
date = start_time
# print start_time, end_time
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
if True:
grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=frame_interval, proj_name='latlong',
base_date = base_date, energy_grids=True,
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, z_bnd=z_bnd_km,
ctr_lon=ctr_lon, ctr_lat=ctr_lat, outpath = outpath,
output_writer = write_cf_netcdf_latlon, output_writer_3d = write_cf_netcdf_3d_latlon,
output_filename_prefix=center_ID, spatial_scale_factor=1.0
)
# Create plots
mapping = { 'source':'lma_source',
'flash_extent':'flash_extent',
'flash_init':'flash_initiation',
'footprint':'flash_footprint',
'specific_energy':'specific_energy',
'flashsize_std':'flashsize_std',
'total_energy': 'total_energy'
}
nc_names = glob.glob(grid_dir+'/20%s/*.nc' %(date.strftime('%y/%b/%d')))
nc_names_3d = glob.glob(grid_dir+'/20%s/*_3d.nc' %(date.strftime('%y/%b/%d')))
nc_names_2d = list(set(nc_names) - set(nc_names_3d))
nc_names_2d.sort()
nc_names_3d.sort()
outpath = plot_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, plot_dir+'/20%s' %(date.strftime('%y/%b')), plot_dir+'/20%s' %(date.strftime('%y'))])
for f in nc_names_2d:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
make_plot(f, var, n_cols=n_cols, x_name='longitude', y_name='latitude', outpath = outpath)
for f in nc_names_3d:
gridtype = f.split('dx_')[-1].replace('.nc', '').replace('_3d', '')
var = mapping[gridtype]
# grid_range = range_mapping[gridtype]
###Read grid files, then plot in either 2d or 3d space###
grid, grid_name, x, y, z, t, grid_t_idx, grid_x_idx, grid_z_idx = read_file_3d(f, var, x_name='longitude', y_name='latitude', z_name='altitude')
make_plot_3d(grid, grid_name, x, y, z, t,
grid_t_idx, grid_x_idx, grid_z_idx,
n_cols = n_cols, outpath = outpath)
#, grid_range=grid_range)
return nc_names_2d, nc_names_3d
def grid_setup(args):
from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid
from lmatools.grid.make_grids import dlonlat_at_grid_center, grid_h5flashfiles
from glmtools.grid.make_grids import grid_GLM_flashes
from glmtools.io.glm import parse_glm_filename
from lmatools.io.LMA_h5_file import parse_lma_h5_filename
from lmatools.grid.fixed import get_GOESR_grid, get_GOESR_coordsys
# When passed None for the minimum event or group counts, the gridder will skip
# the check, saving a bit of time.
min_events = int(args.min_events)
if min_events <= 1:
min_events = None
min_groups = int(args.min_groups)
if min_groups <= 1:
min_groups = None
if args.is_lma:
filename_parser = parse_lma_h5_filename
start_idx = 0
end_idx = 1
else:
filename_parser = parse_glm_filename
start_idx = 3
end_idx = 4
glm_filenames = args.filenames
base_filenames = [os.path.basename(p) for p in glm_filenames]
try:
filename_infos = [filename_parser(f) for f in base_filenames]
# opsenv, algorithm, platform, start, end, created = parse_glm_filename(f)
filename_starts = [info[start_idx] for info in filename_infos]
filename_ends = [info[end_idx] for info in filename_infos]
except ValueError:
log.error("One or more GLM files has a non-standard filename.")
log.error("Assuming that --start and --end have been passed directly.")
from glmtools.io.glm import parse_glm_filename
if args.start is not None:
start_time = datetime.strptime(args.start[:19], '%Y-%m-%dT%H:%M:%S')
else:
start_time = min(filename_starts)
if args.end is not None:
end_time = datetime.strptime(args.end[:19], '%Y-%m-%dT%H:%M:%S')
else:
end_time = max(filename_ends)
date = datetime(start_time.year, start_time.month, start_time.day)
# grid_dir = os.path.join('/data/LCFA-production/', 'grid_test')
# outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
outpath = os.path.join(args.outdir, '20%s' % (date.strftime('%y/%b/%d')))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
# subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
if args.fixed_grid:
proj_name = 'geos'
if (args.goes_position != 'none') & (args.goes_sector != 'none'):
resln = nearest_resolution(args)
view = get_GOESR_grid(position=args.goes_position,
view=args.goes_sector,
resolution=resln)
nadir_lon = view['nadir_lon']
dx = dy = view['resolution']
nx, ny = view['pixelsEW'], view['pixelsNS']
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
if 'centerEW' in view:
x_ctr, y_ctr = view['centerEW'], view['centerNS']
elif args.goes_sector == 'meso':
# use ctr_lon, ctr_lat to get the center of the mesoscale FOV
x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
*grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
elif (args.goes_position != 'none') & (args.goes_sector == 'none'):
# Requires goes_position, a center, and a width. Fully flexible
# in resolution, i.e., doesn't slave it to one of the GOES-R specs
view = get_GOESR_grid(position=args.goes_position,
view='full',
resolution='1.0km')
nadir_lon = view['nadir_lon']
dx1km = dy1km = view['resolution']
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
*grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
# Convert the specified resolution in km given by args.dx to
# a delta in fixed grid coordinates using the 1 km delta from the
# GOES-R PUG.
dx, dy = args.dx * dx1km, args.dy * dy1km
nx, ny = int(args.width / args.dx), int(args.height / args.dy)
else:
raise ValueError(
"Gridding on the fixed grid requires "
"goes_position and dx. For goes_sector='meso', also specify "
"ctr_lon and ctr_lat. Without goes_sector, also include width "
"and height.")
x_bnd = (np.arange(nx, dtype='float') -
nx / 2.0) * dx + x_ctr + 0.5 * dx
y_bnd = (np.arange(ny, dtype='float') -
ny / 2.0) * dy + y_ctr + 0.5 * dy
x_bnd = np.asarray([x_bnd.min(), x_bnd.max()])
y_bnd = np.asarray([y_bnd.min(), y_bnd.max()])
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
ctr_lon, ctr_lat, ctr_alt = grs80lla.fromECEF(
*geofixcs.toECEF(x_ctr, y_ctr, 0.0))
fixed_grid = geofixcs
log.debug((x_bnd, y_bnd, dx, dy, nx, ny))
output_writer = partial(write_cf_netcdf_fixedgrid, nadir_lon=nadir_lon)
else:
# Default
proj_name = 'latlong'
output_writer = write_cf_netcdf_latlon
ctr_lat = float(args.ctr_lat)
ctr_lon = float(args.ctr_lon)
dx_km = float(args.dx) * 1.0e3
dy_km = float(args.dy) * 1.0e3
width, height = 1000.0 * float(args.width), 1000.0 * float(args.height)
x_bnd_km = (-width / 2.0, width / 2.0)
y_bnd_km = (-height / 2.0, height / 2.0)
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# tuples of the corners
corners = np.vstack([(x_bnd[0], y_bnd[0]), (x_bnd[0], y_bnd[1]),
(x_bnd[1], y_bnd[1]), (x_bnd[1], y_bnd[0])])
# print(x_bnd, y_bnd)
if args.is_lma:
gridder = grid_h5flashfiles
output_filename_prefix = 'LMA'
else:
gridder = grid_GLM_flashes
output_filename_prefix = 'GLM'
grid_kwargs = dict(proj_name=proj_name,
base_date=date,
do_3d=False,
dx=dx,
dy=dy,
frame_interval=float(args.dt),
x_bnd=x_bnd,
y_bnd=y_bnd,
ctr_lat=ctr_lat,
ctr_lon=ctr_lon,
outpath=outpath,
min_points_per_flash=min_events,
output_writer=output_writer,
subdivide=args.subdivide_grid,
output_filename_prefix=output_filename_prefix,
spatial_scale_factor=1.0)
if args.fixed_grid:
grid_kwargs['fixed_grid'] = True
grid_kwargs['nadir_lon'] = nadir_lon
if args.split_events:
grid_kwargs['clip_events'] = True
if min_groups is not None:
grid_kwargs['min_groups_per_flash'] = min_groups
if args.is_lma:
grid_kwargs['energy_grids'] = True
else:
grid_kwargs['energy_grids'] = ('total_energy', )
if (proj_name == 'pixel_grid') or (proj_name == 'geos'):
grid_kwargs['pixel_coords'] = fixed_grid
grid_kwargs['ellipse_rev'] = args.ellipse_rev
# if args.corner_points:
# grid_kwargs['corner_pickle'] = args.corner_points
return gridder, glm_filenames, start_time, end_time, grid_kwargs
def grid_setup(args):
from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid
from lmatools.grid.make_grids import dlonlat_at_grid_center, grid_h5flashfiles
from glmtools.grid.make_grids import grid_GLM_flashes
from glmtools.io.glm import parse_glm_filename
from lmatools.io.LMA_h5_file import parse_lma_h5_filename
from lmatools.grid.fixed import get_GOESR_grid, get_GOESR_coordsys
# When passed None for the minimum event or group counts, the gridder will skip
# the check, saving a bit of time.
min_events = int(args.min_events)
if min_events <= 1:
min_events = None
min_groups = int(args.min_groups)
if min_groups <= 1:
min_groups = None
if args.is_lma:
filename_parser = parse_lma_h5_filename
start_idx = 0
end_idx = 1
else:
filename_parser = parse_glm_filename
start_idx = 3
end_idx = 4
glm_filenames = args.filenames
base_filenames = [os.path.basename(p) for p in glm_filenames]
try:
filename_infos = [filename_parser(f) for f in base_filenames]
# opsenv, algorithm, platform, start, end, created = parse_glm_filename(f)
filename_starts = [info[start_idx] for info in filename_infos]
filename_ends = [info[end_idx] for info in filename_infos]
except ValueError:
log.error("One or more GLM files has a non-standard filename.")
log.error("Assuming that --start and --end have been passed directly.")
from glmtools.io.glm import parse_glm_filename
if args.start is not None:
start_time = datetime.strptime(args.start[:19], '%Y-%m-%dT%H:%M:%S')
else:
start_time = min(filename_starts)
if args.end is not None:
end_time = datetime.strptime(args.end[:19], '%Y-%m-%dT%H:%M:%S')
else:
# Used to use max(filename_ends), but on 27 Oct 2020, the filename
# ends started to report the time of the last event in the file,
# causing a slight leakage (usually less than a second) into the
# next minute. This caused two minutes of grids to be produced for every
# three twenty second files passed to this script.
# Instead, we now assume every LCFA file is 20 s long, beginning with
# the start time. No doubt in the future we will see filenames that no
# longer start on an even minute boundary.
end_time = max(filename_starts) + timedelta(0, 20)
date = datetime(start_time.year, start_time.month, start_time.day)
outpath = args.outdir
if args.fixed_grid:
proj_name = 'geos'
if (args.goes_position != 'none') & (args.goes_sector != 'none'):
resln = nearest_resolution(args)
view = get_GOESR_grid(position=args.goes_position,
view=args.goes_sector,
resolution=resln)
nadir_lon = view['nadir_lon']
dx = dy = view['resolution']
nx, ny = view['pixelsEW'], view['pixelsNS']
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
if 'centerEW' in view:
x_ctr, y_ctr = view['centerEW'], view['centerNS']
elif args.goes_sector == 'meso':
# use ctr_lon, ctr_lat to get the center of the mesoscale FOV
x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
*grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
elif (args.goes_position != 'none') & (args.goes_sector == 'none'):
# Requires goes_position, a center, and a width. Fully flexible
# in resolution, i.e., doesn't slave it to one of the GOES-R specs
view = get_GOESR_grid(position=args.goes_position,
view='full',
resolution='1.0km')
nadir_lon = view['nadir_lon']
dx1km = dy1km = view['resolution']
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
*grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
# Convert the specified resolution in km given by args.dx to
# a delta in fixed grid coordinates using the 1 km delta from the
# GOES-R PUG.
dx, dy = args.dx * dx1km, args.dy * dy1km
nx, ny = int(args.width / args.dx), int(args.height / args.dy)
else:
raise ValueError(
"Gridding on the fixed grid requires "
"goes_position and dx. For goes_sector='meso', also specify "
"ctr_lon and ctr_lat. Without goes_sector, also include width "
"and height.")
# Need to use +1 here to convert to xedge, yedge expected by gridder
# instead of the pixel centroids that will result in the final image
nx += 1
ny += 1
x_bnd = (np.arange(nx, dtype='float') -
(nx) / 2.0) * dx + x_ctr + 0.5 * dx
y_bnd = (np.arange(ny, dtype='float') -
(ny) / 2.0) * dy + y_ctr + 0.5 * dy
log.debug(("initial x,y_ctr", x_ctr, y_ctr))
log.debug(("initial x,y_bnd", x_bnd.shape, y_bnd.shape))
x_bnd = np.asarray([x_bnd.min(), x_bnd.max()])
y_bnd = np.asarray([y_bnd.min(), y_bnd.max()])
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
ctr_lon, ctr_lat, ctr_alt = grs80lla.fromECEF(
*geofixcs.toECEF(x_ctr, y_ctr, 0.0))
fixed_grid = geofixcs
log.debug((x_bnd, y_bnd, dx, dy, nx, ny))
output_writer = partial(write_cf_netcdf_fixedgrid, nadir_lon=nadir_lon)
else:
# Default
proj_name = 'latlong'
output_writer = write_cf_netcdf_latlon
ctr_lat = float(args.ctr_lat)
ctr_lon = float(args.ctr_lon)
dx_km = float(args.dx) * 1.0e3
dy_km = float(args.dy) * 1.0e3
width, height = 1000.0 * float(args.width), 1000.0 * float(args.height)
x_bnd_km = (-width / 2.0, width / 2.0)
y_bnd_km = (-height / 2.0, height / 2.0)
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# tuples of the corners
corners = np.vstack([(x_bnd[0], y_bnd[0]), (x_bnd[0], y_bnd[1]),
(x_bnd[1], y_bnd[1]), (x_bnd[1], y_bnd[0])])
# print(x_bnd, y_bnd)
if args.is_lma:
gridder = grid_h5flashfiles
output_filename_prefix = 'LMA'
else:
gridder = grid_GLM_flashes
output_filename_prefix = 'GLM'
grid_kwargs = dict(
proj_name=proj_name,
base_date=date,
do_3d=False,
dx=dx,
dy=dy,
frame_interval=float(args.dt),
x_bnd=x_bnd,
y_bnd=y_bnd,
ctr_lat=ctr_lat,
ctr_lon=ctr_lon,
outpath=outpath,
min_points_per_flash=min_events,
output_writer=output_writer,
subdivide=args.subdivide_grid,
output_filename_prefix=output_filename_prefix,
output_kwargs={'scale_and_offset': args.output_scale_and_offset},
spatial_scale_factor=1.0)
if args.fixed_grid:
grid_kwargs['fixed_grid'] = True
grid_kwargs['nadir_lon'] = nadir_lon
if args.split_events:
grid_kwargs['clip_events'] = True
if min_groups is not None:
grid_kwargs['min_groups_per_flash'] = min_groups
if args.is_lma:
grid_kwargs['energy_grids'] = True
else:
grid_kwargs['energy_grids'] = ('total_energy', )
if (proj_name == 'pixel_grid') or (proj_name == 'geos'):
grid_kwargs['pixel_coords'] = fixed_grid
grid_kwargs['ellipse_rev'] = args.ellipse_rev
# if args.corner_points:
# grid_kwargs['corner_pickle'] = args.corner_points
return gridder, glm_filenames, start_time, end_time, grid_kwargs
h5_filenames = glob.glob(h5_dir+'/20%s/LYLOUT*.dat.flash.h5' %(date.strftime('%y/%b/%d')))
h5_filenames.sort()
frame_interval=60.0*5
ctr_lat = centers[center_ID]['ctr_lat']
ctr_lon = centers[center_ID]['ctr_lon']
dx_km=3.0e3
dy_km=3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
dx=dx_km, dy=dy_km,
x_bnd = x_bnd_km, y_bnd = y_bnd_km )
print(("dx, dy = {0}, {1} deg".format(dx,dy)))
print(("lon_range = {0} deg".format(x_bnd)))
print(("lat_range = {0} deg".format(y_bnd)))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
print(start_time, end_time)
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False: