'ascii_flashes_out':'flashes_out.dat',
'ctr_lat':ctr_lat, 'ctr_lon':ctr_lon,
}
params['thresh_critical_time'] = thresh_critical_time
params['distance'] = distance
# tag = 'thresh-%s_dist-%s' % (thresh_critical_time, distance)
# outdir = os.path.join(base_out_dir, tag)
# os.mkdir(outdir)
# info = open(os.path.join(outdir, 'input_params.py'), 'w')
# info.write(str(params))
# info.close()
h5_filenames = run_files_with_params([LMA_ASCII_outfile], LMA_DATA_DIR, params, retain_ascii_output=False)
# h5_path_searcher = os.path.join(LMA_DATA_DIR,'LYL*.flash.h5')
# h5_filenames = glob.glob(h5_path_searcher)
# ----- Create NetCDF grids -----
from lmatools.make_grids import grid_h5flashfiles
from lmatools.AWIPS_tools import write_AWIPS_netcdf_grid
x_coord, y_coord, lons, lats, extent_density_grid, AWIPS_outfiles, field_names = grid_h5flashfiles(
h5_filenames, start_time, end_time, frame_interval=frame_interval,
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, ctr_lon=ctr_lon, ctr_lat=ctr_lat,
proj_name = target_proj, proj_datum = target_datum, proj_ellipse = target_ellipse,
output_writer = write_AWIPS_netcdf_grid, output_kwargs = {'refresh_minutes':int(frame_interval/60.0)},
outpath = AWIPS_GRID_DIR,
)
'ctr_lon': ctr_lon,
}
params['thresh_critical_time'] = thresh_critical_time
params['distance'] = distance
# tag = 'thresh-%s_dist-%s' % (thresh_critical_time, distance)
# outdir = os.path.join(base_out_dir, tag)
# os.mkdir(outdir)
# info = open(os.path.join(outdir, 'input_params.py'), 'w')
# info.write(str(params))
# info.close()
h5_filenames = run_files_with_params([LMA_ASCII_outfile],
LMA_DATA_DIR,
params,
retain_ascii_output=False)
# h5_path_searcher = os.path.join(LMA_DATA_DIR,'LYL*.flash.h5')
# h5_filenames = glob.glob(h5_path_searcher)
# ----- Create NetCDF grids -----
from lmatools.grid.make_grids import grid_h5flashfiles
from lmatools.grid.AWIPS_tools import write_AWIPS_netcdf_grid
x_coord, y_coord, lons, lats, extent_density_grid, AWIPS_outfiles, field_names = grid_h5flashfiles(
h5_filenames,
start_time,
end_time,
frame_interval=frame_interval,
dx=dx,
dy=dy,
def test_sort_and_grid_and_plot(outpath):
""" Given an output path, run sample data included in lmatools through flash sorting and gridding"""
base_sort_dir = outpath
logger_setup(outpath)
files = get_sample_data_list()
center_ID = 'WTLMA'
ctr_lat, ctr_lon = 33.5, -101.5
params = {
'stations': (6, 13),
'chi2': (0, 1.0),
'ctr_lat': ctr_lat,
'ctr_lon': ctr_lon,
'distance': 3000.0,
'thresh_critical_time': 0.15,
'thresh_duration': 3.0,
'mask_length': 6,
}
h5_dir = os.path.join(base_sort_dir, 'h5_files')
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
y, m, d, H, M, S = tfromfile(files[0])
date = datetime(y, m, d, H, M, S)
# Create HDF5 flash files
base_out_dir = (h5_dir + "/20%s" % (date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call([
'chmod', 'a+w', base_out_dir,
h5_dir + '/20%s' % (date.strftime('%y/%b')),
h5_dir + '/20%s' % (date.strftime('%y'))
])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
run_files_with_params(files,
outdir,
params,
cluster,
retain_ascii_output=False,
cleanup_tmp=True)
# Figure out which HDF5 files were created
h5_filenames = glob.glob(h5_dir + '/20%s/LYLOUT*.dat.flash.h5' %
(date.strftime('%y/%b/%d')))
h5_filenames.sort()
# Create NetCDF gridded data
frame_interval = 60.0 * 2 # seconds
dx_km = 3.0e3 # meters
dy_km = 3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat,
ctr_lon,
dx=dx_km,
dy=dy_km,
x_bnd=x_bnd_km,
y_bnd=y_bnd_km)
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y, m, d, H, M, S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y, m, d, H, M, S)
end_time = start_time + timedelta(0, 600)
# print start_time, end_time
outpath = grid_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
grid_dir + '/20%s' % (date.strftime('%y/%b')),
grid_dir + '/20%s' % (date.strftime('%y'))
])
grid_h5flashfiles(h5_filenames,
start_time,
end_time,
frame_interval=frame_interval,
proj_name='latlong',
dx=dx,
dy=dy,
x_bnd=x_bnd,
y_bnd=y_bnd,
ctr_lon=ctr_lon,
ctr_lat=ctr_lat,
outpath=outpath,
output_writer=write_cf_netcdf_latlon,
output_filename_prefix=center_ID,
spatial_scale_factor=1.0,
energy_grids=True)
# Create plots
n_cols = 2
mapping = {
'source': 'lma_source',
'flash_extent': 'flash_extent',
'flash_init': 'flash_initiation',
'footprint': 'flash_footprint'
}
#nc_names = glob.glob('/home/ebruning/Mar18-19/grids/*.nc')
nc_names = glob.glob(grid_dir + '/20%s/*.nc' % (date.strftime('%y/%b/%d')))
nc_names.sort()
outpath = plot_dir + '/20%s' % (date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call([
'chmod', 'a+w', outpath,
plot_dir + '/20%s' % (date.strftime('%y/%b')),
plot_dir + '/20%s' % (date.strftime('%y'))
])
for f in nc_names:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
# print f
make_plot(f,
var,
n_cols=n_cols,
x_name='longitude',
y_name='latitude',
outpath=outpath)
base_out_dir = (h5_dir + "/20%s" % (date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call([
'chmod', 'a+w', base_out_dir,
h5_dir + '/20%s' % (date.strftime('%y/%b')),
h5_dir + '/20%s' % (date.strftime('%y'))
])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
autorun.run_files_with_params(files, outdir, params, cleanup_tmp=False)
#for thresh_critical_time in np.arange(0.15, 0.20, 0.05):
# for distance in np.arange(3000, 3500, 500):
# params['thresh_critical_time'] = thresh_critical_time
# params['distance'] = distance
# tag = 'thresh-%s_dist-%s' % (thresh_critical_time, distance)
# #tag = ''
# outdir = os.path.join(base_out_dir, tag)
# os.mkdir(outdir)
# info = open(os.path.join(outdir, 'input_params.py'), 'w')
# info.write(str(params))
# info.close()
# autorun.run_files_with_params(files, outdir, params, cleanup_tmp=False)
""" The data used to make the grids are LMA data that have been sorted into flashes. The LMA source data and flash metadata are saved into an HDF5 file."""
#h5_filenames = glob.glob('/home/ebruning/Mar18-19/out/thresh-0.15_dist-3000/LYLOUT*.dat.flash.h5')
template = pkg_resources.resource_filename(resource_package, resource_path)
yield template
files = [f for f in sample_data_paths()]
print files
ctrLat = 35.23833
ctrLon = -97.46028
params = {'stations':(6,13),
'chi2':(0,1.0),
'ascii_flashes_out':'flashes_out.dat',
'ctr_lat':ctrLat, 'ctr_lon':ctrLon,
'thresh_duration':3.0,
'distance':3000.0, 'thresh_critical_time':0.15
}
from lmatools.flashsort.autosort.autorun_sklearn import cluster
run_files_with_params(files, outpath, params, cluster, retain_ascii_output=False, cleanup_tmp=True)
'chmod', 'a+w', base_out_dir,
h5_dir + '/20%s' % (date.strftime('%y/%b')),
h5_dir + '/20%s' % (date.strftime('%y'))
])
# ----- Run flash sorting -----
if True:
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
autorun.run_files_with_params(files,
outdir,
params,
cluster,
cleanup_tmp=True)
#for thresh_critical_time in np.arange(0.15, 0.20, 0.05):
# for distance in np.arange(3000, 3500, 500):
# params['thresh_critical_time'] = thresh_critical_time
# params['distance'] = distance
# tag = 'thresh-%s_dist-%s' % (thresh_critical_time, distance)
# #tag = ''
# outdir = os.path.join(base_out_dir, tag)
# os.mkdir(outdir)
# info = open(os.path.join(outdir, 'input_params.py'), 'w')
# info.write(str(params))
# info.close()
# autorun.run_files_with_params(files, outdir, params, cleanup_tmp=False)
def test_sort_and_grid_and_plot(outpath):
""" Given an output path, run sample data included in lmatools through flash sorting and gridding"""
base_sort_dir = outpath
logger_setup(outpath)
files = get_sample_data_list()
center_ID = 'WTLMA'
ctr_lat, ctr_lon = 33.5, -101.5
params = {'stations':(6,13),
'chi2':(0,1.0),
'ctr_lat':ctr_lat, 'ctr_lon':ctr_lon,
'distance':3000.0, 'thresh_critical_time':0.15,
'mask_length':6,
}
h5_dir = os.path.join(base_sort_dir, 'h5_files')
grid_dir = os.path.join(base_sort_dir, 'grid_files')
plot_dir = os.path.join(base_sort_dir, 'plots')
y,m,d,H,M,S = tfromfile(files[0])
date = datetime(y,m,d, H,M,S)
# Create HDF5 flash files
base_out_dir = (h5_dir+"/20%s" %(date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call(['chmod', 'a+w', base_out_dir, h5_dir+'/20%s' %(date.strftime('%y/%b')), h5_dir+'/20%s' %(date.strftime('%y'))])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
run_files_with_params(files, outdir, params, cluster, retain_ascii_output=False, cleanup_tmp=True)
# Figure out which HDF5 files were created
h5_filenames = glob.glob(h5_dir+'/20%s/LYLOUT*.dat.flash.h5' %(date.strftime('%y/%b/%d')))
h5_filenames.sort()
# Create NetCDF gridded data
frame_interval=60.0*2 # seconds
dx_km=3.0e3 # meters
dy_km=3.0e3
x_bnd_km = (-200e3, 200e3)
y_bnd_km = (-200e3, 200e3)
# There are similar functions in lmatools to grid on a regular x,y grid in some map projection.
dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
dx=dx_km, dy=dy_km,
x_bnd = x_bnd_km, y_bnd = y_bnd_km )
# print("dx, dy = {0}, {1} deg".format(dx,dy))
# print("lon_range = {0} deg".format(x_bnd))
# print("lat_range = {0} deg".format(y_bnd))
for f in h5_filenames:
y,m,d,H,M,S = tfromfile(f)
# print y,m,d,H,M,S
start_time = datetime(y,m,d, H,M,S)
end_time = start_time + timedelta(0,600)
# print start_time, end_time
outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
grid_h5flashfiles(h5_filenames, start_time, end_time, frame_interval=frame_interval, proj_name='latlong',
dx=dx, dy=dy, x_bnd=x_bnd, y_bnd=y_bnd, ctr_lon=ctr_lon, ctr_lat=ctr_lat, outpath = outpath,
output_writer = write_cf_netcdf_latlon, output_filename_prefix=center_ID, spatial_scale_factor=1.0
)
# Create plots
n_cols=2
mapping = { 'source':'lma_source',
'flash_extent':'flash_extent',
'flash_init':'flash_initiation',
'footprint':'flash_footprint'}
#nc_names = glob.glob('/home/ebruning/Mar18-19/grids/*.nc')
nc_names = glob.glob(grid_dir+'/20%s/*.nc' %(date.strftime('%y/%b/%d')))
nc_names.sort()
outpath = plot_dir+'/20%s' %(date.strftime('%y/%b/%d'))
if os.path.exists(outpath) == False:
os.makedirs(outpath)
subprocess.call(['chmod', 'a+w', outpath, plot_dir+'/20%s' %(date.strftime('%y/%b')), plot_dir+'/20%s' %(date.strftime('%y'))])
for f in nc_names:
gridtype = f.split('dx_')[-1].replace('.nc', '')
var = mapping[gridtype]
# print f
make_plot(f, var, n_cols=n_cols, x_name='longitude', y_name='latitude', outpath = outpath)
# date = datetime.datetime(2012, 3, 19, 0) #for use for files not in naming nomenclature
base_out_dir = (h5_dir+"/20%s" %(date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call(['chmod', 'a+w', base_out_dir, h5_dir+'/20%s' %(date.strftime('%y/%b')), h5_dir+'/20%s' %(date.strftime('%y'))])
# ----- Run flash sorting -----
if True:
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
autorun.run_files_with_params(files, outdir, params, cluster, cleanup_tmp=True)
#for thresh_critical_time in np.arange(0.15, 0.20, 0.05):
# for distance in np.arange(3000, 3500, 500):
# params['thresh_critical_time'] = thresh_critical_time
# params['distance'] = distance
# tag = 'thresh-%s_dist-%s' % (thresh_critical_time, distance)
# #tag = ''
# outdir = os.path.join(base_out_dir, tag)
# os.mkdir(outdir)
# info = open(os.path.join(outdir, 'input_params.py'), 'w')
# info.write(str(params))
# info.close()
# autorun.run_files_with_params(files, outdir, params, cleanup_tmp=False)
""" The data used to make the grids are LMA data that have been sorted into flashes. The LMA source data and flash metadata are saved into an HDF5 file."""
template = pkg_resources.resource_filename(resource_package,
resource_path)
yield template
files = [f for f in sample_data_paths()]
print(files)
ctrLat = 35.23833
ctrLon = -97.46028
params = {
'stations': (6, 13),
'chi2': (0, 1.0),
'ascii_flashes_out': 'flashes_out.dat',
'ctr_lat': ctrLat,
'ctr_lon': ctrLon,
'thresh_duration': 3.0,
'distance': 3000.0,
'thresh_critical_time': 0.15
}
from lmatools.flashsort.autosort.autorun_sklearn import cluster
run_files_with_params(files,
outpath,
params,
cluster,
retain_ascii_output=False,
cleanup_tmp=True)
files = sys.argv[1:] # Pass in a list of LMA .dat.gz data files, including the full path name
y,m,d,H,M,S = tfromfile(files[0])
date = datetime(y,m,d, H,M,S)
# date = datetime.datetime(2012, 3, 19, 0) #for use for files not in naming nomenclature
base_out_dir = (h5_dir+"/20%s" %(date.strftime('%y/%b/%d')))
if os.path.exists(base_out_dir) == False:
os.makedirs(base_out_dir)
subprocess.call(['chmod', 'a+w', base_out_dir, h5_dir+'/20%s' %(date.strftime('%y/%b')), h5_dir+'/20%s' %(date.strftime('%y'))])
tag = ''
outdir = os.path.join(base_out_dir, tag)
info = open(os.path.join(outdir, 'input_params.py'), 'w')
info.write(str(params))
info.close()
autorun.run_files_with_params(files, outdir, params, cleanup_tmp=False)
#for thresh_critical_time in np.arange(0.15, 0.20, 0.05):
# for distance in np.arange(3000, 3500, 500):
# params['thresh_critical_time'] = thresh_critical_time
# params['distance'] = distance
# tag = 'thresh-%s_dist-%s' % (thresh_critical_time, distance)
# #tag = ''
# outdir = os.path.join(base_out_dir, tag)
# os.mkdir(outdir)
# info = open(os.path.join(outdir, 'input_params.py'), 'w')
# info.write(str(params))
# info.close()
# autorun.run_files_with_params(files, outdir, params, cleanup_tmp=False)
""" The data used to make the grids are LMA data that have been sorted into flashes. The LMA source data and flash metadata are saved into an HDF5 file."""
