def test_plot():
# read in data
datafiles, _ = read_data_dir("cwb", "us1000chhc")
streams = []
for filename in datafiles:
streams += read_data(filename)
# One plot arias
axes = plot_arias(streams[3])
assert len(axes) == 3
# Multiplot arias
axs = matplotlib.pyplot.subplots(len(streams), 3, figsize=(15, 10))[1]
axs = axs.flatten()
idx = 0
for stream in streams:
axs = plot_arias(
stream,
axes=axs,
axis_index=idx,
minfontsize=15,
show_maximum=False,
title="18km NNE of Hualian, Taiwan",
)
idx += 3
# One plot durations
durations = [(0.05, 0.75), (0.2, 0.8), (0.05, 0.95)]
axes = plot_durations(streams[3], durations)
assert len(axes) == 3
# Multiplot durations
axs = matplotlib.pyplot.subplots(len(streams), 3, figsize=(15, 10))[1]
axs = axs.flatten()
idx = 0
for stream in streams:
axs = plot_durations(
stream,
durations,
axes=axs,
axis_index=idx,
minfontsize=15,
title="18km NNE of Hualian, Taiwan",
)
idx += 3
# Moveout plots
epicenter_lat = 24.14
epicenter_lon = 121.69
plot_moveout(
streams,
epicenter_lat,
epicenter_lon,
"1",
figsize=(15, 10),
minfontsize=16,
normalize=True,
factor=0.1,
)
def process_event(event,
outdir,
pcommands,
config,
input_directory,
process_tag,
files_created,
output_format,
status,
recompute_metrics,
export_dir=None):
# setup logging to write to the input logfile
argthing = namedtuple('args', ['debug', 'quiet'])
args = argthing(debug=True, quiet=False)
setup_logger(args)
logger = logging.getLogger()
stream_handler = logger.handlers[0]
logfile = os.path.join(outdir, '%s.log' % event.id)
fhandler = logging.FileHandler(logfile)
logger.removeHandler(stream_handler)
logger.addHandler(fhandler)
event_dir = os.path.join(outdir, event.id)
if not os.path.exists(event_dir):
os.makedirs(event_dir)
workname = os.path.join(event_dir, WORKSPACE_NAME)
workspace_exists = os.path.isfile(workname)
workspace_has_processed = False
workspace = None
processing_done = False
if workspace_exists:
workspace = StreamWorkspace.open(workname)
labels = workspace.getLabels()
if len(labels):
labels.remove('unprocessed')
elif 'assemble' not in pcommands:
print('No data in workspace. Please run assemble.')
sys.exit(1)
if len(labels) == 1:
process_tag = labels[0]
workspace_has_processed = True
else:
if 'process' not in pcommands:
fmt = '\nThere are %i sets of processed data in %s.'
tpl = (len(labels), workname)
print(fmt % tpl)
print(('This software can only handle one set of '
'processed data. Exiting.\n'))
sys.exit(1)
download_done = False
# Need to initialize rstreams/pstreams
rstreams = []
pstreams = []
rupture_file = None
if 'assemble' in pcommands:
logging.info('Downloading/loading raw streams...')
workspace, workspace_file, rstreams, rupture_file = download(
event, event_dir, config, input_directory)
download_done = True
append_file(files_created, 'Workspace', workname)
else:
if not workspace_exists:
print('\nYou opted not to download or process from input.')
print('No previous HDF workspace file could be found.')
print('Try re-running with the assemble command with or ')
print('without the --directory option.\n')
sys.exit(1)
if 'process' in pcommands:
logging.info('Getting raw streams from workspace...')
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=H5pyDeprecationWarning)
rstreams = workspace.getStreams(event.id,
labels=['unprocessed'])
download_done = True
else:
need_processed = set(['report', 'shakemap'])
need_pstreams = len(need_processed.intersection(pcommands))
if workspace_has_processed:
if need_pstreams:
logging.info('Getting processed streams from workspace...')
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=H5pyDeprecationWarning)
pstreams = workspace.getStreams(event.id,
labels=[process_tag])
download_done = True
processing_done = True
if ('process' in pcommands and download_done and not processing_done
and len(rstreams)):
logging.info('Processing raw streams for event %s...' % event.id)
pstreams = process_streams(rstreams, event, config=config)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=H5pyDeprecationWarning)
workspace.addStreams(event, pstreams, label=process_tag)
workspace.calcMetrics(event.id,
labels=[process_tag],
config=config,
streams=pstreams,
stream_label=process_tag,
rupture_file=rupture_file)
processing_done = True
if 'export' in pcommands:
if export_dir is not None:
if not os.path.isdir(export_dir):
os.makedirs(export_dir)
outdir = export_dir
labels = workspace.getLabels()
if 'unprocessed' not in labels:
fmt = ('Workspace file "%s" appears to have no unprocessed '
'data. Skipping.')
logging.info(fmt % workspace_file)
else:
labels.remove('unprocessed')
if not labels:
fmt = ('Workspace file "%s" appears to have no processed '
'data. Skipping.')
logging.info(fmt % workspace_file)
else:
logging.info('Creating tables for event %s...', event.id)
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=H5pyDeprecationWarning)
if recompute_metrics:
del workspace.dataset.auxiliary_data.WaveFormMetrics
del workspace.dataset.auxiliary_data.StationMetrics
workspace.calcMetrics(event.id,
labels=labels,
config=config,
rupture_file=rupture_file)
event_table, imc_tables, readmes = workspace.getTables(
labels[0], streams=pstreams, stream_label=process_tag)
ev_fit_spec, fit_readme = workspace.getFitSpectraTable(
event.id, labels[0], pstreams)
# Set the precisions for the imc tables, event table, and
# fit_spectra table before writing
imc_tables_formatted = {}
for imc, imc_table in imc_tables.items():
imc_tables_formatted[imc] = set_precisions(imc_table)
event_table_formatted = set_precisions(event_table)
df_fit_spectra_formatted = set_precisions(ev_fit_spec)
if not os.path.isdir(outdir):
os.makedirs(outdir)
filenames = ['events'] + \
[imc.lower() for imc in imc_tables_formatted.keys()] + \
[imc.lower() + '_README' for imc in readmes.keys()] + \
['fit_spectra_parameters', 'fit_spectra_parameters_README']
files = [event_table_formatted] + list(
imc_tables_formatted.values()) + list(readmes.values()) + [
df_fit_spectra_formatted, fit_readme
]
if output_format != 'csv':
output_format = 'xlsx'
for filename, df in dict(zip(filenames, files)).items():
filepath = os.path.join(outdir,
filename + '.%s' % output_format)
if os.path.exists(filepath):
if 'README' in filename:
continue
else:
mode = 'a'
header = False
else:
mode = 'w'
header = True
append_file(files_created, 'Tables', filepath)
if output_format == 'csv':
df.to_csv(filepath,
index=False,
float_format=DEFAULT_FLOAT_FORMAT,
na_rep=DEFAULT_NA_REP,
mode=mode,
header=header)
else:
df.to_excel(filepath,
index=False,
float_format=DEFAULT_FLOAT_FORMAT,
na_rep=DEFAULT_NA_REP,
mode=mode,
header=header)
if ('report' in pcommands and processing_done and len(pstreams)):
logging.info('Creating diagnostic plots for event %s...' % event.id)
plot_dir = os.path.join(event_dir, 'plots')
if not os.path.isdir(plot_dir):
os.makedirs(plot_dir)
for stream in pstreams:
summary_plots(stream, plot_dir, event)
mapfile = draw_stations_map(pstreams, event, event_dir)
plot_moveout(pstreams,
event.latitude,
event.longitude,
file=os.path.join(event_dir, 'moveout_plot.png'))
append_file(files_created, 'Station map', mapfile)
append_file(files_created, 'Moveout plot', 'moveout_plot.png')
logging.info('Creating diagnostic report for event %s...' % event.id)
# Build the summary report?
build_conf = config['build_report']
report_format = build_conf['format']
if report_format == 'latex':
report_file, success = build_report_latex(pstreams,
event_dir,
event,
config=config)
else:
report_file = ''
success = False
if os.path.isfile(report_file) and success:
append_file(files_created, 'Summary report', report_file)
if 'provenance' in pcommands and processing_done and len(pstreams):
logging.info('Creating provenance table for event %s...' % event.id)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=H5pyDeprecationWarning)
provdata = workspace.getProvenance(event.id, labels=[process_tag])
if output_format == 'csv':
csvfile = os.path.join(event_dir, 'provenance.csv')
append_file(files_created, 'Provenance', csvfile)
provdata.to_csv(csvfile)
else:
excelfile = os.path.join(event_dir, 'provenance.xlsx')
append_file(files_created, 'Provenance', excelfile)
provdata.to_excel(excelfile, index=False)
if 'shakemap' in pcommands and processing_done and len(pstreams):
logging.info('Creating shakemap table for event %s...' % event.id)
shakemap_file, jsonfile = save_shakemap_amps(pstreams, event,
event_dir)
append_file(files_created, 'shakemap', shakemap_file)
append_file(files_created, 'shakemap', jsonfile)
if status and processing_done and len(pstreams):
if status == 'short':
index = 'Failure reason'
col = ['Number of records']
elif status == 'long':
index = 'Station ID'
col = ['Failure reason']
elif status == 'net':
index = 'Network'
col = ['Number of passed records', 'Number of failed records']
status_info = pstreams.get_status(status)
status_info.to_csv(os.path.join(event_dir, 'status.csv'),
header=col,
index_label=index)
# since we don't know how many events users will be processing,
# let's guard against memory issues by clearing out the big data
# structures
workspace.close()
logging.info('Finishing event %s' % event.id)
return workname
def main(self, gmrecords):
"""Generate summary report.
This function generates summary plots and then combines them into a
report with latex. If latex (specifically `pdflatex`) is not found on
the system then the PDF report will not be generated but the
constituent plots will be available.
Args:
gmrecords:
GMrecordsApp instance.
"""
logging.info('Running subcommand \'%s\'' % self.command_name)
self.gmrecords = gmrecords
self._get_events()
for event in self.events:
self.eventid = event.id
event_dir = os.path.join(gmrecords.data_path, self.eventid)
workname = os.path.join(event_dir, WORKSPACE_NAME)
if not os.path.isfile(workname):
logging.info(
'No workspace file found for event %s. Please run '
'subcommand \'assemble\' to generate workspace file.' %
self.eventid)
logging.info('Continuing to next event.')
continue
self.workspace = StreamWorkspace.open(workname)
self._get_pstreams()
self.workspace.close()
if not hasattr(self, 'pstreams'):
logging.info('No processed waveforms available. No report '
'generated.')
return
logging.info('Creating diagnostic plots for event %s...' %
self.eventid)
plot_dir = os.path.join(event_dir, 'plots')
if not os.path.isdir(plot_dir):
os.makedirs(plot_dir)
for stream in self.pstreams:
summary_plots(stream, plot_dir, event)
mapfile = draw_stations_map(self.pstreams, event, event_dir)
moveoutfile = os.path.join(event_dir, 'moveout_plot.png')
plot_moveout(self.pstreams,
event.latitude,
event.longitude,
file=moveoutfile)
self.append_file('Station map', mapfile)
self.append_file('Moveout plot', moveoutfile)
logging.info('Generating summary report for event %s...' %
self.eventid)
build_conf = gmrecords.conf['build_report']
report_format = build_conf['format']
if report_format == 'latex':
report_file, success = build_report_latex(
self.pstreams,
event_dir,
event,
prefix="%s_%s" % (gmrecords.project, gmrecords.args.label),
config=gmrecords.conf)
else:
report_file = ''
success = False
if os.path.isfile(report_file) and success:
self.append_file('Summary report', report_file)
self._summarize_files_created()