def _test_colocated(): eventid = "ci38445975" datafiles, event = read_data_dir("fdsn", eventid, "*") datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config_file = os.path.join(datadir, "test_config.yml") with open(config_file, "r", encoding="utf-8") as f: yaml = YAML() yaml.preserve_quotes = True config = yaml.load(f) processed_streams = process_streams(raw_streams, event, config=config) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, "test.hdf") ws = StreamWorkspace(tfile) ws.addEvent(event) ws.addStreams(event, raw_streams, label="raw") ws.addStreams(event, processed_streams, label="processed") ws.calcMetrics(eventid, labels=["processed"], config=config) stasum = ws.getStreamMetrics(eventid, "CI", "MIKB", "processed") np.testing.assert_allclose( stasum.get_pgm("duration", "geometric_mean"), 38.94480068) ws.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def _test_colocated(): eventid = 'ci38445975' datafiles, event = read_data_dir('fdsn', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config_file = os.path.join(datadir, 'test_config.yml') with open(config_file, 'r', encoding='utf-8') as f: config = yaml.load(f, Loader=yaml.FullLoader) processed_streams = process_streams(raw_streams, event, config=config) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') ws = StreamWorkspace(tfile) ws.addEvent(event) ws.addStreams(event, raw_streams, label='raw') ws.addStreams(event, processed_streams, label='processed') ws.calcMetrics(eventid, labels=['processed'], config=config) stasum = ws.getStreamMetrics(eventid, 'CI', 'MIKB', 'processed') np.testing.assert_allclose( stasum.get_pgm('duration', 'geometric_mean'), 38.94480068) ws.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def test_metrics2(): eventid = 'usb000syza' datafiles, event = read_data_dir('knet', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, 'config_min_freq_0p2.yml')) config['metrics']['output_imts'].append('Arias') config['metrics']['output_imcs'].append('arithmetic_mean') # turn off sta/lta check and snr checks newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) processed_streams = process_streams(raw_streams, event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, processed_streams, label='processed') workspace.calcMetrics(event.id, labels=['processed']) etable, imc_tables1, readmes1 = workspace.getTables('processed') assert 'ARITHMETIC_MEAN' not in imc_tables1 assert 'ARITHMETIC_MEAN' not in readmes1 del workspace.dataset.auxiliary_data.WaveFormMetrics del workspace.dataset.auxiliary_data.StationMetrics workspace.calcMetrics(event.id, labels=['processed'], config=config) etable2, imc_tables2, readmes2 = workspace.getTables('processed') assert 'ARITHMETIC_MEAN' in imc_tables2 assert 'ARITHMETIC_MEAN' in readmes2 assert 'ARIAS' in imc_tables2['ARITHMETIC_MEAN'] testarray = readmes2['ARITHMETIC_MEAN']['Column header'].to_numpy() assert 'ARIAS' in testarray except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def test_metrics2(): eventid = 'usb000syza' datafiles, event = read_data_dir('knet', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = get_config() config['metrics']['output_imts'].append('Arias') config['metrics']['output_imcs'].append('arithmetic_mean') # turn off sta/lta check and snr checks newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) processed_streams = process_streams(raw_streams, event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, processed_streams, label='processed') workspace.calcMetrics(event.id, labels=['processed']) etable, imc_tables1 = workspace.getTables('processed') etable2, imc_tables2 = workspace.getTables('processed', config=config) assert 'ARITHMETIC_MEAN' not in imc_tables1 assert 'ARITHMETIC_MEAN' in imc_tables2 assert 'ARIAS' in imc_tables2['ARITHMETIC_MEAN'] except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def test_metrics(): eventid = "usb000syza" datafiles, event = read_data_dir("knet", eventid, "*") datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml")) # turn off sta/lta check and snr checks # newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) # processed_streams = process_streams(raw_streams, event, config=newconfig) newconfig = config.copy() newconfig["processing"].append( {"NNet_QA": { "acceptance_threshold": 0.5, "model_name": "CantWell" }}) processed_streams = process_streams(raw_streams.copy(), event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, "test.hdf") workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, raw_streams, label="raw") workspace.addStreams(event, processed_streams, label="processed") stream1 = raw_streams[0] # Get metrics from station summary for raw streams summary1 = StationSummary.from_config(stream1) s1_df_in = summary1.pgms.sort_values(["IMT", "IMC"]) array1 = s1_df_in["Result"].to_numpy() # Compare to metrics from getStreamMetrics for raw streams workspace.calcMetrics(eventid, labels=["raw"]) summary1_a = workspace.getStreamMetrics(event.id, stream1[0].stats.network, stream1[0].stats.station, "raw") s1_df_out = summary1_a.pgms.sort_values(["IMT", "IMC"]) array2 = s1_df_out["Result"].to_numpy() np.testing.assert_allclose(array1, array2, atol=1e-6, rtol=1e-6) workspace.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def test_metrics(): eventid = 'usb000syza' datafiles, event = read_data_dir('knet', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, 'config_min_freq_0p2.yml')) # turn off sta/lta check and snr checks # newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) # processed_streams = process_streams(raw_streams, event, config=newconfig) newconfig = config.copy() newconfig['processing'].append( {'NNet_QA': { 'acceptance_threshold': 0.5, 'model_name': 'CantWell' }}) processed_streams = process_streams(raw_streams.copy(), event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, raw_streams, label='raw') workspace.addStreams(event, processed_streams, label='processed') stream1 = raw_streams[0] # Get metrics from station summary for raw streams summary1 = StationSummary.from_config(stream1) s1_df_in = summary1.pgms.sort_values(['IMT', 'IMC']) array1 = s1_df_in['Result'].to_numpy() # Compare to metrics from getStreamMetrics for raw streams workspace.calcMetrics(eventid, labels=['raw']) summary1_a = workspace.getStreamMetrics(event.id, stream1[0].stats.network, stream1[0].stats.station, 'raw') s1_df_out = summary1_a.pgms.sort_values(['IMT', 'IMC']) array2 = s1_df_out['Result'].to_numpy() np.testing.assert_allclose(array1, array2, atol=1e-6, rtol=1e-6) workspace.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def _test_metrics2(): eventid = "usb000syza" datafiles, event = read_data_dir("knet", eventid, "*") datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml")) config["metrics"]["output_imts"].append("Arias") config["metrics"]["output_imcs"].append("arithmetic_mean") # Adjust checks so that streams pass checks for this test newconfig = drop_processing(config, ["check_sta_lta"]) csnr = [s for s in newconfig["processing"] if "compute_snr" in s.keys()][0] csnr["compute_snr"]["check"]["threshold"] = -10.0 processed_streams = process_streams(raw_streams, event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, "test.hdf") workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, processed_streams, label="processed") workspace.calcMetrics(event.id, labels=["processed"]) etable, imc_tables1, readmes1 = workspace.getTables("processed") assert "ARITHMETIC_MEAN" not in imc_tables1 assert "ARITHMETIC_MEAN" not in readmes1 del workspace.dataset.auxiliary_data.WaveFormMetrics del workspace.dataset.auxiliary_data.StationMetrics workspace.calcMetrics(event.id, labels=["processed"], config=config) etable2, imc_tables2, readmes2 = workspace.getTables("processed") assert "ARITHMETIC_MEAN" in imc_tables2 assert "ARITHMETIC_MEAN" in readmes2 assert "ARIAS" in imc_tables2["ARITHMETIC_MEAN"] testarray = readmes2["ARITHMETIC_MEAN"]["Column header"].to_numpy() assert "ARIAS" in testarray workspace.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def _test_vs30_dist_metrics(): KNOWN_DISTANCES = { "epicentral": 5.1, "hypocentral": 10.2, "rupture": 2.21, "rupture_var": np.nan, "joyner_boore": 2.21, "joyner_boore_var": np.nan, "gc2_rx": 2.66, "gc2_ry": 3.49, "gc2_ry0": 0.00, "gc2_U": 34.34, "gc2_T": 2.66, } KNOWN_BAZ = 239.46 KNOWN_VS30 = 331.47 eventid = "ci38457511" datafiles, event = read_data_dir("fdsn", eventid, "*") datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml")) processed_streams = process_streams(raw_streams, event, config=config) rupture_file = get_rupture_file(datadir) grid_file = os.path.join(datadir, "test_grid.grd") config["metrics"]["vs30"] = { "vs30": { "file": grid_file, "column_header": "GlobalVs30", "readme_entry": "GlobalVs30", "units": "m/s", } } tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, "test.hdf") ws = StreamWorkspace(tfile) ws.addEvent(event) ws.addStreams(event, raw_streams, label="raw") ws.addStreams(event, processed_streams, label="processed") ws.calcMetrics(event.id, rupture_file=rupture_file, labels=["processed"], config=config) sta_sum = ws.getStreamMetrics(event.id, "CI", "CLC", "processed") for dist in sta_sum.distances: np.testing.assert_allclose(sta_sum.distances[dist], KNOWN_DISTANCES[dist], rtol=0.01) np.testing.assert_allclose(sta_sum._back_azimuth, KNOWN_BAZ, rtol=0.01) np.testing.assert_allclose(sta_sum._vs30["vs30"]["value"], KNOWN_VS30, rtol=0.01) event_df, imc_tables, readme_tables = ws.getTables("processed") ws.close() check_cols = set([ "EpicentralDistance", "HypocentralDistance", "RuptureDistance", "RuptureDistanceVar", "JoynerBooreDistance", "JoynerBooreDistanceVar", "GC2_rx", "GC2_ry", "GC2_ry0", "GC2_U", "GC2_T", "GlobalVs30", "BackAzimuth", ]) assert check_cols.issubset(set(readme_tables["Z"]["Column header"])) assert check_cols.issubset(set(imc_tables["Z"].columns)) except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def _test_vs30_dist_metrics(): KNOWN_DISTANCES = { 'epicentral': 5.1, 'hypocentral': 10.2, 'rupture': 2.21, 'rupture_var': np.nan, 'joyner_boore': 2.21, 'joyner_boore_var': np.nan, 'gc2_rx': 2.66, 'gc2_ry': 3.49, 'gc2_ry0': 0.00, 'gc2_U': 34.34, 'gc2_T': 2.66 } KNOWN_BAZ = 239.46 KNOWN_VS30 = 331.47 eventid = 'ci38457511' datafiles, event = read_data_dir('fdsn', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, 'config_min_freq_0p2.yml')) processed_streams = process_streams(raw_streams, event, config=config) rupture_file = get_rupture_file(datadir) grid_file = os.path.join(datadir, 'test_grid.grd') config['metrics']['vs30'] = { 'vs30': { 'file': grid_file, 'column_header': 'GlobalVs30', 'readme_entry': 'GlobalVs30', 'units': 'm/s' } } tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') ws = StreamWorkspace(tfile) ws.addEvent(event) ws.addStreams(event, raw_streams, label='raw') ws.addStreams(event, processed_streams, label='processed') ws.calcMetrics(event.id, rupture_file=rupture_file, labels=['processed'], config=config) sta_sum = ws.getStreamMetrics(event.id, 'CI', 'CLC', 'processed') for dist in sta_sum.distances: np.testing.assert_allclose(sta_sum.distances[dist], KNOWN_DISTANCES[dist], rtol=0.01) np.testing.assert_allclose(sta_sum._back_azimuth, KNOWN_BAZ, rtol=0.01) np.testing.assert_allclose(sta_sum._vs30['vs30']['value'], KNOWN_VS30, rtol=0.01) event_df, imc_tables, readme_tables = ws.getTables('processed') ws.close() check_cols = set([ 'EpicentralDistance', 'HypocentralDistance', 'RuptureDistance', 'RuptureDistanceVar', 'JoynerBooreDistance', 'JoynerBooreDistanceVar', 'GC2_rx', 'GC2_ry', 'GC2_ry0', 'GC2_U', 'GC2_T', 'GlobalVs30', 'BackAzimuth' ]) assert check_cols.issubset(set(readme_tables['Z']['Column header'])) assert check_cols.issubset(set(imc_tables['Z'].columns)) except Exception as e: raise (e) finally: shutil.rmtree(tdir)