def test(): datafiles, origin = read_data_dir("fdsn", "nc72282711", "BK.CMB*.mseed") streams = [] for datafile in datafiles: streams += read_obspy(datafile) assert streams[0].get_id() == "BK.CMB.HN" datafiles, origin = read_data_dir("fdsn", "nc72282711", "TA.M04C*.mseed") streams = [] for datafile in datafiles: streams += read_obspy(datafile) assert streams[0].get_id() == "TA.M04C.HN" # test assignment of Z channel datafiles, origin = read_data_dir("fdsn", "nc73300395", "BK.VALB*.mseed") streams = [] for datafile in datafiles: streams += read_obspy(datafile) # get all channel names channels = sorted([st[0].stats.channel for st in streams]) assert channels == ["HN2", "HN3", "HNZ"]
def test_num_outliers(): data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed") data_files.sort() streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) num_outliers = [] for st in sc: ping_method = Ping(st) num_outliers.append(ping_method.num_outliers) np.testing.assert_equal(num_outliers, np.array([239, 26, 0, 6, 133, 145]))
def test_channels(): datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A") datafile_v2 = datafiles[0] stream_v2 = read_geonet(datafile_v2)[0] station_summary = StationSummary.from_stream(stream_v2, ["channels"], ["pga"]) pgms = station_summary.pgms np.testing.assert_almost_equal( pgms.loc["PGA", "H2"].Result, 81.28979591836733, decimal=1 ) np.testing.assert_almost_equal( pgms.loc["PGA", "H1"].Result, 99.3173469387755, decimal=1 ) np.testing.assert_almost_equal( pgms.loc["PGA", "Z"].Result, 183.89693877551022, decimal=1 )
def test_nnet(): conf = get_config() update = { "processing": [ { "detrend": { "detrending_method": "demean" } }, { "detrend": { "detrending_method": "linear" } }, { "compute_snr": { "bandwidth": 20.0, "check": { "max_freq": 5.0, "min_freq": 0.2, "threshold": 3.0 }, } }, { "NNet_QA": { "acceptance_threshold": 0.5, "model_name": "CantWell" } }, ] } update_dict(conf, update) data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A") streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) test = process_streams(sc, origin, conf) tstream = test.select(station="HSES")[0] nnet_dict = tstream.getStreamParam("nnet_qa") np.testing.assert_allclose(nnet_dict["score_HQ"], 0.99321798811740059, rtol=1e-3)
def test_metrics(): eventid = "usb000syza" datafiles, event = read_data_dir("knet", eventid, "*") datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml")) # turn off sta/lta check and snr checks # newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) # processed_streams = process_streams(raw_streams, event, config=newconfig) newconfig = config.copy() newconfig["processing"].append( {"NNet_QA": { "acceptance_threshold": 0.5, "model_name": "CantWell" }}) processed_streams = process_streams(raw_streams.copy(), event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, "test.hdf") workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, raw_streams, label="raw") workspace.addStreams(event, processed_streams, label="processed") stream1 = raw_streams[0] # Get metrics from station summary for raw streams summary1 = StationSummary.from_config(stream1) s1_df_in = summary1.pgms.sort_values(["IMT", "IMC"]) array1 = s1_df_in["Result"].to_numpy() # Compare to metrics from getStreamMetrics for raw streams workspace.calcMetrics(eventid, labels=["raw"]) summary1_a = workspace.getStreamMetrics(event.id, stream1[0].stats.network, stream1[0].stats.station, "raw") s1_df_out = summary1_a.pgms.sort_values(["IMT", "IMC"]) array2 = s1_df_out["Result"].to_numpy() np.testing.assert_allclose(array1, array2, atol=1e-6, rtol=1e-6) workspace.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def test_num_clip_intervals(): data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed") data_files.sort() streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) num_clip_intervals = [] for st in sc: hist_method = Histogram(st) num_clip_intervals.append(hist_method.num_clip_intervals) np.testing.assert_equal(num_clip_intervals, np.array([0, 9, 37, 10, 16, 8]))
def test_sac_csn(): # This reads in example SAC data that does not have a separate metadata # file to meet the needs of the Community Seismic Network: # http://csn.caltech.edu/ datafiles, origin = read_data_dir("csn", "ci38457511", "*.sac") datafiles.sort() traces = [] for d in datafiles: traces.append(read_obspy(d)[0][0]) tr_amax = np.zeros(len(traces)) for i, tr in enumerate(traces): tr_amax[i] = np.max(np.abs(tr.data)) target_amax = np.array([4.3384003e-09, 3.42233e-09, 1.0121747e-07]) np.testing.assert_allclose(target_amax, tr_amax)
def test_num_outliers(): data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed") data_files.sort() streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) num_outliers = [] for st in sc: std_dev_method = Std_Dev(st) num_outliers.append(std_dev_method.num_outliers) np.testing.assert_equal(num_outliers, np.array([0, 1086, 131, 1018, 60, 4862]))
def test_check_instrument(): data_files, origin = read_data_dir("fdsn", "nc51194936", "*.mseed") streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) sc.describe() config = update_config(os.path.join(datadir, "config_test_check_instr.yml")) test = process_streams(sc, origin, config=config) for sta, expected in [("CVS", True), ("GASB", True), ("SBT", False)]: st = test.select(station=sta)[0] logging.info(f"Testing stream: {st}") assert st.passed == expected
def test_end_to_end(): datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A") datafile = datafiles[0] stream = read_geonet(datafile)[0] input_imcs = [ "greater_of_two_horizontals", "channels", "rotd50", "rotd100", "invalid", ] input_imts = ["sa1.0", "PGA", "pgv", "invalid"] m = MetricsController(input_imts, input_imcs, stream, config=config) test_pgms = [ ("PGV", "ROTD(100.0)", 114.24894584734818), ("PGV", "ROTD(50.0)", 81.55436750525355), ("PGV", "Z", 37.47740000000001), ("PGV", "H1", 100.81460000000004), ("PGV", "H2", 68.4354), ("PGV", "GREATER_OF_TWO_HORIZONTALS", 100.81460000000004), ("PGA", "ROTD(100.0)", 100.73875535385548), ("PGA", "ROTD(50.0)", 91.40178541935455), ("PGA", "Z", 183.7722361866693), ("PGA", "H1", 99.24999872535474), ("PGA", "H2", 81.23467239067368), ("PGA", "GREATER_OF_TWO_HORIZONTALS", 99.24999872535474), ("SA(1.000)", "ROTD(100.0)", 146.9023350124098), ("SA(1.000)", "ROTD(50.0)", 106.03202302692158), ("SA(1.000)", "Z", 27.74118995438756), ("SA(1.000)", "H1", 136.25041187387063), ("SA(1.000)", "H2", 84.69296738413021), ("SA(1.000)", "GREATER_OF_TWO_HORIZONTALS", 136.25041187387063), ] pgms = m.pgms assert len(pgms) == len(test_pgms) for target in test_pgms: target_imt = target[0] target_imc = target[1] value = target[2] df = pgms.loc[target_imt, target_imc] assert len(df) == 1 np.testing.assert_array_almost_equal(df["Result"], value, decimal=10)
def test_dmg_v1(): file1, _ = read_data_dir("dmg", "ci3144585", files=["LA116TH.RAW"]) file1 = file1[0] assert is_dmg(file1) stream1 = read_dmg(file1)[0] assert stream1.count() == 3 # test that the traces are acceleration for trace in stream1: assert trace.stats["standard"]["units"] == "acc" # test metadata for trace in stream1: stats = trace.stats assert stats["station"] == "14403" assert stats["delta"] == 0.005 assert stats["location"] == "--" assert stats["network"] == "--" dt = "%Y-%m-%dT%H:%M:%SZ" assert stats["starttime"].strftime(dt) == "1994-01-17T12:31:04Z" assert stats.coordinates["latitude"] == 33.929 assert stats.coordinates["longitude"] == -118.26 assert stats.standard["station_name"] == "LOS ANGELES - 116TH ST. SCHOOL" assert stats.standard["instrument"] == "SMA-1" assert stats.standard["sensor_serial_number"] == "3492" if stats["channel"] == "HN1": assert stats.format_specific["sensor_sensitivity"] == 1.915 assert stats.standard["horizontal_orientation"] == 360 assert stats.standard["instrument_period"] == 0.038 assert stats.standard["instrument_damping"] == 0.59 assert stats.format_specific["time_sd"] == 0.115 if stats["channel"] == "HN2": assert stats.standard["horizontal_orientation"] == 90 assert stats.standard["instrument_period"] == 0.04 assert stats.standard["instrument_damping"] == 0.592 assert stats.format_specific["time_sd"] == 0.12 if stats["channel"] == "HNZ": assert stats.standard["horizontal_orientation"] == 0.0 assert stats.standard["instrument_period"] == 0.039 assert stats.standard["instrument_damping"] == 0.556 assert stats.format_specific["time_sd"] == 0.114 assert stats.standard["process_level"] == PROCESS_LEVELS["V1"] assert stats.standard["source_format"] == "dmg" assert stats.standard["source"] == "unknown"
def test_sa(): datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A") datafile_v2 = datafiles[0] stream_v2 = read_geonet(datafile_v2)[0] sa_target = {} for trace in stream_v2: vtrace = trace.copy() vtrace.integrate() sa_target[vtrace.stats["channel"]] = np.abs(vtrace.max()) with warnings.catch_warnings(): warnings.simplefilter("ignore") station_summary = StationSummary.from_stream( stream_v2, [ "greater_of_two_horizontals", "geometric_mean", "rotd50", "arithmetic_mean", "rotd100", "gmrotd50", "channels", ], ["sa1.0", "sa0.01", "saincorrect"], ) pgms = station_summary.pgms assert "SA(1.000)" in pgms.index.get_level_values(0) np.testing.assert_allclose( pgms.loc["SA(1.000)", "ARITHMETIC_MEAN"].Result, 110.47168962900042 ) np.testing.assert_allclose( pgms.loc["SA(1.000)", "GEOMETRIC_MEAN"].Result, 107.42183990654802 ) np.testing.assert_allclose( pgms.loc["SA(1.000)", "ROTD(50.0)"].Result, 106.03202302692158 ) np.testing.assert_allclose( pgms.loc["SA(1.000)", "ROTD(100.0)"].Result, 146.90233501240979 ) # Check high frequency SA np.testing.assert_allclose(pgms.loc["SA(0.010)", "ROTD(100.0)"].Result, 120.187153) np.testing.assert_allclose(pgms.loc["SA(0.010)", "GMROTD(50.0)"].Result, 95.355300) np.testing.assert_allclose(pgms.loc["SA(0.010)", "H1"].Result, 106.716122) np.testing.assert_allclose(pgms.loc["SA(0.010)", "H2"].Result, 90.497883) np.testing.assert_allclose(pgms.loc["SA(0.010)", "GMROTD(50.0)"].Result, 95.355300)
def test_to_dataframe(): cwb_files, event = read_data_dir("geonet", "nz2018p115908") st = read_data(cwb_files[0])[0] df1 = streams_to_dataframe([st, st], event=event) np.testing.assert_array_equal(df1.STATION.tolist(), ["WPWS", "WPWS"]) np.testing.assert_array_equal( df1.NAME.tolist(), ["Waipawa_District_Council", "Waipawa_District_Council"] ) # let's use sets to make sure all the columns are present in whatever order cmp1 = set( [ "ELEVATION", "EPICENTRAL_DISTANCE", "GREATER_OF_TWO_HORIZONTALS", "H1", "H2", "HYPOCENTRAL_DISTANCE", "LAT", "LON", "NAME", "NETID", "SOURCE", "STATION", "Z", ] ) cmp2 = set(["", "PGA", "PGV", "SA(0.300)", "SA(1.000)", "SA(3.000)"]) header1 = set(df1.columns.levels[0]) header2 = set(df1.columns.levels[1]) assert header1 == cmp1 assert header2 == cmp2 # idx = 0 # for s in df1.columns.levels: # for col in s: # try: # assert col == target_levels[idx] # except Exception as e: # x = 1 # idx += 1 # This was previously not being tested """imts = ['PGA', 'PGV', 'SA(0.3)', 'SA(1.0)', 'SA(3.0)']
def test_all_num_clip_intervals(): data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed") data_files.sort() streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) num_clip_intervals = [] for st in sc: hist_method = Histogram(st, test_all=True) num_clip_intervals.append(hist_method.num_clip_intervals) np.testing.assert_equal( num_clip_intervals, np.array([[0, 0, 0], [0, 9, 18], [37, 31, 8], [10, 23, 0], [16, 40, 21], [0, 0, 8]]), )
def test_auto_fchp(): data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A") data_files.sort() streams = [] for f in data_files: streams += read_data(f) sc = StreamCollection(streams) output_fchp = [] config = get_config() config["integration"]["frequency"] = True for st in sc: for tr in st: tr.setParameter( "corner_frequencies", { "type": "constant", "highpass": 0.001, "lowpass": 20 }, ) tmp_st = ridder_fchp(st, config=config) for tr in tmp_st: initial_corners = tr.getParameter("corner_frequencies") output_fchp.append(initial_corners["highpass"]) target_fchp = np.array([ 0.021345158261480087, 0.022839239726168643, 0.02482398434993213, 0.01399481102242619, 0.026850167635921275, 0.004817661513765862, 0.008204101694236587, 0.006429246474225982, 0.004237087327289796, ]) np.testing.assert_allclose(output_fchp, target_fchp, atol=1e-7)
def test_acceleration(): datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A") acc_file = datafiles[0] acc = read_data(acc_file)[0] target_g = acc[0].data * GAL_TO_PCTG target_m = acc[0].data / 100 target_cm = acc[0].data acc_g = get_acceleration(acc, units="%%g") assert acc_g[0].stats["units"] == "%%g" np.testing.assert_allclose(acc_g[0], target_g) acc_m = get_acceleration(acc, units="m/s/s") assert acc_m[0].stats["units"] == "m/s/s" np.testing.assert_allclose(acc_m[0], target_m) acc_cm = get_acceleration(acc, units="cm/s/s") assert acc_cm[0].stats["units"] == "cm/s/s" np.testing.assert_allclose(acc_cm[0], target_cm)
def test_pga(): datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A") datafile_v2 = datafiles[0] stream_v2 = read_geonet(datafile_v2)[0] with warnings.catch_warnings(): warnings.simplefilter("ignore") station_summary = StationSummary.from_stream( stream_v2, [ "channels", "greater_of_two_horizontals", "gmrotd0", "gmrotd50", "gmrotd100", "rotd50", "geometric_mean", "arithmetic_mean", ], ["pga", "sa1.0", "saincorrect"], ) pga_df = station_summary.pgms.loc["PGA"] AM = pga_df.loc["ARITHMETIC_MEAN"].Result GM = pga_df.loc["GEOMETRIC_MEAN"].Result HN1 = pga_df.loc["H1"].Result HN2 = pga_df.loc["H2"].Result HNZ = pga_df.loc["Z"].Result gmrotd0 = pga_df.loc["GMROTD(0.0)"].Result gmrotd50 = pga_df.loc["GMROTD(50.0)"].Result gmrotd100 = pga_df.loc["GMROTD(100.0)"].Result rotd50 = pga_df.loc["ROTD(50.0)"].Result greater = pga_df.loc["GREATER_OF_TWO_HORIZONTALS"].Result np.testing.assert_allclose(AM, 90.242335558014219) np.testing.assert_allclose(GM, 89.791654017670112) np.testing.assert_allclose(HN2, 81.234672390673683) np.testing.assert_allclose(HN1, 99.249998725354743) np.testing.assert_almost_equal(HNZ, 183.77223618666929) np.testing.assert_allclose(greater, 99.249998725354743) np.testing.assert_allclose(gmrotd0, 83.487703753812113) np.testing.assert_allclose(gmrotd50, 86.758642638162982) np.testing.assert_allclose(gmrotd100, 89.791654017670112) np.testing.assert_allclose(rotd50, 91.401785419354567)
def test_v0(): datafiles, origin = read_data_dir("cosmos", "ftbragg") dfile = datafiles[0] # TODO: Fix this problem, or get the data fixed? assert is_cosmos(dfile) try: rstreams = read_cosmos(dfile) tstream = rstreams[0].copy() # raw stream streams = StreamCollection(rstreams) pstream = remove_response(rstreams[0], 0, 0) pstream.detrend(type="demean") for trace in tstream: trace.data /= trace.stats.standard.instrument_sensitivity trace.data *= 100 tstream.detrend(type="demean") np.testing.assert_almost_equal(tstream[0].data, pstream[0].data) except Exception as e: pass
def test_check_clipping(): data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed") data_files.sort() origin = get_event_object("hv70907436") streams = [] for f in data_files: streams += read_data(f) codes = ["HV.TOUO", "HV.MOKD", "HV.MLOD", "HV.HOVE", "HV.HUAD", "HV.HSSD"] passed = [] for code in codes: traces = [] for ss in streams: tcode = f"{ss[0].stats.network}.{ss[0].stats.station}" if tcode == code: traces.append(ss[0]) st = StationStream(traces) check_clipping(st, origin) passed.append(st.passed) assert np.all(~np.array(passed))
def _test_stream_params(): eventid = "us1000778i" datafiles, event = read_data_dir("geonet", eventid, "20161113_110259_WTMC_20.V1A") tdir = tempfile.mkdtemp() streams = [] try: streams += read_data(datafiles[0]) statsdict = {"name": "Fred", "age": 34} streams[0].setStreamParam("stats", statsdict) tfile = os.path.join(tdir, "test.hdf") workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, streams, label="stats") outstreams = workspace.getStreams(event.id, labels=["stats"]) cmpdict = outstreams[0].getStreamParam("stats") assert cmpdict == statsdict workspace.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def _test_metrics2(): eventid = "usb000syza" datafiles, event = read_data_dir("knet", eventid, "*") datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml")) config["metrics"]["output_imts"].append("Arias") config["metrics"]["output_imcs"].append("arithmetic_mean") # Adjust checks so that streams pass checks for this test newconfig = drop_processing(config, ["check_sta_lta"]) csnr = [s for s in newconfig["processing"] if "compute_snr" in s.keys()][0] csnr["compute_snr"]["check"]["threshold"] = -10.0 processed_streams = process_streams(raw_streams, event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, "test.hdf") workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, processed_streams, label="processed") workspace.calcMetrics(event.id, labels=["processed"]) etable, imc_tables1, readmes1 = workspace.getTables("processed") assert "ARITHMETIC_MEAN" not in imc_tables1 assert "ARITHMETIC_MEAN" not in readmes1 del workspace.dataset.auxiliary_data.WaveFormMetrics del workspace.dataset.auxiliary_data.StationMetrics workspace.calcMetrics(event.id, labels=["processed"], config=config) etable2, imc_tables2, readmes2 = workspace.getTables("processed") assert "ARITHMETIC_MEAN" in imc_tables2 assert "ARITHMETIC_MEAN" in readmes2 assert "ARIAS" in imc_tables2["ARITHMETIC_MEAN"] testarray = readmes2["ARITHMETIC_MEAN"]["Column header"].to_numpy() assert "ARIAS" in testarray workspace.close() except Exception as e: raise (e) finally: shutil.rmtree(tdir)
def test_read(): config = get_config() cosmos_files, _ = read_data_dir("cosmos", "ci14155260", "Cosmos12TimeSeriesTest.v1") cwb_files, _ = read_data_dir("cwb", "us1000chhc", "1-EAS.dat") dmg_files, _ = read_data_dir("dmg", "nc71734741", "CE89146.V2") geonet_files, _ = read_data_dir( "geonet", "us1000778i", "20161113_110259_WTMC_20.V1A" ) knet_files, _ = read_data_dir("knet", "us2000cnnl", "AOM0011801241951.EW") smc_files, _ = read_data_dir("smc", "nc216859", "0111a.smc") file_dict = {} file_dict["cosmos"] = cosmos_files[0] file_dict["cwb"] = cwb_files[0] file_dict["dmg"] = dmg_files[0] file_dict["geonet"] = geonet_files[0] file_dict["knet"] = knet_files[0] file_dict["smc"] = smc_files[0] for file_format in file_dict: file_path = file_dict[file_format] assert _get_format(file_path, config) == file_format assert _validate_format(file_path, config, file_format) == file_format assert _validate_format(file_dict["knet"], config, "smc") == "knet" assert _validate_format(file_dict["dmg"], config, "cosmos") == "dmg" assert _validate_format(file_dict["cosmos"], config, "invalid") == "cosmos" for file_format in file_dict: try: stream = read_data(file_dict[file_format], config, file_format)[0] except Exception as e: pass assert stream[0].stats.standard["source_format"] == file_format stream = read_data(file_dict[file_format])[0] assert stream[0].stats.standard["source_format"] == file_format # test exception try: file_path = smc_files[0].replace("0111a.smc", "not_a_file.smc") read_data(file_path)[0] success = True except BaseException: success = False assert success == False
def test_bhrc(): datafiles, origin = read_data_dir("bhrc", "usp000jq5p") # make sure format checker works assert is_bhrc(datafiles[0]) raw_streams = [] for dfile in datafiles: raw_streams += read_bhrc(dfile) peaks = { "5528": 4.793910, "5529": 1.024440, "5522": 1.595120, "5523": 2.291470, "5520": 26.189800, "5526": 1.319720, } for stream in raw_streams: summary = StationSummary.from_config(stream) cmp_value = peaks[summary.station_code] pga = summary.pgms.loc["PGA", "GREATER_OF_TWO_HORIZONTALS"].tolist()[0] np.testing.assert_almost_equal(cmp_value, pga)
def test_free_field(): data_files, origin = read_data_dir("kiknet", "usp000hzq8") raw_streams = [] for dfile in data_files: raw_streams += read_data(dfile) sc = StreamCollection(raw_streams) processed_streams = process_streams(sc, origin) # all of these streams should have failed for different reasons npassed = np.sum([pstream.passed for pstream in processed_streams]) assert npassed == 0 for pstream in processed_streams: is_free = pstream[0].free_field reason = "" for trace in pstream: if trace.hasParameter("failure"): reason = trace.getParameter("failure")["reason"] break if is_free: assert reason.startswith("Failed") else: assert reason == "Failed free field sensor check."
def test_duration(): ddir = os.path.join("data", "testdata") datadir = pkg_resources.resource_filename("gmprocess", ddir) data_file = os.path.join(datadir, "duration_data.json") with open(data_file, "rt", encoding="utf-8") as f: jdict = json.load(f) time = np.array(jdict["time"]) # input output is m/s/s acc = np.array(jdict["acc"]) / 100 target_d595 = jdict["d595"] delta = time[2] - time[1] sr = 1 / delta header = { "delta": delta, "sampling_rate": sr, "npts": len(acc), "channel": "HN1", "standard": { "corner_frequency": np.nan, "station_name": "", "source": "json", "source_file": "", "instrument": "", "instrument_period": np.nan, "source_format": "json", "comments": "", "structure_type": "", "sensor_serial_number": "", "process_level": "raw counts", "process_time": "", "horizontal_orientation": np.nan, "vertical_orientation": np.nan, "units": "m/s/s", "units_type": "acc", "instrument_sensitivity": np.nan, "volts_to_counts": np.nan, "instrument_damping": np.nan, }, } # input is cm/s/s output is m/s/s trace = StationTrace(data=acc * 100, header=header) trace2 = trace.copy() trace2.stats.channel = "HN2" stream = StationStream([trace, trace2]) for tr in stream: response = {"input_units": "counts", "output_units": "cm/s^2"} tr.setProvenance("remove_response", response) station = StationSummary.from_stream(stream, ["ARITHMETIC_MEAN"], ["duration5-95"]) pgms = station.pgms d595 = pgms.loc["DURATION5-95", "ARITHMETIC_MEAN"].Result np.testing.assert_allclose(d595, target_d595, atol=1e-4, rtol=1e-4) # Test other components data_files, _ = read_data_dir("cwb", "us1000chhc", "2-ECU.dat") stream = read_data(data_files[0])[0] station = StationSummary.from_stream( stream, [ "channels", "gmrotd", "rotd50", "greater_of_two_horizontals", "ARITHMETIC_MEAN", "geometric_mean", ], ["duration5-95"], ) # Currently disallowed assert "gmrotd" not in station.pgms.index.get_level_values(1) assert "rotd50" not in station.pgms.index.get_level_values(1) print(station)
def test_dmg(): file1, _ = read_data_dir("dmg", "nc71734741", files=["CE89146.V2"]) file2, _ = read_data_dir("dmg", "ci15481673", files=["CIWLT.V2"]) file3, _ = read_data_dir("dmg", "nc72282711", files=["CE58667.V2"]) file1 = file1[0] file2 = file2[0] file3 = file3[0] for filename in [file1, file2]: assert is_dmg(file1) # test acceleration from the file stream1 = read_dmg(filename)[0] # test for three traces assert stream1.count() == 3 # test that the traces are acceleration for trace in stream1: assert trace.stats["standard"]["units"] == "acc" # Test metadata stream = read_dmg(file1)[0] for trace in stream: stats = trace.stats assert stats["station"] == "89146" assert stats["delta"] == 0.005000 assert stats["location"] == "--" assert stats["network"] == "--" dt = "%Y-%m-%dT%H:%M:%SZ" assert stats["starttime"].strftime(dt) == "2012-02-13T21:06:45Z" assert stats.coordinates["latitude"] == 40.941 assert stats.coordinates["longitude"] == -123.633 assert stats.standard["station_name"] == "Willow Creek" assert stats.standard["instrument"] == "Etna" assert stats.standard["sensor_serial_number"] == "2500" if stats["channel"] == "H1": assert stats.format_specific["sensor_sensitivity"] == 629 assert stats.standard["horizontal_orientation"] == 360 assert stats.standard["instrument_period"] == 0.0108814 assert stats.standard["instrument_damping"] == 0.6700000 if stats["channel"] == "H2": assert stats.standard["horizontal_orientation"] == 90 assert stats.standard["instrument_period"] == 0.0100000 assert stats.standard["instrument_damping"] == 0.6700000 if stats["channel"] == "Z": assert stats.standard["horizontal_orientation"] == 500 assert stats.standard["instrument_period"] == 0.0102354 assert stats.standard["instrument_damping"] == 0.6700000 assert stats.standard["process_level"] == PROCESS_LEVELS["V2"] assert stats.standard["source_format"] == "dmg" assert stats.standard["source"] == "unknown" assert str(stats.format_specific["time_sd"]) == "nan" assert stats.format_specific["scaling_factor"] == 980.665 assert stats.format_specific["low_filter_corner"] == 0.3 assert stats.format_specific["high_filter_corner"] == 40 stream = read_dmg(file2)[0] for trace in stream: stats = trace.stats assert stats["station"] == "WLT" assert stats["delta"] == 0.0200000 assert stats["location"] == "--" assert stats["network"] == "CI" dt = "%Y-%m-%dT%H:%M:%SZ" assert stats["starttime"].strftime(dt) == "2014-03-29T04:09:34Z" assert stats.coordinates["latitude"] == 34.009 assert stats.coordinates["longitude"] == -117.951 assert stats.standard["station_name"] == "Hacienda Heights" assert stats.standard["instrument"] == "" assert stats.standard["sensor_serial_number"] == "4310" assert ( stats.standard["source"] == "Southern California Seismic " + "Network, California Institute of Technology (Caltech)" ) # test acceleration from the file stream3 = read_dmg(filename)[0] assert len(stream3) == 3 # Test for wrong format exception success = True try: file3, _ = read_data_dir("cwb", "us1000chhc", files=["1-EAS.dat"]) file3 = file3[0] read_dmg(file3)[0] except Exception: success = False assert success == False # Test for bad date in header warning try: file4, _ = read_data_dir("dmg", "nc71734741", files=["BadHeader.V2"]) file4 = file4[0] read_dmg(file4)[0] except BaseException: success = False assert success == False # Test alternate defaults no_stream = """RESPONSE AND FOURIER AMPLITUDE SPECTRA CORRECTED ACCELEROGRAM UNCORRECTED ACCELEROGRAM DATA""" temp_dir = tempfile.mkdtemp() try: tmp = os.path.join(temp_dir, "tfile.txt") with open(tmp, "w", encoding="utf-8") as f: f.write(no_stream) with pytest.raises(BaseException): read_dmg(tmp)[0] except Exception as ex: raise (ex) finally: shutil.rmtree(temp_dir)
def test_cosmos(): one_channel, event = read_data_dir("cosmos", "ci14155260", ["Cosmos12TimeSeriesTest.v1"]) two_channels, _ = read_data_dir("cosmos", "ci14155260", ["Cosmos12TimeSeriesTest2.v1"]) one_channel = one_channel[0] two_channels = two_channels[0] assert is_cosmos(one_channel) assert is_cosmos(os.path.abspath(__file__)) is False # test a one channel cosmos file stream1 = read_cosmos(one_channel)[0] stats = stream1[0].stats assert stats["station"] == "J2236" assert stats["delta"] == 0.01 # was .005 assert stats["location"] == "02" assert stats["network"] == "CE" dt = "%Y-%m-%dT%H:%M:%SZ" assert stats["starttime"].strftime(dt) == "2005-06-16T20:53:04Z" assert stats.coordinates["latitude"] == 34.046 assert stats.coordinates["longitude"] == -117.035 assert stats.coordinates["elevation"] == 15 assert stats.standard["station_name"] == "Yucaipa - Bryant & Oak Glen" assert stats.standard["instrument"] == "Kinemetrics FBA-11 accelerometer" assert stats.standard["sensor_serial_number"] == "1889" dt = "%Y-%m-%dT%H:%M:%SZ" assert stats.standard["process_time"] == "2005-06-17T12:01:00Z" assert stats.format_specific["sensor_sensitivity"] == 220 assert stats.standard["horizontal_orientation"] == 340 assert stats.standard["instrument_period"] == 1.0 / 25 assert stats.standard["instrument_damping"] == 0.20 assert stats.standard["process_level"] == PROCESS_LEVELS["V2"] assert stats.standard["source_format"] == "cosmos" assert stats.standard["structure_type"] == "Building" assert stats.standard["source"] == "California Geological Survey" assert stats.format_specific["scaling_factor"] == 1 assert stats.format_specific["v30"] == 120 assert stats.format_specific["physical_units"] == "cm/s/s" assert stats.format_specific["least_significant_bit"] == 123.45 assert stats.format_specific[ "low_filter_type"] == "Butterworth single direction" assert stats.format_specific["low_filter_corner"] == 4 assert stats.format_specific["low_filter_decay"] == 3 assert stats.format_specific["high_filter_type"] == "Rectangular" assert stats.format_specific["high_filter_corner"] == 40 assert stats.format_specific["high_filter_decay"] == 4 assert stats.format_specific["maximum"] == -161.962 assert stats.format_specific["maximum_time"] == 27.85 assert stats.format_specific["station_code"] == 10 assert stats.format_specific["record_flag"] == "No problem" # test that one channel is created assert len(stream1) == 1 # read the maximum from the text header check that the trace max # is the equivalent when rounded to the same number of decimal places with open(one_channel, "rt", encoding="utf-8") as f: file_line = f.readlines()[10].replace(" ", "").lower() file_max = file_line[file_line.find("max=") + 4:file_line.find("cm")] assert np.round(stream1[0].max(), 3) == float(file_max) # test a two channel cosmos file should fail because deg is not a # converted unit failed = False try: stream2 = read_cosmos(two_channels)[0] except BaseException: failed = True assert failed == True # test that reading a file that is a valid station type returns a # stream with traces building_code = 10 stream3 = read_cosmos(one_channel, valid_station_types=[building_code])[0] assert stream3.count() == 1 # test that reading a file that is not a valid station type returns an # empty stream stream4 = read_cosmos(one_channel, valid_station_types=[1, 2, 3, 4])[0] assert stream4.count() == 0 # test that reading a file that is a valid station type returns a # stream with traces building_code = 10 stream3 = read_cosmos(one_channel, valid_station_types=[building_code])[0] assert stream3.count() == 1 # Test location overrride stream = read_cosmos(one_channel, location="test")[0] assert stream[0].stats.location == "test"
def test(): cwb_file, _ = read_data_dir("cwb", "us1000chhc", files=["1-EAS.dat"]) cwb_file = cwb_file[0] assert is_cwb(cwb_file) assert is_cwb(os.path.abspath(__file__)) is False stream = read_cwb(cwb_file)[0] np.testing.assert_almost_equal(np.abs(stream[0].max()), 0.83699999999999997) assert stream[0].stats["sampling_rate"] == 50 cwb_file, _ = read_data_dir("cwb", "us1000chhc", files=["2-ECU.dat"]) cwb_file = cwb_file[0] assert is_cwb(cwb_file) assert is_cwb(os.path.abspath(__file__)) is False stream = read_cwb(cwb_file)[0] for trace in stream: stats = trace.stats assert stats["station"] == "ECU" assert stats["sampling_rate"] == 50 dt = "%Y-%m-%dT%H:%M:%SZ" assert stats["starttime"].strftime(dt) == "2018-02-06T15:50:29Z" assert stats.standard["station_name"] == "Chulu" assert stats.standard["instrument"] == "FBA" assert stats.coordinates["latitude"] == 22.860 assert stats.coordinates["longitude"] == 121.092 assert stats.format_specific["dc_offset_z"] == -1.017 assert stats.format_specific["dc_offset_h1"] == -2.931 assert stats.format_specific["dc_offset_h2"] == -2.811 defaulted = [ "instrument_period", "instrument_damping", "corner_frequency" ] for default in defaulted: assert str(stats.standard[default]) == "nan" defaulted = [ "comments", "structure_type", "sensor_serial_number", "process_time", ] for default in defaulted: assert stats.standard[default] == "" # Test alternate defaults missing_info = """#Earthquake Information \n#Origin Time(GMT+08): 2018/02/06-23:50:42 \n#EpicenterLongitude(E): 121.69 \n#EpicenterLatitude(N): 24.14 \n#Depth(km): 10.0 \n#Magnitude(Ml): 6.0 \n#Station Information \n#StationCode: ECU \n#StartTime(GMT+08): 2018/02/06-23:50:29.000 \n#RecordLength(sec): 120 \n#SampleRate(Hz): 50 \n#AmplitudeUnit: gal. DCoffset(corr) \n#DataSequence: Time U(+); N(+); E(+) \n#Data: 4F10.3 0.000 0.000 0.000 0.000 0.020 0.000 0.000 0.000 0.040 0.000 0.000 0.000 0.060 0.000 0.000 0.000 0.080 0.000 0.000 0.000 0.100 0.000 0.000 0.000 0.120 0.000 0.000 0.000 """ data = stream[0].data data = np.reshape(data, (int(len(data) / 2), 2), order="C") temp_dir = tempfile.mkdtemp() try: tfile = os.path.join(temp_dir, "tfile.txt") with open(tfile, "w", encoding="utf-8") as f: f.write(missing_info) metadata = _get_header_info(open(tfile, "rt"), data) except Exception as e: raise (e) finally: shutil.rmtree(temp_dir) assert str(metadata["coordinates"]["longitude"]) == "nan" assert str(metadata["coordinates"]["latitude"]) == "nan" assert metadata["standard"]["station_name"] == "" assert metadata["standard"]["instrument"] == "" assert str(metadata["format_specific"]["dc_offset_z"]) == "nan" assert str(metadata["format_specific"]["dc_offset_h2"]) == "nan" assert str(metadata["format_specific"]["dc_offset_h1"]) == "nan"
def test_spectral(): datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A") acc_file = datafiles[0] acc = read_data(acc_file)[0] get_spectral(1.0, acc, 0.05)
def _test_workspace(): eventid = "us1000778i" datafiles, event = read_data_dir("geonet", eventid, "*.V1A") tdir = tempfile.mkdtemp() try: with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=H5pyDeprecationWarning) warnings.filterwarnings("ignore", category=YAMLError) warnings.filterwarnings("ignore", category=FutureWarning) config = update_config( os.path.join(datadir, "config_min_freq_0p2.yml")) tfile = os.path.join(tdir, "test.hdf") raw_streams = [] for dfile in datafiles: raw_streams += read_data(dfile) workspace = StreamWorkspace(tfile) t1 = time.time() workspace.addStreams(event, raw_streams, label="raw") t2 = time.time() print("Adding %i streams took %.2f seconds" % (len(raw_streams), (t2 - t1))) str_repr = workspace.__repr__() assert str_repr == "Events: 1 Stations: 3 Streams: 3" eventobj = workspace.getEvent(eventid) assert eventobj.origins[0].latitude == event.origins[0].latitude assert eventobj.magnitudes[0].mag == event.magnitudes[0].mag stations = workspace.getStations() assert sorted(stations) == ["HSES", "THZ", "WTMC"] stations = workspace.getStations() assert sorted(stations) == ["HSES", "THZ", "WTMC"] # test retrieving event that doesn't exist with pytest.raises(KeyError): workspace.getEvent("foo") instream = None for stream in raw_streams: if stream[0].stats.station.lower() == "hses": instream = stream break if instream is None: raise ValueError("Instream should not be none.") outstream = workspace.getStreams(eventid, stations=["HSES"], labels=["raw"])[0] compare_streams(instream, outstream) label_summary = workspace.summarizeLabels() assert label_summary.iloc[0]["Label"] == "raw" assert label_summary.iloc[0]["Software"] == "gmprocess" sc = StreamCollection(raw_streams) processed_streams = process_streams(sc, event, config=config) workspace.addStreams(event, processed_streams, "processed") idlist = workspace.getEventIds() assert idlist[0] == eventid outstream = workspace.getStreams(eventid, stations=["HSES"], labels=["processed"])[0] provenance = workspace.getProvenance(eventid, labels=["processed"]) first_row = pd.Series({ "Record": "NZ.HSES.--.HN1_us1000778i_processed", "Processing Step": "Remove Response", "Step Attribute": "input_units", "Attribute Value": "counts", }) last_row = pd.Series({ "Record": "NZ.WTMC.--.HNZ_us1000778i_processed", "Processing Step": "Lowpass Filter", "Step Attribute": "number_of_passes", "Attribute Value": 2, }) assert provenance.iloc[0].equals(first_row) assert provenance.iloc[-1].equals(last_row) # compare the parameters from the input processed stream # to it's output equivalent instream = None for stream in processed_streams: if stream[0].stats.station.lower() == "hses": instream = stream break if instream is None: raise ValueError("Instream should not be none.") compare_streams(instream, outstream) workspace.close() # read in data from a second event and stash it in the workspace eventid = "nz2018p115908" datafiles, event = read_data_dir("geonet", eventid, "*.V2A") raw_streams = [] for dfile in datafiles: raw_streams += read_data(dfile) workspace = StreamWorkspace.open(tfile) workspace.addStreams(event, raw_streams, label="foo") stations = workspace.getStations() eventids = workspace.getEventIds() assert eventids == ["us1000778i", "nz2018p115908"] instation = raw_streams[0][0].stats.station this_stream = workspace.getStreams(eventid, stations=[instation], labels=["foo"])[0] assert instation == this_stream[0].stats.station usid = "us1000778i" inventory = workspace.getInventory(usid) workspace.close() codes = [ station.code for station in inventory.networks[0].stations ] assert sorted(set(codes)) == ["HSES", "THZ", "WPWS", "WTMC"] except Exception as e: raise (e) finally: shutil.rmtree(tdir)