def test():
datafiles, origin = read_data_dir("fdsn", "nc72282711", "BK.CMB*.mseed")
streams = []
for datafile in datafiles:
streams += read_obspy(datafile)
assert streams[0].get_id() == "BK.CMB.HN"
datafiles, origin = read_data_dir("fdsn", "nc72282711", "TA.M04C*.mseed")
streams = []
for datafile in datafiles:
streams += read_obspy(datafile)
assert streams[0].get_id() == "TA.M04C.HN"
# test assignment of Z channel
datafiles, origin = read_data_dir("fdsn", "nc73300395", "BK.VALB*.mseed")
streams = []
for datafile in datafiles:
streams += read_obspy(datafile)
# get all channel names
channels = sorted([st[0].stats.channel for st in streams])
assert channels == ["HN2", "HN3", "HNZ"]
def test_num_outliers():
data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed")
data_files.sort()
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
num_outliers = []
for st in sc:
ping_method = Ping(st)
num_outliers.append(ping_method.num_outliers)
np.testing.assert_equal(num_outliers, np.array([239, 26, 0, 6, 133, 145]))
def test_channels():
datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A")
datafile_v2 = datafiles[0]
stream_v2 = read_geonet(datafile_v2)[0]
station_summary = StationSummary.from_stream(stream_v2, ["channels"], ["pga"])
pgms = station_summary.pgms
np.testing.assert_almost_equal(
pgms.loc["PGA", "H2"].Result, 81.28979591836733, decimal=1
)
np.testing.assert_almost_equal(
pgms.loc["PGA", "H1"].Result, 99.3173469387755, decimal=1
)
np.testing.assert_almost_equal(
pgms.loc["PGA", "Z"].Result, 183.89693877551022, decimal=1
)
def test_nnet():
conf = get_config()
update = {
"processing": [
{
"detrend": {
"detrending_method": "demean"
}
},
{
"detrend": {
"detrending_method": "linear"
}
},
{
"compute_snr": {
"bandwidth": 20.0,
"check": {
"max_freq": 5.0,
"min_freq": 0.2,
"threshold": 3.0
},
}
},
{
"NNet_QA": {
"acceptance_threshold": 0.5,
"model_name": "CantWell"
}
},
]
}
update_dict(conf, update)
data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A")
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
test = process_streams(sc, origin, conf)
tstream = test.select(station="HSES")[0]
nnet_dict = tstream.getStreamParam("nnet_qa")
np.testing.assert_allclose(nnet_dict["score_HQ"],
0.99321798811740059,
rtol=1e-3)
def test_metrics():
eventid = "usb000syza"
datafiles, event = read_data_dir("knet", eventid, "*")
datadir = os.path.split(datafiles[0])[0]
raw_streams = StreamCollection.from_directory(datadir)
config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml"))
# turn off sta/lta check and snr checks
# newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr'])
# processed_streams = process_streams(raw_streams, event, config=newconfig)
newconfig = config.copy()
newconfig["processing"].append(
{"NNet_QA": {
"acceptance_threshold": 0.5,
"model_name": "CantWell"
}})
processed_streams = process_streams(raw_streams.copy(),
event,
config=newconfig)
tdir = tempfile.mkdtemp()
try:
tfile = os.path.join(tdir, "test.hdf")
workspace = StreamWorkspace(tfile)
workspace.addEvent(event)
workspace.addStreams(event, raw_streams, label="raw")
workspace.addStreams(event, processed_streams, label="processed")
stream1 = raw_streams[0]
# Get metrics from station summary for raw streams
summary1 = StationSummary.from_config(stream1)
s1_df_in = summary1.pgms.sort_values(["IMT", "IMC"])
array1 = s1_df_in["Result"].to_numpy()
# Compare to metrics from getStreamMetrics for raw streams
workspace.calcMetrics(eventid, labels=["raw"])
summary1_a = workspace.getStreamMetrics(event.id,
stream1[0].stats.network,
stream1[0].stats.station,
"raw")
s1_df_out = summary1_a.pgms.sort_values(["IMT", "IMC"])
array2 = s1_df_out["Result"].to_numpy()
np.testing.assert_allclose(array1, array2, atol=1e-6, rtol=1e-6)
workspace.close()
except Exception as e:
raise (e)
finally:
shutil.rmtree(tdir)
def test_num_clip_intervals():
data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed")
data_files.sort()
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
num_clip_intervals = []
for st in sc:
hist_method = Histogram(st)
num_clip_intervals.append(hist_method.num_clip_intervals)
np.testing.assert_equal(num_clip_intervals, np.array([0, 9, 37, 10, 16,
8]))
def test_sac_csn():
# This reads in example SAC data that does not have a separate metadata
# file to meet the needs of the Community Seismic Network:
# http://csn.caltech.edu/
datafiles, origin = read_data_dir("csn", "ci38457511", "*.sac")
datafiles.sort()
traces = []
for d in datafiles:
traces.append(read_obspy(d)[0][0])
tr_amax = np.zeros(len(traces))
for i, tr in enumerate(traces):
tr_amax[i] = np.max(np.abs(tr.data))
target_amax = np.array([4.3384003e-09, 3.42233e-09, 1.0121747e-07])
np.testing.assert_allclose(target_amax, tr_amax)
def test_num_outliers():
data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed")
data_files.sort()
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
num_outliers = []
for st in sc:
std_dev_method = Std_Dev(st)
num_outliers.append(std_dev_method.num_outliers)
np.testing.assert_equal(num_outliers,
np.array([0, 1086, 131, 1018, 60, 4862]))
def test_check_instrument():
data_files, origin = read_data_dir("fdsn", "nc51194936", "*.mseed")
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
sc.describe()
config = update_config(os.path.join(datadir,
"config_test_check_instr.yml"))
test = process_streams(sc, origin, config=config)
for sta, expected in [("CVS", True), ("GASB", True), ("SBT", False)]:
st = test.select(station=sta)[0]
logging.info(f"Testing stream: {st}")
assert st.passed == expected
def test_end_to_end():
datafiles, _ = read_data_dir("geonet", "us1000778i",
"20161113_110259_WTMC_20.V2A")
datafile = datafiles[0]
stream = read_geonet(datafile)[0]
input_imcs = [
"greater_of_two_horizontals",
"channels",
"rotd50",
"rotd100",
"invalid",
]
input_imts = ["sa1.0", "PGA", "pgv", "invalid"]
m = MetricsController(input_imts, input_imcs, stream, config=config)
test_pgms = [
("PGV", "ROTD(100.0)", 114.24894584734818),
("PGV", "ROTD(50.0)", 81.55436750525355),
("PGV", "Z", 37.47740000000001),
("PGV", "H1", 100.81460000000004),
("PGV", "H2", 68.4354),
("PGV", "GREATER_OF_TWO_HORIZONTALS", 100.81460000000004),
("PGA", "ROTD(100.0)", 100.73875535385548),
("PGA", "ROTD(50.0)", 91.40178541935455),
("PGA", "Z", 183.7722361866693),
("PGA", "H1", 99.24999872535474),
("PGA", "H2", 81.23467239067368),
("PGA", "GREATER_OF_TWO_HORIZONTALS", 99.24999872535474),
("SA(1.000)", "ROTD(100.0)", 146.9023350124098),
("SA(1.000)", "ROTD(50.0)", 106.03202302692158),
("SA(1.000)", "Z", 27.74118995438756),
("SA(1.000)", "H1", 136.25041187387063),
("SA(1.000)", "H2", 84.69296738413021),
("SA(1.000)", "GREATER_OF_TWO_HORIZONTALS", 136.25041187387063),
]
pgms = m.pgms
assert len(pgms) == len(test_pgms)
for target in test_pgms:
target_imt = target[0]
target_imc = target[1]
value = target[2]
df = pgms.loc[target_imt, target_imc]
assert len(df) == 1
np.testing.assert_array_almost_equal(df["Result"], value, decimal=10)
def test_dmg_v1():
file1, _ = read_data_dir("dmg", "ci3144585", files=["LA116TH.RAW"])
file1 = file1[0]
assert is_dmg(file1)
stream1 = read_dmg(file1)[0]
assert stream1.count() == 3
# test that the traces are acceleration
for trace in stream1:
assert trace.stats["standard"]["units"] == "acc"
# test metadata
for trace in stream1:
stats = trace.stats
assert stats["station"] == "14403"
assert stats["delta"] == 0.005
assert stats["location"] == "--"
assert stats["network"] == "--"
dt = "%Y-%m-%dT%H:%M:%SZ"
assert stats["starttime"].strftime(dt) == "1994-01-17T12:31:04Z"
assert stats.coordinates["latitude"] == 33.929
assert stats.coordinates["longitude"] == -118.26
assert stats.standard["station_name"] == "LOS ANGELES - 116TH ST. SCHOOL"
assert stats.standard["instrument"] == "SMA-1"
assert stats.standard["sensor_serial_number"] == "3492"
if stats["channel"] == "HN1":
assert stats.format_specific["sensor_sensitivity"] == 1.915
assert stats.standard["horizontal_orientation"] == 360
assert stats.standard["instrument_period"] == 0.038
assert stats.standard["instrument_damping"] == 0.59
assert stats.format_specific["time_sd"] == 0.115
if stats["channel"] == "HN2":
assert stats.standard["horizontal_orientation"] == 90
assert stats.standard["instrument_period"] == 0.04
assert stats.standard["instrument_damping"] == 0.592
assert stats.format_specific["time_sd"] == 0.12
if stats["channel"] == "HNZ":
assert stats.standard["horizontal_orientation"] == 0.0
assert stats.standard["instrument_period"] == 0.039
assert stats.standard["instrument_damping"] == 0.556
assert stats.format_specific["time_sd"] == 0.114
assert stats.standard["process_level"] == PROCESS_LEVELS["V1"]
assert stats.standard["source_format"] == "dmg"
assert stats.standard["source"] == "unknown"
def test_sa():
datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A")
datafile_v2 = datafiles[0]
stream_v2 = read_geonet(datafile_v2)[0]
sa_target = {}
for trace in stream_v2:
vtrace = trace.copy()
vtrace.integrate()
sa_target[vtrace.stats["channel"]] = np.abs(vtrace.max())
with warnings.catch_warnings():
warnings.simplefilter("ignore")
station_summary = StationSummary.from_stream(
stream_v2,
[
"greater_of_two_horizontals",
"geometric_mean",
"rotd50",
"arithmetic_mean",
"rotd100",
"gmrotd50",
"channels",
],
["sa1.0", "sa0.01", "saincorrect"],
)
pgms = station_summary.pgms
assert "SA(1.000)" in pgms.index.get_level_values(0)
np.testing.assert_allclose(
pgms.loc["SA(1.000)", "ARITHMETIC_MEAN"].Result, 110.47168962900042
)
np.testing.assert_allclose(
pgms.loc["SA(1.000)", "GEOMETRIC_MEAN"].Result, 107.42183990654802
)
np.testing.assert_allclose(
pgms.loc["SA(1.000)", "ROTD(50.0)"].Result, 106.03202302692158
)
np.testing.assert_allclose(
pgms.loc["SA(1.000)", "ROTD(100.0)"].Result, 146.90233501240979
)
# Check high frequency SA
np.testing.assert_allclose(pgms.loc["SA(0.010)", "ROTD(100.0)"].Result, 120.187153)
np.testing.assert_allclose(pgms.loc["SA(0.010)", "GMROTD(50.0)"].Result, 95.355300)
np.testing.assert_allclose(pgms.loc["SA(0.010)", "H1"].Result, 106.716122)
np.testing.assert_allclose(pgms.loc["SA(0.010)", "H2"].Result, 90.497883)
np.testing.assert_allclose(pgms.loc["SA(0.010)", "GMROTD(50.0)"].Result, 95.355300)
def test_to_dataframe():
cwb_files, event = read_data_dir("geonet", "nz2018p115908")
st = read_data(cwb_files[0])[0]
df1 = streams_to_dataframe([st, st], event=event)
np.testing.assert_array_equal(df1.STATION.tolist(), ["WPWS", "WPWS"])
np.testing.assert_array_equal(
df1.NAME.tolist(), ["Waipawa_District_Council", "Waipawa_District_Council"]
)
# let's use sets to make sure all the columns are present in whatever order
cmp1 = set(
[
"ELEVATION",
"EPICENTRAL_DISTANCE",
"GREATER_OF_TWO_HORIZONTALS",
"H1",
"H2",
"HYPOCENTRAL_DISTANCE",
"LAT",
"LON",
"NAME",
"NETID",
"SOURCE",
"STATION",
"Z",
]
)
cmp2 = set(["", "PGA", "PGV", "SA(0.300)", "SA(1.000)", "SA(3.000)"])
header1 = set(df1.columns.levels[0])
header2 = set(df1.columns.levels[1])
assert header1 == cmp1
assert header2 == cmp2
# idx = 0
# for s in df1.columns.levels:
# for col in s:
# try:
# assert col == target_levels[idx]
# except Exception as e:
# x = 1
# idx += 1
# This was previously not being tested
"""imts = ['PGA', 'PGV', 'SA(0.3)', 'SA(1.0)', 'SA(3.0)']
def test_all_num_clip_intervals():
data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed")
data_files.sort()
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
num_clip_intervals = []
for st in sc:
hist_method = Histogram(st, test_all=True)
num_clip_intervals.append(hist_method.num_clip_intervals)
np.testing.assert_equal(
num_clip_intervals,
np.array([[0, 0, 0], [0, 9, 18], [37, 31, 8], [10, 23, 0],
[16, 40, 21], [0, 0, 8]]),
)
def test_auto_fchp():
data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A")
data_files.sort()
streams = []
for f in data_files:
streams += read_data(f)
sc = StreamCollection(streams)
output_fchp = []
config = get_config()
config["integration"]["frequency"] = True
for st in sc:
for tr in st:
tr.setParameter(
"corner_frequencies",
{
"type": "constant",
"highpass": 0.001,
"lowpass": 20
},
)
tmp_st = ridder_fchp(st, config=config)
for tr in tmp_st:
initial_corners = tr.getParameter("corner_frequencies")
output_fchp.append(initial_corners["highpass"])
target_fchp = np.array([
0.021345158261480087,
0.022839239726168643,
0.02482398434993213,
0.01399481102242619,
0.026850167635921275,
0.004817661513765862,
0.008204101694236587,
0.006429246474225982,
0.004237087327289796,
])
np.testing.assert_allclose(output_fchp, target_fchp, atol=1e-7)
def test_acceleration():
datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A")
acc_file = datafiles[0]
acc = read_data(acc_file)[0]
target_g = acc[0].data * GAL_TO_PCTG
target_m = acc[0].data / 100
target_cm = acc[0].data
acc_g = get_acceleration(acc, units="%%g")
assert acc_g[0].stats["units"] == "%%g"
np.testing.assert_allclose(acc_g[0], target_g)
acc_m = get_acceleration(acc, units="m/s/s")
assert acc_m[0].stats["units"] == "m/s/s"
np.testing.assert_allclose(acc_m[0], target_m)
acc_cm = get_acceleration(acc, units="cm/s/s")
assert acc_cm[0].stats["units"] == "cm/s/s"
np.testing.assert_allclose(acc_cm[0], target_cm)
def test_pga():
datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A")
datafile_v2 = datafiles[0]
stream_v2 = read_geonet(datafile_v2)[0]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
station_summary = StationSummary.from_stream(
stream_v2,
[
"channels",
"greater_of_two_horizontals",
"gmrotd0",
"gmrotd50",
"gmrotd100",
"rotd50",
"geometric_mean",
"arithmetic_mean",
],
["pga", "sa1.0", "saincorrect"],
)
pga_df = station_summary.pgms.loc["PGA"]
AM = pga_df.loc["ARITHMETIC_MEAN"].Result
GM = pga_df.loc["GEOMETRIC_MEAN"].Result
HN1 = pga_df.loc["H1"].Result
HN2 = pga_df.loc["H2"].Result
HNZ = pga_df.loc["Z"].Result
gmrotd0 = pga_df.loc["GMROTD(0.0)"].Result
gmrotd50 = pga_df.loc["GMROTD(50.0)"].Result
gmrotd100 = pga_df.loc["GMROTD(100.0)"].Result
rotd50 = pga_df.loc["ROTD(50.0)"].Result
greater = pga_df.loc["GREATER_OF_TWO_HORIZONTALS"].Result
np.testing.assert_allclose(AM, 90.242335558014219)
np.testing.assert_allclose(GM, 89.791654017670112)
np.testing.assert_allclose(HN2, 81.234672390673683)
np.testing.assert_allclose(HN1, 99.249998725354743)
np.testing.assert_almost_equal(HNZ, 183.77223618666929)
np.testing.assert_allclose(greater, 99.249998725354743)
np.testing.assert_allclose(gmrotd0, 83.487703753812113)
np.testing.assert_allclose(gmrotd50, 86.758642638162982)
np.testing.assert_allclose(gmrotd100, 89.791654017670112)
np.testing.assert_allclose(rotd50, 91.401785419354567)
def test_v0():
datafiles, origin = read_data_dir("cosmos", "ftbragg")
dfile = datafiles[0]
# TODO: Fix this problem, or get the data fixed?
assert is_cosmos(dfile)
try:
rstreams = read_cosmos(dfile)
tstream = rstreams[0].copy() # raw stream
streams = StreamCollection(rstreams)
pstream = remove_response(rstreams[0], 0, 0)
pstream.detrend(type="demean")
for trace in tstream:
trace.data /= trace.stats.standard.instrument_sensitivity
trace.data *= 100
tstream.detrend(type="demean")
np.testing.assert_almost_equal(tstream[0].data, pstream[0].data)
except Exception as e:
pass
def test_check_clipping():
data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed")
data_files.sort()
origin = get_event_object("hv70907436")
streams = []
for f in data_files:
streams += read_data(f)
codes = ["HV.TOUO", "HV.MOKD", "HV.MLOD", "HV.HOVE", "HV.HUAD", "HV.HSSD"]
passed = []
for code in codes:
traces = []
for ss in streams:
tcode = f"{ss[0].stats.network}.{ss[0].stats.station}"
if tcode == code:
traces.append(ss[0])
st = StationStream(traces)
check_clipping(st, origin)
passed.append(st.passed)
assert np.all(~np.array(passed))
def _test_stream_params():
eventid = "us1000778i"
datafiles, event = read_data_dir("geonet", eventid,
"20161113_110259_WTMC_20.V1A")
tdir = tempfile.mkdtemp()
streams = []
try:
streams += read_data(datafiles[0])
statsdict = {"name": "Fred", "age": 34}
streams[0].setStreamParam("stats", statsdict)
tfile = os.path.join(tdir, "test.hdf")
workspace = StreamWorkspace(tfile)
workspace.addEvent(event)
workspace.addStreams(event, streams, label="stats")
outstreams = workspace.getStreams(event.id, labels=["stats"])
cmpdict = outstreams[0].getStreamParam("stats")
assert cmpdict == statsdict
workspace.close()
except Exception as e:
raise (e)
finally:
shutil.rmtree(tdir)
def _test_metrics2():
eventid = "usb000syza"
datafiles, event = read_data_dir("knet", eventid, "*")
datadir = os.path.split(datafiles[0])[0]
raw_streams = StreamCollection.from_directory(datadir)
config = update_config(os.path.join(datadir, "config_min_freq_0p2.yml"))
config["metrics"]["output_imts"].append("Arias")
config["metrics"]["output_imcs"].append("arithmetic_mean")
# Adjust checks so that streams pass checks for this test
newconfig = drop_processing(config, ["check_sta_lta"])
csnr = [s for s in newconfig["processing"] if "compute_snr" in s.keys()][0]
csnr["compute_snr"]["check"]["threshold"] = -10.0
processed_streams = process_streams(raw_streams, event, config=newconfig)
tdir = tempfile.mkdtemp()
try:
tfile = os.path.join(tdir, "test.hdf")
workspace = StreamWorkspace(tfile)
workspace.addEvent(event)
workspace.addStreams(event, processed_streams, label="processed")
workspace.calcMetrics(event.id, labels=["processed"])
etable, imc_tables1, readmes1 = workspace.getTables("processed")
assert "ARITHMETIC_MEAN" not in imc_tables1
assert "ARITHMETIC_MEAN" not in readmes1
del workspace.dataset.auxiliary_data.WaveFormMetrics
del workspace.dataset.auxiliary_data.StationMetrics
workspace.calcMetrics(event.id, labels=["processed"], config=config)
etable2, imc_tables2, readmes2 = workspace.getTables("processed")
assert "ARITHMETIC_MEAN" in imc_tables2
assert "ARITHMETIC_MEAN" in readmes2
assert "ARIAS" in imc_tables2["ARITHMETIC_MEAN"]
testarray = readmes2["ARITHMETIC_MEAN"]["Column header"].to_numpy()
assert "ARIAS" in testarray
workspace.close()
except Exception as e:
raise (e)
finally:
shutil.rmtree(tdir)
def test_read():
config = get_config()
cosmos_files, _ = read_data_dir("cosmos", "ci14155260", "Cosmos12TimeSeriesTest.v1")
cwb_files, _ = read_data_dir("cwb", "us1000chhc", "1-EAS.dat")
dmg_files, _ = read_data_dir("dmg", "nc71734741", "CE89146.V2")
geonet_files, _ = read_data_dir(
"geonet", "us1000778i", "20161113_110259_WTMC_20.V1A"
)
knet_files, _ = read_data_dir("knet", "us2000cnnl", "AOM0011801241951.EW")
smc_files, _ = read_data_dir("smc", "nc216859", "0111a.smc")
file_dict = {}
file_dict["cosmos"] = cosmos_files[0]
file_dict["cwb"] = cwb_files[0]
file_dict["dmg"] = dmg_files[0]
file_dict["geonet"] = geonet_files[0]
file_dict["knet"] = knet_files[0]
file_dict["smc"] = smc_files[0]
for file_format in file_dict:
file_path = file_dict[file_format]
assert _get_format(file_path, config) == file_format
assert _validate_format(file_path, config, file_format) == file_format
assert _validate_format(file_dict["knet"], config, "smc") == "knet"
assert _validate_format(file_dict["dmg"], config, "cosmos") == "dmg"
assert _validate_format(file_dict["cosmos"], config, "invalid") == "cosmos"
for file_format in file_dict:
try:
stream = read_data(file_dict[file_format], config, file_format)[0]
except Exception as e:
pass
assert stream[0].stats.standard["source_format"] == file_format
stream = read_data(file_dict[file_format])[0]
assert stream[0].stats.standard["source_format"] == file_format
# test exception
try:
file_path = smc_files[0].replace("0111a.smc", "not_a_file.smc")
read_data(file_path)[0]
success = True
except BaseException:
success = False
assert success == False
def test_bhrc():
datafiles, origin = read_data_dir("bhrc", "usp000jq5p")
# make sure format checker works
assert is_bhrc(datafiles[0])
raw_streams = []
for dfile in datafiles:
raw_streams += read_bhrc(dfile)
peaks = {
"5528": 4.793910,
"5529": 1.024440,
"5522": 1.595120,
"5523": 2.291470,
"5520": 26.189800,
"5526": 1.319720,
}
for stream in raw_streams:
summary = StationSummary.from_config(stream)
cmp_value = peaks[summary.station_code]
pga = summary.pgms.loc["PGA", "GREATER_OF_TWO_HORIZONTALS"].tolist()[0]
np.testing.assert_almost_equal(cmp_value, pga)
def test_free_field():
data_files, origin = read_data_dir("kiknet", "usp000hzq8")
raw_streams = []
for dfile in data_files:
raw_streams += read_data(dfile)
sc = StreamCollection(raw_streams)
processed_streams = process_streams(sc, origin)
# all of these streams should have failed for different reasons
npassed = np.sum([pstream.passed for pstream in processed_streams])
assert npassed == 0
for pstream in processed_streams:
is_free = pstream[0].free_field
reason = ""
for trace in pstream:
if trace.hasParameter("failure"):
reason = trace.getParameter("failure")["reason"]
break
if is_free:
assert reason.startswith("Failed")
else:
assert reason == "Failed free field sensor check."
def test_duration():
ddir = os.path.join("data", "testdata")
datadir = pkg_resources.resource_filename("gmprocess", ddir)
data_file = os.path.join(datadir, "duration_data.json")
with open(data_file, "rt", encoding="utf-8") as f:
jdict = json.load(f)
time = np.array(jdict["time"])
# input output is m/s/s
acc = np.array(jdict["acc"]) / 100
target_d595 = jdict["d595"]
delta = time[2] - time[1]
sr = 1 / delta
header = {
"delta": delta,
"sampling_rate": sr,
"npts": len(acc),
"channel": "HN1",
"standard": {
"corner_frequency": np.nan,
"station_name": "",
"source": "json",
"source_file": "",
"instrument": "",
"instrument_period": np.nan,
"source_format": "json",
"comments": "",
"structure_type": "",
"sensor_serial_number": "",
"process_level": "raw counts",
"process_time": "",
"horizontal_orientation": np.nan,
"vertical_orientation": np.nan,
"units": "m/s/s",
"units_type": "acc",
"instrument_sensitivity": np.nan,
"volts_to_counts": np.nan,
"instrument_damping": np.nan,
},
}
# input is cm/s/s output is m/s/s
trace = StationTrace(data=acc * 100, header=header)
trace2 = trace.copy()
trace2.stats.channel = "HN2"
stream = StationStream([trace, trace2])
for tr in stream:
response = {"input_units": "counts", "output_units": "cm/s^2"}
tr.setProvenance("remove_response", response)
station = StationSummary.from_stream(stream, ["ARITHMETIC_MEAN"],
["duration5-95"])
pgms = station.pgms
d595 = pgms.loc["DURATION5-95", "ARITHMETIC_MEAN"].Result
np.testing.assert_allclose(d595, target_d595, atol=1e-4, rtol=1e-4)
# Test other components
data_files, _ = read_data_dir("cwb", "us1000chhc", "2-ECU.dat")
stream = read_data(data_files[0])[0]
station = StationSummary.from_stream(
stream,
[
"channels",
"gmrotd",
"rotd50",
"greater_of_two_horizontals",
"ARITHMETIC_MEAN",
"geometric_mean",
],
["duration5-95"],
)
# Currently disallowed
assert "gmrotd" not in station.pgms.index.get_level_values(1)
assert "rotd50" not in station.pgms.index.get_level_values(1)
print(station)
def test_dmg():
file1, _ = read_data_dir("dmg", "nc71734741", files=["CE89146.V2"])
file2, _ = read_data_dir("dmg", "ci15481673", files=["CIWLT.V2"])
file3, _ = read_data_dir("dmg", "nc72282711", files=["CE58667.V2"])
file1 = file1[0]
file2 = file2[0]
file3 = file3[0]
for filename in [file1, file2]:
assert is_dmg(file1)
# test acceleration from the file
stream1 = read_dmg(filename)[0]
# test for three traces
assert stream1.count() == 3
# test that the traces are acceleration
for trace in stream1:
assert trace.stats["standard"]["units"] == "acc"
# Test metadata
stream = read_dmg(file1)[0]
for trace in stream:
stats = trace.stats
assert stats["station"] == "89146"
assert stats["delta"] == 0.005000
assert stats["location"] == "--"
assert stats["network"] == "--"
dt = "%Y-%m-%dT%H:%M:%SZ"
assert stats["starttime"].strftime(dt) == "2012-02-13T21:06:45Z"
assert stats.coordinates["latitude"] == 40.941
assert stats.coordinates["longitude"] == -123.633
assert stats.standard["station_name"] == "Willow Creek"
assert stats.standard["instrument"] == "Etna"
assert stats.standard["sensor_serial_number"] == "2500"
if stats["channel"] == "H1":
assert stats.format_specific["sensor_sensitivity"] == 629
assert stats.standard["horizontal_orientation"] == 360
assert stats.standard["instrument_period"] == 0.0108814
assert stats.standard["instrument_damping"] == 0.6700000
if stats["channel"] == "H2":
assert stats.standard["horizontal_orientation"] == 90
assert stats.standard["instrument_period"] == 0.0100000
assert stats.standard["instrument_damping"] == 0.6700000
if stats["channel"] == "Z":
assert stats.standard["horizontal_orientation"] == 500
assert stats.standard["instrument_period"] == 0.0102354
assert stats.standard["instrument_damping"] == 0.6700000
assert stats.standard["process_level"] == PROCESS_LEVELS["V2"]
assert stats.standard["source_format"] == "dmg"
assert stats.standard["source"] == "unknown"
assert str(stats.format_specific["time_sd"]) == "nan"
assert stats.format_specific["scaling_factor"] == 980.665
assert stats.format_specific["low_filter_corner"] == 0.3
assert stats.format_specific["high_filter_corner"] == 40
stream = read_dmg(file2)[0]
for trace in stream:
stats = trace.stats
assert stats["station"] == "WLT"
assert stats["delta"] == 0.0200000
assert stats["location"] == "--"
assert stats["network"] == "CI"
dt = "%Y-%m-%dT%H:%M:%SZ"
assert stats["starttime"].strftime(dt) == "2014-03-29T04:09:34Z"
assert stats.coordinates["latitude"] == 34.009
assert stats.coordinates["longitude"] == -117.951
assert stats.standard["station_name"] == "Hacienda Heights"
assert stats.standard["instrument"] == ""
assert stats.standard["sensor_serial_number"] == "4310"
assert (
stats.standard["source"]
== "Southern California Seismic "
+ "Network, California Institute of Technology (Caltech)"
)
# test acceleration from the file
stream3 = read_dmg(filename)[0]
assert len(stream3) == 3
# Test for wrong format exception
success = True
try:
file3, _ = read_data_dir("cwb", "us1000chhc", files=["1-EAS.dat"])
file3 = file3[0]
read_dmg(file3)[0]
except Exception:
success = False
assert success == False
# Test for bad date in header warning
try:
file4, _ = read_data_dir("dmg", "nc71734741", files=["BadHeader.V2"])
file4 = file4[0]
read_dmg(file4)[0]
except BaseException:
success = False
assert success == False
# Test alternate defaults
no_stream = """RESPONSE AND FOURIER AMPLITUDE SPECTRA
CORRECTED ACCELEROGRAM
UNCORRECTED ACCELEROGRAM DATA"""
temp_dir = tempfile.mkdtemp()
try:
tmp = os.path.join(temp_dir, "tfile.txt")
with open(tmp, "w", encoding="utf-8") as f:
f.write(no_stream)
with pytest.raises(BaseException):
read_dmg(tmp)[0]
except Exception as ex:
raise (ex)
finally:
shutil.rmtree(temp_dir)
def test_cosmos():
one_channel, event = read_data_dir("cosmos", "ci14155260",
["Cosmos12TimeSeriesTest.v1"])
two_channels, _ = read_data_dir("cosmos", "ci14155260",
["Cosmos12TimeSeriesTest2.v1"])
one_channel = one_channel[0]
two_channels = two_channels[0]
assert is_cosmos(one_channel)
assert is_cosmos(os.path.abspath(__file__)) is False
# test a one channel cosmos file
stream1 = read_cosmos(one_channel)[0]
stats = stream1[0].stats
assert stats["station"] == "J2236"
assert stats["delta"] == 0.01 # was .005
assert stats["location"] == "02"
assert stats["network"] == "CE"
dt = "%Y-%m-%dT%H:%M:%SZ"
assert stats["starttime"].strftime(dt) == "2005-06-16T20:53:04Z"
assert stats.coordinates["latitude"] == 34.046
assert stats.coordinates["longitude"] == -117.035
assert stats.coordinates["elevation"] == 15
assert stats.standard["station_name"] == "Yucaipa - Bryant & Oak Glen"
assert stats.standard["instrument"] == "Kinemetrics FBA-11 accelerometer"
assert stats.standard["sensor_serial_number"] == "1889"
dt = "%Y-%m-%dT%H:%M:%SZ"
assert stats.standard["process_time"] == "2005-06-17T12:01:00Z"
assert stats.format_specific["sensor_sensitivity"] == 220
assert stats.standard["horizontal_orientation"] == 340
assert stats.standard["instrument_period"] == 1.0 / 25
assert stats.standard["instrument_damping"] == 0.20
assert stats.standard["process_level"] == PROCESS_LEVELS["V2"]
assert stats.standard["source_format"] == "cosmos"
assert stats.standard["structure_type"] == "Building"
assert stats.standard["source"] == "California Geological Survey"
assert stats.format_specific["scaling_factor"] == 1
assert stats.format_specific["v30"] == 120
assert stats.format_specific["physical_units"] == "cm/s/s"
assert stats.format_specific["least_significant_bit"] == 123.45
assert stats.format_specific[
"low_filter_type"] == "Butterworth single direction"
assert stats.format_specific["low_filter_corner"] == 4
assert stats.format_specific["low_filter_decay"] == 3
assert stats.format_specific["high_filter_type"] == "Rectangular"
assert stats.format_specific["high_filter_corner"] == 40
assert stats.format_specific["high_filter_decay"] == 4
assert stats.format_specific["maximum"] == -161.962
assert stats.format_specific["maximum_time"] == 27.85
assert stats.format_specific["station_code"] == 10
assert stats.format_specific["record_flag"] == "No problem"
# test that one channel is created
assert len(stream1) == 1
# read the maximum from the text header check that the trace max
# is the equivalent when rounded to the same number of decimal places
with open(one_channel, "rt", encoding="utf-8") as f:
file_line = f.readlines()[10].replace(" ", "").lower()
file_max = file_line[file_line.find("max=") + 4:file_line.find("cm")]
assert np.round(stream1[0].max(), 3) == float(file_max)
# test a two channel cosmos file should fail because deg is not a
# converted unit
failed = False
try:
stream2 = read_cosmos(two_channels)[0]
except BaseException:
failed = True
assert failed == True
# test that reading a file that is a valid station type returns a
# stream with traces
building_code = 10
stream3 = read_cosmos(one_channel, valid_station_types=[building_code])[0]
assert stream3.count() == 1
# test that reading a file that is not a valid station type returns an
# empty stream
stream4 = read_cosmos(one_channel, valid_station_types=[1, 2, 3, 4])[0]
assert stream4.count() == 0
# test that reading a file that is a valid station type returns a
# stream with traces
building_code = 10
stream3 = read_cosmos(one_channel, valid_station_types=[building_code])[0]
assert stream3.count() == 1
# Test location overrride
stream = read_cosmos(one_channel, location="test")[0]
assert stream[0].stats.location == "test"
def test():
cwb_file, _ = read_data_dir("cwb", "us1000chhc", files=["1-EAS.dat"])
cwb_file = cwb_file[0]
assert is_cwb(cwb_file)
assert is_cwb(os.path.abspath(__file__)) is False
stream = read_cwb(cwb_file)[0]
np.testing.assert_almost_equal(np.abs(stream[0].max()),
0.83699999999999997)
assert stream[0].stats["sampling_rate"] == 50
cwb_file, _ = read_data_dir("cwb", "us1000chhc", files=["2-ECU.dat"])
cwb_file = cwb_file[0]
assert is_cwb(cwb_file)
assert is_cwb(os.path.abspath(__file__)) is False
stream = read_cwb(cwb_file)[0]
for trace in stream:
stats = trace.stats
assert stats["station"] == "ECU"
assert stats["sampling_rate"] == 50
dt = "%Y-%m-%dT%H:%M:%SZ"
assert stats["starttime"].strftime(dt) == "2018-02-06T15:50:29Z"
assert stats.standard["station_name"] == "Chulu"
assert stats.standard["instrument"] == "FBA"
assert stats.coordinates["latitude"] == 22.860
assert stats.coordinates["longitude"] == 121.092
assert stats.format_specific["dc_offset_z"] == -1.017
assert stats.format_specific["dc_offset_h1"] == -2.931
assert stats.format_specific["dc_offset_h2"] == -2.811
defaulted = [
"instrument_period", "instrument_damping", "corner_frequency"
]
for default in defaulted:
assert str(stats.standard[default]) == "nan"
defaulted = [
"comments",
"structure_type",
"sensor_serial_number",
"process_time",
]
for default in defaulted:
assert stats.standard[default] == ""
# Test alternate defaults
missing_info = """#Earthquake Information
\n#Origin Time(GMT+08): 2018/02/06-23:50:42
\n#EpicenterLongitude(E): 121.69
\n#EpicenterLatitude(N): 24.14
\n#Depth(km): 10.0
\n#Magnitude(Ml): 6.0
\n#Station Information
\n#StationCode: ECU
\n#StartTime(GMT+08): 2018/02/06-23:50:29.000
\n#RecordLength(sec): 120
\n#SampleRate(Hz): 50
\n#AmplitudeUnit: gal. DCoffset(corr)
\n#DataSequence: Time U(+); N(+); E(+)
\n#Data: 4F10.3
0.000 0.000 0.000 0.000
0.020 0.000 0.000 0.000
0.040 0.000 0.000 0.000
0.060 0.000 0.000 0.000
0.080 0.000 0.000 0.000
0.100 0.000 0.000 0.000
0.120 0.000 0.000 0.000
"""
data = stream[0].data
data = np.reshape(data, (int(len(data) / 2), 2), order="C")
temp_dir = tempfile.mkdtemp()
try:
tfile = os.path.join(temp_dir, "tfile.txt")
with open(tfile, "w", encoding="utf-8") as f:
f.write(missing_info)
metadata = _get_header_info(open(tfile, "rt"), data)
except Exception as e:
raise (e)
finally:
shutil.rmtree(temp_dir)
assert str(metadata["coordinates"]["longitude"]) == "nan"
assert str(metadata["coordinates"]["latitude"]) == "nan"
assert metadata["standard"]["station_name"] == ""
assert metadata["standard"]["instrument"] == ""
assert str(metadata["format_specific"]["dc_offset_z"]) == "nan"
assert str(metadata["format_specific"]["dc_offset_h2"]) == "nan"
assert str(metadata["format_specific"]["dc_offset_h1"]) == "nan"
def test_spectral():
datafiles, _ = read_data_dir("geonet", "us1000778i", "20161113_110259_WTMC_20.V2A")
acc_file = datafiles[0]
acc = read_data(acc_file)[0]
get_spectral(1.0, acc, 0.05)
def _test_workspace():
eventid = "us1000778i"
datafiles, event = read_data_dir("geonet", eventid, "*.V1A")
tdir = tempfile.mkdtemp()
try:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=H5pyDeprecationWarning)
warnings.filterwarnings("ignore", category=YAMLError)
warnings.filterwarnings("ignore", category=FutureWarning)
config = update_config(
os.path.join(datadir, "config_min_freq_0p2.yml"))
tfile = os.path.join(tdir, "test.hdf")
raw_streams = []
for dfile in datafiles:
raw_streams += read_data(dfile)
workspace = StreamWorkspace(tfile)
t1 = time.time()
workspace.addStreams(event, raw_streams, label="raw")
t2 = time.time()
print("Adding %i streams took %.2f seconds" % (len(raw_streams),
(t2 - t1)))
str_repr = workspace.__repr__()
assert str_repr == "Events: 1 Stations: 3 Streams: 3"
eventobj = workspace.getEvent(eventid)
assert eventobj.origins[0].latitude == event.origins[0].latitude
assert eventobj.magnitudes[0].mag == event.magnitudes[0].mag
stations = workspace.getStations()
assert sorted(stations) == ["HSES", "THZ", "WTMC"]
stations = workspace.getStations()
assert sorted(stations) == ["HSES", "THZ", "WTMC"]
# test retrieving event that doesn't exist
with pytest.raises(KeyError):
workspace.getEvent("foo")
instream = None
for stream in raw_streams:
if stream[0].stats.station.lower() == "hses":
instream = stream
break
if instream is None:
raise ValueError("Instream should not be none.")
outstream = workspace.getStreams(eventid,
stations=["HSES"],
labels=["raw"])[0]
compare_streams(instream, outstream)
label_summary = workspace.summarizeLabels()
assert label_summary.iloc[0]["Label"] == "raw"
assert label_summary.iloc[0]["Software"] == "gmprocess"
sc = StreamCollection(raw_streams)
processed_streams = process_streams(sc, event, config=config)
workspace.addStreams(event, processed_streams, "processed")
idlist = workspace.getEventIds()
assert idlist[0] == eventid
outstream = workspace.getStreams(eventid,
stations=["HSES"],
labels=["processed"])[0]
provenance = workspace.getProvenance(eventid, labels=["processed"])
first_row = pd.Series({
"Record": "NZ.HSES.--.HN1_us1000778i_processed",
"Processing Step": "Remove Response",
"Step Attribute": "input_units",
"Attribute Value": "counts",
})
last_row = pd.Series({
"Record": "NZ.WTMC.--.HNZ_us1000778i_processed",
"Processing Step": "Lowpass Filter",
"Step Attribute": "number_of_passes",
"Attribute Value": 2,
})
assert provenance.iloc[0].equals(first_row)
assert provenance.iloc[-1].equals(last_row)
# compare the parameters from the input processed stream
# to it's output equivalent
instream = None
for stream in processed_streams:
if stream[0].stats.station.lower() == "hses":
instream = stream
break
if instream is None:
raise ValueError("Instream should not be none.")
compare_streams(instream, outstream)
workspace.close()
# read in data from a second event and stash it in the workspace
eventid = "nz2018p115908"
datafiles, event = read_data_dir("geonet", eventid, "*.V2A")
raw_streams = []
for dfile in datafiles:
raw_streams += read_data(dfile)
workspace = StreamWorkspace.open(tfile)
workspace.addStreams(event, raw_streams, label="foo")
stations = workspace.getStations()
eventids = workspace.getEventIds()
assert eventids == ["us1000778i", "nz2018p115908"]
instation = raw_streams[0][0].stats.station
this_stream = workspace.getStreams(eventid,
stations=[instation],
labels=["foo"])[0]
assert instation == this_stream[0].stats.station
usid = "us1000778i"
inventory = workspace.getInventory(usid)
workspace.close()
codes = [
station.code for station in inventory.networks[0].stations
]
assert sorted(set(codes)) == ["HSES", "THZ", "WPWS", "WTMC"]
except Exception as e:
raise (e)
finally:
shutil.rmtree(tdir)
