def df_with_nrl_response(self):
"""
Add NRL response information to the dataframe.
Note: The datalogger is probably not correct for the test station.
It is just to ensure the NRL can be used to get a response.
"""
_inv = obsplus.load_dataset(
"bingham_test").station_client.get_stations()
inv = _inv.select(station="NOQ")
with suppress_warnings():
df = obsplus.stations_to_df(inv)
# set instrument str
sensor_keys = ("Nanometrics", "Trillium 120 Horizon",
"Trillium 120 Horizon")
# get digitizer keys
datalogger_keys = (
"Nanometrics",
"Centaur",
"1 Vpp (40)",
"Off",
"Linear phase",
"100",
)
# keep one as a tuple and convert the other to str
df["sensor_keys"] = [sensor_keys for _ in range(len(df))]
df["datalogger_keys"] = json.dumps(datalogger_keys)
# drop seed id
df = df.drop(columns="seed_id")
return df
def test_radius_degrees(self, catalog):
"""ensure the max_radius works with degrees specified."""
lat, lon = 39.342, 41.044
with suppress_warnings():
cat = catalog.get_events(latitude=lat, longitude=lon, maxradius=8)
# For the example catalog there should be exactly 2 events included
assert len(cat) == 2
def test_cat_cat(self, cat, spatial_calc):
"""ensure it works with two catalogs"""
with suppress_warnings():
df = spatial_calc(cat, cat)
event_ids = {str(x.resource_id) for x in cat}
combinations = set(itertools.product(event_ids, event_ids))
assert combinations == set(df.index)
def set_stations(self, stations: fetcher_station_type):
"""
Set the station state in fetcher.
Parameters
----------
stations
Data representing stations, from which a client or dataframe
can be inferred.
"""
try:
self.station_client = get_station_client(stations)
except TypeError:
self.station_client = getattr(self, "station_client", None)
try:
# since its common for inventories to have far out enddates this
# can raise a warning. These are safe to ignore.
with suppress_warnings(category=TimeOverflowWarning):
self.station_df = stations_to_df(stations)
except TypeError:
# if unable to get station info from stations use waveform client
try:
self.station_df = stations_to_df(self.waveform_client)
except TypeError:
# if no waveforms try events
try:
self.station_df = stations_to_df(self.event_client)
except TypeError:
self.station_df = None
# make sure seed_id is set
if self.station_df is not None:
self.station_df["seed_id"] = get_seed_id_series(self.station_df)
def _write_update(self, df: pd.DataFrame, update_time=None):
""" convert updates to dataframe, then append to index table """
# read in dataframe and cast to correct types
assert not df.duplicated().any(), "update index has duplicate entries"
# set both dfs to use index of event_id
df = df.set_index("event_id")
# get current events, but dont allow it to update again
current = self.read_index(event_id=set(df.index), _allow_update=False)
indicies_to_update = set(current["event_id"]) & set(df.index)
# populate index store and update metadata
with sql_connection(self.index_path) as con:
if indicies_to_update: # delete rows that will be re-entered
_drop_rows(self._index_node, con, event_id=indicies_to_update)
node = self._index_node
df.to_sql(node, con, if_exists="append", index_label="event_id")
tables = _get_tables(con)
if self._meta_node not in tables:
meta = self._make_meta_table()
meta.to_sql(self._meta_node, con, if_exists="replace")
# update timestamp
with suppress_warnings(): # ignore pandas collection warning
if self.allow_update_timestamp:
timestamp = update_time or time.time()
dft = pd.DataFrame(timestamp, index=[0], columns=["time"])
dft.to_sql(self._time_node,
con,
if_exists="replace",
index=False)
self._metadata = meta
self._index = None
def test_list_of_tuples(self, spatial_calc):
"""Test a list of tuples."""
input1 = [(45, -111, 0), (46, -111, 0), (49, -112, 0)]
input2 = obspy.read_events()
with suppress_warnings():
out = spatial_calc(input1, input2)
assert len(out) == 9
assert not out.isnull().any().any()
def test_read_inventory_directories(self, read_inventory, inventory):
"""Tests for reading a directory of inventories."""
with suppress_warnings():
inv_df = stations_to_df(inventory)
assert (read_inventory.columns == inv_df.columns).all()
assert not read_inventory.empty
assert len(inv_df) == len(read_inventory)
assert set(inv_df["seed_id"]) == set(read_inventory["seed_id"])
def test_response_one_missing(self, df_with_partial_responses):
"""Ensure responses which can be got are fetched."""
df = df_with_partial_responses
with suppress_warnings():
inv = df_to_inventory(df)
missing = df["sensor_keys"].isnull() | df["datalogger_keys"].isnull()
missing_seed_ids = set(get_seed_id_series(df[missing]))
assert self.has_valid_response(inv, missing_seed_ids)
def test_get_stations_client(self, df_with_get_stations_kwargs):
"""Ensure get_station_kwargs results responses."""
df = df_with_get_stations_kwargs
col = "get_station_kwargs"
missing = df[col].isnull() | (df[col] == "")
missing_seed_ids = set(get_seed_id_series(df[missing]))
with suppress_warnings():
inv = df_to_inventory(df)
assert self.has_valid_response(inv, missing_seed_ids)
def test_init_with_banks(self, bingham_dataset):
"""Ensure the fetcher can be init'ed with all bank inputs."""
wbank = obsplus.WaveBank(bingham_dataset.waveform_path).update_index()
ebank = obsplus.EventBank(bingham_dataset.event_path).update_index()
sbank = bingham_dataset.station_client
# ignore warnings (endtimes of inv are out of range)
with suppress_warnings():
fetcher = Fetcher(waveforms=wbank, events=ebank, stations=sbank)
edf = fetcher.event_df
sdf = fetcher.station_df
for df in [edf, sdf]:
assert isinstance(df, pd.DataFrame)
assert not df.empty
def test_dataframe_input(self, cat, spatial_calc):
"""
Any dataframe should be valid input provided it has the required columns.
"""
data = [[10.1, 10.1, 0, "some_id"]]
cols = ["latitude", "longitude", "elevation", "id"]
df = pd.DataFrame(data, columns=cols).set_index("id")
with suppress_warnings():
dist_df = spatial_calc(df, cat)
# make sure output has expected shape and ids
assert len(dist_df) == len(cat) * len(df)
id1 = dist_df.index.get_level_values("id1")
assert "some_id" in set(id1)
def ebank_low_version(self, tmpdir, monkeypatch):
""" return the default bank with a negative version number. """
# monkey patch obsplus version so that a low version is saved to disk
monkeypatch.setattr(obsplus, "__last_version__", self.low_version_str)
cat = obspy.read_events()
ebank = EventBank(tmpdir).put_events(cat, update_index=False)
# write index with negative version
with suppress_warnings():
ebank.update_index()
monkeypatch.undo()
assert ebank._index_version == self.low_version_str
assert obsplus.__last_version__ != self.low_version_str
return ebank
def _generic_op_test(self, wf1, number, op):
"""Apply generic operation tests."""
# div 0 raises runtime warning, this is ok.
with suppress_warnings():
wf2 = op(wf1, number)
data1 = op(wf1.data.values, number)
assert isinstance(wf2, WaveFrame)
assert wf2 is not wf1
data2 = wf2.data.values
close = np.isclose(data1, data2)
not_finite = ~np.isfinite(data1)
assert np.all(close | not_finite)
def test_with_tuple_no_id(self, spatial_calc):
"""Test getting relations with tuple and catalog."""
cat = obspy.read_events()
df = obsplus.events_to_df(cat)
ser = df.iloc[0]
# first test with no id
tuple1 = (ser["latitude"], ser["longitude"], -ser["depth"])
with suppress_warnings():
out1 = spatial_calc(cat, tuple1)
# the default index should be sequential
assert set(out1.index.get_level_values("id2")) == {0}
# expected len is 3
assert len(out1) == 3
def test_query_circular(self, bing_ebank):
"""Test circular queries in bank."""
latitude, longitude, minradius, maxradius = (40.5, -112.12, 0.035,
0.05)
with suppress_warnings(): # suppress install geographiclib warning
df = bing_ebank.read_index(
latitude=latitude,
longitude=longitude,
maxradius=minradius,
minradius=maxradius,
)
for lat, lon in zip(df["latitude"], df["longitude"]):
dist, _, _ = gps2dist_azimuth(latitude, longitude, lat, lon)
assert minradius <= kilometer2degrees(dist / 1000.0) <= maxradius
def test_raise_on_missing_waveform(self, fetcher_missing_events,
monkeypatch):
"""Ensure yield waveform raises on missing event when specified."""
def _func(*args, **kwargs):
raise ValueError("Something went wrong!")
fet = fetcher_missing_events
monkeypatch.setattr(fet.waveform_client, "get_waveforms_bulk", _func)
# this should raise
with pytest.raises(ValueError):
list(fet.yield_event_waveforms(1, 10, raise_on_fail=True))
# this should not, it should just return an empty list
with suppress_warnings(UserWarning):
out = list(fet.yield_event_waveforms(1, 10, raise_on_fail=False))
assert out == []
def test_tuple_with_id(self, spatial_calc):
"""Ensure if a 4th column is given the id works."""
tuple1 = (45, -111, 0, "bob")
tuple2 = (45, -111, 0)
with suppress_warnings():
out = spatial_calc(tuple1, tuple2)
# distances should be close to 0
dist_cols = ["distance_m", "distance_degrees", "vertical_distance_m"]
distances = out[dist_cols].values
assert np.allclose(distances, 0)
# azimuths should be increments of 180
assert np.allclose(out["azimuth"] % 180, 0)
assert np.allclose(out["back_azimuth"] % 180, 0)
# check length and index names
assert len(out) == 1
assert set(out.index.to_list()) == {("bob", 0)}
def test_from_path(self, tmpdir):
"""Put events should work with a path to a directory of events."""
cat = obspy.read_events()
event_path = Path(tmpdir) / "events.xml"
bank1 = EventBank(event_path.parent / "catalog_dir1")
bank2 = EventBank(event_path.parent / "catalog_dir2")
# a slightly invalid uri is used, just ignore
with suppress_warnings():
cat.write(str(event_path), "quakeml")
# test works with a Path instance
bank1.put_events(event_path)
assert Path(bank1.bank_path).exists()
assert not bank1.read_index().empty
# tests with a string
bank2.put_events(str(event_path))
assert Path(bank2.bank_path).exists()
assert not bank2.read_index().empty
def test_query_circular(self, bing_ebank, bingham_catalog):
""" The bank query should return the same as get_events on catalog. """
latitude, longitude, minradius, maxradius = (40.5, -112.12, 0.035,
0.05)
with suppress_warnings():
cat1 = bing_ebank.get_events(
latitude=latitude,
longitude=longitude,
maxradius=minradius,
minradius=maxradius,
)
cat2 = bingham_catalog.get_events(
latitude=latitude,
longitude=longitude,
maxradius=minradius,
minradius=maxradius,
)
assert cat1 == cat2
def test_max_min_radius_m(self, catalog):
"""Ensure max and min radius work in m (ie when degrees=False)."""
minrad = 10000
maxrad = 800_000
lat, lon = 39.342, 41.044
kwargs = dict(
latitude=lat,
longitude=lon,
minradius=minrad,
maxradius=maxrad,
degrees=False,
)
with suppress_warnings(): # suppress geograhiclib warning
df = catalog.get_events(**kwargs).get_event_summary()
for _, row in df.iterrows():
args = (row.latitude, row.longitude, lat, lon)
dist, _, _ = gps2dist_azimuth(*args)
assert minrad < dist < maxrad
def df_with_get_stations_kwargs(self):
"""
Add response information to the dataframe using get_stations_kwargs.
Add an additional station which will need to get all data from other
columns.
"""
_inv = obsplus.load_dataset(
"bingham_test").station_client.get_stations()
inv = _inv.select(station="NOQ")
with suppress_warnings():
df = obsplus.stations_to_df(inv).reset_index()
# set get_station_kwargs for last two channels, leave first empty
kwargs_list = [""]
for ind, row in df.iloc[1:].iterrows():
kwargs = {x: row[x] for x in NSLC}
kwargs["endafter"] = str(to_utc(row["start_date"]))
kwargs_list.append(kwargs)
# set last kwargs to str to simulate reading from csv
kwargs_list[-1] = str(kwargs_list[-1])
df["get_station_kwargs"] = kwargs_list
# set the first kwargs to a string to make sure it can be parsed
# this is important for eg reading data from a csv.
df.loc[0, "get_station_kwargs"] = str(df.loc[0, "get_station_kwargs"])
# now add a row with an empty get_station_kwargs column
old = dict(df.iloc[0])
new = {
"station": "CWU",
"network": "UU",
"channel": "EHZ",
"location": "01",
"seed_id": "UU.CWU.01.EHZ",
"get_station_kwargs": "{}",
}
old.update(new)
ser = pd.Series(old)
return df.append(ser, ignore_index=True)
def data_fetcher(self, request):
"""Return a datafetcher from all datasets."""
with suppress_warnings(UserWarning):
return obsplus.load_dataset(request.param).get_fetcher()
def distance_df(self, cat, inv, spatial_calc):
"""Return a dataframe from all the crandall_test events and stations."""
with suppress_warnings():
return spatial_calc(entity_1=cat, entity_2=inv)
def test_nrl_responses(self, df_with_nrl_response):
"""Ensure the NRL is used to pull responses."""
with suppress_warnings():
inv = df_to_inventory(df_with_nrl_response)
assert self.has_valid_response(inv)
def test_circular_search_no_events(self, catalog):
"""Ensure no errors if no events within rectangular region. #177"""
lat, lon = 31.0, 30.0
with suppress_warnings():
cat = catalog.get_events(latitude=lat, longitude=lon, maxradius=1)
assert len(cat) == 0
def _func(*args, **kwargs):
with suppress_warnings():
return old_func(*args, **kwargs)
def read_inventory(self, inv_directory):
"""Convert the inventory directory to a dataframe."""
with suppress_warnings():
return stations_to_df(inv_directory)
