def _download_crandall(self):
"""download waveform/station info for dataset."""
bank = WaveBank(self.waveform_path)
domain = CircularDomain(
self.latitude,
self.longitude,
minradius=0,
maxradius=kilometers2degrees(self.max_dist),
)
cat = obspy.read_events(str(self.source_path / "events.xml"))
df = events_to_df(cat)
for _, row in df.iterrows():
starttime = row.time - self.time_before
endtime = row.time + self.time_after
restrictions = Restrictions(
starttime=UTC(starttime),
endtime=UTC(endtime),
minimum_length=0.90,
minimum_interstation_distance_in_m=100,
channel_priorities=["HH[ZNE]", "BH[ZNE]"],
location_priorities=["", "00", "01", "--"],
)
kwargs = dict(
domain=domain,
restrictions=restrictions,
mseed_storage=str(self.waveform_path),
stationxml_storage=str(self.station_path),
)
MassDownloader(providers=[self._download_client]).download(
**kwargs)
# ensure data have actually been downloaded
bank.update_index()
assert not bank.read_index(starttime=starttime,
endtime=endtime).empty
def __init__(self, waveform_client: obsplus.WaveBank):
assert hasattr(waveform_client, "read_index")
# dicts that will be used as caches for processed/unprocessed streams
self._stream_cache = {}
self._processed_cache = {}
# list of callables that should take a single waveforms and return
self._stream_processors = defaultdict(list)
# load all of the bank index into memory
self.bank = waveform_client
self.index = waveform_client.read_index()
# make a column of callables for getting waveforms
self.index["st_call"] = self.index.apply(self._make_st_calls, axis=1)
# get a column that identifies rows that have the same resource
self.index.rename(columns={"path": "unique_key"}, inplace=True)
class WavebankWrapper(AnxcorDatabase):
def __init__(self, directory):
super().__init__()
self.bank = WaveBank(
directory, name_structure='{network}.{station}.{channel}.{time}')
self.bank.update_index()
def get_waveforms(self, **kwargs):
stream = self.bank.get_waveforms(**kwargs)
traces = []
for trace in stream:
data = trace.data[:-1]
if isinstance(data, np.ma.MaskedArray):
data = np.ma.filled(data, fill_value=np.nan)
header = {
'delta': trace.stats.delta,
'station': trace.stats.station,
'starttime': trace.stats.starttime,
'channel': trace.stats.channel,
'network': trace.stats.network
}
traces.append(Trace(data, header=header))
return Stream(traces=traces)
def get_stations(self):
df = self.bank.get_availability_df()
uptime = self.bank.get_uptime_df()
def create_seed(row):
network = row['network']
station = row['station']
return network + '.' + station
df['seed'] = df.apply(lambda row: create_seed(row), axis=1)
unique_stations = df['seed'].unique().tolist()
return unique_stations
def continuous_sbank(self, download_data):
""" return an sbank pointed at the temp_dir_path """
sb = WaveBank(download_data)
sb.update_index()
return sb
def kem_bank_no_bulk(self, kem_archive, monkeypatch):
""" remove the get_waveforms_bulk from Sbank class """
monkeypatch.delattr(WaveBank, "get_waveforms_bulk")
monkeypatch.delattr(WaveBank, "get_waveforms_by_seed")
# return a bank
yield WaveBank(kem_archive)
def download_waveforms(self):
"""Download TA waveforms."""
st = self._download_client.get_waveforms_bulk(self.bulk)
self.build_archive(st)
# update the index
WaveBank(self.waveform_path).update_index()
def run_real_time_matched_filter(**kwargs):
config = read_config(config_file=kwargs.get("config_file", None))
debug = kwargs.get("debug", False)
if debug:
config.log_level = "DEBUG"
print("Using the following configuration:\n{0}".format(config))
config.setup_logging()
Logger.debug("Running in debug mode - expect lots of output!")
client = config.rt_match_filter.get_client()
if kwargs.get("local_archive"):
local_wave_bank = WaveBank(
base_path=config.rt_match_filter.local_wave_bank)
client = ClientBank(wave_bank=local_wave_bank,
event_bank=client,
station_bank=client)
triggering_eventid = kwargs.get("eventid", None)
if triggering_eventid:
triggering_event = client.get_events(eventid=triggering_eventid)[0]
region = estimate_region(triggering_event)
tribe_name = triggering_eventid
else:
triggering_event = None
region = {
"latitude": kwargs.get("latitude", None),
"longitude": kwargs.get("longitude", None),
"maxradius": kwargs.get("maxradius", None)
}
tribe_name = "{0}_{1}_{2}".format(kwargs.get("latitude", "lat"),
kwargs.get("longitude", "long"),
kwargs.get("maxradius", "rad"))
starttime = kwargs.get("starttime", None)
endtime = kwargs.get("endtime", None)
rt_client_starttime = kwargs.get("rt_client_starttime", None)
if starttime is not None:
region.update({"starttime": starttime})
if endtime is not None:
region.update({"endtime": endtime})
elif rt_client_starttime is not None:
region.update({"endtime": rt_client_starttime})
bank = TemplateBank(base_path=config.database_manager.event_path,
name_structure=config.database_manager.name_structure,
event_format=config.database_manager.event_format,
path_structure=config.database_manager.path_structure,
event_ext=config.database_manager.event_ext)
Logger.info("Region: {0}".format(region))
df = bank.get_event_summary(**region)
Logger.info("{0} events within region".format(len(df)))
if len(df) == 0:
return Party()
Logger.info("Reading in Tribe")
tribe = bank.get_templates(**region)
Logger.info("Read in tribe of {0} templates".format(len(tribe)))
_detection_starttime = rt_client_starttime or UTCDateTime.now()
inventory = get_inventory(client,
tribe,
triggering_event=triggering_event,
location=region,
starttime=_detection_starttime,
max_distance=1000,
n_stations=10)
used_channels = {
"{net}.{sta}.{loc}.{chan}".format(net=net.code,
sta=sta.code,
loc=chan.location_code,
chan=chan.code)
for net in inventory for sta in net for chan in sta
}
tribe = check_tribe_quality(
tribe=tribe,
seed_ids=used_channels,
min_stations=config.database_manager.min_stations,
**config.template)
Logger.info("After some QC there are {0} templates in the Tribe".format(
len(tribe)))
if rt_client_starttime is not None:
config.streaming.starttime = rt_client_starttime
config.streaming.speed_up = kwargs.get("speed_up", 1.0)
config.plot.offline = True
rt_client = config.streaming.get_streaming_client()
real_time_tribe = RealTimeTribe(
tribe=tribe,
inventory=inventory,
rt_client=rt_client,
detect_interval=config.rt_match_filter.detect_interval,
plot=config.rt_match_filter.plot,
name=tribe_name,
plot_options=config.plot,
wavebank=config.rt_match_filter.local_wave_bank)
try:
real_time_tribe._speed_up = config.streaming.speed_up
except AttributeError:
real_time_tribe._speed_up = 1
party = None
try:
party = real_time_tribe.run(**config.rt_match_filter)
except KeyboardInterrupt as e:
Logger.error(e)
finally:
real_time_tribe.stop()
return party
def deposited_bank(self, bank: obsplus.WaveBank):
""" deposit the waveforms in the bank, return the bank """
bank.put_waveforms(self.st_mod)
bank.put_waveforms(self.st)
return bank
def ta_bank_no_bulk(self, ta_dataset, monkeypatch):
"""remove the get_waveforms_bulk from WaveBank class"""
monkeypatch.delattr(WaveBank, "get_waveforms_bulk")
# return a bank
yield WaveBank(ta_dataset.waveform_path)
def wavecache(bingham_dataset):
"""return a wavecache of bingham_test archive"""
bank = WaveBank(bingham_dataset.waveform_client)
return WaveCache(bank)
def __init__(self, directory):
super().__init__()
self.bank = WaveBank(
directory, name_structure='{network}.{station}.{channel}.{time}')
self.bank.update_index()
def synthesise_real_time(
triggering_event: Event,
database_duration: float,
config: Config,
detection_runtime: float = 3600.0,
make_templates: bool = True,
speed_up: float = 1,
debug: bool = False,
query_interval: float = 60,
):
"""
Synthesise a real-time matched-filter process for old data.
Parameters
----------
triggering_event:
The Event that should trigger the system (must have happened in the
past)
database_duration:
The duration to create the template database for in days prior to the
triggering event
config:
Configuration for this synthesis
detection_runtime:
Maximum run-time for the detector in seconds
make_templates:
Whether templates need to be made or not.
speed_up:
Speed-up factor for detector - stream data faster than "real-time".
debug:
Whether to run logging in debug or not
query_interval:
How often to query the waveform server in seconds. Smaller numbers
will query more often, but this is limited by disk read speeds - make
sure you don't go too small and make your system stall!
"""
if debug:
config.log_level = "DEBUG"
print("Using the following configuration:\n{0}".format(config))
config.setup_logging()
client = config.rt_match_filter.get_client()
trigger_origin = (triggering_event.preferred_origin()
or triggering_event.origins[0])
region = estimate_region(triggering_event)
database_starttime = trigger_origin.time - (database_duration * 86400)
database_endtime = trigger_origin.time
trigger_func = partial(
magnitude_rate_trigger_func,
magnitude_threshold=config.reactor.magnitude_threshold,
rate_threshold=config.reactor.rate_threshold,
rate_bin=config.reactor.rate_radius,
minimum_events_in_bin=config.reactor.minimum_events_in_bin)
template_bank = TemplateBank(
config.database_manager.event_path,
name_structure=config.database_manager.name_structure,
event_format=config.database_manager.event_format,
path_structure=config.database_manager.path_structure,
event_ext=config.database_manager.event_ext,
executor=None)
if make_templates:
Logger.info("Downloading template events")
catalog = client.get_events(starttime=database_starttime,
endtime=database_endtime,
**region)
Logger.info(f"Downloaded {len(catalog)} events")
Logger.info("Building template database")
template_bank.make_templates(catalog=catalog,
client=client,
**config.template)
else:
template_bank.update_index()
tribe = template_bank.get_templates(starttime=database_starttime,
endtime=database_endtime,
**region)
inventory = get_inventory(client,
tribe,
triggering_event=triggering_event,
max_distance=config.rt_match_filter.max_distance,
n_stations=config.rt_match_filter.n_stations)
config.plot.update({"offline": True}) # Use to use data time-stamps
Logger.info("Downloading data")
wavebank = WaveBank("simulation_wavebank")
for network in inventory:
for station in network:
for channel in station:
try:
st = client.get_waveforms(
network=network.code,
station=station.code,
channel=channel.code,
location=channel.location_code,
starttime=trigger_origin.time - 60.,
endtime=trigger_origin.time + detection_runtime)
except Exception as e:
Logger.error("Could not download data for "
f"{network.code}.{station.code}."
f"{channel.location_code}.{channel.code}")
Logger.error(e)
continue
wavebank.put_waveforms(st)
# Set up config to use the wavebank rather than FDSN.
config.streaming.update({
"rt_client_url": str(wavebank.bank_path),
"rt_client_type": "obsplus",
"starttime": trigger_origin.time - 60,
"speed_up": speed_up,
"query_interval": 1.0
})
listener = CatalogListener(client=client,
catalog_lookup_kwargs=region,
template_bank=template_bank,
interval=query_interval,
keep=86400,
catalog=None,
waveform_client=client)
listener._speed_up = speed_up
listener._test_start_step = UTCDateTime.now() - trigger_origin.time
listener._test_start_step += 60 # Start up 1 minute before the event
reactor = Reactor(client=client,
listener=listener,
trigger_func=trigger_func,
template_database=template_bank,
config=config)
Logger.info("Starting reactor")
reactor.run(max_run_length=config.reactor.max_run_length)