def _download_crandall(self): """download waveform/station info for dataset.""" bank = WaveBank(self.waveform_path) domain = CircularDomain( self.latitude, self.longitude, minradius=0, maxradius=kilometers2degrees(self.max_dist), ) cat = obspy.read_events(str(self.source_path / "events.xml")) df = events_to_df(cat) for _, row in df.iterrows(): starttime = row.time - self.time_before endtime = row.time + self.time_after restrictions = Restrictions( starttime=UTC(starttime), endtime=UTC(endtime), minimum_length=0.90, minimum_interstation_distance_in_m=100, channel_priorities=["HH[ZNE]", "BH[ZNE]"], location_priorities=["", "00", "01", "--"], ) kwargs = dict( domain=domain, restrictions=restrictions, mseed_storage=str(self.waveform_path), stationxml_storage=str(self.station_path), ) MassDownloader(providers=[self._download_client]).download( **kwargs) # ensure data have actually been downloaded bank.update_index() assert not bank.read_index(starttime=starttime, endtime=endtime).empty
def __init__(self, waveform_client: obsplus.WaveBank): assert hasattr(waveform_client, "read_index") # dicts that will be used as caches for processed/unprocessed streams self._stream_cache = {} self._processed_cache = {} # list of callables that should take a single waveforms and return self._stream_processors = defaultdict(list) # load all of the bank index into memory self.bank = waveform_client self.index = waveform_client.read_index() # make a column of callables for getting waveforms self.index["st_call"] = self.index.apply(self._make_st_calls, axis=1) # get a column that identifies rows that have the same resource self.index.rename(columns={"path": "unique_key"}, inplace=True)
class WavebankWrapper(AnxcorDatabase): def __init__(self, directory): super().__init__() self.bank = WaveBank( directory, name_structure='{network}.{station}.{channel}.{time}') self.bank.update_index() def get_waveforms(self, **kwargs): stream = self.bank.get_waveforms(**kwargs) traces = [] for trace in stream: data = trace.data[:-1] if isinstance(data, np.ma.MaskedArray): data = np.ma.filled(data, fill_value=np.nan) header = { 'delta': trace.stats.delta, 'station': trace.stats.station, 'starttime': trace.stats.starttime, 'channel': trace.stats.channel, 'network': trace.stats.network } traces.append(Trace(data, header=header)) return Stream(traces=traces) def get_stations(self): df = self.bank.get_availability_df() uptime = self.bank.get_uptime_df() def create_seed(row): network = row['network'] station = row['station'] return network + '.' + station df['seed'] = df.apply(lambda row: create_seed(row), axis=1) unique_stations = df['seed'].unique().tolist() return unique_stations
def continuous_sbank(self, download_data): """ return an sbank pointed at the temp_dir_path """ sb = WaveBank(download_data) sb.update_index() return sb
def kem_bank_no_bulk(self, kem_archive, monkeypatch): """ remove the get_waveforms_bulk from Sbank class """ monkeypatch.delattr(WaveBank, "get_waveforms_bulk") monkeypatch.delattr(WaveBank, "get_waveforms_by_seed") # return a bank yield WaveBank(kem_archive)
def download_waveforms(self): """Download TA waveforms.""" st = self._download_client.get_waveforms_bulk(self.bulk) self.build_archive(st) # update the index WaveBank(self.waveform_path).update_index()
def run_real_time_matched_filter(**kwargs): config = read_config(config_file=kwargs.get("config_file", None)) debug = kwargs.get("debug", False) if debug: config.log_level = "DEBUG" print("Using the following configuration:\n{0}".format(config)) config.setup_logging() Logger.debug("Running in debug mode - expect lots of output!") client = config.rt_match_filter.get_client() if kwargs.get("local_archive"): local_wave_bank = WaveBank( base_path=config.rt_match_filter.local_wave_bank) client = ClientBank(wave_bank=local_wave_bank, event_bank=client, station_bank=client) triggering_eventid = kwargs.get("eventid", None) if triggering_eventid: triggering_event = client.get_events(eventid=triggering_eventid)[0] region = estimate_region(triggering_event) tribe_name = triggering_eventid else: triggering_event = None region = { "latitude": kwargs.get("latitude", None), "longitude": kwargs.get("longitude", None), "maxradius": kwargs.get("maxradius", None) } tribe_name = "{0}_{1}_{2}".format(kwargs.get("latitude", "lat"), kwargs.get("longitude", "long"), kwargs.get("maxradius", "rad")) starttime = kwargs.get("starttime", None) endtime = kwargs.get("endtime", None) rt_client_starttime = kwargs.get("rt_client_starttime", None) if starttime is not None: region.update({"starttime": starttime}) if endtime is not None: region.update({"endtime": endtime}) elif rt_client_starttime is not None: region.update({"endtime": rt_client_starttime}) bank = TemplateBank(base_path=config.database_manager.event_path, name_structure=config.database_manager.name_structure, event_format=config.database_manager.event_format, path_structure=config.database_manager.path_structure, event_ext=config.database_manager.event_ext) Logger.info("Region: {0}".format(region)) df = bank.get_event_summary(**region) Logger.info("{0} events within region".format(len(df))) if len(df) == 0: return Party() Logger.info("Reading in Tribe") tribe = bank.get_templates(**region) Logger.info("Read in tribe of {0} templates".format(len(tribe))) _detection_starttime = rt_client_starttime or UTCDateTime.now() inventory = get_inventory(client, tribe, triggering_event=triggering_event, location=region, starttime=_detection_starttime, max_distance=1000, n_stations=10) used_channels = { "{net}.{sta}.{loc}.{chan}".format(net=net.code, sta=sta.code, loc=chan.location_code, chan=chan.code) for net in inventory for sta in net for chan in sta } tribe = check_tribe_quality( tribe=tribe, seed_ids=used_channels, min_stations=config.database_manager.min_stations, **config.template) Logger.info("After some QC there are {0} templates in the Tribe".format( len(tribe))) if rt_client_starttime is not None: config.streaming.starttime = rt_client_starttime config.streaming.speed_up = kwargs.get("speed_up", 1.0) config.plot.offline = True rt_client = config.streaming.get_streaming_client() real_time_tribe = RealTimeTribe( tribe=tribe, inventory=inventory, rt_client=rt_client, detect_interval=config.rt_match_filter.detect_interval, plot=config.rt_match_filter.plot, name=tribe_name, plot_options=config.plot, wavebank=config.rt_match_filter.local_wave_bank) try: real_time_tribe._speed_up = config.streaming.speed_up except AttributeError: real_time_tribe._speed_up = 1 party = None try: party = real_time_tribe.run(**config.rt_match_filter) except KeyboardInterrupt as e: Logger.error(e) finally: real_time_tribe.stop() return party
def deposited_bank(self, bank: obsplus.WaveBank): """ deposit the waveforms in the bank, return the bank """ bank.put_waveforms(self.st_mod) bank.put_waveforms(self.st) return bank
def ta_bank_no_bulk(self, ta_dataset, monkeypatch): """remove the get_waveforms_bulk from WaveBank class""" monkeypatch.delattr(WaveBank, "get_waveforms_bulk") # return a bank yield WaveBank(ta_dataset.waveform_path)
def wavecache(bingham_dataset): """return a wavecache of bingham_test archive""" bank = WaveBank(bingham_dataset.waveform_client) return WaveCache(bank)
def __init__(self, directory): super().__init__() self.bank = WaveBank( directory, name_structure='{network}.{station}.{channel}.{time}') self.bank.update_index()
def synthesise_real_time( triggering_event: Event, database_duration: float, config: Config, detection_runtime: float = 3600.0, make_templates: bool = True, speed_up: float = 1, debug: bool = False, query_interval: float = 60, ): """ Synthesise a real-time matched-filter process for old data. Parameters ---------- triggering_event: The Event that should trigger the system (must have happened in the past) database_duration: The duration to create the template database for in days prior to the triggering event config: Configuration for this synthesis detection_runtime: Maximum run-time for the detector in seconds make_templates: Whether templates need to be made or not. speed_up: Speed-up factor for detector - stream data faster than "real-time". debug: Whether to run logging in debug or not query_interval: How often to query the waveform server in seconds. Smaller numbers will query more often, but this is limited by disk read speeds - make sure you don't go too small and make your system stall! """ if debug: config.log_level = "DEBUG" print("Using the following configuration:\n{0}".format(config)) config.setup_logging() client = config.rt_match_filter.get_client() trigger_origin = (triggering_event.preferred_origin() or triggering_event.origins[0]) region = estimate_region(triggering_event) database_starttime = trigger_origin.time - (database_duration * 86400) database_endtime = trigger_origin.time trigger_func = partial( magnitude_rate_trigger_func, magnitude_threshold=config.reactor.magnitude_threshold, rate_threshold=config.reactor.rate_threshold, rate_bin=config.reactor.rate_radius, minimum_events_in_bin=config.reactor.minimum_events_in_bin) template_bank = TemplateBank( config.database_manager.event_path, name_structure=config.database_manager.name_structure, event_format=config.database_manager.event_format, path_structure=config.database_manager.path_structure, event_ext=config.database_manager.event_ext, executor=None) if make_templates: Logger.info("Downloading template events") catalog = client.get_events(starttime=database_starttime, endtime=database_endtime, **region) Logger.info(f"Downloaded {len(catalog)} events") Logger.info("Building template database") template_bank.make_templates(catalog=catalog, client=client, **config.template) else: template_bank.update_index() tribe = template_bank.get_templates(starttime=database_starttime, endtime=database_endtime, **region) inventory = get_inventory(client, tribe, triggering_event=triggering_event, max_distance=config.rt_match_filter.max_distance, n_stations=config.rt_match_filter.n_stations) config.plot.update({"offline": True}) # Use to use data time-stamps Logger.info("Downloading data") wavebank = WaveBank("simulation_wavebank") for network in inventory: for station in network: for channel in station: try: st = client.get_waveforms( network=network.code, station=station.code, channel=channel.code, location=channel.location_code, starttime=trigger_origin.time - 60., endtime=trigger_origin.time + detection_runtime) except Exception as e: Logger.error("Could not download data for " f"{network.code}.{station.code}." f"{channel.location_code}.{channel.code}") Logger.error(e) continue wavebank.put_waveforms(st) # Set up config to use the wavebank rather than FDSN. config.streaming.update({ "rt_client_url": str(wavebank.bank_path), "rt_client_type": "obsplus", "starttime": trigger_origin.time - 60, "speed_up": speed_up, "query_interval": 1.0 }) listener = CatalogListener(client=client, catalog_lookup_kwargs=region, template_bank=template_bank, interval=query_interval, keep=86400, catalog=None, waveform_client=client) listener._speed_up = speed_up listener._test_start_step = UTCDateTime.now() - trigger_origin.time listener._test_start_step += 60 # Start up 1 minute before the event reactor = Reactor(client=client, listener=listener, trigger_func=trigger_func, template_database=template_bank, config=config) Logger.info("Starting reactor") reactor.run(max_run_length=config.reactor.max_run_length)