示例#1
0
 def _download_crandall(self):
     """download waveform/station info for dataset."""
     bank = WaveBank(self.waveform_path)
     domain = CircularDomain(
         self.latitude,
         self.longitude,
         minradius=0,
         maxradius=kilometers2degrees(self.max_dist),
     )
     cat = obspy.read_events(str(self.source_path / "events.xml"))
     df = events_to_df(cat)
     for _, row in df.iterrows():
         starttime = row.time - self.time_before
         endtime = row.time + self.time_after
         restrictions = Restrictions(
             starttime=UTC(starttime),
             endtime=UTC(endtime),
             minimum_length=0.90,
             minimum_interstation_distance_in_m=100,
             channel_priorities=["HH[ZNE]", "BH[ZNE]"],
             location_priorities=["", "00", "01", "--"],
         )
         kwargs = dict(
             domain=domain,
             restrictions=restrictions,
             mseed_storage=str(self.waveform_path),
             stationxml_storage=str(self.station_path),
         )
         MassDownloader(providers=[self._download_client]).download(
             **kwargs)
         # ensure data have actually been downloaded
         bank.update_index()
         assert not bank.read_index(starttime=starttime,
                                    endtime=endtime).empty
示例#2
0
 def __init__(self, waveform_client: obsplus.WaveBank):
     assert hasattr(waveform_client, "read_index")
     # dicts that will be used as caches for processed/unprocessed streams
     self._stream_cache = {}
     self._processed_cache = {}
     # list of callables that should take a single waveforms and return
     self._stream_processors = defaultdict(list)
     # load all of the bank index into memory
     self.bank = waveform_client
     self.index = waveform_client.read_index()
     # make a column of callables for getting waveforms
     self.index["st_call"] = self.index.apply(self._make_st_calls, axis=1)
     # get a column that identifies rows that have the same resource
     self.index.rename(columns={"path": "unique_key"}, inplace=True)
示例#3
0
class WavebankWrapper(AnxcorDatabase):
    def __init__(self, directory):
        super().__init__()
        self.bank = WaveBank(
            directory, name_structure='{network}.{station}.{channel}.{time}')
        self.bank.update_index()

    def get_waveforms(self, **kwargs):
        stream = self.bank.get_waveforms(**kwargs)
        traces = []
        for trace in stream:
            data = trace.data[:-1]
            if isinstance(data, np.ma.MaskedArray):
                data = np.ma.filled(data, fill_value=np.nan)
            header = {
                'delta': trace.stats.delta,
                'station': trace.stats.station,
                'starttime': trace.stats.starttime,
                'channel': trace.stats.channel,
                'network': trace.stats.network
            }
            traces.append(Trace(data, header=header))
        return Stream(traces=traces)

    def get_stations(self):
        df = self.bank.get_availability_df()
        uptime = self.bank.get_uptime_df()

        def create_seed(row):
            network = row['network']
            station = row['station']
            return network + '.' + station

        df['seed'] = df.apply(lambda row: create_seed(row), axis=1)
        unique_stations = df['seed'].unique().tolist()
        return unique_stations
示例#4
0
 def continuous_sbank(self, download_data):
     """ return an sbank pointed at the temp_dir_path """
     sb = WaveBank(download_data)
     sb.update_index()
     return sb
示例#5
0
 def kem_bank_no_bulk(self, kem_archive, monkeypatch):
     """ remove the get_waveforms_bulk from Sbank class """
     monkeypatch.delattr(WaveBank, "get_waveforms_bulk")
     monkeypatch.delattr(WaveBank, "get_waveforms_by_seed")
     # return a bank
     yield WaveBank(kem_archive)
示例#6
0
 def download_waveforms(self):
     """Download TA waveforms."""
     st = self._download_client.get_waveforms_bulk(self.bulk)
     self.build_archive(st)
     # update the index
     WaveBank(self.waveform_path).update_index()
示例#7
0
def run_real_time_matched_filter(**kwargs):
    config = read_config(config_file=kwargs.get("config_file", None))
    debug = kwargs.get("debug", False)
    if debug:
        config.log_level = "DEBUG"
        print("Using the following configuration:\n{0}".format(config))
    config.setup_logging()
    Logger.debug("Running in debug mode - expect lots of output!")

    client = config.rt_match_filter.get_client()
    if kwargs.get("local_archive"):
        local_wave_bank = WaveBank(
            base_path=config.rt_match_filter.local_wave_bank)
        client = ClientBank(wave_bank=local_wave_bank,
                            event_bank=client,
                            station_bank=client)

    triggering_eventid = kwargs.get("eventid", None)

    if triggering_eventid:
        triggering_event = client.get_events(eventid=triggering_eventid)[0]
        region = estimate_region(triggering_event)
        tribe_name = triggering_eventid
    else:
        triggering_event = None
        region = {
            "latitude": kwargs.get("latitude", None),
            "longitude": kwargs.get("longitude", None),
            "maxradius": kwargs.get("maxradius", None)
        }
        tribe_name = "{0}_{1}_{2}".format(kwargs.get("latitude", "lat"),
                                          kwargs.get("longitude", "long"),
                                          kwargs.get("maxradius", "rad"))
    starttime = kwargs.get("starttime", None)
    endtime = kwargs.get("endtime", None)
    rt_client_starttime = kwargs.get("rt_client_starttime", None)
    if starttime is not None:
        region.update({"starttime": starttime})
    if endtime is not None:
        region.update({"endtime": endtime})
    elif rt_client_starttime is not None:
        region.update({"endtime": rt_client_starttime})
    bank = TemplateBank(base_path=config.database_manager.event_path,
                        name_structure=config.database_manager.name_structure,
                        event_format=config.database_manager.event_format,
                        path_structure=config.database_manager.path_structure,
                        event_ext=config.database_manager.event_ext)
    Logger.info("Region: {0}".format(region))
    df = bank.get_event_summary(**region)
    Logger.info("{0} events within region".format(len(df)))
    if len(df) == 0:
        return Party()
    Logger.info("Reading in Tribe")

    tribe = bank.get_templates(**region)

    Logger.info("Read in tribe of {0} templates".format(len(tribe)))

    _detection_starttime = rt_client_starttime or UTCDateTime.now()
    inventory = get_inventory(client,
                              tribe,
                              triggering_event=triggering_event,
                              location=region,
                              starttime=_detection_starttime,
                              max_distance=1000,
                              n_stations=10)

    used_channels = {
        "{net}.{sta}.{loc}.{chan}".format(net=net.code,
                                          sta=sta.code,
                                          loc=chan.location_code,
                                          chan=chan.code)
        for net in inventory for sta in net for chan in sta
    }

    tribe = check_tribe_quality(
        tribe=tribe,
        seed_ids=used_channels,
        min_stations=config.database_manager.min_stations,
        **config.template)

    Logger.info("After some QC there are {0} templates in the Tribe".format(
        len(tribe)))

    if rt_client_starttime is not None:
        config.streaming.starttime = rt_client_starttime
        config.streaming.speed_up = kwargs.get("speed_up", 1.0)
        config.plot.offline = True
    rt_client = config.streaming.get_streaming_client()
    real_time_tribe = RealTimeTribe(
        tribe=tribe,
        inventory=inventory,
        rt_client=rt_client,
        detect_interval=config.rt_match_filter.detect_interval,
        plot=config.rt_match_filter.plot,
        name=tribe_name,
        plot_options=config.plot,
        wavebank=config.rt_match_filter.local_wave_bank)
    try:
        real_time_tribe._speed_up = config.streaming.speed_up
    except AttributeError:
        real_time_tribe._speed_up = 1

    party = None
    try:
        party = real_time_tribe.run(**config.rt_match_filter)
    except KeyboardInterrupt as e:
        Logger.error(e)
    finally:
        real_time_tribe.stop()
    return party
 def deposited_bank(self, bank: obsplus.WaveBank):
     """ deposit the waveforms in the bank, return the bank """
     bank.put_waveforms(self.st_mod)
     bank.put_waveforms(self.st)
     return bank
示例#9
0
 def ta_bank_no_bulk(self, ta_dataset, monkeypatch):
     """remove the get_waveforms_bulk from WaveBank class"""
     monkeypatch.delattr(WaveBank, "get_waveforms_bulk")
     # return a bank
     yield WaveBank(ta_dataset.waveform_path)
示例#10
0
def wavecache(bingham_dataset):
    """return a wavecache of bingham_test archive"""
    bank = WaveBank(bingham_dataset.waveform_client)
    return WaveCache(bank)
示例#11
0
 def __init__(self, directory):
     super().__init__()
     self.bank = WaveBank(
         directory, name_structure='{network}.{station}.{channel}.{time}')
     self.bank.update_index()
示例#12
0
def synthesise_real_time(
    triggering_event: Event,
    database_duration: float,
    config: Config,
    detection_runtime: float = 3600.0,
    make_templates: bool = True,
    speed_up: float = 1,
    debug: bool = False,
    query_interval: float = 60,
):
    """
    Synthesise a real-time matched-filter process for old data.

    Parameters
    ----------
    triggering_event:
        The Event that should trigger the system (must have happened in the
        past)
    database_duration:
        The duration to create the template database for in days prior to the
        triggering event
    config:
        Configuration for this synthesis
    detection_runtime:
        Maximum run-time for the detector in seconds
    make_templates:
        Whether templates need to be made or not.
    speed_up:
        Speed-up factor for detector - stream data faster than "real-time".
    debug:
        Whether to run logging in debug or not
    query_interval:
        How often to query the waveform server in seconds.  Smaller numbers
        will query more often, but this is limited by disk read speeds - make
        sure you don't go too small and make your system stall!
    """
    if debug:
        config.log_level = "DEBUG"
        print("Using the following configuration:\n{0}".format(config))
    config.setup_logging()

    client = config.rt_match_filter.get_client()

    trigger_origin = (triggering_event.preferred_origin()
                      or triggering_event.origins[0])
    region = estimate_region(triggering_event)
    database_starttime = trigger_origin.time - (database_duration * 86400)
    database_endtime = trigger_origin.time

    trigger_func = partial(
        magnitude_rate_trigger_func,
        magnitude_threshold=config.reactor.magnitude_threshold,
        rate_threshold=config.reactor.rate_threshold,
        rate_bin=config.reactor.rate_radius,
        minimum_events_in_bin=config.reactor.minimum_events_in_bin)

    template_bank = TemplateBank(
        config.database_manager.event_path,
        name_structure=config.database_manager.name_structure,
        event_format=config.database_manager.event_format,
        path_structure=config.database_manager.path_structure,
        event_ext=config.database_manager.event_ext,
        executor=None)

    if make_templates:
        Logger.info("Downloading template events")
        catalog = client.get_events(starttime=database_starttime,
                                    endtime=database_endtime,
                                    **region)
        Logger.info(f"Downloaded {len(catalog)} events")
        Logger.info("Building template database")
        template_bank.make_templates(catalog=catalog,
                                     client=client,
                                     **config.template)
    else:
        template_bank.update_index()
    tribe = template_bank.get_templates(starttime=database_starttime,
                                        endtime=database_endtime,
                                        **region)
    inventory = get_inventory(client,
                              tribe,
                              triggering_event=triggering_event,
                              max_distance=config.rt_match_filter.max_distance,
                              n_stations=config.rt_match_filter.n_stations)

    config.plot.update({"offline": True})  # Use to use data time-stamps

    Logger.info("Downloading data")
    wavebank = WaveBank("simulation_wavebank")
    for network in inventory:
        for station in network:
            for channel in station:
                try:
                    st = client.get_waveforms(
                        network=network.code,
                        station=station.code,
                        channel=channel.code,
                        location=channel.location_code,
                        starttime=trigger_origin.time - 60.,
                        endtime=trigger_origin.time + detection_runtime)
                except Exception as e:
                    Logger.error("Could not download data for "
                                 f"{network.code}.{station.code}."
                                 f"{channel.location_code}.{channel.code}")
                    Logger.error(e)
                    continue
                wavebank.put_waveforms(st)

    # Set up config to use the wavebank rather than FDSN.
    config.streaming.update({
        "rt_client_url": str(wavebank.bank_path),
        "rt_client_type": "obsplus",
        "starttime": trigger_origin.time - 60,
        "speed_up": speed_up,
        "query_interval": 1.0
    })

    listener = CatalogListener(client=client,
                               catalog_lookup_kwargs=region,
                               template_bank=template_bank,
                               interval=query_interval,
                               keep=86400,
                               catalog=None,
                               waveform_client=client)
    listener._speed_up = speed_up
    listener._test_start_step = UTCDateTime.now() - trigger_origin.time
    listener._test_start_step += 60  # Start up 1 minute before the event

    reactor = Reactor(client=client,
                      listener=listener,
                      trigger_func=trigger_func,
                      template_database=template_bank,
                      config=config)
    Logger.info("Starting reactor")
    reactor.run(max_run_length=config.reactor.max_run_length)