def test_station_information_extraction(): """ Station information can either be passed or read from sac files. """ import obspy.station config = pyflex.Config(min_period=50.0, max_period=150.0) # If not passed, it is read from sac files, if available. ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config) assert abs(ws.station.latitude - 37.930401) < 1E-5 assert abs(ws.station.longitude - 58.1189) < 1E-5 # The other option is an inventory object. Assemble a dummy one. inv = obspy.station.Inventory(networks=[], source="local") net = obspy.station.Network(code=OBS_DATA[0].stats.network) sta = obspy.station.Station(code=OBS_DATA[0].stats.station, latitude=1.0, longitude=2.0, elevation=3.0) inv.networks = [net] net.stations = [sta] ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config, station=inv) assert ws.station == pyflex.Station(1.0, 2.0)
def test_custom_weight_function(): """ Test the custom weight function. Set the weight of every window with a CC of smaller then 95 to 0.0. """ def weight_function(win): if win.max_cc_value < 0.95: return 0.0 else: return 10.0 config = pyflex.Config( min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0, window_weight_fct=weight_function) windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) assert np.all(np.array([_i.max_cc_value for _i in windows]) >= 0.95) # Not setting it will result in the default value. config.window_weight_fct = None windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) assert bool(np.all(np.array([_i.max_cc_value for _i in windows]) >= 0.95)) is False
def test_settings_arrays_as_config_values(): """ Tests that arrays can be set as config values. """ npts = OBS_DATA[0].stats.npts stalta_waterlevel = 0.08 * np.ones(npts) tshift_acceptance_level = 15.0 * np.ones(npts) dlna_acceptance_level = 1.0 * np.ones(npts) cc_acceptance_level = 0.80 * np.ones(npts) s2n_limit = 1.5 * np.ones(npts) config = pyflex.Config(min_period=50.0, max_period=150.0, stalta_waterlevel=stalta_waterlevel, tshift_acceptance_level=tshift_acceptance_level, dlna_acceptance_level=dlna_acceptance_level, cc_acceptance_level=cc_acceptance_level, s2n_limit=s2n_limit, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) assert len(windows) == 9
def test_window_selection(): """ This WILL need to be adjusted if any part of the algorithm changes! The settings for this test are more or less the same as for the test data example in the original FLEXWIN package. """ config = pyflex.Config(min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) assert len(windows) == 9 lefties = np.array([_i.left for _i in windows]) righties = np.array([_i.right for _i in windows]) np.testing.assert_allclose( lefties, np.array([1551, 2221, 2709, 2960, 3353, 3609, 3983, 4715, 4962]), atol=3) np.testing.assert_allclose( righties, np.array([1985, 2709, 2960, 3172, 3609, 3920, 4442, 4962, 5207]), atol=3) np.testing.assert_allclose(np.array([_i.max_cc_value for _i in windows]), np.array([ 0.95740629, 0.96646804, 0.96335716, 0.98249547, 0.96838754, 0.88501979, 0.82529382, 0.92953344, 0.92880873 ]), rtol=1E-2) assert [_i.cc_shift for _i in windows] == [-3, 0, -5, -5, -6, 4, -9, -1, 7] np.testing.assert_allclose(np.array([_i.dlnA for _i in windows]), np.array([ 0.07469, 0.12808, -0.19277, 0.185563, 0.093674, -0.118859, -0.638657, 0.25942, 0.106571 ]), rtol=1E-2) # Assert the phases of the first window. assert sorted([_i["phase_name"] for _i in windows[0].phase_arrivals]) == \ ['PKIKP', 'PKIKS', 'PKiKP', 'PP', 'SKIKP', 'SKiKP', 'pPKIKP', 'pPKiKP', 'sPKIKP', 'sPKiKP']
def test_window_plotting(tmpdir): reset_matplotlib() config = pyflex.Config( min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) pyflex.select_windows(OBS_DATA, SYNTH_DATA, config, plot=True) images_are_identical("picked_windows", str(tmpdir))
def select_windows(parameters, paths, merge_flag, obs_tag, syn_tag): """ Selects windows by comparing observed and synthetic traces; writes results to a json file :param parameters: dictionary passed directly to pyflex.Config :param paths.obs: ASDF observed data filename :param paths.syn: ASDF synthetic data filename :param paths.output: windows will be written to a JSON file with this name :param paths.log: information about the quantity and quality of windows will be written to a JSON file with this name :param obs_tag: observed data are read using this ASDF tag :param syn_tag: synthetic data are read using this ASDF tag """ from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.rank cwd = dirname(__file__) # read data fullpath = join(cwd, paths.obs) obs = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") event = obs.events[0] # read synthetics fullpath = join(cwd, paths.syn) syn = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") # generate pyflex.Config objects config = {} for channel, param in parameters.items(): config[channel] = pyflex.Config(**param) # wrapper is required for ASDF processing def wrapped_function(obs, syn): return pytomo3d.window.window_on_stream(obs[obs_tag], syn[syn_tag], config, station=obs.StationXML, event=event, user_modules=None, figure_mode=False, figure_dir=None, _verbose=False) # run window selection windows = obs.process_two_files_without_parallel_output( syn, wrapped_function) # save results if rank == 0: if merge_flag: windows = merge(windows) write_windows_json(paths.output, windows)
def test_cc_config_setting(): """ Make sure setting the CC threshold does something. """ config = pyflex.Config( min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.95, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) assert np.all(np.array([_i.max_cc_value for _i in windows]) >= 0.95)
def test_window_merging_strategy(): """ Pyflex can also merge windows. """ config = pyflex.Config( min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0, resolution_strategy="merge") windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) assert len(windows) == 4
def test_run_with_data_quality_checks(): """ Run with data quality checks. """ config = pyflex.Config( min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0, check_global_data_quality=True) windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config) # The data in this case is so good that nothing should have changed. assert len(windows) == 9
def load_window_config_yaml(filename): """ Load yaml and setup pyflex.Config object :param filename: :return: """ with open(filename) as fh: data = yaml.load(fh) if data["min_period"] > data["max_period"]: raise ValueError("min_period is larger than max_period in config " "file: %s" % filename) return pyflex.Config(**data)
def test_runs_without_event_information(recwarn): """ Make sure it runs without event information. Some things will not work but it will at least not crash. """ config = pyflex.Config(min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) obs = OBS_DATA[0].copy() syn = SYNTH_DATA[0].copy() # Remove the sac header information. del obs.stats.sac del syn.stats.sac recwarn.clear() windows = pyflex.select_windows(obs, syn, config) # This will actually result in a bunch more windows as before. So it # is always a good idea to specify the event and station information! assert len(windows) == 12 assert len(recwarn.list) == 1 w = recwarn.list[0] assert w.category == pyflex.PyflexWarning assert "Event and/or station information is not available".lower() in \ str(w.message).lower() # No phases should be attached as they cannot be calculated. phases = [] for win in windows: phases.extend(win.phase_arrivals) assert phases == []
def load_window_config(param): config_dict = {} flag_list = [] for key, value in param.iteritems(): # pop the "instrument_merge_flag" value out flag_list.append(value["instrument_merge_flag"]) value.pop("instrument_merge_flag") check_param_with_function_args(value) config_dict[key] = pyflex.Config(**value) if not all(_e == flag_list[0] for _e in flag_list): raise ValueError("Instrument_merge_flag not consistent amonge" "different parameter yaml files(%s). Check!" % flag_list) return config_dict, flag_list[0]
def test_event_information_extraction(): """ Event information can either be passed or read from sac files. """ config = pyflex.Config(min_period=50.0, max_period=150.0) # If not passed, it is read from sac files, if available. ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config) assert abs(ws.event.latitude - -3.77) <= 1E-5 assert abs(ws.event.longitude - -77.07) <= 1E-5 assert abs(ws.event.depth_in_m - 112800.00305) <= 1E-5 assert ws.event.origin_time == \ obspy.UTCDateTime(1995, 5, 2, 6, 6, 13, 900000) # If it passed, the passed event will be used. ev = pyflex.Event(1, 2, 3, obspy.UTCDateTime(2012, 1, 1)) ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config, event=ev) assert ws.event == ev # Alternatively, an ObsPy Catalog or Event object can be passed which # opens the gate to more complex workflows. cat = obspy.readEvents() cat.events = cat.events[:1] event = cat[0] ev = pyflex.Event(event.origins[0].latitude, event.origins[0].longitude, event.origins[0].depth, event.origins[0].time) # Test catalog. ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config, event=cat) assert ws.event == ev # Test event. ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config, event=cat[0]) assert ws.event == ev
def compute_windows(conf_file): conf = load_config(conf_file) if conf.window.use_pyflex: if conf.window.pyflex_conf_file is None: raise Exception( "pyflex_conf_file should be set to compute windows.") # NOQA logger = logging.getLogger("pyflex") logger.setLevel(logging.ERROR) pyflex_args = utils.load_yaml(conf.window.pyflex_conf_file) pyflex_conf = pyflex.Config(**pyflex_args) mpi.run_with_mpi( partial(compute_windows_pyflex, pyflex_conf=pyflex_conf, conf=conf), conf.get_event_names()) elif conf.window.max_vel == 0 or conf.window.min_vel == 0: raise Exception( "Max & min velocity should be set for window selection.") # NOQA else: mpi.run_with_mpi(partial(compute_windows_by_vel, conf=conf), conf.get_event_names())
outputbasedir = os.path.join(basedir, "window") datadir = os.path.join(databasedir, eventdir) syntdir = os.path.join(syntbasedir, eventdir) outputdir = os.path.join(outputbasedir, eventdir) if not os.path.exists(outputdir): os.makedirs(outputdir) figdir = os.path.join(outputdir, "figures") if not os.path.exists(figdir): os.makedirs(figdir) config_bw = pyflex.Config( min_period=27.0, max_period=60.0, stalta_waterlevel=0.10, tshift_acceptance_level=10.0, dlna_acceptance_level=0.80, cc_acceptance_level=0.80, s2n_limit=2.0, check_global_data_quality=True, c_0=0.7, c_1=2.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0, window_signal_to_noise_type="amplitude") config_sw = pyflex.Config( min_period=60.0, max_period=120.0, stalta_waterlevel=0.10, tshift_acceptance_level=18.0, dlna_acceptance_level=0.80, cc_acceptance_level=0.80, s2n_limit=2.0, check_global_data_quality=True, c_0=0.7, c_1=3.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0, window_signal_to_noise_type="amplitude") if (max(period_band)<65): print "Body wave config" #body wave(short period)
new_config.cc_acceptance_level = cc new_config.s2n_limit = s2n new_config.signal_end_index = r_index return new_config conf = pyflex.Config( 40, 80, stalta_waterlevel=0.08, tshift_acceptance_level=10.0, tshift_reference=0.0, dlna_acceptance_level=0.8, cc_acceptance_level=0.85, check_global_data_quality=True, snr_integrate_base=3.5, snr_max_base=12.0, c_0=0.7, c_1=3.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0 ) ds_obsd = pyasdf.ASDFDataSet("observed.h5") ds_synt = pyasdf.ASDFDataSet("synthetic_more_perturbed.h5") event = ds_synt.events[0] origin = event.preferred_origin() synt_tag = "synthetic" obsd_tag = "synthetic"
def test_reading_and_writing_windows(tmpdir): """ Tests reading and writing of window sets. """ config = pyflex.Config(min_period=50.0, max_period=150.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.80, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) ws = pyflex.WindowSelector(observed=OBS_DATA, synthetic=SYNTH_DATA, config=config) windows = ws.select_windows() assert len(windows) > 0 # Write/read to/from file. filename = os.path.join(str(tmpdir), "window.json") ws.write(filename) ws2 = pyflex.WindowSelector(observed=OBS_DATA, synthetic=SYNTH_DATA, config=config) ws2.load(filename) assert ws.windows == ws2.windows os.remove(filename) # Write/read to/from open file. with open(filename, "w") as fh: ws.write(fh) ws2 = pyflex.WindowSelector(observed=OBS_DATA, synthetic=SYNTH_DATA, config=config) fh.seek(0, 0) ws2.load(filename) assert ws.windows == ws2.windows # Write/read to/from StringIO. with io.StringIO() as fh: ws.write(fh) ws2 = pyflex.WindowSelector(observed=OBS_DATA, synthetic=SYNTH_DATA, config=config) fh.seek(0, 0) ws2.load(filename) assert ws.windows == ws2.windows # Write/read to/from BytesIO. with io.BytesIO() as fh: ws.write(fh) ws2 = pyflex.WindowSelector(observed=OBS_DATA, synthetic=SYNTH_DATA, config=config) fh.seek(0, 0) ws2.load(filename) assert ws.windows == ws2.windows
# preprocessing obs_st.detrend("linear") obs_st.taper(max_percentage=0.05, type="hann") obs_st.filter("lowpass", freq=0.25, corners=2, zerophase=True) syn_st.detrend("linear") syn_st.taper(max_percentage=0.05, type="hann") syn_st.filter("lowpass", freq=0.25, corners=2, zerophase=True) # user defined configuration config = pyflex.Config(min_period=1.0, max_period=100.0, stalta_waterlevel=0.08, tshift_acceptance_level=15.0, dlna_acceptance_level=1.0, cc_acceptance_level=0.50, c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0) windows = pyflex.select_windows(obs_st, syn_st, config, plot=True) pprint.pprint(windows) #win = windows[0] # print("Relative times in seconds: %s - %s" % (win.relative_starttime, # win.relative_endtime))
event = ds.events[0] def weight_function(win): return win.max_cc_value config = pyflex.Config(min_period=27.0, max_period=60.0, stalta_waterlevel=0.11, tshift_acceptance_level=15.0, dlna_acceptance_level=2.5, cc_acceptance_level=0.6, c_0=0.7, c_1=2.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0, s2n_limit=0.5, max_time_before_first_arrival=-50.0, min_surface_wave_velocity=3.0, window_signal_to_noise_type="energy", window_weight_fct=weight_function) def process(this_station_group, other_station_group): # Make sure everything thats required is there. if not hasattr(this_station_group, "StationXML") or \ not hasattr(this_station_group, "preprocessed_27s_to_60s") or \ not hasattr(other_station_group,
def _process(self, inputs): # Weighting function for the windows. This particular choice will tend # to favor lots of smaller windows if "interval_scheduling" is chosen # as overlap resolution strategy. ip = inputs["input"] param = ip["parameters"] config = pyflex.Config( min_period=param["min_period"], max_period=param["max_period"], stalta_waterlevel=param["stalta_waterlevel"], s2n_limit=param["s2n_limit"], snr_max_base=param["snr_max_base"], tshift_acceptance_level=param["tshift_acceptance_level"], tshift_reference=param["tshift_reference"], dlna_acceptance_level=param["dlna_acceptance_level"], dlna_reference=param["dlna_reference"], cc_acceptance_level=param["cc_acceptance_level"], earth_model=param["earth_model"], min_surface_wave_velocity=param["min_surface_wave_velocity"], max_time_before_first_arrival=param[ "max_time_before_first_arrival"], c_0=param["c_0"], c_1=param["c_1"], c_2=param["c_2"], c_3a=param["c_3a"], c_3b=param["c_3b"], c_4a=param["c_4a"], c_4b=param["c_4b"], check_global_data_quality=param["check_global_data_quality"], snr_integrate_base=param["snr_integrate_base"], noise_start_index=param["noise_start_index"], noise_end_index=param["noise_end_index"], signal_start_index=param["signal_start_index"], signal_end_index=param["signal_end_index"], window_signal_to_noise_type=param["window_signal_to_noise_type"], resolution_strategy=param["resolution_strategy"], window_weight_fct=weighting_function) ws = WindowSelector(observed=ip["data_trace"], synthetic=ip["synthetic_trace"], config=config, event=[_i for _i in obspy.readEvents(ip["quakeml"]) if _i.resource_id.id == ip["event_id"]][0], station=obspy.read_inventory( ip["stationxml"], format="stationxml")) windows = ws.select_windows() data = ip["data_trace"] station_id = "%s.%s" % (data.stats.network, data.stats.station) component = data.stats.channel[-1] # HACK! with io.BytesIO() as _: ws.plot(_) fig = plt.gcf() fig.suptitle("Component: %s" % component, fontsize=15, horizontalalignment="center", x=0.8, y=1) with io.BytesIO() as buf: fig.savefig(buf) buf.seek(0, 0) # Not Python 3 compatible but who cares. image_string = buf.read() plt.close(fig) image_type = "pyflex_windows" # Metadata for Alessandro! metadata = [{"type": "pyflex", "station_id": station_id, "component": component, "channel_id": win.channel_id, "starttime": str(win.absolute_starttime), "endtime": str(win.absolute_endtime), "max_cc_value": win.max_cc_value, "cc_shift": win.cc_shift, "dlnA": win.dlnA} for win in windows] self.parameters=param self.write( "image", ({"image_string": image_string, "component": component, "output_folder": ip["output_folder"]}, image_type, station_id),metadata=metadata) #control={'con:skip':True} # self.write( # "windows", { # "record_type": "pyflex_windows", # "output_folder": ip["output_folder"], # "windows": windows, # "station_id": station_id, # "component": component},metadata=metadata) if ip["misfit_type"] == "pyflex_and_time_frequency": ip["windows"] = [{ "starttime": win.absolute_starttime, "endtime": win.absolute_endtime} for win in windows] self.write("window_tapering", ip, metadata=metadata)
def load_window_config(param): config_dict = {} for key, value in param.iteritems(): config_dict[key] = pyflex.Config(**value) return config_dict