def test_station_information_extraction():
"""
Station information can either be passed or read from sac files.
"""
import obspy.station
config = pyflex.Config(min_period=50.0, max_period=150.0)
# If not passed, it is read from sac files, if available.
ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config)
assert abs(ws.station.latitude - 37.930401) < 1E-5
assert abs(ws.station.longitude - 58.1189) < 1E-5
# The other option is an inventory object. Assemble a dummy one.
inv = obspy.station.Inventory(networks=[], source="local")
net = obspy.station.Network(code=OBS_DATA[0].stats.network)
sta = obspy.station.Station(code=OBS_DATA[0].stats.station,
latitude=1.0,
longitude=2.0,
elevation=3.0)
inv.networks = [net]
net.stations = [sta]
ws = pyflex.window_selector.WindowSelector(OBS_DATA,
SYNTH_DATA,
config,
station=inv)
assert ws.station == pyflex.Station(1.0, 2.0)
def test_custom_weight_function():
"""
Test the custom weight function. Set the weight of every window with a
CC of smaller then 95 to 0.0.
"""
def weight_function(win):
if win.max_cc_value < 0.95:
return 0.0
else:
return 10.0
config = pyflex.Config(
min_period=50.0, max_period=150.0,
stalta_waterlevel=0.08, tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0, cc_acceptance_level=0.80,
c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0,
window_weight_fct=weight_function)
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
assert np.all(np.array([_i.max_cc_value for _i in windows]) >= 0.95)
# Not setting it will result in the default value.
config.window_weight_fct = None
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
assert bool(np.all(np.array([_i.max_cc_value for _i in windows]) >=
0.95)) is False
def test_settings_arrays_as_config_values():
"""
Tests that arrays can be set as config values.
"""
npts = OBS_DATA[0].stats.npts
stalta_waterlevel = 0.08 * np.ones(npts)
tshift_acceptance_level = 15.0 * np.ones(npts)
dlna_acceptance_level = 1.0 * np.ones(npts)
cc_acceptance_level = 0.80 * np.ones(npts)
s2n_limit = 1.5 * np.ones(npts)
config = pyflex.Config(min_period=50.0,
max_period=150.0,
stalta_waterlevel=stalta_waterlevel,
tshift_acceptance_level=tshift_acceptance_level,
dlna_acceptance_level=dlna_acceptance_level,
cc_acceptance_level=cc_acceptance_level,
s2n_limit=s2n_limit,
c_0=0.7,
c_1=4.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0)
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
assert len(windows) == 9
def test_window_selection():
"""
This WILL need to be adjusted if any part of the algorithm changes!
The settings for this test are more or less the same as for the test
data example in the original FLEXWIN package.
"""
config = pyflex.Config(min_period=50.0,
max_period=150.0,
stalta_waterlevel=0.08,
tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0,
cc_acceptance_level=0.80,
c_0=0.7,
c_1=4.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0)
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
assert len(windows) == 9
lefties = np.array([_i.left for _i in windows])
righties = np.array([_i.right for _i in windows])
np.testing.assert_allclose(
lefties,
np.array([1551, 2221, 2709, 2960, 3353, 3609, 3983, 4715, 4962]),
atol=3)
np.testing.assert_allclose(
righties,
np.array([1985, 2709, 2960, 3172, 3609, 3920, 4442, 4962, 5207]),
atol=3)
np.testing.assert_allclose(np.array([_i.max_cc_value for _i in windows]),
np.array([
0.95740629, 0.96646804, 0.96335716,
0.98249547, 0.96838754, 0.88501979,
0.82529382, 0.92953344, 0.92880873
]),
rtol=1E-2)
assert [_i.cc_shift for _i in windows] == [-3, 0, -5, -5, -6, 4, -9, -1, 7]
np.testing.assert_allclose(np.array([_i.dlnA for _i in windows]),
np.array([
0.07469, 0.12808, -0.19277, 0.185563,
0.093674, -0.118859, -0.638657, 0.25942,
0.106571
]),
rtol=1E-2)
# Assert the phases of the first window.
assert sorted([_i["phase_name"] for _i in windows[0].phase_arrivals]) == \
['PKIKP', 'PKIKS', 'PKiKP', 'PP', 'SKIKP', 'SKiKP', 'pPKIKP', 'pPKiKP',
'sPKIKP', 'sPKiKP']
def test_window_plotting(tmpdir):
reset_matplotlib()
config = pyflex.Config(
min_period=50.0, max_period=150.0,
stalta_waterlevel=0.08, tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0, cc_acceptance_level=0.80,
c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0)
pyflex.select_windows(OBS_DATA, SYNTH_DATA, config, plot=True)
images_are_identical("picked_windows", str(tmpdir))
def select_windows(parameters, paths, merge_flag, obs_tag, syn_tag):
"""
Selects windows by comparing observed and synthetic traces;
writes results to a json file
:param parameters: dictionary passed directly to pyflex.Config
:param paths.obs: ASDF observed data filename
:param paths.syn: ASDF synthetic data filename
:param paths.output: windows will be written to a JSON file with this name
:param paths.log: information about the quantity and quality of windows
will be written to a JSON file with this name
:param obs_tag: observed data are read using this ASDF tag
:param syn_tag: synthetic data are read using this ASDF tag
"""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.rank
cwd = dirname(__file__)
# read data
fullpath = join(cwd, paths.obs)
obs = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a")
event = obs.events[0]
# read synthetics
fullpath = join(cwd, paths.syn)
syn = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a")
# generate pyflex.Config objects
config = {}
for channel, param in parameters.items():
config[channel] = pyflex.Config(**param)
# wrapper is required for ASDF processing
def wrapped_function(obs, syn):
return pytomo3d.window.window_on_stream(obs[obs_tag],
syn[syn_tag],
config,
station=obs.StationXML,
event=event,
user_modules=None,
figure_mode=False,
figure_dir=None,
_verbose=False)
# run window selection
windows = obs.process_two_files_without_parallel_output(
syn, wrapped_function)
# save results
if rank == 0:
if merge_flag: windows = merge(windows)
write_windows_json(paths.output, windows)
def test_cc_config_setting():
"""
Make sure setting the CC threshold does something.
"""
config = pyflex.Config(
min_period=50.0, max_period=150.0,
stalta_waterlevel=0.08, tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0, cc_acceptance_level=0.95,
c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0)
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
assert np.all(np.array([_i.max_cc_value for _i in windows]) >= 0.95)
def test_window_merging_strategy():
"""
Pyflex can also merge windows.
"""
config = pyflex.Config(
min_period=50.0, max_period=150.0,
stalta_waterlevel=0.08, tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0, cc_acceptance_level=0.80,
c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0,
resolution_strategy="merge")
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
assert len(windows) == 4
def test_run_with_data_quality_checks():
"""
Run with data quality checks.
"""
config = pyflex.Config(
min_period=50.0, max_period=150.0,
stalta_waterlevel=0.08, tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0, cc_acceptance_level=0.80,
c_0=0.7, c_1=4.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0,
check_global_data_quality=True)
windows = pyflex.select_windows(OBS_DATA, SYNTH_DATA, config)
# The data in this case is so good that nothing should have changed.
assert len(windows) == 9
def load_window_config_yaml(filename):
"""
Load yaml and setup pyflex.Config object
:param filename:
:return:
"""
with open(filename) as fh:
data = yaml.load(fh)
if data["min_period"] > data["max_period"]:
raise ValueError("min_period is larger than max_period in config "
"file: %s" % filename)
return pyflex.Config(**data)
def test_runs_without_event_information(recwarn):
"""
Make sure it runs without event information. Some things will not work
but it will at least not crash.
"""
config = pyflex.Config(min_period=50.0,
max_period=150.0,
stalta_waterlevel=0.08,
tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0,
cc_acceptance_level=0.80,
c_0=0.7,
c_1=4.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0)
obs = OBS_DATA[0].copy()
syn = SYNTH_DATA[0].copy()
# Remove the sac header information.
del obs.stats.sac
del syn.stats.sac
recwarn.clear()
windows = pyflex.select_windows(obs, syn, config)
# This will actually result in a bunch more windows as before. So it
# is always a good idea to specify the event and station information!
assert len(windows) == 12
assert len(recwarn.list) == 1
w = recwarn.list[0]
assert w.category == pyflex.PyflexWarning
assert "Event and/or station information is not available".lower() in \
str(w.message).lower()
# No phases should be attached as they cannot be calculated.
phases = []
for win in windows:
phases.extend(win.phase_arrivals)
assert phases == []
def load_window_config(param):
config_dict = {}
flag_list = []
for key, value in param.iteritems():
# pop the "instrument_merge_flag" value out
flag_list.append(value["instrument_merge_flag"])
value.pop("instrument_merge_flag")
check_param_with_function_args(value)
config_dict[key] = pyflex.Config(**value)
if not all(_e == flag_list[0] for _e in flag_list):
raise ValueError("Instrument_merge_flag not consistent amonge"
"different parameter yaml files(%s). Check!"
% flag_list)
return config_dict, flag_list[0]
def test_event_information_extraction():
"""
Event information can either be passed or read from sac files.
"""
config = pyflex.Config(min_period=50.0, max_period=150.0)
# If not passed, it is read from sac files, if available.
ws = pyflex.window_selector.WindowSelector(OBS_DATA, SYNTH_DATA, config)
assert abs(ws.event.latitude - -3.77) <= 1E-5
assert abs(ws.event.longitude - -77.07) <= 1E-5
assert abs(ws.event.depth_in_m - 112800.00305) <= 1E-5
assert ws.event.origin_time == \
obspy.UTCDateTime(1995, 5, 2, 6, 6, 13, 900000)
# If it passed, the passed event will be used.
ev = pyflex.Event(1, 2, 3, obspy.UTCDateTime(2012, 1, 1))
ws = pyflex.window_selector.WindowSelector(OBS_DATA,
SYNTH_DATA,
config,
event=ev)
assert ws.event == ev
# Alternatively, an ObsPy Catalog or Event object can be passed which
# opens the gate to more complex workflows.
cat = obspy.readEvents()
cat.events = cat.events[:1]
event = cat[0]
ev = pyflex.Event(event.origins[0].latitude, event.origins[0].longitude,
event.origins[0].depth, event.origins[0].time)
# Test catalog.
ws = pyflex.window_selector.WindowSelector(OBS_DATA,
SYNTH_DATA,
config,
event=cat)
assert ws.event == ev
# Test event.
ws = pyflex.window_selector.WindowSelector(OBS_DATA,
SYNTH_DATA,
config,
event=cat[0])
assert ws.event == ev
def compute_windows(conf_file):
conf = load_config(conf_file)
if conf.window.use_pyflex:
if conf.window.pyflex_conf_file is None:
raise Exception(
"pyflex_conf_file should be set to compute windows.") # NOQA
logger = logging.getLogger("pyflex")
logger.setLevel(logging.ERROR)
pyflex_args = utils.load_yaml(conf.window.pyflex_conf_file)
pyflex_conf = pyflex.Config(**pyflex_args)
mpi.run_with_mpi(
partial(compute_windows_pyflex, pyflex_conf=pyflex_conf,
conf=conf), conf.get_event_names())
elif conf.window.max_vel == 0 or conf.window.min_vel == 0:
raise Exception(
"Max & min velocity should be set for window selection.") # NOQA
else:
mpi.run_with_mpi(partial(compute_windows_by_vel, conf=conf),
conf.get_event_names())
outputbasedir = os.path.join(basedir, "window")
datadir = os.path.join(databasedir, eventdir)
syntdir = os.path.join(syntbasedir, eventdir)
outputdir = os.path.join(outputbasedir, eventdir)
if not os.path.exists(outputdir):
os.makedirs(outputdir)
figdir = os.path.join(outputdir, "figures")
if not os.path.exists(figdir):
os.makedirs(figdir)
config_bw = pyflex.Config(
min_period=27.0, max_period=60.0,
stalta_waterlevel=0.10, tshift_acceptance_level=10.0,
dlna_acceptance_level=0.80, cc_acceptance_level=0.80, s2n_limit=2.0,
check_global_data_quality=True,
c_0=0.7, c_1=2.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0,
window_signal_to_noise_type="amplitude")
config_sw = pyflex.Config(
min_period=60.0, max_period=120.0,
stalta_waterlevel=0.10, tshift_acceptance_level=18.0,
dlna_acceptance_level=0.80, cc_acceptance_level=0.80, s2n_limit=2.0,
check_global_data_quality=True,
c_0=0.7, c_1=3.0, c_2=0.0, c_3a=1.0, c_3b=2.0, c_4a=3.0, c_4b=10.0,
window_signal_to_noise_type="amplitude")
if (max(period_band)<65):
print "Body wave config"
#body wave(short period)
new_config.cc_acceptance_level = cc
new_config.s2n_limit = s2n
new_config.signal_end_index = r_index
return new_config
conf = pyflex.Config(
40, 80,
stalta_waterlevel=0.08,
tshift_acceptance_level=10.0,
tshift_reference=0.0,
dlna_acceptance_level=0.8,
cc_acceptance_level=0.85,
check_global_data_quality=True,
snr_integrate_base=3.5,
snr_max_base=12.0,
c_0=0.7,
c_1=3.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0
)
ds_obsd = pyasdf.ASDFDataSet("observed.h5")
ds_synt = pyasdf.ASDFDataSet("synthetic_more_perturbed.h5")
event = ds_synt.events[0]
origin = event.preferred_origin()
synt_tag = "synthetic"
obsd_tag = "synthetic"
def test_reading_and_writing_windows(tmpdir):
"""
Tests reading and writing of window sets.
"""
config = pyflex.Config(min_period=50.0,
max_period=150.0,
stalta_waterlevel=0.08,
tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0,
cc_acceptance_level=0.80,
c_0=0.7,
c_1=4.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0)
ws = pyflex.WindowSelector(observed=OBS_DATA,
synthetic=SYNTH_DATA,
config=config)
windows = ws.select_windows()
assert len(windows) > 0
# Write/read to/from file.
filename = os.path.join(str(tmpdir), "window.json")
ws.write(filename)
ws2 = pyflex.WindowSelector(observed=OBS_DATA,
synthetic=SYNTH_DATA,
config=config)
ws2.load(filename)
assert ws.windows == ws2.windows
os.remove(filename)
# Write/read to/from open file.
with open(filename, "w") as fh:
ws.write(fh)
ws2 = pyflex.WindowSelector(observed=OBS_DATA,
synthetic=SYNTH_DATA,
config=config)
fh.seek(0, 0)
ws2.load(filename)
assert ws.windows == ws2.windows
# Write/read to/from StringIO.
with io.StringIO() as fh:
ws.write(fh)
ws2 = pyflex.WindowSelector(observed=OBS_DATA,
synthetic=SYNTH_DATA,
config=config)
fh.seek(0, 0)
ws2.load(filename)
assert ws.windows == ws2.windows
# Write/read to/from BytesIO.
with io.BytesIO() as fh:
ws.write(fh)
ws2 = pyflex.WindowSelector(observed=OBS_DATA,
synthetic=SYNTH_DATA,
config=config)
fh.seek(0, 0)
ws2.load(filename)
assert ws.windows == ws2.windows
# preprocessing
obs_st.detrend("linear")
obs_st.taper(max_percentage=0.05, type="hann")
obs_st.filter("lowpass", freq=0.25, corners=2, zerophase=True)
syn_st.detrend("linear")
syn_st.taper(max_percentage=0.05, type="hann")
syn_st.filter("lowpass", freq=0.25, corners=2, zerophase=True)
# user defined configuration
config = pyflex.Config(min_period=1.0,
max_period=100.0,
stalta_waterlevel=0.08,
tshift_acceptance_level=15.0,
dlna_acceptance_level=1.0,
cc_acceptance_level=0.50,
c_0=0.7,
c_1=4.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0)
windows = pyflex.select_windows(obs_st, syn_st, config, plot=True)
pprint.pprint(windows)
#win = windows[0]
# print("Relative times in seconds: %s - %s" % (win.relative_starttime,
# win.relative_endtime))
event = ds.events[0]
def weight_function(win):
return win.max_cc_value
config = pyflex.Config(min_period=27.0,
max_period=60.0,
stalta_waterlevel=0.11,
tshift_acceptance_level=15.0,
dlna_acceptance_level=2.5,
cc_acceptance_level=0.6,
c_0=0.7,
c_1=2.0,
c_2=0.0,
c_3a=1.0,
c_3b=2.0,
c_4a=3.0,
c_4b=10.0,
s2n_limit=0.5,
max_time_before_first_arrival=-50.0,
min_surface_wave_velocity=3.0,
window_signal_to_noise_type="energy",
window_weight_fct=weight_function)
def process(this_station_group, other_station_group):
# Make sure everything thats required is there.
if not hasattr(this_station_group, "StationXML") or \
not hasattr(this_station_group, "preprocessed_27s_to_60s") or \
not hasattr(other_station_group,
def _process(self, inputs):
# Weighting function for the windows. This particular choice will tend
# to favor lots of smaller windows if "interval_scheduling" is chosen
# as overlap resolution strategy.
ip = inputs["input"]
param = ip["parameters"]
config = pyflex.Config(
min_period=param["min_period"],
max_period=param["max_period"],
stalta_waterlevel=param["stalta_waterlevel"],
s2n_limit=param["s2n_limit"],
snr_max_base=param["snr_max_base"],
tshift_acceptance_level=param["tshift_acceptance_level"],
tshift_reference=param["tshift_reference"],
dlna_acceptance_level=param["dlna_acceptance_level"],
dlna_reference=param["dlna_reference"],
cc_acceptance_level=param["cc_acceptance_level"],
earth_model=param["earth_model"],
min_surface_wave_velocity=param["min_surface_wave_velocity"],
max_time_before_first_arrival=param[
"max_time_before_first_arrival"],
c_0=param["c_0"],
c_1=param["c_1"],
c_2=param["c_2"],
c_3a=param["c_3a"],
c_3b=param["c_3b"],
c_4a=param["c_4a"],
c_4b=param["c_4b"],
check_global_data_quality=param["check_global_data_quality"],
snr_integrate_base=param["snr_integrate_base"],
noise_start_index=param["noise_start_index"],
noise_end_index=param["noise_end_index"],
signal_start_index=param["signal_start_index"],
signal_end_index=param["signal_end_index"],
window_signal_to_noise_type=param["window_signal_to_noise_type"],
resolution_strategy=param["resolution_strategy"],
window_weight_fct=weighting_function)
ws = WindowSelector(observed=ip["data_trace"],
synthetic=ip["synthetic_trace"],
config=config,
event=[_i for _i in obspy.readEvents(ip["quakeml"])
if _i.resource_id.id == ip["event_id"]][0],
station=obspy.read_inventory(
ip["stationxml"], format="stationxml"))
windows = ws.select_windows()
data = ip["data_trace"]
station_id = "%s.%s" % (data.stats.network,
data.stats.station)
component = data.stats.channel[-1]
# HACK!
with io.BytesIO() as _:
ws.plot(_)
fig = plt.gcf()
fig.suptitle("Component: %s" % component, fontsize=15,
horizontalalignment="center", x=0.8, y=1)
with io.BytesIO() as buf:
fig.savefig(buf)
buf.seek(0, 0)
# Not Python 3 compatible but who cares.
image_string = buf.read()
plt.close(fig)
image_type = "pyflex_windows"
# Metadata for Alessandro!
metadata = [{"type": "pyflex",
"station_id": station_id,
"component": component,
"channel_id": win.channel_id,
"starttime": str(win.absolute_starttime),
"endtime": str(win.absolute_endtime),
"max_cc_value": win.max_cc_value,
"cc_shift": win.cc_shift,
"dlnA": win.dlnA} for win in windows]
self.parameters=param
self.write(
"image",
({"image_string": image_string, "component": component,
"output_folder": ip["output_folder"]},
image_type, station_id),metadata=metadata)
#control={'con:skip':True}
# self.write(
# "windows", {
# "record_type": "pyflex_windows",
# "output_folder": ip["output_folder"],
# "windows": windows,
# "station_id": station_id,
# "component": component},metadata=metadata)
if ip["misfit_type"] == "pyflex_and_time_frequency":
ip["windows"] = [{
"starttime": win.absolute_starttime,
"endtime": win.absolute_endtime} for win in windows]
self.write("window_tapering", ip, metadata=metadata)
def load_window_config(param):
config_dict = {}
for key, value in param.iteritems():
config_dict[key] = pyflex.Config(**value)
return config_dict
