def test_input_file_invocation(project):
"""
Tests if the input file generation actually creates some files and works in
the first place.
Does not test the input files. That is the responsibility of the input file
generator module.
"""
assert os.listdir(project.paths["output"]) == []
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
# Normal simulation.
project.generate_input_files(
"1", "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11", "normal simulation")
output_dir = [_i for _i in os.listdir(project.paths["output"])
if "normal_simulation" in _i][0]
assert len(os.listdir(os.path.join(
project.paths["output"], output_dir))) != 0
# Adjoint forward.
project.generate_input_files(
"1", "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11", "adjoint forward")
output_dir = [_i for _i in os.listdir(project.paths["output"])
if "adjoint_forward" in _i][0]
assert len(os.listdir(os.path.join(
project.paths["output"], output_dir))) != 0
# Adjoint reverse.
project.generate_input_files(
"1", "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11", "adjoint reverse")
output_dir = [_i for _i in os.listdir(project.paths["output"])
if "adjoint_reverse" in _i][0]
assert len(os.listdir(os.path.join(
project.paths["output"], output_dir))) != 0
def test_preprocessing_runs(project):
"""
Simple tests to assure the preprocessing actually runs. Does not test if it
does the right thing but will at least assure the program flow works as
expected.
"""
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
processing_tag = project._get_iteration("1").get_processing_tag()
event_data_dir = os.path.join(
project.paths["data"], "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11")
processing_dir = os.path.join(event_data_dir, processing_tag)
assert not os.path.exists(processing_dir)
# This will process only one event.
project.preprocess_data("1", ["GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"],
waiting_time=0.0)
assert os.path.exists(processing_dir)
assert len(os.listdir(processing_dir)) == 4
# Remove and try again, this time not specifying the event which will
# simply use all events. Should have the same result.
shutil.rmtree(processing_dir)
assert not os.path.exists(processing_dir)
project.preprocess_data("1", waiting_time=0.0)
assert os.path.exists(processing_dir)
assert len(os.listdir(processing_dir)) == 4
def test_iteration_status(project):
"""
Tests the iteration status commands.
"""
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
event = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"
# Currenty the project has 4 files, that are not preprocessed.
status = project.get_iteration_status("1")
assert len(status["channels_not_yet_preprocessed"]) == 4
assert status["stations_in_iteration_that_do_not_exist"] == []
# The project only has synthetics for two stations.
assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA",
"KO.RSDY"]
# Preprocess some files.
project.preprocess_data("1", [event],
waiting_time=0.0)
status = project.get_iteration_status("1")
assert status["channels_not_yet_preprocessed"] == []
assert status["stations_in_iteration_that_do_not_exist"] == []
assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA",
"KO.RSDY"]
# Remove one of the waveform files. This has the effect that the iteration
# contains a file that is not actually in existance. This should be
# detected.
proc_folder = os.path.join(
project.paths["data"], event,
project._get_iteration("1").get_processing_tag())
data_folder = os.path.join(project.paths["data"], event, "raw")
data_file = sorted(glob.glob(os.path.join(data_folder, "*")))[0]
proc_file = sorted(glob.glob(os.path.join(proc_folder, "*")))[0]
os.remove(data_file)
os.remove(proc_file)
status = project.get_iteration_status("1")
assert status["channels_not_yet_preprocessed"] == []
assert len(status["stations_in_iteration_that_do_not_exist"]) == 1
assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA",
"KO.RSDY"]
# Now remove all synthetics. This should have the result that all
# synthetics are missing.
for folder in os.listdir(project.paths["synthetics"]):
shutil.rmtree(os.path.join(project.paths["synthetics"], folder))
status = project.get_iteration_status("1")
assert status["channels_not_yet_preprocessed"] == []
assert len(status["stations_in_iteration_that_do_not_exist"]) == 1
# HL.ARG has been remove before.
assert sorted(status["synthetic_data_missing"][event]) == \
["HT.SIGR", "KO.KULA", "KO.RSDY"]
def test_Q_model_plotting(project):
"""
Tests the Q model plotting.
"""
project.create_new_iteration("1", "ses3d_4_1", 11, 111)
with mock.patch("lasif.tools.Q_discrete.plot") as patch:
project.plot_Q_model("1")
patch.assert_called_once()
kwargs = patch.call_args[1]
assert round(kwargs["f_min"] - 1.0 / 111.0, 5) == 0
assert round(kwargs["f_max"] - 1.0 / 11.0, 5) == 0
def test_discover_available_data(project):
"""
Tests the discover available data method.
"""
event = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"
# At the beginning it contains nothing, except a raw vertical component
assert project.discover_available_data(event, "HL.ARG") == \
{"processed": {}, "synthetic": {}, "raw": {"raw": ["Z"]}}
# Create a new iteration. At this point it should contain some synthetics.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
assert project.discover_available_data(event, "HL.ARG") == \
{"processed": {},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
# A new iteration without data does not add anything.
project.create_new_iteration("2", "ses3d_4_1", 8, 100)
assert project.discover_available_data(event, "HL.ARG") == \
{"processed": {},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
# Data is also available for a second station. But not for another one.
assert project.discover_available_data(event, "HT.SIGR") == \
{"processed": {},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
assert project.discover_available_data(event, "KO.KULA") == \
{"processed": {},
"synthetic": {},
"raw": {"raw": ["Z"]}}
# Requesting data for a non-existent station raises.
with pytest.raises(LASIFException):
project.discover_available_data(event, "NET.STA")
# Now preprocess some data that then should appear.
processing_tag = project._get_iteration("1").get_processing_tag()
project.preprocess_data("1", [event], waiting_time=0.0)
assert project.discover_available_data(event, "HT.SIGR") == \
{"processed": {processing_tag: ["Z"]},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
assert project.discover_available_data(event, "KO.KULA") == \
{"processed": {processing_tag: ["Z"]},
"synthetic": {},
"raw": {"raw": ["Z"]}}
def test_data_synthetic_iterator(project, recwarn):
"""
Tests that the data synthetic iterator works as expected.
"""
# It requires an existing iteration with processed data.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
project.preprocess_data("1", "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11",
waiting_time=0.0)
iterator = project.data_synthetic_iterator(
"GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11", "1")
# The example project only contains synthetics for two stations.
expected = {
"HL.ARG..BHZ": {"latitude": 36.216, "local_depth_in_m": 0.0,
"elevation_in_m": 170.0, "longitude": 28.126},
"HT.SIGR..HHZ": {"latitude": 39.2114, "local_depth_in_m": 0.0,
"elevation_in_m": 93.0, "longitude": 25.8553}}
found = []
no_synthetics_found_count = 0
assert recwarn.list == []
for _i in iterator:
if _i is None:
# If the data is not found it should always warn.
w = recwarn.pop(UserWarning)
assert "No synthetics found" in w.message.args[0]
no_synthetics_found_count += 1
continue
data = _i["data"]
synth = _i["synthetics"]
coods = _i["coordinates"]
assert recwarn.list == []
# Only one real data is present at all times.
assert len(data) == 1
# Three synthetic components.
assert len(synth) == 3
found.append(data[0].id)
assert data[0].id in expected
assert expected[data[0].id] == coods
assert sorted(found) == sorted(expected.keys())
# No synthetics exists for the other two.
assert no_synthetics_found_count == 2
def test_iteration_handling(project):
"""
Tests the managing of the iterations.
"""
# First create two iterations.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
project.create_new_iteration("2", "ses3d_4_1", 8, 100)
assert sorted(os.listdir(project.paths["iterations"])) == \
sorted(["ITERATION_1.xml", "ITERATION_2.xml"])
# Make sure they are found correctly.
assert project.get_iteration_dict() == {key: os.path.join(
project.paths["iterations"], "ITERATION_" + key + ".xml")
for key in ["1", "2"]}
iteration = project._get_iteration("1")
# Assert that the aspects of the example project did get picked up by the
# iteration. Only one event will be available as the other is empty.
assert len(iteration.events) == 1
assert len(iteration.events["GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"]
["stations"]) == 4
assert iteration.iteration_name == "1"
assert iteration.source_time_function == "Filtered Heaviside"
assert iteration.data_preprocessing["lowpass_period"] == 8.0
assert iteration.data_preprocessing["highpass_period"] == 100.0
# Assert the processing parameters. This is somewhat redundant and should
# rather be tested in the iteration test suite.
process_params = iteration.get_process_params()
assert process_params["npts"] == 500
assert process_params["dt"] == 0.75
assert process_params["stf"] == "Filtered Heaviside"
assert process_params["lowpass"] == 0.125
assert process_params["highpass"] == 0.01
def test_generating_new_iteration(project):
"""
Tests that iteration creation works.
"""
assert os.listdir(project.paths["iterations"]) == []
# Using an invalid solver raises.
with pytest.raises(LASIFException) as excinfo:
project.create_new_iteration("1", "unknown_solver", 8, 100)
msg = excinfo.value.message
assert "not known" in msg
assert "unknown_solver" in msg
# Nothing should have happened.
assert os.listdir(project.paths["iterations"]) == []
# Now actually create a new iteration.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
assert os.listdir(project.paths["iterations"]) == ["ITERATION_1.xml"]
# Creating an already existing iteration raises.
with pytest.raises(LASIFException) as excinfo:
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
assert excinfo.value.message.lower() == "iteration already exists."
def test_coordinate_retrieval(project):
"""
Tests the retrieval of coordinates.
"""
event_name = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"
# Remove the station file for KO_KULA. This is necessary for some tests
# later on.
os.remove(os.path.join(project.paths["dataless_seed"],
"dataless.KO_KULA"))
# Force a rebuilding of the station cache. This usually only happens once.
project._Project__update_station_cache()
# The first two files have coordinates from SEED.
filename = os.path.join(project.paths["data"], event_name, "raw",
"HL.ARG..BHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "raw", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["data"], event_name, "raw",
"HT.SIGR..HHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "raw", "HT", "SIGR", event_name) == \
{"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0,
"longitude": 25.8553}
# This should also work for the synthetics! In that case the coordinates
# are also found from the SEED files.
filename = os.path.join(project.paths["synthetics"], event_name,
"ITERATION_1", "HL.ARG__.___.x")
assert project._get_coordinates_for_waveform_file(
filename, "synthetic", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["synthetics"], event_name,
"ITERATION_1", "HL.ARG__.___.z")
assert project._get_coordinates_for_waveform_file(
filename, "synthetic", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["synthetics"], event_name,
"ITERATION_1", "HT.SIGR_.___.y")
assert project._get_coordinates_for_waveform_file(
filename, "synthetic", "HT", "SIGR", event_name) == \
{"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0,
"longitude": 25.8553}
# Also with processed data. We thus need to create a new iteration and
# create processed data.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
project.preprocess_data("1", [event_name], waiting_time=0.0)
processing_tag = project._get_iteration("1").get_processing_tag()
filename = os.path.join(project.paths["data"], event_name, processing_tag,
"HL.ARG..BHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "processed", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["data"], event_name, processing_tag,
"HT.SIGR..HHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "processed", "HT", "SIGR", event_name) == \
{"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0,
"longitude": 25.8553}
# The file exists, but has no corresponding station file.
filename = os.path.join(project.paths["data"], event_name, "raw",
"KO.KULA..BHZ.mseed")
# Now check what happens if not station coordinate file is available. It
# will first attempt retrieve files from the waveform cache which will only
# be filled if it is a SAC file.
result = {"latitude": 1, "local_depth_in_m": 2, "elevation_in_m": 3,
"longitude": 4}
with mock.patch("lasif.tools.waveform_cache.WaveformCache.get_details") \
as patch:
patch.return_value = [result]
assert project._get_coordinates_for_waveform_file(
filename, "raw", "KO", "KULA", event_name) == result
assert patch.called_once_with("KO", "KULA")
# Otherwise the inventory database will be called.
with mock.patch("lasif.tools.inventory_db.get_station_coordinates") \
as patch:
patch.return_value = result
assert project._get_coordinates_for_waveform_file(
filename, "raw", "KO", "KULA", event_name) == result
patch.assert_called_once()
