def test_preprocessing_runs(project):
"""
Simple tests to assure the preprocessing actually runs. Does not test if it
does the right thing but will at least assure the program flow works as
expected.
"""
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
processing_tag = project._get_iteration("1").get_processing_tag()
event_data_dir = os.path.join(
project.paths["data"], "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11")
processing_dir = os.path.join(event_data_dir, processing_tag)
assert not os.path.exists(processing_dir)
# This will process only one event.
project.preprocess_data("1", ["GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"],
waiting_time=0.0)
assert os.path.exists(processing_dir)
assert len(os.listdir(processing_dir)) == 4
# Remove and try again, this time not specifying the event which will
# simply use all events. Should have the same result.
shutil.rmtree(processing_dir)
assert not os.path.exists(processing_dir)
project.preprocess_data("1", waiting_time=0.0)
assert os.path.exists(processing_dir)
assert len(os.listdir(processing_dir)) == 4
def test_iteration_status(project):
"""
Tests the iteration status commands.
"""
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
event = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"
# Currenty the project has 4 files, that are not preprocessed.
status = project.get_iteration_status("1")
assert len(status["channels_not_yet_preprocessed"]) == 4
assert status["stations_in_iteration_that_do_not_exist"] == []
# The project only has synthetics for two stations.
assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA",
"KO.RSDY"]
# Preprocess some files.
project.preprocess_data("1", [event],
waiting_time=0.0)
status = project.get_iteration_status("1")
assert status["channels_not_yet_preprocessed"] == []
assert status["stations_in_iteration_that_do_not_exist"] == []
assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA",
"KO.RSDY"]
# Remove one of the waveform files. This has the effect that the iteration
# contains a file that is not actually in existance. This should be
# detected.
proc_folder = os.path.join(
project.paths["data"], event,
project._get_iteration("1").get_processing_tag())
data_folder = os.path.join(project.paths["data"], event, "raw")
data_file = sorted(glob.glob(os.path.join(data_folder, "*")))[0]
proc_file = sorted(glob.glob(os.path.join(proc_folder, "*")))[0]
os.remove(data_file)
os.remove(proc_file)
status = project.get_iteration_status("1")
assert status["channels_not_yet_preprocessed"] == []
assert len(status["stations_in_iteration_that_do_not_exist"]) == 1
assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA",
"KO.RSDY"]
# Now remove all synthetics. This should have the result that all
# synthetics are missing.
for folder in os.listdir(project.paths["synthetics"]):
shutil.rmtree(os.path.join(project.paths["synthetics"], folder))
status = project.get_iteration_status("1")
assert status["channels_not_yet_preprocessed"] == []
assert len(status["stations_in_iteration_that_do_not_exist"]) == 1
# HL.ARG has been remove before.
assert sorted(status["synthetic_data_missing"][event]) == \
["HT.SIGR", "KO.KULA", "KO.RSDY"]
def test_discover_available_data(project):
"""
Tests the discover available data method.
"""
event = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"
# At the beginning it contains nothing, except a raw vertical component
assert project.discover_available_data(event, "HL.ARG") == \
{"processed": {}, "synthetic": {}, "raw": {"raw": ["Z"]}}
# Create a new iteration. At this point it should contain some synthetics.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
assert project.discover_available_data(event, "HL.ARG") == \
{"processed": {},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
# A new iteration without data does not add anything.
project.create_new_iteration("2", "ses3d_4_1", 8, 100)
assert project.discover_available_data(event, "HL.ARG") == \
{"processed": {},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
# Data is also available for a second station. But not for another one.
assert project.discover_available_data(event, "HT.SIGR") == \
{"processed": {},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
assert project.discover_available_data(event, "KO.KULA") == \
{"processed": {},
"synthetic": {},
"raw": {"raw": ["Z"]}}
# Requesting data for a non-existent station raises.
with pytest.raises(LASIFException):
project.discover_available_data(event, "NET.STA")
# Now preprocess some data that then should appear.
processing_tag = project._get_iteration("1").get_processing_tag()
project.preprocess_data("1", [event], waiting_time=0.0)
assert project.discover_available_data(event, "HT.SIGR") == \
{"processed": {processing_tag: ["Z"]},
"synthetic": {"1": ["Z", "N", "E"]},
"raw": {"raw": ["Z"]}}
assert project.discover_available_data(event, "KO.KULA") == \
{"processed": {processing_tag: ["Z"]},
"synthetic": {},
"raw": {"raw": ["Z"]}}
def test_data_synthetic_iterator(project, recwarn):
"""
Tests that the data synthetic iterator works as expected.
"""
# It requires an existing iteration with processed data.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
project.preprocess_data("1", "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11",
waiting_time=0.0)
iterator = project.data_synthetic_iterator(
"GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11", "1")
# The example project only contains synthetics for two stations.
expected = {
"HL.ARG..BHZ": {"latitude": 36.216, "local_depth_in_m": 0.0,
"elevation_in_m": 170.0, "longitude": 28.126},
"HT.SIGR..HHZ": {"latitude": 39.2114, "local_depth_in_m": 0.0,
"elevation_in_m": 93.0, "longitude": 25.8553}}
found = []
no_synthetics_found_count = 0
assert recwarn.list == []
for _i in iterator:
if _i is None:
# If the data is not found it should always warn.
w = recwarn.pop(UserWarning)
assert "No synthetics found" in w.message.args[0]
no_synthetics_found_count += 1
continue
data = _i["data"]
synth = _i["synthetics"]
coods = _i["coordinates"]
assert recwarn.list == []
# Only one real data is present at all times.
assert len(data) == 1
# Three synthetic components.
assert len(synth) == 3
found.append(data[0].id)
assert data[0].id in expected
assert expected[data[0].id] == coods
assert sorted(found) == sorted(expected.keys())
# No synthetics exists for the other two.
assert no_synthetics_found_count == 2
def test_coordinate_retrieval(project):
"""
Tests the retrieval of coordinates.
"""
event_name = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"
# Remove the station file for KO_KULA. This is necessary for some tests
# later on.
os.remove(os.path.join(project.paths["dataless_seed"],
"dataless.KO_KULA"))
# Force a rebuilding of the station cache. This usually only happens once.
project._Project__update_station_cache()
# The first two files have coordinates from SEED.
filename = os.path.join(project.paths["data"], event_name, "raw",
"HL.ARG..BHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "raw", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["data"], event_name, "raw",
"HT.SIGR..HHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "raw", "HT", "SIGR", event_name) == \
{"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0,
"longitude": 25.8553}
# This should also work for the synthetics! In that case the coordinates
# are also found from the SEED files.
filename = os.path.join(project.paths["synthetics"], event_name,
"ITERATION_1", "HL.ARG__.___.x")
assert project._get_coordinates_for_waveform_file(
filename, "synthetic", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["synthetics"], event_name,
"ITERATION_1", "HL.ARG__.___.z")
assert project._get_coordinates_for_waveform_file(
filename, "synthetic", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["synthetics"], event_name,
"ITERATION_1", "HT.SIGR_.___.y")
assert project._get_coordinates_for_waveform_file(
filename, "synthetic", "HT", "SIGR", event_name) == \
{"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0,
"longitude": 25.8553}
# Also with processed data. We thus need to create a new iteration and
# create processed data.
project.create_new_iteration("1", "ses3d_4_1", 8, 100)
project.preprocess_data("1", [event_name], waiting_time=0.0)
processing_tag = project._get_iteration("1").get_processing_tag()
filename = os.path.join(project.paths["data"], event_name, processing_tag,
"HL.ARG..BHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "processed", "HL", "ARG", event_name) == \
{"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0,
"longitude": 28.126}
filename = os.path.join(project.paths["data"], event_name, processing_tag,
"HT.SIGR..HHZ.mseed")
assert project._get_coordinates_for_waveform_file(
filename, "processed", "HT", "SIGR", event_name) == \
{"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0,
"longitude": 25.8553}
# The file exists, but has no corresponding station file.
filename = os.path.join(project.paths["data"], event_name, "raw",
"KO.KULA..BHZ.mseed")
# Now check what happens if not station coordinate file is available. It
# will first attempt retrieve files from the waveform cache which will only
# be filled if it is a SAC file.
result = {"latitude": 1, "local_depth_in_m": 2, "elevation_in_m": 3,
"longitude": 4}
with mock.patch("lasif.tools.waveform_cache.WaveformCache.get_details") \
as patch:
patch.return_value = [result]
assert project._get_coordinates_for_waveform_file(
filename, "raw", "KO", "KULA", event_name) == result
assert patch.called_once_with("KO", "KULA")
# Otherwise the inventory database will be called.
with mock.patch("lasif.tools.inventory_db.get_station_coordinates") \
as patch:
patch.return_value = result
assert project._get_coordinates_for_waveform_file(
filename, "raw", "KO", "KULA", event_name) == result
patch.assert_called_once()