def test_preprocessing_runs(project): """ Simple tests to assure the preprocessing actually runs. Does not test if it does the right thing but will at least assure the program flow works as expected. """ project.create_new_iteration("1", "ses3d_4_1", 8, 100) processing_tag = project._get_iteration("1").get_processing_tag() event_data_dir = os.path.join( project.paths["data"], "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11") processing_dir = os.path.join(event_data_dir, processing_tag) assert not os.path.exists(processing_dir) # This will process only one event. project.preprocess_data("1", ["GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"], waiting_time=0.0) assert os.path.exists(processing_dir) assert len(os.listdir(processing_dir)) == 4 # Remove and try again, this time not specifying the event which will # simply use all events. Should have the same result. shutil.rmtree(processing_dir) assert not os.path.exists(processing_dir) project.preprocess_data("1", waiting_time=0.0) assert os.path.exists(processing_dir) assert len(os.listdir(processing_dir)) == 4
def test_iteration_status(project): """ Tests the iteration status commands. """ project.create_new_iteration("1", "ses3d_4_1", 8, 100) event = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11" # Currenty the project has 4 files, that are not preprocessed. status = project.get_iteration_status("1") assert len(status["channels_not_yet_preprocessed"]) == 4 assert status["stations_in_iteration_that_do_not_exist"] == [] # The project only has synthetics for two stations. assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA", "KO.RSDY"] # Preprocess some files. project.preprocess_data("1", [event], waiting_time=0.0) status = project.get_iteration_status("1") assert status["channels_not_yet_preprocessed"] == [] assert status["stations_in_iteration_that_do_not_exist"] == [] assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA", "KO.RSDY"] # Remove one of the waveform files. This has the effect that the iteration # contains a file that is not actually in existance. This should be # detected. proc_folder = os.path.join( project.paths["data"], event, project._get_iteration("1").get_processing_tag()) data_folder = os.path.join(project.paths["data"], event, "raw") data_file = sorted(glob.glob(os.path.join(data_folder, "*")))[0] proc_file = sorted(glob.glob(os.path.join(proc_folder, "*")))[0] os.remove(data_file) os.remove(proc_file) status = project.get_iteration_status("1") assert status["channels_not_yet_preprocessed"] == [] assert len(status["stations_in_iteration_that_do_not_exist"]) == 1 assert sorted(status["synthetic_data_missing"][event]) == ["KO.KULA", "KO.RSDY"] # Now remove all synthetics. This should have the result that all # synthetics are missing. for folder in os.listdir(project.paths["synthetics"]): shutil.rmtree(os.path.join(project.paths["synthetics"], folder)) status = project.get_iteration_status("1") assert status["channels_not_yet_preprocessed"] == [] assert len(status["stations_in_iteration_that_do_not_exist"]) == 1 # HL.ARG has been remove before. assert sorted(status["synthetic_data_missing"][event]) == \ ["HT.SIGR", "KO.KULA", "KO.RSDY"]
def test_discover_available_data(project): """ Tests the discover available data method. """ event = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11" # At the beginning it contains nothing, except a raw vertical component assert project.discover_available_data(event, "HL.ARG") == \ {"processed": {}, "synthetic": {}, "raw": {"raw": ["Z"]}} # Create a new iteration. At this point it should contain some synthetics. project.create_new_iteration("1", "ses3d_4_1", 8, 100) assert project.discover_available_data(event, "HL.ARG") == \ {"processed": {}, "synthetic": {"1": ["Z", "N", "E"]}, "raw": {"raw": ["Z"]}} # A new iteration without data does not add anything. project.create_new_iteration("2", "ses3d_4_1", 8, 100) assert project.discover_available_data(event, "HL.ARG") == \ {"processed": {}, "synthetic": {"1": ["Z", "N", "E"]}, "raw": {"raw": ["Z"]}} # Data is also available for a second station. But not for another one. assert project.discover_available_data(event, "HT.SIGR") == \ {"processed": {}, "synthetic": {"1": ["Z", "N", "E"]}, "raw": {"raw": ["Z"]}} assert project.discover_available_data(event, "KO.KULA") == \ {"processed": {}, "synthetic": {}, "raw": {"raw": ["Z"]}} # Requesting data for a non-existent station raises. with pytest.raises(LASIFException): project.discover_available_data(event, "NET.STA") # Now preprocess some data that then should appear. processing_tag = project._get_iteration("1").get_processing_tag() project.preprocess_data("1", [event], waiting_time=0.0) assert project.discover_available_data(event, "HT.SIGR") == \ {"processed": {processing_tag: ["Z"]}, "synthetic": {"1": ["Z", "N", "E"]}, "raw": {"raw": ["Z"]}} assert project.discover_available_data(event, "KO.KULA") == \ {"processed": {processing_tag: ["Z"]}, "synthetic": {}, "raw": {"raw": ["Z"]}}
def test_iteration_handling(project): """ Tests the managing of the iterations. """ # First create two iterations. project.create_new_iteration("1", "ses3d_4_1", 8, 100) project.create_new_iteration("2", "ses3d_4_1", 8, 100) assert sorted(os.listdir(project.paths["iterations"])) == \ sorted(["ITERATION_1.xml", "ITERATION_2.xml"]) # Make sure they are found correctly. assert project.get_iteration_dict() == {key: os.path.join( project.paths["iterations"], "ITERATION_" + key + ".xml") for key in ["1", "2"]} iteration = project._get_iteration("1") # Assert that the aspects of the example project did get picked up by the # iteration. Only one event will be available as the other is empty. assert len(iteration.events) == 1 assert len(iteration.events["GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"] ["stations"]) == 4 assert iteration.iteration_name == "1" assert iteration.source_time_function == "Filtered Heaviside" assert iteration.data_preprocessing["lowpass_period"] == 8.0 assert iteration.data_preprocessing["highpass_period"] == 100.0 # Assert the processing parameters. This is somewhat redundant and should # rather be tested in the iteration test suite. process_params = iteration.get_process_params() assert process_params["npts"] == 500 assert process_params["dt"] == 0.75 assert process_params["stf"] == "Filtered Heaviside" assert process_params["lowpass"] == 0.125 assert process_params["highpass"] == 0.01
def test_coordinate_retrieval(project): """ Tests the retrieval of coordinates. """ event_name = "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11" # Remove the station file for KO_KULA. This is necessary for some tests # later on. os.remove(os.path.join(project.paths["dataless_seed"], "dataless.KO_KULA")) # Force a rebuilding of the station cache. This usually only happens once. project._Project__update_station_cache() # The first two files have coordinates from SEED. filename = os.path.join(project.paths["data"], event_name, "raw", "HL.ARG..BHZ.mseed") assert project._get_coordinates_for_waveform_file( filename, "raw", "HL", "ARG", event_name) == \ {"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0, "longitude": 28.126} filename = os.path.join(project.paths["data"], event_name, "raw", "HT.SIGR..HHZ.mseed") assert project._get_coordinates_for_waveform_file( filename, "raw", "HT", "SIGR", event_name) == \ {"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0, "longitude": 25.8553} # This should also work for the synthetics! In that case the coordinates # are also found from the SEED files. filename = os.path.join(project.paths["synthetics"], event_name, "ITERATION_1", "HL.ARG__.___.x") assert project._get_coordinates_for_waveform_file( filename, "synthetic", "HL", "ARG", event_name) == \ {"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0, "longitude": 28.126} filename = os.path.join(project.paths["synthetics"], event_name, "ITERATION_1", "HL.ARG__.___.z") assert project._get_coordinates_for_waveform_file( filename, "synthetic", "HL", "ARG", event_name) == \ {"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0, "longitude": 28.126} filename = os.path.join(project.paths["synthetics"], event_name, "ITERATION_1", "HT.SIGR_.___.y") assert project._get_coordinates_for_waveform_file( filename, "synthetic", "HT", "SIGR", event_name) == \ {"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0, "longitude": 25.8553} # Also with processed data. We thus need to create a new iteration and # create processed data. project.create_new_iteration("1", "ses3d_4_1", 8, 100) project.preprocess_data("1", [event_name], waiting_time=0.0) processing_tag = project._get_iteration("1").get_processing_tag() filename = os.path.join(project.paths["data"], event_name, processing_tag, "HL.ARG..BHZ.mseed") assert project._get_coordinates_for_waveform_file( filename, "processed", "HL", "ARG", event_name) == \ {"latitude": 36.216, "local_depth_in_m": 0.0, "elevation_in_m": 170.0, "longitude": 28.126} filename = os.path.join(project.paths["data"], event_name, processing_tag, "HT.SIGR..HHZ.mseed") assert project._get_coordinates_for_waveform_file( filename, "processed", "HT", "SIGR", event_name) == \ {"latitude": 39.2114, "local_depth_in_m": 0.0, "elevation_in_m": 93.0, "longitude": 25.8553} # The file exists, but has no corresponding station file. filename = os.path.join(project.paths["data"], event_name, "raw", "KO.KULA..BHZ.mseed") # Now check what happens if not station coordinate file is available. It # will first attempt retrieve files from the waveform cache which will only # be filled if it is a SAC file. result = {"latitude": 1, "local_depth_in_m": 2, "elevation_in_m": 3, "longitude": 4} with mock.patch("lasif.tools.waveform_cache.WaveformCache.get_details") \ as patch: patch.return_value = [result] assert project._get_coordinates_for_waveform_file( filename, "raw", "KO", "KULA", event_name) == result assert patch.called_once_with("KO", "KULA") # Otherwise the inventory database will be called. with mock.patch("lasif.tools.inventory_db.get_station_coordinates") \ as patch: patch.return_value = result assert project._get_coordinates_for_waveform_file( filename, "raw", "KO", "KULA", event_name) == result patch.assert_called_once()