def test_endtime_on_sample(self): """ [T0, T1), T0 should be included; T1 excluded Test to see whether points on starttime are included, and samples on the endtime are excluded. Total number of samples for this test = 2308 Trace in file runs from [00:01.625000Z to 00:59.300000Z] """ file = os.path.join(self.path, "tiny_quality_file.mseed") # Set T0 and T1 on sample (N-1) starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 625000) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_samples"] == 2307) self.assertTrue(md.meta["end_time"] == endtime) # Set T0 on sample and T1 1μ after sample (N) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300001) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_samples"] == 2308) self.assertTrue(md.meta["end_time"] == endtime) # Set T0 and T1 1μ after sample (N-1) starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 625001) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_samples"] == 2307) self.assertTrue(md.meta["start_time"] == starttime) # Set T0 1μ after sample and T1 on sample (N-2) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_samples"] == 2306) self.assertTrue(md.meta["start_time"] == starttime)
def test_start_gap(self): """ Tests whether a gap in the beginning of the file is interpreted A gap at the beginning of the window is ignored if a sample can be found before the starttime, and is continuous Trace in file runs from [00:01.625000Z to 00:59.300000Z] """ file = os.path.join(self.path, "tiny_quality_file.mseed") # first sample is 625000, so we introduce a gap of 0.025 starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 600000) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime, add_c_segments=True) self.assertTrue(md.meta["num_gaps"] == 1) self.assertTrue(md.meta["sum_gaps"] == 0.025) self.assertTrue(md.meta["start_gap"] == 0.025) # Test single continuous segments # Gap in beginning, cseg starts at first sample cseg = md.meta['c_segments'][0] self.assertEqual(cseg['start_time'], obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 625000)) self.assertEqual(cseg['end_time'], obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000)) # Start beyond a sample, but it is padded to the left, no gap starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 630000) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime, add_c_segments=True) self.assertTrue(md.meta["num_gaps"] == 0) self.assertTrue(md.meta["start_gap"] is None) # Test single continuous segments # first sample is padded, first cseg starttime is window starttime cseg = md.meta['c_segments'][0] self.assertEqual(cseg['start_time'], obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 630000)) self.assertEqual(cseg['end_time'], obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000)) md = MSEEDMetadata([file], add_c_segments=True) self.assertTrue(md.meta["num_gaps"] == 0) self.assertTrue(md.meta["start_gap"] is None) # Test single continuous segments # No window, start time is first sample, end time is last sample + dt cseg = md.meta['c_segments'][0] self.assertEqual(cseg['start_time'], obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 625000)) self.assertEqual(cseg['end_time'], obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 325000))
def test_json_serialization(self): """ Just tests that it actually works and raises no error. We tested the dictionaries enough - now we just test the JSON encoder. """ with NamedTemporaryFile() as tf: obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(0)}).write( tf.name, format="mseed") md = MSEEDMetadata(files=[tf.name]) self.assertTrue(md.get_json_meta())
def test_raise_unmatching_ids(self): """ Test error raised for multiple stream identifiers """ with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: obspy.Trace(data=np.arange(10, dtype=np.int32), header={ "starttime": obspy.UTCDateTime(0), "network": "NL", "station": "HGN", "location": "02", "channel": "BHZ" }).write(tf1.name, format="mseed", encoding="STEIM1", reclen=256) obspy.Trace(data=np.arange(10, dtype=np.float32), header={ "starttime": obspy.UTCDateTime(100), "sampling_rate": 2.0, "network": "BW", "station": "ALTM", "location": "00", "channel": "EHE" }).write(tf2.name, format="mseed", encoding="FLOAT32", reclen=1024) with self.assertRaises(ValueError) as e: MSEEDMetadata([tf1.name, tf2.name]) self.assertEqual(e.exception.args[0], "All traces must have the same SEED id and quality.")
def test_clock_locked_percentage(self): """ 7/10 records with io_flag clock_locked set to 1 for which we count the percentage of record time of total time """ file = os.path.join(self.path, "tiny_quality_file.mseed") starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 625000) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 300000) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime, add_flags=True) # These are the record lengths # The final record is cut off by the endtime record_lengths = [ 5.7, 6.05, 5.9, 5.55, 5.475, 5.725, 6.025, 5.775, 5.7, 5.775001 ] # Bit 5 is flagged for for these records: [0, 2, 3, 4, 5, 8, 9] record_lengths_flagged = [5.7, 5.9, 5.55, 5.475, 5.725, 5.7, 5.775001] # Check if the record lenghts matches the total length total_time = md.meta["end_time"] - md.meta["start_time"] self.assertTrue(abs(sum(record_lengths) - (total_time)) < 1e-6) # Calculate the percentage of clock_locked seconds percentage = 100 * sum(record_lengths_flagged) / sum(record_lengths) meta = md.meta["miniseed_header_percentages"] meta_io = meta['io_and_clock_flags'] self.assertTrue(abs(meta_io["clock_locked"] - percentage) < 1e-6)
def test_extraction_of_basic_mseed_headers(self): """ Tests extraction of basic features. """ # Mixed files. with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(0), "network": "BW", "station": "ALTM", "location": "00", "channel": "EHE"}).write( tf1.name, format="mseed", encoding="STEIM1", reclen=256) obspy.Trace(data=np.arange(10, dtype=np.float32), header={"starttime": obspy.UTCDateTime(100), "sampling_rate": 2.0, "network": "BW", "station": "ALTM", "location": "00", "channel": "EHE"}).write( tf2.name, format="mseed", encoding="FLOAT32", reclen=1024) md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True) self.assertEqual(md.meta["network"], "BW") self.assertEqual(md.meta["station"], "ALTM") self.assertEqual(md.meta["location"], "00") self.assertEqual(md.meta["channel"], "EHE") self.assertEqual(md.meta["quality"], "D") self.assertEqual(md.meta["start_time"], obspy.UTCDateTime(0)) self.assertEqual(md.meta["end_time"], obspy.UTCDateTime(105)) self.assertEqual(md.meta["num_records"], 2) self.assertEqual(md.meta["num_samples"], 20) self.assertEqual(md.meta["sample_rate"], [1.0, 2.0]) self.assertEqual(md.meta["record_length"], [256, 1024]) self.assertEqual(md.meta["encoding"], ["FLOAT32", "STEIM1"])
def test_schema_validation(self): with NamedTemporaryFile() as tf: obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(0)}).write( tf.name, format="mseed") md = MSEEDMetadata(files=[tf.name]) # One can either directly validate the metrics. md.validate_qc_metrics(md.meta) # Or do it during the serialization. md.get_json_meta(validate=True) # Also try with extracting the flags. md = MSEEDMetadata(files=[tf.name], add_flags=True) md.validate_qc_metrics(md.meta) md.get_json_meta(validate=True)
def test_no_files_given(self): """ Tests the raised exception if no file is given. """ with self.assertRaises(ValueError) as e: MSEEDMetadata(files=[]) self.assertEqual(e.exception.args[0], "No data within the temporal constraints.")
def test_extraction_header_flags_complex(self): """ Tests the flag extraction in a complex record situation Three records, with records 2 & 3 two overlapping 50% and a 50% record length gap between record 1 and 2. Rules for overlaps with different bits are as follows: Records are sorted from end to start by endtime and processed in this order. Each consecutive record occupies a time-range that can no longer be used by another record. In the following example, the third record is dominant over the second record because it is processed first and occupies the time range. Therefore the bit in this entire range is set to 1, despite partially overlapping with a record with its bit set to 0 Bits in the test are set as shown [ ==1== ] [ ==1== ]...[ ==0== ] | | START END 25 125 [RECORD 1 (1)] = 0 - 50 [clock_locked: 1] [RECORD 2 (0)] = 75 - 125 [clock_locked: 0] [RECORD 3 (1)] = 100 - 150 [clock_locked: 1] With starttime = 25 and endtime = 125 The clock_locked percentage should thus be exactly 50.0% """ # Couldn't properly break this line following PEP8 so use # shorter notation ..... short = NamedTemporaryFile with short() as tf1, short() as tf2, short() as tf3: _create_mseed_file(tf1.name, record_count=1, starttime=obspy.UTCDateTime(0), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 1}}) _create_mseed_file(tf2.name, record_count=1, starttime=obspy.UTCDateTime(75), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 0}}) _create_mseed_file(tf3.name, record_count=1, starttime=obspy.UTCDateTime(100), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 1}}) md = MSEEDMetadata([tf1.name, tf2.name, tf3.name], starttime=obspy.UTCDateTime(25), endtime=obspy.UTCDateTime(125), add_flags=True) io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"] self.assertEqual(io_f["clock_locked"], 50.0)
def test_overall_sample_metrics(self): """ Tests the global metrics on the samples. """ with NamedTemporaryFile() as tf: obspy.Trace(data=np.arange(10, dtype=np.int32), header={ "starttime": obspy.UTCDateTime(0) }).write(tf.name, format="mseed") md = MSEEDMetadata(files=[tf.name]) self.assertEqual(md.meta["sample_min"], 0) self.assertEqual(md.meta["sample_max"], 9) self.assertEqual(md.meta["sample_mean"], 4.5) self.assertTrue(md.meta["sample_stdev"] - 2.8722813232 < 1E-6) self.assertTrue(md.meta["sample_rms"] - 5.33853912602 < 1E-6) self.assertTrue(md.meta["sample_median"], 4.5) self.assertTrue(md.meta["sample_lower_quartile"], 2.25) self.assertTrue(md.meta["sample_upper_quartile"], 6.75) # Make sure they also work if split up across two arrays. d = np.arange(10, dtype=np.int32) with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: obspy.Trace(data=d[:5], header={ "starttime": obspy.UTCDateTime(0) }).write(tf1.name, format="mseed") obspy.Trace(data=d[5:], header={ "starttime": obspy.UTCDateTime(10) }).write(tf2.name, format="mseed") md = MSEEDMetadata(files=[tf1.name, tf2.name]) self.assertEqual(md.meta["sample_min"], 0) self.assertEqual(md.meta["sample_max"], 9) self.assertEqual(md.meta["sample_mean"], 4.5) self.assertTrue(md.meta["sample_stdev"] - 2.8722813232 < 1E-6) self.assertTrue(md.meta["sample_rms"] - 7.14142842854 < 1E-6) self.assertEqual(md.meta["sample_median"], 4.5) self.assertTrue(md.meta["sample_lower_quartile"], 2.25) self.assertTrue(md.meta["sample_upper_quartile"], 6.75)
def test_int_overflow(self): """ Tests calculations for an array of big numbers that will implicitly cause an int32 overflow in the RMS calculation For an array of y = a, RMS should a """ d = np.empty(10000, dtype=np.int32) d.fill(np.iinfo(np.int32).max) with NamedTemporaryFile() as tf1: obspy.Trace(data=d, header={ "starttime": obspy.UTCDateTime(10) }).write(tf1.name, format="mseed") md = MSEEDMetadata([tf1.name]) self.assertTrue(md.meta["sample_rms"] == np.iinfo(np.int32).max)
def test_end_gap(self): """ Test for the end gap. A gap should be found if the endtime exceeds the last sample + delta + time tolerance Trace in file runs from [00:01.625000Z to 00:59.300000Z] """ file = os.path.join(self.path, "tiny_quality_file.mseed") # Last sample is at 300000, but this sample covers the trace # up to 300000 + delta (0.025) => 325000 - no gaps starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 1, 625000) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 325000) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_gaps"] == 0) self.assertTrue(md.meta["sum_gaps"] == 0.0) self.assertTrue(md.meta["end_gap"] is None) # Add 1μs; exceed projected sample plus time tolerance - GAP! endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 59, 350001) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_gaps"] == 1) self.assertTrue(md.meta["sum_gaps"] == 0.025001) self.assertTrue(md.meta["end_gap"] == 0.025001)
def test_file_with_no_timing_quality(self): """ Tests timing quality extraction in files with no timing quality. """ with NamedTemporaryFile() as tf1: obspy.Trace(data=np.arange(10, dtype=np.int32), header={ "starttime": obspy.UTCDateTime(0) }).write(tf1.name, format="mseed") mseed_metadata = MSEEDMetadata([tf1.name], add_flags=True) ref = mseed_metadata.meta['miniseed_header_percentages'] self.assertEqual(ref['timing_quality_max'], None) self.assertEqual(ref['timing_quality_min'], None) self.assertEqual(ref['timing_quality_mean'], None)
def test_continuous_segments_combined(self): """ Test continuous segments from traces in two files that are continuous. Also test a continuous segment that is continuous but has a different sampling rate """ tr_1 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(0)}) tr_2 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(10)}) tr_3 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={ "starttime": obspy.UTCDateTime(20), "sampling_rate": 0.5 }) st = obspy.Stream(traces=[tr_1, tr_3]) st2 = obspy.Stream(traces=[tr_2]) with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: st.write(tf1.name, format="mseed") st2.write(tf2.name, format="mseed") md = MSEEDMetadata(files=[tf1.name, tf2.name]) c_seg = md.meta["c_segments"] self.assertEqual(len(c_seg), 2) c = c_seg[0] self.assertEqual(c["start_time"], obspy.UTCDateTime(0)) self.assertEqual(c["end_time"], obspy.UTCDateTime(20)) self.assertEqual(c["segment_length"], 20) self.assertEqual(c["sample_min"], 0) self.assertEqual(c["sample_max"], 9) self.assertEqual(c["num_samples"], 20) self.assertEqual(c["sample_median"], 4.5) self.assertEqual(c["sample_lower_quartile"], 2.0) self.assertEqual(c["sample_upper_quartile"], 7.0) self.assertEqual(c["sample_rate"], 1.0) # Not continuous because of different sampling_rate (0.5) c = c_seg[1] self.assertEqual(c["start_time"], obspy.UTCDateTime(20)) self.assertEqual(c["end_time"], obspy.UTCDateTime(40)) self.assertEqual(c["segment_length"], 20) self.assertEqual(c["sample_min"], 0) self.assertEqual(c["sample_max"], 9) self.assertEqual(c["num_samples"], 10) self.assertEqual(c["sample_median"], 4.5) self.assertEqual(c["sample_lower_quartile"], 2.25) self.assertEqual(c["sample_upper_quartile"], 6.75) self.assertEqual(c["sample_rate"], 0.5)
def _callObsPyMetadata(self, files, start, end, granule): """ WFCatalogCollector._callObsPyMetadata wrapper function to call obspy.signal.MSEEDMetdata """ # Throw an exception after the timeout (UNIX only) signal.signal(signal.SIGALRM, self.handler) signal.alarm(self.config['PROCESSING_TIMEOUT']) try: # Catch mSEED reading warnings with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') # Skip continuous segments for hourly granules if granule == 'daily': metadata = MSEEDMetadata(files, starttime=start, endtime=end, add_flags=self.args['flags'], add_c_segments=self.args['csegs']) elif granule == 'hourly': metadata = MSEEDMetadata(files, starttime=start, endtime=end, add_flags=self.args['flags'], add_c_segments=False) metadata.meta.update({'warnings': len(w) > 0}) finally: # Disable alarm signal.alarm(0) return metadata.meta
def test_root_mean_square(self): """ Test the RMS calculation on a sine wave with amplitude sqrt(2) For this discrete sine wave RMS should be sqrt(2)/sqrt(2) = 1 Within a certain precision """ d = np.sqrt(2) * np.sin(np.linspace(-np.pi, np.pi, 10000)) with NamedTemporaryFile() as tf1: obspy.Trace(data=d, header={ "starttime": obspy.UTCDateTime(10) }).write(tf1.name, format="mseed") md = MSEEDMetadata([tf1.name]) self.assertTrue(np.fabs(md.meta["sample_rms"] - 1) < 1E-3) for c_segment in md.meta["c_segments"]: self.assertTrue(np.fabs(c_segment["sample_rms"] - 1) < 1E-3)
def test_timing_quality(self): """ Test extraction of timing quality with a file that actually has it. """ # Test file is constructed and orignally from the obspy.io.mseed # test suite. md = MSEEDMetadata( files=[os.path.join(self.path, "timingquality.mseed")], add_flags=True) ref = md.meta['miniseed_header_percentages'] self.assertEqual(ref['timing_quality_mean'], 50.0) self.assertEqual(ref['timing_quality_min'], 0.0) self.assertEqual(ref['timing_quality_max'], 100.0) self.assertEqual(ref['timing_quality_median'], 50.0) self.assertEqual(ref['timing_quality_lower_quartile'], 25.0) self.assertEqual(ref['timing_quality_upper_quartile'], 75.0)
def test_random_window(self): """ Tests a random window within a continuous trace, expect no gaps Continuous trace in file runs from [00:01.625000Z to 00:59.300000Z] """ file = os.path.join(self.path, "tiny_quality_file.mseed") # Go randomly somewhere between two samples, find no gaps because # the trace is continuous everywhere between start-end starttime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 22, 646572) endtime = obspy.UTCDateTime(2015, 10, 16, 0, 0, 38, 265749) md = MSEEDMetadata([file], starttime=starttime, endtime=endtime) self.assertTrue(md.meta["num_gaps"] == 0) self.assertTrue(md.meta["num_overlaps"] == 0) # Slicing to a start & end without asking for flags, num_records # will be set to None self.assertEqual(md.meta["num_records"], None)
def test_gaps_between_multiple_files(self): """ Test gap counting between multiple files. Simple test but there is no effective difference between having multiple files and a single one with many Traces as internally it is all parsed to a single Stream object. """ with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: # Two files, same ids but a gap in-between. obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(0)}).write( tf1.name, format="mseed") obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(100)}).write( tf2.name, format="mseed") # Don't calculate statistics on the single segments. mseed_metadata = MSEEDMetadata([tf1.name, tf2.name], add_c_segments=False) self.assertEqual(mseed_metadata.meta['num_gaps'], 1) self.assertNotIn("c_segments", mseed_metadata.meta)
def test_gaps_and_overlaps(self): """ Test gaps and overlaps. """ # Create a file. No gap between 1 and 2, 10 second gap between 2 and # 3, 5 second overlap between 3 and 4, and another 10 second gap # between 4 and 5. tr_1 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(0)}) tr_2 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(10)}) tr_3 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(30)}) tr_4 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(35)}) tr_5 = obspy.Trace(data=np.arange(10, dtype=np.int32), header={"starttime": obspy.UTCDateTime(55)}) st = obspy.Stream(traces=[tr_1, tr_2, tr_3, tr_4, tr_5]) with NamedTemporaryFile() as tf: st.write(tf.name, format="mseed") mseed_metadata = MSEEDMetadata(files=[tf.name]) self.assertEqual(mseed_metadata.meta['num_gaps'], 2) self.assertEqual(mseed_metadata.meta['num_overlaps'], 1) self.assertEqual(mseed_metadata.meta['sum_overlaps'], 5.0) self.assertEqual(mseed_metadata.meta['sum_gaps'], 20.0) self.assertEqual(mseed_metadata.meta['percent_availability'], 45.0 / 65.0 * 100.0) # Same again but this time with start-and end time settings. mseed_metadata = MSEEDMetadata(files=[tf.name], starttime=obspy.UTCDateTime(5), endtime=obspy.UTCDateTime(60)) self.assertEqual(mseed_metadata.meta['num_gaps'], 2) self.assertEqual(mseed_metadata.meta['num_overlaps'], 1) self.assertEqual(mseed_metadata.meta['sum_overlaps'], 5.0) self.assertEqual(mseed_metadata.meta['sum_gaps'], 20.0) self.assertEqual(mseed_metadata.meta['percent_availability'], 35.0 / 55.0 * 100.0) # Head and tail gaps. mseed_metadata = MSEEDMetadata(files=[tf.name], starttime=obspy.UTCDateTime(-10), endtime=obspy.UTCDateTime(80)) self.assertEqual(mseed_metadata.meta['num_gaps'], 4) self.assertEqual(mseed_metadata.meta['num_overlaps'], 1) self.assertEqual(mseed_metadata.meta['sum_overlaps'], 5.0) self.assertEqual(mseed_metadata.meta['sum_gaps'], 45.0) self.assertEqual(mseed_metadata.meta['percent_availability'], 45.0 / 90.0 * 100.0) # Tail gap must be larger than 1 delta, otherwise it does not # count. mseed_metadata = MSEEDMetadata(files=[tf.name], endtime=obspy.UTCDateTime(64)) self.assertEqual(mseed_metadata.meta['num_gaps'], 2) self.assertEqual(mseed_metadata.meta['sum_gaps'], 20.0) self.assertEqual(mseed_metadata.meta['percent_availability'], 44.0 / 64.0 * 100.0) mseed_metadata = MSEEDMetadata(files=[tf.name], endtime=obspy.UTCDateTime(65)) self.assertEqual(mseed_metadata.meta['num_gaps'], 2) self.assertEqual(mseed_metadata.meta['sum_gaps'], 20.0) self.assertEqual(mseed_metadata.meta['percent_availability'], 45.0 / 65.0 * 100.0) mseed_metadata = MSEEDMetadata(files=[tf.name], endtime=obspy.UTCDateTime(66)) self.assertEqual(mseed_metadata.meta['num_gaps'], 3) self.assertEqual(mseed_metadata.meta['sum_gaps'], 21.0) self.assertEqual(mseed_metadata.meta['percent_availability'], 45.0 / 66.0 * 100.0)
from obspy.signal.quality_control import MSEEDMetadata # IO Directories INPUT = "/var/input" OUTPUT = "/var/output/metadata.json" ERROR = "/var/output/error.log" # Get the filename from the input directory files = [ os.path.join(INPUT, file) for file in os.listdir(INPUT) ] # Calculate the metadata try: MD = MSEEDMetadata( files, add_c_segments=True, add_flags=True ) # Write result to file with open(OUTPUT, "w") as outfile: outfile.write(MD.get_json_meta()); except Exception as e: # Write error to file with open(ERROR, "w") as outfile: outfile.write("Could not complete metric calculation: %s" % e);
def test_extraction_fixed_header_flags(self): # Had to put positive_leap count to 0 to prevent # end time from being wrong with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: _create_mseed_file(tf1.name, record_count=35, starttime=obspy.UTCDateTime(0), seed=12345, flags={ 'data_quality_flags': { "amplifier_saturation": 25, "digitizer_clipping": 12, "spikes": 30, "glitches": 6, "missing_padded_data": 15, "telemetry_sync_error": 16, "digital_filter_charging": 4, "suspect_time_tag": 8 }, 'activity_flags': { "calibration_signal": 10, "time_correction_applied": 20, "event_begin": 33, "event_end": 33, "positive_leap": 0, "negative_leap": 10, "event_in_progress": 15 }, 'io_and_clock_flags': { "station_volume": 8, "long_record_read": 33, "short_record_read": 24, "start_time_series": 31, "end_time_series": 24, "clock_locked": 32 } }) # Previous file ends exactly on 1750, start new file # to prevent overlapping records. When records overlap # their contributions should NOT be summed _create_mseed_file(tf2.name, record_count=23, starttime=obspy.UTCDateTime(1750), seed=12345, flags={ 'data_quality_flags': { "amplifier_saturation": 5, "digitizer_clipping": 7, "spikes": 5, "glitches": 3, "missing_padded_data": 5, "telemetry_sync_error": 3, "digital_filter_charging": 4, "suspect_time_tag": 2 }, 'activity_flags': { "calibration_signal": 1, "time_correction_applied": 0, "event_begin": 3, "event_end": 3, "positive_leap": 0, "negative_leap": 1, "event_in_progress": 5 }, 'io_and_clock_flags': { "station_volume": 1, "long_record_read": 3, "short_record_read": 2, "start_time_series": 3, "end_time_series": 4, "clock_locked": 2 } }) md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True) def _assert_float_equal(a, b): """ Supplementary function to test floats to precision of 1E-6 """ self.assertTrue(abs(a - b) < 1E-6) # Sum up contributions from both files. # Check percentages meta = md.meta['miniseed_header_counts'] meta_dq = meta['data_quality_flags'] self.assertEqual(meta_dq['glitches'], 9) self.assertEqual(meta_dq['amplifier_saturation'], 30) self.assertEqual(meta_dq['digital_filter_charging'], 8) self.assertEqual(meta_dq['digitizer_clipping'], 19) self.assertEqual(meta_dq['missing_padded_data'], 20) self.assertEqual(meta_dq['spikes'], 35) self.assertEqual(meta_dq['suspect_time_tag'], 10) self.assertEqual(meta_dq['telemetry_sync_error'], 19) meta_af = meta['activity_flags'] self.assertEqual(meta_af['calibration_signal'], 11) self.assertEqual(meta_af['event_begin'], 36) self.assertEqual(meta_af['event_end'], 36) self.assertEqual(meta_af['event_in_progress'], 20) self.assertEqual(meta_af['time_correction_applied'], 20) meta_io = meta['io_and_clock_flags'] self.assertEqual(meta_io['clock_locked'], 34) self.assertEqual(meta_io['station_volume'], 9) self.assertEqual(meta_io['long_record_read'], 36) self.assertEqual(meta_io['short_record_read'], 26) self.assertEqual(meta_io['start_time_series'], 34) self.assertEqual(meta_io['end_time_series'], 28) meta = md.meta['miniseed_header_percentages'] meta_dq = meta['data_quality_flags'] _assert_float_equal(meta_dq['glitches'], 9 / 0.58) _assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58) _assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58) _assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58) _assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58) _assert_float_equal(meta_dq['spikes'], 35 / 0.58) _assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58) _assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58) meta_af = meta['activity_flags'] _assert_float_equal(meta_af['calibration_signal'], 11 / 0.58) _assert_float_equal(meta_af['event_begin'], 36 / 0.58) _assert_float_equal(meta_af['event_end'], 36 / 0.58) _assert_float_equal(meta_af['event_in_progress'], 20 / 0.58) _assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58) meta_io = meta['io_and_clock_flags'] _assert_float_equal(meta_io['clock_locked'], 34 / 0.58) _assert_float_equal(meta_io['station_volume'], 9 / 0.58) _assert_float_equal(meta_io['long_record_read'], 36 / 0.58) _assert_float_equal(meta_io['short_record_read'], 26 / 0.58) _assert_float_equal(meta_io['start_time_series'], 34 / 0.58) _assert_float_equal(meta_io['end_time_series'], 28 / 0.58) ref = md.meta['miniseed_header_percentages'] self.assertEqual(ref['timing_quality_mean'], None) self.assertEqual(ref['timing_quality_min'], None) self.assertEqual(ref['timing_quality_max'], None)
def test_continuous_segments_sample_metrics(self): """ Tests the metrics on each segment. """ d = np.arange(10, dtype=np.int32) with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2, \ NamedTemporaryFile() as tf3: obspy.Trace(data=d[:5], header={ "starttime": obspy.UTCDateTime(0) }).write(tf1.name, format="mseed") obspy.Trace(data=d[5:], header={ "starttime": obspy.UTCDateTime(10) }).write(tf2.name, format="mseed") obspy.Trace(data=np.arange(10, dtype=np.int32), header={ "starttime": obspy.UTCDateTime(20) }).write(tf3.name, format="mseed") md = MSEEDMetadata(files=[tf1.name, tf2.name, tf3.name]) c_seg = md.meta["c_segments"] self.assertEqual(len(c_seg), 3) c = c_seg[0] self.assertEqual(c["start_time"], obspy.UTCDateTime(0)) self.assertEqual(c["end_time"], obspy.UTCDateTime(5)) self.assertEqual(c["sample_min"], 0) self.assertEqual(c["sample_max"], 4) self.assertEqual(c["sample_mean"], 2.0) self.assertTrue(c["sample_rms"] - 2.4494897427831779 < 1E-6) self.assertTrue(c["sample_stdev"] - 1.4142135623730951 < 1E-6) self.assertEqual(c["num_samples"], 5) self.assertEqual(c["segment_length"], 5.0) self.assertEqual(c["sample_median"], 2) self.assertTrue(c["sample_lower_quartile"], 1.0) self.assertTrue(c["sample_upper_quartile"], 3.0) c = c_seg[1] self.assertEqual(c["start_time"], obspy.UTCDateTime(10)) self.assertEqual(c["end_time"], obspy.UTCDateTime(15)) self.assertEqual(c["sample_min"], 5) self.assertEqual(c["sample_max"], 9) self.assertEqual(c["sample_mean"], 7.0) self.assertTrue(c["sample_rms"] - 7.1414284285428504 < 1E-6) self.assertTrue(c["sample_stdev"] - 1.4142135623730951 < 1E-6) self.assertEqual(c["num_samples"], 5) self.assertEqual(c["segment_length"], 5.0) self.assertEqual(c["sample_median"], 7) self.assertTrue(c["sample_lower_quartile"], 6.0) self.assertTrue(c["sample_upper_quartile"], 8.0) c = c_seg[2] self.assertEqual(c["start_time"], obspy.UTCDateTime(20)) self.assertEqual(c["end_time"], obspy.UTCDateTime(30)) self.assertEqual(c["num_samples"], 10) self.assertEqual(c["segment_length"], 10.0) self.assertEqual(c["sample_min"], 0) self.assertEqual(c["sample_max"], 9) self.assertEqual(c["sample_mean"], 4.5) self.assertTrue(c["sample_stdev"] - 2.8722813232 < 1E-6) self.assertTrue(c["sample_rms"] - 5.3385391260156556 < 1E-6) self.assertEqual(c["sample_median"], 4.5) self.assertTrue(c["sample_lower_quartile"], 2.25) self.assertTrue(c["sample_upper_quartile"], 6.25)
def _rapid_gap_testing(start, end): md = MSEEDMetadata(files=[tf.name], starttime=obspy.UTCDateTime(start), endtime=obspy.UTCDateTime(end)) return md.meta['sum_gaps']
def test_schema_validation(self): with NamedTemporaryFile() as tf: obspy.Trace(data=np.arange(10, dtype=np.int32), header={ "starttime": obspy.UTCDateTime(0) }).write(tf.name, format="mseed") md = MSEEDMetadata(files=[tf.name]) # One can either directly validate the metrics. md.validate_qc_metrics(md.meta) # Or do it during the serialization. md.get_json_meta(validate=True) # Also try with extracting the flags. md = MSEEDMetadata(files=[tf.name], add_flags=True) md.validate_qc_metrics(md.meta) md.get_json_meta(validate=True)
import os from obspy.signal.quality_control import MSEEDMetadata # IO Directories INPUT = "/var/input" OUTPUT = "/var/output/metadata.json" ERROR = "/var/output/error.log" # Get the filename from the input directory files = [os.path.join(INPUT, file) for file in os.listdir(INPUT)] # Calculate the metadata try: MD = MSEEDMetadata(files, add_c_segments=True, add_flags=True) # Write result to file with open(OUTPUT, "w") as outfile: outfile.write(MD.get_json_meta()) except Exception as e: # Write error to file with open(ERROR, "w") as outfile: outfile.write("Could not complete metric calculation: %s" % e)