def test_extraction_header_flags_complex(self): """ Tests the flag extraction in a complex record situation Three records, with records 2 & 3 two overlapping 50% and a 50% record length gap between record 1 and 2. Rules for overlaps with different bits are as follows: Records are sorted from end to start by endtime and processed in this order. Each consecutive record occupies a time-range that can no longer be used by another record. In the following example, the third record is dominant over the second record because it is processed first and occupies the time range. Therefore the bit in this entire range is set to 1, despite partially overlapping with a record with its bit set to 0 Bits in the test are set as shown [ ==1== ] [ ==1== ]...[ ==0== ] | | START END 25 125 [RECORD 1 (1)] = 0 - 50 [clock_locked: 1] [RECORD 2 (0)] = 75 - 125 [clock_locked: 0] [RECORD 3 (1)] = 100 - 150 [clock_locked: 1] With starttime = 25 and endtime = 125 The clock_locked percentage should thus be exactly 50.0% """ # Couldn't properly break this line following PEP8 so use # shorter notation ..... short = NamedTemporaryFile with short() as tf1, short() as tf2, short() as tf3: _create_mseed_file(tf1.name, record_count=1, starttime=obspy.UTCDateTime(0), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 1}}) _create_mseed_file(tf2.name, record_count=1, starttime=obspy.UTCDateTime(75), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 0}}) _create_mseed_file(tf3.name, record_count=1, starttime=obspy.UTCDateTime(100), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 1}}) md = MSEEDMetadata([tf1.name, tf2.name, tf3.name], starttime=obspy.UTCDateTime(25), endtime=obspy.UTCDateTime(125), add_flags=True) io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"] self.assertEqual(io_f["clock_locked"], 50.0)
def test_extraction_header_flags_complex(self): """ Tests the flag extraction in a complex record situation Three records, with records 2 & 3 two overlapping 50% and a 50% record length gap between record 1 and 2. Rules for overlaps with different bits are as follows: Records are sorted from end to start by endtime and processed in this order. Each consecutive record occupies a time-range that can no longer be used by another record. In the following example, the third record is dominant over the second record because it is processed first and occupies the time range. Therefore the bit in this entire range is set to 1, despite partially overlapping with a record with its bit set to 0 Bits in the test are set as shown [ ==1== ] [ ==1== ]...[ ==0== ] | | START END 25 125 [RECORD 1 (1)] = 0 - 50 [clock_locked: 1] [RECORD 2 (0)] = 75 - 125 [clock_locked: 0] [RECORD 3 (1)] = 100 - 150 [clock_locked: 1] With starttime = 25 and endtime = 125 The clock_locked percentage should thus be exactly 50.0% """ # Couldn't properly break this line following PEP8 so use # shorter notation ..... short = NamedTemporaryFile with short() as tf1, short() as tf2, short() as tf3: _create_mseed_file(tf1.name, record_count=1, starttime=obspy.UTCDateTime(0), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 1}}) _create_mseed_file(tf2.name, record_count=1, starttime=obspy.UTCDateTime(75), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 0}}) _create_mseed_file(tf3.name, record_count=1, starttime=obspy.UTCDateTime(100), seed=12345, flags={ 'io_and_clock_flags': { "clock_locked": 1}}) md = MSEEDMetadata([tf1.name, tf2.name, tf3.name], starttime=obspy.UTCDateTime(25), endtime=obspy.UTCDateTime(125), add_flags=True) io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"] self.assertEqual(io_f["clock_locked"], 50.0)
def test_extraction_fixed_header_flags(self): # Had to put positive_leap count to 0 to prevent # end time from being wrong with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: _create_mseed_file(tf1.name, record_count=35, starttime=obspy.UTCDateTime(0), seed=12345, flags={ 'data_quality_flags': { "amplifier_saturation": 25, "digitizer_clipping": 12, "spikes": 30, "glitches": 6, "missing_padded_data": 15, "telemetry_sync_error": 16, "digital_filter_charging": 4, "suspect_time_tag": 8 }, 'activity_flags': { "calibration_signal": 10, "time_correction_applied": 20, "event_begin": 33, "event_end": 33, "positive_leap": 0, "negative_leap": 10, "event_in_progress": 15 }, 'io_and_clock_flags': { "station_volume": 8, "long_record_read": 33, "short_record_read": 24, "start_time_series": 31, "end_time_series": 24, "clock_locked": 32 } }) # Previous file ends exactly on 1750, start new file # to prevent overlapping records. When records overlap # their contributions should NOT be summed _create_mseed_file(tf2.name, record_count=23, starttime=obspy.UTCDateTime(1750), seed=12345, flags={ 'data_quality_flags': { "amplifier_saturation": 5, "digitizer_clipping": 7, "spikes": 5, "glitches": 3, "missing_padded_data": 5, "telemetry_sync_error": 3, "digital_filter_charging": 4, "suspect_time_tag": 2 }, 'activity_flags': { "calibration_signal": 1, "time_correction_applied": 0, "event_begin": 3, "event_end": 3, "positive_leap": 0, "negative_leap": 1, "event_in_progress": 5 }, 'io_and_clock_flags': { "station_volume": 1, "long_record_read": 3, "short_record_read": 2, "start_time_series": 3, "end_time_series": 4, "clock_locked": 2 } }) md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True) def _assert_float_equal(a, b): """ Supplementary function to test floats to precision of 1E-6 """ self.assertTrue(abs(a - b) < 1E-6) # Sum up contributions from both files. # Check percentages meta = md.meta['miniseed_header_counts'] meta_dq = meta['data_quality_flags'] self.assertEqual(meta_dq['glitches'], 9) self.assertEqual(meta_dq['amplifier_saturation'], 30) self.assertEqual(meta_dq['digital_filter_charging'], 8) self.assertEqual(meta_dq['digitizer_clipping'], 19) self.assertEqual(meta_dq['missing_padded_data'], 20) self.assertEqual(meta_dq['spikes'], 35) self.assertEqual(meta_dq['suspect_time_tag'], 10) self.assertEqual(meta_dq['telemetry_sync_error'], 19) meta_af = meta['activity_flags'] self.assertEqual(meta_af['calibration_signal'], 11) self.assertEqual(meta_af['event_begin'], 36) self.assertEqual(meta_af['event_end'], 36) self.assertEqual(meta_af['event_in_progress'], 20) self.assertEqual(meta_af['time_correction_applied'], 20) meta_io = meta['io_and_clock_flags'] self.assertEqual(meta_io['clock_locked'], 34) self.assertEqual(meta_io['station_volume'], 9) self.assertEqual(meta_io['long_record_read'], 36) self.assertEqual(meta_io['short_record_read'], 26) self.assertEqual(meta_io['start_time_series'], 34) self.assertEqual(meta_io['end_time_series'], 28) meta = md.meta['miniseed_header_percentages'] meta_dq = meta['data_quality_flags'] _assert_float_equal(meta_dq['glitches'], 9 / 0.58) _assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58) _assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58) _assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58) _assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58) _assert_float_equal(meta_dq['spikes'], 35 / 0.58) _assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58) _assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58) meta_af = meta['activity_flags'] _assert_float_equal(meta_af['calibration_signal'], 11 / 0.58) _assert_float_equal(meta_af['event_begin'], 36 / 0.58) _assert_float_equal(meta_af['event_end'], 36 / 0.58) _assert_float_equal(meta_af['event_in_progress'], 20 / 0.58) _assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58) meta_io = meta['io_and_clock_flags'] _assert_float_equal(meta_io['clock_locked'], 34 / 0.58) _assert_float_equal(meta_io['station_volume'], 9 / 0.58) _assert_float_equal(meta_io['long_record_read'], 36 / 0.58) _assert_float_equal(meta_io['short_record_read'], 26 / 0.58) _assert_float_equal(meta_io['start_time_series'], 34 / 0.58) _assert_float_equal(meta_io['end_time_series'], 28 / 0.58) ref = md.meta['miniseed_header_percentages'] self.assertEqual(ref['timing_quality_mean'], None) self.assertEqual(ref['timing_quality_min'], None) self.assertEqual(ref['timing_quality_max'], None)
def test_extraction_fixed_header_flags(self): # Had to put positive_leap count to 0 to prevent # end time from being wrong with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2: _create_mseed_file(tf1.name, record_count=35, starttime=obspy.UTCDateTime(0), seed=12345, flags={ 'data_quality_flags': { "amplifier_saturation": 25, "digitizer_clipping": 12, "spikes": 30, "glitches": 6, "missing_padded_data": 15, "telemetry_sync_error": 16, "digital_filter_charging": 4, "suspect_time_tag": 8}, 'activity_flags': { "calibration_signal": 10, "time_correction_applied": 20, "event_begin": 33, "event_end": 33, "positive_leap": 0, "negative_leap": 10, "event_in_progress": 15}, 'io_and_clock_flags': { "station_volume": 8, "long_record_read": 33, "short_record_read": 24, "start_time_series": 31, "end_time_series": 24, "clock_locked": 32}}) # Previous file ends exactly on 1750, start new file # to prevent overlapping records. When records overlap # their contributions should NOT be summed _create_mseed_file(tf2.name, record_count=23, starttime=obspy.UTCDateTime(1750), seed=12345, flags={ 'data_quality_flags': { "amplifier_saturation": 5, "digitizer_clipping": 7, "spikes": 5, "glitches": 3, "missing_padded_data": 5, "telemetry_sync_error": 3, "digital_filter_charging": 4, "suspect_time_tag": 2}, 'activity_flags': { "calibration_signal": 1, "time_correction_applied": 0, "event_begin": 3, "event_end": 3, "positive_leap": 0, "negative_leap": 1, "event_in_progress": 5}, 'io_and_clock_flags': { "station_volume": 1, "long_record_read": 3, "short_record_read": 2, "start_time_series": 3, "end_time_series": 4, "clock_locked": 2}}) md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True) def _assert_float_equal(a, b): """ Supplementary function to test floats to precision of 1E-6 """ self.assertTrue(abs(a - b) < 1E-6) # Sum up contributions from both files. # Check percentages meta = md.meta['miniseed_header_counts'] meta_dq = meta['data_quality_flags'] self.assertEqual(meta_dq['glitches'], 9) self.assertEqual(meta_dq['amplifier_saturation'], 30) self.assertEqual(meta_dq['digital_filter_charging'], 8) self.assertEqual(meta_dq['digitizer_clipping'], 19) self.assertEqual(meta_dq['missing_padded_data'], 20) self.assertEqual(meta_dq['spikes'], 35) self.assertEqual(meta_dq['suspect_time_tag'], 10) self.assertEqual(meta_dq['telemetry_sync_error'], 19) meta_af = meta['activity_flags'] self.assertEqual(meta_af['calibration_signal'], 11) self.assertEqual(meta_af['event_begin'], 36) self.assertEqual(meta_af['event_end'], 36) self.assertEqual(meta_af['event_in_progress'], 20) self.assertEqual(meta_af['time_correction_applied'], 20) meta_io = meta['io_and_clock_flags'] self.assertEqual(meta_io['clock_locked'], 34) self.assertEqual(meta_io['station_volume'], 9) self.assertEqual(meta_io['long_record_read'], 36) self.assertEqual(meta_io['short_record_read'], 26) self.assertEqual(meta_io['start_time_series'], 34) self.assertEqual(meta_io['end_time_series'], 28) meta = md.meta['miniseed_header_percentages'] meta_dq = meta['data_quality_flags'] _assert_float_equal(meta_dq['glitches'], 9 / 0.58) _assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58) _assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58) _assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58) _assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58) _assert_float_equal(meta_dq['spikes'], 35 / 0.58) _assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58) _assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58) meta_af = meta['activity_flags'] _assert_float_equal(meta_af['calibration_signal'], 11 / 0.58) _assert_float_equal(meta_af['event_begin'], 36 / 0.58) _assert_float_equal(meta_af['event_end'], 36 / 0.58) _assert_float_equal(meta_af['event_in_progress'], 20 / 0.58) _assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58) meta_io = meta['io_and_clock_flags'] _assert_float_equal(meta_io['clock_locked'], 34 / 0.58) _assert_float_equal(meta_io['station_volume'], 9 / 0.58) _assert_float_equal(meta_io['long_record_read'], 36 / 0.58) _assert_float_equal(meta_io['short_record_read'], 26 / 0.58) _assert_float_equal(meta_io['start_time_series'], 34 / 0.58) _assert_float_equal(meta_io['end_time_series'], 28 / 0.58) ref = md.meta['miniseed_header_percentages'] self.assertEqual(ref['timing_quality_mean'], None) self.assertEqual(ref['timing_quality_min'], None) self.assertEqual(ref['timing_quality_max'], None)