Beispiel #1
0
    def test_extraction_header_flags_complex(self):
        """
        Tests the flag extraction in a complex record situation
        Three records, with records 2 & 3 two overlapping 50% and
        a 50% record length gap between record 1 and 2.

        Rules for overlaps with different bits are as follows:
        Records are sorted from end to start by endtime and processed
        in this order. Each consecutive record occupies a time-range
        that can no longer be used by another record. In the following
        example, the third record is dominant over the second record
        because it is processed first and occupies the time range.
        Therefore the bit in this entire range is set to 1, despite
        partially overlapping with a record with its bit set to 0

        Bits in the test are set as shown

                        [ ==1== ]
        [ ==1== ]...[ ==0== ]
            |               |
          START            END
           25              125

        [RECORD 1 (1)] = 0 - 50 [clock_locked: 1]
        [RECORD 2 (0)] = 75 - 125 [clock_locked: 0]
        [RECORD 3 (1)] = 100 - 150 [clock_locked: 1]
        With starttime = 25 and endtime = 125

        The clock_locked percentage should thus be exactly 50.0%
        """

        # Couldn't properly break this line following PEP8 so use
        # shorter notation .....
        short = NamedTemporaryFile
        with short() as tf1, short() as tf2, short() as tf3:
            _create_mseed_file(tf1.name, record_count=1,
                               starttime=obspy.UTCDateTime(0),
                               seed=12345, flags={
                                   'io_and_clock_flags': {
                                       "clock_locked": 1}})
            _create_mseed_file(tf2.name, record_count=1,
                               starttime=obspy.UTCDateTime(75),
                               seed=12345, flags={
                                   'io_and_clock_flags': {
                                       "clock_locked": 0}})
            _create_mseed_file(tf3.name, record_count=1,
                               starttime=obspy.UTCDateTime(100),
                               seed=12345, flags={
                                   'io_and_clock_flags': {
                                       "clock_locked": 1}})

            md = MSEEDMetadata([tf1.name, tf2.name, tf3.name],
                               starttime=obspy.UTCDateTime(25),
                               endtime=obspy.UTCDateTime(125), add_flags=True)
            io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"]
            self.assertEqual(io_f["clock_locked"], 50.0)
Beispiel #2
0
    def test_extraction_header_flags_complex(self):
        """
        Tests the flag extraction in a complex record situation
        Three records, with records 2 & 3 two overlapping 50% and
        a 50% record length gap between record 1 and 2.

        Rules for overlaps with different bits are as follows:
        Records are sorted from end to start by endtime and processed
        in this order. Each consecutive record occupies a time-range
        that can no longer be used by another record. In the following
        example, the third record is dominant over the second record
        because it is processed first and occupies the time range.
        Therefore the bit in this entire range is set to 1, despite
        partially overlapping with a record with its bit set to 0

        Bits in the test are set as shown

                        [ ==1== ]
        [ ==1== ]...[ ==0== ]
            |               |
          START            END
           25              125

        [RECORD 1 (1)] = 0 - 50 [clock_locked: 1]
        [RECORD 2 (0)] = 75 - 125 [clock_locked: 0]
        [RECORD 3 (1)] = 100 - 150 [clock_locked: 1]
        With starttime = 25 and endtime = 125

        The clock_locked percentage should thus be exactly 50.0%
        """

        # Couldn't properly break this line following PEP8 so use
        # shorter notation .....
        short = NamedTemporaryFile
        with short() as tf1, short() as tf2, short() as tf3:
            _create_mseed_file(tf1.name, record_count=1,
                               starttime=obspy.UTCDateTime(0),
                               seed=12345, flags={
                                'io_and_clock_flags': {
                                    "clock_locked": 1}})
            _create_mseed_file(tf2.name, record_count=1,
                               starttime=obspy.UTCDateTime(75),
                               seed=12345, flags={
                                'io_and_clock_flags': {
                                    "clock_locked": 0}})
            _create_mseed_file(tf3.name, record_count=1,
                               starttime=obspy.UTCDateTime(100),
                               seed=12345, flags={
                                'io_and_clock_flags': {
                                    "clock_locked": 1}})

            md = MSEEDMetadata([tf1.name, tf2.name, tf3.name],
                               starttime=obspy.UTCDateTime(25),
                               endtime=obspy.UTCDateTime(125), add_flags=True)
            io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"]
            self.assertEqual(io_f["clock_locked"], 50.0)
    def test_extraction_fixed_header_flags(self):
        # Had to put positive_leap count to 0 to prevent
        # end time from being wrong
        with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2:
            _create_mseed_file(tf1.name,
                               record_count=35,
                               starttime=obspy.UTCDateTime(0),
                               seed=12345,
                               flags={
                                   'data_quality_flags': {
                                       "amplifier_saturation": 25,
                                       "digitizer_clipping": 12,
                                       "spikes": 30,
                                       "glitches": 6,
                                       "missing_padded_data": 15,
                                       "telemetry_sync_error": 16,
                                       "digital_filter_charging": 4,
                                       "suspect_time_tag": 8
                                   },
                                   'activity_flags': {
                                       "calibration_signal": 10,
                                       "time_correction_applied": 20,
                                       "event_begin": 33,
                                       "event_end": 33,
                                       "positive_leap": 0,
                                       "negative_leap": 10,
                                       "event_in_progress": 15
                                   },
                                   'io_and_clock_flags': {
                                       "station_volume": 8,
                                       "long_record_read": 33,
                                       "short_record_read": 24,
                                       "start_time_series": 31,
                                       "end_time_series": 24,
                                       "clock_locked": 32
                                   }
                               })
            # Previous file ends exactly on 1750, start new file
            # to prevent overlapping records. When records overlap
            # their contributions should NOT be summed
            _create_mseed_file(tf2.name,
                               record_count=23,
                               starttime=obspy.UTCDateTime(1750),
                               seed=12345,
                               flags={
                                   'data_quality_flags': {
                                       "amplifier_saturation": 5,
                                       "digitizer_clipping": 7,
                                       "spikes": 5,
                                       "glitches": 3,
                                       "missing_padded_data": 5,
                                       "telemetry_sync_error": 3,
                                       "digital_filter_charging": 4,
                                       "suspect_time_tag": 2
                                   },
                                   'activity_flags': {
                                       "calibration_signal": 1,
                                       "time_correction_applied": 0,
                                       "event_begin": 3,
                                       "event_end": 3,
                                       "positive_leap": 0,
                                       "negative_leap": 1,
                                       "event_in_progress": 5
                                   },
                                   'io_and_clock_flags': {
                                       "station_volume": 1,
                                       "long_record_read": 3,
                                       "short_record_read": 2,
                                       "start_time_series": 3,
                                       "end_time_series": 4,
                                       "clock_locked": 2
                                   }
                               })

            md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True)

            def _assert_float_equal(a, b):
                """
                Supplementary function to test floats to precision of 1E-6
                """
                self.assertTrue(abs(a - b) < 1E-6)

            # Sum up contributions from both files.
            # Check percentages
            meta = md.meta['miniseed_header_counts']
            meta_dq = meta['data_quality_flags']
            self.assertEqual(meta_dq['glitches'], 9)
            self.assertEqual(meta_dq['amplifier_saturation'], 30)
            self.assertEqual(meta_dq['digital_filter_charging'], 8)
            self.assertEqual(meta_dq['digitizer_clipping'], 19)
            self.assertEqual(meta_dq['missing_padded_data'], 20)
            self.assertEqual(meta_dq['spikes'], 35)
            self.assertEqual(meta_dq['suspect_time_tag'], 10)
            self.assertEqual(meta_dq['telemetry_sync_error'], 19)

            meta_af = meta['activity_flags']
            self.assertEqual(meta_af['calibration_signal'], 11)
            self.assertEqual(meta_af['event_begin'], 36)
            self.assertEqual(meta_af['event_end'], 36)
            self.assertEqual(meta_af['event_in_progress'], 20)
            self.assertEqual(meta_af['time_correction_applied'], 20)

            meta_io = meta['io_and_clock_flags']
            self.assertEqual(meta_io['clock_locked'], 34)
            self.assertEqual(meta_io['station_volume'], 9)
            self.assertEqual(meta_io['long_record_read'], 36)
            self.assertEqual(meta_io['short_record_read'], 26)
            self.assertEqual(meta_io['start_time_series'], 34)
            self.assertEqual(meta_io['end_time_series'], 28)

            meta = md.meta['miniseed_header_percentages']
            meta_dq = meta['data_quality_flags']
            _assert_float_equal(meta_dq['glitches'], 9 / 0.58)
            _assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58)
            _assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58)
            _assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58)
            _assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58)
            _assert_float_equal(meta_dq['spikes'], 35 / 0.58)
            _assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58)
            _assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58)

            meta_af = meta['activity_flags']
            _assert_float_equal(meta_af['calibration_signal'], 11 / 0.58)
            _assert_float_equal(meta_af['event_begin'], 36 / 0.58)
            _assert_float_equal(meta_af['event_end'], 36 / 0.58)
            _assert_float_equal(meta_af['event_in_progress'], 20 / 0.58)
            _assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58)

            meta_io = meta['io_and_clock_flags']
            _assert_float_equal(meta_io['clock_locked'], 34 / 0.58)
            _assert_float_equal(meta_io['station_volume'], 9 / 0.58)
            _assert_float_equal(meta_io['long_record_read'], 36 / 0.58)
            _assert_float_equal(meta_io['short_record_read'], 26 / 0.58)
            _assert_float_equal(meta_io['start_time_series'], 34 / 0.58)
            _assert_float_equal(meta_io['end_time_series'], 28 / 0.58)

            ref = md.meta['miniseed_header_percentages']
            self.assertEqual(ref['timing_quality_mean'], None)
            self.assertEqual(ref['timing_quality_min'], None)
            self.assertEqual(ref['timing_quality_max'], None)
Beispiel #4
0
    def test_extraction_fixed_header_flags(self):
        # Had to put positive_leap count to 0 to prevent
        # end time from being wrong
        with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2:
            _create_mseed_file(tf1.name, record_count=35,
                               starttime=obspy.UTCDateTime(0),
                               seed=12345, flags={
                                   'data_quality_flags': {
                                       "amplifier_saturation": 25,
                                       "digitizer_clipping": 12,
                                       "spikes": 30,
                                       "glitches": 6,
                                       "missing_padded_data": 15,
                                       "telemetry_sync_error": 16,
                                       "digital_filter_charging": 4,
                                       "suspect_time_tag": 8},
                                   'activity_flags': {
                                       "calibration_signal": 10,
                                       "time_correction_applied": 20,
                                       "event_begin": 33,
                                       "event_end": 33,
                                       "positive_leap": 0,
                                       "negative_leap": 10,
                                       "event_in_progress": 15},
                                   'io_and_clock_flags': {
                                       "station_volume": 8,
                                       "long_record_read": 33,
                                       "short_record_read": 24,
                                       "start_time_series": 31,
                                       "end_time_series": 24,
                                       "clock_locked": 32}})
            # Previous file ends exactly on 1750, start new file
            # to prevent overlapping records. When records overlap
            # their contributions should NOT be summed
            _create_mseed_file(tf2.name, record_count=23,
                               starttime=obspy.UTCDateTime(1750),
                               seed=12345, flags={
                                   'data_quality_flags': {
                                       "amplifier_saturation": 5,
                                       "digitizer_clipping": 7,
                                       "spikes": 5,
                                       "glitches": 3,
                                       "missing_padded_data": 5,
                                       "telemetry_sync_error": 3,
                                       "digital_filter_charging": 4,
                                       "suspect_time_tag": 2},
                                   'activity_flags': {
                                       "calibration_signal": 1,
                                       "time_correction_applied": 0,
                                       "event_begin": 3,
                                       "event_end": 3,
                                       "positive_leap": 0,
                                       "negative_leap": 1,
                                       "event_in_progress": 5},
                                   'io_and_clock_flags': {
                                       "station_volume": 1,
                                       "long_record_read": 3,
                                       "short_record_read": 2,
                                       "start_time_series": 3,
                                       "end_time_series": 4,
                                       "clock_locked": 2}})

            md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True)

            def _assert_float_equal(a, b):
                """
                Supplementary function to test floats to precision of 1E-6
                """
                self.assertTrue(abs(a - b) < 1E-6)

            # Sum up contributions from both files.
            # Check percentages
            meta = md.meta['miniseed_header_counts']
            meta_dq = meta['data_quality_flags']
            self.assertEqual(meta_dq['glitches'], 9)
            self.assertEqual(meta_dq['amplifier_saturation'], 30)
            self.assertEqual(meta_dq['digital_filter_charging'], 8)
            self.assertEqual(meta_dq['digitizer_clipping'], 19)
            self.assertEqual(meta_dq['missing_padded_data'], 20)
            self.assertEqual(meta_dq['spikes'], 35)
            self.assertEqual(meta_dq['suspect_time_tag'], 10)
            self.assertEqual(meta_dq['telemetry_sync_error'], 19)

            meta_af = meta['activity_flags']
            self.assertEqual(meta_af['calibration_signal'], 11)
            self.assertEqual(meta_af['event_begin'], 36)
            self.assertEqual(meta_af['event_end'], 36)
            self.assertEqual(meta_af['event_in_progress'], 20)
            self.assertEqual(meta_af['time_correction_applied'], 20)

            meta_io = meta['io_and_clock_flags']
            self.assertEqual(meta_io['clock_locked'], 34)
            self.assertEqual(meta_io['station_volume'], 9)
            self.assertEqual(meta_io['long_record_read'], 36)
            self.assertEqual(meta_io['short_record_read'], 26)
            self.assertEqual(meta_io['start_time_series'], 34)
            self.assertEqual(meta_io['end_time_series'], 28)

            meta = md.meta['miniseed_header_percentages']
            meta_dq = meta['data_quality_flags']
            _assert_float_equal(meta_dq['glitches'], 9 / 0.58)
            _assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58)
            _assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58)
            _assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58)
            _assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58)
            _assert_float_equal(meta_dq['spikes'], 35 / 0.58)
            _assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58)
            _assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58)

            meta_af = meta['activity_flags']
            _assert_float_equal(meta_af['calibration_signal'], 11 / 0.58)
            _assert_float_equal(meta_af['event_begin'], 36 / 0.58)
            _assert_float_equal(meta_af['event_end'], 36 / 0.58)
            _assert_float_equal(meta_af['event_in_progress'], 20 / 0.58)
            _assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58)

            meta_io = meta['io_and_clock_flags']
            _assert_float_equal(meta_io['clock_locked'], 34 / 0.58)
            _assert_float_equal(meta_io['station_volume'], 9 / 0.58)
            _assert_float_equal(meta_io['long_record_read'], 36 / 0.58)
            _assert_float_equal(meta_io['short_record_read'], 26 / 0.58)
            _assert_float_equal(meta_io['start_time_series'], 34 / 0.58)
            _assert_float_equal(meta_io['end_time_series'], 28 / 0.58)

            ref = md.meta['miniseed_header_percentages']
            self.assertEqual(ref['timing_quality_mean'], None)
            self.assertEqual(ref['timing_quality_min'], None)
            self.assertEqual(ref['timing_quality_max'], None)