def test_extraction_header_flags_complex(self):
"""
Tests the flag extraction in a complex record situation
Three records, with records 2 & 3 two overlapping 50% and
a 50% record length gap between record 1 and 2.
Rules for overlaps with different bits are as follows:
Records are sorted from end to start by endtime and processed
in this order. Each consecutive record occupies a time-range
that can no longer be used by another record. In the following
example, the third record is dominant over the second record
because it is processed first and occupies the time range.
Therefore the bit in this entire range is set to 1, despite
partially overlapping with a record with its bit set to 0
Bits in the test are set as shown
[ ==1== ]
[ ==1== ]...[ ==0== ]
| |
START END
25 125
[RECORD 1 (1)] = 0 - 50 [clock_locked: 1]
[RECORD 2 (0)] = 75 - 125 [clock_locked: 0]
[RECORD 3 (1)] = 100 - 150 [clock_locked: 1]
With starttime = 25 and endtime = 125
The clock_locked percentage should thus be exactly 50.0%
"""
# Couldn't properly break this line following PEP8 so use
# shorter notation .....
short = NamedTemporaryFile
with short() as tf1, short() as tf2, short() as tf3:
_create_mseed_file(tf1.name, record_count=1,
starttime=obspy.UTCDateTime(0),
seed=12345, flags={
'io_and_clock_flags': {
"clock_locked": 1}})
_create_mseed_file(tf2.name, record_count=1,
starttime=obspy.UTCDateTime(75),
seed=12345, flags={
'io_and_clock_flags': {
"clock_locked": 0}})
_create_mseed_file(tf3.name, record_count=1,
starttime=obspy.UTCDateTime(100),
seed=12345, flags={
'io_and_clock_flags': {
"clock_locked": 1}})
md = MSEEDMetadata([tf1.name, tf2.name, tf3.name],
starttime=obspy.UTCDateTime(25),
endtime=obspy.UTCDateTime(125), add_flags=True)
io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"]
self.assertEqual(io_f["clock_locked"], 50.0)
def test_extraction_header_flags_complex(self):
"""
Tests the flag extraction in a complex record situation
Three records, with records 2 & 3 two overlapping 50% and
a 50% record length gap between record 1 and 2.
Rules for overlaps with different bits are as follows:
Records are sorted from end to start by endtime and processed
in this order. Each consecutive record occupies a time-range
that can no longer be used by another record. In the following
example, the third record is dominant over the second record
because it is processed first and occupies the time range.
Therefore the bit in this entire range is set to 1, despite
partially overlapping with a record with its bit set to 0
Bits in the test are set as shown
[ ==1== ]
[ ==1== ]...[ ==0== ]
| |
START END
25 125
[RECORD 1 (1)] = 0 - 50 [clock_locked: 1]
[RECORD 2 (0)] = 75 - 125 [clock_locked: 0]
[RECORD 3 (1)] = 100 - 150 [clock_locked: 1]
With starttime = 25 and endtime = 125
The clock_locked percentage should thus be exactly 50.0%
"""
# Couldn't properly break this line following PEP8 so use
# shorter notation .....
short = NamedTemporaryFile
with short() as tf1, short() as tf2, short() as tf3:
_create_mseed_file(tf1.name, record_count=1,
starttime=obspy.UTCDateTime(0),
seed=12345, flags={
'io_and_clock_flags': {
"clock_locked": 1}})
_create_mseed_file(tf2.name, record_count=1,
starttime=obspy.UTCDateTime(75),
seed=12345, flags={
'io_and_clock_flags': {
"clock_locked": 0}})
_create_mseed_file(tf3.name, record_count=1,
starttime=obspy.UTCDateTime(100),
seed=12345, flags={
'io_and_clock_flags': {
"clock_locked": 1}})
md = MSEEDMetadata([tf1.name, tf2.name, tf3.name],
starttime=obspy.UTCDateTime(25),
endtime=obspy.UTCDateTime(125), add_flags=True)
io_f = md.meta["miniseed_header_percentages"]["io_and_clock_flags"]
self.assertEqual(io_f["clock_locked"], 50.0)
def test_extraction_fixed_header_flags(self):
# Had to put positive_leap count to 0 to prevent
# end time from being wrong
with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2:
_create_mseed_file(tf1.name,
record_count=35,
starttime=obspy.UTCDateTime(0),
seed=12345,
flags={
'data_quality_flags': {
"amplifier_saturation": 25,
"digitizer_clipping": 12,
"spikes": 30,
"glitches": 6,
"missing_padded_data": 15,
"telemetry_sync_error": 16,
"digital_filter_charging": 4,
"suspect_time_tag": 8
},
'activity_flags': {
"calibration_signal": 10,
"time_correction_applied": 20,
"event_begin": 33,
"event_end": 33,
"positive_leap": 0,
"negative_leap": 10,
"event_in_progress": 15
},
'io_and_clock_flags': {
"station_volume": 8,
"long_record_read": 33,
"short_record_read": 24,
"start_time_series": 31,
"end_time_series": 24,
"clock_locked": 32
}
})
# Previous file ends exactly on 1750, start new file
# to prevent overlapping records. When records overlap
# their contributions should NOT be summed
_create_mseed_file(tf2.name,
record_count=23,
starttime=obspy.UTCDateTime(1750),
seed=12345,
flags={
'data_quality_flags': {
"amplifier_saturation": 5,
"digitizer_clipping": 7,
"spikes": 5,
"glitches": 3,
"missing_padded_data": 5,
"telemetry_sync_error": 3,
"digital_filter_charging": 4,
"suspect_time_tag": 2
},
'activity_flags': {
"calibration_signal": 1,
"time_correction_applied": 0,
"event_begin": 3,
"event_end": 3,
"positive_leap": 0,
"negative_leap": 1,
"event_in_progress": 5
},
'io_and_clock_flags': {
"station_volume": 1,
"long_record_read": 3,
"short_record_read": 2,
"start_time_series": 3,
"end_time_series": 4,
"clock_locked": 2
}
})
md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True)
def _assert_float_equal(a, b):
"""
Supplementary function to test floats to precision of 1E-6
"""
self.assertTrue(abs(a - b) < 1E-6)
# Sum up contributions from both files.
# Check percentages
meta = md.meta['miniseed_header_counts']
meta_dq = meta['data_quality_flags']
self.assertEqual(meta_dq['glitches'], 9)
self.assertEqual(meta_dq['amplifier_saturation'], 30)
self.assertEqual(meta_dq['digital_filter_charging'], 8)
self.assertEqual(meta_dq['digitizer_clipping'], 19)
self.assertEqual(meta_dq['missing_padded_data'], 20)
self.assertEqual(meta_dq['spikes'], 35)
self.assertEqual(meta_dq['suspect_time_tag'], 10)
self.assertEqual(meta_dq['telemetry_sync_error'], 19)
meta_af = meta['activity_flags']
self.assertEqual(meta_af['calibration_signal'], 11)
self.assertEqual(meta_af['event_begin'], 36)
self.assertEqual(meta_af['event_end'], 36)
self.assertEqual(meta_af['event_in_progress'], 20)
self.assertEqual(meta_af['time_correction_applied'], 20)
meta_io = meta['io_and_clock_flags']
self.assertEqual(meta_io['clock_locked'], 34)
self.assertEqual(meta_io['station_volume'], 9)
self.assertEqual(meta_io['long_record_read'], 36)
self.assertEqual(meta_io['short_record_read'], 26)
self.assertEqual(meta_io['start_time_series'], 34)
self.assertEqual(meta_io['end_time_series'], 28)
meta = md.meta['miniseed_header_percentages']
meta_dq = meta['data_quality_flags']
_assert_float_equal(meta_dq['glitches'], 9 / 0.58)
_assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58)
_assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58)
_assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58)
_assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58)
_assert_float_equal(meta_dq['spikes'], 35 / 0.58)
_assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58)
_assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58)
meta_af = meta['activity_flags']
_assert_float_equal(meta_af['calibration_signal'], 11 / 0.58)
_assert_float_equal(meta_af['event_begin'], 36 / 0.58)
_assert_float_equal(meta_af['event_end'], 36 / 0.58)
_assert_float_equal(meta_af['event_in_progress'], 20 / 0.58)
_assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58)
meta_io = meta['io_and_clock_flags']
_assert_float_equal(meta_io['clock_locked'], 34 / 0.58)
_assert_float_equal(meta_io['station_volume'], 9 / 0.58)
_assert_float_equal(meta_io['long_record_read'], 36 / 0.58)
_assert_float_equal(meta_io['short_record_read'], 26 / 0.58)
_assert_float_equal(meta_io['start_time_series'], 34 / 0.58)
_assert_float_equal(meta_io['end_time_series'], 28 / 0.58)
ref = md.meta['miniseed_header_percentages']
self.assertEqual(ref['timing_quality_mean'], None)
self.assertEqual(ref['timing_quality_min'], None)
self.assertEqual(ref['timing_quality_max'], None)
def test_extraction_fixed_header_flags(self):
# Had to put positive_leap count to 0 to prevent
# end time from being wrong
with NamedTemporaryFile() as tf1, NamedTemporaryFile() as tf2:
_create_mseed_file(tf1.name, record_count=35,
starttime=obspy.UTCDateTime(0),
seed=12345, flags={
'data_quality_flags': {
"amplifier_saturation": 25,
"digitizer_clipping": 12,
"spikes": 30,
"glitches": 6,
"missing_padded_data": 15,
"telemetry_sync_error": 16,
"digital_filter_charging": 4,
"suspect_time_tag": 8},
'activity_flags': {
"calibration_signal": 10,
"time_correction_applied": 20,
"event_begin": 33,
"event_end": 33,
"positive_leap": 0,
"negative_leap": 10,
"event_in_progress": 15},
'io_and_clock_flags': {
"station_volume": 8,
"long_record_read": 33,
"short_record_read": 24,
"start_time_series": 31,
"end_time_series": 24,
"clock_locked": 32}})
# Previous file ends exactly on 1750, start new file
# to prevent overlapping records. When records overlap
# their contributions should NOT be summed
_create_mseed_file(tf2.name, record_count=23,
starttime=obspy.UTCDateTime(1750),
seed=12345, flags={
'data_quality_flags': {
"amplifier_saturation": 5,
"digitizer_clipping": 7,
"spikes": 5,
"glitches": 3,
"missing_padded_data": 5,
"telemetry_sync_error": 3,
"digital_filter_charging": 4,
"suspect_time_tag": 2},
'activity_flags': {
"calibration_signal": 1,
"time_correction_applied": 0,
"event_begin": 3,
"event_end": 3,
"positive_leap": 0,
"negative_leap": 1,
"event_in_progress": 5},
'io_and_clock_flags': {
"station_volume": 1,
"long_record_read": 3,
"short_record_read": 2,
"start_time_series": 3,
"end_time_series": 4,
"clock_locked": 2}})
md = MSEEDMetadata([tf1.name, tf2.name], add_flags=True)
def _assert_float_equal(a, b):
"""
Supplementary function to test floats to precision of 1E-6
"""
self.assertTrue(abs(a - b) < 1E-6)
# Sum up contributions from both files.
# Check percentages
meta = md.meta['miniseed_header_counts']
meta_dq = meta['data_quality_flags']
self.assertEqual(meta_dq['glitches'], 9)
self.assertEqual(meta_dq['amplifier_saturation'], 30)
self.assertEqual(meta_dq['digital_filter_charging'], 8)
self.assertEqual(meta_dq['digitizer_clipping'], 19)
self.assertEqual(meta_dq['missing_padded_data'], 20)
self.assertEqual(meta_dq['spikes'], 35)
self.assertEqual(meta_dq['suspect_time_tag'], 10)
self.assertEqual(meta_dq['telemetry_sync_error'], 19)
meta_af = meta['activity_flags']
self.assertEqual(meta_af['calibration_signal'], 11)
self.assertEqual(meta_af['event_begin'], 36)
self.assertEqual(meta_af['event_end'], 36)
self.assertEqual(meta_af['event_in_progress'], 20)
self.assertEqual(meta_af['time_correction_applied'], 20)
meta_io = meta['io_and_clock_flags']
self.assertEqual(meta_io['clock_locked'], 34)
self.assertEqual(meta_io['station_volume'], 9)
self.assertEqual(meta_io['long_record_read'], 36)
self.assertEqual(meta_io['short_record_read'], 26)
self.assertEqual(meta_io['start_time_series'], 34)
self.assertEqual(meta_io['end_time_series'], 28)
meta = md.meta['miniseed_header_percentages']
meta_dq = meta['data_quality_flags']
_assert_float_equal(meta_dq['glitches'], 9 / 0.58)
_assert_float_equal(meta_dq['amplifier_saturation'], 30 / 0.58)
_assert_float_equal(meta_dq['digital_filter_charging'], 8 / 0.58)
_assert_float_equal(meta_dq['digitizer_clipping'], 19 / 0.58)
_assert_float_equal(meta_dq['missing_padded_data'], 20 / 0.58)
_assert_float_equal(meta_dq['spikes'], 35 / 0.58)
_assert_float_equal(meta_dq['suspect_time_tag'], 10 / 0.58)
_assert_float_equal(meta_dq['telemetry_sync_error'], 19 / 0.58)
meta_af = meta['activity_flags']
_assert_float_equal(meta_af['calibration_signal'], 11 / 0.58)
_assert_float_equal(meta_af['event_begin'], 36 / 0.58)
_assert_float_equal(meta_af['event_end'], 36 / 0.58)
_assert_float_equal(meta_af['event_in_progress'], 20 / 0.58)
_assert_float_equal(meta_af['time_correction_applied'], 20 / 0.58)
meta_io = meta['io_and_clock_flags']
_assert_float_equal(meta_io['clock_locked'], 34 / 0.58)
_assert_float_equal(meta_io['station_volume'], 9 / 0.58)
_assert_float_equal(meta_io['long_record_read'], 36 / 0.58)
_assert_float_equal(meta_io['short_record_read'], 26 / 0.58)
_assert_float_equal(meta_io['start_time_series'], 34 / 0.58)
_assert_float_equal(meta_io['end_time_series'], 28 / 0.58)
ref = md.meta['miniseed_header_percentages']
self.assertEqual(ref['timing_quality_mean'], None)
self.assertEqual(ref['timing_quality_min'], None)
self.assertEqual(ref['timing_quality_max'], None)
