def test_read_reftek130_steim2(self):
"""
Test reading a steim2 encoded data file.
Unpacking of data is tested separately so just checking a few samples
at the start should suffice.
"""
st = _read_reftek130(
self.reftek_file_steim2, network="XX", location="01",
component_codes=["1", "2", "3"],
sort_permuted_package_sequence=True)
# note: test data has stream name defined as 'DS 1', so we end up with
# non-SEED conforming channel codes which is expected
self.assertEqual(len(st), 3)
self.assertEqual(len(st[0]), 3788)
self.assertEqual(len(st[1]), 3788)
self.assertEqual(len(st[2]), 3788)
self.assertEqual(st[0].id, 'XX.TL01.01.DS 11')
self.assertEqual(st[1].id, 'XX.TL01.01.DS 12')
self.assertEqual(st[2].id, 'XX.TL01.01.DS 13')
np.testing.assert_array_equal(
st[0].data[:5], [26814, 26823, 26878, 26941, 26942])
np.testing.assert_array_equal(
st[1].data[:5], [-1987, -1984, -1959, -1966, -1978])
np.testing.assert_array_equal(
st[2].data[:5], [-2404, -2376, -2427, -2452, -2452])
def test_read_file_perturbed_packet_sequence(self):
"""
Test data read from file when packet sequence is perturbed. This makes
sure that data is read correctly even when the array storing the
packets gets permuted and thus becomes incontiguous for C.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and move some packets around in
# the file
# (packets are 1024 byte each)
tmp1 = fh2.read(1024 * 2)
tmp2 = fh2.read(1024 * 4)
tmp3 = fh2.read(1024 * 1)
tmp4 = fh2.read(1024 * 2)
tmp5 = fh2.read(1024 * 3)
tmp6 = fh2.read()
fh.write(tmp1)
fh.write(tmp3)
fh.write(tmp5)
fh.write(tmp6)
fh.write(tmp2)
fh.write(tmp4)
fh.seek(0)
# try to read file, finding a non-contiguous packet sequence
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"])
st_reftek.merge(-1)
self.assertEqual(len(w), 1)
self.assertEqual(str(w[0].message),
'Detected permuted packet sequence, sorting.')
self._assert_reftek130_test_stream(st_reftek)
def test_truncated_last_packet(self):
"""
Test that reading the file works, if the ET packet at the end is
truncated and thus omitted.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and truncate last (ET) packet
# (packets are 1024 byte each)
tmp = fh2.read()
tmp = tmp[:-10]
fh.write(tmp)
fh.seek(0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(fh.name,
network="XX",
location="01",
component_codes=["1", "2", "3"])
self.assertEqual(len(w), 2)
# we get two warnings, one about the truncated packet and one about the
# missing last (ET) packet
self.assertEqual(
str(w[0].message), 'Length of data not a multiple of '
'1024. Data might be truncated. Dropping 1014 '
'byte(s) at the end.')
self.assertEqual(
str(w[1].message),
'No event trailer (ET) packets in packet sequence. File might be '
'truncated.')
# data should be read OK aside from the warnings
self._assert_reftek130_test_stream(st_reftek)
def test_read_reftek130_no_component_codes_specified(self):
"""
Test reading reftek 130 data file not providing component codes
(relying on the information in header packet).
rt_mseed fills in network as "XX", location as "01" and channels as
"001", "002", "003".
"""
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(
self.reftek_file, network="XX", location="01",
sort_permuted_package_sequence=True)
self.assertEqual(len(w), 8)
for w_ in w:
self.assertEqual(
str(w_.message),
'No channel code specified in the data file and no component '
'codes specified. Using stream label and number of channel in '
'file as channel codes.')
# check that channel codes are set with stream label from EH packet +
# enumerated channel number starting at 0
for tr, cha in zip(st_reftek, ('EH0', 'EH0', 'EH0', 'EH1', 'EH1',
'EH1', 'EH2', 'EH2')):
self.assertEqual(tr.stats.channel, cha)
for tr in st_reftek:
# need to adapt channel codes to compare against mseed stream now..
tr.stats.channel = (
'EH' + {'0': '1', '1': '2', '2': '3'}[tr.stats.channel[-1]])
self._assert_reftek130_test_stream(st_reftek)
def test_read_reftek130_no_component_codes_specified(self):
"""
Test reading reftek 130 data file not providing component codes
(relying on the information in header packet).
rt_mseed fills in network as "XX", location as "01" and channels as
"001", "002", "003".
"""
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(self.reftek_file,
network="XX",
location="01")
self.assertEqual(len(w), 8)
for w_ in w:
self.assertEqual(
str(w_.message),
'No channel code specified in the data file and no component '
'codes specified. Using stream label and number of channel in '
'file as channel codes.')
# check that channel codes are set with stream label from EH packet +
# enumerated channel number starting at 0
for tr, cha in zip(
st_reftek,
('EH0', 'EH0', 'EH0', 'EH1', 'EH1', 'EH1', 'EH2', 'EH2')):
self.assertEqual(tr.stats.channel, cha)
for tr in st_reftek:
# need to adapt channel codes to compare against mseed stream now..
tr.stats.channel = ('EH' + {
'0': '1',
'1': '2',
'2': '3'
}[tr.stats.channel[-1]])
self._assert_reftek130_test_stream(st_reftek)
def test_missing_event_trailer_packet(self):
"""
Test that reading the file if the ET packet is missing works and a
warning is shown.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and omit last (ET) packet
# (packets are 1024 byte each)
tmp = fh2.read()
tmp = tmp[:-1024]
fh.write(tmp)
fh.seek(0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"],
sort_permuted_package_sequence=True)
self.assertEqual(len(w), 1)
self.assertEqual(
str(w[0].message),
'No event trailer (ET) packets in packet sequence. File might be '
'truncated.')
self._assert_reftek130_test_stream(st_reftek)
def test_truncated_last_packet(self):
"""
Test that reading the file works, if the ET packet at the end is
truncated and thus omitted.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and truncate last (ET) packet
# (packets are 1024 byte each)
tmp = fh2.read()
tmp = tmp[:-10]
fh.write(tmp)
fh.seek(0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"],
sort_permuted_package_sequence=True)
self.assertEqual(len(w), 2)
# we get two warnings, one about the truncated packet and one about the
# missing last (ET) packet
self.assertEqual(str(w[0].message), 'Length of data not a multiple of '
'1024. Data might be truncated. Dropping 1014 '
'byte(s) at the end.')
self.assertEqual(
str(w[1].message),
'No event trailer (ET) packets in packet sequence. File might be '
'truncated.')
# data should be read OK aside from the warnings
self._assert_reftek130_test_stream(st_reftek)
def test_read_reftek130_steim2(self):
"""
Test reading a steim2 encoded data file.
Unpacking of data is tested separately so just checking a few samples
at the start should suffice.
"""
st = _read_reftek130(self.reftek_file_steim2,
network="XX",
location="01",
component_codes=["1", "2", "3"])
# note: test data has stream name defined as 'DS 1', so we end up with
# non-SEED conforming channel codes which is expected
self.assertEqual(len(st), 3)
self.assertEqual(len(st[0]), 3788)
self.assertEqual(len(st[1]), 3788)
self.assertEqual(len(st[2]), 3788)
self.assertEqual(st[0].id, 'XX.TL01.01.DS 11')
self.assertEqual(st[1].id, 'XX.TL01.01.DS 12')
self.assertEqual(st[2].id, 'XX.TL01.01.DS 13')
np.testing.assert_array_equal(st[0].data[:5],
[26814, 26823, 26878, 26941, 26942])
np.testing.assert_array_equal(st[1].data[:5],
[-1987, -1984, -1959, -1966, -1978])
np.testing.assert_array_equal(st[2].data[:5],
[-2404, -2376, -2427, -2452, -2452])
def test_read_file_perturbed_packet_sequence(self):
"""
Test data read from file when packet sequence is perturbed. This makes
sure that data is read correctly even when the array storing the
packets gets permuted and thus becomes incontiguous for C.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and move some packets around in
# the file
# (packets are 1024 byte each)
tmp1 = fh2.read(1024 * 2)
tmp2 = fh2.read(1024 * 4)
tmp3 = fh2.read(1024 * 1)
tmp4 = fh2.read(1024 * 2)
tmp5 = fh2.read(1024 * 3)
tmp6 = fh2.read()
fh.write(tmp1)
fh.write(tmp3)
fh.write(tmp5)
fh.write(tmp6)
fh.write(tmp2)
fh.write(tmp4)
fh.seek(0)
# try to read file, finding a non-contiguous packet sequence
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"])
st_reftek.merge(-1)
self.assertEqual(len(w), 1)
self.assertEqual(str(w[0].message),
'Detected permuted packet sequence, sorting.')
self._assert_reftek130_test_stream(st_reftek)
def test_missing_event_trailer_packet(self):
"""
Test that reading the file if the ET packet is missing works and a
warning is shown.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and omit last (ET) packet
# (packets are 1024 byte each)
tmp = fh2.read()
tmp = tmp[:-1024]
fh.write(tmp)
fh.seek(0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
st_reftek = _read_reftek130(fh.name,
network="XX",
location="01",
component_codes=["1", "2", "3"])
self.assertEqual(len(w), 1)
self.assertEqual(
str(w[0].message),
'No event trailer (ET) packets in packet sequence. File might be '
'truncated.')
self._assert_reftek130_test_stream(st_reftek)
def test_no_eh_et_packet(self):
"""
Test error messages when reading a file without any EH/ET packet.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and omit first and last (EH/ET)
# packet
# (packets are 1024 byte each)
tmp = fh2.read()
fh.write(tmp[1024:-1024])
fh.seek(0)
with self.assertRaises(Reftek130Exception) as context:
_read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"])
self.assertEqual(
str(context.exception),
"Reftek data contains data packets without corresponding header "
"or trailer packet.")
def test_no_eh_et_packet(self):
"""
Test error messages when reading a file without any EH/ET packet.
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and omit first and last (EH/ET)
# packet
# (packets are 1024 byte each)
tmp = fh2.read()
fh.write(tmp[1024:-1024])
fh.seek(0)
with self.assertRaises(Reftek130Exception) as context:
_read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"])
self.assertEqual(
str(context.exception),
"Reftek data contains data packets without corresponding header "
"or trailer packet.")
def test_drop_not_implemented_packets(self):
"""
Test error message when some not implemented packet types are dropped
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and write the last three packets
# with a different packet type
# (packets are 1024 byte each)
tmp = fh2.read()[:-(1024 * 3)]
fh2.seek(-(1024 * 3), 2)
tmp2 = fh2.read(1024)
tmp3 = fh2.read(1024)
tmp4 = fh2.read(1024)
# write last three packages with some different packet types
fh.write(tmp)
fh.write(b"AA")
fh.write(tmp2[2:])
fh.write(b"AA")
fh.write(tmp3[2:])
fh.write(b"BB")
fh.write(tmp4[2:])
fh.seek(0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
_read_reftek130(fh.name,
network="XX",
location="01",
component_codes=["1", "2", "3"],
sort_permuted_package_sequence=True)
self.assertEqual(len(w), 2)
self.assertTrue(
re.match(
r"Encountered some packets of types that are not implemented "
r"yet \(types: \[b?'AA', b?'BB'\]\). Dropped 3 packets "
r"overall.", str(w[0].message)))
# this message we get because ET packet at end is now missing
self.assertEqual(
str(w[1].message),
'No event trailer (ET) packets in packet sequence. File might be '
'truncated.')
def test_drop_not_implemented_packets(self):
"""
Test error message when some not implemented packet types are dropped
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and write the last three packets
# with a different packet type
# (packets are 1024 byte each)
tmp = fh2.read()[:-(1024 * 3)]
fh2.seek(-(1024 * 3), 2)
tmp2 = fh2.read(1024)
tmp3 = fh2.read(1024)
tmp4 = fh2.read(1024)
# write last three packages with some different packet types
fh.write(tmp)
fh.write(b"AA")
fh.write(tmp2[2:])
fh.write(b"AA")
fh.write(tmp3[2:])
fh.write(b"BB")
fh.write(tmp4[2:])
fh.seek(0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
_read_reftek130(
fh.name, network="XX", location="01",
component_codes=["1", "2", "3"],
sort_permuted_package_sequence=True)
self.assertEqual(len(w), 2)
self.assertTrue(
re.match(
r"Encountered some packets of types that are not implemented "
r"yet \(types: \[b?'AA', b?'BB'\]\). Dropped 3 packets "
r"overall.",
str(w[0].message)))
# this message we get because ET packet at end is now missing
self.assertEqual(
str(w[1].message),
'No event trailer (ET) packets in packet sequence. File might be '
'truncated.')
def test_warning_disturbed_packet_sequence(self):
"""
Test warning message when packet sequence is non-contiguous (one packet
missing).
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and omit one packet
# (packets are 1024 byte each)
fh.write(fh2.read(1024 * 5))
fh2.seek(1024 * 6)
fh.write(fh2.read())
fh.seek(0)
# try to read file, finding a non-contiguous packet sequence
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
_read_reftek130(fh.name, network="XX", location="01",
component_codes=["1", "2", "3"])
self.assertEqual(len(w), 1)
self.assertEqual(str(w[0].message),
'Detected a non-contiguous packet sequence!')
def test_read_reftek130_steim1(self):
"""
Test original reftek 130 data file against miniseed files converted
using "rt_mseed" utility from Trimble/Reftek.
rt_mseed fills in network as "XX", location as "01" and channels as
"001", "002", "003".
"""
st_reftek = _read_reftek130(
self.reftek_file, network="XX", location="01",
component_codes=["1", "2", "3"])
self._assert_reftek130_test_stream(st_reftek)
def test_warning_disturbed_packet_sequence(self):
"""
Test warning message when packet sequence is non-contiguous (one packet
missing).
"""
with NamedTemporaryFile() as fh:
with open(self.reftek_file, 'rb') as fh2:
# write packages to the file and omit one packet
# (packets are 1024 byte each)
fh.write(fh2.read(1024 * 5))
fh2.seek(1024 * 6)
fh.write(fh2.read())
fh.seek(0)
# try to read file, finding a non-contiguous packet sequence
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
_read_reftek130(fh.name, network="XX", location="01",
component_codes=["1", "2", "3"])
self.assertEqual(len(w), 1)
self.assertEqual(str(w[0].message),
'Detected a non-contiguous packet sequence!')
def test_read_reftek130(self):
"""
Test original reftek 130 data file against miniseed files converted
using "rt_mseed" utility from Trimble/Reftek.
rt_mseed fills in network as "XX", location as "01" and channels as
"001", "002", "003".
"""
st_reftek = _read_reftek130(
self.reftek_file, network="XX", location="01",
component_codes=["1", "2", "3"])
self._assert_reftek130_test_stream(st_reftek)
def test_reading_file_with_encoding_32(self):
"""
Test reading a file with encoding '32' (uncompressed 32 bit integer)
Only tests unpacked sample data, everything else should be covered by
existing tests.
"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
st = _read_reftek130(self.reftek_file_32)
# read expected data
npz = np.load(self.reftek_file_32_npz)
# compare it
self.assertEqual(len(st), 3)
for tr, (_, expected) in zip(st, sorted(npz.items())):
np.testing.assert_array_equal(expected, tr.data)
def test_reading_file_with_encoding_32(self):
"""
Test reading a file with encoding '32' (uncompressed 32 bit integer)
Only tests unpacked sample data, everything else should be covered by
existing tests.
"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
st = _read_reftek130(self.reftek_file_32)
# read expected data
npz = np.load(self.reftek_file_32_npz)
# compare it
self.assertEqual(len(st), 3)
for tr, (_, expected) in zip(st, sorted(npz.items())):
np.testing.assert_array_equal(expected, tr.data)