def test_get_multi(self):
"""
Test that the parser will read the data from a file and return multiple
valid particles and values within the particles.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 0}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
self.test01['byte_count'] = [1338, 1338, 1338]
# get 3 records from the parser, and compare to expected values
particles = self.parser.get_records(3)
self.parse_particles(particles)
self.assert_result(self.test01, self.parsed_data, particles)
def test_get_data(self):
"""
Test that the parser will read the data from a file and return valid
particles and values within the particles.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 0}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
self.test01['byte_count'] = [446, 892, 1338]
# get three records from the parser, one at a time, and compare to expected values
for i in range(0, 2):
particle = self.parser.get_records(1)
self.parse_particles(particle)
self.assert_result(self.test01, self.parsed_data, particle, i)
def test_set_state(self):
"""
Test that the parser will start up at some mid-point in the file, and
continue reading data from a file, returning valid particles and values
within the particles.
This tests the ability to set the POSITION state and have the parser
pick up from there.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 10704}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
# get 3 records from the parser, and compare to expected values
particles = self.parser.get_records(3)
self.parse_particles(particles)
self.assert_result(self.test03, self.parsed_data, particles)
def _build_parser(self, parser_state, infile):
config = self._parser_config
config.update({
'particle_module': 'mi.dataset.parser.adcpa',
'particle_class': 'ADCPA_PD0_PARSED_DataParticle'
})
log.debug("MYCONFIG: %s", config)
self._parser = AdcpaParser(config, parser_state, infile,
self._save_parser_state,
self._data_callback)
return self._parser
def test_get_bottom(self):
"""
Test that the parser will start up at some mid-point in the file, and
continue reading data from a file, returning valid particles and values
within the particles.
This duplicates test_set_state above, and extends it by grabbing the
additional bottom-tracking (BT) data that will appear at this point in
the file. This validates that the parser will correctly start adding
the BT data when it is encountered. The first 2 particles will not have
BT data, while the last three will.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 90538}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
# get 3 records from the parser, and compare to expected values
particles = self.parser.get_records(5)
self.parse_particles(particles)
self.assert_result(self.test04, self.parsed_data, particles)
class AdcpaParserUnitTestCase(ParserUnitTestCase):
"""
ADCPA Parser unit test suite
"""
# Set the source of the test data to a known file (collected during the PVT
# testing off the Oregon coast in 2011). This file contains both regular
# ensembles and bottom-tracking data (starts about 2/3 of the way through
# the file).
TEST_DIR = os.path.dirname(__file__)
TEST_DATA = os.path.join(TEST_DIR, 'LA101636.PD0')
def setUp(self):
ParserUnitTestCase.setUp(self)
# configure parser for testing
self.config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.adcpa',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'ADCPA_PD0_PARSED_DataParticle'
}
self.stream_handle = open(AdcpaParserUnitTestCase.TEST_DATA, 'rb')
self.position_callback_value = None
self.publish_callback_value = None
# Set known values for key elements of the particle to validate
# against. These values were created by opening the test data file in
# Matlab using a function called PD0decoder provided by Jerry Mullison
# <*****@*****.**> of Teledyne RD Instruments. Results of
# decoding the file using that file are compared to the results of this
# parser to confirm functionality.
### Test 1 and 2, Read first 3 ensembles, either one at a time or all
### at once, and compare results to values below.
self.test01 = {}
self.test01['internal_timestamp'] = [
3535202174.300002,
3535202175.940005,
3535202177.479998
]
self.test01['ensemble_start_time'] = [
1326213374.300002,
1326213375.940005,
1326213377.479998,
]
self.test01['echo_intensity_beam1'] = [
[114, 96, 84, 74, 73, 71, 68, 68, 68, 68, 67, 67, 67, 66, 66],
[113, 95, 81, 76, 72, 71, 70, 68, 68, 68, 68, 67, 67, 66, 67],
[117, 93, 82, 77, 74, 72, 69, 69, 68, 68, 68, 67, 67, 67, 67]
]
self.test01['correlation_magnitude_beam1'] = [
[106, 126, 106, 89, 66, 49, 40, 34, 24, 17, 15, 12, 0, 0, 0],
[105, 121, 101, 99, 65, 51, 40, 33, 29, 19, 15, 13, 0, 0, 0],
[106, 118, 105, 97, 72, 52, 45, 38, 22, 21, 20, 16, 0, 0, 0]
]
self.test01['percent_good_3beam'] = [
[0, 0, 20, 100, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 20, 100, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 20, 80, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
]
self.test01['water_velocity_east'] = [
[23, 23, 45, 110, 68, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-94, -94, -167, -454, -176, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-42, -42, -76, -184, -121, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test01['water_velocity_north'] = [
[-62, -62, -53, -27, -17, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[252, 252, 222, 105, 41, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[111, 111, 97, 42, 28, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test01['water_velocity_up'] = [
[84, 84, 84, 86, 53, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-339, -339, -341, -348, -136, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-150, -150, -151, -141, -93, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test01['error_velocity'] = [
[0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test01['eastward_bt_velocity'] = []
self.test01['northward_bt_velocity'] = []
self.test01['upward_bt_velocity'] = []
self.test01['error_bt_velocity'] = []
### Test 3, Skip ahead 25 ensembles (11150 bytes), get 3 records and
### compare results to values below.
self.test03 = {}
self.test03['byte_count'] = [12042, 12042, 12042]
self.test03['internal_timestamp'] = [
3535202213.179996,
3535202214.900003,
3535202216.600003
]
self.test03['ensemble_start_time'] = [
1326213413.179996,
1326213414.900003,
1326213416.600004
]
self.test03['echo_intensity_beam1'] = [
[123, 94, 84, 80, 72, 69, 70, 69, 68, 68, 67, 67, 67, 67, 67],
[123, 93, 83, 76, 71, 71, 69, 69, 69, 67, 67, 67, 67, 67, 67],
[127, 89, 80, 76, 73, 71, 71, 69, 68, 69, 68, 67, 67, 68, 67]
]
self.test03['correlation_magnitude_beam1'] = [
[96, 123, 107, 106, 55, 55, 41, 25, 20, 15, 18, 14, 0, 0, 0],
[97, 117, 114, 91, 71, 52, 44, 34, 16, 16, 15, 13, 0, 0, 0],
[92, 113, 109, 94, 72, 61, 37, 36, 22, 16, 13, 17, 0, 0, 0]
]
self.test03['percent_good_3beam'] = [
[0, 0, 60, 100, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 60, 100, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 40, 90, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
]
self.test03['water_velocity_east'] = [
[22, 22, 110, 107, 360, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-77, -77, -268, -372, -349, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-37, -37, -88, -166, -432, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test03['water_velocity_north'] = [
[-59, -59, -23, -24, -82, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[206, 206, 128, 85, 80, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[98, 98, 78, 38, 100, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test03['water_velocity_up'] = [
[79, 79, 81, 81, 275, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-277, -277, -282, -285, -267, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[-133, -133, -134, -127, -331, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test03['error_velocity'] = [
[0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test03['eastward_bt_velocity'] = []
self.test03['northward_bt_velocity'] = []
self.test03['upward_bt_velocity'] = []
self.test03['error_bt_velocity'] = []
### Test 4, Skip ahead to records containing bottom-tracking. Read 2
### ensembles prior to bottom tracking and three after (ensembles
### 204-208, byte 90984), all at once and compare results to values
### below.
self.test04 = {}
self.test04['byte_count'] = [93017, 93017, 93017, 93017, 93017]
self.test04['internal_timestamp'] = [
3535202503.630000,
3535202505.349996,
3535202506.880003,
3535202509.079999,
3535202514.970000
]
self.test04['ensemble_start_time'] = [
1326213703.630000,
1326213705.349996,
1326213706.880002,
1326213709.079999,
1326213714.970000
]
self.test04['echo_intensity_beam1'] = [
[95, 86, 80, 79, 76, 71, 74, 86, 76, 67, 66, 66, 66, 66, 67],
[91, 92, 84, 79, 74, 72, 76, 86, 75, 67, 66, 67, 66, 66, 67],
[94, 88, 84, 80, 76, 71, 74, 76, 69, 67, 67, 71, 72, 71, 69],
[101, 88, 83, 78, 74, 72, 77, 75, 70, 69, 69, 69, 71, 74, 71],
[100, 91, 85, 77, 75, 79, 76, 72, 73, 74, 71, 72, 69, 68, 66]
]
self.test04['correlation_magnitude_beam1'] = [
[128, 118, 104, 109, 95, 74, 80, 109, 73, 21, 13, 12, 12, 11, 9],
[118, 125, 113, 107, 86, 76, 82, 109, 70, 19, 12, 11, 10, 13, 13],
[119, 119, 115, 113, 94, 70, 69, 63, 35, 12, 18, 33, 36, 41, 30],
[124, 119, 115, 107, 85, 73, 74, 57, 39, 27, 26, 26, 40, 45, 38],
[126, 121, 115, 98, 88, 93, 73, 48, 38, 48, 36, 35, 29, 22, 13]
]
self.test04['percent_good_3beam'] = [
[0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 10, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 50, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 10, 10, 0, 0]
]
self.test04['water_velocity_east'] = [
[32, 32, 32, 32, 32, 43, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[29, 29, 29, 29, 29, 36, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[27, 27, 27, 27, 28, 80, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[33, 33, 33, 33, 37, 58, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[35, 35, 35, 35, 35, 58, -32768, -32768, -32768, -32768, -32768, 43, 40, -32768, -32768]
]
self.test04['water_velocity_north'] = [
[8, 8, 8, 8, 8, -115, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[7, 7, 7, 7, 7, -55, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[6, 6, 6, 6, -2, -20, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[7, 7, 7, 7, -49, -93, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[7, 7, 7, 7, 7, -68, -32768, -32768, -32768, -32768, -32768, -115, -107, -32768, -32768]
]
self.test04['water_velocity_up'] = [
[127, 127, 127, 127, 127, 138, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[113, 113, 113, 113, 113, 123, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[106, 106, 106, 106, 107, 79, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[130, 130, 130, 130, 132, 128, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[127, 127, 127, 127, 127, 125, -32768, -32768, -32768, -32768, -32768, 133, 123, -32768, -32768]
]
self.test04['error_velocity'] = [
[0, 0, 0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, 0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768],
[0, 0, 0, 0, 0, 0, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768, -32768]
]
self.test04['eastward_bt_velocity'] = [-32768, -550, -356]
self.test04['northward_bt_velocity'] = [-32768, -200, -195]
self.test04['upward_bt_velocity'] = [-32768, 316, 273]
self.test04['error_bt_velocity'] = [-32768, -32768, -32768]
def tearDown(self):
self.stream_handle.close()
def pos_callback(self, pos, file_ingested):
""" Call back method to watch what comes in via the position callback """
self.position_callback_value = pos
self.file_ingested = file_ingested
def pub_callback(self, pub):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_value = pub
def parse_particles(self, particles):
"""
Parse a particle to get at some of the data for comparison purposes
"""
self.parsed_data = collections.defaultdict(list)
for i in range(0, len(particles)):
data = particles[i].generate_dict()
# parse the values from the dictionary
values = {}
for value in data['values']:
values[value['value_id']] = value['value']
# start extracting the data for use -- all ensembles
self.parsed_data['byte_count'].append(self.position_callback_value['position'])
self.parsed_data['internal_timestamp'].append(data['internal_timestamp'])
self.parsed_data['ensemble_start_time'].append(values['ensemble_start_time'])
self.parsed_data['echo_intensity_beam1'].append(values['echo_intensity_beam1'])
self.parsed_data['correlation_magnitude_beam1'].append(values['correlation_magnitude_beam1'])
self.parsed_data['percent_good_3beam'].append(values['percent_good_3beam'])
self.parsed_data['water_velocity_east'].append(values['water_velocity_east'])
self.parsed_data['water_velocity_north'].append(values['water_velocity_north'])
self.parsed_data['water_velocity_up'].append(values['water_velocity_up'])
self.parsed_data['error_velocity'].append(values['error_velocity'])
# ensembles containing bottom-tracking data
if 'eastward_bt_velocity' in values:
self.parsed_data['eastward_bt_velocity'].append(values['eastward_bt_velocity'])
self.parsed_data['northward_bt_velocity'].append(values['northward_bt_velocity'])
self.parsed_data['upward_bt_velocity'].append(values['upward_bt_velocity'])
self.parsed_data['error_bt_velocity'].append(values['error_bt_velocity'])
else:
self.parsed_data['eastward_bt_velocity'] = []
self.parsed_data['northward_bt_velocity'] = []
self.parsed_data['upward_bt_velocity'] = []
self.parsed_data['error_bt_velocity'] = []
def assert_result(self, test, parsed, particle, index=-1):
"""
Suite of tests to run against each returned particle and the parsed
data from same.
"""
self.assertEqual(self.publish_callback_value[:], particle[:])
self.assert_(isinstance(self.publish_callback_value, list))
# compare each key in the test and parsed dictionaries
for key in test:
# modify the test data to account for using only 1 index for
# test_get_data versus all three indexs for test_set_state.
if index == -1:
tdata = test[key]
else:
if len(test[key]) >= 1:
tdata = [test[key][index]]
else:
tdata = test[key]
# test the keys
if 'internal_timestamp' in key or 'ensemble_start_time' in key:
# slightly different test for these values as they are floats.
# this asserts that the difference between these values is less
# than 0.1 milliseconds.
compare = np.abs(np.array(tdata) - np.array(parsed[key])) <= 1e-5
self.assertTrue(compare.all())
else:
# otherwise they are all ints and should be exactly equal
self.assertEqual(tdata, parsed[key])
def test_get_data(self):
"""
Test that the parser will read the data from a file and return valid
particles and values within the particles.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 0}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
self.test01['byte_count'] = [446, 892, 1338]
# get three records from the parser, one at a time, and compare to expected values
for i in range(0, 2):
particle = self.parser.get_records(1)
self.parse_particles(particle)
self.assert_result(self.test01, self.parsed_data, particle, i)
def test_get_multi(self):
"""
Test that the parser will read the data from a file and return multiple
valid particles and values within the particles.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 0}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
self.test01['byte_count'] = [1338, 1338, 1338]
# get 3 records from the parser, and compare to expected values
particles = self.parser.get_records(3)
self.parse_particles(particles)
self.assert_result(self.test01, self.parsed_data, particles)
def test_set_state(self):
"""
Test that the parser will start up at some mid-point in the file, and
continue reading data from a file, returning valid particles and values
within the particles.
This tests the ability to set the POSITION state and have the parser
pick up from there.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 10704}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
# get 3 records from the parser, and compare to expected values
particles = self.parser.get_records(3)
self.parse_particles(particles)
self.assert_result(self.test03, self.parsed_data, particles)
def test_get_bottom(self):
"""
Test that the parser will start up at some mid-point in the file, and
continue reading data from a file, returning valid particles and values
within the particles.
This duplicates test_set_state above, and extends it by grabbing the
additional bottom-tracking (BT) data that will appear at this point in
the file. This validates that the parser will correctly start adding
the BT data when it is encountered. The first 2 particles will not have
BT data, while the last three will.
"""
# reset the parser
self.stream_handle.seek(0)
self.position = {StateKey.POSITION: 90538}
self.parser = AdcpaParser(self.config, self.position, self.stream_handle,
self.pos_callback, self.pub_callback)
# get 3 records from the parser, and compare to expected values
particles = self.parser.get_records(5)
self.parse_particles(particles)
self.assert_result(self.test04, self.parsed_data, particles)
