def test_bad_config(self):
"""
Test that a set of bad configurations produces the expected exceptions
"""
file_handle = open(os.path.join(RESOURCE_PATH, 'single.mrg'), 'rU')
# confirm a configuration exception occurs if no config is passed in
with self.assertRaises(ConfigurationException):
GliderParser({}, file_handle, self.exception_callback)
# confirm a config missing the particle class causes an exception
bad_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.glider'
}
with self.assertRaises(ConfigurationException):
GliderParser(bad_config, file_handle, self.exception_callback)
# confirm a config with a non existing class causes an exception
bad_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.glider',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'BadDataParticle'
}
with self.assertRaises(AttributeError):
GliderParser(bad_config, file_handle, self.exception_callback)
def test_bad_headers(self):
"""
Test that a file with a short header raises a sample exception
"""
# this file does not have enough header lines
file_handle = open(os.path.join(RESOURCE_PATH, 'short_header.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
# this file specifies a number of header lines other than 14
file_handle = open(os.path.join(RESOURCE_PATH, 'bad_num_header_lines.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
# this file specifies a number of label lines other than 3
file_handle = open(os.path.join(RESOURCE_PATH, 'bad_num_label_lines.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
def test_missing_time(self):
"""
Test that a file which is missing the required m_present_time field for timestamps raises a sample exception
"""
# this file is missing the m_present_time label
file_handle = open(os.path.join(RESOURCE_PATH, 'no_time_label.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
def test_missing_time(self):
"""
Test that a file which is missing the required m_present_time field for timestamps raises a sample exception
"""
# this file is missing the m_present_time label
file_handle = open(os.path.join(RESOURCE_PATH, 'no_time_label.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
def test_short_units(self):
"""
Test that if the number of label columns does not match the units number of columns an exception occurs
"""
# this file is has two columns removed from the data libe
file_handle = open(os.path.join(RESOURCE_PATH, 'short_units.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
def test_short_units(self):
"""
Test that if the number of label columns does not match the units number of columns an exception occurs
"""
# this file is has two columns removed from the data libe
file_handle = open(os.path.join(RESOURCE_PATH, 'short_units.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
parser.get_records(1)
def test_full(self):
"""
Test a full file and confirm the right number of particles is returned
"""
with open(os.path.join(RESOURCE_PATH, 'unit_514-2014-351-2-0.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(40)
# requested more than are available in file, should only be 10
self.assertEquals(len(particles), 31)
self.assertEqual(self.exception_callback_value, [])
def test_empty(self):
"""
An empty file will return a sample exception since it cannot read the header
"""
file_handle = open(os.path.join(RESOURCE_PATH, 'empty.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(1)
# requested more than are available in file, should only be 10
self.assertEquals(len(particles), 0)
def test_full(self):
"""
Test a full file and confirm the right number of particles is returned
"""
with open(os.path.join(RESOURCE_PATH, 'unit_514-2014-351-2-0.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(40)
# requested more than are available in file, should only be 10
self.assertEquals(len(particles), 31)
self.assertEqual(self.exception_callback_value, [])
def test_simple(self):
"""
Test a simple case that we can parse a single message
"""
with open(os.path.join(RESOURCE_PATH, 'single.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(1)
self.assert_particles(particles, "single.yml", RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_empty(self):
"""
An empty file will return a sample exception since it cannot read the header
"""
file_handle = open(os.path.join(RESOURCE_PATH, 'empty.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(1)
# requested more than are available in file, should only be 10
self.assertEquals(len(particles), 0)
def test_simple(self):
"""
Test a simple case that we can parse a single message
"""
with open(os.path.join(RESOURCE_PATH, 'single.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(1)
self.assert_particles(particles, "single.yml", RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_many(self):
"""
Test a simple case with more messages
"""
with open(os.path.join(RESOURCE_PATH, 'many.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(12)
# requested more than are available in file, should only be 10
self.assertEquals(len(particles), 10)
self.assert_particles(particles, "many.yml", RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_many(self):
"""
Test a simple case with more messages
"""
with open(os.path.join(RESOURCE_PATH, 'many.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(12)
# requested more than are available in file, should only be 10
self.assertEquals(len(particles), 10)
self.assert_particles(particles, "many.yml", RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_bad_sensors_per_cycle(self):
"""
Test that if the number of sensors per cycle from the header does not match that in the header that an
exception in the callback occurs, but processing continues
"""
with open(os.path.join(RESOURCE_PATH, 'bad_sensors_per_cycle.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(1)
self.assert_particles(particles, "single.yml", RESOURCE_PATH)
self.assertEqual(len(self.exception_callback_value), 1)
self.assertIsInstance(self.exception_callback_value[0], SampleException)
def test_bad_sensors_per_cycle(self):
"""
Test that if the number of sensors per cycle from the header does not match that in the header that an
exception in the callback occurs, but processing continues
"""
with open(os.path.join(RESOURCE_PATH, 'bad_sensors_per_cycle.mrg'), 'rU') as file_handle:
parser = GliderParser(self.config, file_handle, self.exception_callback)
particles = parser.get_records(1)
self.assert_particles(particles, "single.yml", RESOURCE_PATH)
self.assertEqual(len(self.exception_callback_value), 1)
self.assertIsInstance(self.exception_callback_value[0], SampleException)
def _build_parser(self, stream_handle):
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.glider',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'NutnrMDataParticle'
}
return GliderParser(config, stream_handle, self._exception_callback)
def _build_parser(self, parser_state, infile):
config = self._parser_config
config.update({
'particle_module': 'mi.dataset.parser.glider',
'particle_class': 'GgldrParadDelayedDataParticle'
})
log.debug("MYCONFIG: %s", config)
self._parser = GliderParser(config, parser_state, infile,
self._save_parser_state,
self._data_callback)
return self._parser
def process(self):
with open(self._source_file_path, 'rb') as stream_handle:
def exp_callback(exception):
self._particle_data_handler.setParticleDataCaptureFailure()
parser = GliderParser(self._config, stream_handle, exp_callback)
driver = DataSetDriver(parser, self._particle_data_handler)
driver.processFileStream()
return self._particle_data_handler
def process(self):
with open(self._sourceFilePath, 'rb') as stream_handle:
parser_state = None
def exp_callback(exception):
self._particleDataHdlrObj.setParticleDataCaptureFailure()
parser = GliderParser(self._config, stream_handle, exp_callback)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
return self._particleDataHdlrObj
def process(self):
log = get_logger()
with open(self._sourceFilePath, "rb") as file_handle:
def exception_callback(exception):
log.debug("Exception: %s", exception)
self._particleDataHdlrObj.setParticleDataCaptureFailure()
parser = GliderParser(self._parser_config, file_handle,
exception_callback)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
return self._particleDataHdlrObj
def process(self):
log = get_logger()
with open(self._sourceFilePath, 'rb') as file_handle:
def exception_callback(exception):
log.debug('Exception: %s', exception)
self._particleDataHdlrObj.setParticleDataCaptureFailure()
parser = GliderParser(self._config, None, file_handle,
lambda state, ingested: None,
lambda data: None, exception_callback)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
return self._particleDataHdlrObj
def process(self):
log = get_logger()
with open(self._source_file_path, 'rb') as file_handle:
def exception_callback(exception):
log.debug('Exception: %s', exception)
self._particle_data_handler.setParticleDataCaptureFailure()
parser = GliderParser(self._config,
file_handle,
exception_callback)
driver = DataSetDriver(parser, self._particle_data_handler)
driver.processFileStream()
return self._particle_data_handler
def _build_ctdgv_recovered_parser(self, parser_state, infile, data_key):
"""
Build and return the specified parser as indicated by the data_key.
"""
config = self._parser_config
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.glider',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'CtdgvRecoveredDataParticle'
})
parser = GliderParser(config,
parser_state,
infile,
lambda state, ingested: self._save_parser_state(state, data_key, ingested),
self._data_callback,
self._sample_exception_callback)
return parser
def reset_parser(self, state = {}):
self.state_callback_values = []
self.publish_callback_values = []
self.parser = GliderParser(self.config, state, self.test_data,
self.state_callback, self.pub_callback, self.error_callback)
class GliderParserUnitTestCase(ParserUnitTestCase):
"""
Glider Parser unit test base class and common tests.
"""
config = {}
def state_callback(self, state, file_ingested):
""" Call back method to watch what comes in via the state callback """
self.state_callback_values.append(state)
self.file_ingested = file_ingested
def pub_callback(self, particle):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_values.append(particle)
def error_callback(self, error):
""" Call back method to watch what comes in via the state callback """
self.error_callback_values.append(error)
def setUp(self):
ParserUnitTestCase.setUp(self)
def set_data(self, *args):
"""
Accept strings of data in args[] joined together and then a file handle
to the concatenated string is returned.
"""
io = StringIO()
for count, data in enumerate(args):
io.write(data)
log.debug("Test data file: %s", io.getvalue())
io.seek(0)
self.test_data = io
def set_data_file(self, filename):
"""
Set test to read from a file.
"""
self.test_data = open(filename, "r")
def reset_parser(self, state = {}):
self.state_callback_values = []
self.publish_callback_values = []
self.parser = GliderParser(self.config, state, self.test_data,
self.state_callback, self.pub_callback, self.error_callback)
def get_published_value(self):
return self.publish_callback_values.pop(0)
def get_state_value(self):
return self.state_callback_values.pop(0)
def assert_state(self, expected_position):
"""
Verify the state
"""
state = self.parser._read_state
log.debug("Current state: %s", state)
position = state.get(StateKey.POSITION)
self.assertEqual(position, expected_position)
def assert_no_more_data(self):
"""
Verify we don't find any other records in the data file.
"""
records = self.parser.get_records(1)
self.assertEqual(len(records), 0)
def assert_generate_particle(self, particle_type, values_dict = None, expected_position = None):
"""
Verify that we can generate a particle of the correct type and that
the state is set properly.
@param particle_type type of particle we are producing
@param values_dict key value pairs to test in the particle.
@param expected_position upon publication of the particle, what should the state position indicate.
"""
# ensure the callback queues are empty before we start
self.assertEqual(len(self.publish_callback_values), 0)
self.assertEqual(len(self.state_callback_values), 0)
records = self.parser.get_records(1)
self.assertIsNotNone(records)
self.assertIsInstance(records, list)
self.assertEqual(len(records), 1)
self.assertEqual(len(self.publish_callback_values), 1)
self.assertEqual(len(self.state_callback_values), 1)
particles = self.get_published_value()
self.assertEqual(len(particles), 1)
# Verify the data
if values_dict:
self.assert_particle_values(particles[0], values_dict)
# Verify the parser state
state = self.get_state_value()
log.debug("Published state: %s", state)
if expected_position:
position = state.get(StateKey.POSITION)
self.assertEqual(position, expected_position)
def assert_particle_values(self, particle, expected_values):
"""
Verify the data in expected values is the data in the particle
"""
data_dict = particle.generate_dict()
log.debug("Data in particle: %s", data_dict)
log.debug("Expected Data: %s", expected_values)
for key in expected_values.keys():
for value in data_dict['values']:
if value['value_id'] == key:
self.assertEqual(value['value'], expected_values[key])
def assert_type(self, records, particle_type):
for particle in records:
str_of_type = particle.type()
self.assertEqual(particle_type, str_of_type)
def assert_timestamp(self, ntp_timestamp, unix_timestamp):
ntp_stamp = ntplib.system_to_ntp_time(unix_timestamp)
assertion = np.allclose(ntp_timestamp, ntp_stamp)
self.assertTrue(assertion)
def test_init(self):
"""
Verify we can initialize
"""
self.set_data(HEADER)
self.reset_parser()
self.assert_state(1003)
self.set_data(HEADER2)
self.reset_parser()
self.assert_state(1004)
def test_exception(self):
with self.assertRaises(SampleException):
self.set_data("Foo")
self.reset_parser()
def test_chunker(self):
"""
Verify the chunker is returning values we expect.
"""
self.set_data(HEADER, CHUNKER_TEST)
self.reset_parser()
records = CHUNKER_TEST.strip("\n").split("\n")
log.debug("Expected Records: %s", records)
self.assertEqual(len(records), 2)
# Load all data into the chunker
self.parser.get_block(1024)
self.assertEqual(CHUNKER_TEST.strip("\n"), self.parser._chunker.buffer.strip("\n"))
(timestamp, data_record, start, end) = self.parser._chunker.get_next_data_with_index()
log.debug("Data Record: %s", data_record)
self.assertEqual(records[0]+"\n", data_record)
(timestamp, data_record, start, end) = self.parser._chunker.get_next_data_with_index()
self.assertEqual(records[1]+"\n", data_record)
def reset_parser(self):
self.test_file.seek(0)
self.position_callback_value = None
self.publish_callback_value = None
self.parser = GliderParser(self.config, self.position, self.test_file,
self.pos_callback, self.pub_callback)
class GliderParserUnitTestCase(ParserUnitTestCase):
"""
Glider Parser unit test base class and common tests.
"""
config = {}
def state_callback(self, state):
""" Call back method to watch what comes in via the state callback """
self.state_callback_values.append(state)
def pub_callback(self, particle):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_values.append(particle)
def setUp(self):
ParserUnitTestCase.setUp(self)
def set_data(self, *args):
"""
Accept strings of data in args[] joined together and then a file handle
to the concatenated string is returned.
"""
io = StringIO()
for count, data in enumerate(args):
io.write(data)
log.debug("Test data file: %s", io.getvalue())
io.seek(0)
self.test_data = io
def set_data_file(self, filename):
"""
Set test to read from a file.
"""
self.test_data = open(filename, "r")
def reset_parser(self, state = {}):
self.state_callback_values = []
self.publish_callback_values = []
self.parser = GliderParser(self.config, state, self.test_data,
self.state_callback, self.pub_callback)
def get_published_value(self):
return self.publish_callback_values.pop(0)
def get_state_value(self):
return self.state_callback_values.pop(0)
def assert_state(self, expected_position):
"""
Verify the state
"""
state = self.parser._read_state
log.debug("Current state: %s", state)
position = state.get(StateKey.POSITION)
self.assertEqual(position, expected_position)
def assert_no_more_data(self):
"""
Verify we don't find any other records in the data file.
"""
records = self.parser.get_records(1)
self.assertEqual(len(records), 0)
def assert_generate_particle(self, particle_type, values_dict = None, expected_position = None):
"""
Verify that we can generate a particle of the correct type and that
the state is set properly.
@param particle_type type of particle we are producing
@param values_dict key value pairs to test in the particle.
@param expected_position upon publication of the particle, what should the state position indicate.
"""
# ensure the callback queues are empty before we start
self.assertEqual(len(self.publish_callback_values), 0)
self.assertEqual(len(self.state_callback_values), 0)
records = self.parser.get_records(1)
self.assertIsNotNone(records)
self.assertIsInstance(records, list)
self.assertEqual(len(records), 1)
self.assertEqual(len(self.publish_callback_values), 1)
self.assertEqual(len(self.state_callback_values), 1)
particles = self.get_published_value()
self.assertEqual(len(particles), 1)
# Verify the data
if values_dict:
self.assert_particle_values(particles[0], values_dict)
# Verify the parser state
state = self.get_state_value()
log.debug("Published state: %s", state)
if expected_position:
position = state.get(StateKey.POSITION)
self.assertEqual(position, expected_position)
def assert_particle_values(self, particle, expected_values):
"""
Verify the data in expected values is the data in the particle
"""
data_dict = particle.generate_dict()
log.debug("Data in particle: %s", data_dict)
log.debug("Expected Data: %s", expected_values)
for key in expected_values.keys():
for value in data_dict['values']:
if value['value_id'] == key:
self.assertEqual(value['value'], expected_values[key])
def assert_type(self, records, particle_type):
for particle in records:
str_of_type = particle.type()
self.assertEqual(particle_type, str_of_type)
def assert_timestamp(self, ntp_timestamp, unix_timestamp):
ntp_stamp = ntplib.system_to_ntp_time(unix_timestamp)
assertion = np.allclose(ntp_timestamp, ntp_stamp)
self.assertTrue(assertion)
def test_init(self):
"""
Verify we can initialize
"""
self.set_data(HEADER)
self.reset_parser()
self.assert_state(1003)
self.set_data(HEADER2)
self.reset_parser()
self.assert_state(1004)
def test_exception(self):
with self.assertRaises(SampleException):
self.set_data("Foo")
self.reset_parser()
def test_chunker(self):
"""
Verify the chunker is returning values we expect.
"""
self.set_data(HEADER, CHUNKER_TEST)
self.reset_parser()
records = CHUNKER_TEST.strip("\n").split("\n")
log.debug("Expected Records: %s", records)
self.assertEqual(len(records), 2)
# Load all data into the chunker
self.parser.get_block(1024)
self.assertEqual(CHUNKER_TEST.strip("\n"), self.parser._chunker.buffer.strip("\n"))
(timestamp, data_record, start, end) = self.parser._chunker.get_next_data_with_index()
log.debug("Data Record: %s", data_record)
self.assertEqual(records[0]+"\n", data_record)
(timestamp, data_record, start, end) = self.parser._chunker.get_next_data_with_index()
self.assertEqual(records[1]+"\n", data_record)
def reset_parser(self, state = {}):
self.state_callback_values = []
self.publish_callback_values = []
self.parser = GliderParser(self.config, state, self.test_data,
self.state_callback, self.pub_callback)
def test_bad_headers(self):
"""
Test that a file with a short header raises a sample exception
"""
# this file does not have enough header lines
file_handle = open(os.path.join(RESOURCE_PATH, 'short_header.mrg'),
'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle,
self.exception_callback)
parser.get_records(1)
# this file specifies a number of header lines other than 14
file_handle = open(
os.path.join(RESOURCE_PATH, 'bad_num_header_lines.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle,
self.exception_callback)
parser.get_records(1)
# this file specifies a number of label lines other than 3
file_handle = open(
os.path.join(RESOURCE_PATH, 'bad_num_label_lines.mrg'), 'rU')
with self.assertRaises(DatasetParserException):
parser = GliderParser(self.config, file_handle,
self.exception_callback)
parser.get_records(1)
class GliderParserUnitTestCase(ParserUnitTestCase):
"""
Glider Parser unit test suite
"""
#log.debug('###########!! PWD is %s !!##########' % os.getcwd())
test_file = open('mi/dataset/parser/test/test_glider_data.mrg', 'r')
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.glider'}
def pos_callback(self, pos):
""" Call back method to watch what comes in via the position callback """
self.position_callback_value = pos
def pub_callback(self, pub):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_value = pub
def setUp(self):
ParserUnitTestCase.setUp(self)
# not a DataSourceLocation...its just the parser
self.position = {StateKey.POSITION: 0}
# take that, rewind it back ...
self.test_file.seek(0)
self.position_callback_value = None
self.publish_callback_value = None
def reset_parser(self):
self.test_file.seek(0)
self.position_callback_value = None
self.publish_callback_value = None
self.parser = GliderParser(self.config, self.position, self.test_file,
self.pos_callback, self.pub_callback)
def generic_particle_parse(self, particle_type, num_of_records, start_index=0):
self.reset_parser()
records = self.parser.get_records(num_of_records)
while records:
end_index = start_index + len(records)-1
self.assertEqual(records, self.publish_callback_value)
self.assert_type(records, particle_type)
self.assert_particles(records, start_index, end_index)
self.assertEqual(self.parser._state[StateKey.POSITION], positions[end_index])
self.assertEqual(self.position_callback_value[StateKey.POSITION], positions[end_index])
start_index += len(records)
records = self.parser.get_records(num_of_records)
def assert_type(self, records, particle_type):
for particle in records:
str_of_type = particle.type()
self.assertEqual(particle_type, str_of_type)
def assert_timestamp(self, ntp_timestamp, unix_timestamp):
ntp_stamp = ntplib.system_to_ntp_time(unix_timestamp)
assertion = np.allclose(ntp_timestamp, ntp_stamp)
self.assertTrue(assertion)
def assert_particles(self, records, lower_index, upper_index):
indexes = range(lower_index, upper_index + 1)
index = indexes.pop(0)
for particle in records:
test_dict = particle.generate_dict()
values = {}
for value in test_dict['values']:
values[value['value_id']] = value['value']
for key in values:
assertion = np.allclose(values[key], glider_test_data[key][index])
self.assertTrue(assertion)
self.assert_timestamp(test_dict['internal_timestamp'], values['m_present_time'])
if indexes:
index = indexes.pop(0)
def test_get_ctdgv_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid ctdgv data particle.
"""
self.config[DataSetDriverConfigKeys.PARTICLE_CLASS] = 'GgldrCtdgvDelayedDataParticle'
particle_type = DataParticleType.GGLDR_CTDGV_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_get_dosta_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid dosta data particle.
"""
self.config[DataSetDriverConfigKeys.PARTICLE_CLASS] = 'GgldrDostaDelayedDataParticle'
particle_type = DataParticleType.GGLDR_DOSTA_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_get_flord_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid flord data particle.
"""
self.config[DataSetDriverConfigKeys.PARTICLE_CLASS] = 'GgldrFlordDelayedDataParticle'
particle_type = DataParticleType.GGLDR_FLORD_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_get_eng_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid engineering data particle.
"""
self.config[DataSetDriverConfigKeys.PARTICLE_CLASS] = 'GgldrEngDelayedDataParticle'
particle_type = DataParticleType.GGLDR_ENG_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_set_state(self):
self.config[DataSetDriverConfigKeys.PARTICLE_CLASS] = 'GgldrCtdgvDelayedDataParticle'
particle_type = DataParticleType.GGLDR_CTDGV_DELAYED
index = 3
self.position[StateKey.POSITION] = positions[index]
self.generic_particle_parse(particle_type, 1, start_index=index + 1)
def test_particle_exceptions(self):
self.assertRaises(
SampleException, GliderParticle,
'this is testing that GliderParticle raises a SampleException')
self.assertRaises(
SampleException, GgldrCtdgvDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
self.assertRaises(
SampleException, GgldrDostaDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
self.assertRaises(
SampleException, GgldrFlordDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
self.assertRaises(
SampleException, GgldrEngDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
def reset_parser(self):
self.test_file.seek(0)
self.position_callback_value = None
self.publish_callback_value = None
self.parser = GliderParser(self.config, self.position, self.test_file,
self.pos_callback, self.pub_callback)
class GliderParserUnitTestCase(ParserUnitTestCase):
"""
Glider Parser unit test suite
"""
#log.debug('###########!! PWD is %s !!##########' % os.getcwd())
test_file = open('mi/dataset/parser/test/test_glider_data.mrg', 'r')
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.glider'
}
def pos_callback(self, pos):
""" Call back method to watch what comes in via the position callback """
self.position_callback_value = pos
def pub_callback(self, pub):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_value = pub
def setUp(self):
ParserUnitTestCase.setUp(self)
# not a DataSourceLocation...its just the parser
self.position = {StateKey.POSITION: 0}
# take that, rewind it back ...
self.test_file.seek(0)
self.position_callback_value = None
self.publish_callback_value = None
def reset_parser(self):
self.test_file.seek(0)
self.position_callback_value = None
self.publish_callback_value = None
self.parser = GliderParser(self.config, self.position, self.test_file,
self.pos_callback, self.pub_callback)
def generic_particle_parse(self,
particle_type,
num_of_records,
start_index=0):
self.reset_parser()
records = self.parser.get_records(num_of_records)
while records:
end_index = start_index + len(records) - 1
self.assertEqual(records, self.publish_callback_value)
self.assert_type(records, particle_type)
self.assert_particles(records, start_index, end_index)
self.assertEqual(self.parser._state[StateKey.POSITION],
positions[end_index])
self.assertEqual(self.position_callback_value[StateKey.POSITION],
positions[end_index])
start_index += len(records)
records = self.parser.get_records(num_of_records)
def assert_type(self, records, particle_type):
for particle in records:
str_of_type = particle.type()
self.assertEqual(particle_type, str_of_type)
def assert_timestamp(self, ntp_timestamp, unix_timestamp):
ntp_stamp = ntplib.system_to_ntp_time(unix_timestamp)
assertion = np.allclose(ntp_timestamp, ntp_stamp)
self.assertTrue(assertion)
def assert_particles(self, records, lower_index, upper_index):
indexes = range(lower_index, upper_index + 1)
index = indexes.pop(0)
for particle in records:
test_dict = particle.generate_dict()
values = {}
for value in test_dict['values']:
values[value['value_id']] = value['value']
for key in values:
assertion = np.allclose(values[key],
glider_test_data[key][index])
self.assertTrue(assertion)
self.assert_timestamp(test_dict['internal_timestamp'],
values['m_present_time'])
if indexes:
index = indexes.pop(0)
def test_get_ctdgv_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid ctdgv data particle.
"""
self.config[DataSetDriverConfigKeys.
PARTICLE_CLASS] = 'GgldrCtdgvDelayedDataParticle'
particle_type = DataParticleType.GGLDR_CTDGV_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_get_dosta_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid dosta data particle.
"""
self.config[DataSetDriverConfigKeys.
PARTICLE_CLASS] = 'GgldrDostaDelayedDataParticle'
particle_type = DataParticleType.GGLDR_DOSTA_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_get_flord_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid flord data particle.
"""
self.config[DataSetDriverConfigKeys.
PARTICLE_CLASS] = 'GgldrFlordDelayedDataParticle'
particle_type = DataParticleType.GGLDR_FLORD_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_get_eng_data(self):
"""
Test the path of operations where the parser takes the input
and spits out a valid engineering data particle.
"""
self.config[DataSetDriverConfigKeys.
PARTICLE_CLASS] = 'GgldrEngDelayedDataParticle'
particle_type = DataParticleType.GGLDR_ENG_DELAYED
self.generic_particle_parse(particle_type, 1)
self.generic_particle_parse(particle_type, 5)
self.generic_particle_parse(particle_type, 12)
def test_set_state(self):
self.config[DataSetDriverConfigKeys.
PARTICLE_CLASS] = 'GgldrCtdgvDelayedDataParticle'
particle_type = DataParticleType.GGLDR_CTDGV_DELAYED
index = 3
self.position[StateKey.POSITION] = positions[index]
self.generic_particle_parse(particle_type, 1, start_index=index + 1)
def test_particle_exceptions(self):
self.assertRaises(
SampleException, GliderParticle,
'this is testing that GliderParticle raises a SampleException')
self.assertRaises(
SampleException, GgldrCtdgvDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
self.assertRaises(
SampleException, GgldrDostaDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
self.assertRaises(
SampleException, GgldrFlordDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')
self.assertRaises(
SampleException, GgldrEngDelayedDataParticle,
'this is testing that Ctdgv Particle raises a SampleException')