def test_bad_config(self):
"""
Test that configurations with a missing data particle dict and missing
data particle class key causes a configuration exception
"""
# test a config with a missing particle classes dict
config = {}
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "r")
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(
config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback
)
# test a config with a missing data particle class key
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle
}
}
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(
config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback
)
def test_bad_config(self):
"""
Test that configurations with a missing data particle dict and missing
data particle class key causes a configuration exception
"""
# test a config with a missing particle classes dict
config = {}
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'r')
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# test a config with a missing data particle class key
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
}
}
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
def create_parser(self, stream_handle, telem_flag=True, state=None):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.nutnr_j_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle
}
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.nutnr_j_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle
}
}
self.parser = NutnrJCsppParser(config, state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
def create_parser(self, stream_handle, telem_flag=True):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
NutnrJCsppMetadataTelemeteredDataParticle,
LIGHT_PARTICLE_CLASS_KEY:
NutnrJCsppTelemeteredDataParticle,
DARK_PARTICLE_CLASS_KEY:
NutnrJCsppDarkTelemeteredDataParticle
}
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
NutnrJCsppMetadataRecoveredDataParticle,
LIGHT_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle,
DARK_PARTICLE_CLASS_KEY:
NutnrJCsppDarkRecoveredDataParticle
}
}
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
def create_parser(self, stream_handle, telem_flag=True):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
LIGHT_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle,
DARK_PARTICLE_CLASS_KEY: NutnrJCsppDarkTelemeteredDataParticle
}
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
LIGHT_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle,
DARK_PARTICLE_CLASS_KEY: NutnrJCsppDarkRecoveredDataParticle
}
}
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
def create_parser(self, stream_handle, telem_flag=True, state=None):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: "mi.dataset.parser.nutnr_j_cspp",
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle,
},
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: "mi.dataset.parser.nutnr_j_cspp",
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle,
},
}
self.parser = NutnrJCsppParser(
config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback
)
def _build_parser(self, stream_handle):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle
}
}
parser = NutnrJCsppParser(parser_config, stream_handle,
self._exception_callback)
return parser
def _build_parser(self, parser_state, stream_in, data_key):
"""
Build and return a parser for the data_key type parser
@param parser_state starting parser state to pass to parser
@param stream_in Handle of open file to pass to parser
@param data_key Key to determine which parser type is built
"""
# build the parser based on which key is passed in
if data_key == DataSourceKey.NUTNR_J_CSPP_TELEMETERED:
config = self._parser_config.get(DataSourceKey.NUTNR_J_CSPP_TELEMETERED)
config.update({
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle
}
})
elif data_key == DataSourceKey.NUTNR_J_CSPP_RECOVERED:
config = self._parser_config.get(DataSourceKey.NUTNR_J_CSPP_RECOVERED)
config.update({
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle
}
})
else:
log.warn("Cannot build parser for unknown data source key %s", data_key)
raise ConfigurationException("Cannot build parser for unknown data source key %s" % \
data_key)
parser = NutnrJCsppParser(
config,
parser_state,
stream_in,
lambda state, ingested: self._save_parser_state(state, data_key,
ingested),
self._data_callback,
self._sample_exception_callback
)
return parser
class NutnrJCsppParserUnitTestCase(ParserUnitTestCase):
"""
nutnr_j_cspp Parser unit test suite
"""
def particle_to_yml(self, particles, filename, mode='w'):
"""
This is added as a testing helper, not actually as part of the parser tests. Since the same particles
will be used for the driver test it is helpful to write them to .yml in the same form they need in the
results.yml fids here.
"""
# open write append, if you want to start from scratch manually delete this fid
fid = open(os.path.join(RESOURCE_PATH, filename), mode)
fid.write('header:\n')
fid.write(" particle_object: 'MULTIPLE'\n")
fid.write(" particle_type: 'MULTIPLE'\n")
fid.write('data:\n')
for i in range(0, len(particles)):
particle_dict = particles[i].generate_dict()
fid.write(' - _index: %d\n' % (i+1))
fid.write(' particle_object: %s\n' % particles[i].__class__.__name__)
fid.write(' particle_type: %s\n' % particle_dict.get('stream_name'))
fid.write(' internal_timestamp: %16.3f\n' %
particle_dict.get('internal_timestamp'))
for val in particle_dict.get('values'):
if isinstance(val.get('value'), float):
if val.get('value_id') is "time_of_sample":
fid.write(' %s: %16.5f\n' % (val.get('value_id'), val.get('value')))
else:
fid.write(' %s: %16.3f\n' % (val.get('value_id'), val.get('value')))
elif isinstance(val.get('value'), str):
fid.write(" %s: '%s'\n" % (val.get('value_id'), val.get('value')))
else:
fid.write(' %s: %s\n' % (val.get('value_id'), val.get('value')))
fid.close()
def create_yml(self):
"""
This utility creates a yml file
"""
fid = open(os.path.join(RESOURCE_PATH,
'11079419_SNA_SNA.txt'),
MODE_ASCII_READ)
stream_handle = fid
self.create_parser(stream_handle, True)
particles = self.parser.get_records(1000)
self.particle_to_yml(particles, '11079419_SNA_SNA_telem.yml')
fid.close()
def create_parser(self, stream_handle, telem_flag=True):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
LIGHT_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle,
DARK_PARTICLE_CLASS_KEY: NutnrJCsppDarkTelemeteredDataParticle
}
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
LIGHT_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle,
DARK_PARTICLE_CLASS_KEY: NutnrJCsppDarkRecoveredDataParticle
}
}
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
def test_simple_telem(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get and compare the metadata particle
particles = self.parser.get_records(6)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_telem.yml', RESOURCE_PATH)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_simple_recov(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle, telem_flag=False)
# get and compare the metadata particle
particles = self.parser.get_records(6)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_recov.yml', RESOURCE_PATH)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists. Confirm
that exceptions occur.
"""
# bad data file has:
# 1 bad status
# particle A has bad timestamp
# particle B has bad dark fit
# particle C has bad frame type
# particle D has bad year
stream_handle = open(os.path.join(RESOURCE_PATH,
'bad_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle, telem_flag=False)
# get E, since it is first it will generate a metadata
particles = self.parser.get_records(2)
# check all the values against expected results.
self.assert_particles(particles, 'last_and_meta_SNA_recov.yml', RESOURCE_PATH)
# should have had 5 exceptions by now
self.assertEqual(len(self.exception_callback_value), 5)
for exception in self.exception_callback_value:
self.assert_(isinstance(exception, RecoverableSampleException))
def test_missing_source_file(self):
"""
Test that a file with a missing source file path in the header
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'no_source_file_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get A-E, without metadata
particles = self.parser.get_records(5)
# check all the values against expected results.
self.assert_particles(particles, 'no_source_file_SNA_SNA_telem.yml', RESOURCE_PATH)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_no_header(self):
"""
Test that a file with no header lines
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'no_header_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get A-E, without metadata
particles = self.parser.get_records(5)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_telem_no_meta.yml', RESOURCE_PATH)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_partial_header(self):
"""
Test a case where we are missing part of the header, but it is not
the source file so we still want to create the header
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'part_header_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get A-E, also metadata
particles = self.parser.get_records(6)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_telem_part.yml', RESOURCE_PATH)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_bad_config(self):
"""
Test that configurations with a missing data particle dict and missing
data particle class key causes a configuration exception
"""
# test a config with a missing particle classes dict
config = {}
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), MODE_ASCII_READ)
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
# test a config with a missing data particle class key
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
}
}
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
def test_real_file(self):
"""
Read test data from IDD and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
log.info('===== START TEST BYTE LOSS =====')
# Recovered
file_path = os.path.join(RESOURCE_PATH, '11079364_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle, telem_flag=False)
particles = self.parser.get_records(182)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_SNA_SNA_recov.yml', RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
# Telemetered
file_path = os.path.join(RESOURCE_PATH, '11079419_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle)
particles = self.parser.get_records(172)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079419_SNA_SNA_telem.yml', RESOURCE_PATH)
stream_handle.close()
log.info('===== END TEST REAL FILE =====')
def test_bad_match(self):
"""
Test that a file that has a data sample that is causing the regex
matcher to hang. This test confirms
the fix doesn't hang and causes exceptions for not matching data.
"""
log.info('===== START TEST BAD MATCH =====')
# Telemetered
file_path = os.path.join(RESOURCE_PATH, '11129553_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle)
particles = self.parser.get_records(57)
log.debug("*** test_bad_match Num particles %s", len(particles))
# 2 bad samples
self.assertEqual(len(self.exception_callback_value), 2)
stream_handle.close()
log.info('===== END TEST BAD MATCH =====')
def test_byte_loss(self):
"""
Test that a file with known byte loss occurring in the form of hex ascii
lines of data creates an exception.
"""
log.info('===== START TEST REAL FILE =====')
# Telemetered
file_path = os.path.join(RESOURCE_PATH, '11330408_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle)
particles = self.parser.get_records(3)
log.debug("*** test_byte_loss Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, 'byte_loss.yml', RESOURCE_PATH)
stream_handle.close()
log.info('===== END TEST BYTE LOSS =====')
class NutnrJCsppParserUnitTestCase(ParserUnitTestCase):
"""
nutnr_j_cspp Parser unit test suite
"""
def particle_to_yml(self, particles, filename, mode='w'):
"""
This is added as a testing helper, not actually as part of the parser tests. Since the same particles
will be used for the driver test it is helpful to write them to .yml in the same form they need in the
results.yml fids here.
"""
# open write append, if you want to start from scratch manually delete this fid
fid = open(os.path.join(RESOURCE_PATH, filename), mode)
fid.write('header:\n')
fid.write(" particle_object: 'MULTIPLE'\n")
fid.write(" particle_type: 'MULTIPLE'\n")
fid.write('data:\n')
for i in range(0, len(particles)):
particle_dict = particles[i].generate_dict()
fid.write(' - _index: %d\n' % (i+1))
fid.write(' particle_object: %s\n' % particles[i].__class__.__name__)
fid.write(' particle_type: %s\n' % particle_dict.get('stream_name'))
fid.write(' internal_timestamp: %16.3f\n' %
particle_dict.get('internal_timestamp'))
for val in particle_dict.get('values'):
if isinstance(val.get('value'), float):
if val.get('value_id') is "time_of_sample":
fid.write(' %s: %16.5f\n' % (val.get('value_id'), val.get('value')))
else:
fid.write(' %s: %16.3f\n' % (val.get('value_id'), val.get('value')))
elif isinstance(val.get('value'), str):
fid.write(" %s: '%s'\n" % (val.get('value_id'), val.get('value')))
else:
fid.write(' %s: %s\n' % (val.get('value_id'), val.get('value')))
fid.close()
def create_yml(self):
"""
This utility creates a yml file
"""
fid = open(os.path.join(RESOURCE_PATH,
'no_source_file_SNA_SNA.txt'),
MODE_ASCII_READ)
stream_handle = fid
self.create_parser(stream_handle)
particles = self.parser.get_records(182)
self.particle_to_yml(particles, 'no_source_file_SNA_SNA_telem.yml')
fid.close()
def create_parser(self, stream_handle, telem_flag=True):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle
}
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle
}
}
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
def test_simple_telem(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get and compare the metadata particle
particles = self.parser.get_records(6)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_telem.yml', RESOURCE_PATH)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_simple_recov(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle, telem_flag=False)
# get and compare the metadata particle
particles = self.parser.get_records(6)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_recov.yml', RESOURCE_PATH)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists. Confirm
that exceptions occur.
"""
# bad data file has:
# 1 bad status
# particle A has bad timestamp
# particle B has bad dark fit
# particle C has bad frame type
# particle D has bad year
stream_handle = open(os.path.join(RESOURCE_PATH,
'bad_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle, telem_flag=False)
# get E, since it is first it will generate a metadata
particles = self.parser.get_records(2)
# check all the values against expected results.
self.assert_particles(particles, 'last_and_meta_SNA_recov.yml', RESOURCE_PATH)
# should have had 5 exceptions by now
self.assertEqual(len(self.exception_callback_value), 5)
for exception in self.exception_callback_value:
self.assert_(isinstance(exception, RecoverableSampleException))
def test_missing_source_file(self):
"""
Test that a file with a missing source file path in the header
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'no_source_file_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get A-E, without metadata
particles = self.parser.get_records(5)
# check all the values against expected results.
self.assert_particles(particles, 'no_source_file_SNA_SNA_telem.yml', RESOURCE_PATH)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_no_header(self):
"""
Test that a file with no header lines
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'no_header_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get A-E, without metadata
particles = self.parser.get_records(5)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_telem_no_meta.yml', RESOURCE_PATH)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_partial_header(self):
"""
Test a case where we are missing part of the header, but it is not
the source file so we still want to create the header
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'part_header_SNA_SNA.txt'), MODE_ASCII_READ)
self.create_parser(stream_handle)
# get A-E, also metadata
particles = self.parser.get_records(6)
# check all the values against expected results.
self.assert_particles(particles, 'short_SNA_telem_part.yml', RESOURCE_PATH)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_bad_config(self):
"""
Test that configurations with a missing data particle dict and missing
data particle class key causes a configuration exception
"""
# test a config with a missing particle classes dict
config = {}
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), MODE_ASCII_READ)
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
# test a config with a missing data particle class key
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
}
}
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, stream_handle,
self.exception_callback)
def test_real_file(self):
"""
Read test data from IDD and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
log.info('===== START TEST BYTE LOSS =====')
# Recovered
file_path = os.path.join(RESOURCE_PATH, '11079364_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle, telem_flag=False)
particles = self.parser.get_records(182)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_SNA_SNA_recov.yml', RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
# Telemetered
file_path = os.path.join(RESOURCE_PATH, '11079419_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle)
particles = self.parser.get_records(172)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079419_SNA_SNA_telem.yml', RESOURCE_PATH)
stream_handle.close()
log.info('===== END TEST REAL FILE =====')
def test_bad_match(self):
"""
Test that a file that has a data sample that is causing the regex
matcher to hang. This test confirms
the fix doesn't hang and causes exceptions for not matching data.
"""
log.info('===== START TEST BAD MATCH =====')
# Telemetered
file_path = os.path.join(RESOURCE_PATH, '11129553_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle)
particles = self.parser.get_records(57)
log.debug("*** test_bad_match Num particles %s", len(particles))
# 2 bad samples
self.assertEqual(len(self.exception_callback_value), 2)
stream_handle.close()
log.info('===== END TEST BAD MATCH =====')
def test_byte_loss(self):
"""
Test that a file with known byte loss occurring in the form of hex ascii
lines of data creates an exception.
"""
log.info('===== START TEST REAL FILE =====')
# Telemetered
file_path = os.path.join(RESOURCE_PATH, '11330408_SNA_SNA.txt')
stream_handle = open(file_path, MODE_ASCII_READ)
self.create_parser(stream_handle)
particles = self.parser.get_records(3)
log.debug("*** test_byte_loss Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, 'byte_loss.yml', RESOURCE_PATH)
stream_handle.close()
log.info('===== END TEST BYTE LOSS =====')
class NutnrJCsppParserUnitTestCase(ParserUnitTestCase):
"""
nutnr_j_cspp Parser unit test suite
"""
def state_callback(self, state, file_ingested):
""" Call back method to watch what comes in via the position callback """
self.state_callback_value = state
self.file_ingested_value = file_ingested
def pub_callback(self, pub):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_value = pub
def exception_callback(self, exception_val):
""" Call back method to watch what comes in via the exception callback """
self.exception_callback_value.append(exception_val)
def setUp(self):
ParserUnitTestCase.setUp(self)
# metadata header dictionary
header_dict = {
DefaultHeaderKey.SOURCE_FILE: "D:\\Storage\\Programs\\OOI\\Group 5\\Correspondence\\07_18_14 sample files\\files\\11079419.SNA",
DefaultHeaderKey.PROCESSED: "07/18/2014 13:49:01",
DefaultHeaderKey.USING_VERSION: "1.11",
DefaultHeaderKey.DEVICE: "SNA",
DefaultHeaderKey.START_DATE: "04/17/2014",
}
# from 3222:4444
particle_a_match = DATA_MATCHER.match(
"1397774268.772\t0.000\ty\tSDB"
"\t2014\t107\t22.627037\t0.000\t0.000\t0.000\t0.000\t0.000\t484\t0\t1\t497\t490\t484\t"
"494\t481\t490\t486\t473\t495\t496\t503\t493\t494\t486\t480\t503\t482\t484\t494\t488\t"
"493\t503\t485\t493\t492\t484\t483\t486\t491\t481\t485\t485\t493\t481\t483\t490\t490\t"
"485\t488\t483\t499\t501\t489\t488\t488\t481\t487\t479\t491\t485\t496\t488\t495\t474\t"
"485\t474\t484\t482\t487\t497\t501\t489\t492\t485\t486\t488\t481\t477\t497\t478\t493\t"
"488\t491\t488\t491\t484\t485\t493\t493\t477\t487\t483\t485\t487\t485\t473\t461\t460\t"
"462\t482\t485\t489\t485\t489\t484\t492\t481\t490\t491\t482\t481\t487\t467\t483\t480\t"
"488\t474\t487\t490\t483\t469\t483\t489\t486\t481\t489\t482\t481\t477\t484\t487\t489\t"
"471\t471\t486\t468\t481\t488\t484\t481\t482\t489\t483\t485\t482\t489\t500\t488\t488\t"
"496\t483\t484\t481\t480\t499\t492\t503\t482\t482\t486\t481\t477\t490\t473\t491\t484\t"
"489\t493\t470\t469\t467\t468\t483\t478\t483\t482\t485\t481\t485\t491\t473\t491\t478\t"
"484\t490\t480\t491\t470\t475\t474\t465\t483\t473\t483\t464\t477\t479\t478\t491\t483\t"
"493\t480\t487\t482\t480\t481\t487\t485\t485\t477\t483\t501\t484\t491\t473\t487\t478\t"
"486\t499\t489\t485\t476\t480\t490\t493\t489\t482\t489\t470\t487\t482\t482\t488\t479\t"
"481\t485\t489\t480\t482\t465\t467\t474\t475\t485\t488\t487\t477\t477\t483\t480\t479\t"
"483\t474\t486\t482\t475\t483\t485\t492\t484\t483\t490\t478\t468\t475\t408\t0.000\t0.000"
"\t0.000\t246604\t35.657\t11.969\t0.092\t5.039\t57.381\t0.000\t0.000\t0.000\t0.000\t0.000"
"\t0\t-1.000\t-1.000\t-1.000\t36\n"
)
# metadata arguments is a tuple, 1st item header dictionary, 2nd item 1st particle data match
self.meta_telem_particle = NutnrJCsppMetadataTelemeteredDataParticle((header_dict, particle_a_match))
# generate the dictionary so the timestamp is set
self.meta_telem_particle.generate_dict()
# from 3222:4444
self.telem_particle_a = NutnrJCsppTelemeteredDataParticle(particle_a_match)
# generate the dictionary so the timestamp is set
self.telem_particle_a.generate_dict()
# from 4444:6136
particle_b_match = DATA_MATCHER.match(
"1397774270.706\t0.000\ty\tSLB\t2014"
"\t107\t22.627543\t11.962\t0.168\t-0.029\t-0.027\t0.000\t24886\t484\t1\t516\t515\t511\t515"
"\t523\t529\t528\t517\t531\t512\t515\t519\t537\t521\t517\t518\t519\t521\t528\t544\t569\t621"
"\t743\t999\t1388\t2020\t2951\t4265\t6010\t8258\t11016\t14300\t18061\t22265\t26749\t31334\t"
"35848\t40005\t43493\t46201\t47859\t48525\t48346\t47436\t46097\t44624\t43146\t41812\t40772"
"\t40016\t39545\t39401\t39552\t39975\t40661\t41554\t42696\t44017\t45536\t47208\t49022\t50842"
"\t52696\t54441\t55934\t57125\t57914\t58107\t57764\t56797\t55317\t53413\t51132\t48700\t46208"
"\t43741\t41442\t39306\t37373\t35653\t34187\t32882\t31780\t30855\t30151\t29578\t29201\t28970"
"\t28898\t28982\t29196\t29597\t30070\t30705\t31448\t32312\t33195\t34092\t35017\t35836\t36571"
"\t37108\t37426\t37533\t37352\t36905\t36209\t35337\t34291\t33155\t31949\t30740\t29591\t28465\t"
"27451\t26551\t25702\t24994\t24394\t23877\t23495\t23189\t23013\t22920\t22918\t23001\t23200\t"
"23481\t23800\t24218\t24708\t25230\t25829\t26463\t27147\t27884\t28640\t29406\t30237\t31060\t"
"31856\t32633\t33373\t34075\t34701\t35240\t35707\t36087\t36347\t36461\t36470\t36346\t36112\t"
"35790\t35288\t34739\t34113\t33394\t32685\t31936\t31123\t30329\t29575\t28812\t28077\t27350\t"
"26626\t25929\t25210\t24519\t23847\t23193\t22583\t22006\t21510\t21067\t20653\t20296\t19991\t"
"19733\t19515\t19318\t19179\t19022\t18897\t18752\t18617\t18512\t18389\t18273\t18160\t18060\t"
"17966\t17896\t17857\t17828\t17796\t17799\t17773\t17792\t17809\t17867\t17908\t17940\t17970\t"
"18012\t18050\t18081\t18113\t18103\t18110\t18098\t18051\t17997\t17927\t17815\t17691\t17550\t"
"17393\t17242\t17096\t16896\t16622\t16315\t15969\t15632\t15331\t15063\t14804\t14547\t14319\t"
"14112\t13884\t13699\t13607\t13496\t13336\t13064\t12810\t12586\t12391\t12173\t11995\t11838\t"
"11702\t11617\t11557\t11461\t11339\t11214\t11163\t11075\t10838\t10362\t9568\t8565\t11.438\t"
"11.063\t12.563\t246606\t35.743\t11.677\t12.002\t4.991\t511.803\t24.446\t2.634\t-2.808\t0.052"
"\t0.000\t0\t-1.000\t-1.000\t-1.000\t8D\n"
)
self.telem_particle_b = NutnrJCsppTelemeteredDataParticle(particle_b_match)
# generate the dictionary so the timestamp is set
self.telem_particle_b.generate_dict()
# from 6136:7828
particle_c_match = DATA_MATCHER.match(
"1397774271.656\t0.000\ty\tSLB\t2014"
"\t107\t22.627810\t12.084\t0.169\t-0.029\t-0.027\t0.000\t24863\t484\t1\t514\t501\t507\t510\t"
"515\t516\t523\t525\t517\t515\t512\t512\t511\t535\t531\t531\t531\t535\t536\t550\t565\t641\t"
"764\t978\t1375\t1990\t2923\t4245\t6006\t8238\t10990\t14277\t18028\t22218\t26677\t31248\t"
"35767\t39910\t43402\t46121\t47783\t48422\t48259\t47374\t46036\t44554\t43076\t41750\t40702"
"\t39941\t39488\t39373\t"
"39523\t39932\t40613\t41498\t42629\t43944\t45464\t47144\t48960\t"
"50781\t52636\t54380\t55870\t57061\t57856\t58049\t57683\t56725\t55255\t53330\t51063\t48617"
"\t46149\t43706\t41403\t39264\t37357\t35633\t34135\t32843\t31742\t30833\t30106\t29544\t29186"
"\t28937\t28873\t28963\t29177\t29553\t30049\t30670\t31424\t32272\t33147\t34090\t34981\t35812"
"\t36553\t37089\t37447\t37530\t37347\t36881\t36196\t35306\t34260\t33108\t31918\t30721\t29553"
"\t28439\t27419\t26513\t25672\t24961\t24349\t23872\t23453\t23163\t22972\t22893\t22913\t22986"
"\t23169\t23457\t23785\t24203\t24686\t25210\t25796\t26434\t27104\t27841\t28617\t29394\t30211"
"\t31027\t31825\t32605\t33348\t34050\t34670\t35216\t35666\t36060\t36285\t36411\t36419\t36321"
"\t36066\t35751\t35258\t34701\t34076\t33355\t32640\t31899\t31105\t30313\t29550\t28787\t28058"
"\t27331\t26616\t25917\t25197\t24496\t23827\t23179\t22557\t21981\t21485\t21050\t20646\t20300"
"\t19987\t19755\t19530\t19349\t19185\t19031\t18884\t18745\t18603\t18478\t18380\t18275\t18147"
"\t18055\t17972\t17921\t17865\t17821\t17799\t17789\t17757\t17787\t17800\t17852\t17908\t17934"
"\t17973\t17997\t18055\t18073\t18123\t18105\t18100\t18105\t18048\t17978\t17902\t17803\t17672"
"\t17542\t17402\t17262\t17101\t16890\t16599\t16315\t15956\t15623\t15329\t15042\t14796\t14532"
"\t14322\t14112\t13897\t13725\t13605\t13505\t13330\t13064\t12793\t12577\t12377\t12169\t11989"
"\t11836\t11693\t11606\t11555\t11470\t11341\t11216\t11159\t11064\t10839\t10361\t9578\t8554\t"
"11.438\t11.063\t12.563\t246607\t35.873\t11.739\t11.970\t5.032\t517.250\t24.266\t3.037\t-2.909"
"\t0.118\t0.000\t0\t-1.000\t-1.000\t-1.000\tD9\n"
)
self.telem_particle_c = NutnrJCsppTelemeteredDataParticle(particle_c_match)
# generate the dictionary so the timestamp is set
self.telem_particle_c.generate_dict()
# from 7828:9055
particle_d_match = DATA_MATCHER.match(
"1397774272.683\t0.000\tn\tSDB\t2014\t107\t22.628126\t0.000"
"\t0.000\t0.000\t0.000\t0.000\t479\t0\t1\t489\t492\t483\t477\t481\t481\t487\t474\t481\t486\t475"
"\t487\t492\t485\t491\t485\t478\t472\t486\t489\t477\t491\t478\t475\t469\t483\t486\t473\t483\t468"
"\t461\t469\t465\t489\t485\t488\t489\t491\t478\t486\t475\t482\t489\t489\t473\t481\t481\t467\t481"
"\t484\t482\t474\t481\t477\t483\t488\t478\t484\t481\t481\t477\t494\t497\t473\t471\t487\t473\t485"
"\t478\t467\t474\t485\t475\t491\t496\t485\t488\t478\t479\t483\t481\t471\t470\t487\t470\t471\t461"
"\t473\t478\t474\t475\t477\t497\t483\t471\t475\t477\t484\t480\t499\t485\t474\t487\t493\t480\t479"
"\t476\t467\t459\t487\t470\t471\t468\t487\t473\t490\t485\t485\t481\t477\t479\t482\t472\t471\t465"
"\t471\t458\t487\t482\t479\t487\t482\t477\t493\t477\t473\t491\t473\t478\t488\t481\t492\t490\t483"
"\t477\t484\t472\t475\t465\t477\t488\t477\t477\t491\t477\t475\t477\t474\t478\t473\t479\t468\t481"
"\t488\t478\t486\t473\t467\t467\t471\t458\t476\t461\t481\t488\t487\t479\t481\t486\t477\t473\t485"
"\t478\t485\t490\t489\t492\t483\t481\t480\t472\t469\t460\t471\t473\t472\t477\t489\t473\t481\t480"
"\t469\t471\t467\t473\t477\t469\t471\t484\t491\t478\t494\t484\t495\t485\t484\t471\t471\t467\t475"
"\t471\t477\t491\t483\t487\t471\t467\t470\t470\t472\t488\t490\t486\t490\t484\t473\t473\t464\t488"
"\t480\t479\t475\t483\t493\t487\t471\t481\t470\t461\t459\t462\t474\t474\t482\t467\t423\t11.438\t"
"11.063\t12.563\t246606\t35.799\t11.866\t11.963\t5.010\t112.953\t0.000\t0.000\t0.000\t0.000\t0.000"
"\t0\t-1.000\t-1.000\t-1.000\t16\n"
)
self.telem_particle_d = NutnrJCsppTelemeteredDataParticle(particle_d_match)
# generate the dictionary so the timestamp is set
self.telem_particle_d.generate_dict()
particle_e_match = DATA_MATCHER.match(
"1397774273.831\t0.000\ty\tSDB\t2014\t107\t22.628356\t0.000\t"
"0.000\t0.000\t0.000\t0.000\t481\t0\t1\t489\t486\t479\t477\t475\t492\t487\t488\t490\t501\t491\t"
"488\t484\t477\t471\t490\t474\t477\t477\t487\t490\t491\t497\t473\t491\t489\t483\t473\t481\t482\t"
"494\t489\t497\t492\t475\t477\t484\t485\t473\t485\t478\t471\t477\t472\t459\t471\t490\t476\t475\t"
"485\t477\t485\t480\t483\t473\t486\t491\t489\t473\t487\t491\t493\t503\t489\t485\t480\t477\t473\t"
"480\t487\t493\t477\t485\t477\t485\t470\t494\t484\t479\t489\t491\t485\t479\t503\t484\t487\t477\t"
"477\t483\t481\t486\t491\t493\t477\t487\t492\t500\t492\t494\t489\t480\t485\t484\t487\t483\t475\t"
"487\t490\t489\t481\t481\t485\t472\t471\t481\t477\t483\t477\t473\t468\t475\t474\t474\t471\t474\t"
"470\t469\t479\t477\t474\t485\t478\t477\t482\t469\t476\t482\t481\t481\t499\t482\t473\t473\t470\t"
"477\t468\t469\t481\t489\t486\t493\t478\t479\t483\t482\t474\t487\t478\t480\t472\t475\t489\t484\t"
"473\t483\t477\t477\t483\t486\t473\t473\t477\t480\t491\t477\t467\t483\t484\t493\t495\t487\t479\t"
"485\t482\t465\t491\t476\t485\t488\t477\t471\t467\t458\t471\t472\t489\t491\t489\t481\t494\t473\t"
"480\t471\t481\t474\t480\t482\t483\t485\t470\t475\t477\t473\t472\t485\t475\t481\t488\t477\t482\t"
"473\t471\t467\t483\t482\t493\t493\t483\t491\t493\t485\t471\t487\t482\t475\t474\t478\t482\t485\t"
"484\t472\t471\t476\t486\t489\t497\t481\t483\t497\t475\t473\t477\t477\t474\t473\t394\t11.438\t"
"11.125\t12.563\t246606\t35.805\t11.911\t11.965\t5.049\t111.832\t0.000\t0.000\t0.000\t0.000\t0.000"
"\t0\t-1.000\t-1.000\t-1.000\t93\n"
)
self.telem_particle_e = NutnrJCsppTelemeteredDataParticle(particle_e_match)
# generate the dictionary so the timestamp is set
self.telem_particle_e.generate_dict()
self.meta_telem_e = NutnrJCsppMetadataTelemeteredDataParticle((header_dict, particle_e_match))
# generate the dictionary so the timestamp is set
self.meta_telem_e.generate_dict()
# the particle at the end of the long file 224798:226490
particle_long_end_match = DATA_MATCHER.match(
"1397774471.010\t0.000\tn\tSLB\t2014\t107"
"\t22.683184\t0.717\t0.010\t-0.022\t-0.022\t0.000\t24747\t482\t1\t517\t517\t508\t506\t500\t515\t522\t"
"523\t522\t531\t527\t521\t541\t521\t511\t502\t505\t521\t517\t532\t579\t654\t801\t1074\t1555\t2280\t"
"3357\t4853\t6855\t9373\t12459\t16069\t20157\t24643\t29400\t34168\t38801\t42966\t46394\t48931\t50337"
"\t50690\t50177\t48948\t47287\t45540\t43811\t42299\t41061\t40156\t39572\t39341\t39376\t39706\t40324\t"
"41152\t42189\t43440\t44902\t46512\t48277\t50040\t51858\t53556\t55002\t56169\t56916\t57085\t56756\t"
"55800\t54332\t52490\t50237\t47824\t45403\t43002\t40749\t38656\t36759\t35078\t33608\t32339\t31264\t"
"30381\t29670\t29113\t28756\t28544\t28461\t28567\t28789\t29176\t29656\t30283\t31026\t31858\t32728\t"
"33637\t34534\t35337\t36071\t36609\t36939\t37031\t36856\t36417\t35738\t34859\t33819\t32720\t31517\t"
"30349\t29192\t28092\t27080\t26165\t25355\t24635\t24048\t23543\t23143\t22871\t22676\t22585\t22611\t"
"22698\t22893\t23145\t23489\t23900\t24345\t24859\t25459\t26070\t26725\t27465\t28224\t28995\t29793\t"
"30585\t31380\t32165\t32906\t33587\t34202\t34747\t35187\t35568\t35817\t35926\t35943\t35834\t35614\t"
"35248\t34794\t34238\t33620\t32904\t32195\t31461\t30669\t29921\t29156\t28419\t27700\t26972\t26264\t"
"25577\t24866\t24175\t23514\t22852\t22267\t21712\t21220\t20795\t20396\t20039\t19735\t19502\t19285\t"
"19096\t18946\t18786\t18646\t18514\t18358\t18247\t18175\t18048\t17932\t17837\t17756\t17702\t17653\t"
"17614\t17600\t17585\t17561\t17595\t17590\t17655\t17716\t17744\t17780\t17818\t17867\t17909\t17921\t"
"17911\t17912\t17898\t17854\t17789\t17715\t17626\t17497\t17367\t17221\t17075\t16923\t16727\t16456\t"
"16152\t15793\t15469\t15189\t14904\t14641\t14390\t14165\t13957\t13747\t13568\t13465\t13354\t13191\t"
"12936\t12676\t12460\t12258\t12053\t11863\t11717\t11582\t11506\t11437\t11349\t11225\t11093\t11037\t"
"10955\t10741\t10251\t9467\t8475\t12.313\t13.188\t13.188\t246633\t34.853\t11.745\t11.996\t4.988\t"
"493.629\t26.120\t0.411\t-3.311\t0.583\t0.000\t0\t-1.000\t-1.000\t-1.000\t43\n"
)
self.telem_particle_long = NutnrJCsppTelemeteredDataParticle(particle_long_end_match)
# generate the dictionary so the timestamp is set
self.telem_particle_long.generate_dict()
# recovered particles using the same data as telemetered particles
self.meta_recov_particle = NutnrJCsppMetadataRecoveredDataParticle((header_dict, particle_a_match))
self.meta_recov_particle.generate_dict()
self.recov_particle_a = NutnrJCsppRecoveredDataParticle(particle_a_match)
self.recov_particle_a.generate_dict()
self.recov_particle_b = NutnrJCsppRecoveredDataParticle(particle_b_match)
self.recov_particle_b.generate_dict()
self.recov_particle_c = NutnrJCsppRecoveredDataParticle(particle_c_match)
self.recov_particle_c.generate_dict()
self.recov_particle_d = NutnrJCsppRecoveredDataParticle(particle_d_match)
self.recov_particle_d.generate_dict()
self.recov_particle_e = NutnrJCsppRecoveredDataParticle(particle_e_match)
self.recov_particle_e.generate_dict()
self.file_ingested_value = None
self.state_callback_value = None
self.publish_callback_value = None
self.exception_callback_value = []
def assert_equal_regex(self, match1, match2):
"""
A data regex match cannot be directly equal to another data regex match,
pull out the matching raw data result to see if the regexes matched the same
data
"""
self.assertEqual(match1.group(0), match2.group(0))
def assert_result(self, result, position, particle, ingested, is_metadata=False):
"""
Assert two particles and the states after that particle are equal
"""
self.assert_(isinstance(self.publish_callback_value, list))
if is_metadata:
self.assert_metadata(result, particle)
else:
self.assert_data(result, particle)
self.assert_timestamp(result, particle)
self.assert_position(position)
# compare if the file is ingested
self.assertEqual(self.file_ingested_value, ingested)
def assert_metadata(self, result, particle, result_index=0):
"""
Assert two metadata particles raw data matches
"""
# the data regex match is passed in as the second argument to the metadata,
# need to pull out the string for comparison since regex results cannot be
# directly compared
self.assertEqual(result[result_index].raw_data[0], particle.raw_data[0])
self.assert_equal_regex(result[result_index].raw_data[1], particle.raw_data[1])
self.assertEqual(self.publish_callback_value[result_index].raw_data[0], particle.raw_data[0])
self.assert_equal_regex(self.publish_callback_value[result_index].raw_data[1], particle.raw_data[1])
def assert_data(self, result, particle, result_index=0):
# the data regex match is passed in as the argument to the data particle,
# need to pull out the string for comparison since regex results cannot be
# directly compared
self.assert_equal_regex(result[result_index].raw_data, particle.raw_data)
self.assert_equal_regex(self.publish_callback_value[result_index].raw_data, particle.raw_data)
def assert_timestamp(self, result, particle, result_index=0):
"""
Compare the timestamp of two particles
"""
allowed_diff = 0.000001
self.assertTrue(
abs(
result[result_index].contents[DataParticleKey.INTERNAL_TIMESTAMP]
- particle.contents[DataParticleKey.INTERNAL_TIMESTAMP]
)
<= allowed_diff
)
def assert_data_and_timestamp(self, result, particle, result_index=0):
"""
Combine asserting data and timestamp
"""
self.assert_data(result, particle, result_index)
self.assert_timestamp(result, particle, result_index)
def assert_metadata_and_timestamp(self, result, particle, result_index=0):
self.assert_metadata(result, particle, result_index)
self.assert_timestamp(result, particle, result_index)
def assert_position(self, position):
"""
Assert that the position in the parser state and state callback match
the expected position
"""
self.assertEqual(self.parser._state[StateKey.POSITION], position)
self.assertEqual(self.state_callback_value[StateKey.POSITION], position)
def create_parser(self, stream_handle, telem_flag=True, state=None):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: "mi.dataset.parser.nutnr_j_cspp",
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle,
},
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: "mi.dataset.parser.nutnr_j_cspp",
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle,
},
}
self.parser = NutnrJCsppParser(
config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback
)
def test_simple_telem(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "rb")
self.create_parser(stream_handle)
# get and compare the metadata particle
result = self.parser.get_records(1)
self.assert_result(result, 0, self.meta_telem_particle, False, is_metadata=True)
# get and compare the first data particle
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_A_POS, self.telem_particle_a, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_B_POS, self.telem_particle_b, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_C_POS, self.telem_particle_c, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_D_POS, self.telem_particle_d, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_E_POS, self.telem_particle_e, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_simple_recov(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "rb")
self.create_parser(stream_handle, telem_flag=False)
# get and compare the metadata particle
result = self.parser.get_records(1)
self.assert_result(result, 0, self.meta_recov_particle, False, is_metadata=True)
# get and compare the first data particle
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_A_POS, self.recov_particle_a, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_B_POS, self.recov_particle_b, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_C_POS, self.recov_particle_c, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_D_POS, self.recov_particle_d, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_E_POS, self.recov_particle_e, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "rb")
self.create_parser(stream_handle)
result = self.parser.get_records(6)
self.assert_(isinstance(self.publish_callback_value, list))
# compare particle raw data values
self.assert_metadata_and_timestamp(result, self.meta_telem_particle)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=4)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=5)
# compare position and file ingested from the state
self.assert_position(PARTICLE_E_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_long_stream(self):
"""
Test a long stream
"""
stream_handle = open(os.path.join(RESOURCE_PATH, "11079419_SNA_SNA.txt"), "rb")
self.create_parser(stream_handle)
result = self.parser.get_records(172)
self.assertEqual(len(result), 172)
self.assert_(isinstance(self.publish_callback_value, list))
# compare particle raw data values, check the particles we know from
# the beginning, then the last particle in the file
self.assert_metadata_and_timestamp(result, self.meta_telem_particle)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=4)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=5)
self.assert_data_and_timestamp(result, self.telem_particle_long, result_index=-1)
# compare position and file ingested from the state
self.assert_position(END_LONG_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
"""
# set the state so that the metadata, particle A and B have been read
new_state = {StateKey.POSITION: PARTICLE_B_POS, StateKey.METADATA_EXTRACTED: True}
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "rb")
self.create_parser(stream_handle, state=new_state)
# ask for more records but should only get 3 back: c, d, and e
result = self.parser.get_records(6)
self.assertEqual(len(result), 3)
self.assert_(isinstance(self.publish_callback_value, list))
# compare particle raw data values
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=2)
# compare position and file ingested from the state
self.assert_position(PARTICLE_E_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
# set the state so that the metadata, particle A, B, and C have been read
new_state = {StateKey.POSITION: PARTICLE_C_POS, StateKey.METADATA_EXTRACTED: True}
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "r")
self.create_parser(stream_handle)
# get metadata and A
result = self.parser.get_records(2)
# compare particle raw data values and timestamp
self.assert_metadata_and_timestamp(result, self.meta_telem_particle)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=1)
# compare position and file ingested from the state
self.assert_position(PARTICLE_A_POS)
self.assertEqual(self.file_ingested_value, False)
# now change the state to skip over B and C
self.parser.set_state(new_state)
# confirm we get D and E now
result = self.parser.get_records(2)
# compare particle raw data values and timestamp
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=1)
# compare position and file ingested from the state
self.assert_position(PARTICLE_E_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# bad data file has:
# 1 bad status
# particle A has bad timestamp
# particle B has bad dark fit
# particle C has bad frame type
# particle D has bad year
stream_handle = open(os.path.join(RESOURCE_PATH, "bad_SNA_SNA.txt"), "r")
self.create_parser(stream_handle)
# get E, since it is first it will generate a metadata
result = self.parser.get_records(1)
self.assert_result(result, 0, self.meta_telem_e, False, is_metadata=True)
result = self.parser.get_records(1)
# bytes in file changed due to making file 'bad'
self.assert_result(result, 10257, self.telem_particle_e, True)
# should have had 5 exceptions by now
self.assertEqual(len(self.exception_callback_value), 5)
for exception in self.exception_callback_value:
self.assert_(isinstance(exception, RecoverableSampleException))
def test_missing_source_file(self):
"""
Test that a file with a missing source file path in the header
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH, "no_source_file_SNA_SNA.txt"), "r")
self.create_parser(stream_handle)
# get A-E, without metadata
result = self.parser.get_records(5)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=4)
# compare file ingested from the state
self.assertEqual(self.file_ingested_value, True)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_no_header(self):
"""
Test that a file with no header lines
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH, "no_header_SNA_SNA.txt"), "r")
self.create_parser(stream_handle)
# get A-E, without metadata
result = self.parser.get_records(5)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=4)
# compare file ingested from the state
self.assertEqual(self.file_ingested_value, True)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_bad_config(self):
"""
Test that configurations with a missing data particle dict and missing
data particle class key causes a configuration exception
"""
# test a config with a missing particle classes dict
config = {}
stream_handle = open(os.path.join(RESOURCE_PATH, "short_SNA_SNA.txt"), "r")
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(
config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback
)
# test a config with a missing data particle class key
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle
}
}
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(
config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback
)
class NutnrJCsppParserUnitTestCase(ParserUnitTestCase):
"""
nutnr_j_cspp Parser unit test suite
"""
def state_callback(self, state, file_ingested):
""" Call back method to watch what comes in via the position callback """
self.state_callback_value = state
self.file_ingested_value = file_ingested
def pub_callback(self, pub):
""" Call back method to watch what comes in via the publish callback """
self.publish_callback_value = pub
def exception_callback(self, exception_val):
""" Call back method to watch what comes in via the exception callback """
self.exception_callback_value.append(exception_val)
def setUp(self):
ParserUnitTestCase.setUp(self)
# metadata header dictionary
header_dict = {
DefaultHeaderKey.SOURCE_FILE: 'D:\\Storage\\Programs\\OOI\\Group 5\\Correspondence\\07_18_14 sample files\\files\\11079419.SNA',
DefaultHeaderKey.PROCESSED: '07/18/2014 13:49:01',
DefaultHeaderKey.USING_VERSION: '1.11',
DefaultHeaderKey.DEVICE: 'SNA',
DefaultHeaderKey.START_DATE: '04/17/2014'}
# from 3222:4444
particle_a_match = DATA_MATCHER.match('1397774268.772\t0.000\ty\tSDB' \
'\t2014\t107\t22.627037\t0.000\t0.000\t0.000\t0.000\t0.000\t484\t0\t1\t497\t490\t484\t' \
'494\t481\t490\t486\t473\t495\t496\t503\t493\t494\t486\t480\t503\t482\t484\t494\t488\t' \
'493\t503\t485\t493\t492\t484\t483\t486\t491\t481\t485\t485\t493\t481\t483\t490\t490\t' \
'485\t488\t483\t499\t501\t489\t488\t488\t481\t487\t479\t491\t485\t496\t488\t495\t474\t' \
'485\t474\t484\t482\t487\t497\t501\t489\t492\t485\t486\t488\t481\t477\t497\t478\t493\t' \
'488\t491\t488\t491\t484\t485\t493\t493\t477\t487\t483\t485\t487\t485\t473\t461\t460\t' \
'462\t482\t485\t489\t485\t489\t484\t492\t481\t490\t491\t482\t481\t487\t467\t483\t480\t' \
'488\t474\t487\t490\t483\t469\t483\t489\t486\t481\t489\t482\t481\t477\t484\t487\t489\t' \
'471\t471\t486\t468\t481\t488\t484\t481\t482\t489\t483\t485\t482\t489\t500\t488\t488\t' \
'496\t483\t484\t481\t480\t499\t492\t503\t482\t482\t486\t481\t477\t490\t473\t491\t484\t' \
'489\t493\t470\t469\t467\t468\t483\t478\t483\t482\t485\t481\t485\t491\t473\t491\t478\t' \
'484\t490\t480\t491\t470\t475\t474\t465\t483\t473\t483\t464\t477\t479\t478\t491\t483\t' \
'493\t480\t487\t482\t480\t481\t487\t485\t485\t477\t483\t501\t484\t491\t473\t487\t478\t' \
'486\t499\t489\t485\t476\t480\t490\t493\t489\t482\t489\t470\t487\t482\t482\t488\t479\t' \
'481\t485\t489\t480\t482\t465\t467\t474\t475\t485\t488\t487\t477\t477\t483\t480\t479\t' \
'483\t474\t486\t482\t475\t483\t485\t492\t484\t483\t490\t478\t468\t475\t408\t0.000\t0.000' \
'\t0.000\t246604\t35.657\t11.969\t0.092\t5.039\t57.381\t0.000\t0.000\t0.000\t0.000\t0.000' \
'\t0\t-1.000\t-1.000\t-1.000\t36\n')
# metadata arguments is a tuple, 1st item header dictionary, 2nd item 1st particle data match
self.meta_telem_particle = NutnrJCsppMetadataTelemeteredDataParticle((header_dict, particle_a_match))
# generate the dictionary so the timestamp is set
self.meta_telem_particle.generate_dict()
# from 3222:4444
self.telem_particle_a = NutnrJCsppTelemeteredDataParticle(particle_a_match)
# generate the dictionary so the timestamp is set
self.telem_particle_a.generate_dict()
# from 4444:6136
particle_b_match = DATA_MATCHER.match('1397774270.706\t0.000\ty\tSLB\t2014' \
'\t107\t22.627543\t11.962\t0.168\t-0.029\t-0.027\t0.000\t24886\t484\t1\t516\t515\t511\t515' \
'\t523\t529\t528\t517\t531\t512\t515\t519\t537\t521\t517\t518\t519\t521\t528\t544\t569\t621' \
'\t743\t999\t1388\t2020\t2951\t4265\t6010\t8258\t11016\t14300\t18061\t22265\t26749\t31334\t' \
'35848\t40005\t43493\t46201\t47859\t48525\t48346\t47436\t46097\t44624\t43146\t41812\t40772' \
'\t40016\t39545\t39401\t39552\t39975\t40661\t41554\t42696\t44017\t45536\t47208\t49022\t50842' \
'\t52696\t54441\t55934\t57125\t57914\t58107\t57764\t56797\t55317\t53413\t51132\t48700\t46208' \
'\t43741\t41442\t39306\t37373\t35653\t34187\t32882\t31780\t30855\t30151\t29578\t29201\t28970' \
'\t28898\t28982\t29196\t29597\t30070\t30705\t31448\t32312\t33195\t34092\t35017\t35836\t36571' \
'\t37108\t37426\t37533\t37352\t36905\t36209\t35337\t34291\t33155\t31949\t30740\t29591\t28465\t' \
'27451\t26551\t25702\t24994\t24394\t23877\t23495\t23189\t23013\t22920\t22918\t23001\t23200\t' \
'23481\t23800\t24218\t24708\t25230\t25829\t26463\t27147\t27884\t28640\t29406\t30237\t31060\t' \
'31856\t32633\t33373\t34075\t34701\t35240\t35707\t36087\t36347\t36461\t36470\t36346\t36112\t' \
'35790\t35288\t34739\t34113\t33394\t32685\t31936\t31123\t30329\t29575\t28812\t28077\t27350\t' \
'26626\t25929\t25210\t24519\t23847\t23193\t22583\t22006\t21510\t21067\t20653\t20296\t19991\t' \
'19733\t19515\t19318\t19179\t19022\t18897\t18752\t18617\t18512\t18389\t18273\t18160\t18060\t' \
'17966\t17896\t17857\t17828\t17796\t17799\t17773\t17792\t17809\t17867\t17908\t17940\t17970\t' \
'18012\t18050\t18081\t18113\t18103\t18110\t18098\t18051\t17997\t17927\t17815\t17691\t17550\t' \
'17393\t17242\t17096\t16896\t16622\t16315\t15969\t15632\t15331\t15063\t14804\t14547\t14319\t' \
'14112\t13884\t13699\t13607\t13496\t13336\t13064\t12810\t12586\t12391\t12173\t11995\t11838\t' \
'11702\t11617\t11557\t11461\t11339\t11214\t11163\t11075\t10838\t10362\t9568\t8565\t11.438\t' \
'11.063\t12.563\t246606\t35.743\t11.677\t12.002\t4.991\t511.803\t24.446\t2.634\t-2.808\t0.052' \
'\t0.000\t0\t-1.000\t-1.000\t-1.000\t8D\n')
self.telem_particle_b = NutnrJCsppTelemeteredDataParticle(particle_b_match)
# generate the dictionary so the timestamp is set
self.telem_particle_b.generate_dict()
# from 6136:7828
particle_c_match = DATA_MATCHER.match('1397774271.656\t0.000\ty\tSLB\t2014' \
'\t107\t22.627810\t12.084\t0.169\t-0.029\t-0.027\t0.000\t24863\t484\t1\t514\t501\t507\t510\t' \
'515\t516\t523\t525\t517\t515\t512\t512\t511\t535\t531\t531\t531\t535\t536\t550\t565\t641\t' \
'764\t978\t1375\t1990\t2923\t4245\t6006\t8238\t10990\t14277\t18028\t22218\t26677\t31248\t' \
'35767\t39910\t43402\t46121\t47783\t48422\t48259\t47374\t46036\t44554\t43076\t41750\t40702' \
'\t39941\t39488\t39373\t''39523\t39932\t40613\t41498\t42629\t43944\t45464\t47144\t48960\t' \
'50781\t52636\t54380\t55870\t57061\t57856\t58049\t57683\t56725\t55255\t53330\t51063\t48617' \
'\t46149\t43706\t41403\t39264\t37357\t35633\t34135\t32843\t31742\t30833\t30106\t29544\t29186' \
'\t28937\t28873\t28963\t29177\t29553\t30049\t30670\t31424\t32272\t33147\t34090\t34981\t35812' \
'\t36553\t37089\t37447\t37530\t37347\t36881\t36196\t35306\t34260\t33108\t31918\t30721\t29553' \
'\t28439\t27419\t26513\t25672\t24961\t24349\t23872\t23453\t23163\t22972\t22893\t22913\t22986' \
'\t23169\t23457\t23785\t24203\t24686\t25210\t25796\t26434\t27104\t27841\t28617\t29394\t30211' \
'\t31027\t31825\t32605\t33348\t34050\t34670\t35216\t35666\t36060\t36285\t36411\t36419\t36321' \
'\t36066\t35751\t35258\t34701\t34076\t33355\t32640\t31899\t31105\t30313\t29550\t28787\t28058' \
'\t27331\t26616\t25917\t25197\t24496\t23827\t23179\t22557\t21981\t21485\t21050\t20646\t20300' \
'\t19987\t19755\t19530\t19349\t19185\t19031\t18884\t18745\t18603\t18478\t18380\t18275\t18147' \
'\t18055\t17972\t17921\t17865\t17821\t17799\t17789\t17757\t17787\t17800\t17852\t17908\t17934' \
'\t17973\t17997\t18055\t18073\t18123\t18105\t18100\t18105\t18048\t17978\t17902\t17803\t17672' \
'\t17542\t17402\t17262\t17101\t16890\t16599\t16315\t15956\t15623\t15329\t15042\t14796\t14532' \
'\t14322\t14112\t13897\t13725\t13605\t13505\t13330\t13064\t12793\t12577\t12377\t12169\t11989' \
'\t11836\t11693\t11606\t11555\t11470\t11341\t11216\t11159\t11064\t10839\t10361\t9578\t8554\t' \
'11.438\t11.063\t12.563\t246607\t35.873\t11.739\t11.970\t5.032\t517.250\t24.266\t3.037\t-2.909' \
'\t0.118\t0.000\t0\t-1.000\t-1.000\t-1.000\tD9\n')
self.telem_particle_c = NutnrJCsppTelemeteredDataParticle(particle_c_match)
# generate the dictionary so the timestamp is set
self.telem_particle_c.generate_dict()
# from 7828:9055
particle_d_match = DATA_MATCHER.match('1397774272.683\t0.000\tn\tSDB\t2014\t107\t22.628126\t0.000' \
'\t0.000\t0.000\t0.000\t0.000\t479\t0\t1\t489\t492\t483\t477\t481\t481\t487\t474\t481\t486\t475' \
'\t487\t492\t485\t491\t485\t478\t472\t486\t489\t477\t491\t478\t475\t469\t483\t486\t473\t483\t468' \
'\t461\t469\t465\t489\t485\t488\t489\t491\t478\t486\t475\t482\t489\t489\t473\t481\t481\t467\t481' \
'\t484\t482\t474\t481\t477\t483\t488\t478\t484\t481\t481\t477\t494\t497\t473\t471\t487\t473\t485' \
'\t478\t467\t474\t485\t475\t491\t496\t485\t488\t478\t479\t483\t481\t471\t470\t487\t470\t471\t461' \
'\t473\t478\t474\t475\t477\t497\t483\t471\t475\t477\t484\t480\t499\t485\t474\t487\t493\t480\t479' \
'\t476\t467\t459\t487\t470\t471\t468\t487\t473\t490\t485\t485\t481\t477\t479\t482\t472\t471\t465' \
'\t471\t458\t487\t482\t479\t487\t482\t477\t493\t477\t473\t491\t473\t478\t488\t481\t492\t490\t483' \
'\t477\t484\t472\t475\t465\t477\t488\t477\t477\t491\t477\t475\t477\t474\t478\t473\t479\t468\t481' \
'\t488\t478\t486\t473\t467\t467\t471\t458\t476\t461\t481\t488\t487\t479\t481\t486\t477\t473\t485' \
'\t478\t485\t490\t489\t492\t483\t481\t480\t472\t469\t460\t471\t473\t472\t477\t489\t473\t481\t480' \
'\t469\t471\t467\t473\t477\t469\t471\t484\t491\t478\t494\t484\t495\t485\t484\t471\t471\t467\t475' \
'\t471\t477\t491\t483\t487\t471\t467\t470\t470\t472\t488\t490\t486\t490\t484\t473\t473\t464\t488' \
'\t480\t479\t475\t483\t493\t487\t471\t481\t470\t461\t459\t462\t474\t474\t482\t467\t423\t11.438\t' \
'11.063\t12.563\t246606\t35.799\t11.866\t11.963\t5.010\t112.953\t0.000\t0.000\t0.000\t0.000\t0.000' \
'\t0\t-1.000\t-1.000\t-1.000\t16\n')
self.telem_particle_d = NutnrJCsppTelemeteredDataParticle(particle_d_match)
# generate the dictionary so the timestamp is set
self.telem_particle_d.generate_dict()
particle_e_match = DATA_MATCHER.match('1397774273.831\t0.000\ty\tSDB\t2014\t107\t22.628356\t0.000\t' \
'0.000\t0.000\t0.000\t0.000\t481\t0\t1\t489\t486\t479\t477\t475\t492\t487\t488\t490\t501\t491\t' \
'488\t484\t477\t471\t490\t474\t477\t477\t487\t490\t491\t497\t473\t491\t489\t483\t473\t481\t482\t' \
'494\t489\t497\t492\t475\t477\t484\t485\t473\t485\t478\t471\t477\t472\t459\t471\t490\t476\t475\t' \
'485\t477\t485\t480\t483\t473\t486\t491\t489\t473\t487\t491\t493\t503\t489\t485\t480\t477\t473\t' \
'480\t487\t493\t477\t485\t477\t485\t470\t494\t484\t479\t489\t491\t485\t479\t503\t484\t487\t477\t' \
'477\t483\t481\t486\t491\t493\t477\t487\t492\t500\t492\t494\t489\t480\t485\t484\t487\t483\t475\t' \
'487\t490\t489\t481\t481\t485\t472\t471\t481\t477\t483\t477\t473\t468\t475\t474\t474\t471\t474\t' \
'470\t469\t479\t477\t474\t485\t478\t477\t482\t469\t476\t482\t481\t481\t499\t482\t473\t473\t470\t' \
'477\t468\t469\t481\t489\t486\t493\t478\t479\t483\t482\t474\t487\t478\t480\t472\t475\t489\t484\t' \
'473\t483\t477\t477\t483\t486\t473\t473\t477\t480\t491\t477\t467\t483\t484\t493\t495\t487\t479\t' \
'485\t482\t465\t491\t476\t485\t488\t477\t471\t467\t458\t471\t472\t489\t491\t489\t481\t494\t473\t' \
'480\t471\t481\t474\t480\t482\t483\t485\t470\t475\t477\t473\t472\t485\t475\t481\t488\t477\t482\t' \
'473\t471\t467\t483\t482\t493\t493\t483\t491\t493\t485\t471\t487\t482\t475\t474\t478\t482\t485\t' \
'484\t472\t471\t476\t486\t489\t497\t481\t483\t497\t475\t473\t477\t477\t474\t473\t394\t11.438\t' \
'11.125\t12.563\t246606\t35.805\t11.911\t11.965\t5.049\t111.832\t0.000\t0.000\t0.000\t0.000\t0.000' \
'\t0\t-1.000\t-1.000\t-1.000\t93\n')
self.telem_particle_e = NutnrJCsppTelemeteredDataParticle(particle_e_match)
# generate the dictionary so the timestamp is set
self.telem_particle_e.generate_dict()
self.meta_telem_e = NutnrJCsppMetadataTelemeteredDataParticle((header_dict, particle_e_match))
# generate the dictionary so the timestamp is set
self.meta_telem_e.generate_dict()
# the particle at the end of the long file 224798:226490
particle_long_end_match = DATA_MATCHER.match('1397774471.010\t0.000\tn\tSLB\t2014\t107' \
'\t22.683184\t0.717\t0.010\t-0.022\t-0.022\t0.000\t24747\t482\t1\t517\t517\t508\t506\t500\t515\t522\t' \
'523\t522\t531\t527\t521\t541\t521\t511\t502\t505\t521\t517\t532\t579\t654\t801\t1074\t1555\t2280\t' \
'3357\t4853\t6855\t9373\t12459\t16069\t20157\t24643\t29400\t34168\t38801\t42966\t46394\t48931\t50337' \
'\t50690\t50177\t48948\t47287\t45540\t43811\t42299\t41061\t40156\t39572\t39341\t39376\t39706\t40324\t' \
'41152\t42189\t43440\t44902\t46512\t48277\t50040\t51858\t53556\t55002\t56169\t56916\t57085\t56756\t' \
'55800\t54332\t52490\t50237\t47824\t45403\t43002\t40749\t38656\t36759\t35078\t33608\t32339\t31264\t' \
'30381\t29670\t29113\t28756\t28544\t28461\t28567\t28789\t29176\t29656\t30283\t31026\t31858\t32728\t' \
'33637\t34534\t35337\t36071\t36609\t36939\t37031\t36856\t36417\t35738\t34859\t33819\t32720\t31517\t' \
'30349\t29192\t28092\t27080\t26165\t25355\t24635\t24048\t23543\t23143\t22871\t22676\t22585\t22611\t' \
'22698\t22893\t23145\t23489\t23900\t24345\t24859\t25459\t26070\t26725\t27465\t28224\t28995\t29793\t' \
'30585\t31380\t32165\t32906\t33587\t34202\t34747\t35187\t35568\t35817\t35926\t35943\t35834\t35614\t' \
'35248\t34794\t34238\t33620\t32904\t32195\t31461\t30669\t29921\t29156\t28419\t27700\t26972\t26264\t' \
'25577\t24866\t24175\t23514\t22852\t22267\t21712\t21220\t20795\t20396\t20039\t19735\t19502\t19285\t' \
'19096\t18946\t18786\t18646\t18514\t18358\t18247\t18175\t18048\t17932\t17837\t17756\t17702\t17653\t' \
'17614\t17600\t17585\t17561\t17595\t17590\t17655\t17716\t17744\t17780\t17818\t17867\t17909\t17921\t' \
'17911\t17912\t17898\t17854\t17789\t17715\t17626\t17497\t17367\t17221\t17075\t16923\t16727\t16456\t' \
'16152\t15793\t15469\t15189\t14904\t14641\t14390\t14165\t13957\t13747\t13568\t13465\t13354\t13191\t' \
'12936\t12676\t12460\t12258\t12053\t11863\t11717\t11582\t11506\t11437\t11349\t11225\t11093\t11037\t' \
'10955\t10741\t10251\t9467\t8475\t12.313\t13.188\t13.188\t246633\t34.853\t11.745\t11.996\t4.988\t' \
'493.629\t26.120\t0.411\t-3.311\t0.583\t0.000\t0\t-1.000\t-1.000\t-1.000\t43\n')
self.telem_particle_long = NutnrJCsppTelemeteredDataParticle(particle_long_end_match)
# generate the dictionary so the timestamp is set
self.telem_particle_long.generate_dict()
# recovered particles using the same data as telemetered particles
self.meta_recov_particle = NutnrJCsppMetadataRecoveredDataParticle((header_dict, particle_a_match))
self.meta_recov_particle.generate_dict()
self.recov_particle_a = NutnrJCsppRecoveredDataParticle(particle_a_match)
self.recov_particle_a.generate_dict()
self.recov_particle_b = NutnrJCsppRecoveredDataParticle(particle_b_match)
self.recov_particle_b.generate_dict()
self.recov_particle_c = NutnrJCsppRecoveredDataParticle(particle_c_match)
self.recov_particle_c.generate_dict()
self.recov_particle_d = NutnrJCsppRecoveredDataParticle(particle_d_match)
self.recov_particle_d.generate_dict()
self.recov_particle_e = NutnrJCsppRecoveredDataParticle(particle_e_match)
self.recov_particle_e.generate_dict()
self.file_ingested_value = None
self.state_callback_value = None
self.publish_callback_value = None
self.exception_callback_value = []
def assert_equal_regex(self, match1, match2):
"""
A data regex match cannot be directly equal to another data regex match,
pull out the matching raw data result to see if the regexes matched the same
data
"""
self.assertEqual(match1.group(0), match2.group(0))
def assert_result(self, result, position, particle, ingested, is_metadata=False):
"""
Assert two particles and the states after that particle are equal
"""
self.assert_(isinstance(self.publish_callback_value, list))
if is_metadata:
self.assert_metadata(result, particle)
else:
self.assert_data(result, particle)
self.assert_timestamp(result, particle)
self.assert_position(position)
# compare if the file is ingested
self.assertEqual(self.file_ingested_value, ingested)
def assert_metadata(self, result, particle, result_index=0):
"""
Assert two metadata particles raw data matches
"""
# the data regex match is passed in as the second argument to the metadata,
# need to pull out the string for comparison since regex results cannot be
# directly compared
self.assertEqual(result[result_index].raw_data[0], particle.raw_data[0])
self.assert_equal_regex(result[result_index].raw_data[1], particle.raw_data[1])
self.assertEqual(self.publish_callback_value[result_index].raw_data[0],
particle.raw_data[0])
self.assert_equal_regex(self.publish_callback_value[result_index].raw_data[1],
particle.raw_data[1])
def assert_data(self, result, particle, result_index=0):
# the data regex match is passed in as the argument to the data particle,
# need to pull out the string for comparison since regex results cannot be
# directly compared
self.assert_equal_regex(result[result_index].raw_data, particle.raw_data)
self.assert_equal_regex(self.publish_callback_value[result_index].raw_data,
particle.raw_data)
def assert_timestamp(self, result, particle, result_index=0):
"""
Compare the timestamp of two particles
"""
allowed_diff = .000001
self.assertTrue(abs(result[result_index].contents[DataParticleKey.INTERNAL_TIMESTAMP] - \
particle.contents[DataParticleKey.INTERNAL_TIMESTAMP]) <= allowed_diff)
def assert_data_and_timestamp(self, result, particle, result_index=0):
"""
Combine asserting data and timestamp
"""
self.assert_data(result, particle, result_index)
self.assert_timestamp(result, particle, result_index)
def assert_metadata_and_timestamp(self, result, particle, result_index=0):
self.assert_metadata(result, particle, result_index)
self.assert_timestamp(result, particle, result_index)
def assert_position(self, position):
"""
Assert that the position in the parser state and state callback match
the expected position
"""
self.assertEqual(self.parser._state[StateKey.POSITION], position)
self.assertEqual(self.state_callback_value[StateKey.POSITION], position)
def create_parser(self, stream_handle, telem_flag=True, state=None):
"""
Initialize the parser with the given stream handle, using the
telemetered config if the flag is set, recovered if it is not
"""
if telem_flag:
# use telemetered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.nutnr_j_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppTelemeteredDataParticle
}
}
else:
# use recovered config
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.nutnr_j_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: NutnrJCsppRecoveredDataParticle
}
}
self.parser = NutnrJCsppParser(config, state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
def test_simple_telem(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'rb')
self.create_parser(stream_handle)
# get and compare the metadata particle
result = self.parser.get_records(1)
self.assert_result(result, 0, self.meta_telem_particle, False, is_metadata=True)
# get and compare the first data particle
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_A_POS, self.telem_particle_a, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_B_POS, self.telem_particle_b, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_C_POS, self.telem_particle_c, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_D_POS, self.telem_particle_d, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_E_POS, self.telem_particle_e, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_simple_recov(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'rb')
self.create_parser(stream_handle, telem_flag=False)
# get and compare the metadata particle
result = self.parser.get_records(1)
self.assert_result(result, 0, self.meta_recov_particle, False, is_metadata=True)
# get and compare the first data particle
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_A_POS, self.recov_particle_a, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_B_POS, self.recov_particle_b, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_C_POS, self.recov_particle_c, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_D_POS, self.recov_particle_d, False)
result = self.parser.get_records(1)
self.assert_result(result, PARTICLE_E_POS, self.recov_particle_e, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'rb')
self.create_parser(stream_handle)
result = self.parser.get_records(6)
self.assert_(isinstance(self.publish_callback_value, list))
# compare particle raw data values
self.assert_metadata_and_timestamp(result, self.meta_telem_particle)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=4)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=5)
# compare position and file ingested from the state
self.assert_position(PARTICLE_E_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_long_stream(self):
"""
Test a long stream
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'11079419_SNA_SNA.txt'), 'rb')
self.create_parser(stream_handle)
result = self.parser.get_records(172)
self.assertEqual(len(result), 172)
self.assert_(isinstance(self.publish_callback_value, list))
# compare particle raw data values, check the particles we know from
# the beginning, then the last particle in the file
self.assert_metadata_and_timestamp(result, self.meta_telem_particle)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=4)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=5)
self.assert_data_and_timestamp(result, self.telem_particle_long, result_index=-1)
# compare position and file ingested from the state
self.assert_position(END_LONG_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
"""
# set the state so that the metadata, particle A and B have been read
new_state = {StateKey.POSITION: PARTICLE_B_POS,
StateKey.METADATA_EXTRACTED: True}
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'rb')
self.create_parser(stream_handle, state=new_state)
# ask for more records but should only get 3 back: c, d, and e
result = self.parser.get_records(6)
self.assertEqual(len(result), 3)
self.assert_(isinstance(self.publish_callback_value, list))
# compare particle raw data values
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=2)
# compare position and file ingested from the state
self.assert_position(PARTICLE_E_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
# set the state so that the metadata, particle A, B, and C have been read
new_state = {StateKey.POSITION: PARTICLE_C_POS,
StateKey.METADATA_EXTRACTED: True}
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'r')
self.create_parser(stream_handle)
# get metadata and A
result = self.parser.get_records(2)
# compare particle raw data values and timestamp
self.assert_metadata_and_timestamp(result, self.meta_telem_particle)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=1)
# compare position and file ingested from the state
self.assert_position(PARTICLE_A_POS)
self.assertEqual(self.file_ingested_value, False)
# now change the state to skip over B and C
self.parser.set_state(new_state)
# confirm we get D and E now
result = self.parser.get_records(2)
# compare particle raw data values and timestamp
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=1)
# compare position and file ingested from the state
self.assert_position(PARTICLE_E_POS)
self.assertEqual(self.file_ingested_value, True)
# confirm no exceptions occurred
self.assertEqual(self.exception_callback_value, [])
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# bad data file has:
# 1 bad status
# particle A has bad timestamp
# particle B has bad dark fit
# particle C has bad frame type
# particle D has bad year
stream_handle = open(os.path.join(RESOURCE_PATH,
'bad_SNA_SNA.txt'), 'r')
self.create_parser(stream_handle)
# get E, since it is first it will generate a metadata
result = self.parser.get_records(1)
self.assert_result(result, 0, self.meta_telem_e, False, is_metadata=True)
result = self.parser.get_records(1)
# bytes in file changed due to making file 'bad'
self.assert_result(result, 10257, self.telem_particle_e, True)
# should have had 5 exceptions by now
self.assertEqual(len(self.exception_callback_value), 5)
for exception in self.exception_callback_value:
self.assert_(isinstance(exception, RecoverableSampleException))
def test_missing_source_file(self):
"""
Test that a file with a missing source file path in the header
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'no_source_file_SNA_SNA.txt'), 'r')
self.create_parser(stream_handle)
# get A-E, without metadata
result = self.parser.get_records(5)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=4)
# compare file ingested from the state
self.assertEqual(self.file_ingested_value, True)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_no_header(self):
"""
Test that a file with no header lines
fails to create a metadata particle and throws an exception
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'no_header_SNA_SNA.txt'), 'r')
self.create_parser(stream_handle)
# get A-E, without metadata
result = self.parser.get_records(5)
self.assert_data_and_timestamp(result, self.telem_particle_a, result_index=0)
self.assert_data_and_timestamp(result, self.telem_particle_b, result_index=1)
self.assert_data_and_timestamp(result, self.telem_particle_c, result_index=2)
self.assert_data_and_timestamp(result, self.telem_particle_d, result_index=3)
self.assert_data_and_timestamp(result, self.telem_particle_e, result_index=4)
# compare file ingested from the state
self.assertEqual(self.file_ingested_value, True)
# confirm an exception occurred
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_bad_config(self):
"""
Test that configurations with a missing data particle dict and missing
data particle class key causes a configuration exception
"""
# test a config with a missing particle classes dict
config = {}
stream_handle = open(os.path.join(RESOURCE_PATH,
'short_SNA_SNA.txt'), 'r')
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# test a config with a missing data particle class key
config = {
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: NutnrJCsppMetadataTelemeteredDataParticle,
}
}
with self.assertRaises(ConfigurationException):
self.parser = NutnrJCsppParser(config, None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
