def test_get_many(self):
"""
Read test data and pull out data particles in smaller groups.
Assert that the results are those we expected.
"""
with open(os.path.join(RESOURCE_PATH, '11079419_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
# Attempt to retrieve 20 total particles
particles = parser.get_records(5)
particles2 = parser.get_records(10)
particles.extend(particles2)
particles3 = parser.get_records(5)
particles.extend(particles3)
# Should end up with 20 particles
self.assertTrue(len(particles) == 20)
self.assert_particles(particles, '11079419_PPB_OPT.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
def test_midstate_start(self):
"""
This test makes sure that we retrieve the correct particles upon starting with an offsetted state.
"""
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
initial_state = {
StateKey.POSITION: 332,
StateKey.METADATA_EXTRACTED: True
}
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
initial_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback)
particles = parser.get_records(2)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + 2], particles[i])
log.info("******** Read State: %s", parser._read_state)
log.info("******** State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 518,
StateKey.METADATA_EXTRACTED: True
})
def test_air_saturation_preset(self):
"""
Ensure that input files containing the air saturation field are parsed correctly.
Redmine #10238 Identified additional parameter enabled after first deployment
"""
with open(os.path.join(RESOURCE_PATH, 'ucspp_32260420_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
# get the metadata particle and first 2 instrument particles and verify values.
particles = parser.get_records(3)
self.assertTrue(len(particles) == 3)
self.assertEqual(self.exception_callback_value, [])
self.assert_particles(particles, 'ucspp_32260420_PPB_OPT.yml',
RESOURCE_PATH)
# get remaining particles and verify parsed without error.
particles = parser.get_records(100)
self.assertTrue(len(particles) == 93)
self.assertEqual(self.exception_callback_value, [])
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
# Let's attempt to retrieve 20 particles
particles = parser.get_records(20)
log.info("Exception callback value: %s", self.exception_callback_value)
# Should end up with 20 particles
self.assertTrue(len(particles) == 20)
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
def test_simple(self):
"""
Read test data and pull out data particles.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(5)
log.info("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is None)
self.assertTrue(len(particles) == 5)
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
stream_handle.close()
def _build_parser(self, stream_handle):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dosta_abcdjm_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: DostaAbcdjmCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: DostaAbcdjmCsppInstrumentRecoveredDataParticle,
}
}
return DostaAbcdjmCsppParser(parser_config, stream_handle, self._exception_callback)
def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the parser
"""
config = self._parser_config.get(data_key)
#
# If the key is DOSTA_ABCDJM_CSPP_RECOVERED, build the WFP parser.
#
if data_key == DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dosta_abcdjm_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
DostaAbcdjmCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY:
DostaAbcdjmCsppInstrumentRecoveredDataParticle
}
})
#
# If the key is DOSTA_ABCDJM_CSPP_TELEMETERED, build the WFP SIO Mule parser.
#
elif data_key == DataTypeKey.DOSTA_ABCDJM_CSPP_TELEMETERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dosta_abcdjm_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
DostaAbcdjmCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY:
DostaAbcdjmCsppInstrumentTelemeteredDataParticle
}
})
#
# If the key is one that we're not expecting, don't build any parser.
#
else:
raise ConfigurationException(
"Invalid data_key supplied to build parser")
parser = DostaAbcdjmCsppParser(
config, parser_state, infile,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
return parser
def test_bad_header_start_date(self):
"""
Ensure that bad data is skipped when it exists.
"""
with open(
os.path.join(RESOURCE_PATH,
'BadHeaderProcessedData_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
# parser should return metadata without start date filled in
parser.get_records(1)
self.assertEqual(self.exception_callback_value, [])
def test_linux_source_path_handling(self):
"""
Read linux source path test data and assert that the results are those we expected.
"""
with open(os.path.join(RESOURCE_PATH, 'linux_11079894_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
particles = parser.get_records(20)
self.assertTrue(len(particles) == 20)
self.assert_particles(particles, 'linux.yml', RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_long_stream(self):
"""
Test a long stream
"""
with open(os.path.join(RESOURCE_PATH, '11079364_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
# Let's attempt to retrieve more particles than are available
particles = parser.get_records(300)
# Should end up with 272 particles
self.assertTrue(len(particles) == 272)
self.assertEquals(self.exception_callback_value, [])
def test_bad_header_source_file_name(self):
"""
Ensure that bad source file name produces an error
"""
with open(
os.path.join(RESOURCE_PATH,
'BadHeaderSourceFileName_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
parser.get_records(1)
self.assertEquals(len(self.exception_callback_value), 1)
self.assertIsInstance(self.exception_callback_value[0],
SampleEncodingException)
def test_bad_data_record(self):
"""
Ensure that bad data creates a recoverable sample exception and parsing continues
"""
with open(os.path.join(RESOURCE_PATH, 'BadDataRecord_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
particles = parser.get_records(19)
self.assert_particles(particles, 'BadDataRecord_PPB_OPT.yml',
RESOURCE_PATH)
self.assertEquals(len(self.exception_callback_value), 1)
self.assertIsInstance(self.exception_callback_value[0],
RecoverableSampleException)
def create_yml(self):
"""
This utility creates a yml file
"""
fid = open(os.path.join(RESOURCE_PATH, '11194982_PPD_OPT.txt'))
test_buffer = fid.read()
fid.close()
self.stream_handle = StringIO(test_buffer)
self.parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_TELEMETERED), None,
self.stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = self.parser.get_records(20)
log.info("Exception callback value: %s", self.exception_callback_value)
self.particle_to_yml(particles, '11194982_PPD_OPT.yml')
def test_bad_header_start_date(self):
"""
Ensure that bad data is skipped when it exists.
"""
file_path = os.path.join(RESOURCE_PATH,
'BadHeaderProcessedData_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
log.info(self.exception_callback_value)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
# parser should return metadata without start date filled in
parser.get_records(1)
self.assertEqual(self.exception_callback_value, None)
stream_handle.close()
def test_bad_data_record(self):
"""
Ensure that bad data is skipped when it exists.
"""
file_path = os.path.join(RESOURCE_PATH, 'BadDataRecord_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
log.info(self.exception_callback_value)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
parser.get_records(1)
log.info("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
stream_handle.close()
def test_simple_telem(self):
"""
Read test data and pull out data particles.
Assert that the results are those we expected.
"""
with open(os.path.join(RESOURCE_PATH, '11194982_PPD_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_telemetered,
stream_handle,
self.exception_callback)
# Attempt to retrieve 20 particles, there are only 18 in the file though
particles = parser.get_records(20)
# Should end up with 18 particles
self.assertTrue(len(particles) == 18)
self.assert_particles(particles, '11194982_PPD_OPT.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
def test_simple_recov(self):
"""
Read test data and pull out data particles.
Assert that the results are those we expected.
"""
with open(os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt'),
'rU') as stream_handle:
parser = DostaAbcdjmCsppParser(self.config_recovered,
stream_handle,
self.exception_callback)
# Attempt to retrieve 20 particles, there are more in this file but only verify 20
particles = parser.get_records(20)
# Should end up with 20 particles
self.assertTrue(len(particles) == 20)
self.assert_particles(particles, '11079894_PPB_OPT.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
def test_long_stream(self):
"""
Test a long stream
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
# Let's attempt to retrieve 2000 particles
particles = parser.get_records(300)
log.info("Num particles: %s", len(particles))
log.info("Exception callback value: %s", self.exception_callback_value)
# Should end up with 272 particles
self.assertTrue(len(particles) == 272)
stream_handle.close()
def test_state_after_one_record_retrieval(self):
"""
This test makes sure that we get the correct particles upon requesting one record at
a time.
"""
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 1)
log.info("11111111 Read State: %s", parser._read_state)
log.info("11111111 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 0,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][0], particles[0])
new_state = copy.copy(parser._state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("22222222 Read State: %s", parser._read_state)
log.info("22222222 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 332,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][1], particles[0])
new_state = copy.copy(parser._state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("33333333 Read State: %s", parser._read_state)
log.info("33333333 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 425,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][2], particles[0])
new_state = copy.copy(parser._state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("44444444 Read State: %s", parser._read_state)
log.info("44444444 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 518,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][3], particles[0])
new_state = copy.copy(parser._state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("55555555 Read State: %s", parser._read_state)
log.info("55555555 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 611,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][4], particles[0])
def test_state_after_two_record_retrievals(self):
"""
This test makes sure that we get the correct particles upon requesting two records at
a time.
"""
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(2)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
log.info("11111111 Read State: %s", parser._read_state)
log.info("11111111 State: %s", parser._state)
the_new_state = {
StateKey.POSITION: 332,
StateKey.METADATA_EXTRACTED: True
}
log.info("11111111 new parser state: %s", the_new_state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
the_new_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback)
particles = parser.get_records(2)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + 2], particles[i])
log.info("22222222 Read State: %s", parser._read_state)
log.info("22222222 State: %s", parser._state)
the_new_state = {
StateKey.POSITION: 480,
StateKey.METADATA_EXTRACTED: True
}
log.info("22222222 new parser state: %s", the_new_state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
the_new_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback)
particles = parser.get_records(2)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + 4], particles[i])
log.info("33333333 Read State: %s", parser._read_state)
log.info("33333333 State: %s", parser._state)
def test_position_and_metadata_extracted_state(self):
"""
This test makes sure that we retrieve the metadata record upon resetting the state position to 0
and setting the METADATA_EXTRACTED to False.
"""
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 1)
log.info("11111111 Read State: %s", parser._read_state)
log.info("11111111 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 0,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][0], particles[0])
new_state = {StateKey.POSITION: 0, StateKey.METADATA_EXTRACTED: False}
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
self.assertTrue(len(particles) == 1)
log.info("22222222 Read State: %s", parser._read_state)
log.info("22222222 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 0,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][0], particles[0])
new_state = {StateKey.POSITION: 0, StateKey.METADATA_EXTRACTED: True}
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
self.assertTrue(len(particles) == 1)
log.info("22222222 Read State: %s", parser._read_state)
log.info("22222222 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 332,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][1], particles[0])
def test_state_reset(self):
"""
This test makes sure that we retrieve the correct particles upon resetting the state to a prior position.
"""
expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
stream_handle = open(file_path, 'rb')
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 1)
log.info("11111111 Read State: %s", parser._read_state)
log.info("11111111 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 0,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][0], particles[0])
new_state = copy.copy(parser._state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("22222222 Read State: %s", parser._read_state)
log.info("22222222 State: %s", parser._state)
self.assertTrue(len(particles) == 1)
self.assertTrue(parser._state == {
StateKey.POSITION: 332,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][1], particles[0])
new_state = copy.copy(parser._state)
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(1)
log.info("33333333 Read State: %s", parser._read_state)
log.info("33333333 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 425,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][2], particles[0])
new_state = {StateKey.POSITION: 0, StateKey.METADATA_EXTRACTED: True}
parser = DostaAbcdjmCsppParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
new_state, stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
# Now retrieve two particles. We should end up with the metadata and first data record
particles = parser.get_records(1)
log.info("44444444 Read State: %s", parser._read_state)
log.info("44444444 State: %s", parser._state)
self.assertTrue(parser._state == {
StateKey.POSITION: 332,
StateKey.METADATA_EXTRACTED: True
})
self.assert_result(expected_results['data'][1], particles[0])