def test_recov(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
The contents of ADCP_data_20130702.000 are the expected results
from the IDD. These results for the that record were manually verified
and are the entire parsed particle is represented in ADCP_data_20130702.yml
"""
# ND072022.PD0 contains a single ADCPA ensemble
with open(os.path.join(RESOURCE_PATH, '20140424.adcpt.log'),
'rU') as stream_handle:
parser = AdcptAcfgmDclPd0Parser(self.config_recov, stream_handle,
self.exception_callback,
self.state_callback,
self.publish_callback)
particles = parser.get_records(31)
log.debug('got back %d particles', len(particles))
# Note the yml file was produced from the parser output but was hand verified
# against the sample outputs provided in the IDD
self.assert_particles(particles, '20140424.recov.adcpt.yml',
RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_telem(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
The contents of ADCP_data_20130702.000 are the expected results
from the IDD. These results for the that record were manually verified
and are the entire parsed particle is represented in ADCP_data_20130702.yml
"""
# ND072022.PD0 contains a single ADCPA ensemble
with open(os.path.join(RESOURCE_PATH, '20140424.adcpt.log'),
'rb') as stream_handle:
parser = AdcptAcfgmDclPd0Parser(self.config_telem, stream_handle,
self.exception_callback,
self.state_callback,
self.publish_callback)
particles = parser.get_records(20)
# ask for 20 but should only get 15
log.debug('got back %d particles', len(particles))
self.assert_particles(particles, '20140424.telem.adcpt.yml',
RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
#20140424.adcpt_BAD.log has a corrupt record in it
with open(os.path.join(RESOURCE_PATH, '20140424.adcpt_BAD.log'),
'rb') as stream_handle:
parser = AdcptAcfgmDclPd0Parser(self.config_recov, stream_handle,
self.exception_callback,
self.state_callback,
self.publish_callback)
#try to get a particle, should get none
parser.get_records(1)
self.assert_(
isinstance(self.exception_callback_value[0],
RecoverableSampleException))
def process(self):
with open(self._sourceFilePath, "r") as file_handle:
def exception_callback(exception):
log.debug("Exception: %s", exception)
self._particleDataHdlrObj.setParticleDataCaptureFailure()
parser = AdcptAcfgmDclPd0Parser(self._parser_config, file_handle,
exception_callback,
lambda state, ingested: None,
lambda data: None)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
return self._particleDataHdlrObj
def test_live_data(self):
files_without_records = [
'20140424.adcpt_BAD.log',
'20141007.adcpt.log',
'20141008.adcpt.log',
]
for filename in os.listdir(RESOURCE_PATH):
if filename.endswith('.log'):
log.info('Testing file: %s', filename)
with open(os.path.join(RESOURCE_PATH, filename), 'rb') as fh:
parser = AdcptAcfgmDclPd0Parser(self.config_telem, fh,
self.exception_callback,
self.state_callback,
self.publish_callback)
particles = parser.get_records(100)
log.debug('got back %d particles', len(particles))
if filename not in files_without_records:
self.assertGreater(len(particles), 0)