def test_build_yml_file(self):
"""
Read test data. Should detect that there is a decimation factor in the data.
Check that the data matches the expected results.
"""
log.debug('CAG TEST: START BUILDING YML FILE')
stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp_sio_mule/resource/BIG_DATA_FILE.dat', 'rb')
self.parser = CtdpfCklWfpSioMuleParser(self.config, None, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback)
result = self.parser.get_records(50000)
self.particle_to_yml(result, 'BIG_DATA_FILE.yml')
log.debug('CAG TEST: FINISHED BUILDING YML FILE')
def test_simple_with_input_too_short(self):
"""
Read test data. Should detect that the input stream ?????
Data stream should be rejected.
"""
log.debug('CAG TEST: FILE IS TOO SHORT')
stream_handle = StringIO(TEST_DATA_wts)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(1)
if result:
log.debug('CAG TEST: FAILED TO DETECT FILE IS TOO SHORT CASE')
else:
log.debug('CAG TEST: FILE IS TOO SHORT DETECTED')
def test_simple_with_wrong_header(self):
"""
Read test data. Should detect that the input stream ?????
Data stream should be rejected.
"""
log.debug('CAG TEST: FILE HAS THE WRONG HEADER')
stream_handle = StringIO(TEST_DATA_wwh)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(1)
if result:
log.debug('CAG TEST: FAILED TO DETECT WRONG HEADER')
else:
log.debug('CAG TEST: WRONG HEADER DETECTED')
def test_simple_with_no_data_recs(self):
"""
Read test data. Should detect that there is no data between the header and footer.
Data out should be a metadata particle only
"""
log.debug('CAG TEST: NO DATA RECORDS')
stream_handle = StringIO(TEST_DATA_ndr)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(1)
if result:
log.debug('CAG TEST: FAILED TO DETECT NO DATA RECORDS CASE')
else:
log.debug('CAG TEST: NO DATA RECORDS DETECTED')
def test_simple_with_no_eop(self):
"""
Read test data. Should detect that the End of Profile (eop) is missing.
Data stream should be rejected.
"""
log.debug('CAG TEST: MISSING END OF PROFILE')
stream_handle = StringIO(TEST_DATA_neop)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(4)
if result:
log.debug('CAG TEST: FAILED TO DETECT MISSING END OF PROFILE')
else:
log.debug('CAG TEST: MISSING END OF PROFILE DETECTED')
def test_simple_with_no_time_stamp(self):
"""
Read test data. Should detect that the data is missing the time stamp.
Data stream should be rejected.
"""
log.debug('CAG TEST: NO TIME STAMP')
stream_handle = StringIO(TEST_DATA_nts)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(4)
if result:
log.debug('CAG TEST: FAILED TO DETECT NO TIME STAMP')
else:
log.debug('CAG TEST: NO TIME STAMP DETECTED')
def test_simple_with_incorrect_header(self):
"""
Read test data. Should detect that the header is NOT for a WC SIO block
Data stream should be rejected.
"""
log.debug('CAG TEST: INCORRECT HEADER')
stream_handle = StringIO(TEST_DATA_bts)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(4)
if result:
log.debug('CAG TEST: FAILED TO DETECT INCORRECT HEADER')
else:
log.debug('CAG TEST: INCORRECT HEADER DETECTED')
def test_simple_with_no_decimation_factor(self):
"""
Read test data. Should detect that there is NO decimation factor in the data.
Check that the data matches the expected results.
"""
log.debug('CAG TEST: FILE HAS NO DECIMATION FACTOR')
stream_handle = StringIO(TEST_DATA_ndf)
self.parser = CtdpfCklWfpSioMuleParser(self.config, None,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# next get records
result = self.parser.get_records(4)
self.assertEqual(result, [
self.particle_meta_ndf, self.particle_a, self.particle_b,
self.particle_c
])
log.debug('CAG TEST: NO DECIMATION FACTOR TEST PASSES')
def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the parser
"""
# Default the parser to None
parser = None
config = self._parser_config.get(data_key)
#
# If the key is CTDPF_CKL_WFP, build the ctdpf_ckl_wfp parser and
# provide a config that includes the specific recovered particle types.
#
if data_key == DataTypeKey.CTDPF_CKL_WFP:
log.debug('CAG DRIVER - build parser for %s. State is %s',
data_key, parser_state)
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.ctdpf_ckl_wfp_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
INSTRUMENT_DATA_PARTICLE_CLASS:
CtdpfCklWfpRecoveredDataParticle,
METADATA_PARTICLE_CLASS:
CtdpfCklWfpRecoveredMetadataParticle
}
})
parser = CtdpfCklWfpParser(
config, parser_state, infile,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback, os.path.getsize(infile.name))
#
# If the key is CTDPF_CKL_WFP_SIO_MULE, build the ctdpf_ckl_wfp_sio_mule parser and
# provide a config that includes the specific telemetered particle types.
#
elif data_key == DataTypeKey.CTDPF_CKL_WFP_SIO_MULE:
log.debug('CAG DRIVER - build parser for %s. State is %s',
data_key, parser_state)
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.ctdpf_ckl_wfp_sio_mule',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
INSTRUMENT_DATA_PARTICLE_CLASS:
CtdpfCklWfpSioMuleDataParticle,
METADATA_PARTICLE_CLASS: CtdpfCklWfpSioMuleMetadataParticle
}
})
parser = CtdpfCklWfpSioMuleParser(
config, parser_state, infile,
lambda state: self._save_parser_state(
state, DataTypeKey.CTDPF_CKL_WFP_SIO_MULE),
self._data_callback, self._sample_exception_callback)
else:
raise ConfigurationException(
'Bad Configuration: %s - Failed to build ctdpf_ckl_wfp parser',
config)
return parser
