def test_bad_eop_data(self):
"""
If the last "data" record in the file is not 11 byes of 0xFF, raise a sample exception
and do not parse the file.
"""
stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_EOP)
#********************************************
# Test the "recovered" version of the parser
#********************************************
with self.assertRaises(SampleException):
recovered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED),
self.recovered_start_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
with self.assertRaises(SampleException):
telemetered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED),
self.telemetered_start_state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
def test_bad_size_data(self):
"""
If any of the data records in the file are not 11 bytes long, raise a sample exception
and do not parse the file.
"""
stream_handle = StringIO(
DofstKWfpParserUnitTestCase.TEST_DATA_BAD_SIZE)
#********************************************
# Test the "recovered" version of the parser
#********************************************
with self.assertRaises(SampleException):
recovered_parser = DofstKWfpParser(
self.config_recovered, self.recovered_start_state,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_SIZE))
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
with self.assertRaises(SampleException):
telemetered_parser = DofstKWfpParser(
self.config_telemetered, self.telemetered_start_state,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_SIZE))
def test_simple(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA)
#********************************************
# Test the "recovered" version of the parser
#********************************************
recovered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = recovered_parser
# next get records
recovered_result = recovered_parser.get_records(1)
self.assert_result(recovered_result, 0, self.particle_meta, False, 0, True)
recovered_result = self.parser.get_records(1)
self.assert_result(recovered_result, 11, self.particle_a, False, 1, True)
recovered_result = self.parser.get_records(1)
self.assert_result(recovered_result, 22, self.particle_b, False, 2, True)
recovered_result = self.parser.get_records(1)
self.assert_result(recovered_result, 33, self.particle_c, True, 3, True)
# no data left, dont move the position
recovered_result = recovered_parser.get_records(1)
self.assertEqual(recovered_result, [])
self.assertEqual(recovered_parser._state[StateKey.POSITION], 33)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
self.assert_(isinstance(self.publish_callback_value, list))
self.assertEqual(self.publish_callback_value[0], self.particle_c)
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
telemetered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = telemetered_parser
# next get records
telemetered_result = telemetered_parser.get_records(1)
self.assert_result(telemetered_result, 0, self.particle_meta, False, 0, True)
telemetered_result = self.parser.get_records(1)
self.assert_result(telemetered_result, 11, self.particle_a, False, 1, True)
telemetered_result = self.parser.get_records(1)
self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True)
telemetered_result = self.parser.get_records(1)
self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)
# no data left, dont move the position
telemetered_result = telemetered_parser.get_records(1)
self.assertEqual(telemetered_result, [])
self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
self.assert_(isinstance(self.publish_callback_value, list))
self.assertEqual(self.publish_callback_value[0], self.particle_c)
def test_bad_time_data(self):
"""
If the timestamps are missing, raise a sample exception and do not parse the file
"""
stream_handle = StringIO(
DofstKWfpParserUnitTestCase.TEST_DATA_BAD_TIME)
#********************************************
# Test the "recovered" version of the parser
#********************************************
with self.assertRaises(SampleException):
recovered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED),
self.recovered_start_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_TIME))
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
with self.assertRaises(SampleException):
telemetered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED),
self.telemetered_start_state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_TIME))
def _build_parser(self, parser_state, file_handle, data_key=None):
"""
Build and return the parser
"""
# Default the parser to None
parser = None
config = self._parser_config.get(data_key)
#
# If the key is DOFST_K_WFP_RECOVERED, build the dofst_k_wfp parser and
# provide a config that includes the specific recovered particle types.
#
if data_key == DataTypeKey.DOFST_K_WFP_RECOVERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dofst_k_wfp_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'instrument_data_particle_class': DofstKWfpRecoveredDataParticle,
'metadata_particle_class': DofstKWfpRecoveredMetadataParticle
}
})
log.debug("My Config: %s", config)
parser = DofstKWfpParser(
config,
parser_state,
file_handle,
lambda state, ingested: self._save_parser_state(state, data_key, ingested),
self._data_callback,
self._sample_exception_callback,
os.path.getsize(file_handle.name))
#
# If the key is DOFST_K_WFP_TELEMETERED, build the dofst_k_wfp parser and
# provide a config that includes the specific telemetered particle types.
#
elif data_key == DataTypeKey.DOFST_K_WFP_TELEMETERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dofst_k_wfp_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'instrument_data_particle_class': DofstKWfpTelemeteredDataParticle,
'metadata_particle_class': DofstKWfpTelemeteredMetadataParticle
}
})
log.debug("My Config: %s", config)
parser = DofstKWfpParser(
config,
parser_state,
file_handle,
lambda state, ingested: self._save_parser_state(state, data_key, ingested),
self._data_callback,
self._sample_exception_callback,
os.path.getsize(file_handle.name))
else:
raise ConfigurationException\
('Bad Configuration: %s - Failed to build ctdpf_ckl_wfp parser',config)
return parser
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA)
#********************************************
# Test the "recovered" version of the parser
#********************************************
recovered_new_state = {StateKey.POSITION: 11,
StateKey.RECORDS_READ: 1,
StateKey.METADATA_SENT: True}
recovered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = recovered_parser
recovered_result = recovered_parser.get_records(1)
self.assert_result(recovered_result, 0, self.particle_meta, False, 0, True)
# essentially skips particle a
recovered_parser.set_state(recovered_new_state)
recovered_result = recovered_parser.get_records(1)
self.assert_result(recovered_result, 22, self.particle_b, False, 2, True)
recovered_result = recovered_parser.get_records(1)
self.assert_result(recovered_result, 33, self.particle_c, True, 3, True)
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
telemetered_new_state = {StateKey.POSITION: 11,
StateKey.RECORDS_READ: 1,
StateKey.METADATA_SENT: True}
telemetered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = telemetered_parser
telemetered_result = telemetered_parser.get_records(1)
self.assert_result(telemetered_result, 0, self.particle_meta, False, 0, True)
# essentially skips particle a
telemetered_parser.set_state(telemetered_new_state)
telemetered_result = telemetered_parser.get_records(1)
self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True)
telemetered_result = telemetered_parser.get_records(1)
self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA)
#********************************************
# Test the "recovered" version of the parser
#********************************************
recovered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = recovered_parser
# next get records
# NOTE - while we ask for 4 records, the metadata is a special case
# the number of records read is actually 3. See the IDD for details.
recovered_result = recovered_parser.get_records(4)
self.assertEqual(recovered_result, [self.particle_meta, self.particle_a,
self.particle_b, self.particle_c])
self.assertEqual(recovered_parser._state[StateKey.POSITION], 33)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
self.assertEqual(self.publish_callback_value[0], self.particle_meta)
self.assertEqual(self.publish_callback_value[1], self.particle_a)
self.assertEqual(self.publish_callback_value[2], self.particle_b)
self.assertEqual(self.publish_callback_value[3], self.particle_c)
self.assertEqual(self.file_ingested_value, True)
self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 3)
self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
self.assertEqual(recovered_parser._state[StateKey.METADATA_SENT], True)
self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
telemetered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
self.state_callback, self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = telemetered_parser
# next get records
telemetered_result = telemetered_parser.get_records(4)
self.assertEqual(telemetered_result, [self.particle_meta, self.particle_a,
self.particle_b, self.particle_c])
self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
self.assertEqual(self.publish_callback_value[0], self.particle_meta)
self.assertEqual(self.publish_callback_value[1], self.particle_a)
self.assertEqual(self.publish_callback_value[2], self.particle_b)
self.assertEqual(self.publish_callback_value[3], self.particle_c)
self.assertEqual(self.file_ingested_value, True)
self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 3)
self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
self.assertEqual(telemetered_parser._state[StateKey.METADATA_SENT], True)
self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
"""
stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA)
#********************************************
# Test the "recovered" version of the parser
#********************************************
# set the state after the metadata and first record
recovered_new_state = {
StateKey.POSITION: 11,
StateKey.RECORDS_READ: 1,
StateKey.METADATA_SENT: True
}
recovered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED),
recovered_new_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = recovered_parser
recovered_result = recovered_parser.get_records(1)
self.assert_result(recovered_result, 22, self.particle_b, False, 2,
True)
recovered_result = self.parser.get_records(1)
self.assert_result(recovered_result, 33, self.particle_c, True, 3,
True)
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
# set the state after the metadata and first record
telemetered_new_state = {
StateKey.POSITION: 11,
StateKey.RECORDS_READ: 1,
StateKey.METADATA_SENT: True
}
telemetered_parser = DofstKWfpParser(
self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED),
telemetered_new_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback,
len(DofstKWfpParserUnitTestCase.TEST_DATA))
self.parser = telemetered_parser
telemetered_result = telemetered_parser.get_records(1)
self.assert_result(telemetered_result, 22, self.particle_b, False, 2,
True)
telemetered_result = self.parser.get_records(1)
self.assert_result(telemetered_result, 33, self.particle_c, True, 3,
True)
def test_long_stream(self):
"""
Test a long stream
"""
filepath = os.path.join(RESOURCE_PATH, 'C0000038.DAT')
filesize = os.path.getsize(filepath)
stream_handle = open(filepath)
#********************************************
# Test the "recovered" version of the parser
#********************************************
recovered_parser = DofstKWfpParser(self.config_recovered,
self.recovered_start_state,
stream_handle, self.state_callback,
self.pub_callback,
self.exception_callback, filesize)
self.parser = recovered_parser
# next get records
recovered_result = self.parser.get_records(271)
self.assertEqual(recovered_result[0], self.recov_particle_meta_long)
self.assertEqual(recovered_result[1], self.recov_particle_a_long)
self.assertEqual(recovered_result[2], self.recov_particle_b_long)
self.assertEqual(recovered_result[-1], self.recov_particle_last)
self.assertEqual(recovered_parser._state[StateKey.POSITION], 2970)
self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 270)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970)
self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270)
self.assertEqual(self.publish_callback_value[-1],
self.recov_particle_last)
#**********************************************
# Test the "telemetered" version of the parser
#**********************************************
telemetered_parser = DofstKWfpParser(self.config_telemetered,
self.telemetered_start_state,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback, filesize)
self.parser = telemetered_parser
# next get records
telemetered_result = self.parser.get_records(271)
self.assertEqual(telemetered_result[0], self.telem_particle_meta_long)
self.assertEqual(telemetered_result[1], self.telem_particle_a_long)
self.assertEqual(telemetered_result[2], self.telem_particle_b_long)
self.assertEqual(telemetered_result[-1], self.telem_particle_last)
self.assertEqual(telemetered_parser._state[StateKey.POSITION], 2970)
self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 270)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970)
self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270)
self.assertEqual(self.publish_callback_value[-1],
self.telem_particle_last)
def process(self):
log = get_logger()
stream_handle = open(self._sourceFilePath, 'rb')
filesize = os.path.getsize(stream_handle.name)
def state_callback(state, ingested):
pass
def pub_callback(data):
log.trace("Found data: %s", data)
def exp_callback(exception):
self._particleDataHdlrObj.setParticleDataCaptureFailure()
try:
parser = DofstKWfpParser(self._config, None, stream_handle,
state_callback, pub_callback,
exp_callback, filesize)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
finally:
stream_handle.close()
return self._particleDataHdlrObj
def _build_parser(self, stream_handle):
filesize = os.path.getsize(stream_handle.name)
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dofs_k_wfp_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'instrument_data_particle_class':
DofstKWfpRecoveredDataParticle,
'metadata_particle_class': DofstKWfpRecoveredMetadataParticle
}
}
parser = DofstKWfpParser(config, None, stream_handle,
lambda state, ingested: None,
lambda data: None, self._exception_callback,
filesize)
return parser
def _build_parser(self, stream_handle):
filesize = os.path.getsize(stream_handle.name)
config = {
WfpCFileCommonConfigKeys.PRESSURE_FIELD_C_FILE:
DofstKWfpDataParticleKey.PRESSURE,
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dofs_k_wfp_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS:
None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'instrument_data_particle_class':
DofstKWfpTelemeteredDataParticle,
'metadata_particle_class': DofstKWfpTelemeteredMetadataParticle
}
}
parser = DofstKWfpParser(config, None, stream_handle,
lambda state, ingested: None,
lambda data: None, self._exception_callback,
filesize, self._e_file_time_pressure_tuples)
return parser
