def test_bad_eop_data(self): """ If the last "data" record in the file is not 11 byes of 0xFF, raise a sample exception and do not parse the file. """ stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_EOP) #******************************************** # Test the "recovered" version of the parser #******************************************** with self.assertRaises(SampleException): recovered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_EOP)) #********************************************** # Test the "telemetered" version of the parser #********************************************** with self.assertRaises(SampleException): telemetered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
def test_bad_size_data(self): """ If any of the data records in the file are not 11 bytes long, raise a sample exception and do not parse the file. """ stream_handle = StringIO( DofstKWfpParserUnitTestCase.TEST_DATA_BAD_SIZE) #******************************************** # Test the "recovered" version of the parser #******************************************** with self.assertRaises(SampleException): recovered_parser = DofstKWfpParser( self.config_recovered, self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_SIZE)) #********************************************** # Test the "telemetered" version of the parser #********************************************** with self.assertRaises(SampleException): telemetered_parser = DofstKWfpParser( self.config_telemetered, self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_SIZE))
def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA) #******************************************** # Test the "recovered" version of the parser #******************************************** recovered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = recovered_parser # next get records recovered_result = recovered_parser.get_records(1) self.assert_result(recovered_result, 0, self.particle_meta, False, 0, True) recovered_result = self.parser.get_records(1) self.assert_result(recovered_result, 11, self.particle_a, False, 1, True) recovered_result = self.parser.get_records(1) self.assert_result(recovered_result, 22, self.particle_b, False, 2, True) recovered_result = self.parser.get_records(1) self.assert_result(recovered_result, 33, self.particle_c, True, 3, True) # no data left, dont move the position recovered_result = recovered_parser.get_records(1) self.assertEqual(recovered_result, []) self.assertEqual(recovered_parser._state[StateKey.POSITION], 33) self.assertEqual(self.state_callback_value[StateKey.POSITION], 33) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_c) #********************************************** # Test the "telemetered" version of the parser #********************************************** telemetered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = telemetered_parser # next get records telemetered_result = telemetered_parser.get_records(1) self.assert_result(telemetered_result, 0, self.particle_meta, False, 0, True) telemetered_result = self.parser.get_records(1) self.assert_result(telemetered_result, 11, self.particle_a, False, 1, True) telemetered_result = self.parser.get_records(1) self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True) telemetered_result = self.parser.get_records(1) self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True) # no data left, dont move the position telemetered_result = telemetered_parser.get_records(1) self.assertEqual(telemetered_result, []) self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33) self.assertEqual(self.state_callback_value[StateKey.POSITION], 33) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_c)
def test_bad_time_data(self): """ If the timestamps are missing, raise a sample exception and do not parse the file """ stream_handle = StringIO( DofstKWfpParserUnitTestCase.TEST_DATA_BAD_TIME) #******************************************** # Test the "recovered" version of the parser #******************************************** with self.assertRaises(SampleException): recovered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_TIME)) #********************************************** # Test the "telemetered" version of the parser #********************************************** with self.assertRaises(SampleException): telemetered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA_BAD_TIME))
def _build_parser(self, parser_state, file_handle, data_key=None): """ Build and return the parser """ # Default the parser to None parser = None config = self._parser_config.get(data_key) # # If the key is DOFST_K_WFP_RECOVERED, build the dofst_k_wfp parser and # provide a config that includes the specific recovered particle types. # if data_key == DataTypeKey.DOFST_K_WFP_RECOVERED: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dofst_k_wfp_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'instrument_data_particle_class': DofstKWfpRecoveredDataParticle, 'metadata_particle_class': DofstKWfpRecoveredMetadataParticle } }) log.debug("My Config: %s", config) parser = DofstKWfpParser( config, parser_state, file_handle, lambda state, ingested: self._save_parser_state(state, data_key, ingested), self._data_callback, self._sample_exception_callback, os.path.getsize(file_handle.name)) # # If the key is DOFST_K_WFP_TELEMETERED, build the dofst_k_wfp parser and # provide a config that includes the specific telemetered particle types. # elif data_key == DataTypeKey.DOFST_K_WFP_TELEMETERED: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dofst_k_wfp_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'instrument_data_particle_class': DofstKWfpTelemeteredDataParticle, 'metadata_particle_class': DofstKWfpTelemeteredMetadataParticle } }) log.debug("My Config: %s", config) parser = DofstKWfpParser( config, parser_state, file_handle, lambda state, ingested: self._save_parser_state(state, data_key, ingested), self._data_callback, self._sample_exception_callback, os.path.getsize(file_handle.name)) else: raise ConfigurationException\ ('Bad Configuration: %s - Failed to build ctdpf_ckl_wfp parser',config) return parser
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA) #******************************************** # Test the "recovered" version of the parser #******************************************** recovered_new_state = {StateKey.POSITION: 11, StateKey.RECORDS_READ: 1, StateKey.METADATA_SENT: True} recovered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = recovered_parser recovered_result = recovered_parser.get_records(1) self.assert_result(recovered_result, 0, self.particle_meta, False, 0, True) # essentially skips particle a recovered_parser.set_state(recovered_new_state) recovered_result = recovered_parser.get_records(1) self.assert_result(recovered_result, 22, self.particle_b, False, 2, True) recovered_result = recovered_parser.get_records(1) self.assert_result(recovered_result, 33, self.particle_c, True, 3, True) #********************************************** # Test the "telemetered" version of the parser #********************************************** telemetered_new_state = {StateKey.POSITION: 11, StateKey.RECORDS_READ: 1, StateKey.METADATA_SENT: True} telemetered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = telemetered_parser telemetered_result = telemetered_parser.get_records(1) self.assert_result(telemetered_result, 0, self.particle_meta, False, 0, True) # essentially skips particle a telemetered_parser.set_state(telemetered_new_state) telemetered_result = telemetered_parser.get_records(1) self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True) telemetered_result = telemetered_parser.get_records(1) self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)
def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA) #******************************************** # Test the "recovered" version of the parser #******************************************** recovered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = recovered_parser # next get records # NOTE - while we ask for 4 records, the metadata is a special case # the number of records read is actually 3. See the IDD for details. recovered_result = recovered_parser.get_records(4) self.assertEqual(recovered_result, [self.particle_meta, self.particle_a, self.particle_b, self.particle_c]) self.assertEqual(recovered_parser._state[StateKey.POSITION], 33) self.assertEqual(self.state_callback_value[StateKey.POSITION], 33) self.assertEqual(self.publish_callback_value[0], self.particle_meta) self.assertEqual(self.publish_callback_value[1], self.particle_a) self.assertEqual(self.publish_callback_value[2], self.particle_b) self.assertEqual(self.publish_callback_value[3], self.particle_c) self.assertEqual(self.file_ingested_value, True) self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 3) self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3) self.assertEqual(recovered_parser._state[StateKey.METADATA_SENT], True) self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True) #********************************************** # Test the "telemetered" version of the parser #********************************************** telemetered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_TELEMETERED), self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = telemetered_parser # next get records telemetered_result = telemetered_parser.get_records(4) self.assertEqual(telemetered_result, [self.particle_meta, self.particle_a, self.particle_b, self.particle_c]) self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33) self.assertEqual(self.state_callback_value[StateKey.POSITION], 33) self.assertEqual(self.publish_callback_value[0], self.particle_meta) self.assertEqual(self.publish_callback_value[1], self.particle_a) self.assertEqual(self.publish_callback_value[2], self.particle_b) self.assertEqual(self.publish_callback_value[3], self.particle_c) self.assertEqual(self.file_ingested_value, True) self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 3) self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3) self.assertEqual(telemetered_parser._state[StateKey.METADATA_SENT], True) self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)
def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ stream_handle = StringIO(DofstKWfpParserUnitTestCase.TEST_DATA) #******************************************** # Test the "recovered" version of the parser #******************************************** # set the state after the metadata and first record recovered_new_state = { StateKey.POSITION: 11, StateKey.RECORDS_READ: 1, StateKey.METADATA_SENT: True } recovered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), recovered_new_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = recovered_parser recovered_result = recovered_parser.get_records(1) self.assert_result(recovered_result, 22, self.particle_b, False, 2, True) recovered_result = self.parser.get_records(1) self.assert_result(recovered_result, 33, self.particle_c, True, 3, True) #********************************************** # Test the "telemetered" version of the parser #********************************************** # set the state after the metadata and first record telemetered_new_state = { StateKey.POSITION: 11, StateKey.RECORDS_READ: 1, StateKey.METADATA_SENT: True } telemetered_parser = DofstKWfpParser( self.config.get(DataTypeKey.DOFST_K_WFP_RECOVERED), telemetered_new_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, len(DofstKWfpParserUnitTestCase.TEST_DATA)) self.parser = telemetered_parser telemetered_result = telemetered_parser.get_records(1) self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True) telemetered_result = self.parser.get_records(1) self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)
def test_long_stream(self): """ Test a long stream """ filepath = os.path.join(RESOURCE_PATH, 'C0000038.DAT') filesize = os.path.getsize(filepath) stream_handle = open(filepath) #******************************************** # Test the "recovered" version of the parser #******************************************** recovered_parser = DofstKWfpParser(self.config_recovered, self.recovered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, filesize) self.parser = recovered_parser # next get records recovered_result = self.parser.get_records(271) self.assertEqual(recovered_result[0], self.recov_particle_meta_long) self.assertEqual(recovered_result[1], self.recov_particle_a_long) self.assertEqual(recovered_result[2], self.recov_particle_b_long) self.assertEqual(recovered_result[-1], self.recov_particle_last) self.assertEqual(recovered_parser._state[StateKey.POSITION], 2970) self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 270) self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970) self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270) self.assertEqual(self.publish_callback_value[-1], self.recov_particle_last) #********************************************** # Test the "telemetered" version of the parser #********************************************** telemetered_parser = DofstKWfpParser(self.config_telemetered, self.telemetered_start_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback, filesize) self.parser = telemetered_parser # next get records telemetered_result = self.parser.get_records(271) self.assertEqual(telemetered_result[0], self.telem_particle_meta_long) self.assertEqual(telemetered_result[1], self.telem_particle_a_long) self.assertEqual(telemetered_result[2], self.telem_particle_b_long) self.assertEqual(telemetered_result[-1], self.telem_particle_last) self.assertEqual(telemetered_parser._state[StateKey.POSITION], 2970) self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 270) self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970) self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270) self.assertEqual(self.publish_callback_value[-1], self.telem_particle_last)
def process(self): log = get_logger() stream_handle = open(self._sourceFilePath, 'rb') filesize = os.path.getsize(stream_handle.name) def state_callback(state, ingested): pass def pub_callback(data): log.trace("Found data: %s", data) def exp_callback(exception): self._particleDataHdlrObj.setParticleDataCaptureFailure() try: parser = DofstKWfpParser(self._config, None, stream_handle, state_callback, pub_callback, exp_callback, filesize) driver = DataSetDriver(parser, self._particleDataHdlrObj) driver.processFileStream() finally: stream_handle.close() return self._particleDataHdlrObj
def _build_parser(self, stream_handle): filesize = os.path.getsize(stream_handle.name) config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dofs_k_wfp_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'instrument_data_particle_class': DofstKWfpRecoveredDataParticle, 'metadata_particle_class': DofstKWfpRecoveredMetadataParticle } } parser = DofstKWfpParser(config, None, stream_handle, lambda state, ingested: None, lambda data: None, self._exception_callback, filesize) return parser
def _build_parser(self, stream_handle): filesize = os.path.getsize(stream_handle.name) config = { WfpCFileCommonConfigKeys.PRESSURE_FIELD_C_FILE: DofstKWfpDataParticleKey.PRESSURE, DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dofs_k_wfp_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'instrument_data_particle_class': DofstKWfpTelemeteredDataParticle, 'metadata_particle_class': DofstKWfpTelemeteredMetadataParticle } } parser = DofstKWfpParser(config, None, stream_handle, lambda state, ingested: None, lambda data: None, self._exception_callback, filesize, self._e_file_time_pressure_tuples) return parser