def test_bad_data(self): """ Ensure that the missing timestamp field causes a sample exception """ with self.assertRaises(SampleException): stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1)
def test_encoding(self): """ Create an encoding error in the data and make sure an encoding error shows up """ stream_handle = open( os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) errors = result[0].get_encoding_errors() log.debug("encoding errors: %s", errors) self.assertNotEqual(errors, [])
def test_simple_particles(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH) self.assertEqual(self.exception_callback_value, None)
def test_bad_data(self): """ Ensure that the missing timestamp field causes a sample exception """ with self.assertRaises(SampleException): stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) self.parser.get_records(1)
def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, self.particle_a, True) # no data left, do not move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None)
def test_encoding(self): """ Create an encoding error in the data and make sure an encoding error shows up """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) errors = result[0].get_encoding_errors() log.debug("encoding errors: %s", errors) self.assertNotEqual(errors, [])
def test_get_many(self): """ Read test data and try to pull out multiple data particles at one time, but we should only get 1 . Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(4) self.assert_result(result, self.particle_a, True) self.assertEqual(len(self.publish_callback_value), 1) # no data left, dont move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None)
def test_simple_particles(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH) self.assertEqual(self.exception_callback_value, None)
def test_generate(self): """ Ensure we can generate the particle dictionary and compare it to expected ones """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) res_dict = result[0].generate_dict() # assert two lists of generated dictionaries are the same for cdict in self.comparison_list: for rdict in res_dict['values']: if cdict.get('value_id') == rdict.get('value_id'): if cdict.get('value') != rdict.get('value'): log.error("mismatch for key %s, values '%s' '%s'", cdict.get('value_id'), cdict.get('value'), rdict.get('value')) self.fail("mismatch for key %s, values '%s', '%s'" % (cdict.get('value_id'), cdict.get('value'), rdict.get('value')))
def process(self): log = get_logger() def exception_callback(exception): log.debug("ERROR: %r", exception) self._particle_data_handler.setParticleDataCaptureFailure() with open(self._source_file_path, 'rb') as stream_handle: parser = CgStcEngStcParser(self._parser_config, None, stream_handle, lambda state, ingested: None, lambda data: None, exception_callback) driver = DataSetDriver(parser, self._particle_data_handler) driver.processFileStream() return self._particle_data_handler
def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, self.particle_a, True) # no data left, do not move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None)
def test_get_many(self): """ Read test data and try to pull out multiple data particles at one time, but we should only get 1 . Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(4) self.assert_result(result, self.particle_a, True) self.assertEqual(len(self.publish_callback_value), 1) # no data left, dont move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None)
def test_generate(self): """ Ensure we can generate the particle dictionary and compare it to expected ones """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) res_dict = result[0].generate_dict() # assert two lists of generated dictionaries are the same for cdict in self.comparison_list: for rdict in res_dict['values']: if cdict.get('value_id') == rdict.get('value_id'): if cdict.get('value') != rdict.get('value'): log.error("mismatch for key %s, values '%s' '%s'", cdict.get('value_id'), cdict.get('value'), rdict.get('value')) self.fail("mismatch for key %s, values '%s', '%s'" % (cdict.get('value_id'), cdict.get('value'), rdict.get('value')))
class CgParserUnitTestCase(ParserUnitTestCase): """ Cg_stc_eng_stc Parser unit test suite """ def state_callback(self, state, file_ingested): """ Call back method to watch what comes in via the position callback """ self.state_callback_value = state self.file_ingested_value = file_ingested def pub_callback(self, pub): """ Call back method to watch what comes in via the publish callback """ self.publish_callback_value = pub def exception_callback(self, exception): """ Callback method to watch what comes in via the exception callback """ self.exception_callback_value = exception def setUp(self): ParserUnitTestCase.setUp(self) self.config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserDataParticle' } # Define test data particles and their associated timestamps which will be # compared with returned results fid = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) data = fid.read() fid.close() utime_grp = re.search(r'Platform.utime=(.+?)(\r\n?|\n)', data) self.timestamp_a = ntplib.system_to_ntp_time(float(utime_grp.group(1))) self.particle_a = CgStcEngStcParserDataParticle(data, internal_timestamp=self.timestamp_a, preferred_timestamp=DataParticleKey.INTERNAL_TIMESTAMP) self.comparison_list = [{DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_TIME, DataParticleKey.VALUE: '2013/10/04 16:07:02.253'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_UTIME, DataParticleKey.VALUE: 1380902822.253}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_GPS, DataParticleKey.VALUE: 83}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_NTP, DataParticleKey.VALUE: 0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_PPS, DataParticleKey.VALUE: 4}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_POWER_SYS, DataParticleKey.VALUE: 0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_SUPERV, DataParticleKey.VALUE: 7}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_TELEM, DataParticleKey.VALUE: 0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_GPS, DataParticleKey.VALUE: 1}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_PPS, DataParticleKey.VALUE: 1}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_TELEM_SYS, DataParticleKey.VALUE: 3}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_GPS, DataParticleKey.VALUE: '***Warning, BAD GPS CHECKSUM'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_PPS, DataParticleKey.VALUE: 'C_PPS: Warning: Pulse delta [790] above warning level [500], still within window [900]'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_TELEM_SYS, DataParticleKey.VALUE: ' "***Error turning on fb1 [ret=No Device]'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_UPTIME, DataParticleKey.VALUE: '0 days 00:01:22'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD1, DataParticleKey.VALUE: 1.03}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD5, DataParticleKey.VALUE: 0.36}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD15, DataParticleKey.VALUE: 0.12}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_RAM, DataParticleKey.VALUE: 127460}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_FREE, DataParticleKey.VALUE: 93396}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_NPROC, DataParticleKey.VALUE: 76}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_EFLAG, DataParticleKey.VALUE: 0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_V, DataParticleKey.VALUE: 17.90}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_C, DataParticleKey.VALUE: 379.20}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_V, DataParticleKey.VALUE: 0.0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_C, DataParticleKey.VALUE: 0.0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP1, DataParticleKey.VALUE: 25.0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP2, DataParticleKey.VALUE: 23.3}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_HUMID, DataParticleKey.VALUE: 31.6}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_PRESS, DataParticleKey.VALUE: 14.7}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GF_ENA, DataParticleKey.VALUE: 15}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT1, DataParticleKey.VALUE: 7.7}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT2, DataParticleKey.VALUE: 5.2}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT3, DataParticleKey.VALUE: 2.8}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT4, DataParticleKey.VALUE: 4.0}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_LD_ENA, DataParticleKey.VALUE: 3}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_DATE, DataParticleKey.VALUE: 41013}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_TIME, DataParticleKey.VALUE: 160701}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LATSTR, DataParticleKey.VALUE: '4132.1353 N'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LONSTR, DataParticleKey.VALUE: '07038.8306 W'}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LAT, DataParticleKey.VALUE: 41.535588}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LON, DataParticleKey.VALUE: -70.647177}, {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_SPD, DataParticleKey.VALUE: 0.0}] # uncomment the following to write the above comparison list in yml format to a file # self.write_comparison_to_yml() self.file_ingested_value = None self.state_callback_value = None self.publish_callback_value = None self.exception_callback_value = None def write_comparison_to_yml(self): """ Helper class to create a yml file for driver tests """ fid = open('particle.yml', 'a') fid.write('header:\n') fid.write(' particle_object: CgStcEngStcParserDataParticle\n') fid.write(' particle_type: cg_stc_eng_stc\n') fid.write('data:\n') fid.write(' - _index: 1\n') fid.write(' internal_timestamp: 0.0\n') for item in self.comparison_list: if isinstance(item.get('value'), float): fid.write(' %s: %16.20f\n' % (item.get('value_id'), item.get('value'))) else: fid.write(' %s: %s\n' % (item.get('value_id'), item.get('value'))) fid.close() def assert_result(self, result, particle, ingested): if result[0].raw_data == particle.raw_data: log.debug("raw data match") log.debug("comparing result %s, particle %s", result[0].contents, particle.contents) self.assertEqual(result, [particle]) self.assertEqual(self.file_ingested_value, ingested) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], particle) def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, self.particle_a, True) # no data left, do not move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None) def test_simple_particles(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH) self.assertEqual(self.exception_callback_value, None) def test_get_many(self): """ Read test data and try to pull out multiple data particles at one time, but we should only get 1 . Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(4) self.assert_result(result, self.particle_a, True) self.assertEqual(len(self.publish_callback_value), 1) # no data left, do not move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None) def test_generate(self): """ Ensure we can generate the particle dictionary and compare it to expected ones """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) res_dict = result[0].generate_dict() # assert two lists of generated dictionaries are the same for cdict in self.comparison_list: for rdict in res_dict['values']: if cdict.get('value_id') == rdict.get('value_id'): if cdict.get('value') != rdict.get('value'): log.error("mismatch for key %s, values '%s' '%s'", cdict.get('value_id'), cdict.get('value'), rdict.get('value')) self.fail("mismatch for key %s, values '%s', '%s'" % (cdict.get('value_id'), cdict.get('value'), rdict.get('value'))) def test_bad_data(self): """ Ensure that the missing timestamp field causes a sample exception """ with self.assertRaises(SampleException): stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) self.parser.get_records(1) def test_encoding(self): """ Create an encoding error in the data and make sure an encoding error shows up """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) errors = result[0].get_encoding_errors() log.debug("encoding errors: %s", errors) self.assertNotEqual(errors, [])
class CgParserUnitTestCase(ParserUnitTestCase): """ Cg_stc_eng_stc Parser unit test suite """ def state_callback(self, state, file_ingested): """ Call back method to watch what comes in via the position callback """ self.state_callback_value = state self.file_ingested_value = file_ingested def pub_callback(self, pub): """ Call back method to watch what comes in via the publish callback """ self.publish_callback_value = pub def exception_callback(self, exception): """ Callback method to watch what comes in via the exception callback """ self.exception_callback_value = exception def setUp(self): ParserUnitTestCase.setUp(self) self.config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserDataParticle' } # Define test data particles and their associated timestamps which will be # compared with returned results fid = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) data = fid.read() fid.close() utime_grp = re.search(r'Platform.utime=(.+?)(\r\n?|\n)', data) self.timestamp_a = ntplib.system_to_ntp_time(float(utime_grp.group(1))) self.particle_a = CgStcEngStcParserDataParticle( data, internal_timestamp=self.timestamp_a) self.comparison_list = [{ DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_TIME, DataParticleKey.VALUE: '2013/10/04 16:07:02.253' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_UTIME, DataParticleKey.VALUE: 1380902822.253 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_GPS, DataParticleKey.VALUE: 83 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_NTP, DataParticleKey.VALUE: 0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_PPS, DataParticleKey.VALUE: 4 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_POWER_SYS, DataParticleKey.VALUE: 0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_SUPERV, DataParticleKey.VALUE: 7 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_TELEM, DataParticleKey.VALUE: 0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_GPS, DataParticleKey.VALUE: 1 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_PPS, DataParticleKey.VALUE: 1 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_TELEM_SYS, DataParticleKey.VALUE: 3 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_GPS, DataParticleKey.VALUE: '***Warning, BAD GPS CHECKSUM' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_PPS, DataParticleKey.VALUE: 'C_PPS: Warning: Pulse delta [790] above warning level [500], still within window [900]' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_TELEM_SYS, DataParticleKey.VALUE: ' "***Error turning on fb1 [ret=No Device]' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_UPTIME, DataParticleKey.VALUE: '0 days 00:01:22' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD1, DataParticleKey.VALUE: 1.03 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD5, DataParticleKey.VALUE: 0.36 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD15, DataParticleKey.VALUE: 0.12 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_RAM, DataParticleKey.VALUE: 127460 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_FREE, DataParticleKey.VALUE: 93396 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_NPROC, DataParticleKey.VALUE: 76 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_EFLAG, DataParticleKey.VALUE: 0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_V, DataParticleKey.VALUE: 17.90 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_C, DataParticleKey.VALUE: 379.20 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_V, DataParticleKey.VALUE: 0.0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_C, DataParticleKey.VALUE: 0.0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP1, DataParticleKey.VALUE: 25.0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP2, DataParticleKey.VALUE: 23.3 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_HUMID, DataParticleKey.VALUE: 31.6 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_PRESS, DataParticleKey.VALUE: 14.7 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GF_ENA, DataParticleKey.VALUE: 15 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT1, DataParticleKey.VALUE: 7.7 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT2, DataParticleKey.VALUE: 5.2 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT3, DataParticleKey.VALUE: 2.8 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT4, DataParticleKey.VALUE: 4.0 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_LD_ENA, DataParticleKey.VALUE: 3 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_DATE, DataParticleKey.VALUE: 41013 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_TIME, DataParticleKey.VALUE: 160701 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LATSTR, DataParticleKey.VALUE: '4132.1353 N' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LONSTR, DataParticleKey.VALUE: '07038.8306 W' }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LAT, DataParticleKey.VALUE: 41.535588 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LON, DataParticleKey.VALUE: -70.647177 }, { DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_SPD, DataParticleKey.VALUE: 0.0 }] # uncomment the following to write the above comparison list in yml format to a file # self.write_comparison_to_yml() self.file_ingested_value = None self.state_callback_value = None self.publish_callback_value = None self.exception_callback_value = None def write_comparison_to_yml(self): """ Helper class to create a yml file for driver tests """ fid = open('particle.yml', 'a') fid.write('header:\n') fid.write(' particle_object: CgStcEngStcParserDataParticle\n') fid.write(' particle_type: cg_stc_eng_stc\n') fid.write('data:\n') fid.write(' - _index: 1\n') fid.write(' internal_timestamp: 0.0\n') for item in self.comparison_list: if isinstance(item.get('value'), float): fid.write(' %s: %16.20f\n' % (item.get('value_id'), item.get('value'))) else: fid.write(' %s: %s\n' % (item.get('value_id'), item.get('value'))) fid.close() def assert_result(self, result, particle, ingested): if result[0].raw_data == particle.raw_data: log.debug("raw data match") log.debug("comparing result %s, particle %s", result[0].contents, particle.contents) self.assertEqual(result, [particle]) self.assertEqual(self.file_ingested_value, ingested) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], particle) def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, self.particle_a, True) # no data left, do not move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None) def test_simple_particles(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH) self.assertEqual(self.exception_callback_value, None) def test_get_many(self): """ Read test data and try to pull out multiple data particles at one time, but we should only get 1 . Assert that the results are those we expected. """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(4) self.assert_result(result, self.particle_a, True) self.assertEqual(len(self.publish_callback_value), 1) # no data left, do not move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.exception_callback_value, None) def test_generate(self): """ Ensure we can generate the particle dictionary and compare it to expected ones """ stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) res_dict = result[0].generate_dict() # assert two lists of generated dictionaries are the same for cdict in self.comparison_list: for rdict in res_dict['values']: if cdict.get('value_id') == rdict.get('value_id'): if cdict.get('value') != rdict.get('value'): log.error("mismatch for key %s, values '%s' '%s'", cdict.get('value_id'), cdict.get('value'), rdict.get('value')) self.fail("mismatch for key %s, values '%s', '%s'" % (cdict.get('value_id'), cdict.get('value'), rdict.get('value'))) def test_bad_data(self): """ Ensure that the missing timestamp field causes a sample exception """ with self.assertRaises(SampleException): stream_handle = open( os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) self.parser.get_records(1) def test_encoding(self): """ Create an encoding error in the data and make sure an encoding error shows up """ stream_handle = open( os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt')) self.parser = CgStcEngStcParser(self.config, None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) errors = result[0].get_encoding_errors() log.debug("encoding errors: %s", errors) self.assertNotEqual(errors, [])
def _build_parser(self, parser_state, stream_in, data_key, file_in): """ Build the parser based on which data_key is input. The file name is only needed for mopak, and it just not passed in to the other parser builders @param parser_state previous parser state to initialize parser with @param stream_in handle of the opened file to parse @param data_key harvester / parser key @param file_in file name """ # get the config for the correct parser instance config = self._parser_config.get(data_key) if config is None: log.warn( 'Parser config does not exist for key = %s. Not building parser', data_key) raise ConfigurationException if data_key == DataTypeKey.CG_STC_ENG_TELEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserDataParticle' }) parser = CgStcEngStcParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.CG_STC_ENG_RECOV: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserRecoveredDataParticle' }) parser = CgStcEngStcParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.MOPAK_TELEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.mopak_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: None, # particle_class configuration does nothing for multi-particle parsers # put the class names in specific config parameters so the parser can get them # use real classes as objects instead of strings to make it easier DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { MopakParticleClassType.ACCEL_PARTCICLE_CLASS: MopakODclAccelParserDataParticle, MopakParticleClassType.RATE_PARTICLE_CLASS: MopakODclRateParserDataParticle } }) parser = MopakODclParser( config, parser_state, stream_in, file_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.MOPAK_RECOV: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.mopak_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: None, # particle_class configuration does nothing for multi-particle parsers # put the class names in specific config parameters so the parser can get them # use real classes as objects instead of strings to make it easier DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { MopakParticleClassType.ACCEL_PARTCICLE_CLASS: MopakODclAccelParserRecoveredDataParticle, MopakParticleClassType.RATE_PARTICLE_CLASS: MopakODclRateParserRecoveredDataParticle } }) parser = MopakODclParser( config, parser_state, stream_in, file_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.RTE_TELEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.rte_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: 'RteODclParserDataParticle' }) parser = RteODclParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.RTE_RECOV: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.rte_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: 'RteODclParserRecoveredDataParticle' }) parser = RteODclParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) else: log.warn('Invalid Data_Key %s. Not building parser', data_key) raise ConfigurationException return parser