Example #1
0
 def test_bad_data(self):
     """
     Ensure that the missing timestamp field causes a sample exception
     """
     with self.assertRaises(SampleException):
         stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt'))
         self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                         self.state_callback, self.pub_callback,
                                         self.exception_callback)
         result = self.parser.get_records(1)
 def test_encoding(self):
     """
     Create an encoding error in the data and make sure an encoding error shows up
     """
     stream_handle = open(
         os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt'))
     self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                     self.state_callback, self.pub_callback,
                                     self.exception_callback)
     result = self.parser.get_records(1)
     errors = result[0].get_encoding_errors()
     log.debug("encoding errors: %s", errors)
     self.assertNotEqual(errors, [])
    def test_simple_particles(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH)
        self.assertEqual(self.exception_callback_value, None)
 def test_bad_data(self):
     """
     Ensure that the missing timestamp field causes a sample exception
     """
     with self.assertRaises(SampleException):
         stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt'))
         self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                         self.state_callback, self.pub_callback,
                                         self.exception_callback)
         self.parser.get_records(1)
    def test_simple(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_result(result, self.particle_a, True)

        # no data left, do not move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)
 def test_encoding(self):
     """
     Create an encoding error in the data and make sure an encoding error shows up
     """
     stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt'))
     self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                     self.state_callback, self.pub_callback,
                                     self.exception_callback)
     result = self.parser.get_records(1)
     errors = result[0].get_encoding_errors()
     log.debug("encoding errors: %s", errors)
     self.assertNotEqual(errors, [])
Example #7
0
    def test_get_many(self):
        """
        Read test data and try to pull out multiple data particles at one time,
        but we should only get 1 .
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(4)
        self.assert_result(result, self.particle_a, True)
        self.assertEqual(len(self.publish_callback_value), 1)

        # no data left, dont move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)
    def test_simple_particles(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH)
        self.assertEqual(self.exception_callback_value, None)
    def test_generate(self):
        """
        Ensure we can generate the particle dictionary and compare it to expected ones
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        res_dict = result[0].generate_dict()
        # assert two lists of generated dictionaries are the same
        for cdict in self.comparison_list:
            for rdict in res_dict['values']:
                if cdict.get('value_id') == rdict.get('value_id'):
                    if cdict.get('value') != rdict.get('value'):
                        log.error("mismatch for key %s, values '%s' '%s'",
                                  cdict.get('value_id'), cdict.get('value'),
                                  rdict.get('value'))
                        self.fail("mismatch for key %s, values '%s', '%s'" %
                                  (cdict.get('value_id'), cdict.get('value'),
                                   rdict.get('value')))
    def process(self):
        log = get_logger()

        def exception_callback(exception):
            log.debug("ERROR: %r", exception)
            self._particle_data_handler.setParticleDataCaptureFailure()

        with open(self._source_file_path, 'rb') as stream_handle:
            parser = CgStcEngStcParser(self._parser_config, None,
                                       stream_handle,
                                       lambda state, ingested: None,
                                       lambda data: None, exception_callback)

            driver = DataSetDriver(parser, self._particle_data_handler)
            driver.processFileStream()

        return self._particle_data_handler
    def test_simple(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_result(result, self.particle_a, True)

        # no data left, do not move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)
    def test_get_many(self):
        """
        Read test data and try to pull out multiple data particles at one time,
        but we should only get 1 .
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(4)
        self.assert_result(result, self.particle_a, True)
        self.assertEqual(len(self.publish_callback_value), 1)

        # no data left, dont move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)
    def test_generate(self):
        """
        Ensure we can generate the particle dictionary and compare it to expected ones
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        res_dict = result[0].generate_dict()
        # assert two lists of generated dictionaries are the same
        for cdict in self.comparison_list:
            for rdict in res_dict['values']:
                if cdict.get('value_id') == rdict.get('value_id'):
                    if cdict.get('value') != rdict.get('value'):
                        log.error("mismatch for key %s, values '%s' '%s'", cdict.get('value_id'),
                                  cdict.get('value'),
                                  rdict.get('value'))
                        self.fail("mismatch for key %s, values '%s', '%s'" % (cdict.get('value_id'),
                                                                              cdict.get('value'),
                                                                              rdict.get('value')))
class CgParserUnitTestCase(ParserUnitTestCase):
    """
    Cg_stc_eng_stc Parser unit test suite
    """
    def state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.state_callback_value = state
        self.file_ingested_value = file_ingested

    def pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.publish_callback_value = pub

    def exception_callback(self, exception):
        """ Callback method to watch what comes in via the exception callback """
        self.exception_callback_value = exception

    def setUp(self):
        ParserUnitTestCase.setUp(self)
        self.config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc',
            DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserDataParticle'
        }
        # Define test data particles and their associated timestamps which will be 
        # compared with returned results
        fid = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        data = fid.read()
        fid.close()
        utime_grp = re.search(r'Platform.utime=(.+?)(\r\n?|\n)', data)
        self.timestamp_a = ntplib.system_to_ntp_time(float(utime_grp.group(1)))
        self.particle_a = CgStcEngStcParserDataParticle(data,
                                                        internal_timestamp=self.timestamp_a,
                                                        preferred_timestamp=DataParticleKey.INTERNAL_TIMESTAMP)

        self.comparison_list = [{DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_TIME,
                                 DataParticleKey.VALUE: '2013/10/04 16:07:02.253'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_UTIME,
                                 DataParticleKey.VALUE: 1380902822.253},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_GPS,
                                 DataParticleKey.VALUE: 83},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_NTP,
                                 DataParticleKey.VALUE: 0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_PPS,
                                 DataParticleKey.VALUE: 4},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_POWER_SYS,
                                 DataParticleKey.VALUE: 0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_SUPERV,
                                 DataParticleKey.VALUE: 7},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_TELEM,
                                 DataParticleKey.VALUE: 0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_GPS,
                                 DataParticleKey.VALUE: 1},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_PPS,
                                 DataParticleKey.VALUE: 1},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_TELEM_SYS,
                                 DataParticleKey.VALUE: 3},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_GPS,
                                 DataParticleKey.VALUE: '***Warning, BAD GPS CHECKSUM'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_PPS,
                                 DataParticleKey.VALUE: 'C_PPS: Warning: Pulse delta [790] above warning level [500], still within window [900]'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_TELEM_SYS,
                                 DataParticleKey.VALUE: ' "***Error turning on fb1 [ret=No Device]'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_UPTIME,
                                 DataParticleKey.VALUE: '0 days 00:01:22'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD1,
                                 DataParticleKey.VALUE: 1.03},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD5,
                                 DataParticleKey.VALUE: 0.36},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD15,
                                 DataParticleKey.VALUE: 0.12},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_RAM,
                                 DataParticleKey.VALUE: 127460},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_FREE,
                                 DataParticleKey.VALUE: 93396},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_NPROC,
                                 DataParticleKey.VALUE: 76},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_EFLAG,
                                 DataParticleKey.VALUE: 0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_V,
                                 DataParticleKey.VALUE: 17.90},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_C,
                                 DataParticleKey.VALUE: 379.20},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_V,
                                 DataParticleKey.VALUE: 0.0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_C,
                                 DataParticleKey.VALUE: 0.0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP1,
                                 DataParticleKey.VALUE: 25.0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP2,
                                 DataParticleKey.VALUE: 23.3},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_HUMID,
                                 DataParticleKey.VALUE: 31.6},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_PRESS,
                                 DataParticleKey.VALUE: 14.7},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GF_ENA,
                                 DataParticleKey.VALUE: 15},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT1,
                                 DataParticleKey.VALUE: 7.7},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT2,
                                 DataParticleKey.VALUE: 5.2},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT3,
                                 DataParticleKey.VALUE: 2.8},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT4,
                                 DataParticleKey.VALUE: 4.0},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_LD_ENA,
                                 DataParticleKey.VALUE: 3},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_DATE,
                                 DataParticleKey.VALUE: 41013},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_TIME,
                                 DataParticleKey.VALUE: 160701},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LATSTR,
                                 DataParticleKey.VALUE: '4132.1353 N'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LONSTR,
                                 DataParticleKey.VALUE: '07038.8306 W'},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LAT,
                                 DataParticleKey.VALUE: 41.535588},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LON,
                                 DataParticleKey.VALUE: -70.647177},
                                {DataParticleKey.VALUE_ID: CgStcEngStcParserDataParticleKey.CG_ENG_GPS_SPD,
                                 DataParticleKey.VALUE: 0.0}]
        # uncomment the following to write the above comparison list in yml format to a file
        # self.write_comparison_to_yml()

        self.file_ingested_value = None
        self.state_callback_value = None
        self.publish_callback_value = None
        self.exception_callback_value = None

    def write_comparison_to_yml(self):
        """
        Helper class to create a yml file for driver tests
        """
        fid = open('particle.yml', 'a')
        fid.write('header:\n')
        fid.write('    particle_object: CgStcEngStcParserDataParticle\n')
        fid.write('    particle_type: cg_stc_eng_stc\n')
        fid.write('data:\n')
        fid.write('  - _index: 1\n')
        fid.write('    internal_timestamp: 0.0\n')

        for item in self.comparison_list:
            if isinstance(item.get('value'), float):
                fid.write('    %s: %16.20f\n' % (item.get('value_id'), item.get('value')))
            else:
                fid.write('    %s: %s\n' % (item.get('value_id'), item.get('value')))
        fid.close()

    def assert_result(self, result, particle, ingested):
        if result[0].raw_data == particle.raw_data:
            log.debug("raw data match")
        log.debug("comparing result %s, particle %s", result[0].contents, particle.contents)
        self.assertEqual(result, [particle])
        self.assertEqual(self.file_ingested_value, ingested)

        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], particle)

    def test_simple(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_result(result, self.particle_a, True)

        # no data left, do not move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)

    def test_simple_particles(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH)
        self.assertEqual(self.exception_callback_value, None)

    def test_get_many(self):
        """
        Read test data and try to pull out multiple data particles at one time,
        but we should only get 1 .
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(4)
        self.assert_result(result, self.particle_a, True)
        self.assertEqual(len(self.publish_callback_value), 1)

        # no data left, do not move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)

    def test_generate(self):
        """
        Ensure we can generate the particle dictionary and compare it to expected ones
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        res_dict = result[0].generate_dict()
        # assert two lists of generated dictionaries are the same
        for cdict in self.comparison_list:
            for rdict in res_dict['values']:
                if cdict.get('value_id') == rdict.get('value_id'):
                    if cdict.get('value') != rdict.get('value'):
                        log.error("mismatch for key %s, values '%s' '%s'", cdict.get('value_id'),
                                  cdict.get('value'),
                                  rdict.get('value'))
                        self.fail("mismatch for key %s, values '%s', '%s'" % (cdict.get('value_id'),
                                                                              cdict.get('value'),
                                                                              rdict.get('value')))

    def test_bad_data(self):
        """
        Ensure that the missing timestamp field causes a sample exception
        """
        with self.assertRaises(SampleException):
            stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt'))
            self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                            self.state_callback, self.pub_callback,
                                            self.exception_callback)
            self.parser.get_records(1)

    def test_encoding(self):
        """
        Create an encoding error in the data and make sure an encoding error shows up
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)
        result = self.parser.get_records(1)
        errors = result[0].get_encoding_errors()
        log.debug("encoding errors: %s", errors)
        self.assertNotEqual(errors, [])
Example #15
0
class CgParserUnitTestCase(ParserUnitTestCase):
    """
    Cg_stc_eng_stc Parser unit test suite
    """
    def state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.state_callback_value = state
        self.file_ingested_value = file_ingested

    def pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.publish_callback_value = pub

    def exception_callback(self, exception):
        """ Callback method to watch what comes in via the exception callback """
        self.exception_callback_value = exception

    def setUp(self):
        ParserUnitTestCase.setUp(self)
        self.config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE:
            'mi.dataset.parser.cg_stc_eng_stc',
            DataSetDriverConfigKeys.PARTICLE_CLASS:
            'CgStcEngStcParserDataParticle'
        }
        # Define test data particles and their associated timestamps which will be
        # compared with returned results
        fid = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        data = fid.read()
        fid.close()
        utime_grp = re.search(r'Platform.utime=(.+?)(\r\n?|\n)', data)
        self.timestamp_a = ntplib.system_to_ntp_time(float(utime_grp.group(1)))
        self.particle_a = CgStcEngStcParserDataParticle(
            data, internal_timestamp=self.timestamp_a)

        self.comparison_list = [{
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_TIME,
            DataParticleKey.VALUE:
            '2013/10/04 16:07:02.253'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_PLATFORM_UTIME,
            DataParticleKey.VALUE:
            1380902822.253
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_GPS,
            DataParticleKey.VALUE:
            83
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_NTP,
            DataParticleKey.VALUE:
            0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_PPS,
            DataParticleKey.VALUE:
            4
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_POWER_SYS,
            DataParticleKey.VALUE:
            0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_SUPERV,
            DataParticleKey.VALUE:
            7
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MSG_CNTS_C_TELEM,
            DataParticleKey.VALUE:
            0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_GPS,
            DataParticleKey.VALUE:
            1
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_PPS,
            DataParticleKey.VALUE:
            1
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_ERR_C_TELEM_SYS,
            DataParticleKey.VALUE:
            3
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_GPS,
            DataParticleKey.VALUE:
            '***Warning, BAD GPS CHECKSUM'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_PPS,
            DataParticleKey.VALUE:
            'C_PPS: Warning: Pulse delta [790] above warning level [500], still within window [900]'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_ERRMSG_C_TELEM_SYS,
            DataParticleKey.VALUE:
            ' "***Error turning on fb1 [ret=No Device]'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_CPU_UPTIME,
            DataParticleKey.VALUE:
            '0 days 00:01:22'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD1,
            DataParticleKey.VALUE:
            1.03
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD5,
            DataParticleKey.VALUE:
            0.36
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_CPU_LOAD15,
            DataParticleKey.VALUE:
            0.12
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_RAM,
            DataParticleKey.VALUE:
            127460
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MEMORY_FREE,
            DataParticleKey.VALUE:
            93396
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_NPROC,
            DataParticleKey.VALUE:
            76
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_EFLAG,
            DataParticleKey.VALUE:
            0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_V,
            DataParticleKey.VALUE:
            17.90
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_MAIN_C,
            DataParticleKey.VALUE:
            379.20
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_V,
            DataParticleKey.VALUE:
            0.0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_BAT_C,
            DataParticleKey.VALUE:
            0.0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP1,
            DataParticleKey.VALUE:
            25.0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_TEMP2,
            DataParticleKey.VALUE:
            23.3
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_HUMID,
            DataParticleKey.VALUE:
            31.6
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_PRESS,
            DataParticleKey.VALUE:
            14.7
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GF_ENA,
            DataParticleKey.VALUE:
            15
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT1,
            DataParticleKey.VALUE:
            7.7
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT2,
            DataParticleKey.VALUE:
            5.2
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT3,
            DataParticleKey.VALUE:
            2.8
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_GFLT4,
            DataParticleKey.VALUE:
            4.0
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_MPIC_LD_ENA,
            DataParticleKey.VALUE:
            3
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_DATE,
            DataParticleKey.VALUE:
            41013
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_TIME,
            DataParticleKey.VALUE:
            160701
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LATSTR,
            DataParticleKey.VALUE:
            '4132.1353 N'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LONSTR,
            DataParticleKey.VALUE:
            '07038.8306 W'
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LAT,
            DataParticleKey.VALUE:
            41.535588
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_LON,
            DataParticleKey.VALUE:
            -70.647177
        }, {
            DataParticleKey.VALUE_ID:
            CgStcEngStcParserDataParticleKey.CG_ENG_GPS_SPD,
            DataParticleKey.VALUE:
            0.0
        }]
        # uncomment the following to write the above comparison list in yml format to a file
        # self.write_comparison_to_yml()

        self.file_ingested_value = None
        self.state_callback_value = None
        self.publish_callback_value = None
        self.exception_callback_value = None

    def write_comparison_to_yml(self):
        """
        Helper class to create a yml file for driver tests
        """
        fid = open('particle.yml', 'a')
        fid.write('header:\n')
        fid.write('    particle_object: CgStcEngStcParserDataParticle\n')
        fid.write('    particle_type: cg_stc_eng_stc\n')
        fid.write('data:\n')
        fid.write('  - _index: 1\n')
        fid.write('    internal_timestamp: 0.0\n')

        for item in self.comparison_list:
            if isinstance(item.get('value'), float):
                fid.write('    %s: %16.20f\n' %
                          (item.get('value_id'), item.get('value')))
            else:
                fid.write('    %s: %s\n' %
                          (item.get('value_id'), item.get('value')))
        fid.close()

    def assert_result(self, result, particle, ingested):
        if result[0].raw_data == particle.raw_data:
            log.debug("raw data match")
        log.debug("comparing result %s, particle %s", result[0].contents,
                  particle.contents)
        self.assertEqual(result, [particle])
        self.assertEqual(self.file_ingested_value, ingested)

        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], particle)

    def test_simple(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_result(result, self.particle_a, True)

        # no data left, do not move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)

    def test_simple_particles(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status2.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        self.assert_particles(result, 'stc_first2.result.yml', RESOURCE_PATH)
        self.assertEqual(self.exception_callback_value, None)

    def test_get_many(self):
        """
        Read test data and try to pull out multiple data particles at one time,
        but we should only get 1 .
        Assert that the results are those we expected.
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(4)
        self.assert_result(result, self.particle_a, True)
        self.assertEqual(len(self.publish_callback_value), 1)

        # no data left, do not move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_a)
        self.assertEqual(self.exception_callback_value, None)

    def test_generate(self):
        """
        Ensure we can generate the particle dictionary and compare it to expected ones
        """
        stream_handle = open(os.path.join(RESOURCE_PATH, 'stc_status.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)

        result = self.parser.get_records(1)
        res_dict = result[0].generate_dict()
        # assert two lists of generated dictionaries are the same
        for cdict in self.comparison_list:
            for rdict in res_dict['values']:
                if cdict.get('value_id') == rdict.get('value_id'):
                    if cdict.get('value') != rdict.get('value'):
                        log.error("mismatch for key %s, values '%s' '%s'",
                                  cdict.get('value_id'), cdict.get('value'),
                                  rdict.get('value'))
                        self.fail("mismatch for key %s, values '%s', '%s'" %
                                  (cdict.get('value_id'), cdict.get('value'),
                                   rdict.get('value')))

    def test_bad_data(self):
        """
        Ensure that the missing timestamp field causes a sample exception
        """
        with self.assertRaises(SampleException):
            stream_handle = open(
                os.path.join(RESOURCE_PATH, 'stc_status_missing_time.txt'))
            self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                            self.state_callback,
                                            self.pub_callback,
                                            self.exception_callback)
            self.parser.get_records(1)

    def test_encoding(self):
        """
        Create an encoding error in the data and make sure an encoding error shows up
        """
        stream_handle = open(
            os.path.join(RESOURCE_PATH, 'stc_status_bad_encode.txt'))
        self.parser = CgStcEngStcParser(self.config, None, stream_handle,
                                        self.state_callback, self.pub_callback,
                                        self.exception_callback)
        result = self.parser.get_records(1)
        errors = result[0].get_encoding_errors()
        log.debug("encoding errors: %s", errors)
        self.assertNotEqual(errors, [])
Example #16
0
    def _build_parser(self, parser_state, stream_in, data_key, file_in):
        """
        Build the parser based on which data_key is input.  The file name is only
        needed for mopak, and it just not passed in to the other parser builders
        @param parser_state previous parser state to initialize parser with
        @param stream_in handle of the opened file to parse
        @param data_key harvester / parser key 
        @param file_in file name
        """

        # get the config for the correct parser instance
        config = self._parser_config.get(data_key)

        if config is None:
            log.warn(
                'Parser config does not exist for key = %s.  Not building parser',
                data_key)
            raise ConfigurationException

        if data_key == DataTypeKey.CG_STC_ENG_TELEM:
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.cg_stc_eng_stc',
                DataSetDriverConfigKeys.PARTICLE_CLASS:
                'CgStcEngStcParserDataParticle'
            })
            parser = CgStcEngStcParser(
                config, parser_state, stream_in,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        elif data_key == DataTypeKey.CG_STC_ENG_RECOV:
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.cg_stc_eng_stc',
                DataSetDriverConfigKeys.PARTICLE_CLASS:
                'CgStcEngStcParserRecoveredDataParticle'
            })
            parser = CgStcEngStcParser(
                config, parser_state, stream_in,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        elif data_key == DataTypeKey.MOPAK_TELEM:

            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.mopak_o_dcl',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                # particle_class configuration does nothing for multi-particle parsers
                # put the class names in specific config parameters so the parser can get them
                # use real classes as objects instead of strings to make it easier
                DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                    MopakParticleClassType.ACCEL_PARTCICLE_CLASS:
                    MopakODclAccelParserDataParticle,
                    MopakParticleClassType.RATE_PARTICLE_CLASS:
                    MopakODclRateParserDataParticle
                }
            })

            parser = MopakODclParser(
                config, parser_state, stream_in, file_in,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        elif data_key == DataTypeKey.MOPAK_RECOV:

            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.mopak_o_dcl',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                # particle_class configuration does nothing for multi-particle parsers
                # put the class names in specific config parameters so the parser can get them
                # use real classes as objects instead of strings to make it easier
                DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                    MopakParticleClassType.ACCEL_PARTCICLE_CLASS:
                    MopakODclAccelParserRecoveredDataParticle,
                    MopakParticleClassType.RATE_PARTICLE_CLASS:
                    MopakODclRateParserRecoveredDataParticle
                }
            })

            parser = MopakODclParser(
                config, parser_state, stream_in, file_in,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        elif data_key == DataTypeKey.RTE_TELEM:

            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.rte_o_dcl',
                DataSetDriverConfigKeys.PARTICLE_CLASS:
                'RteODclParserDataParticle'
            })
            parser = RteODclParser(
                config, parser_state, stream_in,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        elif data_key == DataTypeKey.RTE_RECOV:

            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.rte_o_dcl',
                DataSetDriverConfigKeys.PARTICLE_CLASS:
                'RteODclParserRecoveredDataParticle'
            })
            parser = RteODclParser(
                config, parser_state, stream_in,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        else:
            log.warn('Invalid Data_Key %s.  Not building parser', data_key)
            raise ConfigurationException

        return parser