def test_bad_eop_data(self):
        """
        If the last "data" record in the file is not 11 byes of 0xFF, raise a sample exception
        and do not parse the file.
        """
        filepath = os.path.join(RESOURCE_PATH, 'bad_eop_data.dat')
        filesize = os.path.getsize(filepath)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        with self.assertRaises(SampleException):
            with open(filepath, 'rb') as stream_handle:
                CtdpfCklWfpParser(
                    self._recov_config, stream_handle,
                    self.exception_callback,
                    filesize)
        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        with self.assertRaises(SampleException):
            with open(filepath, 'rb') as stream_handle:
                CtdpfCklWfpParser(
                    self._recov_config, stream_handle,
                    self.exception_callback,
                    filesize)
 def test_bad_eop_data(self):
     """
     If the last "data" record in the file is not 11 byes of 0xFF, raise a sample exception
     and do not parse the file.
     """
     stream_handle = StringIO(
         CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP)
     #********************************************
     # Test the "recovered" version of the parser
     #********************************************
     with self.assertRaises(SampleException):
         recovered_parser = CtdpfCklWfpParser(
             self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
             self.recovered_start_state, stream_handle, self.state_callback,
             self.pub_callback, self.exception_callback,
             len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
     #**********************************************
     # Test the "telemetered" version of the parser
     #**********************************************
     with self.assertRaises(SampleException):
         telemetered_parser = CtdpfCklWfpParser(
             self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
             self.telemetered_start_state, stream_handle,
             self.state_callback, self.pub_callback,
             self.exception_callback,
             len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
    def test_long_stream(self):
        """
        Test a long stream
        """
        filepath = os.path.join(RESOURCE_PATH, 'C0000038.dat')
        filesize = os.path.getsize(filepath)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        with open(filepath) as stream_handle:
        
            recovered_parser = CtdpfCklWfpParser(
                self._recov_config, stream_handle,
                self.exception_callback, filesize)
            self.parser = recovered_parser

            recovered_result = self.parser.get_records(271)

            self.assert_particles(recovered_result, 'C0000038_recov.yml', RESOURCE_PATH)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        with open(filepath) as stream_handle:

            telemetered_parser = CtdpfCklWfpParser(
                self._telem_config, stream_handle,
                self.exception_callback, filesize)
            self.parser = telemetered_parser

            telemetered_result = self.parser.get_records(271)

            self.assert_particles(telemetered_result, 'C0000038_telem.yml', RESOURCE_PATH)
    def test_bad_time_data(self):
        """
        If the timestamps are missing, raise a sample exception and do not parse the file
        """
        filepath = os.path.join(RESOURCE_PATH, 'bad_time_data.dat')
        filesize = os.path.getsize(filepath)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        with self.assertRaises(SampleException):
            with open(filepath, 'rb') as stream_handle:
                CtdpfCklWfpParser(
                    self._recov_config, stream_handle,
                    self.exception_callback,
                    filesize)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        with self.assertRaises(SampleException):
            with open(filepath, 'rb') as stream_handle:
                CtdpfCklWfpParser(
                    self._telem_config, stream_handle,
                    self.exception_callback,
                    filesize)
 def test_bad_time_data(self):
     """
     If the timestamps are missing, raise a sample exception and do not parse the file
     """
     stream_handle = StringIO(
         CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_TIME)
     #********************************************
     # Test the "recovered" version of the parser
     #********************************************
     with self.assertRaises(SampleException):
         recovered_parser = CtdpfCklWfpParser(
             self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
             self.recovered_start_state, stream_handle, self.state_callback,
             self.pub_callback, self.exception_callback,
             len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_TIME))
     #**********************************************
     # Test the "telemetered" version of the parser
     #**********************************************
     with self.assertRaises(SampleException):
         telemetered_parser = CtdpfCklWfpParser(
             self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED),
             self.telemetered_start_state, stream_handle,
             self.state_callback, self.pub_callback,
             self.exception_callback,
             len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_TIME))
Esempio n. 6
0
    def _build_parser(self, parser_state, file_handle, data_key=None):
        """
        Build and return the parser
        """
        # Default the parser to None
        parser = None

        config = self._parser_config.get(data_key)

        #
        # If the key is CTDPF_CKL_WFP_RECOVERED, build the ctdpf_ckl_wfp parser and
        # provide a config that includes the specific recovered particle types.
        #
        if data_key == DataTypeKey.CTDPF_CKL_WFP_RECOVERED:
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.ctdpf_ckl_wfp_particles',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                    'instrument_data_particle_class':
                    CtdpfCklWfpRecoveredDataParticle,
                    'metadata_particle_class':
                    CtdpfCklWfpRecoveredMetadataParticle
                }
            })
            log.debug("My Config: %s", config)
            parser = CtdpfCklWfpParser(
                config, parser_state, file_handle,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback,
                os.path.getsize(file_handle.name))
        #
        # If the key is CTDPF_CKL_WFP_TELEMETERED, build the ctdpf_ckl_wfp parser and
        # provide a config that includes the specific telemetered particle types.
        #
        elif data_key == DataTypeKey.CTDPF_CKL_WFP_TELEMETERED:
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.ctdpf_ckl_wfp_particles',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                    'instrument_data_particle_class':
                    CtdpfCklWfpTelemeteredDataParticle,
                    'metadata_particle_class':
                    CtdpfCklWfpTelemeteredMetadataParticle
                }
            })
            log.debug("My Config: %s", config)
            parser = CtdpfCklWfpParser(
                config, parser_state, file_handle,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback,
                os.path.getsize(file_handle.name))
        else:
            raise ConfigurationException\
                ('Bad Configuration: %s - Failed to build ctdpf_ckl_wfp parser',config)

        return parser
    def test_get_many(self):
        """
        Read test data and pull out multiple data particles at one time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_parser =  CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        self.parser = recovered_parser
        # next get records
        # NOTE - while we ask for 4 records, the metadata is a special case
        # the number of records read is actually 3. See the IDD for details.
        recovered_result = recovered_parser.get_records(4)
        self.assertEqual(recovered_result, [self.particle_meta, self.particle_a,
                                            self.particle_b, self.particle_c])
        self.assertEqual(recovered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assertEqual(self.publish_callback_value[0], self.particle_meta)
        self.assertEqual(self.publish_callback_value[1], self.particle_a)
        self.assertEqual(self.publish_callback_value[2], self.particle_b)
        self.assertEqual(self.publish_callback_value[3], self.particle_c)
        self.assertEqual(self.file_ingested_value, True)
        self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 3)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
        self.assertEqual(recovered_parser._state[StateKey.METADATA_SENT], True)
        self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_parser =  CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        self.parser = telemetered_parser
        # next get records
        telemetered_result = telemetered_parser.get_records(4)
        self.assertEqual(telemetered_result, [self.particle_meta, self.particle_a,
                                            self.particle_b, self.particle_c])
        self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assertEqual(self.publish_callback_value[0], self.particle_meta)
        self.assertEqual(self.publish_callback_value[1], self.particle_a)
        self.assertEqual(self.publish_callback_value[2], self.particle_b)
        self.assertEqual(self.publish_callback_value[3], self.particle_c)
        self.assertEqual(self.file_ingested_value, True)
        self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 3)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
        self.assertEqual(telemetered_parser._state[StateKey.METADATA_SENT], True)
        self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)
    def test_simple_pad(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD))
        # next get records
        result = self.parser.get_records(1)
        self.assert_result(result, 0, self.particle_meta, False, 0, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 11, self.particle_a, False, 1, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)

        # no data left, dont move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)
    def process(self):

        log = get_logger()

        try:
            file_handle = open(self._sourceFilePath, 'rb')
            filesize = os.path.getsize(file_handle.name)
            state = None
            parser_state = None

            def state_callback(state, ingested):
                pass

            def pub_callback(data):
                log.trace("Found data: %s", data)

            def sample_exception_callback(exception):
                self._particleDataHdlrObj.setParticleDataCaptureFailure()

            parser = CtdpfCklWfpParser(
                self._config, parser_state, file_handle,
                lambda state, ingested: state_callback(state, ingested),
                pub_callback, sample_exception_callback, filesize)

            driver = DataSetDriver(parser, self._particleDataHdlrObj)

            driver.processFileStream()
        finally:
            file_handle.close()

        return self._particleDataHdlrObj
    def test_long_stream(self):
        """
        Test a long stream
        """
        filepath = os.path.join(RESOURCE_PATH, 'C0000038.DAT')
        filesize = os.path.getsize(filepath)
        stream_handle = open(filepath)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
            self.recovered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback, filesize)
        self.parser = recovered_parser
        # next get records
        recovered_result = self.parser.get_records(271)
        self.assertEqual(recovered_result[0], self.particle_meta_long)
        self.assertEqual(recovered_result[1], self.particle_a_long)
        self.assertEqual(recovered_result[2], self.particle_b_long)
        self.assertEqual(recovered_result[-1], self.particle_last)
        self.assertEqual(recovered_parser._state[StateKey.POSITION], 2970)
        self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 270)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270)
        self.assertEqual(self.publish_callback_value[-1], self.particle_last)
        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED),
            self.telemetered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback, filesize)
        self.parser = telemetered_parser
        # next get records
        telemetered_result = self.parser.get_records(271)
        self.assertEqual(telemetered_result[0], self.particle_meta_long)
        self.assertEqual(telemetered_result[1], self.particle_a_long)
        self.assertEqual(telemetered_result[2], self.particle_b_long)
        self.assertEqual(telemetered_result[-1], self.particle_last)
        self.assertEqual(telemetered_parser._state[StateKey.POSITION], 2970)
        self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 270)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270)
        self.assertEqual(self.publish_callback_value[-1], self.particle_last)
 def test_bad_eop_data(self):
     """
     Ensure that missing timestamps causes us to sample exception and not parse the file
     """
     with self.assertRaises(SampleException):
         stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP)
         self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                         self.state_callback, self.pub_callback, self.exception_callback,
                                         len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
    def test_mid_state_start(self):
        """
        Test starting the parser in a state in the middle of processing
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        # set the state after the metadata and first record
        recovered_new_state = {StateKey.POSITION: 11,
                               StateKey.RECORDS_READ: 1,
                               StateKey.METADATA_SENT: True}

        recovered_parser =  CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), recovered_new_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = recovered_parser

        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 22, self.particle_b, False, 2, True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 33, self.particle_c, True, 3, True)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        # set the state after the metadata and first record
        telemetered_new_state = {StateKey.POSITION: 11,
                                 StateKey.RECORDS_READ: 1,
                                 StateKey.METADATA_SENT: True}
        telemetered_parser =  CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), telemetered_new_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = telemetered_parser

        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)
    def test_simple(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        self.parser = recovered_parser

        # next get records
        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 0, self.particle_meta, False, 0, True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 11, self.particle_a, False, 1, True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 22, self.particle_b, False, 2, True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 33, self.particle_c, True, 3, True)

        # no data left, dont move the position
        recovered_result = recovered_parser.get_records(1)
        self.assertEqual(recovered_result, [])
        self.assertEqual(recovered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        self.parser = telemetered_parser

        # next get records
        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 0, self.particle_meta, False, 0, True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 11, self.particle_a, False, 1, True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)

        # no data left, dont move the position
        telemetered_result = telemetered_parser.get_records(1)
        self.assertEqual(telemetered_result, [])
        self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)
    def test_simple_pad(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        filepath = os.path.join(RESOURCE_PATH, 'simple_pad.dat')
        filesize = os.path.getsize(filepath)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        with open(filepath, 'rb') as stream_handle:

            recovered_parser = CtdpfCklWfpParser(
                self._recov_config, stream_handle,
                self.exception_callback,
                filesize)

            particles = recovered_parser.get_records(5)

            self.assert_particles(particles, 'test_simple_pad_recov.yml', RESOURCE_PATH)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        with open(filepath, 'rb') as stream_handle:

            telemetered_parser = CtdpfCklWfpParser(
                self._telem_config, stream_handle,
                self.exception_callback,
                filesize)

            particles = telemetered_parser.get_records(1)

            self.assert_particles(particles, 'test_simple_pad_telem.yml', RESOURCE_PATH)
    def test_build_yml_file(self):
        """
        Read test data. Should detect that there is a decimation factor in the data.
        Check that the data matches the expected results.
        """
        log.debug('CAG TEST: START BUILDING YML FILES')

        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp_sio_mule/resource/BIG_C0000038.dat', 'rb')
        filesize = os.path.getsize(stream_handle.name)
        self.recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback, filesize)
        result = self.recovered_parser.get_records(50000)
        log.debug('CAG Number of Results %d', len(result))
        self.particle_to_yml(result, 'BIG_C0000038.yml')
        stream_handle.close()

#        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp/resource/first.DAT', 'rb')
#        filesize = os.path.getsize(stream_handle.name)
#        self.recovered_parser = CtdpfCklWfpParser(
#            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
#            self.state_callback, self.pub_callback, self.exception_callback, filesize)
#        result = self.recovered_parser.get_records(4)
#        log.debug('CAG Number of Results %d', len(result))
#        self.particle_to_yml(result, 'first.result.yml')
#        stream_handle.close()

#        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp/resource/C0000038.DAT', 'rb')
#        filesize = os.path.getsize(stream_handle.name)
#        self.telemetered_parser = CtdpfCklWfpParser(
#            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
#            self.state_callback, self.pub_callback, self.exception_callback, filesize)
#        result = self.telemetered_parser.get_records(272)
#        log.debug('CAG Number of Results %d', len(result))
#        self.particle_to_yml(result, 'C0000038.yml')
#        stream_handle.close()

#        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp/resource/second.DAT', 'rb')
#        filesize = os.path.getsize(stream_handle.name)
#        self.telemetered_parser = CtdpfCklWfpParser(
#            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
#            self.state_callback, self.pub_callback, self.exception_callback, filesize)
#        result = self.telemetered_parser.get_records(7)
#        log.debug('CAG Number of Results %d', len(result))
#        self.particle_to_yml(result, 'second.result.yml')
#        stream_handle.close()

        log.debug('CAG TEST: FINISHED BUILDING YML FILES')
    def test_mid_state_start(self):
        """
        Test starting the parser in a state in the middle of processing
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        # set the state after the metadata and first record
        recovered_new_state = {
            StateKey.POSITION: 11,
            StateKey.RECORDS_READ: 1,
            StateKey.METADATA_SENT: True
        }

        recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
            recovered_new_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = recovered_parser

        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 22, self.particle_b, False, 2,
                           True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 33, self.particle_c, True, 3,
                           True)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        # set the state after the metadata and first record
        telemetered_new_state = {
            StateKey.POSITION: 11,
            StateKey.RECORDS_READ: 1,
            StateKey.METADATA_SENT: True
        }
        telemetered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
            telemetered_new_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = telemetered_parser

        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 22, self.particle_b, False, 2,
                           True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 33, self.particle_c, True, 3,
                           True)
    def _build_parser(self, stream_handle):

        parser_config = {
            DataSetDriverConfigKeys.PARTICLE_CLASS: None,
            DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                METADATA_PARTICLE_CLASS_KEY:
                CtdpfCklWfpRecoveredMetadataParticle,
                DATA_PARTICLE_CLASS_KEY: CtdpfCklWfpRecoveredDataParticle
            }
        }

        file_size = os.path.getsize(stream_handle.name)

        parser = CtdpfCklWfpParser(parser_config, stream_handle,
                                   self._exception_callback, file_size)

        return parser
    def test_mid_state_start(self):
        """
        Test starting the parser in a state in the middle of processing
        """
        # set the state after the metadata and first record
        new_state = {StateKey.POSITION: 11,
                     StateKey.RECORDS_READ: 1,
                     StateKey.METADATA_SENT: True}
        self.stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        self.parser =  CtdpfCklWfpParser(self.config, new_state, self.stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)
 def test_long_stream(self):
     """
     Test a long stream
     """
     filepath = os.path.join(RESOURCE_PATH, 'C0000038.dat')
     filesize = os.path.getsize(filepath)
     stream_handle = open(filepath)
     self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                     self.state_callback, self.pub_callback, self.exception_callback,
                                     filesize)
     result = self.parser.get_records(271)
     self.assertEqual(result[0], self.particle_meta_long)
     self.assertEqual(result[1], self.particle_a_long)
     self.assertEqual(result[2], self.particle_b_long)
     self.assertEqual(result[-1], self.particle_last)
     self.assertEqual(self.parser._state[StateKey.POSITION], 2970)
     self.assertEqual(self.parser._state[StateKey.RECORDS_READ], 270)
     self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970)
     self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270)
     self.assertEqual(self.publish_callback_value[-1], self.particle_last)
    def _build_parser(self, stream_handle):

        parser_config = {
            WfpCFileCommonConfigKeys.PRESSURE_FIELD_C_FILE: CtdpfCklWfpDataParticleKey.PRESSURE,
            DataSetDriverConfigKeys.PARTICLE_CLASS: None,
            DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                METADATA_PARTICLE_CLASS_KEY: CtdpfCklWfpTelemeteredMetadataParticle,
                DATA_PARTICLE_CLASS_KEY: CtdpfCklWfpTelemeteredDataParticle
            }
        }

        file_size = os.path.getsize(stream_handle.name)

        parser = CtdpfCklWfpParser(parser_config,
                                   stream_handle,
                                   self._exception_callback,
                                   file_size,
                                   self._e_file_time_pressure_tuples)

        return parser
    def test_set_state(self):
        """
        Test changing to a new state after initializing the parser and
        reading data, as if new data has been found and the state has
        changed
        """
        new_state = {StateKey.POSITION: 11,
                     StateKey.RECORDS_READ: 1,
                     StateKey.METADATA_SENT: True}
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        result = self.parser.get_records(1)
        self.assert_result(result, 0, self.particle_meta, False, 0, True)

        # essentially skips particle a
        self.parser.set_state(new_state)
        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)
 def test_get_many(self):
     """
     Read test data and pull out multiple data particles at one time.
     Assert that the results are those we expected.
     """
     self.stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
     self.parser =  CtdpfCklWfpParser(self.config, self.start_state, self.stream_handle,
                                     self.state_callback, self.pub_callback, self.exception_callback,
                                     len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
     # next get records
     result = self.parser.get_records(4)
     self.assertEqual(result, [self.particle_meta, self.particle_a, self.particle_b, self.particle_c])
     self.assertEqual(self.parser._state[StateKey.POSITION], 33)
     self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
     self.assertEqual(self.publish_callback_value[0], self.particle_meta)
     self.assertEqual(self.publish_callback_value[1], self.particle_a)
     self.assertEqual(self.publish_callback_value[2], self.particle_b)
     self.assertEqual(self.publish_callback_value[3], self.particle_c)
     self.assertEqual(self.file_ingested_value, True)
     self.assertEqual(self.parser._state[StateKey.RECORDS_READ], 3)
     self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
     self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True)
     self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)
Esempio n. 23
0
    def _build_parser(self, parser_state, infile, data_key=None):
        """
        Build and return the parser
        """
        # Default the parser to None
        parser = None

        config = self._parser_config.get(data_key)

        #
        # If the key is CTDPF_CKL_WFP, build the ctdpf_ckl_wfp parser and
        # provide a config that includes the specific recovered particle types.
        #
        if data_key == DataTypeKey.CTDPF_CKL_WFP:
            log.debug('CAG DRIVER - build parser for %s. State is %s',
                      data_key, parser_state)
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.ctdpf_ckl_wfp_particles',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                    INSTRUMENT_DATA_PARTICLE_CLASS:
                    CtdpfCklWfpRecoveredDataParticle,
                    METADATA_PARTICLE_CLASS:
                    CtdpfCklWfpRecoveredMetadataParticle
                }
            })

            parser = CtdpfCklWfpParser(
                config, parser_state, infile,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback, os.path.getsize(infile.name))
        #
        # If the key is CTDPF_CKL_WFP_SIO_MULE, build the ctdpf_ckl_wfp_sio_mule parser and
        # provide a config that includes the specific telemetered particle types.
        #
        elif data_key == DataTypeKey.CTDPF_CKL_WFP_SIO_MULE:
            log.debug('CAG DRIVER - build parser for %s. State is %s',
                      data_key, parser_state)
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.ctdpf_ckl_wfp_sio_mule',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
                    INSTRUMENT_DATA_PARTICLE_CLASS:
                    CtdpfCklWfpSioMuleDataParticle,
                    METADATA_PARTICLE_CLASS: CtdpfCklWfpSioMuleMetadataParticle
                }
            })

            parser = CtdpfCklWfpSioMuleParser(
                config, parser_state, infile,
                lambda state: self._save_parser_state(
                    state, DataTypeKey.CTDPF_CKL_WFP_SIO_MULE),
                self._data_callback, self._sample_exception_callback)
        else:
            raise ConfigurationException(
                'Bad Configuration: %s - Failed to build ctdpf_ckl_wfp parser',
                config)

        return parser
class CtdpfCklWfpParserUnitTestCase(ParserUnitTestCase):
    """
    ctdpf_ckl_wfp_sio_mule Parser unit test suite
    """
    recovered_start_state = {StateKey.POSITION: 0,
                                  StateKey.RECORDS_READ: 0,
                                  StateKey.METADATA_SENT: False}

    telemetered_start_state = {StateKey.POSITION: 0,
                                    StateKey.RECORDS_READ: 0,
                                    StateKey.METADATA_SENT: False}

    def state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.file_ingested_value = file_ingested
        state = None

    def pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.publish_callback_value = pub

    def exception_callback(self, exception):
        """ Callback method to watch what comes in via the exception callback """
        self.exception_callback_value = exception

    def setUp(self):

        ParserUnitTestCase.setUp(self)

        self.config = {
            DataTypeKey.CTDPF_CKL_WFP_RECOVERED: {
                DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.ctdpf_ckl_wfp',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                'particle_classes_dict': {
                    'instrument_data_particle_class': CtdpfCklWfpRecoveredDataParticle,
                    'metadata_particle_class': CtdpfCklWfpRecoveredMetadataParticle
                },
            },
            DataTypeKey.CTDPF_CKL_WFP_TELEMETERED: {
                DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.ctdpf_ckl_wfp',
                DataSetDriverConfigKeys.PARTICLE_CLASS: None,
                'particle_classes_dict': {
                    'instrument_data_particle_class': CtdpfCklWfpTelemeteredDataParticle,
                    'metadata_particle_class': CtdpfCklWfpTelemeteredMetadataParticle
                }
            }
        }

        self.file_ingested_value = None
        self.state_callback_value = None
        self.publish_callback_value = None

    def calc_timestamp(self, start, increment, sample_idx):
        new_time = start + (increment * sample_idx)
        return float(ntplib.system_to_ntp_time(new_time))

    def assert_result(self, result, particle, ingested):
        self.assertEqual(result, [particle])
        self.assertEqual(self.file_ingested_value, ingested)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], particle)

    def particle_to_yml(self, particles, filename, mode='w'):
        """
        This is added as a testing helper, not actually as part of the parser tests. Since the same particles
        will be used for the driver test it is helpful to write them to .yml in the same form they need in the
        results.yml fids here.
        """
        # open write append, if you want to start from scratch manually delete this fid
        fid = open(os.path.join(RESOURCE_PATH, filename), mode)

        fid.write('header:\n')
        fid.write("    particle_object: 'MULTIPLE'\n")
        fid.write("    particle_type: 'MULTIPLE'\n")
        fid.write('data:\n')

        for i in range(0, len(particles)):
            particle_dict = particles[i].generate_dict()

            fid.write('  - _index: %d\n' %(i+1))

            fid.write('    particle_object: %s\n' % particles[i].__class__.__name__)
            fid.write('    particle_type: %s\n' % particle_dict.get('stream_name'))
            fid.write('    internal_timestamp: %f\n' % particle_dict.get('internal_timestamp'))

            for val in particle_dict.get('values'):
                if isinstance(val.get('value'), float):
                    fid.write('    %s: %16.16f\n' % (val.get('value_id'), val.get('value')))
                else:
                    fid.write('    %s: %s\n' % (val.get('value_id'), val.get('value')))
        fid.close()

    def test_build_yml_file(self):
        """
        Read test data. Should detect that there is a decimation factor in the data.
        Check that the data matches the expected results.
        """
        log.debug('CAG TEST: START BUILDING YML FILES')

        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp_sio_mule/resource/BIG_C0000038.dat', 'rb')
        filesize = os.path.getsize(stream_handle.name)
        self.recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback, filesize)
        result = self.recovered_parser.get_records(50000)
        log.debug('CAG Number of Results %d', len(result))
        self.particle_to_yml(result, 'BIG_C0000038.yml')
        stream_handle.close()

#        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp/resource/first.DAT', 'rb')
#        filesize = os.path.getsize(stream_handle.name)
#        self.recovered_parser = CtdpfCklWfpParser(
#            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
#            self.state_callback, self.pub_callback, self.exception_callback, filesize)
#        result = self.recovered_parser.get_records(4)
#        log.debug('CAG Number of Results %d', len(result))
#        self.particle_to_yml(result, 'first.result.yml')
#        stream_handle.close()

#        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp/resource/C0000038.DAT', 'rb')
#        filesize = os.path.getsize(stream_handle.name)
#        self.telemetered_parser = CtdpfCklWfpParser(
#            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
#            self.state_callback, self.pub_callback, self.exception_callback, filesize)
#        result = self.telemetered_parser.get_records(272)
#        log.debug('CAG Number of Results %d', len(result))
#        self.particle_to_yml(result, 'C0000038.yml')
#        stream_handle.close()

#        stream_handle = open('/home/cgoodrich/Workspace/code/marine-integrations/mi/dataset/driver/ctdpf_ckl/wfp/resource/second.DAT', 'rb')
#        filesize = os.path.getsize(stream_handle.name)
#        self.telemetered_parser = CtdpfCklWfpParser(
#            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
#            self.state_callback, self.pub_callback, self.exception_callback, filesize)
#        result = self.telemetered_parser.get_records(7)
#        log.debug('CAG Number of Results %d', len(result))
#        self.particle_to_yml(result, 'second.result.yml')
#        stream_handle.close()

        log.debug('CAG TEST: FINISHED BUILDING YML FILES')
class CtdpfCklWfpParserUnitTestCase(ParserUnitTestCase):
    TEST_DATA = b"\x00\x1a\x88\x03\xe3\x3b\x00\x03\xeb\x0a\xc8\x00\x1a\x8c\x03\xe2" + \
    "\xc0\x00\x03\xeb\x0a\x81\x00\x1a\x90\x03\xe1\x5b\x00\x03\xeb\x0a\x65\xff\xff" + \
    "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x52\x4e\x75\x82\x52\x4e\x76\x9a"

    TEST_DATA_PAD = b"\x00\x1a\x88\x03\xe3\x3b\x00\x03\xeb\x0a\xc8\x00\x1a\x8c\x03\xe2" + \
    "\xc0\x00\x03\xeb\x0a\x81\x00\x1a\x90\x03\xe1\x5b\x00\x03\xeb\x0a\x65\xff\xff" + \
    "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x52\x4e\x75\x82\x52\x4e\x76\x9a\x0a"

    # not enough bytes for final timestamps
    TEST_DATA_BAD_TIME = b"\x00\x1a\x88\x03\xe3\x3b\x00\x03\xeb\x0a\xc8\x00\x1a\x8c\x03\xe2" + \
    "\xc0\x00\x03\xeb\x0a\x81\x00\x1a\x90\x03\xe1\x5b\x00\x03\xeb\x0a\x65\xff\xff" + \
    "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x52\x4e\x75\x82\x52\x4e"

    TEST_DATA_BAD_SIZE = b"\x00\x1a\x88\x03\xe3\x3b\xc8\x00\x1a\x8c\x03\xe2" + \
    "\xc0\x00\x03\xeb\x0a\x81\x00\x1a\x90\x03\xe1\x5b\x00\x03\xeb\x0a\x65\xff\xff" + \
    "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x52\x4e\x75\x82\x52\x4e\x76\x9a"

    TEST_DATA_BAD_EOP = b"\x00\x1a\x88\x03\xe3\x3b\x00\x03\xeb\x0a\xc8\x00\x1a\x8c\x03\xe2" + \
    "\xc0\x00\x03\xeb\x0a\x81\x00\x1a\x90\x03\xe1\x5b\x00\x03\xeb\x0a\x65" + \
    "\xff\xff\xff\xff\x52\x4e\x75\x82\x52\x4e\x76\x9a"

    """
    ctdpf_ckl_wfp Parser unit test suite
    """
    def state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.state_callback_value = state
        self.file_ingested_value = file_ingested

    def pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.publish_callback_value = pub

    def exception_callback(self, exception):
        """ Callback method to watch what comes in via the exception callback """
        self.exception_callback_value = exception

    def setUp(self):
        ParserUnitTestCase.setUp(self)
        self.config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.ctdpf_ckl_wfp',
            DataSetDriverConfigKeys.PARTICLE_CLASS: ['CtdpfCklWfpParserDataParticle',
                                                     'CtdpfCklWfpMetadataParserDataParticle']
            }
        self.start_state = {StateKey.POSITION: 0,
                            StateKey.TIMESTAMP: 0.0,
                            StateKey.RECORDS_READ: 0,
                            StateKey.METADATA_SENT: False}
        # Define test data particles and their associated timestamps which will be
        # compared with returned results
        timefields = struct.unpack('>II', '\x52\x4e\x75\x82\x52\x4e\x76\x9a')
        start_time = int(timefields[0])
        end_time = int(timefields[1])
        # even though there are only 3 samples in TEST_DATA, there are 270 samples in the original file,
        # so this needs to be used to determine the time increment for each time sample
        time_increment_3 = float(end_time - start_time) / 3.0
        time_increment_270 = float(end_time - start_time) / 270.0

        self.start_timestamp = self.calc_timestamp(start_time, time_increment_3, 0)
        self.particle_meta = CtdpfCklWfpMetadataParserDataParticle((b"\x52\x4e\x75\x82\x52\x4e\x76\x9a", 3.0),
            internal_timestamp=self.start_timestamp)
        self.start_timestamp_long = self.calc_timestamp(start_time, time_increment_270, 0)
        self.particle_meta_long = CtdpfCklWfpMetadataParserDataParticle((b"\x52\x4e\x75\x82\x52\x4e\x76\x9a", 270.0),
            internal_timestamp=self.start_timestamp_long)

        self.particle_a = CtdpfCklWfpParserDataParticle(b"\x00\x1a\x88\x03\xe3\x3b\x00\x03\xeb\x0a\xc8",
                                                          internal_timestamp=self.start_timestamp)
        self.particle_a_long = CtdpfCklWfpParserDataParticle(b"\x00\x1a\x88\x03\xe3\x3b\x00\x03\xeb\x0a\xc8",
                                                               internal_timestamp=self.start_timestamp_long)
        self.timestamp_2 = self.calc_timestamp(start_time, time_increment_3, 1)
        self.particle_b = CtdpfCklWfpParserDataParticle(b"\x00\x1a\x8c\x03\xe2\xc0\x00\x03\xeb\x0a\x81",
                                                          internal_timestamp=self.timestamp_2)
        self.timestamp_2_long = self.calc_timestamp(start_time, time_increment_270, 1)
        self.particle_b_long = CtdpfCklWfpParserDataParticle(b"\x00\x1a\x8c\x03\xe2\xc0\x00\x03\xeb\x0a\x81",
                                                          internal_timestamp=self.timestamp_2_long)
        timestamp_3 = self.calc_timestamp(start_time, time_increment_3, 2)
        self.particle_c = CtdpfCklWfpParserDataParticle(b"\x00\x1a\x90\x03\xe1\x5b\x00\x03\xeb\x0a\x65",
                                                          internal_timestamp=timestamp_3)
        timestamp_last = self.calc_timestamp(start_time, time_increment_270, 269)
        self.particle_last = CtdpfCklWfpParserDataParticle(b"\x00\x1a\x8f\x03\xe5\x91\x00\x03\xeb\x0bS",
                                                            internal_timestamp=timestamp_last)

	# uncomment to generate yml
	#self.particle_to_yml(self.particle_meta)
	#self.particle_to_yml(self.particle_a)
	#self.particle_to_yml(self.particle_b)
	#self.particle_to_yml(self.particle_c)

        self.file_ingested_value = None
        self.state_callback_value = None
        self.publish_callback_value = None

    def calc_timestamp(self, start, increment, sample_idx):
        new_time = start + (increment*sample_idx)
        return float(ntplib.system_to_ntp_time(new_time))

    def assert_result(self, result, position, particle, ingested, rec_read, metadata_sent):
        self.assertEqual(result, [particle])
        self.assertEqual(self.file_ingested_value, ingested)

        self.assertEqual(self.parser._state[StateKey.POSITION], position)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], position)

        self.assertEqual(self.parser._state[StateKey.METADATA_SENT], metadata_sent)
        self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], metadata_sent)

        self.assertEqual(self.parser._state[StateKey.RECORDS_READ], rec_read)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], rec_read)

        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], particle)

    def particle_to_yml(self, particle):
        """
        This is added as a testing helper, not actually as part of the parser tests. Since the same particles
        will be used for the driver test it is helpful to write them to .yml in the same form they need in the
        results.yml files here.
        """
        particle_dict = particle.generate_dict()
        # open write append, if you want to start from scratch manually delete this file
        fid = open('particle.yml', 'a')
        fid.write('  - _index: 1\n')
        fid.write('    internal_timestamp: %f\n' % particle_dict.get('internal_timestamp'))
        fid.write('    particle_object: %s\n' % particle.__class__.__name__)
        fid.write('    particle_type: %s\n' % particle_dict.get('stream_name'))
        for val in particle_dict.get('values'):
            if isinstance(val.get('value'), float):
                fid.write('    %s: %16.20f\n' % (val.get('value_id'), val.get('value')))
            else:
                fid.write('    %s: %s\n' % (val.get('value_id'), val.get('value')))
        fid.close()

    def test_simple(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        # next get records
        result = self.parser.get_records(1)
        self.assert_result(result, 0, self.particle_meta, False, 0, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 11, self.particle_a, False, 1, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)

        # no data left, dont move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)

    def test_simple_pad(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD))
        # next get records
        result = self.parser.get_records(1)
        self.assert_result(result, 0, self.particle_meta, False, 0, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 11, self.particle_a, False, 1, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)

        # no data left, dont move the position
        result = self.parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)

    def test_get_many(self):
        """
        Read test data and pull out multiple data particles at one time.
        Assert that the results are those we expected.
        """
        self.stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, self.stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        # next get records
        result = self.parser.get_records(4)
        self.assertEqual(result, [self.particle_meta, self.particle_a, self.particle_b, self.particle_c])
        self.assertEqual(self.parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assertEqual(self.publish_callback_value[0], self.particle_meta)
        self.assertEqual(self.publish_callback_value[1], self.particle_a)
        self.assertEqual(self.publish_callback_value[2], self.particle_b)
        self.assertEqual(self.publish_callback_value[3], self.particle_c)
        self.assertEqual(self.file_ingested_value, True)
        self.assertEqual(self.parser._state[StateKey.RECORDS_READ], 3)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
        self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True)
        self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT], True)

    def test_long_stream(self):
        """
        Test a long stream
        """
        filepath = os.path.join(RESOURCE_PATH, 'C0000038.dat')
        filesize = os.path.getsize(filepath)
        stream_handle = open(filepath)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        filesize)
        result = self.parser.get_records(271)
        self.assertEqual(result[0], self.particle_meta_long)
        self.assertEqual(result[1], self.particle_a_long)
        self.assertEqual(result[2], self.particle_b_long)
        self.assertEqual(result[-1], self.particle_last)
        self.assertEqual(self.parser._state[StateKey.POSITION], 2970)
        self.assertEqual(self.parser._state[StateKey.RECORDS_READ], 270)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 2970)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 270)
        self.assertEqual(self.publish_callback_value[-1], self.particle_last)

    def test_mid_state_start(self):
        """
        Test starting the parser in a state in the middle of processing
        """
        # set the state after the metadata and first record
        new_state = {StateKey.POSITION: 11,
                     StateKey.TIMESTAMP: self.start_timestamp,
                     StateKey.RECORDS_READ: 1,
                     StateKey.METADATA_SENT: True}
        self.stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        self.parser =  CtdpfCklWfpParser(self.config, new_state, self.stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)

    def test_set_state(self):
        """
        Test changing to a new state after initializing the parser and
        reading data, as if new data has been found and the state has
        changed
        """
        new_state = {StateKey.POSITION: 11,
                     StateKey.TIMESTAMP: self.start_timestamp,
                     StateKey.RECORDS_READ: 1,
                     StateKey.METADATA_SENT: True}
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                        self.state_callback, self.pub_callback, self.exception_callback,
                                        len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        result = self.parser.get_records(1)
        self.assert_result(result, 0, self.particle_meta, False, 0, True)

        # essentially skips particle a
        self.parser.set_state(new_state)
        result = self.parser.get_records(1)
        self.assert_result(result, 22, self.particle_b, False, 2, True)
        result = self.parser.get_records(1)
        self.assert_result(result, 33, self.particle_c, True, 3, True)

    def test_bad_time_data(self):
        """
        Ensure that missing timestamps causes us to sample exception and not parse the file
        """
        with self.assertRaises(SampleException):
            stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_TIME)
            self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                            self.state_callback, self.pub_callback, self.exception_callback,
                                            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_TIME))

    def test_bad_size_data(self):
        """
        Ensure that missing timestamps causes us to sample exception and not parse the file
        """
        with self.assertRaises(SampleException):
            stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_SIZE)
            self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                            self.state_callback, self.pub_callback, self.exception_callback,
                                            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_SIZE))

    def test_bad_eop_data(self):
        """
        Ensure that missing timestamps causes us to sample exception and not parse the file
        """
        with self.assertRaises(SampleException):
            stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP)
            self.parser =  CtdpfCklWfpParser(self.config, self.start_state, stream_handle,
                                            self.state_callback, self.pub_callback, self.exception_callback,
                                            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_BAD_EOP))
    def test_set_state(self):
        """
        Test changing to a new state after initializing the parser and
        reading data, as if new data has been found and the state has
        changed
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_new_state = {StateKey.POSITION: 11,
                               StateKey.RECORDS_READ: 1,
                               StateKey.METADATA_SENT: True}
        
        recovered_parser =  CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED), self.recovered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = recovered_parser

        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 0, self.particle_meta, False, 0, True)

        # essentially skips particle a
        recovered_parser.set_state(recovered_new_state)
        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 22, self.particle_b, False, 2, True)
        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 33, self.particle_c, True, 3, True)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_new_state = {StateKey.POSITION: 11,
                                 StateKey.RECORDS_READ: 1,
                                 StateKey.METADATA_SENT: True}
        
        telemetered_parser =  CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED), self.telemetered_start_state, stream_handle,
            self.state_callback, self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = telemetered_parser

        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 0, self.particle_meta, False, 0, True)

        # essentially skips particle a
        telemetered_parser.set_state(telemetered_new_state)
        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 22, self.particle_b, False, 2, True)
        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 33, self.particle_c, True, 3, True)
    def test_simple_pad(self):
        """
        Read test data and pull out data particles one at a time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
            self.recovered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD))
        self.parser = recovered_parser

        # next get records
        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 0, self.particle_meta, False, 0,
                           True)

        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 11, self.particle_a, False, 1,
                           True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 22, self.particle_b, False, 2,
                           True)
        recovered_result = self.parser.get_records(1)
        self.assert_result(recovered_result, 33, self.particle_c, True, 3,
                           True)

        # no data left, dont move the position
        recovered_result = recovered_parser.get_records(1)
        self.assertEqual(recovered_result, [])
        self.assertEqual(recovered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED),
            self.telemetered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA_PAD))
        self.parser = telemetered_parser

        # next get records
        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 0, self.particle_meta, False, 0,
                           True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 11, self.particle_a, False, 1,
                           True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 22, self.particle_b, False, 2,
                           True)
        telemetered_result = self.parser.get_records(1)
        self.assert_result(telemetered_result, 33, self.particle_c, True, 3,
                           True)

        # no data left, dont move the position
        telemetered_result = telemetered_parser.get_records(1)
        self.assertEqual(telemetered_result, [])
        self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assert_(isinstance(self.publish_callback_value, list))
        self.assertEqual(self.publish_callback_value[0], self.particle_c)
    def test_get_many(self):
        """
        Read test data and pull out multiple data particles at one time.
        Assert that the results are those we expected.
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)
        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
            self.recovered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        self.parser = recovered_parser
        # next get records
        # NOTE - while we ask for 4 records, the metadata is a special case
        # the number of records read is actually 3. See the IDD for details.
        recovered_result = recovered_parser.get_records(4)
        self.assertEqual(recovered_result, [
            self.particle_meta, self.particle_a, self.particle_b,
            self.particle_c
        ])
        self.assertEqual(recovered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assertEqual(self.publish_callback_value[0], self.particle_meta)
        self.assertEqual(self.publish_callback_value[1], self.particle_a)
        self.assertEqual(self.publish_callback_value[2], self.particle_b)
        self.assertEqual(self.publish_callback_value[3], self.particle_c)
        self.assertEqual(self.file_ingested_value, True)
        self.assertEqual(recovered_parser._state[StateKey.RECORDS_READ], 3)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
        self.assertEqual(recovered_parser._state[StateKey.METADATA_SENT], True)
        self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT],
                         True)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED),
            self.telemetered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))
        self.parser = telemetered_parser
        # next get records
        telemetered_result = telemetered_parser.get_records(4)
        self.assertEqual(telemetered_result, [
            self.particle_meta, self.particle_a, self.particle_b,
            self.particle_c
        ])
        self.assertEqual(telemetered_parser._state[StateKey.POSITION], 33)
        self.assertEqual(self.state_callback_value[StateKey.POSITION], 33)
        self.assertEqual(self.publish_callback_value[0], self.particle_meta)
        self.assertEqual(self.publish_callback_value[1], self.particle_a)
        self.assertEqual(self.publish_callback_value[2], self.particle_b)
        self.assertEqual(self.publish_callback_value[3], self.particle_c)
        self.assertEqual(self.file_ingested_value, True)
        self.assertEqual(telemetered_parser._state[StateKey.RECORDS_READ], 3)
        self.assertEqual(self.state_callback_value[StateKey.RECORDS_READ], 3)
        self.assertEqual(telemetered_parser._state[StateKey.METADATA_SENT],
                         True)
        self.assertEqual(self.state_callback_value[StateKey.METADATA_SENT],
                         True)
    def test_set_state(self):
        """
        Test changing to a new state after initializing the parser and
        reading data, as if new data has been found and the state has
        changed
        """
        stream_handle = StringIO(CtdpfCklWfpParserUnitTestCase.TEST_DATA)

        #********************************************
        # Test the "recovered" version of the parser
        #********************************************
        recovered_new_state = {
            StateKey.POSITION: 11,
            StateKey.RECORDS_READ: 1,
            StateKey.METADATA_SENT: True
        }

        recovered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_RECOVERED),
            self.recovered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = recovered_parser

        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 0, self.particle_meta, False, 0,
                           True)

        # essentially skips particle a
        recovered_parser.set_state(recovered_new_state)
        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 22, self.particle_b, False, 2,
                           True)
        recovered_result = recovered_parser.get_records(1)
        self.assert_result(recovered_result, 33, self.particle_c, True, 3,
                           True)

        #**********************************************
        # Test the "telemetered" version of the parser
        #**********************************************
        telemetered_new_state = {
            StateKey.POSITION: 11,
            StateKey.RECORDS_READ: 1,
            StateKey.METADATA_SENT: True
        }

        telemetered_parser = CtdpfCklWfpParser(
            self.config.get(DataTypeKey.CTDPF_CKL_WFP_TELEMETERED),
            self.telemetered_start_state, stream_handle, self.state_callback,
            self.pub_callback, self.exception_callback,
            len(CtdpfCklWfpParserUnitTestCase.TEST_DATA))

        self.parser = telemetered_parser

        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 0, self.particle_meta, False, 0,
                           True)

        # essentially skips particle a
        telemetered_parser.set_state(telemetered_new_state)
        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 22, self.particle_b, False, 2,
                           True)
        telemetered_result = telemetered_parser.get_records(1)
        self.assert_result(telemetered_result, 33, self.particle_c, True, 3,
                           True)