def test_simple(self):
        """
        This test reads in a small number of particles and verifies the result of one of the particles.
        """

        file_path = os.path.join(RESOURCE_PATH,
                                 'flntu_1_20131124T005004_458.mpk')
        stream_handle = open(file_path, 'rb')

        parser = FlntuXMmpCdsParser(self.config, None, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(6)

        for particle in particles:
            print particle.generate_dict()

        test_data = self.get_dict_from_yml('first.yml')
        self.assert_result(test_data['data'][0], particles[0])

        test_data = self.get_dict_from_yml('second.yml')
        self.assert_result(test_data['data'][0], particles[1])

        test_data = self.get_dict_from_yml('good.yml')
        self.assert_result(test_data['data'][0], particles[5])

        stream_handle.close()
    def test_get_many(self):
        """
        This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles
         and verifies the 30th particle.
        """

        file_path = os.path.join(RESOURCE_PATH,
                                 'flntu_1_20131124T005004_458.mpk')
        stream_handle = open(file_path, 'rb')

        parser = FlntuXMmpCdsParser(self.config, None, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(20)

        # Should end up with 20 particles
        self.assertTrue(len(particles) == 20)

        test_data = self.get_dict_from_yml('first.yml')
        self.assert_result(test_data['data'][0], particles[0])
        test_data = self.get_dict_from_yml('get_many_one.yml')
        self.assert_result(test_data['data'][0], particles[19])

        particles = parser.get_records(30)

        # Should end up with 30 particles
        self.assertTrue(len(particles) == 30)

        test_data = self.get_dict_from_yml('get_many_two.yml')
        self.assert_result(test_data['data'][0], particles[29])

        stream_handle.close()
    def test_mid_state_start(self):
        """
        This test exercises setting the state past one chunk, retrieving particles and verify the result of one
        of the particles.
        """

        # Using two concatenated msgpack files to simulate two chunks.
        file_path = os.path.join(RESOURCE_PATH, 'set_state.mpk')
        stream_handle = open(file_path, 'rb')

        # Moving the file position to the end of the first chunk
        state = {StateKey.PARTICLES_RETURNED: 20}

        parser = FlntuXMmpCdsParser(self.config, state, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(4)

        log.info(len(particles))

        # Should end up with 4 particles
        self.assertTrue(len(particles) == 4)

        test_data = self.get_dict_from_yml('set_state.yml')
        self.assert_result(test_data['data'][23], particles[3])
        self.assert_result(test_data['data'][22], particles[2])
        self.assert_result(test_data['data'][20], particles[0])

        stream_handle.close()
    def test_bad_data_two(self):
        """
        This test verifies that a SampleException is raised when an entire msgpack buffer is not msgpack.
        """

        file_path = os.path.join(RESOURCE_PATH, 'not-msg-pack.mpk')
        stream_handle = open(file_path, 'rb')

        parser = FlntuXMmpCdsParser(self.config, None, stream_handle,
                                    self.state_callback, self.pub_callback)

        with self.assertRaises(SampleException):
            parser.get_records(1)

        stream_handle.close()
Пример #5
0
    def test_bad_data_one(self):
        """
        This test verifies that a SampleException is raised when msgpack data is malformed.
        """

        file_path = os.path.join(RESOURCE_PATH, 'flntu_1_20131124T005004_458-BAD.mpk')
        stream_handle = open(file_path, 'rb')

        parser = FlntuXMmpCdsParser(self.config, None, stream_handle,
                                      self.state_callback, self.pub_callback)

        with self.assertRaises(SampleException):
            parser.get_records(1)

        stream_handle.close()
Пример #6
0
    def test_long_stream(self):
        """
        This test exercises retrieve approximately 200 particles.
        """

        # Using two concatenated msgpack files to simulate two chunks to get more particles.
        file_path = os.path.join(RESOURCE_PATH, 'flntu_concat.mpk')
        stream_handle = open(file_path, 'rb')

        state = {StateKey.PARTICLES_RETURNED: 0}

        parser = FlntuXMmpCdsParser(self.config, None, stream_handle,
                                      self.state_callback, self.pub_callback)

        # Attempt to retrieve 200 particles, but we will retrieve less
        particles = parser.get_records(200)

        # Should end up with 172 particles
        self.assertTrue(len(particles) == 184)

        stream_handle.close()
Пример #7
0
    def _build_parser(self, parser_state, infile, data_key=None):
        """
        Build and return the parser
        """

        parser = None

        if data_key == DataParticleType.FLNTU_X_MMP_CDS_INSTRUMENT:

            config = self._parser_config.get(
                DataParticleType.FLNTU_X_MMP_CDS_INSTRUMENT)
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.flntu_x_mmp_cds',
                DataSetDriverConfigKeys.PARTICLE_CLASS:
                'FlntuXMmpCdsParserDataParticle'
            })

            log.debug("My Config: %s", config)
            parser = FlntuXMmpCdsParser(
                config, parser_state, infile,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)

        elif data_key == DataParticleType.FLCDR_X_MMP_CDS_INSTRUMENT:

            config = self._parser_config.get(
                DataParticleType.FLCDR_X_MMP_CDS_INSTRUMENT)
            config.update({
                DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.flcdr_x_mmp_cds',
                DataSetDriverConfigKeys.PARTICLE_CLASS:
                'FlcdrXMmpCdsParserDataParticle'
            })

            log.debug("My Config: %s", config)
            parser = FlcdrXMmpCdsParser(
                config, parser_state, infile,
                lambda state, ingested: self._save_parser_state(
                    state, data_key, ingested), self._data_callback,
                self._sample_exception_callback)
        else:
            raise ConfigurationException(
                'flntu/flcdr parser not built due to missing key')
        if parser is None:
            raise ConfigurationException(
                'flntu/flcdr parser not built due to failed instantiation')
        return parser
Пример #8
0
    def _build_parser(self, stream_handle):

        parser_config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.flntu_x_mmp_cds',
            DataSetDriverConfigKeys.PARTICLE_CLASS: 'FlntuXMmpCdsParserDataParticle'
        }

        parser = FlntuXMmpCdsParser(parser_config,
                                    None,
                                    stream_handle,
                                    lambda state, ingested: None,
                                    lambda data: None,
                                    self._exception_callback)

        return parser
    def test_set_state(self):
        """
        This test exercises setting the state past one chunk, retrieving particles, verifying one
        of the particles, and then setting the state back to the beginning, retrieving a few particles, and
        verifying one of the particles.
        """

        # Using the default mspack test file.
        file_path = os.path.join(RESOURCE_PATH, 'set_state.mpk')
        stream_handle = open(file_path, 'rb')

        # Moving the file position to the beginning

        parser = FlntuXMmpCdsParser(self.config, None, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(4)

        # Should end up with 4 particles
        self.assertTrue(len(particles) == 4)

        log.info(parser._state)

        stat_info = os.stat(file_path)

        test_data = self.get_dict_from_yml('set_state.yml')
        self.assert_result(test_data['data'][3], particles[3])
        self.assert_result(test_data['data'][1], particles[1])
        self.assert_result(test_data['data'][0], particles[0])

        state = copy.copy(parser._state)

        log.info(state)

        parser = FlntuXMmpCdsParser(self.config, state, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(4)

        # Should end up with 4 particles
        self.assertTrue(len(particles) == 4)

        self.assert_result(test_data['data'][4], particles[0])
        self.assert_result(test_data['data'][6], particles[2])
        self.assert_result(test_data['data'][7], particles[3])

        # Give a bad position which will be ignored
        state = {StateKey.PARTICLES_RETURNED: 0}

        parser = FlntuXMmpCdsParser(self.config, state, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(1)

        self.assertTrue(len(particles) == 1)

        # Give a bad position which will be ignored
        state = {StateKey.PARTICLES_RETURNED: 0}

        parser = FlntuXMmpCdsParser(self.config, state, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(1000)

        self.assertTrue(len(particles) == 30)

        self.assert_result(test_data['data'][0], particles[0])
        self.assert_result(test_data['data'][14], particles[14])
        self.assert_result(test_data['data'][29], particles[29])

        # Provide a bad particles returned
        state = {StateKey.PARTICLES_RETURNED: 80}

        parser = FlntuXMmpCdsParser(self.config, state, stream_handle,
                                    self.state_callback, self.pub_callback)

        particles = parser.get_records(1)

        self.assertTrue(len(particles) == 0)

        stream_handle.close()