def test_get_many(self): """ This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles and verifies the 30th particle. """ file_path = os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_458.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(20) # Should end up with 20 particles self.assertTrue(len(particles) == 20) test_data = self.get_dict_from_yml('first_cdr.yml') self.assert_result(test_data['data'][0], particles[0]) test_data = self.get_dict_from_yml('get_many_one_cdr.yml') self.assert_result(test_data['data'][0], particles[19]) particles = parser.get_records(30) # Should end up with 30 particles self.assertTrue(len(particles) == 30) test_data = self.get_dict_from_yml('get_many_two_cdr.yml') self.assert_result(test_data['data'][0], particles[29]) stream_handle.close()
def test_bad_data_two(self): """ This test verifies that a SampleException is raised when an entire msgpack buffer is not msgpack. """ file_path = os.path.join(RESOURCE_PATH, "not-msg-pack.mpk") stream_handle = open(file_path, "rb") parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) with self.assertRaises(SampleException): parser.get_records(1) stream_handle.close()
def test_bad_data_one(self): """ This test verifies that a SampleException is raised when msgpack data is malformed. """ file_path = os.path.join(RESOURCE_PATH, "flcdr_1_20131124T005004_458-BAD.mpk") stream_handle = open(file_path, "rb") parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) with self.assertRaises(SampleException): parser.get_records(1) stream_handle.close()
def test_bad_data_two(self): """ This test verifies that a SampleException is raised when an entire msgpack buffer is not msgpack. """ file_path = os.path.join(RESOURCE_PATH, 'not-msg-pack.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) with self.assertRaises(SampleException): parser.get_records(1) stream_handle.close()
def test_bad_data_one(self): """ This test verifies that a SampleException is raised when msgpack data is malformed. """ file_path = os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_458-BAD.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) with self.assertRaises(SampleException): parser.get_records(1) stream_handle.close()
def test_simple(self): """ This test reads in a small number of particles and verifies the result of one of the particles. """ file_path = os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_458.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(6) for particle in particles: print particle.generate_dict() test_data = self.get_dict_from_yml('good_cdr.yml') self.assert_result(test_data['data'][0], particles[5]) test_data = self.get_dict_from_yml('second_cdr.yml') self.assert_result(test_data['data'][0], particles[1]) test_data = self.get_dict_from_yml('first_cdr.yml') self.assert_result(test_data['data'][0], particles[0]) stream_handle.close()
def test_mid_state_start(self): """ This test exercises setting the state past one chunk, retrieving particles and verify the result of one of the particles. """ # Using two concatenated msgpack files to simulate two chunks. file_path = os.path.join(RESOURCE_PATH, 'set_state_cdr.mpk') stream_handle = open(file_path, 'rb') stat_info = os.stat(file_path) # Moving the file position to the end of the first chunk state = {StateKey.PARTICLES_RETURNED: 20} parser = FlcdrXMmpCdsParser(self.config, state, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(4) log.info(len(particles)) # Should end up with 4 particles self.assertTrue(len(particles) == 4) test_data = self.get_dict_from_yml('set_state_cdr.yml') self.assert_result(test_data['data'][20], particles[0]) test_data = self.get_dict_from_yml('set_state_cdr.yml') self.assert_result(test_data['data'][23], particles[3]) stream_handle.close()
def test_verify_with_yml(self): """ This test exercises retrieving 4 particles and verifies them using a yml file """ file_path = os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_459.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(4) # Should end up with 4 particles self.assertTrue(len(particles) == 4) self.assert_particles(particles, 'first_four_cdr.yml', RESOURCE_PATH) stream_handle.close()
def test_long_stream(self): """ This test exercises retrieve approximately 200 particles. """ # Using two concatenated msgpack files to simulate two chunks to get more particles. file_path = os.path.join(RESOURCE_PATH, 'flcdr_concat.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) # Attempt to retrieve 200 particles, but we will retrieve less particles = parser.get_records(400) # Should end up with 172 particles self.assertTrue(len(particles) == 394) stream_handle.close()
def test_set_state(self): """ This test exercises setting the state past one chunk, retrieving particles, verifying one of the particles, and then setting the state back to the beginning, retrieving a few particles, and verifying one of the particles. """ # Using the default mspack test file. file_path = os.path.join(RESOURCE_PATH, 'set_state_cdr.mpk') stream_handle = open(file_path, 'rb') parser = FlcdrXMmpCdsParser(self.config, None, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(4) # Should end up with 4 particles self.assertTrue(len(particles) == 4) log.info(parser._state) test_data = self.get_dict_from_yml('set_state_cdr.yml') self.assert_result(test_data['data'][0], particles[0]) self.assert_result(test_data['data'][2], particles[2]) self.assert_result(test_data['data'][3], particles[3]) state = copy.copy(parser._state) log.info(state) parser = FlcdrXMmpCdsParser(self.config, state, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(4) # Should end up with 4 particles self.assertTrue(len(particles) == 4) self.assert_result(test_data['data'][6], particles[2]) self.assert_result(test_data['data'][7], particles[3]) # Give a bad position which will be ignored state = {StateKey.PARTICLES_RETURNED: 0} parser = FlcdrXMmpCdsParser(self.config, state, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(1) self.assertTrue(len(particles) == 1) # Give a bad position which will be ignored state = {StateKey.PARTICLES_RETURNED: 0} parser = FlcdrXMmpCdsParser(self.config, state, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(1000) self.assertTrue(len(particles) == 30) self.assert_result(test_data['data'][29], particles[29]) self.assert_result(test_data['data'][20], particles[20]) self.assert_result(test_data['data'][0], particles[0]) # Provide a bad particles returned state = {StateKey.PARTICLES_RETURNED: 80} parser = FlcdrXMmpCdsParser(self.config, state, stream_handle, self.state_callback, self.pub_callback) particles = parser.get_records(1) self.assertTrue(len(particles) == 0) stream_handle.close()