def test_get_many(self): """ This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles and verifies the 30th particle. """ with open(os.path.join(RESOURCE_PATH, 'ctd_1_20131124T005004_458.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(20) # Should end up with 20 particles self.assertTrue(len(particles) == 20) # the yml file only has particle 19 in it self.assert_particles(particles[19:20], 'get_many_one.yml', RESOURCE_PATH) particles = parser.get_records(30) # Should end up with 30 particles self.assertTrue(len(particles) == 30) # the yml file only has particle 29 in it self.assert_particles(particles[29:30], 'get_many_two.yml', RESOURCE_PATH)
def test_simple(self): """ This test reads in a small number of particles and verifies the result of one of the particles. """ with open( os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_458.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(6) # this yml file only has particle 5 in it self.assert_particles(particles[5:6], 'good_cdr.yml', RESOURCE_PATH) # this yml file only has particle 1 in it self.assert_particles(particles[1:2], 'second_cdr.yml', RESOURCE_PATH) # this yml file only has particle 0 in it self.assert_particles(particles[0:1], 'first_cdr.yml', RESOURCE_PATH)
def test_get_many(self): """ This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles and verifies the 30th particle. """ with open( os.path.join(RESOURCE_PATH, 'flntu_1_20131124T005004_458.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(20) # Should end up with 20 particles self.assertTrue(len(particles) == 20) # this yml file only has particle 0 in it self.assert_particles(particles[0:1], 'first.yml', RESOURCE_PATH) # this yml file only has particle 19 in it self.assert_particles(particles[19:20], 'get_many_one.yml', RESOURCE_PATH) particles = parser.get_records(30) # Should end up with 30 particles self.assertTrue(len(particles) == 30) # this yml file only has particle 29 in it self.assert_particles(particles[29:30], 'get_many_two.yml', RESOURCE_PATH)
def test_simple(self): """ This test reads in a small number of particles and verifies the result of one of the particles. """ with open(os.path.join(RESOURCE_PATH, 'simple.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(1) self.assert_particles(particles, 'simple.yml', RESOURCE_PATH)
def test_bad_data_two(self): """ This test verifies that a SampleException is raised when an entire msgpack buffer is not msgpack. """ with open(os.path.join(RESOURCE_PATH, 'not-msg-pack.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) parser.get_records(1) self.assertTrue(len(self.exception_callback_value) >= 1) self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_bad_data_one(self): """ This test verifies that a SampleException is raised when msgpack data is malformed. """ with open(os.path.join(RESOURCE_PATH, 'ctd_1_20131124T005004_BAD.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) parser.get_records(1) self.assertEqual(len(self.exception_callback_value), 1) self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_simple(self): """ This test reads in a small number of particles and verifies the result of one of the particles. """ with open(os.path.join(RESOURCE_PATH, 'optode_1_20131124T005004_458.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(6) # the yml file only has particle 5 in it self.assert_particles(particles[5:6], 'good.yml', RESOURCE_PATH)
def test_bad_data_one(self): """ This test verifies that a SampleException is raised when msgpack data is malformed. """ with open(os.path.join(RESOURCE_PATH, 'acm_1_20131124T005004_458-BAD.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) parser.get_records(1) self.assertEqual(len(self.exception_callback_value), 1) self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_verify_with_yml(self): """ This test exercises retrieving 4 particles and verifies them using a yml file """ with open(os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_459.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(4) # Should end up with 4 particles self.assertTrue(len(particles) == 4) self.assert_particles(particles, 'first_four_cdr.yml', RESOURCE_PATH)
def test_long_stream(self): """ This test exercises retrieve approximately 200 particles. """ # Using two concatenated msgpack files to simulate two chunks to get more particles. with open(os.path.join(RESOURCE_PATH, 'acm_concat.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) # Attempt to retrieve 400 particles, but we will retrieve less particles = parser.get_records(400) # Should end up with 386 particles self.assertTrue(len(particles) == 386)
def test_long_stream(self): """ This test exercises retrieve approximately 200 particles. """ # Using two concatenated msgpack files to simulate two chunks to get more particles. with open(os.path.join(RESOURCE_PATH, 'ctd_concat.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) # Attempt to retrieve 200 particles, but we will retrieve less particles = parser.get_records(200) # Should end up with 172 particles self.assertEqual(len(particles), 172)
def test_long_stream(self): """ This test exercises retrieve approximately 200 particles. """ with open(os.path.join(RESOURCE_PATH, 'large_import.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) # Attempt to retrieve 1000 particles particles = parser.get_records(1000) # Should end up with 1000 particles self.assertTrue(len(particles) == 1000) self.assert_particles(particles, 'large_import.yml', RESOURCE_PATH)
def test_get_many(self): """ This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles and verifies the 30th particle. """ with open(os.path.join(RESOURCE_PATH, 'get_many.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(50) # Should end up with 50 particles self.assertTrue(len(particles) == 50) self.assert_particles(particles, 'get_many.yml', RESOURCE_PATH)
def test_simple(self): """ This test reads in a small number of particles and verifies the result of one of the particles. """ with open(os.path.join(RESOURCE_PATH, 'ctd_1_20131124T005004_458.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) # parser.get_records(4) # throw first 4 particles away, yml file has 5th particle particles = parser.get_records(6) # the yml file only has particle 5 in it self.assert_particles(particles[5:6], 'good.yml', RESOURCE_PATH)
def test_bad_data_one(self): """ This test verifies that a SampleException is raised when msgpack data is malformed. """ with open(os.path.join(RESOURCE_PATH, 'acs_archive.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(100) self.assertTrue(len(particles) == 40) with open(os.path.join(RESOURCE_PATH, 'acs_archive_BAD.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) parser.get_records(1) self.assertTrue(len(self.exception_callback_value) >= 1) self.assert_( isinstance(self.exception_callback_value[0], SampleException))
def _build_parser(self, stream_handle): parser_config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.flntu_x_mmp_cds', DataSetDriverConfigKeys.PARTICLE_CLASS: 'FlntuXMmpCdsParserDataParticle' } parser = MmpCdsParser(parser_config, stream_handle, self._exception_callback) return parser
def test_verify_with_yml(self): """ This test exercises retrieving 4 particles and verifies them using a yml file """ with open( os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_459.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(4) # Should end up with 4 particles self.assertTrue(len(particles) == 4) self.assert_particles(particles, 'first_four_cdr.yml', RESOURCE_PATH)
def test_bad_data_one(self): """ This test verifies that a SampleException is raised when msgpack data is malformed. """ with open(os.path.join(RESOURCE_PATH, 'acs_archive.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) particles = parser.get_records(100) self.assertTrue(len(particles) == 40) with open(os.path.join(RESOURCE_PATH, 'acs_archive_BAD.mpk'), 'rb') as stream_handle: parser = MmpCdsParser(self.config, stream_handle, self.exception_callback) parser.get_records(1) self.assertTrue(len(self.exception_callback_value) >= 1) self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def parse(unused, source_file_path, particle_data_handler): parser_config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dosta_abcdjm_mmp_cds', DataSetDriverConfigKeys.PARTICLE_CLASS: 'DostaAbcdjmMmpCdsParserDataParticle' } def exception_callback(exception): log.debug("ERROR: %r", exception) particle_data_handler.setParticleDataCaptureFailure() with open(source_file_path, 'rb') as stream_handle: parser = MmpCdsParser(parser_config, stream_handle, exception_callback) driver = DataSetDriver(parser, particle_data_handler) driver.processFileStream() return particle_data_handler
def parse(basePythonCodePath, sourceFilePath, particleDataHdlrObj): config.add_configuration( os.path.join(basePythonCodePath, 'res', 'config', 'mi-logging.yml')) parser_config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dosta_abcdjm_mmp_cds', DataSetDriverConfigKeys.PARTICLE_CLASS: 'DostaAbcdjmMmpCdsParserDataParticle' } def exception_callback(exception): log.debug("ERROR: %r", exception) particleDataHdlrObj.setParticleDataCaptureFailure() with open(sourceFilePath, 'rb') as stream_handle: parser = MmpCdsParser(parser_config, stream_handle, exception_callback) driver = DataSetDriver(parser, particleDataHdlrObj) driver.processFileStream() return particleDataHdlrObj