def test_get_many(self):
"""
This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles
and verifies the 30th particle.
"""
with open(os.path.join(RESOURCE_PATH, 'ctd_1_20131124T005004_458.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(20)
# Should end up with 20 particles
self.assertTrue(len(particles) == 20)
# the yml file only has particle 19 in it
self.assert_particles(particles[19:20], 'get_many_one.yml', RESOURCE_PATH)
particles = parser.get_records(30)
# Should end up with 30 particles
self.assertTrue(len(particles) == 30)
# the yml file only has particle 29 in it
self.assert_particles(particles[29:30], 'get_many_two.yml', RESOURCE_PATH)
def test_simple(self):
"""
This test reads in a small number of particles and verifies the result of one of the particles.
"""
with open(
os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_458.mpk'),
'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle,
self.exception_callback)
particles = parser.get_records(6)
# this yml file only has particle 5 in it
self.assert_particles(particles[5:6], 'good_cdr.yml',
RESOURCE_PATH)
# this yml file only has particle 1 in it
self.assert_particles(particles[1:2], 'second_cdr.yml',
RESOURCE_PATH)
# this yml file only has particle 0 in it
self.assert_particles(particles[0:1], 'first_cdr.yml',
RESOURCE_PATH)
def test_get_many(self):
"""
This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles
and verifies the 30th particle.
"""
with open(
os.path.join(RESOURCE_PATH, 'flntu_1_20131124T005004_458.mpk'),
'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle,
self.exception_callback)
particles = parser.get_records(20)
# Should end up with 20 particles
self.assertTrue(len(particles) == 20)
# this yml file only has particle 0 in it
self.assert_particles(particles[0:1], 'first.yml', RESOURCE_PATH)
# this yml file only has particle 19 in it
self.assert_particles(particles[19:20], 'get_many_one.yml',
RESOURCE_PATH)
particles = parser.get_records(30)
# Should end up with 30 particles
self.assertTrue(len(particles) == 30)
# this yml file only has particle 29 in it
self.assert_particles(particles[29:30], 'get_many_two.yml',
RESOURCE_PATH)
def test_simple(self):
"""
This test reads in a small number of particles and verifies the result of one of the particles.
"""
with open(os.path.join(RESOURCE_PATH, 'simple.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(1)
self.assert_particles(particles, 'simple.yml', RESOURCE_PATH)
def test_bad_data_two(self):
"""
This test verifies that a SampleException is raised when an entire msgpack buffer is not msgpack.
"""
with open(os.path.join(RESOURCE_PATH, 'not-msg-pack.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
parser.get_records(1)
self.assertTrue(len(self.exception_callback_value) >= 1)
self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_simple(self):
"""
This test reads in a small number of particles and verifies the result of one of the particles.
"""
with open(os.path.join(RESOURCE_PATH, 'simple.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(1)
self.assert_particles(particles, 'simple.yml', RESOURCE_PATH)
def test_bad_data_two(self):
"""
This test verifies that a SampleException is raised when an entire msgpack buffer is not msgpack.
"""
with open(os.path.join(RESOURCE_PATH, 'not-msg-pack.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
parser.get_records(1)
self.assertTrue(len(self.exception_callback_value) >= 1)
self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_bad_data_one(self):
"""
This test verifies that a SampleException is raised when msgpack data is malformed.
"""
with open(os.path.join(RESOURCE_PATH, 'ctd_1_20131124T005004_BAD.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
parser.get_records(1)
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_simple(self):
"""
This test reads in a small number of particles and verifies the result of one of the particles.
"""
with open(os.path.join(RESOURCE_PATH, 'optode_1_20131124T005004_458.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(6)
# the yml file only has particle 5 in it
self.assert_particles(particles[5:6], 'good.yml', RESOURCE_PATH)
def test_bad_data_one(self):
"""
This test verifies that a SampleException is raised when msgpack data is malformed.
"""
with open(os.path.join(RESOURCE_PATH, 'acm_1_20131124T005004_458-BAD.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
parser.get_records(1)
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def test_verify_with_yml(self):
"""
This test exercises retrieving 4 particles and verifies them using a yml file
"""
with open(os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_459.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(4)
# Should end up with 4 particles
self.assertTrue(len(particles) == 4)
self.assert_particles(particles, 'first_four_cdr.yml', RESOURCE_PATH)
def test_long_stream(self):
"""
This test exercises retrieve approximately 200 particles.
"""
# Using two concatenated msgpack files to simulate two chunks to get more particles.
with open(os.path.join(RESOURCE_PATH, 'acm_concat.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
# Attempt to retrieve 400 particles, but we will retrieve less
particles = parser.get_records(400)
# Should end up with 386 particles
self.assertTrue(len(particles) == 386)
def test_long_stream(self):
"""
This test exercises retrieve approximately 200 particles.
"""
# Using two concatenated msgpack files to simulate two chunks to get more particles.
with open(os.path.join(RESOURCE_PATH, 'ctd_concat.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
# Attempt to retrieve 200 particles, but we will retrieve less
particles = parser.get_records(200)
# Should end up with 172 particles
self.assertEqual(len(particles), 172)
def test_long_stream(self):
"""
This test exercises retrieve approximately 200 particles.
"""
with open(os.path.join(RESOURCE_PATH, 'large_import.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
# Attempt to retrieve 1000 particles
particles = parser.get_records(1000)
# Should end up with 1000 particles
self.assertTrue(len(particles) == 1000)
self.assert_particles(particles, 'large_import.yml', RESOURCE_PATH)
def test_get_many(self):
"""
This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles
and verifies the 30th particle.
"""
with open(os.path.join(RESOURCE_PATH, 'get_many.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(50)
# Should end up with 50 particles
self.assertTrue(len(particles) == 50)
self.assert_particles(particles, 'get_many.yml', RESOURCE_PATH)
def test_simple(self):
"""
This test reads in a small number of particles and verifies the result of one of the particles.
"""
with open(os.path.join(RESOURCE_PATH, 'ctd_1_20131124T005004_458.mpk'),
'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle,
self.exception_callback)
# parser.get_records(4) # throw first 4 particles away, yml file has 5th particle
particles = parser.get_records(6)
# the yml file only has particle 5 in it
self.assert_particles(particles[5:6], 'good.yml', RESOURCE_PATH)
def test_long_stream(self):
"""
This test exercises retrieve approximately 200 particles.
"""
with open(os.path.join(RESOURCE_PATH, 'large_import.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
# Attempt to retrieve 1000 particles
particles = parser.get_records(1000)
# Should end up with 1000 particles
self.assertTrue(len(particles) == 1000)
self.assert_particles(particles, 'large_import.yml', RESOURCE_PATH)
def test_get_many(self):
"""
This test exercises retrieving 20 particles, verifying the 20th particle, then retrieves 30 particles
and verifies the 30th particle.
"""
with open(os.path.join(RESOURCE_PATH, 'get_many.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(50)
# Should end up with 50 particles
self.assertTrue(len(particles) == 50)
self.assert_particles(particles, 'get_many.yml', RESOURCE_PATH)
def test_bad_data_one(self):
"""
This test verifies that a SampleException is raised when msgpack data is malformed.
"""
with open(os.path.join(RESOURCE_PATH, 'acs_archive.mpk'),
'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle,
self.exception_callback)
particles = parser.get_records(100)
self.assertTrue(len(particles) == 40)
with open(os.path.join(RESOURCE_PATH, 'acs_archive_BAD.mpk'),
'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle,
self.exception_callback)
parser.get_records(1)
self.assertTrue(len(self.exception_callback_value) >= 1)
self.assert_(
isinstance(self.exception_callback_value[0], SampleException))
def _build_parser(self, stream_handle):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.flntu_x_mmp_cds',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'FlntuXMmpCdsParserDataParticle'
}
parser = MmpCdsParser(parser_config, stream_handle, self._exception_callback)
return parser
def test_verify_with_yml(self):
"""
This test exercises retrieving 4 particles and verifies them using a yml file
"""
with open(
os.path.join(RESOURCE_PATH, 'flcdr_1_20131124T005004_459.mpk'),
'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle,
self.exception_callback)
particles = parser.get_records(4)
# Should end up with 4 particles
self.assertTrue(len(particles) == 4)
self.assert_particles(particles, 'first_four_cdr.yml',
RESOURCE_PATH)
def test_bad_data_one(self):
"""
This test verifies that a SampleException is raised when msgpack data is malformed.
"""
with open(os.path.join(RESOURCE_PATH, 'acs_archive.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
particles = parser.get_records(100)
self.assertTrue(len(particles) == 40)
with open(os.path.join(RESOURCE_PATH, 'acs_archive_BAD.mpk'), 'rb') as stream_handle:
parser = MmpCdsParser(self.config, stream_handle, self.exception_callback)
parser.get_records(1)
self.assertTrue(len(self.exception_callback_value) >= 1)
self.assert_(isinstance(self.exception_callback_value[0], SampleException))
def parse(unused, source_file_path, particle_data_handler):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dosta_abcdjm_mmp_cds',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'DostaAbcdjmMmpCdsParserDataParticle'
}
def exception_callback(exception):
log.debug("ERROR: %r", exception)
particle_data_handler.setParticleDataCaptureFailure()
with open(source_file_path, 'rb') as stream_handle:
parser = MmpCdsParser(parser_config, stream_handle, exception_callback)
driver = DataSetDriver(parser, particle_data_handler)
driver.processFileStream()
return particle_data_handler
def parse(basePythonCodePath, sourceFilePath, particleDataHdlrObj):
config.add_configuration(
os.path.join(basePythonCodePath, 'res', 'config', 'mi-logging.yml'))
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dosta_abcdjm_mmp_cds',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'DostaAbcdjmMmpCdsParserDataParticle'
}
def exception_callback(exception):
log.debug("ERROR: %r", exception)
particleDataHdlrObj.setParticleDataCaptureFailure()
with open(sourceFilePath, 'rb') as stream_handle:
parser = MmpCdsParser(parser_config, stream_handle, exception_callback)
driver = DataSetDriver(parser, particleDataHdlrObj)
driver.processFileStream()
return particleDataHdlrObj
