def test_no_sensor_data(self):
"""
Read a file containing no sensor data records
and verify that no particles are produced.
"""
log.debug("===== START TEST NO SENSOR DATA RECOVERED =====")
with self.open_file(FILE1) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
# Try to get a record and verify that none are produced.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.exception_callback_value, [])
log.debug("===== START TEST NO SENSOR DATA TELEMETERED =====")
with self.open_file(FILE1) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback)
# Try to get a record and verify that none are produced.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.exception_callback_value, [])
log.debug("===== END TEST NO SENSOR DATA =====")
def test_big_giant_input(self):
"""
Read a large file and verify that all expected particles can be read.
Verification is not done at this time, but will be done during
integration and qualification testing.
"""
log.debug("===== START TEST BIG GIANT INPUT RECOVERED =====")
with self.open_file(FILE6) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
# In a single read, get all particles in this file.
number_expected_results = RECORDS_FILE6
result = parser.get_records(number_expected_results)
self.assertEqual(len(result), number_expected_results)
self.assertEqual(self.exception_callback_value, [])
log.debug("===== START TEST BIG GIANT INPUT TELEMETERED =====")
with self.open_file(FILE7) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback)
# In a single read, get all particles in this file.
number_expected_results = RECORDS_FILE7
result = parser.get_records(number_expected_results)
self.assertEqual(len(result), number_expected_results)
self.assertEqual(self.exception_callback_value, [])
log.debug("===== END TEST BIG GIANT INPUT =====")
def test_many_with_yml(self):
"""
Read a file and verify that all records can be read.
Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug("===== START TEST MANY WITH YML RECOVERED =====")
num_particles = 30
with self.open_file(FILE2) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
particles = parser.get_records(num_particles)
log.debug("Num particles: %d", len(particles))
self.assert_particles(particles, "rec_20020215.flort2.yml", RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
log.debug("===== START TEST MANY WITH YML TELEMETERED =====")
with self.open_file(FILE2) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback)
particles = parser.get_records(num_particles)
log.debug("Num particles: %d", len(particles))
self.assert_particles(particles, "tel_20020215.flort2.yml", RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
log.debug("===== END TEST MANY WITH YML =====")
def test_get_many(self):
"""
Read a file and pull out multiple data particles at one time.
Verify that the results are those we expected.
"""
log.debug('===== START TEST GET MANY RECOVERED =====')
expected_particle = 24
with self.open_file(FILE5) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
# In a single read, get all particles for this file.
result = parser.get_records(expected_particle)
self.assertEqual(len(result), expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== START TEST GET MANY TELEMETERED =====')
with self.open_file(FILE4) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback)
# In a single read, get all particles for this file.
result = parser.get_records(expected_particle)
self.assertEqual(len(result), expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== END TEST GET MANY =====')
def test_invalid_sensor_data_records(self):
"""
Read data from a file containing invalid sensor data records.
Verify that no instrument particles are produced
and the correct number of exceptions are detected.
"""
log.debug("===== START TEST INVALID SENSOR DATA RECOVERED =====")
with self.open_file(FILE8) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
# Try to get records and verify that none are returned.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(len(self.exception_callback_value), EXCEPTIONS_FILE8)
log.debug("===== START TEST INVALID SENSOR DATA TELEMETERED =====")
self.exception_callback_value = [] # reset exceptions
with self.open_file(FILE8) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback)
# Try to get records and verify that none are returned.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(len(self.exception_callback_value), EXCEPTIONS_FILE8)
log.debug("===== END TEST INVALID SENSOR DATA =====")
def test_bug_9692(self):
"""
This test verifies a fix to accommodate DCL timestamps with Seconds >59
The test file is a trimmed down copy of a recovered file from a real deployment
"""
with self.open_file("20151023.flort.log") as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
particles = parser.get_records(5)
log.debug("Num particles: %d", len(particles))
self.assertEquals(len(particles), 3)
self.assertEquals(self.exception_callback_value, [])
def test_bug_9692(self):
"""
This test verifies a fix to accommodate DCL timestamps with Seconds >59
The test file is a trimmed down copy of a recovered file from a real deployment
"""
with self.open_file('20151023.flort.log') as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
particles = parser.get_records(5)
log.debug("Num particles: %d", len(particles))
self.assertEquals(len(particles), 3)
self.assertEquals(self.exception_callback_value, [])
def test_bug_13245(self):
"""
This test verifies that Issue #13245, where the re.match call in the parser
never returns, has been fixed.
"""
log.debug('===== START TEST BUG 13245 =====')
with self.open_file('20141204.flort2.log') as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
particles = parser.get_records(5)
log.debug("Num particles: %d", len(particles))
self.assertEquals(len(particles), 0)
self.assertEquals(len(self.exception_callback_value), 1)
log.debug('===== END TEST BUG 13245 =====')
def test_get_many(self):
"""
Read a file and pull out multiple data particles at one time.
Verify that the results are those we expected.
"""
log.debug("===== START TEST GET MANY RECOVERED =====")
with self.open_file(FILE5) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback)
# Generate a list of expected result particles.
expected_particle = []
for expected in EXPECTED_FILE5:
particle = FlortDjDclRecoveredInstrumentDataParticle(expected)
particle.generate_dict()
expected_particle.append(particle)
# In a single read, get all particles for this file.
result = parser.get_records(len(expected_particle))
self.assertEqual(result, expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug("===== START TEST GET MANY TELEMETERED =====")
with self.open_file(FILE4) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback)
# Generate a list of expected result particles.
expected_particle = []
for expected in EXPECTED_FILE4:
particle = FlortDjDclTelemeteredInstrumentDataParticle(expected)
particle.generate_dict()
expected_particle.append(particle)
# In a single read, get all particles for this file.
result = parser.get_records(len(expected_particle))
self.assertEqual(result, expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug("===== END TEST GET MANY =====")
def test_many_with_yml(self):
"""
Read a file and verify that all records can be read.
Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug('===== START TEST MANY WITH YML RECOVERED =====')
num_particles = 30
with self.open_file(FILE2) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
particles = parser.get_records(num_particles)
log.debug("Num particles: %d", len(particles))
self.assert_particles(particles, "rec_20020215.flort2.yml",
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== START TEST MANY WITH YML TELEMETERED =====')
with self.open_file(FILE2) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file,
self.exception_callback)
particles = parser.get_records(num_particles)
log.debug("Num particles: %d", len(particles))
self.assert_particles(particles, "tel_20020215.flort2.yml",
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST MANY WITH YML =====')
def test_no_sensor_data(self):
"""
Read a file containing no sensor data records
and verify that no particles are produced.
"""
log.debug('===== START TEST NO SENSOR DATA RECOVERED =====')
with self.open_file(FILE1) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
# Try to get a record and verify that none are produced.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.exception_callback_value, [])
log.debug('===== START TEST NO SENSOR DATA TELEMETERED =====')
with self.open_file(FILE1) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file,
self.exception_callback)
# Try to get a record and verify that none are produced.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.exception_callback_value, [])
log.debug('===== END TEST NO SENSOR DATA =====')
def test_invalid_sensor_data_records(self):
"""
Read data from a file containing invalid sensor data records.
Verify that no instrument particles are produced
and the correct number of exceptions are detected.
"""
log.debug('===== START TEST INVALID SENSOR DATA RECOVERED =====')
with self.open_file(FILE8) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
# Try to get records and verify that none are returned.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(len(self.exception_callback_value),
EXCEPTIONS_FILE8)
log.debug('===== START TEST INVALID SENSOR DATA TELEMETERED =====')
self.exception_callback_value = [] # reset exceptions
with self.open_file(FILE8) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file,
self.exception_callback)
# Try to get records and verify that none are returned.
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(len(self.exception_callback_value),
EXCEPTIONS_FILE8)
log.debug('===== END TEST INVALID SENSOR DATA =====')
def test_get_many(self):
"""
Read a file and pull out multiple data particles at one time.
Verify that the results are those we expected.
"""
log.debug('===== START TEST GET MANY RECOVERED =====')
expected_particle = 24
with self.open_file(FILE5) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
# In a single read, get all particles for this file.
result = parser.get_records(expected_particle)
self.assertEqual(len(result), expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== START TEST GET MANY TELEMETERED =====')
with self.open_file(FILE4) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file,
self.exception_callback)
# In a single read, get all particles for this file.
result = parser.get_records(expected_particle)
self.assertEqual(len(result), expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== END TEST GET MANY =====')
def test_big_giant_input(self):
"""
Read a large file and verify that all expected particles can be read.
Verification is not done at this time, but will be done during
integration and qualification testing.
"""
log.debug('===== START TEST BIG GIANT INPUT RECOVERED =====')
with self.open_file(FILE6) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
# In a single read, get all particles in this file.
number_expected_results = RECORDS_FILE6
result = parser.get_records(number_expected_results)
self.assertEqual(len(result), number_expected_results)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== START TEST BIG GIANT INPUT TELEMETERED =====')
with self.open_file(FILE7) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file,
self.exception_callback)
# In a single read, get all particles in this file.
number_expected_results = RECORDS_FILE7
result = parser.get_records(number_expected_results)
self.assertEqual(len(result), number_expected_results)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== END TEST BIG GIANT INPUT =====')
def process(self):
log = get_logger()
with open(self._source_file_path, "r") as file_handle:
def exception_callback(exception):
log.debug("Exception: %s", exception)
self._particle_data_handler.setParticleDataCaptureFailure()
parser = FlortDjDclParser(self._parser_config, file_handle,
exception_callback)
driver = DataSetDriver(parser, self._particle_data_handler)
driver.processFileStream()
return self._particle_data_handler
def test_get_many(self):
"""
Read a file and pull out multiple data particles at one time.
Verify that the results are those we expected.
"""
log.debug('===== START TEST GET MANY RECOVERED =====')
with self.open_file(FILE5) as in_file:
parser = FlortDjDclParser(self.rec_config, in_file,
self.exception_callback)
# Generate a list of expected result particles.
expected_particle = []
for expected in EXPECTED_FILE5:
particle = FlortDjDclRecoveredInstrumentDataParticle(expected)
particle.generate_dict()
expected_particle.append(particle)
# In a single read, get all particles for this file.
result = parser.get_records(len(expected_particle))
self.assertEqual(result, expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== START TEST GET MANY TELEMETERED =====')
with self.open_file(FILE4) as in_file:
parser = FlortDjDclParser(self.tel_config, in_file,
self.exception_callback)
# Generate a list of expected result particles.
expected_particle = []
for expected in EXPECTED_FILE4:
particle = FlortDjDclTelemeteredInstrumentDataParticle(
expected)
particle.generate_dict()
expected_particle.append(particle)
# In a single read, get all particles for this file.
result = parser.get_records(len(expected_particle))
self.assertEqual(result, expected_particle)
self.assertEqual(self.exception_callback_value, [])
log.debug('===== END TEST GET MANY =====')
