def test_no_sensor_data(self): """ Read a file containing no sensor data records and verify that no particles are produced. """ log.debug("===== START TEST NO SENSOR DATA RECOVERED =====") with self.open_file(FILE1) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # Try to get a record and verify that none are produced. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.exception_callback_value, []) log.debug("===== START TEST NO SENSOR DATA TELEMETERED =====") with self.open_file(FILE1) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # Try to get a record and verify that none are produced. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.exception_callback_value, []) log.debug("===== END TEST NO SENSOR DATA =====")
def test_big_giant_input(self): """ Read a large file and verify that all expected particles can be read. Verification is not done at this time, but will be done during integration and qualification testing. """ log.debug("===== START TEST BIG GIANT INPUT RECOVERED =====") with self.open_file(FILE6) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # In a single read, get all particles in this file. number_expected_results = RECORDS_FILE6 result = parser.get_records(number_expected_results) self.assertEqual(len(result), number_expected_results) self.assertEqual(self.exception_callback_value, []) log.debug("===== START TEST BIG GIANT INPUT TELEMETERED =====") with self.open_file(FILE7) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # In a single read, get all particles in this file. number_expected_results = RECORDS_FILE7 result = parser.get_records(number_expected_results) self.assertEqual(len(result), number_expected_results) self.assertEqual(self.exception_callback_value, []) log.debug("===== END TEST BIG GIANT INPUT =====")
def test_many_with_yml(self): """ Read a file and verify that all records can be read. Verify that the contents of the particles are correct. There should be no exceptions generated. """ log.debug("===== START TEST MANY WITH YML RECOVERED =====") num_particles = 30 with self.open_file(FILE2) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) particles = parser.get_records(num_particles) log.debug("Num particles: %d", len(particles)) self.assert_particles(particles, "rec_20020215.flort2.yml", RESOURCE_PATH) self.assertEquals(self.exception_callback_value, []) log.debug("===== START TEST MANY WITH YML TELEMETERED =====") with self.open_file(FILE2) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) particles = parser.get_records(num_particles) log.debug("Num particles: %d", len(particles)) self.assert_particles(particles, "tel_20020215.flort2.yml", RESOURCE_PATH) self.assertEquals(self.exception_callback_value, []) log.debug("===== END TEST MANY WITH YML =====")
def test_get_many(self): """ Read a file and pull out multiple data particles at one time. Verify that the results are those we expected. """ log.debug('===== START TEST GET MANY RECOVERED =====') expected_particle = 24 with self.open_file(FILE5) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # In a single read, get all particles for this file. result = parser.get_records(expected_particle) self.assertEqual(len(result), expected_particle) self.assertEqual(self.exception_callback_value, []) log.debug('===== START TEST GET MANY TELEMETERED =====') with self.open_file(FILE4) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # In a single read, get all particles for this file. result = parser.get_records(expected_particle) self.assertEqual(len(result), expected_particle) self.assertEqual(self.exception_callback_value, []) log.debug('===== END TEST GET MANY =====')
def test_invalid_sensor_data_records(self): """ Read data from a file containing invalid sensor data records. Verify that no instrument particles are produced and the correct number of exceptions are detected. """ log.debug("===== START TEST INVALID SENSOR DATA RECOVERED =====") with self.open_file(FILE8) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # Try to get records and verify that none are returned. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(len(self.exception_callback_value), EXCEPTIONS_FILE8) log.debug("===== START TEST INVALID SENSOR DATA TELEMETERED =====") self.exception_callback_value = [] # reset exceptions with self.open_file(FILE8) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # Try to get records and verify that none are returned. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(len(self.exception_callback_value), EXCEPTIONS_FILE8) log.debug("===== END TEST INVALID SENSOR DATA =====")
def test_bug_9692(self): """ This test verifies a fix to accommodate DCL timestamps with Seconds >59 The test file is a trimmed down copy of a recovered file from a real deployment """ with self.open_file("20151023.flort.log") as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) particles = parser.get_records(5) log.debug("Num particles: %d", len(particles)) self.assertEquals(len(particles), 3) self.assertEquals(self.exception_callback_value, [])
def test_bug_9692(self): """ This test verifies a fix to accommodate DCL timestamps with Seconds >59 The test file is a trimmed down copy of a recovered file from a real deployment """ with self.open_file('20151023.flort.log') as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) particles = parser.get_records(5) log.debug("Num particles: %d", len(particles)) self.assertEquals(len(particles), 3) self.assertEquals(self.exception_callback_value, [])
def test_bug_13245(self): """ This test verifies that Issue #13245, where the re.match call in the parser never returns, has been fixed. """ log.debug('===== START TEST BUG 13245 =====') with self.open_file('20141204.flort2.log') as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) particles = parser.get_records(5) log.debug("Num particles: %d", len(particles)) self.assertEquals(len(particles), 0) self.assertEquals(len(self.exception_callback_value), 1) log.debug('===== END TEST BUG 13245 =====')
def test_get_many(self): """ Read a file and pull out multiple data particles at one time. Verify that the results are those we expected. """ log.debug("===== START TEST GET MANY RECOVERED =====") with self.open_file(FILE5) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # Generate a list of expected result particles. expected_particle = [] for expected in EXPECTED_FILE5: particle = FlortDjDclRecoveredInstrumentDataParticle(expected) particle.generate_dict() expected_particle.append(particle) # In a single read, get all particles for this file. result = parser.get_records(len(expected_particle)) self.assertEqual(result, expected_particle) self.assertEqual(self.exception_callback_value, []) log.debug("===== START TEST GET MANY TELEMETERED =====") with self.open_file(FILE4) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # Generate a list of expected result particles. expected_particle = [] for expected in EXPECTED_FILE4: particle = FlortDjDclTelemeteredInstrumentDataParticle(expected) particle.generate_dict() expected_particle.append(particle) # In a single read, get all particles for this file. result = parser.get_records(len(expected_particle)) self.assertEqual(result, expected_particle) self.assertEqual(self.exception_callback_value, []) log.debug("===== END TEST GET MANY =====")
def test_many_with_yml(self): """ Read a file and verify that all records can be read. Verify that the contents of the particles are correct. There should be no exceptions generated. """ log.debug('===== START TEST MANY WITH YML RECOVERED =====') num_particles = 30 with self.open_file(FILE2) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) particles = parser.get_records(num_particles) log.debug("Num particles: %d", len(particles)) self.assert_particles(particles, "rec_20020215.flort2.yml", RESOURCE_PATH) self.assertEquals(self.exception_callback_value, []) log.debug('===== START TEST MANY WITH YML TELEMETERED =====') with self.open_file(FILE2) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) particles = parser.get_records(num_particles) log.debug("Num particles: %d", len(particles)) self.assert_particles(particles, "tel_20020215.flort2.yml", RESOURCE_PATH) self.assertEquals(self.exception_callback_value, []) log.debug('===== END TEST MANY WITH YML =====')
def test_no_sensor_data(self): """ Read a file containing no sensor data records and verify that no particles are produced. """ log.debug('===== START TEST NO SENSOR DATA RECOVERED =====') with self.open_file(FILE1) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # Try to get a record and verify that none are produced. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.exception_callback_value, []) log.debug('===== START TEST NO SENSOR DATA TELEMETERED =====') with self.open_file(FILE1) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # Try to get a record and verify that none are produced. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.exception_callback_value, []) log.debug('===== END TEST NO SENSOR DATA =====')
def test_invalid_sensor_data_records(self): """ Read data from a file containing invalid sensor data records. Verify that no instrument particles are produced and the correct number of exceptions are detected. """ log.debug('===== START TEST INVALID SENSOR DATA RECOVERED =====') with self.open_file(FILE8) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # Try to get records and verify that none are returned. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(len(self.exception_callback_value), EXCEPTIONS_FILE8) log.debug('===== START TEST INVALID SENSOR DATA TELEMETERED =====') self.exception_callback_value = [] # reset exceptions with self.open_file(FILE8) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # Try to get records and verify that none are returned. result = parser.get_records(1) self.assertEqual(result, []) self.assertEqual(len(self.exception_callback_value), EXCEPTIONS_FILE8) log.debug('===== END TEST INVALID SENSOR DATA =====')
def test_big_giant_input(self): """ Read a large file and verify that all expected particles can be read. Verification is not done at this time, but will be done during integration and qualification testing. """ log.debug('===== START TEST BIG GIANT INPUT RECOVERED =====') with self.open_file(FILE6) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # In a single read, get all particles in this file. number_expected_results = RECORDS_FILE6 result = parser.get_records(number_expected_results) self.assertEqual(len(result), number_expected_results) self.assertEqual(self.exception_callback_value, []) log.debug('===== START TEST BIG GIANT INPUT TELEMETERED =====') with self.open_file(FILE7) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # In a single read, get all particles in this file. number_expected_results = RECORDS_FILE7 result = parser.get_records(number_expected_results) self.assertEqual(len(result), number_expected_results) self.assertEqual(self.exception_callback_value, []) log.debug('===== END TEST BIG GIANT INPUT =====')
def process(self): log = get_logger() with open(self._source_file_path, "r") as file_handle: def exception_callback(exception): log.debug("Exception: %s", exception) self._particle_data_handler.setParticleDataCaptureFailure() parser = FlortDjDclParser(self._parser_config, file_handle, exception_callback) driver = DataSetDriver(parser, self._particle_data_handler) driver.processFileStream() return self._particle_data_handler
def test_get_many(self): """ Read a file and pull out multiple data particles at one time. Verify that the results are those we expected. """ log.debug('===== START TEST GET MANY RECOVERED =====') with self.open_file(FILE5) as in_file: parser = FlortDjDclParser(self.rec_config, in_file, self.exception_callback) # Generate a list of expected result particles. expected_particle = [] for expected in EXPECTED_FILE5: particle = FlortDjDclRecoveredInstrumentDataParticle(expected) particle.generate_dict() expected_particle.append(particle) # In a single read, get all particles for this file. result = parser.get_records(len(expected_particle)) self.assertEqual(result, expected_particle) self.assertEqual(self.exception_callback_value, []) log.debug('===== START TEST GET MANY TELEMETERED =====') with self.open_file(FILE4) as in_file: parser = FlortDjDclParser(self.tel_config, in_file, self.exception_callback) # Generate a list of expected result particles. expected_particle = [] for expected in EXPECTED_FILE4: particle = FlortDjDclTelemeteredInstrumentDataParticle( expected) particle.generate_dict() expected_particle.append(particle) # In a single read, get all particles for this file. result = parser.get_records(len(expected_particle)) self.assertEqual(result, expected_particle) self.assertEqual(self.exception_callback_value, []) log.debug('===== END TEST GET MANY =====')