def test_random_diag_record(self):
"""
The file used in this test has a bad checksum in the head configuration record.
Instrument metadata will still be produced but the fields from
the head config will NOT be included.
"""
log.debug('===== START TEST RANDOM DIAGNOSTIC RECORD FOUND =====')
with open(os.path.join(RESOURCE_PATH, 'random_diag_record_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'random_diag_record_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST RANDOM DIAGNOSTIC RECORD FOUND =====')
def test_invalid_metadata_timestamp(self):
"""
The file used in this test has error in the timestamp for the first metadata record.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST INVALID METADATA TIMESTAMP =====')
with open(
os.path.join(RESOURCE_PATH, 'invalid_metadata_timestamp.txt'),
'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 9
parser = Pco2wAbcParser(self._parser_config, file_handle,
self.exception_callback, None, None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "invalid_metadata_timestamp.yml",
RESOURCE_PATH)
self.assertEqual(self._exception_occurred, True)
log.debug('===== END TEST INVALID METADATA TIMESTAMP =====')
def test_real_file(self):
"""
The file used in this test, is a real file from the IDD. It contains 14 records:
7 power records, 6 CO2 records (normal) and 1 CO2 record (blank).
(No control files are in the real file.)
Verify that the correct number of particles are generated
from a real file.
"""
log.debug('===== START TEST REAL FILE =====')
with open(os.path.join(RESOURCE_PATH, '20140507.pco2w1.log'), 'r') as file_handle:
num_particles_to_request = 2500
num_expected_particles = 14
parser = Pco2wAbcDclParser(self._recovered_parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(num_particles_to_request)
log.info(len(particles))
self.assertEquals(len(particles), num_expected_particles)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST REAL FILE =====')
def test_invalid_checksum(self):
"""
The last five records in the file used in this test have a length that does not match the Length
field in the record. This tests for this requirement:
If the beginning of another instrument data record (* character), is encountered before "Length"
bytes have been found, where "Length" is the record length specified in a record, then we can not
reliably parse the record.
This results in 5 particles being retrieved instead of 6, and also result in the exception
callback being called.
"""
log.debug('===== START TEST INVALID CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_checksum.log'),
'r') as file_handle:
num_particles_to_request = 1
num_expected_particles = 1
parser = Pco2wAbcDclParser(self._recovered_parser_config,
file_handle, self.exception_callback,
None, None)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "invalid_checksum.yml",
RESOURCE_PATH)
# No exception should be thrown
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST INVALID CHECKSUM =====')
def test_no_particles(self):
"""
The file used in this test only contains DCL Logging records.
Verify that no particles are produced if the input file
has no instrument data records, i.e., they just contain DCL Logging records.
"""
log.debug('===== START TEST NO PARTICLES =====')
with open(os.path.join(RESOURCE_PATH, 'no_particles.log'), 'r') as file_handle:
num_particles_to_request = 2
num_expected_particles = 0
parser = Pco2wAbcDclParser(self._recovered_parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST NO PARTICLES =====')
def test_invalid_checksum(self):
"""
The last five records in the file used in this test have a length that does not match the Length
field in the record. This tests for this requirement:
If the beginning of another instrument data record (* character), is encountered before "Length"
bytes have been found, where "Length" is the record length specified in a record, then we can not
reliably parse the record.
This results in 5 particles being retrieved instead of 6, and also result in the exception
callback being called.
"""
log.debug('===== START TEST INVALID CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_checksum.log'), 'r') as file_handle:
num_particles_to_request = 1
num_expected_particles = 1
parser = Pco2wAbcDclParser(self._recovered_parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "invalid_checksum.yml", RESOURCE_PATH)
# No exception should be thrown
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST INVALID CHECKSUM =====')
def test_real_file(self):
"""
The file used in this test, is a real file from the acquisition server.
It contains 20 pH records:
Verify that 20 instrument particles and one metadata particle are generated
from the real file.
"""
log.debug('===== START TEST REAL FILE =====')
with open(os.path.join(RESOURCE_PATH, 'phsen1_20140730_190554.DAT'), O_MODE) as file_handle:
num_particles_to_request = 25
num_expected_particles = 21
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
log.info(len(particles))
self.assertEquals(len(particles), num_expected_particles)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST REAL FILE =====')
def test_schema_generation(self):
self.maxDiff = None
result = self.param_dict.generate_dict()
json_result = json.dumps(result, indent=4, sort_keys=True)
log.debug("Expected: %s", self.target_schema)
log.debug("Result: %s", json_result)
self.assertEqual(result, self.target_schema)
def test_no_particles(self):
"""
Verify that no particles are produced if the input file
has no instrument records.
"""
log.debug('===== START TEST NO PARTICLES =====')
with open(os.path.join(RESOURCE_PATH, 'no_particles.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 0
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assertEqual(self._exception_occurred, False)
log.debug('===== END TEST NO PARTICLES =====')
def test_blank_line(self):
"""
Read file and verify that all expected particles can be read.
Verify that the contents of the particles are correct. There are
blank lines interspersed in the file. This test verifies that
these blank lines do not adversely affect the parser. Only one
test is run as the content of the input files is the same for
recovered or telemetered.
"""
log.debug('===== START TEST BLANK LINE =====')
num_particles_to_request = 25
num_expected_particles = 20
with open(os.path.join(RESOURCE_PATH, '20141207sbl.pwrsys.log'),
'rU') as file_handle:
parser = FuelCellEngDclParser(self._recovered_parser_config,
file_handle, self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
log.debug('===== END TEST BLANK LINE =====')
def test_invalid_data_recov(self):
"""
Read files and verify that all expected particles can be read.
Verify that invalid data is handled appropriately with the
correct exceptions being reported.
"""
log.debug('===== START TEST INVALID DATA RECOVERED =====')
num_particles_to_request = 10
num_expected_particles = 7
with open(os.path.join(RESOURCE_PATH, 'pco2wXYZ_11212014_1625.DAT'),
'r') as file_handle:
parser = Pco2wAbcImodemParser(self._recov_parser_config,
file_handle, self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
"pco2wXYZ_11212014_1625.recov.yml",
RESOURCE_PATH)
self.assertEquals(len(self.exception_callback_value), 2)
for exception in self.exception_callback_value:
self.assertIsInstance(exception, RecoverableSampleException)
log.debug('===== END TEST INVALID DATA RECOVERED =====')
def test_missing_file_time_recov(self):
"""
Read a file that is missing the file time metadata
A RecoverableException should be reported.
"""
log.debug('===== START TEST MISSING FILE TIME RECOV =====')
num_particles_to_request = 10
num_expected_particles = 6
with open(os.path.join(RESOURCE_PATH, 'pco2wXYZ_11212014_1628.DAT'),
'r') as file_handle:
parser = Pco2wAbcImodemParser(self._recov_parser_config,
file_handle, self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
"pco2wXYZ_11212014_1628.recov.yml",
RESOURCE_PATH)
self.assertEquals(len(self.exception_callback_value), 1)
for exception in self.exception_callback_value:
self.assertIsInstance(exception, RecoverableSampleException)
log.debug('===== END TEST MISSING FILE TIME RECOV =====')
def test_invalid_header_fields(self):
"""
The header in the file used in this test has in invalid Voltage and Number of Samples Written.
A metadata particle will still get created, but there will be None in some of the parameters
(an exception will be generated).
"""
log.debug('===== START TEST INVALID HEADER FIELDS =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_header_fields.DAT'), O_MODE) as file_handle:
num_particles_to_request = 5
num_expected_particles = 5
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "invalid_header_fields_rec.yml", RESOURCE_PATH)
self.assert_(isinstance(self.exception_callback_value[0], UnexpectedDataException))
log.debug('===== END TEST INVALID HEADER FIELDS =====')
def test_invalid_checksum(self):
"""
The first record in the file used in this test has in invalid checksum. An instrument particle will
still get created, but the passed_checksum parameter will be 0 (no warning or error msg generated).
"""
log.debug('===== START TEST INVALID CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_checksum.DAT'), O_MODE) as file_handle:
num_particles_to_request = 5
num_expected_particles = 5
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "invalid_checksum_rec.yml", RESOURCE_PATH)
# No exception should be thrown
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST INVALID CHECKSUM =====')
def test_no_science_particles(self):
"""
The file used in this test only contains header and footer records.
Verify that no science (pH or Control) particles are produced if the input file
has no pH data records or control data, i.e., they just contain header and footer records.
In this case only the metadata particle will get created.
"""
log.debug('===== START TEST NO SCIENCE PARTICLES =====')
with open(os.path.join(RESOURCE_PATH, 'header_and_footer_only.DAT'), O_MODE) as file_handle:
num_particles_to_request = 2
num_expected_particles = 1
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "header_and_footer_only_rec.yml", RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST NO SCIENCE PARTICLES =====')
def test_real_file(self):
"""
The file used in this test, is a real file from the IDD. It contains 14 records:
7 power records, 6 CO2 records (normal) and 1 CO2 record (blank).
(No control files are in the real file.)
Verify that the correct number of particles are generated
from a real file.
"""
log.debug('===== START TEST REAL FILE =====')
with open(os.path.join(RESOURCE_PATH, '20140507.pco2w1.log'),
'r') as file_handle:
num_particles_to_request = 2500
num_expected_particles = 14
parser = Pco2wAbcDclParser(self._recovered_parser_config,
file_handle, self.exception_callback,
None, None)
particles = parser.get_records(num_particles_to_request)
log.info(len(particles))
self.assertEquals(len(particles), num_expected_particles)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST REAL FILE =====')
def test_invalid_data_recov(self):
"""
Read files and verify that all expected particles can be read.
Verify that invalid data is handled appropriately with the
correct exceptions being reported.
"""
log.debug('===== START TEST INVALID DATA RECOVERED =====')
num_particles_to_request = 10
num_expected_particles = 7
with open(os.path.join(RESOURCE_PATH, 'pco2wXYZ_11212014_1625.DAT'), 'r') as file_handle:
parser = Pco2wAbcImodemParser(self._recov_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "pco2wXYZ_11212014_1625.recov.yml", RESOURCE_PATH)
self.assertEquals(len(self.exception_callback_value), 2)
for exception in self.exception_callback_value:
self.assertIsInstance(exception, RecoverableSampleException)
log.debug('===== END TEST INVALID DATA RECOVERED =====')
def launch(self):
"""
Launches the simulator process as indicated by _COMMAND.
@return (rsn_oms, uri) A pair with the CIOMSSimulator instance and the
associated URI to establish connection with it.
"""
log.debug("[OMSim] Launching: %s", _COMMAND)
self._process = self._spawn(_COMMAND)
if not self._process or not self.poll():
msg = "[OMSim] Failed to launch simulator: %s" % _COMMAND
log.error(msg)
raise Exception(msg)
log.debug("[OMSim] process started, pid: %s", self.getpid())
# give it some time to start up
sleep(5)
# get URI:
uri = None
with open("logs/rsn_oms_simulator.yml", buffering=1) as f:
# we expect one of the first few lines to be of the form:
# rsn_oms_simulator_uri=xxxx
# where xxxx is the uri -- see oms_simulator_server.
while uri is None:
line = f.readline()
if line.index("rsn_oms_simulator_uri=") == 0:
uri = line[len("rsn_oms_simulator_uri="):].strip()
self._rsn_oms = CIOMSClientFactory.create_instance(uri)
return self._rsn_oms, uri
def test_missing_file_time_recov(self):
"""
Read a file that is missing the file time metadata
A RecoverableException should be reported.
"""
log.debug('===== START TEST MISSING FILE TIME RECOV =====')
num_particles_to_request = 10
num_expected_particles = 6
with open(os.path.join(RESOURCE_PATH, 'pco2wXYZ_11212014_1628.DAT'), 'r') as file_handle:
parser = Pco2wAbcImodemParser(self._recov_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "pco2wXYZ_11212014_1628.recov.yml", RESOURCE_PATH)
self.assertEquals(len(self.exception_callback_value), 1)
for exception in self.exception_callback_value:
self.assertIsInstance(exception, RecoverableSampleException)
log.debug('===== END TEST MISSING FILE TIME RECOV =====')
def generate_and_notify_event(self):
if self._index >= len(EventInfo.EVENT_TYPES):
self._index = 0
event_type = EventInfo.EVENT_TYPES.values()[self._index]
self._index += 1
platform_id = "TODO_some_platform_id"
message = "%s (synthetic event generated from simulator)" % event_type[
'name']
group = event_type['group']
timestamp = ntplib.system_to_ntp_time(time.time())
first_time_timestamp = timestamp
severity = event_type['severity']
event_instance = {
'message': message,
'platform_id': platform_id,
'timestamp': timestamp,
'first_time_timestamp': first_time_timestamp,
'severity': severity,
'group': group,
}
log.debug("notifying event_instance=%s", str(event_instance))
self._notifier.notify(event_instance)
def test_bad_diag_checksum_21_recs(self):
"""
The file used in this test has a power record with a missing timestamp.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug(
'===== START TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS ====='
)
with open(os.path.join(RESOURCE_PATH, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 118
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles,
'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug(
'===== END TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS ====='
)
def launch(self):
"""
Launches the simulator process as indicated by _COMMAND.
@return (rsn_oms, uri) A pair with the CIOMSSimulator instance and the
associated URI to establish connection with it.
"""
log.debug("[OMSim] Launching: %s", _COMMAND)
self._process = self._spawn(_COMMAND)
if not self._process or not self.poll():
msg = "[OMSim] Failed to launch simulator: %s" % _COMMAND
log.error(msg)
raise Exception(msg)
log.debug("[OMSim] process started, pid: %s", self.getpid())
# give it some time to start up
sleep(5)
# get URI:
uri = None
with open("logs/rsn_oms_simulator.yml", buffering=1) as f:
# we expect one of the first few lines to be of the form:
# rsn_oms_simulator_uri=xxxx
# where xxxx is the uri -- see oms_simulator_server.
while uri is None:
line = f.readline()
if line.index("rsn_oms_simulator_uri=") == 0:
uri = line[len("rsn_oms_simulator_uri="):].strip()
self._rsn_oms = CIOMSClientFactory.create_instance(uri)
return self._rsn_oms, uri
def _build_parsed_values(self):
"""
Take the velocity header data sample format and parse it into
values with appropriate tags.
@throws SampleException If there is a problem with sample creation
"""
log.debug('VectorVelocityHeaderDataParticle: raw data =%r', self.raw_data)
try:
unpack_string = '<4s6sH8B20sH'
sync, timestamp, number_of_records, noise1, noise2, noise3, _, correlation1, correlation2, correlation3, _,\
_, cksum = struct.unpack(unpack_string, self.raw_data)
if not validate_checksum('<20H', self.raw_data):
log.warn("Failed checksum in %s from instrument (%r)", self._data_particle_type, self.raw_data)
self.contents[DataParticleKey.QUALITY_FLAG] = DataParticleValue.CHECKSUM_FAILED
timestamp = common.convert_time(timestamp)
self.set_internal_timestamp((timestamp-datetime(1900, 1, 1)).total_seconds())
except Exception as e:
log.error('Error creating particle vel3d_cd_data_header, raw data: %r', self.raw_data)
raise SampleException(e)
result = [{VID: VectorVelocityHeaderDataParticleKey.TIMESTAMP, VAL: str(timestamp)},
{VID: VectorVelocityHeaderDataParticleKey.NUMBER_OF_RECORDS, VAL: number_of_records},
{VID: VectorVelocityHeaderDataParticleKey.NOISE1, VAL: noise1},
{VID: VectorVelocityHeaderDataParticleKey.NOISE2, VAL: noise2},
{VID: VectorVelocityHeaderDataParticleKey.NOISE3, VAL: noise3},
{VID: VectorVelocityHeaderDataParticleKey.CORRELATION1, VAL: correlation1},
{VID: VectorVelocityHeaderDataParticleKey.CORRELATION2, VAL: correlation2},
{VID: VectorVelocityHeaderDataParticleKey.CORRELATION3, VAL: correlation3}]
log.debug('VectorVelocityHeaderDataParticle: particle=%s', result)
return result
def test_happy_path(self):
"""
Read files and verify that all expected particles can be read.
Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug('===== START TEST HAPPY PATH =====')
with open(os.path.join(RESOURCE_PATH, 'happy_path.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = NUM_EXPECTED_PARTICLES = 10
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "happy_path.yml", RESOURCE_PATH)
self.assertEqual(self._exception_occurred, False)
log.debug('===== END TEST HAPPY PATH =====')
def test_invalid_record_type(self):
"""
The file used in this test has a record type that does not match any of the expected record types.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST INVALID RECORD TYPE =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_record_type.txt'),
'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 9
parser = Pco2wAbcParser(self._parser_config, file_handle,
self.exception_callback, None, None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "invalid_record_type.yml",
RESOURCE_PATH)
self.assertEqual(self._exception_occurred, True)
log.debug('===== END TEST INVALID RECORD TYPE =====')
def test_invalid_record_type(self):
"""
The file used in this test has a record type that does not match any of the expected record types.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST INVALID RECORD TYPE =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_record_type.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 9
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "invalid_record_type.yml", RESOURCE_PATH)
self.assertEqual(self._exception_occurred, True)
log.debug('===== END TEST INVALID RECORD TYPE =====')
def test_no_particles(self):
"""
Verify that no particles are produced if the input file
has no instrument records.
"""
log.debug('===== START TEST NO PARTICLES =====')
with open(os.path.join(RESOURCE_PATH, 'no_particles.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 0
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assertEqual(self._exception_occurred, False)
log.debug('===== END TEST NO PARTICLES =====')
def test_power_record_missing_timestamp(self):
"""
The file used in this test has a power record with a missing timestamp.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST POWER RECORD MISSING TIMESTAMP =====')
with open(
os.path.join(RESOURCE_PATH,
'power_record_missing_timestamp.txt'),
'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 9
parser = Pco2wAbcParser(self._parser_config, file_handle,
self.exception_callback, None, None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles,
"power_record_missing_timestamp.yml",
RESOURCE_PATH)
self.assertEqual(self._exception_occurred, True)
log.debug('===== END TEST POWER RECORD MISSING TIMESTAMP =====')
def test_jumbled(self):
"""
Read files and verify that all expected particles can be read.
This particular data file has the velocity data records
preceded by the diagnostics records, a situation not likely
to occur on a deployed instrument but anything is possible!
The logic in the parser will not produce an instrument metadata
particle (configuration data) until it encounters a velocity or
a diagnostics record. Assumes that all the configuration records are
at the beginning of the file. This is reasonable as the instrument is
configured before being deployed. So the config records would be stored
first. Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug('===== START TEST SIMPLE NOT IN ORDER =====')
# Test the telemetered version
with open(os.path.join(RESOURCE_PATH, 'jumbled_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'jumbled_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST SIMPLE NOT IN ORDER =====')
def test_hardware_config_bad_checksum(self):
"""
The file used in this test has a bad checksum in the hardware configuration record.
Instrument metadata will still be produced but the fields from
the hardware config will NOT be included.
"""
log.debug('===== START TEST HARDWARE CONFIG BAD CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'bad_checksum_in_hardware_config_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'no_hardware_config_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST HARDWARE CONFIG BAD CHECKSUM =====')
def test_no_particles(self):
"""
The file used in this test only contains DCL Logging records.
Verify that no particles are produced if the input file
has no instrument data records, i.e., they just contain DCL Logging records.
"""
log.debug('===== START TEST NO PARTICLES =====')
with open(os.path.join(RESOURCE_PATH, 'no_particles.log'), 'r') as file_handle:
num_particles_to_request = 2
num_expected_particles = 0
parser = Pco2wAbcDclParser(self._recovered_parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST NO PARTICLES =====')
def test_bad_diagnostic_checksum(self):
"""
The file used in this test has a record with a bad checksum.
This results in 71 particles being retrieved instead of 72.
"""
log.debug('===== START TEST FOUND BAD DIAGNOSTICS CHECKSUM =====')
with open(
os.path.join(
RESOURCE_PATH,
'bad_diag_checksum_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'too_few_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST FOUND BAD DIAGNOSTICS CHECKSUM =====')
def test_missing_all_config(self):
"""
The file used in this test has no user configuration record.
Instrument metadata will still be produced but the fields from
the user config will NOT be included.
"""
log.debug('===== START TEST MISSING ALL CONFIG RECORDS =====')
with open(
os.path.join(RESOURCE_PATH,
'no_config_recs_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'no_config_recs_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST MISSING ALL CONFIG RECORDS =====')
def test_truncated_file(self):
"""
The file used in this test has a malformed (too short) record at
the end of the file.This results in 71 particles being retrieved
instead of 72.
"""
log.debug('===== START TEST FOUND TRUNCATED FILE =====')
with open(
os.path.join(RESOURCE_PATH,
'truncated_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'truncated_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST FOUND TRUNCATED FILE =====')
def test_invalid_record_id(self):
"""
The file used in this test has one record with an invalid ID byte.
This results in 71 particles being retrieved instead of 72.
"""
log.debug('===== START TEST INVALID RECORD ID =====')
with open(
os.path.join(RESOURCE_PATH,
'bad_id_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'bad_id_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST INVALID RECORD ID =====')
def test_invalid_sync_byte(self):
"""
The file used in this test has extra bytes between records which need to be skipped
in order to process the correct number of particles. All records are still processed.
"""
log.debug('===== START TEST INVALID SYNC BYTE =====')
with open(
os.path.join(RESOURCE_PATH,
'extra_bytes_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST INVALID SYNC BYTE =====')
def test_invalid_header_fields(self):
"""
The header in the file used in this test has in invalid Voltage and Number of Samples Written.
A metadata particle will still get created, but there will be None in some of the parameters
(an exception will be generated).
"""
log.debug('===== START TEST INVALID HEADER FIELDS =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_header_fields.DAT'), O_MODE) as file_handle:
num_particles_to_request = 5
num_expected_particles = 5
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "invalid_header_fields_rec.yml", RESOURCE_PATH)
self.assert_(isinstance(self.exception_callback_value[0], UnexpectedDataException))
log.debug('===== END TEST INVALID HEADER FIELDS =====')
def test_blank_line(self):
"""
Read file and verify that all expected particles can be read.
Verify that the contents of the particles are correct. There are
blank lines interspersed in the file. This test verifies that
these blank lines do not adversely affect the parser. Only one
test is run as the content of the input files is the same for
recovered or telemetered.
"""
log.debug('===== START TEST BLANK LINE =====')
num_particles_to_request = 25
num_expected_particles = 20
with open(os.path.join(RESOURCE_PATH, '20141207sbl.pwrsys.log'), 'rU') as file_handle:
parser = FuelCellEngDclParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
log.debug('===== END TEST BLANK LINE =====')
def parse_file(self):
"""
Parse the *.mp4 file.
"""
match = FILE_PATH_MATCHER.match(self._stream_handle.name)
if match:
file_datetime = match.group('Date') + match.group('Time')
time_stamp = ntplib.system_to_ntp_time(
utilities.formatted_timestamp_utc_time(file_datetime,
TIMESTAMP_FORMAT))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(CamhdAInstrumentDataParticle,
None,
match.group('Path'),
internal_timestamp=time_stamp)
log.debug('Parsed particle: %s', particle.generate_dict())
self._record_buffer.append(particle)
else:
# Files retrieved from the instrument should always match the timestamp naming convention
self.recov_exception_callback(
"Unable to extract file time from input file name: %s."
"Expected format REFDES-YYYYmmddTHHMMSSZ.mp4" %
self._stream_handle.name)
def test_simple(self):
"""
Read files and verify that all expected particles can be read.
Verify that the contents of the particles are correct.
This is the happy path test.
"""
log.debug('===== START TEST SIMPLE =====')
# Test the telemetered version
with open(os.path.join(RESOURCE_PATH, 'VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST SIMPLE =====')
def test_too_many_diagnostics_records(self):
"""
The file used in this test has 21 diagnostics records in the second set.
Twenty are expected. The records are all still processed.
The error is simply noted.
"""
log.debug('===== START TEST TOO MANY DIAGNOSTICS RECORDS =====')
with open(
os.path.join(RESOURCE_PATH,
'too_many_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 73
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'too_many_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST TOO MANY DIAGNOSTICS RECORDS =====')
def test_happy_path(self):
"""
Read files and verify that all expected particles can be read.
Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug('===== START TEST HAPPY PATH =====')
with open(os.path.join(RESOURCE_PATH, 'happy_path.txt'),
'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = NUM_EXPECTED_PARTICLES = 10
parser = Pco2wAbcParser(self._parser_config, file_handle,
self.exception_callback, None, None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "happy_path.yml", RESOURCE_PATH)
self.assertEqual(self._exception_occurred, False)
log.debug('===== END TEST HAPPY PATH =====')
def test_schema_generation(self):
self.maxDiff = None
result = self.param_dict.generate_dict()
json_result = json.dumps(result, indent=4, sort_keys=True)
log.debug("Expected: %s", self.target_schema)
log.debug("Result: %s", json_result)
self.assertEqual(result, self.target_schema)
def stop_launched_simulator(cls):
"""
Utility to stop the process launched with launch_simulator.
The stop is attempted a couple of times in case of errors (with a few
seconds of sleep in between).
@return None if process seems to have been stopped properly.
Otherwise the exception of the last attempt to stop it.
"""
if cls._sim_process:
sim_proc, cls._sim_process = cls._sim_process, None
attempts = 3
attempt = 0
while attempt <= attempts:
attempt += 1
log.debug("[OMSim] stopping launched simulator (attempt=%d) ...", attempt)
try:
sim_proc.stop()
log.debug("[OMSim] simulator process seems to have stopped properly")
return None
except Exception as ex:
if attempt < attempts:
sleep(10)
else:
log.warn("[OMSim] error while stopping simulator process: %s", ex)
return ex
def test_power_record_missing_timestamp(self):
"""
The file used in this test has a power record with a missing timestamp.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST POWER RECORD MISSING TIMESTAMP =====')
with open(os.path.join(RESOURCE_PATH, 'power_record_missing_timestamp.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 9
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "power_record_missing_timestamp.yml", RESOURCE_PATH)
self.assertEqual(self._exception_occurred, True)
log.debug('===== END TEST POWER RECORD MISSING TIMESTAMP =====')
def test_real_file(self):
"""
Verify that the correct number of particles are generated
from a real file.
"""
log.debug('===== START TEST REAL FILE =====')
with open(os.path.join(RESOURCE_PATH, 'SAMI_C0069_300614.txt'),
'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 2500
NUM_EXPECTED_PARTICLES = 2063
parser = Pco2wAbcParser(self._parser_config, file_handle,
self.exception_callback, None, None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
log.info(len(particles))
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assertEqual(self._exception_occurred, False)
log.debug('===== END TEST REAL FILE =====')
def test_real_file(self):
"""
Verify that the correct number of particles are generated
from a real file.
"""
log.debug('===== START TEST REAL FILE =====')
with open(os.path.join(RESOURCE_PATH, 'SAMI_C0069_300614.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 2500
NUM_EXPECTED_PARTICLES = 2063
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
log.info(len(particles))
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assertEqual(self._exception_occurred, False)
log.debug('===== END TEST REAL FILE =====')
def test_no_science_particles(self):
"""
The file used in this test only contains header and footer records.
Verify that no science (pH or Control) particles are produced if the input file
has no pH data records or control data, i.e., they just contain header and footer records.
In this case only the metadata particle will get created.
"""
log.debug('===== START TEST NO SCIENCE PARTICLES =====')
with open(os.path.join(RESOURCE_PATH, 'header_and_footer_only.DAT'), O_MODE) as file_handle:
num_particles_to_request = 2
num_expected_particles = 1
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "header_and_footer_only_rec.yml", RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST NO SCIENCE PARTICLES =====')
def test_invalid_metadata_timestamp(self):
"""
The file used in this test has error in the timestamp for the first metadata record.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST INVALID METADATA TIMESTAMP =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_metadata_timestamp.txt'), 'r') as file_handle:
NUM_PARTICLES_TO_REQUEST = 10
NUM_EXPECTED_PARTICLES = 9
parser = Pco2wAbcParser(self._parser_config,
file_handle,
self.exception_callback,
None,
None)
particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)
self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)
self.assert_particles(particles, "invalid_metadata_timestamp.yml", RESOURCE_PATH)
self.assertEqual(self._exception_occurred, True)
log.debug('===== END TEST INVALID METADATA TIMESTAMP =====')
def test_invalid_checksum(self):
"""
The first record in the file used in this test has in invalid checksum. An instrument particle will
still get created, but the passed_checksum parameter will be 0 (no warning or error msg generated).
"""
log.debug('===== START TEST INVALID CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'invalid_checksum.DAT'), O_MODE) as file_handle:
num_particles_to_request = 5
num_expected_particles = 5
parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, "invalid_checksum_rec.yml", RESOURCE_PATH)
# No exception should be thrown
self.assertEquals(self.exception_callback_value, [])
log.debug('===== END TEST INVALID CHECKSUM =====')
def stop_launched_simulator(cls):
"""
Utility to stop the process launched with launch_simulator.
The stop is attempted a couple of times in case of errors (with a few
seconds of sleep in between).
@return None if process seems to have been stopped properly.
Otherwise the exception of the last attempt to stop it.
"""
if cls._sim_process:
sim_proc, cls._sim_process = cls._sim_process, None
attempts = 3
attempt = 0
while attempt <= attempts:
attempt += 1
log.debug(
"[OMSim] stopping launched simulator (attempt=%d) ...",
attempt)
try:
sim_proc.stop()
log.debug(
"[OMSim] simulator process seems to have stopped properly"
)
return None
except Exception as ex:
if attempt < attempts:
sleep(10)
else:
log.warn(
"[OMSim] error while stopping simulator process: %s",
ex)
return ex
def test_bad_diag_checksum_21_recs(self):
"""
The file used in this test has a power record with a missing timestamp.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS =====')
with open(os.path.join(RESOURCE_PATH, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 118
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS =====')
def test_one(self):
particle_data_hdlr_obj = parse(Config().base_dir(), self.sourceFilePath,
ParticleDataHandler())
log.debug("SAMPLES: %s", particle_data_hdlr_obj._samples)
log.debug("FAILURE: %s", particle_data_hdlr_obj._failure)
self.assertEquals(particle_data_hdlr_obj._failure, False)
def test_one(self):
particle_data_handler = parse(None, self.source_file_path,
ParticleDataHandler())
log.debug("SAMPLES: %s", particle_data_handler._samples)
log.debug("FAILURE: %s", particle_data_handler._failure)
self.assertEquals(particle_data_handler._failure, False)
def setUp(self):
DVR_CONFIG['oms_uri'] = self._dispatch_simulator(oms_uri)
log.debug("DVR_CONFIG['oms_uri'] = %s", DVR_CONFIG['oms_uri'])
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
CIOMSClientFactory.destroy_instance(rsn_oms)
self._plat_driver = RSNPlatformDriver(self.evt_recv)
self._configure()
self._connect()
def hearbeat():
n = 0
while cls._sim_process:
sleep(1)
n += 1
if cls._sim_process and n % 20 == 0:
log.debug("[OMSim] heartbeat sent")
try:
cls._rsn_oms.ping()
except Exception:
pass
log.debug("[OMSim] heartbeat ended")
def destroy_instance(cls, instance):
"""
Destroys an instance created with create_instance.
This is mainly a convenience method to deactivate the simulator when
run in embedded form.
"""
cls._inst_count -= 1
if isinstance(instance, CIOMSSimulator):
instance._deactivate_simulator()
log.debug("Embedded CIOMSSimulator instance destroyed")
# else: nothing needed to do.
log.debug("destroy_instance: _inst_count = %d", cls._inst_count)
def test_partial_configuration(self):
"""
Attempt to build a parser with a bad configuration.
"""
log.debug('===== START TEST PARTIAL CONFIGURATION =====')
with open(os.path.join(RESOURCE_PATH, 'VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
with self.assertRaises(ConfigurationException):
parser = VelptAbParser(self._incomplete_parser_config,
file_handle,
self.exception_callback)
log.debug('===== END TEST PARTIAL CONFIGURATION =====')
def sieve_function(raw_data):
""" The method that splits samples
"""
return_list = []
pattern = r'SATPAR(?P<sernum>\d{4}),(?P<timer>\d{1,7}.\d\d),(?P<counts>\d{10}),(?P<checksum>\d{1,3})'
regex = re.compile(pattern)
for match in regex.finditer(raw_data):
return_list.append((match.start(), match.end()))
log.debug("Sieving: %s...%s",
raw_data[match.start():match.start()+5],
raw_data[match.end()-5:match.end()])
return return_list
def sieve_function(cls, raw_data):
"""
The method that detects data sample structures from instrument
Should be in the format [[structure_sync_bytes, structure_len]*]
"""
return_list = []
sieve_matchers = [VELOCITY_DATA_REGEX]
for matcher in sieve_matchers:
for match in matcher.finditer(raw_data):
return_list.append((match.start(), match.end()))
log.debug("sieve_function: regex found %r", raw_data[match.start():match.end()])
return return_list