def test_long_stream(self):
"""
Test a long stream
"""
file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat')
# Obtain statistics on the test sample file
stat_info = os.stat(file_path)
# Init the state to declare the UNPROCESSED_DATA as the full length of the file
state = {
StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]],
StateKey.IN_PROCESS_DATA: [],
StateKey.FILE_SIZE: stat_info.st_size
}
# Open the file holding the test sample data
stream_handle = open(file_path, 'rb')
self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# Attempt to retrieve 200000 particles
particles = self.parser.get_records(20000)
# There is NOT 20000 in the file. Make sure we obtained 11456 particles.
self.assertTrue(len(particles) == 11456)
# Close the file stream as we don't need it anymore
stream_handle.close()
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat')
# Obtain statistics on the test sample file
stat_info = os.stat(file_path)
# Init the state to declare the UNPROCESSED_DATA as the full length of the file
state = {
StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]],
StateKey.IN_PROCESS_DATA: [],
StateKey.FILE_SIZE: stat_info.st_size
}
# Open the file holding the test sample data
stream_handle = open(file_path, 'rb')
self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# Attempt to retrieve 30 particles
particles = self.parser.get_records(30)
log.info(len(particles))
# Make sure we obtained 30 particles
self.assertTrue(len(particles) == 30)
for particle in particles:
log.info(particle.generate_dict())
# Compare one of the particles to its expected result
self.assert_result(self.test_eng_particle1, particles[28])
# Attempt to retrieve 50 particles
particles = self.parser.get_records(50)
# Make sure we obtained 50 particles
self.assertTrue(len(particles) == 50)
for particle in particles:
log.info(particle.generate_dict())
# Compare one of the particles to its expected result
self.assert_result(self.test_start_particle1, particles[48])
stream_handle.close()
def test_simple(self):
"""
Read test data and pull out 5 data particles. Then examine each particle against its expected result.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH,
'wfp_eng_wfp_sio_mule_small.DAT')
# Obtain statistics on the test sample file
stat_info = os.stat(file_path)
# Init the state to declare the UNPROCESSED_DATA as the full length of the file
state = {
StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]],
StateKey.IN_PROCESS_DATA: [],
StateKey.FILE_SIZE: stat_info.st_size
}
# Open the file holding the test sample data
stream_handle = open(file_path, 'rb')
self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# Attempt to retrieve 5 particles
particles = self.parser.get_records(5)
# Close the file stream as we don't need it anymore
stream_handle.close()
# Make sure we obtained 5 particles
self.assertTrue(len(particles) == 5)
# Obtain the expected 5 samples from a yml file
test_data = self.get_dict_from_yml('first_five.yml')
index = 0
for particle in particles:
log.info(particle.generate_dict())
# Make sure each retrieved sample matches its expected result
self.assert_result(test_data['data'][index], particles[index])
index += 1
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
"""
file_path = os.path.join(RESOURCE_PATH,
'wfp_eng_wfp_sio_mule_small.DAT')
# Obtain statistics on the test sample file
stat_info = os.stat(file_path)
# Init the state to declare the UNPROCESSED_DATA as one WE chunk. Declare the IN_PROCESS_DATA as the one WE
# chunk that has 5 samples with 1 of the samples as returned.
initial_state = {
StateKey.UNPROCESSED_DATA: [[2818, 2992]],
StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 1]],
StateKey.FILE_SIZE: stat_info.st_size
}
# Open the file holding the test sample data
stream_handle = open(file_path, 'rb')
self.parser = WfpEngWfpSioMuleParser(self.config, initial_state,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# Attempt to retrieve 1 particle
particles = self.parser.get_records(1)
# Make sure we obtained 1 particle
self.assertTrue(len(particles) == 1)
# Compare the particle to its expected result
self.assert_result(self.test_status_particle1, particles[0])
stream_handle.close()
def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the parser
"""
config = self._parser_config.get(data_key)
#
# If the key is WFP_ENG_STC_IMODEM, build the WFP parser.
#
if data_key == DataTypeKey.WFP_ENG_STC_IMODEM:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.wfp_eng__stc_imodem_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'status_data_particle_class':
WfpEngStcImodemStatusRecoveredDataParticle,
'start_data_particle_class':
WfpEngStcImodemStartRecoveredDataParticle,
'engineering_data_particle_class':
WfpEngStcImodemEngineeringRecoveredDataParticle
}
})
parser = WfpEngStcImodemParser(
config, parser_state, infile,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
#
# If the key is WFP_ENG_WFP_SIO_MULE, build the WFP SIO Mule parser.
#
elif data_key == DataTypeKey.WFP_ENG_WFP_SIO_MULE:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.wfp_eng_wfp_sio_mule',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'start_time_data_particle_class':
WfpEngWfpSioMuleParserDataStartTimeParticle,
'status_data_particle_class':
WfpEngWfpSioMuleParserDataStatusParticle,
'engineering_data_particle_class':
WfpEngWfpSioMuleParserDataEngineeringParticle
}
})
parser = WfpEngWfpSioMuleParser(
config, parser_state, infile,
lambda state: self._save_parser_state(state, data_key),
self._data_callback, self._sample_exception_callback)
#
# If the key is one that we're not expecting, don't build any parser.
#
else:
raise ConfigurationException(
"Invalid data_key supplied to build parser")
return parser
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
file_path = os.path.join(RESOURCE_PATH, 'node58p3.dat')
# Obtain statistics on the test sample file
stat_info = os.stat(file_path)
# Init the state to declare the UNPROCESSED_DATA as the full length of the file
initial_state = {
StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]],
StateKey.IN_PROCESS_DATA: [],
StateKey.FILE_SIZE: stat_info.st_size
}
# Open the file holding the test sample data
stream_handle = open(file_path, 'rb')
self.parser = WfpEngWfpSioMuleParser(self.config, initial_state,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# Attempt to retrieve 3 particles
particles = self.parser.get_records(3)
# Make sure we obtained 3 particles
self.assertTrue(len(particles) == 3)
# Make sure the state indicates:
# 1) The IN_PROCESSED_DATA includes two chunks with one chunk having 5 samples with 3 returned and the other
# chunk with 20 samples and 0 returned.
# 2) The UNPROCESSED_DATA includes 4 chunks.
self.assert_state(
[[2818, 2982, 5, 3], [4059, 4673, 20, 0]],
[[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]])
# Attempt to retrieve 30 particles
particles = self.parser.get_records(30)
# Make sure we obtained 22 particles
self.assertTrue(len(particles) == 22)
# Make sure the parser's new state indicates that there 3 remaining chunks in the UNPROCESSED_DATA, none of
# which have WE samples.
self.assert_state([], [[4058, 4059], [7423, 7424], [7594, 7623]])
# Create a new state to declare the UNPROCESSED_DATA including 4 chunks. Declare the IN_PROCESS_DATA as
# the two chunks with WE samples one having 5 samples with 3 returned and the other having 20 samples with
# 0 returned.
new_state = {
StateKey.UNPROCESSED_DATA: [[2818, 2982], [4058, 4673],
[7423, 7424], [7594, 7623]],
StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 3], [4059, 4673, 20,
0]],
StateKey.FILE_SIZE:
stat_info.st_size
}
# Re-set the parser's state
self.parser.set_state(new_state)
# Attempt to retrieve 2 particles
particles = self.parser.get_records(2)
# Make sure we obtained 2 particles
self.assertTrue(len(particles) == 2)
# Make sure the state indicates:
# 1) The IN_PROCESSED_DATA includes one chunk with 20 samples and 0 returned.
# 2) The UNPROCESSED_DATA includes 3 chunks within one corresponding to the chunk with WE samples. The
# other two chunks contain invalid data.
self.assert_state([[4059, 4673, 20, 0]],
[[4058, 4673], [7423, 7424], [7594, 7623]])
# Close the file stream as we don't need it anymore
stream_handle.close()
def test_start_stop_set_state_and_resume(self):
"""
Test starting the parser in a state in the middle of processing
"""
file_path = os.path.join(RESOURCE_PATH,
'wfp_eng_wfp_sio_mule_small.DAT')
# Obtain statistics on the test sample file
stat_info = os.stat(file_path)
# Init the state to declare the UNPROCESSED_DATA as the full length of the file
initial_state = {
StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]],
StateKey.IN_PROCESS_DATA: [],
StateKey.FILE_SIZE: stat_info.st_size
}
# Open the file holding the test sample data
stream_handle = open(file_path, 'rb')
self.parser = WfpEngWfpSioMuleParser(self.config, initial_state,
stream_handle,
self.state_callback,
self.pub_callback,
self.exception_callback)
# Attempt to retrieve 2 particles
particles = self.parser.get_records(2)
# Make sure we obtained 2 particles
self.assertTrue(len(particles) == 2)
# Create a new state to declare the UNPROCESSED_DATA as one chunk. Declare the IN_PROCESS_DATA as one chunk
# that has 5 samples with 4 of the samples as returned.
new_state = {
StateKey.UNPROCESSED_DATA: [[2818, 2992]],
StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 4]],
StateKey.FILE_SIZE: stat_info.st_size
}
# Re-set the parser's state
self.parser.set_state(new_state)
# Attempt to retrieve 2 particles
particles = self.parser.get_records(2)
# Make sure we only obtained 1 particle since the sample test file only holds one WE chunk with 5 total
# samples with 4 already returned.
self.assertTrue(len(particles) == 1)
# Compare the retrieved particle to its expected result.
self.assert_result(self.test_eng_particle2, particles[0])
# Attempt to retrieve 1 particle
particles = self.parser.get_records(1)
# Make sure we obtained 0 particles
self.assertTrue(len(particles) == 0)
# Close the file stream as we don't need it anymore
stream_handle.close()
