def missing_message_data(path, channels=CHANNELS):
for record in list_records(path):
glog.info("reading records %s" % record)
reader = RecordReader(record)
for channel in channels:
glog.info("has %d messages" % reader.get_messagenumber(channel))
if reader.get_messagenumber(channel) == 0:
return True
return False
def extract_data(self, record_files, output_path, channels,
extraction_rates):
"""
Extract the desired channel messages if channel_list is specified.
Otherwise extract all sensor calibration messages according to
extraction rate, 10% by default.
"""
# all records have identical sensor channels.
sensor_channels = self.get_sensor_channel_list(record_files[0])
if (len(channels) > 0
and not self.validate_channel_list(channels, sensor_channels)):
print('The input channel list is invalid.')
return False
# Extract all the sensor channels if channel_list is empty(no input arguments).
print(sensor_channels)
if len(channels) == 0:
channels = sensor_channels
# Declare logging variables
process_channel_success_num = len(channels)
process_channel_failure_num = 0
process_msg_failure_num = 0
channel_success = {}
channel_occur_time = {}
channel_output_path = {}
# channel_messages = {}
channel_parsers = {}
channel_message_number = {}
channel_processed_msg_num = {}
for channel in channels:
channel_success[channel] = True
channel_occur_time[channel] = -1
topic_name = channel.replace('/', '_')
channel_output_path[channel] = os.path.join(
output_path, topic_name)
self.process_dir(channel_output_path[channel], operation='create')
channel_parsers[channel] =\
self.build_parser(channel, channel_output_path[channel])
channel_message_number[channel] = 0
for record_file in record_files:
record_reader = RecordReader(record_file)
channel_message_number[
channel] += record_reader.get_messagenumber(channel)
channel_message_number[channel] = channel_message_number[
channel] // extraction_rates[channel]
channel_message_number_total = 0
for num in channel_message_number.values():
channel_message_number_total += num
channel_processed_msg_num = 0
# if channel in SMALL_TOPICS:
# channel_messages[channel] = list()
for record_file in record_files:
record_reader = RecordReader(record_file)
for msg in record_reader.read_messages():
if msg.topic in channels:
# Only care about messages in certain time intervals
msg_timestamp_sec = msg.timestamp / 1e9
if not self.in_range(msg_timestamp_sec,
self.start_timestamp,
self.end_timestamp):
continue
channel_occur_time[msg.topic] += 1
# Extract the topic according to extraction_rate
if channel_occur_time[msg.topic] % extraction_rates[
msg.topic] != 0:
continue
ret = channel_parsers[msg.topic].parse_sensor_message(msg)
channel_processed_msg_num += 1
self.progress.percentage = channel_processed_msg_num / \
channel_message_number_total * 90.0
# Calculate parsing statistics
if not ret:
process_msg_failure_num += 1
if channel_success[msg.topic]:
channel_success[msg.topic] = False
process_channel_failure_num += 1
process_channel_success_num -= 1
log_string = (
'Failed to extract data from channel: '
f'{msg.topic} in record {record_file}')
print(log_string)
self.progress.log_string = log_string
self.writer.write(self.progress)
# traverse the dict, if any channel topic stored as a list
# then save the list as a summary file, mostly binary file
for channel, parser in channel_parsers.items():
self.save_combined_messages_info(parser, channel)
# Logging statics about channel extraction
self.print_and_publish(
(f"Extracted sensor channel number {len(channels)} "
f"from record files: {' '.join(record_files)}"))
self.print_and_publish(
(f'Successfully processed {process_channel_success_num} channels, '
f'and {process_channel_failure_num} was failed.'))
if process_msg_failure_num > 0:
self.print_and_publish(
f'Channel extraction failure number is {process_msg_failure_num}.',
preprocess_table_pb2.Status.FAIL)
return True