def missing_message_data(path, channels=CHANNELS): for record in list_records(path): glog.info("reading records %s" % record) reader = RecordReader(record) for channel in channels: glog.info("has %d messages" % reader.get_messagenumber(channel)) if reader.get_messagenumber(channel) == 0: return True return False
def extract_data(self, record_files, output_path, channels, extraction_rates): """ Extract the desired channel messages if channel_list is specified. Otherwise extract all sensor calibration messages according to extraction rate, 10% by default. """ # all records have identical sensor channels. sensor_channels = self.get_sensor_channel_list(record_files[0]) if (len(channels) > 0 and not self.validate_channel_list(channels, sensor_channels)): print('The input channel list is invalid.') return False # Extract all the sensor channels if channel_list is empty(no input arguments). print(sensor_channels) if len(channels) == 0: channels = sensor_channels # Declare logging variables process_channel_success_num = len(channels) process_channel_failure_num = 0 process_msg_failure_num = 0 channel_success = {} channel_occur_time = {} channel_output_path = {} # channel_messages = {} channel_parsers = {} channel_message_number = {} channel_processed_msg_num = {} for channel in channels: channel_success[channel] = True channel_occur_time[channel] = -1 topic_name = channel.replace('/', '_') channel_output_path[channel] = os.path.join( output_path, topic_name) self.process_dir(channel_output_path[channel], operation='create') channel_parsers[channel] =\ self.build_parser(channel, channel_output_path[channel]) channel_message_number[channel] = 0 for record_file in record_files: record_reader = RecordReader(record_file) channel_message_number[ channel] += record_reader.get_messagenumber(channel) channel_message_number[channel] = channel_message_number[ channel] // extraction_rates[channel] channel_message_number_total = 0 for num in channel_message_number.values(): channel_message_number_total += num channel_processed_msg_num = 0 # if channel in SMALL_TOPICS: # channel_messages[channel] = list() for record_file in record_files: record_reader = RecordReader(record_file) for msg in record_reader.read_messages(): if msg.topic in channels: # Only care about messages in certain time intervals msg_timestamp_sec = msg.timestamp / 1e9 if not self.in_range(msg_timestamp_sec, self.start_timestamp, self.end_timestamp): continue channel_occur_time[msg.topic] += 1 # Extract the topic according to extraction_rate if channel_occur_time[msg.topic] % extraction_rates[ msg.topic] != 0: continue ret = channel_parsers[msg.topic].parse_sensor_message(msg) channel_processed_msg_num += 1 self.progress.percentage = channel_processed_msg_num / \ channel_message_number_total * 90.0 # Calculate parsing statistics if not ret: process_msg_failure_num += 1 if channel_success[msg.topic]: channel_success[msg.topic] = False process_channel_failure_num += 1 process_channel_success_num -= 1 log_string = ( 'Failed to extract data from channel: ' f'{msg.topic} in record {record_file}') print(log_string) self.progress.log_string = log_string self.writer.write(self.progress) # traverse the dict, if any channel topic stored as a list # then save the list as a summary file, mostly binary file for channel, parser in channel_parsers.items(): self.save_combined_messages_info(parser, channel) # Logging statics about channel extraction self.print_and_publish( (f"Extracted sensor channel number {len(channels)} " f"from record files: {' '.join(record_files)}")) self.print_and_publish( (f'Successfully processed {process_channel_success_num} channels, ' f'and {process_channel_failure_num} was failed.')) if process_msg_failure_num > 0: self.print_and_publish( f'Channel extraction failure number is {process_msg_failure_num}.', preprocess_table_pb2.Status.FAIL) return True