def test_record_reader(reader_path):
"""
record reader.
"""
freader = record.RecordReader(reader_path)
time.sleep(1)
print "+" * 80
print "+++begin to read..."
count = 1
for channelname, msg, datatype, timestamp in freader.read_messages():
print "=" * 80
print "read [%d] msg" % count
print "chnanel_name -> %s" % channelname
print "msgtime -> %d" % timestamp
print "msgnum -> %d" % freader.get_messagenumber(channelname)
print "msgtype -> %s" % datatype
# print "pbdesc -> %s" % freader.get_protodesc(channelname)
count = count + 1
print "msg is -> %s" % msg
print "***after parse(if need),the msg is ->"
if datatype == MSG_TYPE:
msg_new = SimpleMessage()
msg_new.ParseFromString(msg)
print msg_new
if datatype == MSG_TYPE_CHATTER:
msg_new = Chatter()
msg_new.ParseFromString(msg)
print msg_new
def test_record_reader(reader_path):
"""
Record reader.
"""
freader = record.RecordReader(reader_path)
time.sleep(1)
print('+' * 80)
print('+++ Begin to read +++')
count = 0
for channel_name, msg, datatype, timestamp in freader.read_messages():
count += 1
print('=' * 80)
print('read [%d] messages' % count)
print('channel_name -> %s' % channel_name)
print('msgtime -> %d' % timestamp)
print('msgnum -> %d' % freader.get_messagenumber(channel_name))
print('msgtype -> %s' % datatype)
print('message is -> %s' % msg)
print('***After parse(if needed),the message is ->')
if datatype == MSG_TYPE:
msg_new = SimpleMessage()
msg_new.ParseFromString(msg)
print(msg_new)
elif datatype == MSG_TYPE_CHATTER:
msg_new = Chatter()
msg_new.ParseFromString(msg)
print(msg_new)
def parse_apollo_record(parse_dict, dest_dict, parser_func):
"""
"""
record_folder_path = parse_dict["record_folder"]
parse_type = parse_dict["parse_type"]
record_files = sorted(os.listdir(parse_dict["record_folder"]))
parse_timestamp = []
parse_mod = import_module(parser_func)
print "=" * 60
print('--------- Parsing data for: ' + parse_type + ' ---------')
for rfile in record_files:
print "=" * 60
print("parsing record file: %s" % rfile)
freader = record.RecordReader(record_folder_path + rfile)
time.sleep(.025)
for channelname, msg, datatype, timestamp in freader.read_messages():
if channelname == dest_dict["channel_name"]:
tstamp = parse_mod.parse_data(channelname, msg,
dest_dict['destination_folder'])
parse_timestamp.append(tstamp)
# write radar-timestamp files
with open(dest_dict["timestamp_file"], 'w+') as f:
for item in parse_timestamp:
f.write("%s\n" % item)
print "=" * 60
print('DONE: records parsed and data saved to: \n ' +
dest_dict['destination_folder'])
print "=" * 60
def print_channel_info(file_path):
freader = record.RecordReader(file_path)
channels = freader.get_channellist()
header_msg = freader.get_headerstring()
header = record_pb2.Header()
header.ParseFromString(header_msg)
print('\n++++++++++++Begin Channel Info Statistics++++++++++++++')
print('-' * 40)
print('record version: [%d:%d]' %
(header.major_version, header.minor_version))
print('record message_number: %s' % str(header.message_number))
print('record file size(Byte): %s' % str(header.size))
print('chunk_number: %d' % header.chunk_number)
print('channel count: %d' % len(channels))
print('-' * 40)
count = 0
for channel in channels:
desc = freader.get_protodesc(channel)
count += 1
print('Channel: %s, count: %d, desc size: %d' %
(channel, count, len(desc)))
# print desc
print "++++++++++++Finish Channel Info Statistics++++++++++++++\n"
def print_channel_info(file_path):
freader = record.RecordReader(file_path)
channels = freader.get_channellist()
header_msg = freader.get_headerstring()
header = record_pb2.Header()
header.ParseFromString(header_msg)
print ""
print "++++++++++++Begin Channel Info Statistics++++++++++++++"
print "-" * 40
print "record version: %d.%d" % (header.major_version,
header.minor_version)
print "record message_number: ", header.message_number
print "record file size(Byte) ", header.size
print "chunk_number: ", header.chunk_number
print "channel counts: ", len(channels)
print "-" * 40
counts = 1
for channel in channels:
desc = freader.get_protodesc(channel)
print "[", counts, "]", "channel name: ", channel, "; desc size is ", len(
desc)
counts = counts + 1
# print desc
print "++++++++++++Finish Channel Info Statistics++++++++++++++"
print ""
def dump_record(in_record, out_dir, start_time, duration, filter_topic):
freader = record.RecordReader(in_record)
time.sleep(1)
seq = 0
start_time = start_time * 1e9
duration = duration * 1e9
update_start = 0
for topic, raw_msg, msg_type, t_sec in freader.read_messages():
if start_time and update_start == 0:
update_start = 1
start_time = int(t_sec + start_time)
print("start_time %s t_sec %s duration %s" %
(start_time, t_sec, duration))
if start_time and t_sec < start_time:
print('time not reach yet')
continue
if start_time and t_sec >= start_time + duration:
print('Done')
break
if topic == '/apollo/sensor/mobileye':
continue
if not filter_topic or topic == filter_topic:
message_file = topic.replace("/", "_")
file_path = os.path.join(out_dir,
str(seq) + message_file + ".pb.txt")
meta_msg = g_message_manager.get_msg_meta_by_topic(topic)
msg = meta_msg.msg_type()
if meta_msg is None:
print('Unknown topic: %s' % topic)
continue
msg.ParseFromString(raw_msg)
write_to_file(file_path, msg)
seq += 1
def process_record_file(args):
"""Read record file and extract the message with specified channels"""
freader = record.RecordReader(args.input_file)
glog.info('#processing record file {}'.format(args.input_file))
time.sleep(1)
output_file = os.path.join(args.output_dir, 'gpsimu.bin')
with open(output_file, 'wb') as outfile:
for channel, message, _type, _timestamp in freader.read_messages():
if channel == args.gps_raw_data_channel:
raw_data = RawData()
raw_data.ParseFromString(message)
outfile.write(raw_data.data)
def test_record_writer_read(self):
"""
unit test of record.
"""
self.assertTrue(cyber.init())
# writer
fwriter = record.RecordWriter()
fwriter.set_size_fileseg(200)
fwriter.set_intervaltime_fileseg(10)
self.assertTrue(fwriter.open(TEST_RECORD_FILE))
fwriter.write_channel(CHAN_1, MSG_TYPE, STR_10B)
fwriter.write_message(CHAN_1, STR_10B, TIME)
self.assertEqual(1, fwriter.get_messagenumber(CHAN_1))
self.assertEqual(MSG_TYPE, fwriter.get_messagetype(CHAN_1))
self.assertEqual(STR_10B, fwriter.get_protodesc(CHAN_1))
fwriter.close()
# reader
fread = record.RecordReader(TEST_RECORD_FILE)
channel_list = fread.get_channellist()
self.assertEqual(1, len(channel_list))
self.assertEqual(CHAN_1, channel_list[0])
for channelname, msg, datatype, timestamp in fread.read_messages():
# print "+++"
# print channelname
# print msg, datatype, timestamp
self.assertEqual(CHAN_1, channelname)
self.assertEqual(STR_10B, msg)
self.assertEqual(TIME, timestamp)
self.assertEqual(1, fread.get_messagenumber(channelname))
self.assertEqual(MSG_TYPE, datatype)
self.assertEqual(MSG_TYPE, fread.get_messagetype(channelname))
# print "pbdesc -> %s" % fread.get_protodesc(channelname)
msg = record_pb2.Header()
header_msg = fread.get_headerstring()
msg.ParseFromString(header_msg)
self.assertEqual(1, msg.major_version)
self.assertEqual(0, msg.minor_version)
self.assertEqual(1, msg.chunk_number)
self.assertEqual(1, msg.channel_number)
self.assertTrue(msg.is_complete)
cyber.shutdown()
def test_record_reader(reader_path):
"""
record reader.
"""
freader = record.RecordReader(reader_path)
time.sleep(1)
print "+"*80
print "+++begin to read..."
count = 1
for channelname, msg, datatype, timestamp in freader.read_messages():
print "="*80
print "read [%d] msg" % count
print "chnanel_name -> %s" % channelname
print "msg -> %s" % msg
print "msgtime -> %d" % timestamp
print "msgnum -> %d" % freader.get_messagenumber(channelname)
print "msgtype -> %s" % datatype
# print "pbdesc -> %s" % freader.get_protodesc(channelname)
count = count + 1
def extract_record(in_record, output):
freader = record.RecordReader(in_record)
print("begin to extract from record {}".format(in_record))
time.sleep(1)
seq = 0
localization_topic = '/apollo/localization/pose'
meta_msg = g_message_manager.get_msg_meta_by_topic(localization_topic)
localization_type = meta_msg.msg_type
for channelname, msg_data, datatype, timestamp in freader.read_messages():
topic = channelname
if topic != localization_topic:
continue
msg = localization_type()
msg.ParseFromString(msg_data)
pose = msg.pose
output.write("%s %s %s %s %s %s %s %s\n" %
(msg.measurement_time, pose.position.x, pose.position.y,
pose.position.z, pose.orientation.qx,
pose.orientation.qy, pose.orientation.qz,
pose.orientation.qw))
print("Finished extracting from record {}".format(in_record))
def test_record_writer_read(self):
"""
unit test of record.
"""
# writer
fwriter = record.RecordWriter()
fwriter.set_size_fileseg(0)
fwriter.set_intervaltime_fileseg(0)
self.assertTrue(fwriter.open(TEST_RECORD_FILE))
fwriter.write_channel(CHAN_1, MSG_TYPE, PROTO_DESC)
fwriter.write_message(CHAN_1, MSG_DATA, TIME)
self.assertEqual(1, fwriter.get_messagenumber(CHAN_1))
self.assertEqual(MSG_TYPE, fwriter.get_messagetype(CHAN_1))
self.assertEqual(PROTO_DESC, fwriter.get_protodesc(CHAN_1))
fwriter.close()
# reader
fread = record.RecordReader(TEST_RECORD_FILE)
channel_list = fread.get_channellist()
self.assertEqual(1, len(channel_list))
self.assertEqual(CHAN_1, channel_list[0])
header = record_pb2.Header()
header.ParseFromString(fread.get_headerstring())
self.assertEqual(1, header.major_version)
self.assertEqual(0, header.minor_version)
self.assertEqual(1, header.chunk_number)
self.assertEqual(1, header.channel_number)
self.assertTrue(header.is_complete)
for channelname, msg, datatype, timestamp in fread.read_messages():
self.assertEqual(CHAN_1, channelname)
self.assertEqual(MSG_DATA, msg)
self.assertEqual(TIME, timestamp)
self.assertEqual(1, fread.get_messagenumber(channelname))
self.assertEqual(MSG_TYPE, datatype)
self.assertEqual(MSG_TYPE, fread.get_messagetype(channelname))
def test_record_trans(reader_path):
"""
Record trans.
"""
fwriter = record.RecordWriter()
if not fwriter.open(TEST_RECORD_FILE):
print('Failed to open record writer!')
return
print('+++ Begin to trans +++')
fread = record.RecordReader(reader_path)
count = 0
for channelname, msg, datatype, timestamp in fread.read_messages():
# print channelname, timestamp, fread.get_messagenumber(channelname)
desc = fread.get_protodesc(channelname)
fwriter.write_channel(channelname, datatype, desc)
fwriter.write_message(channelname, msg, timestamp)
count += 1
print('-' * 80)
print('Message count: %d' % count)
print('Channel info: ')
channel_list = fread.get_channellist()
print('Channel count: %d' % len(channel_list))
print(channel_list)
def dump_record(in_record, out_dir, start_time, duration, filter_topic):
freader = record.RecordReader()
if not freader.open(in_record):
print("Failed to open: %s" % in_record)
return
time.sleep(1)
seq = 0
while not freader.endoffile():
read_msg_succ = freader.read_message()
if not read_msg_succ:
print("Read failed")
return
t_sec = freader.currentmessage_time()
if start_time and t_sec < start_time:
print "not yet reached the start time"
continue
if start_time and t_sec >= start_time + duration:
print "done"
break
topic = freader.currentmessage_channelname()
msg_type = freader.get_messagetype(topic)
if topic == "/apollo/sensor/mobileye":
continue
if not filter_topic or topic == filter_topic:
message_file = topic.replace("/", "_")
file_path = os.path.join(out_dir,
str(seq) + message_file + ".pb.txt")
meta_msg = g_message_manager.get_msg_meta_by_topic(topic)
if meta_msg is None:
print("Unknown topic: %s " % topic)
continue
msg = meta_msg.msg_type()
msg.ParseFromString(freader.current_rawmessage())
write_to_file(file_path, msg)
seq += 1
freader.close()
def test_record_trans(reader_path):
"""
Record trans.
"""
fwriter = record.RecordWriter()
if not fwriter.open(TEST_RECORD_FILE):
print "writer open failed!"
return
print "+++ begin to trans..."
fread = record.RecordReader(reader_path)
count = 0
for channelname, msg, datatype, timestamp in fread.read_messages():
# print channelname, timestamp, fread.get_messagenumber(channelname)
desc = fread.get_protodesc(channelname)
fwriter.write_channel(channelname, datatype, desc)
fwriter.write_message(channelname, msg, timestamp)
count = count + 1
print "-" * 80
print "message count is ", count
print "channel info:"
channel_list = fread.get_channellist()
print "channel count is ", len(channel_list)
print channel_list
