e.channel)
ignored_channels.append(e.channel)
continue
try:
msg = lcmtype.decode(e.data)
except:
statusMsg = deleteStatusMsg(statusMsg)
sys.stderr.write("error: couldn't decode msg on channel %s\n" %
e.channel)
continue
msgCount = msgCount + 1
if (msgCount % 5000) == 0:
statusMsg = deleteStatusMsg(statusMsg)
statusMsg = "read % d messages, % d %% done" % (
msgCount, log.tell() / float(log.size()) * 100)
sys.stderr.write(statusMsg)
sys.stderr.flush()
if e.channel in flatteners:
flattener = flatteners[e.channel]
else:
flattener = make_flattener(msg)
flatteners[e.channel] = flattener
data[e.channel] = []
if printFormat:
statusMsg = deleteStatusMsg(statusMsg)
typeStr, fieldCount = make_lcmtype_string(msg)
typeStr.append("%d- log_timestamp" % (fieldCount + 1))
typeStr = "\n#%s %s :\n#[\n#%s\n#]\n" % (e.channel, lcmtype,
def parse_and_save(args, opts={}):
if isinstance(args, (list, tuple)):
fname = args[0]
elif isinstance(args, str):
fname = args
else:
usage()
#default options
lcm_packages = ["botlcm"]
printFname = "stdout"
printFile = sys.stdout
verbose = False
printOutput = False
savePickle = False
printFormat = False
channelsToIgnore = ""
checkIgnore = False
channelsToProcess = ".*"
separator = ' '
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
elif o in ("-p", "--print"):
printOutput = True
elif o in ("-k", "--pickle"):
savePickle = True
elif o in ("-f", "--format"):
printFormat = True
elif o in ("-s", "--separator="):
separator = a
elif o in ("-o", "--outfile="):
outFname = a
printFname = a
elif o in ("-c", "--channelsToProcess="):
channelsToProcess = a
elif o in ("-i", "--ignore="):
channelsToIgnore = a
checkIgnore = True
elif o in ("-l", "--lcm_packages="):
lcm_packages = a.split(",")
else:
assert False, "unhandled option"
try:
outFname
except NameError:
outDir, outFname = os.path.split(os.path.abspath(fname))
if savePickle:
outFname = outDir + "/" + outFname + ".pkl"
else:
outFname = outFname.replace(".", "_")
outFname = outFname.replace("-", "_")
outFname = outDir + "/" + outFname + ".mat"
fullPathName = os.path.abspath(outFname)
dirname = os.path.dirname(fullPathName)
outBaseName = ".".join(os.path.basename(outFname).split(".")[0:-1])
fullBaseName = dirname + "/" + outBaseName
data = {}
print("Searching for LCM types...")
type_db = make_lcmtype_dictionary()
channelsToProcess = re.compile(channelsToProcess)
channelsToIgnore = re.compile(channelsToIgnore)
log = EventLog(fname, "r")
if printOutput:
sys.stderr.write("opened % s, printing output to %s \n" %
(fname, printFname))
if printFname == "stdout":
printFile = sys.stdout
else:
printFile = open(printFname, "w")
else:
sys.stderr.write("opened % s, outputing to % s\n" % (fname, outFname))
ignored_channels = []
msgCount = 0
statusMsg = ""
startTime = 0
# Iterate LCM log file
for e in log:
if msgCount == 0:
startTime = e.timestamp
if e.channel in ignored_channels:
continue
if ((checkIgnore and channelsToIgnore.match(e.channel) and len(channelsToIgnore.match(e.channel).group())==len(e.channel)) \
or (not channelsToProcess.match(e.channel))):
if verbose:
statusMsg = deleteStatusMsg(statusMsg)
sys.stderr.write("ignoring channel %s\n" % e.channel)
ignored_channels.append(e.channel)
continue
packed_fingerprint = e.data[:8]
lcmtype = type_db.get(packed_fingerprint, None)
if not lcmtype:
if verbose:
statusMsg = deleteStatusMsg(statusMsg)
sys.stderr.write(
"ignoring channel %s -not a known LCM type\n" % e.channel)
ignored_channels.append(e.channel)
continue
try:
msg = lcmtype.decode(e.data)
except:
statusMsg = deleteStatusMsg(statusMsg)
sys.stderr.write("error: couldn't decode msg on channel %s\n" %
e.channel)
continue
msgCount = msgCount + 1
if (msgCount % 5000) == 0:
statusMsg = deleteStatusMsg(statusMsg)
statusMsg = "read % d messages, % d %% done" % (
msgCount, log.tell() / float(log.size()) * 100)
sys.stderr.write(statusMsg)
sys.stderr.flush()
msg_to_dict(data, e.channel, msg, statusMsg, verbose,
(e.timestamp - startTime) / 1e6)
deleteStatusMsg(statusMsg)
if not printOutput:
sys.stderr.write("loaded all %d messages, saving to % s\n" %
(msgCount, outFname))
if savePickle:
# Pickle the list/dictonary using the highest protocol available.
output = open(outFname, 'wb')
pickle.dump(data, output, -1)
output.close()
else:
# Matlab format using scipy
if sys.version_info < (2, 6):
scipy.io.mio.savemat(outFname, data)
else:
scipy.io.matlab.mio.savemat(outFname, data, oned_as='row')
mfile = open(dirname + "/" + outBaseName + ".m",
"w",
encoding='utf-8')
loadFunc = """function [d imFnames]={_outBaseName}()
full_fname = '{_outFname}';
fname = '{_fullPathName}';
if (exist(full_fname,'file'))
filename = full_fname;
else
filename = fname;
end
d = load(filename);
""".format(_outBaseName=outBaseName,
_outFname=outFname,
_fullPathName=fullPathName)
print(loadFunc)
mfile.write(unicode(loadFunc))
mfile.close()
imagePoses = {}
imageMeta = {}
imagesWritten =0
for e in log:
if msgCount == 0:
startTime = e.timestamp
if e.channel=="POSE":
lastPose = pose_t.decode(e.data);
elif lastPose.utime<0:
continue
msgCount = msgCount + 1
if (msgCount % 5000) == 0:
statusMsg = deleteStatusMsg(statusMsg)
statusMsg = "read % d messages, % d %% done, %d images written" % (msgCount, log.tell() / float(log.size())*100,imagesWritten)
sys.stderr.write(statusMsg)
sys.stderr.flush()
#if (msgCount>=100000):
#break
isImage = channels.match(e.channel)
if not isImage:
continue
elif not (e.channel in imageChannels):
#first image on this channel... do some setup
imageChannels.append(e.channel)
imageFnames[e.channel] = []
imagePoses[e.channel] = []
imageMeta[e.channel] = []
lcmtype = type_db.find_type(e.data[:8])
if not lcmtype:
if verbose:
sys.stderr.write("ignoring channel %s\n" % e.channel)
ignored_channels.append(e.channel)
continue
try:
msg = lcmtype.decode(e.data)
except:
sys.stderr.write("error: couldn't decode msg on channel %s" % e.channel)
continue
msgCount = msgCount + 1
if (msgCount % 5000) == 0:
statusMsg = deleteStatusMsg(statusMsg)
statusMsg = "read % d messages, % d %% done" % (msgCount, log.tell() / float(log.size())*100)
sys.stderr.write(statusMsg)
sys.stderr.flush()
if e.channel in flatteners:
flattener = flatteners[e.channel]
else:
flattener = make_flattener(msg)
flatteners[e.channel] = flattener
data[e.channel] = []
if printFormat:
statusMsg = deleteStatusMsg(statusMsg)
typeStr, fieldCount = make_lcmtype_string(msg)
typeStr.append("%d- log_timestamp" % (fieldCount + 1))
typeStr = "\n#%s %s :\n#[\n#%s\n#]\n" % (e.channel, lcmtype, "\n#".join(typeStr))
statusMsg = deleteStatusMsg(statusMsg)
sys.stderr.write("ignoring channel %s -not a known LCM type\n" % e.channel)
ignored_channels.append(e.channel)
continue
try:
msg = lcmtype.decode(e.data)
except:
statusMsg = deleteStatusMsg(statusMsg)
sys.stderr.write("error: couldn't decode msg on channel %s\n" % e.channel)
continue
## We were successfully able to decode the message
msgCount = msgCount + 1
if (msgCount % 5000) == 0:
statusMsg = deleteStatusMsg(statusMsg)
statusMsg = "read % d messages, % d %% done" % (msgCount, log.tell() / float(log.size())*100)
sys.stderr.write(statusMsg)
sys.stderr.flush()
## Figure out how to parse this message
if e.channel in flatteners:
flattener = flatteners[e.channel]
else:
flattener = make_flattener(msg)
flatteners[e.channel] = flattener
data[e.channel] = []
if printFormat:
statusMsg = deleteStatusMsg(statusMsg)
typeStr, fieldCount = make_lcmtype_string(msg)
typeStr.append("%d- log_timestamp" % (fieldCount + 1))
imageMeta = {}
imagesWritten = 0
for e in log:
if msgCount == 0:
startTime = e.timestamp
if e.channel == "POSE":
lastPose = pose_t.decode(e.data)
elif lastPose.utime < 0:
continue
msgCount = msgCount + 1
if (msgCount % 5000) == 0:
statusMsg = deleteStatusMsg(statusMsg)
statusMsg = "read % d messages, % d %% done, %d images written" % (
msgCount, log.tell() / float(log.size()) * 100, imagesWritten)
sys.stderr.write(statusMsg)
sys.stderr.flush()
#if (msgCount>=100000):
#break
isImage = channels.match(e.channel)
if not isImage:
continue
elif not (e.channel in imageChannels):
#first image on this channel... do some setup
imageChannels.append(e.channel)
imageFnames[e.channel] = []
imagePoses[e.channel] = []
imageMeta[e.channel] = []
