e.channel) ignored_channels.append(e.channel) continue try: msg = lcmtype.decode(e.data) except: statusMsg = deleteStatusMsg(statusMsg) sys.stderr.write("error: couldn't decode msg on channel %s\n" % e.channel) continue msgCount = msgCount + 1 if (msgCount % 5000) == 0: statusMsg = deleteStatusMsg(statusMsg) statusMsg = "read % d messages, % d %% done" % ( msgCount, log.tell() / float(log.size()) * 100) sys.stderr.write(statusMsg) sys.stderr.flush() if e.channel in flatteners: flattener = flatteners[e.channel] else: flattener = make_flattener(msg) flatteners[e.channel] = flattener data[e.channel] = [] if printFormat: statusMsg = deleteStatusMsg(statusMsg) typeStr, fieldCount = make_lcmtype_string(msg) typeStr.append("%d- log_timestamp" % (fieldCount + 1)) typeStr = "\n#%s %s :\n#[\n#%s\n#]\n" % (e.channel, lcmtype,
def parse_and_save(args, opts={}): if isinstance(args, (list, tuple)): fname = args[0] elif isinstance(args, str): fname = args else: usage() #default options lcm_packages = ["botlcm"] printFname = "stdout" printFile = sys.stdout verbose = False printOutput = False savePickle = False printFormat = False channelsToIgnore = "" checkIgnore = False channelsToProcess = ".*" separator = ' ' for o, a in opts: if o == "-v": verbose = True elif o in ("-h", "--help"): usage() elif o in ("-p", "--print"): printOutput = True elif o in ("-k", "--pickle"): savePickle = True elif o in ("-f", "--format"): printFormat = True elif o in ("-s", "--separator="): separator = a elif o in ("-o", "--outfile="): outFname = a printFname = a elif o in ("-c", "--channelsToProcess="): channelsToProcess = a elif o in ("-i", "--ignore="): channelsToIgnore = a checkIgnore = True elif o in ("-l", "--lcm_packages="): lcm_packages = a.split(",") else: assert False, "unhandled option" try: outFname except NameError: outDir, outFname = os.path.split(os.path.abspath(fname)) if savePickle: outFname = outDir + "/" + outFname + ".pkl" else: outFname = outFname.replace(".", "_") outFname = outFname.replace("-", "_") outFname = outDir + "/" + outFname + ".mat" fullPathName = os.path.abspath(outFname) dirname = os.path.dirname(fullPathName) outBaseName = ".".join(os.path.basename(outFname).split(".")[0:-1]) fullBaseName = dirname + "/" + outBaseName data = {} print("Searching for LCM types...") type_db = make_lcmtype_dictionary() channelsToProcess = re.compile(channelsToProcess) channelsToIgnore = re.compile(channelsToIgnore) log = EventLog(fname, "r") if printOutput: sys.stderr.write("opened % s, printing output to %s \n" % (fname, printFname)) if printFname == "stdout": printFile = sys.stdout else: printFile = open(printFname, "w") else: sys.stderr.write("opened % s, outputing to % s\n" % (fname, outFname)) ignored_channels = [] msgCount = 0 statusMsg = "" startTime = 0 # Iterate LCM log file for e in log: if msgCount == 0: startTime = e.timestamp if e.channel in ignored_channels: continue if ((checkIgnore and channelsToIgnore.match(e.channel) and len(channelsToIgnore.match(e.channel).group())==len(e.channel)) \ or (not channelsToProcess.match(e.channel))): if verbose: statusMsg = deleteStatusMsg(statusMsg) sys.stderr.write("ignoring channel %s\n" % e.channel) ignored_channels.append(e.channel) continue packed_fingerprint = e.data[:8] lcmtype = type_db.get(packed_fingerprint, None) if not lcmtype: if verbose: statusMsg = deleteStatusMsg(statusMsg) sys.stderr.write( "ignoring channel %s -not a known LCM type\n" % e.channel) ignored_channels.append(e.channel) continue try: msg = lcmtype.decode(e.data) except: statusMsg = deleteStatusMsg(statusMsg) sys.stderr.write("error: couldn't decode msg on channel %s\n" % e.channel) continue msgCount = msgCount + 1 if (msgCount % 5000) == 0: statusMsg = deleteStatusMsg(statusMsg) statusMsg = "read % d messages, % d %% done" % ( msgCount, log.tell() / float(log.size()) * 100) sys.stderr.write(statusMsg) sys.stderr.flush() msg_to_dict(data, e.channel, msg, statusMsg, verbose, (e.timestamp - startTime) / 1e6) deleteStatusMsg(statusMsg) if not printOutput: sys.stderr.write("loaded all %d messages, saving to % s\n" % (msgCount, outFname)) if savePickle: # Pickle the list/dictonary using the highest protocol available. output = open(outFname, 'wb') pickle.dump(data, output, -1) output.close() else: # Matlab format using scipy if sys.version_info < (2, 6): scipy.io.mio.savemat(outFname, data) else: scipy.io.matlab.mio.savemat(outFname, data, oned_as='row') mfile = open(dirname + "/" + outBaseName + ".m", "w", encoding='utf-8') loadFunc = """function [d imFnames]={_outBaseName}() full_fname = '{_outFname}'; fname = '{_fullPathName}'; if (exist(full_fname,'file')) filename = full_fname; else filename = fname; end d = load(filename); """.format(_outBaseName=outBaseName, _outFname=outFname, _fullPathName=fullPathName) print(loadFunc) mfile.write(unicode(loadFunc)) mfile.close()
imagePoses = {} imageMeta = {} imagesWritten =0 for e in log: if msgCount == 0: startTime = e.timestamp if e.channel=="POSE": lastPose = pose_t.decode(e.data); elif lastPose.utime<0: continue msgCount = msgCount + 1 if (msgCount % 5000) == 0: statusMsg = deleteStatusMsg(statusMsg) statusMsg = "read % d messages, % d %% done, %d images written" % (msgCount, log.tell() / float(log.size())*100,imagesWritten) sys.stderr.write(statusMsg) sys.stderr.flush() #if (msgCount>=100000): #break isImage = channels.match(e.channel) if not isImage: continue elif not (e.channel in imageChannels): #first image on this channel... do some setup imageChannels.append(e.channel) imageFnames[e.channel] = [] imagePoses[e.channel] = [] imageMeta[e.channel] = []
lcmtype = type_db.find_type(e.data[:8]) if not lcmtype: if verbose: sys.stderr.write("ignoring channel %s\n" % e.channel) ignored_channels.append(e.channel) continue try: msg = lcmtype.decode(e.data) except: sys.stderr.write("error: couldn't decode msg on channel %s" % e.channel) continue msgCount = msgCount + 1 if (msgCount % 5000) == 0: statusMsg = deleteStatusMsg(statusMsg) statusMsg = "read % d messages, % d %% done" % (msgCount, log.tell() / float(log.size())*100) sys.stderr.write(statusMsg) sys.stderr.flush() if e.channel in flatteners: flattener = flatteners[e.channel] else: flattener = make_flattener(msg) flatteners[e.channel] = flattener data[e.channel] = [] if printFormat: statusMsg = deleteStatusMsg(statusMsg) typeStr, fieldCount = make_lcmtype_string(msg) typeStr.append("%d- log_timestamp" % (fieldCount + 1)) typeStr = "\n#%s %s :\n#[\n#%s\n#]\n" % (e.channel, lcmtype, "\n#".join(typeStr))
statusMsg = deleteStatusMsg(statusMsg) sys.stderr.write("ignoring channel %s -not a known LCM type\n" % e.channel) ignored_channels.append(e.channel) continue try: msg = lcmtype.decode(e.data) except: statusMsg = deleteStatusMsg(statusMsg) sys.stderr.write("error: couldn't decode msg on channel %s\n" % e.channel) continue ## We were successfully able to decode the message msgCount = msgCount + 1 if (msgCount % 5000) == 0: statusMsg = deleteStatusMsg(statusMsg) statusMsg = "read % d messages, % d %% done" % (msgCount, log.tell() / float(log.size())*100) sys.stderr.write(statusMsg) sys.stderr.flush() ## Figure out how to parse this message if e.channel in flatteners: flattener = flatteners[e.channel] else: flattener = make_flattener(msg) flatteners[e.channel] = flattener data[e.channel] = [] if printFormat: statusMsg = deleteStatusMsg(statusMsg) typeStr, fieldCount = make_lcmtype_string(msg) typeStr.append("%d- log_timestamp" % (fieldCount + 1))
imageMeta = {} imagesWritten = 0 for e in log: if msgCount == 0: startTime = e.timestamp if e.channel == "POSE": lastPose = pose_t.decode(e.data) elif lastPose.utime < 0: continue msgCount = msgCount + 1 if (msgCount % 5000) == 0: statusMsg = deleteStatusMsg(statusMsg) statusMsg = "read % d messages, % d %% done, %d images written" % ( msgCount, log.tell() / float(log.size()) * 100, imagesWritten) sys.stderr.write(statusMsg) sys.stderr.flush() #if (msgCount>=100000): #break isImage = channels.match(e.channel) if not isImage: continue elif not (e.channel in imageChannels): #first image on this channel... do some setup imageChannels.append(e.channel) imageFnames[e.channel] = [] imagePoses[e.channel] = [] imageMeta[e.channel] = []