def process_mav(self, mlog, timeshift, flightmode_selections, _flightmodes): """process one file""" self.vars = {} idx = 0 all_false = True for s in flightmode_selections: if s: all_false = False while True: msg = mlog.recv_msg() if msg is None: break if msg.get_type() not in self.msg_types: continue if self.condition: if not mavutil.evaluate_condition(self.condition, mlog.messages): continue try: tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp + timeshift)) except ValueError: # this can happen if the log is corrupt # ValueError: year is out of range break if all_false or len(flightmode_selections) == 0: self.add_data(tdays, msg, mlog.messages, mlog.flightmode) else: if idx < len(_flightmodes) and msg._timestamp >= _flightmodes[idx][2]: idx += 1 elif idx < len(flightmode_selections) and flightmode_selections[idx]: self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
def process_mav(self, mlog, timeshift, flightmode_selections, _flightmodes): '''process one file''' self.vars = {} idx = 0 all_false = True for s in flightmode_selections: if s: all_false = False while True: msg = mlog.recv_msg() if msg is None: break if msg.get_type() not in self.msg_types: continue if self.condition: if not mavutil.evaluate_condition(self.condition, mlog.messages): continue try: tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp+timeshift)) except ValueError: # this can happen if the log is corrupt # ValueError: year is out of range break if all_false or len(flightmode_selections) == 0: self.add_data(tdays, msg, mlog.messages, mlog.flightmode) else: if idx < len(_flightmodes) and msg._timestamp >= _flightmodes[idx][2]: idx += 1 elif (idx < len(flightmode_selections) and flightmode_selections[idx]): self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
def process_mav(self, mlog, flightmode_selections): '''process one file''' self.vars = {} idx = 0 all_false = True for s in flightmode_selections: if s: all_false = False # pre-calc right/left axes self.num_fields = len(self.fields) for i in range(0, self.num_fields): f = self.fields[i] if f.endswith(":2"): self.axes[i] = 2 f = f[:-2] if f.endswith(":1"): self.first_only[i] = True f = f[:-2] self.fields[i] = f # see which fields are simple self.simple_field = [] for i in range(0, self.num_fields): f = self.fields[i] m = re.match('^([A-Z][A-Z0-9_]*)[.]([A-Za-z_][A-Za-z0-9_]*)$', f) if m is None: self.simple_field.append(None) else: self.simple_field.append((m.group(1),m.group(2))) if len(self.flightmode_list) > 0: # prime the timestamp conversion timestamp_to_days(self.flightmode_list[0][1], self.timeshift) try: reset_state_data() except Exception: pass while True: msg = mlog.recv_match(type=self.msg_types) if msg is None: break if msg.get_type() not in self.msg_types: continue if self.condition: if not mavutil.evaluate_condition(self.condition, mlog.messages): continue tdays = timestamp_to_days(msg._timestamp, self.timeshift) if all_false or len(flightmode_selections) == 0: self.add_data(tdays, msg, mlog.messages) else: if idx < len(self.flightmode_list) and msg._timestamp >= self.flightmode_list[idx][2]: idx += 1 elif (idx < len(flightmode_selections) and flightmode_selections[idx]): self.add_data(tdays, msg, mlog.messages)
def process(filename): '''process one logfile''' print(("Processing %s" % filename)) mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, robust_parsing=args.robust) ext = os.path.splitext(filename)[1] isbin = ext in ['.bin', '.BIN'] islog = ext in ['.log', '.LOG'] output = None count = 1 dirname = os.path.dirname(filename) if isbin or islog: extension = "bin" else: extension = "tlog" file_header = '' while True: m = mlog.recv_match() if m is None: break if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]: file_header += m.get_msgbuf() if (isbin or islog) and m.get_type() == 'MSG' and m.Message.startswith( "Ardu"): file_header += m.get_msgbuf() if m.get_type() in ['PARAM_VALUE', 'MISSION_ITEM']: timestamp = getattr(m, '_timestamp', None) file_header += struct.pack('>Q', timestamp * 1.0e6) + m.get_msgbuf() if not mavutil.evaluate_condition(args.condition, mlog.messages): continue if mlog.flightmode.upper() == args.mode.upper(): if output is None: path = os.path.join( dirname, "%s%u.%s" % (args.mode, count, extension)) count += 1 print(("Creating %s" % path)) output = open(path, mode='wb') output.write(file_header) else: if output is not None: output.close() output = None if output and m.get_type() != 'BAD_DATA': timestamp = getattr(m, '_timestamp', None) if not isbin: output.write(struct.pack('>Q', timestamp * 1.0e6)) output.write(m.get_msgbuf())
def process(filename): '''process one logfile''' print("Processing %s" % filename) mlog = mavutil.mavlink_connection(filename, notimestamps=opts.notimestamps, robust_parsing=opts.robust) ext = os.path.splitext(filename)[1] isbin = ext in ['.bin', '.BIN'] islog = ext in ['.log', '.LOG'] output = None count = 1 dirname = os.path.dirname(filename) if isbin or islog: extension = "bin" else: extension = "tlog" file_header = '' while True: m = mlog.recv_match() if m is None: break if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]: file_header += m.get_msgbuf() if (isbin or islog ) and m.get_type() == 'MSG' and m.Message.startswith("Ardu"): file_header += m.get_msgbuf() if m.get_type() in ['PARAM_VALUE', 'MISSION_ITEM']: timestamp = getattr(m, '_timestamp', None) file_header += struct.pack('>Q', timestamp * 1.0e6) + m.get_msgbuf() if not mavutil.evaluate_condition(opts.condition, mlog.messages): continue if mlog.flightmode.upper() == opts.mode.upper(): if output is None: path = os.path.join(dirname, "%s%u.%s" % (opts.mode, count, extension)) count += 1 print("Creating %s" % path) output = open(path, mode='wb') output.write(file_header) else: if output is not None: output.close() output = None if output and m.get_type() != 'BAD_DATA': timestamp = getattr(m, '_timestamp', None) if not isbin: output.write(struct.pack('>Q', timestamp * 1.0e6)) output.write(m.get_msgbuf())
def process_mav(self, mlog, flightmode_selections): '''process one file''' self.vars = {} idx = 0 all_false = True for s in flightmode_selections: if s: all_false = False # pre-calc right/left axes self.num_fields = len(self.fields) for i in range(0, self.num_fields): f = self.fields[i] if f.endswith(":2"): self.axes[i] = 2 f = f[:-2] if f.endswith(":1"): self.first_only[i] = True f = f[:-2] self.fields[i] = f # see which fields are simple self.simple_field = [] for i in range(0, self.num_fields): f = self.fields[i] m = re.match('^([A-Z][A-Z0-9_]*)[.]([A-Za-z_][A-Za-z0-9_]*)$', f) if m is None: self.simple_field.append(None) else: self.simple_field.append((m.group(1),m.group(2))) if len(self.flightmode_list) > 0: # prime the timestamp conversion self.timestamp_to_days(self.flightmode_list[0][1]) while True: msg = mlog.recv_match(type=self.msg_types) if msg is None: break if msg.get_type() not in self.msg_types: continue if self.condition: if not mavutil.evaluate_condition(self.condition, mlog.messages): continue tdays = self.timestamp_to_days(msg._timestamp) if all_false or len(flightmode_selections) == 0: self.add_data(tdays, msg, mlog.messages) else: if idx < len(self.flightmode_list) and msg._timestamp >= self.flightmode_list[idx][2]: idx += 1 elif (idx < len(flightmode_selections) and flightmode_selections[idx]): self.add_data(tdays, msg, mlog.messages)
def recv_match(self, condition=None, type=None, blocking=False): '''recv the next message that matches the given condition type can be a string or a list of strings''' if type is not None and not isinstance(type, list): type = [type] while True: m = self.recv_msg() if m is None: return None if type is not None and not m.get_type() in type: continue if not mavutil.evaluate_condition(condition, self.messages): continue return m
def process_mav(self, mlog, timeshift): '''process one file''' self.vars = {} while True: msg = mlog.recv_msg() if msg is None: break if msg.get_type() not in self.msg_types: continue if self.condition: if not mavutil.evaluate_condition(self.condition, mlog.messages): continue tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp+timeshift)) self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
def process(filename): '''process one logfile''' print("Processing %s" % filename) types = args.types if types is not None: types = types.split(',') mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps) base, ext = os.path.splitext(filename) output = None count = 1 dirname = os.path.dirname(filename) extension = "tlog" file_header = bytearray() messages = [] # dictionary of outputs by sysid output = {} while True: m = mlog.recv_match(type=types) if m is None: break if args.condition and not mavutil.evaluate_condition( args.condition, mlog.messages): continue sysid = m.get_srcSystem() if not sysid in output: fname = "%s-%u.%s" % (base, sysid, extension) print("Creating %s" % fname) output[sysid] = open(fname, mode='wb') if output[sysid] and m.get_type() != 'BAD_DATA': timestamp = getattr(m, '_timestamp', None) output[sysid].write(struct.pack('>Q', int(timestamp * 1.0e6))) output[sysid].write(m.get_msgbuf())
def process_mav(self, mlog, timeshift): '''process one file''' self.vars = {} while True: msg = mlog.recv_msg() if msg is None: break if msg.get_type() not in self.msg_types: continue if self.condition: if not mavutil.evaluate_condition(self.condition, mlog.messages): continue try: tdays = matplotlib.dates.date2num( datetime.datetime.fromtimestamp(msg._timestamp + timeshift)) except ValueError: # this can happen if the log is corrupt # ValueError: year is out of range break self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
m = mlog.recv_match(blocking=args.follow) if m is None: # FIXME: Make sure to output the last CSV message before dropping out of this loop break if output is not None: if (isbin or islog) and m.get_type() == "FMT": output.write(m.get_msgbuf()) continue if (isbin or islog) and (m.get_type() == "PARM" and args.parms): output.write(m.get_msgbuf()) continue if m.get_type() == 'PARAM_VALUE' and args.parms: timestamp = getattr(m, '_timestamp', None) output.write(struct.pack('>Q', timestamp*1.0e6) + m.get_msgbuf()) continue if not mavutil.evaluate_condition(args.condition, mlog.messages): continue if types is not None and m.get_type() not in types and m.get_type() != 'BAD_DATA': continue if m.get_type() == 'BAD_DATA' and m.reason == "Bad prefix": continue # Grab the timestamp. timestamp = getattr(m, '_timestamp', 0.0) # If we're just logging, pack in the timestamp and data into the output file. if output: if not (isbin or islog): output.write(struct.pack('>Q', timestamp*1.0e6))
def check_condition(self, condition): '''check if a condition is true''' return mavutil.evaluate_condition(condition, self.messages)
def __init__(self, bin_file, no_timestamps=False, planner=False, robust=False, condition=None, types=None, nottypes=None, dialect="ardupilotmega", zero_time_base=False, source_system=None, source_component=None, link=None, mav10=False): ''' parser.add_argument("--no-timestamps", dest="notimestamps", action='store_true', help="Log doesn't have timestamps") parser.add_argument("--planner", action='store_true', help="use planner file format") parser.add_argument("--robust", action='store_true', help="Enable robust parsing (skip over bad data)") parser.add_argument("--condition", default=None, help="select packets by condition") parser.add_argument("--types", default=None, help="types of messages (comma separated with wildcard)") parser.add_argument("--nottypes", default=None, help="types of messages not to include (comma separated with wildcard)") parser.add_argument("--dialect", default="ardupilotmega", help="MAVLink dialect") parser.add_argument("--zero-time-base", action='store_true', help="use Z time base for DF logs") parser.add_argument("--source-system", type=int, default=None, help="filter by source system ID") parser.add_argument("--source-component", type=int, default=None, help="filter by source component ID") parser.add_argument("--link", type=int, default=None, help="filter by comms link ID") parser.add_argument("--mav10", action='store_true', help="parse as MAVLink1") parser.add_argument("log", metavar="LOG") ''' self._parms = None if not mav10: os.environ['MAVLINK20'] = '1' filename = bin_file mlog = mavutil.mavlink_connection(filename, planner_format=planner, notimestamps=no_timestamps, robust_parsing=robust, dialect=dialect, zero_time_base=zero_time_base) ext = os.path.splitext(filename)[1] isbin = ext in ['.bin', '.BIN', '.px4log'] islog = ext in ['.log', '.LOG'] # NOTE: "islog" does not mean a tlog istlog = ext in ['.tlog', '.TLOG'] # Track types found available_types = set() types = list(set(types + ['PARM'])) if nottypes is not None: nottypes = nottypes.split(',') def match_type(mtype, patterns): '''return True if mtype matches pattern''' for p in patterns: if fnmatch.fnmatch(mtype, p): return True return False # for DF logs pre-calculate types list match_types = None if types is not None and hasattr(mlog, 'name_to_id'): for k in mlog.name_to_id.keys(): if match_type(k, types): if nottypes is not None and match_type(k, nottypes): continue if match_types is None: match_types = [] match_types.append(k) dfs_dicts = {} self.read_types = types while True: m = mlog.recv_match(blocking=False, type=match_types) if m is None: break if not mavutil.evaluate_condition(condition, mlog.messages): continue if source_system is not None and source_system != m.get_srcSystem( ): continue if source_component is not None and source_component != m.get_srcComponent( ): continue if link is not None and link != m._link: continue if types is not None and m.get_type( ) != 'BAD_DATA' and not match_type(m.get_type(), types): continue if nottypes is not None and match_type(m.get_type(), nottypes): continue # Ignore BAD_DATA messages is the user requested or if they're because of a bad prefix. The # latter case is normally because of a mismatched MAVLink version. if m.get_type() == 'BAD_DATA': continue # Grab the timestamp. timestamp = getattr(m, '_timestamp', 0.0) try: dfs_dicts[m.get_type()] except KeyError: dfs_dicts[m.get_type()] = {} dfs_dicts[m.get_type()]['timestamp'] = [] for field in m.get_fieldnames(): dfs_dicts[m.get_type()][field] = [] dfs_dicts[m.get_type()]['timestamp'].append( getattr(m, '_timestamp', 0.0)) for field in m.get_fieldnames(): dfs_dicts[m.get_type()][field].append(getattr(m, field)) self._dfs = {} for msgType in dfs_dicts.keys(): self._dfs[msgType] = pd.DataFrame(data=dfs_dicts[msgType]) new_cols = [] for val in self._dfs[msgType].columns: if val == 'timestamp': new_cols.append(val) else: new_cols.append(msgType + val) self._dfs[msgType].columns = new_cols mlog.filehandle.close()
def process(filename): '''process one logfile''' print("Processing %s" % filename) mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, robust_parsing=args.robust) ext = os.path.splitext(filename)[1] isbin = ext in ['.bin', '.BIN'] islog = ext in ['.log', '.LOG'] output = None count = 1 dirname = os.path.dirname(filename) if isbin or islog: extension = "bin" else: extension = "tlog" file_header = '' messages = [] # we allow a list of modes that map to one mode number. This allows for --mode=AUTO,RTL and consider the RTL as part of AUTO modes = args.mode.upper().split(',') flightmode = None while True: m = mlog.recv_match() if m is None: break if args.link is not None and m._link != args.link: continue mtype = m.get_type() if mtype in messages: if older_message(m, messages[mtype]): continue # we don't use mlog.flightmode as that can be wrong if we are extracting a single link if mtype == 'HEARTBEAT' and m.get_srcComponent() != mavutil.mavlink.MAV_COMP_ID_GIMBAL and m.type != mavutil.mavlink.MAV_TYPE_GCS: flightmode = mavutil.mode_string_v10(m).upper() if mtype == 'MODE': flightmode = mlog.flightmode if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]: file_header += m.get_msgbuf() if (isbin or islog) and m.get_type() == 'MSG' and m.Message.startswith("Ardu"): file_header += m.get_msgbuf() if m.get_type() in ['PARAM_VALUE','MISSION_ITEM']: timestamp = getattr(m, '_timestamp', None) file_header += struct.pack('>Q', timestamp*1.0e6) + m.get_msgbuf() if not mavutil.evaluate_condition(args.condition, mlog.messages): continue if flightmode in modes: if output is None: path = os.path.join(dirname, "%s%u.%s" % (modes[0], count, extension)) count += 1 print("Creating %s" % path) output = open(path, mode='wb') output.write(file_header) else: if output is not None: output.close() output = None if output and m.get_type() != 'BAD_DATA': timestamp = getattr(m, '_timestamp', None) if not isbin: output.write(struct.pack('>Q', timestamp*1.0e6)) output.write(m.get_msgbuf())
def process(filename): '''process one logfile''' print("Processing %s" % filename) mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, robust_parsing=args.robust) ext = os.path.splitext(filename)[1] isbin = ext in ['.bin', '.BIN'] islog = ext in ['.log', '.LOG'] output = None count = 1 dirname = os.path.dirname(filename) if isbin or islog: extension = "bin" else: extension = "tlog" file_header = '' messages = [] # we allow a list of modes that map to one mode number. This allows for --mode=AUTO,RTL and consider the RTL as part of AUTO modes = args.mode.upper().split(',') flightmode = None while True: m = mlog.recv_match() if m is None: break if args.link is not None and m._link != args.link: continue mtype = m.get_type() if mtype in messages: if older_message(m, messages[mtype]): continue # we don't use mlog.flightmode as that can be wrong if we are extracting a single link if mtype == 'HEARTBEAT' and m.get_srcComponent( ) != mavutil.mavlink.MAV_COMP_ID_GIMBAL and m.type != mavutil.mavlink.MAV_TYPE_GCS: flightmode = mavutil.mode_string_v10(m).upper() if mtype == 'MODE': flightmode = mlog.flightmode if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]: file_header += m.get_msgbuf() if (isbin or islog ) and m.get_type() == 'MSG' and m.Message.startswith("Ardu"): file_header += m.get_msgbuf() if m.get_type() in ['PARAM_VALUE', 'MISSION_ITEM']: timestamp = getattr(m, '_timestamp', None) file_header += struct.pack('>Q', timestamp * 1.0e6) + m.get_msgbuf() if not mavutil.evaluate_condition(args.condition, mlog.messages): continue if flightmode in modes: if output is None: path = os.path.join(dirname, "%s%u.%s" % (modes[0], count, extension)) count += 1 print("Creating %s" % path) output = open(path, mode='wb') output.write(file_header) else: if output is not None: output.close() output = None if output and m.get_type() != 'BAD_DATA': timestamp = getattr(m, '_timestamp', None) if not isbin: output.write(struct.pack('>Q', timestamp * 1.0e6)) output.write(m.get_msgbuf())