def process_mav(self, mlog, timeshift, flightmode_selections, _flightmodes):
"""process one file"""
self.vars = {}
idx = 0
all_false = True
for s in flightmode_selections:
if s:
all_false = False
while True:
msg = mlog.recv_msg()
if msg is None:
break
if msg.get_type() not in self.msg_types:
continue
if self.condition:
if not mavutil.evaluate_condition(self.condition, mlog.messages):
continue
try:
tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp + timeshift))
except ValueError:
# this can happen if the log is corrupt
# ValueError: year is out of range
break
if all_false or len(flightmode_selections) == 0:
self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
else:
if idx < len(_flightmodes) and msg._timestamp >= _flightmodes[idx][2]:
idx += 1
elif idx < len(flightmode_selections) and flightmode_selections[idx]:
self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
def process_mav(self, mlog, timeshift, flightmode_selections, _flightmodes):
'''process one file'''
self.vars = {}
idx = 0
all_false = True
for s in flightmode_selections:
if s:
all_false = False
while True:
msg = mlog.recv_msg()
if msg is None:
break
if msg.get_type() not in self.msg_types:
continue
if self.condition:
if not mavutil.evaluate_condition(self.condition, mlog.messages):
continue
try:
tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp+timeshift))
except ValueError:
# this can happen if the log is corrupt
# ValueError: year is out of range
break
if all_false or len(flightmode_selections) == 0:
self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
else:
if idx < len(_flightmodes) and msg._timestamp >= _flightmodes[idx][2]:
idx += 1
elif (idx < len(flightmode_selections) and flightmode_selections[idx]):
self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
def process_mav(self, mlog, flightmode_selections):
'''process one file'''
self.vars = {}
idx = 0
all_false = True
for s in flightmode_selections:
if s:
all_false = False
# pre-calc right/left axes
self.num_fields = len(self.fields)
for i in range(0, self.num_fields):
f = self.fields[i]
if f.endswith(":2"):
self.axes[i] = 2
f = f[:-2]
if f.endswith(":1"):
self.first_only[i] = True
f = f[:-2]
self.fields[i] = f
# see which fields are simple
self.simple_field = []
for i in range(0, self.num_fields):
f = self.fields[i]
m = re.match('^([A-Z][A-Z0-9_]*)[.]([A-Za-z_][A-Za-z0-9_]*)$', f)
if m is None:
self.simple_field.append(None)
else:
self.simple_field.append((m.group(1),m.group(2)))
if len(self.flightmode_list) > 0:
# prime the timestamp conversion
timestamp_to_days(self.flightmode_list[0][1], self.timeshift)
try:
reset_state_data()
except Exception:
pass
while True:
msg = mlog.recv_match(type=self.msg_types)
if msg is None:
break
if msg.get_type() not in self.msg_types:
continue
if self.condition:
if not mavutil.evaluate_condition(self.condition, mlog.messages):
continue
tdays = timestamp_to_days(msg._timestamp, self.timeshift)
if all_false or len(flightmode_selections) == 0:
self.add_data(tdays, msg, mlog.messages)
else:
if idx < len(self.flightmode_list) and msg._timestamp >= self.flightmode_list[idx][2]:
idx += 1
elif (idx < len(flightmode_selections) and flightmode_selections[idx]):
self.add_data(tdays, msg, mlog.messages)
def process(filename):
'''process one logfile'''
print(("Processing %s" % filename))
mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps,
robust_parsing=args.robust)
ext = os.path.splitext(filename)[1]
isbin = ext in ['.bin', '.BIN']
islog = ext in ['.log', '.LOG']
output = None
count = 1
dirname = os.path.dirname(filename)
if isbin or islog:
extension = "bin"
else:
extension = "tlog"
file_header = ''
while True:
m = mlog.recv_match()
if m is None:
break
if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]:
file_header += m.get_msgbuf()
if (isbin or islog) and m.get_type() == 'MSG' and m.Message.startswith(
"Ardu"):
file_header += m.get_msgbuf()
if m.get_type() in ['PARAM_VALUE', 'MISSION_ITEM']:
timestamp = getattr(m, '_timestamp', None)
file_header += struct.pack('>Q',
timestamp * 1.0e6) + m.get_msgbuf()
if not mavutil.evaluate_condition(args.condition, mlog.messages):
continue
if mlog.flightmode.upper() == args.mode.upper():
if output is None:
path = os.path.join(
dirname, "%s%u.%s" %
(args.mode, count, extension))
count += 1
print(("Creating %s" % path))
output = open(path, mode='wb')
output.write(file_header)
else:
if output is not None:
output.close()
output = None
if output and m.get_type() != 'BAD_DATA':
timestamp = getattr(m, '_timestamp', None)
if not isbin:
output.write(struct.pack('>Q', timestamp * 1.0e6))
output.write(m.get_msgbuf())
def process(filename):
'''process one logfile'''
print("Processing %s" % filename)
mlog = mavutil.mavlink_connection(filename,
notimestamps=opts.notimestamps,
robust_parsing=opts.robust)
ext = os.path.splitext(filename)[1]
isbin = ext in ['.bin', '.BIN']
islog = ext in ['.log', '.LOG']
output = None
count = 1
dirname = os.path.dirname(filename)
if isbin or islog:
extension = "bin"
else:
extension = "tlog"
file_header = ''
while True:
m = mlog.recv_match()
if m is None:
break
if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]:
file_header += m.get_msgbuf()
if (isbin or islog
) and m.get_type() == 'MSG' and m.Message.startswith("Ardu"):
file_header += m.get_msgbuf()
if m.get_type() in ['PARAM_VALUE', 'MISSION_ITEM']:
timestamp = getattr(m, '_timestamp', None)
file_header += struct.pack('>Q',
timestamp * 1.0e6) + m.get_msgbuf()
if not mavutil.evaluate_condition(opts.condition, mlog.messages):
continue
if mlog.flightmode.upper() == opts.mode.upper():
if output is None:
path = os.path.join(dirname,
"%s%u.%s" % (opts.mode, count, extension))
count += 1
print("Creating %s" % path)
output = open(path, mode='wb')
output.write(file_header)
else:
if output is not None:
output.close()
output = None
if output and m.get_type() != 'BAD_DATA':
timestamp = getattr(m, '_timestamp', None)
if not isbin:
output.write(struct.pack('>Q', timestamp * 1.0e6))
output.write(m.get_msgbuf())
def process_mav(self, mlog, flightmode_selections):
'''process one file'''
self.vars = {}
idx = 0
all_false = True
for s in flightmode_selections:
if s:
all_false = False
# pre-calc right/left axes
self.num_fields = len(self.fields)
for i in range(0, self.num_fields):
f = self.fields[i]
if f.endswith(":2"):
self.axes[i] = 2
f = f[:-2]
if f.endswith(":1"):
self.first_only[i] = True
f = f[:-2]
self.fields[i] = f
# see which fields are simple
self.simple_field = []
for i in range(0, self.num_fields):
f = self.fields[i]
m = re.match('^([A-Z][A-Z0-9_]*)[.]([A-Za-z_][A-Za-z0-9_]*)$', f)
if m is None:
self.simple_field.append(None)
else:
self.simple_field.append((m.group(1),m.group(2)))
if len(self.flightmode_list) > 0:
# prime the timestamp conversion
self.timestamp_to_days(self.flightmode_list[0][1])
while True:
msg = mlog.recv_match(type=self.msg_types)
if msg is None:
break
if msg.get_type() not in self.msg_types:
continue
if self.condition:
if not mavutil.evaluate_condition(self.condition, mlog.messages):
continue
tdays = self.timestamp_to_days(msg._timestamp)
if all_false or len(flightmode_selections) == 0:
self.add_data(tdays, msg, mlog.messages)
else:
if idx < len(self.flightmode_list) and msg._timestamp >= self.flightmode_list[idx][2]:
idx += 1
elif (idx < len(flightmode_selections) and flightmode_selections[idx]):
self.add_data(tdays, msg, mlog.messages)
def recv_match(self, condition=None, type=None, blocking=False):
'''recv the next message that matches the given condition
type can be a string or a list of strings'''
if type is not None and not isinstance(type, list):
type = [type]
while True:
m = self.recv_msg()
if m is None:
return None
if type is not None and not m.get_type() in type:
continue
if not mavutil.evaluate_condition(condition, self.messages):
continue
return m
def process_mav(self, mlog, timeshift):
'''process one file'''
self.vars = {}
while True:
msg = mlog.recv_msg()
if msg is None:
break
if msg.get_type() not in self.msg_types:
continue
if self.condition:
if not mavutil.evaluate_condition(self.condition, mlog.messages):
continue
tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp+timeshift))
self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
def recv_match(self, condition=None, type=None, blocking=False):
'''recv the next message that matches the given condition
type can be a string or a list of strings'''
if type is not None and not isinstance(type, list):
type = [type]
while True:
m = self.recv_msg()
if m is None:
return None
if type is not None and not m.get_type() in type:
continue
if not mavutil.evaluate_condition(condition, self.messages):
continue
return m
def process(filename):
'''process one logfile'''
print("Processing %s" % filename)
types = args.types
if types is not None:
types = types.split(',')
mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps)
base, ext = os.path.splitext(filename)
output = None
count = 1
dirname = os.path.dirname(filename)
extension = "tlog"
file_header = bytearray()
messages = []
# dictionary of outputs by sysid
output = {}
while True:
m = mlog.recv_match(type=types)
if m is None:
break
if args.condition and not mavutil.evaluate_condition(
args.condition, mlog.messages):
continue
sysid = m.get_srcSystem()
if not sysid in output:
fname = "%s-%u.%s" % (base, sysid, extension)
print("Creating %s" % fname)
output[sysid] = open(fname, mode='wb')
if output[sysid] and m.get_type() != 'BAD_DATA':
timestamp = getattr(m, '_timestamp', None)
output[sysid].write(struct.pack('>Q', int(timestamp * 1.0e6)))
output[sysid].write(m.get_msgbuf())
def process_mav(self, mlog, timeshift):
'''process one file'''
self.vars = {}
while True:
msg = mlog.recv_msg()
if msg is None:
break
if msg.get_type() not in self.msg_types:
continue
if self.condition:
if not mavutil.evaluate_condition(self.condition,
mlog.messages):
continue
try:
tdays = matplotlib.dates.date2num(
datetime.datetime.fromtimestamp(msg._timestamp +
timeshift))
except ValueError:
# this can happen if the log is corrupt
# ValueError: year is out of range
break
self.add_data(tdays, msg, mlog.messages, mlog.flightmode)
m = mlog.recv_match(blocking=args.follow)
if m is None:
# FIXME: Make sure to output the last CSV message before dropping out of this loop
break
if output is not None:
if (isbin or islog) and m.get_type() == "FMT":
output.write(m.get_msgbuf())
continue
if (isbin or islog) and (m.get_type() == "PARM" and args.parms):
output.write(m.get_msgbuf())
continue
if m.get_type() == 'PARAM_VALUE' and args.parms:
timestamp = getattr(m, '_timestamp', None)
output.write(struct.pack('>Q', timestamp*1.0e6) + m.get_msgbuf())
continue
if not mavutil.evaluate_condition(args.condition, mlog.messages):
continue
if types is not None and m.get_type() not in types and m.get_type() != 'BAD_DATA':
continue
if m.get_type() == 'BAD_DATA' and m.reason == "Bad prefix":
continue
# Grab the timestamp.
timestamp = getattr(m, '_timestamp', 0.0)
# If we're just logging, pack in the timestamp and data into the output file.
if output:
if not (isbin or islog):
output.write(struct.pack('>Q', timestamp*1.0e6))
def check_condition(self, condition):
'''check if a condition is true'''
return mavutil.evaluate_condition(condition, self.messages)
def __init__(self,
bin_file,
no_timestamps=False,
planner=False,
robust=False,
condition=None,
types=None,
nottypes=None,
dialect="ardupilotmega",
zero_time_base=False,
source_system=None,
source_component=None,
link=None,
mav10=False):
'''
parser.add_argument("--no-timestamps", dest="notimestamps", action='store_true', help="Log doesn't have timestamps")
parser.add_argument("--planner", action='store_true', help="use planner file format")
parser.add_argument("--robust", action='store_true', help="Enable robust parsing (skip over bad data)")
parser.add_argument("--condition", default=None, help="select packets by condition")
parser.add_argument("--types", default=None, help="types of messages (comma separated with wildcard)")
parser.add_argument("--nottypes", default=None, help="types of messages not to include (comma separated with wildcard)")
parser.add_argument("--dialect", default="ardupilotmega", help="MAVLink dialect")
parser.add_argument("--zero-time-base", action='store_true', help="use Z time base for DF logs")
parser.add_argument("--source-system", type=int, default=None, help="filter by source system ID")
parser.add_argument("--source-component", type=int, default=None, help="filter by source component ID")
parser.add_argument("--link", type=int, default=None, help="filter by comms link ID")
parser.add_argument("--mav10", action='store_true', help="parse as MAVLink1")
parser.add_argument("log", metavar="LOG")
'''
self._parms = None
if not mav10:
os.environ['MAVLINK20'] = '1'
filename = bin_file
mlog = mavutil.mavlink_connection(filename,
planner_format=planner,
notimestamps=no_timestamps,
robust_parsing=robust,
dialect=dialect,
zero_time_base=zero_time_base)
ext = os.path.splitext(filename)[1]
isbin = ext in ['.bin', '.BIN', '.px4log']
islog = ext in ['.log', '.LOG'] # NOTE: "islog" does not mean a tlog
istlog = ext in ['.tlog', '.TLOG']
# Track types found
available_types = set()
types = list(set(types + ['PARM']))
if nottypes is not None:
nottypes = nottypes.split(',')
def match_type(mtype, patterns):
'''return True if mtype matches pattern'''
for p in patterns:
if fnmatch.fnmatch(mtype, p):
return True
return False
# for DF logs pre-calculate types list
match_types = None
if types is not None and hasattr(mlog, 'name_to_id'):
for k in mlog.name_to_id.keys():
if match_type(k, types):
if nottypes is not None and match_type(k, nottypes):
continue
if match_types is None:
match_types = []
match_types.append(k)
dfs_dicts = {}
self.read_types = types
while True:
m = mlog.recv_match(blocking=False, type=match_types)
if m is None:
break
if not mavutil.evaluate_condition(condition, mlog.messages):
continue
if source_system is not None and source_system != m.get_srcSystem(
):
continue
if source_component is not None and source_component != m.get_srcComponent(
):
continue
if link is not None and link != m._link:
continue
if types is not None and m.get_type(
) != 'BAD_DATA' and not match_type(m.get_type(), types):
continue
if nottypes is not None and match_type(m.get_type(), nottypes):
continue
# Ignore BAD_DATA messages is the user requested or if they're because of a bad prefix. The
# latter case is normally because of a mismatched MAVLink version.
if m.get_type() == 'BAD_DATA':
continue
# Grab the timestamp.
timestamp = getattr(m, '_timestamp', 0.0)
try:
dfs_dicts[m.get_type()]
except KeyError:
dfs_dicts[m.get_type()] = {}
dfs_dicts[m.get_type()]['timestamp'] = []
for field in m.get_fieldnames():
dfs_dicts[m.get_type()][field] = []
dfs_dicts[m.get_type()]['timestamp'].append(
getattr(m, '_timestamp', 0.0))
for field in m.get_fieldnames():
dfs_dicts[m.get_type()][field].append(getattr(m, field))
self._dfs = {}
for msgType in dfs_dicts.keys():
self._dfs[msgType] = pd.DataFrame(data=dfs_dicts[msgType])
new_cols = []
for val in self._dfs[msgType].columns:
if val == 'timestamp':
new_cols.append(val)
else:
new_cols.append(msgType + val)
self._dfs[msgType].columns = new_cols
mlog.filehandle.close()
def check_condition(self, condition):
'''check if a condition is true'''
return mavutil.evaluate_condition(condition, self.messages)
def process(filename):
'''process one logfile'''
print("Processing %s" % filename)
mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps,
robust_parsing=args.robust)
ext = os.path.splitext(filename)[1]
isbin = ext in ['.bin', '.BIN']
islog = ext in ['.log', '.LOG']
output = None
count = 1
dirname = os.path.dirname(filename)
if isbin or islog:
extension = "bin"
else:
extension = "tlog"
file_header = ''
messages = []
# we allow a list of modes that map to one mode number. This allows for --mode=AUTO,RTL and consider the RTL as part of AUTO
modes = args.mode.upper().split(',')
flightmode = None
while True:
m = mlog.recv_match()
if m is None:
break
if args.link is not None and m._link != args.link:
continue
mtype = m.get_type()
if mtype in messages:
if older_message(m, messages[mtype]):
continue
# we don't use mlog.flightmode as that can be wrong if we are extracting a single link
if mtype == 'HEARTBEAT' and m.get_srcComponent() != mavutil.mavlink.MAV_COMP_ID_GIMBAL and m.type != mavutil.mavlink.MAV_TYPE_GCS:
flightmode = mavutil.mode_string_v10(m).upper()
if mtype == 'MODE':
flightmode = mlog.flightmode
if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]:
file_header += m.get_msgbuf()
if (isbin or islog) and m.get_type() == 'MSG' and m.Message.startswith("Ardu"):
file_header += m.get_msgbuf()
if m.get_type() in ['PARAM_VALUE','MISSION_ITEM']:
timestamp = getattr(m, '_timestamp', None)
file_header += struct.pack('>Q', timestamp*1.0e6) + m.get_msgbuf()
if not mavutil.evaluate_condition(args.condition, mlog.messages):
continue
if flightmode in modes:
if output is None:
path = os.path.join(dirname, "%s%u.%s" % (modes[0], count, extension))
count += 1
print("Creating %s" % path)
output = open(path, mode='wb')
output.write(file_header)
else:
if output is not None:
output.close()
output = None
if output and m.get_type() != 'BAD_DATA':
timestamp = getattr(m, '_timestamp', None)
if not isbin:
output.write(struct.pack('>Q', timestamp*1.0e6))
output.write(m.get_msgbuf())
def process(filename):
'''process one logfile'''
print("Processing %s" % filename)
mlog = mavutil.mavlink_connection(filename,
notimestamps=args.notimestamps,
robust_parsing=args.robust)
ext = os.path.splitext(filename)[1]
isbin = ext in ['.bin', '.BIN']
islog = ext in ['.log', '.LOG']
output = None
count = 1
dirname = os.path.dirname(filename)
if isbin or islog:
extension = "bin"
else:
extension = "tlog"
file_header = ''
messages = []
# we allow a list of modes that map to one mode number. This allows for --mode=AUTO,RTL and consider the RTL as part of AUTO
modes = args.mode.upper().split(',')
flightmode = None
while True:
m = mlog.recv_match()
if m is None:
break
if args.link is not None and m._link != args.link:
continue
mtype = m.get_type()
if mtype in messages:
if older_message(m, messages[mtype]):
continue
# we don't use mlog.flightmode as that can be wrong if we are extracting a single link
if mtype == 'HEARTBEAT' and m.get_srcComponent(
) != mavutil.mavlink.MAV_COMP_ID_GIMBAL and m.type != mavutil.mavlink.MAV_TYPE_GCS:
flightmode = mavutil.mode_string_v10(m).upper()
if mtype == 'MODE':
flightmode = mlog.flightmode
if (isbin or islog) and m.get_type() in ["FMT", "PARM", "CMD"]:
file_header += m.get_msgbuf()
if (isbin or islog
) and m.get_type() == 'MSG' and m.Message.startswith("Ardu"):
file_header += m.get_msgbuf()
if m.get_type() in ['PARAM_VALUE', 'MISSION_ITEM']:
timestamp = getattr(m, '_timestamp', None)
file_header += struct.pack('>Q',
timestamp * 1.0e6) + m.get_msgbuf()
if not mavutil.evaluate_condition(args.condition, mlog.messages):
continue
if flightmode in modes:
if output is None:
path = os.path.join(dirname,
"%s%u.%s" % (modes[0], count, extension))
count += 1
print("Creating %s" % path)
output = open(path, mode='wb')
output.write(file_header)
else:
if output is not None:
output.close()
output = None
if output and m.get_type() != 'BAD_DATA':
timestamp = getattr(m, '_timestamp', None)
if not isbin:
output.write(struct.pack('>Q', timestamp * 1.0e6))
output.write(m.get_msgbuf())
