def split_logs(filename, seconds2split, prefix=None, verbose=False):
"""
This splits an ArduPilot binary dataflash log into each flight segment
Parameters
----------
filename : str
Name of the file to split.
seconds2split: int
number of seconds gap in the file which will cause a file split
"""
log = DFReader_binary(filename)
# save the ehader
header = '\n'.join(log.data.split('\n', 10)[0:9]) + '\n'
first = True
previous_ts = 0
starting_offset = 0
part = 0
if prefix:
outfile = prefix + filename
else:
outfile = filename
# receive messages and check for large gaps
while True:
m = log.recv_msg()
# if m is none we hit the end of the file so we write out the last segment
if m is None:
with open(outfile + "." + str(part), 'w') as fd:
fd.write(header + log.data[starting_offset:])
print(("split {0} into {1} segments".format(filename, part + 1)))
break
elif not first:
timediff = m._timestamp - previous_ts
if timediff > seconds2split:
if verbose:
print(("Breaking at {0} for type{1}".format(
m._timestamp, m.get_type())))
with open(outfile + "." + str(part), 'w') as fd:
before_message = log.offset - len(m.binary)
# if we are in the first message, no need to repeat the header
if part == 0:
fd.write(log.data[starting_offset:before_message])
# repeat the header for other segments
else:
fd.write(header +
log.data[starting_offset:before_message])
starting_offset = before_message
part += 1
else:
first = False
# each timestamp store the previous timestamp
previous_ts = m._timestamp
def split_logs(filename, seconds2split, prefix=None, verbose=False):
"""
This splits an ArduPilot binary dataflash log into each flight segment
Parameters
----------
filename : str
Name of the file to split.
seconds2split: int
number of seconds gap in the file which will cause a file split
"""
log = DFReader_binary(filename)
# save the ehader
header = '\n'.join(log.data.split('\n', 10)[0:9]) + '\n'
first = True
previous_ts = 0
starting_offset = 0
part = 0
if prefix:
outfile = prefix + filename
else:
outfile = filename
# receive messages and check for large gaps
while True:
m = log.recv_msg()
# if m is none we hit the end of the file so we write out the last segment
if m is None:
with open(outfile + "."+str(part), 'w') as fd :
fd.write(header + log.data[starting_offset:])
print "split {0} into {1} segments".format(filename, part+1)
break
elif not first:
timediff = m._timestamp - previous_ts
if timediff > seconds2split:
if verbose:
print "Breaking at {0} for type{1}".format(m._timestamp, m.get_type())
with open(outfile+"."+str(part), 'w') as fd :
before_message = log.offset-len(m.binary)
# if we are in the first message, no need to repeat the header
if part == 0:
fd.write(log.data[starting_offset:before_message])
# repeat the header for other segments
else:
fd.write(header + log.data[starting_offset:before_message])
starting_offset = before_message
part += 1
else:
first = False
# each timestamp store the previous timestamp
previous_ts = m._timestamp
def extractMAVLINK(filename, outfile, msg_types_to_save):
"""
From dataflash file save an HDF5 store of PANDAS dataframes for each msg
Parameters
----------
filename : str
Name of the file to split.
outfile: str
Name of the output file
msg_types_to_save: list
list of string identifiers of Mavlink Messages to put in Pandas Frame
"""
log = DFReader_binary(filename)
last_m = None
num_msgs = 0
out_dict = {}
first = True
init_time = 0
while True:
# we use mavlinks recv_match function to iterate through logs
m = log.recv_match(type=msg_types_to_save)
if m:
timestamp = m._timestamp
if first:
init_time = timestamp
delta = timestamp - init_time
dt = datetime.datetime.utcfromtimestamp(timestamp + NUMLEAPSECONDS)
msg_timestamp_dict = out_dict.get(m.get_type(), {})
msg_timestamp_dict[dt] = m.to_dict()
out_dict[m.get_type()] = msg_timestamp_dict
else:
if os.path.exists(outfile):
print "Unlinking %s, which already exists!" % outfile
os.unlink(outfile)
f = pd.HDFStore(outfile, mode='w')
try:
tabs = [key for key in out_dict.iterkeys()]
for tab in tabs:
attr = out_dict[tab]
f.put(tab, pd.DataFrame(attr))
if f.get(tab).empty:
warnings.warn('%s is empty.' % tab)
finally:
f.close()
break
def extractMAVLINK(filename, outfile, msg_types_to_save):
"""
From dataflash file save an HDF5 store of PANDAS dataframes for each msg
Parameters
----------
filename : str
Name of the file to split.
outfile: str
Name of the output file
msg_types_to_save: list
list of string identifiers of Mavlink Messages to put in Pandas Frame
"""
log = DFReader_binary(filename)
last_m = None
num_msgs = 0
out_dict = {}
first = True
init_time = 0
while True:
# we use mavlinks recv_match function to iterate through logs
m = log.recv_match(type=msg_types_to_save)
if m:
timestamp = m._timestamp
if first:
init_time = timestamp
delta = timestamp - init_time
dt = datetime.datetime.utcfromtimestamp(timestamp + NUMLEAPSECONDS)
msg_timestamp_dict = out_dict.get(m.get_type(), {})
msg_timestamp_dict[dt] = m.to_dict()
out_dict[m.get_type()] = msg_timestamp_dict
else:
if os.path.exists(outfile):
print("Unlinking %s, which already exists!" % outfile)
os.unlink(outfile)
f = pd.HDFStore(outfile, mode='w')
try:
tabs = [key for key in out_dict.iterkeys()]
for tab in tabs:
attr = out_dict[tab]
f.put(tab, pd.DataFrame(attr))
if f.get(tab).empty:
warnings.warn('%s is empty.' % tab)
finally:
f.close()
break
def extractSBP(filename):
extracted_data = []
log = DFReader_binary(filename)
last_m = None
num_msgs = 0
while True:
# we use mavlinks recv_match function to iterate through logs
# and give us the SBR1 or SBR2 message
# SBR1 msgs are the first 64 bytes of any sbp message, or the entire message
# if the message is smaller than 64 bytes
# SBR2 msgs are the next n bytes if the original message is longer than 64 bytes
m = log.recv_match(type=['SBR1', 'SBR2'])
if m is None:
break
bin_data = None
timestamp = None
msg_type = None
sender_id = None
msg_len = None
if last_m != None and last_m.get_type() == 'SBR1' and m.get_type() == 'SBR2':
# If the last message was an SBR1 and the current message is SBR2
# we combine the two into one SBP message
msg_len = last_m.msg_len
timestamp = getattr(last_m, '_time', 0.0)
binary = last_m.binary
bin_data = bytearray(last_m.binary[SBR1_DATASTART:SBR1_DATASTART+64]
+ m.binary[SBR2_DATASTART:SBR2_DATASTART+msg_len-64])
assert len(bin_data) == msg_len, "Length of binary data decoded \
from dataflash does not match msg_len in header"
msg_type = last_m.msg_type
sender_id = last_m.sender_id
last_m = m
elif last_m and last_m.get_type() == 'SBR1' and m.get_type() == 'SBR1':
# If the last message was SBR1 and this one is SBR1, we extract the last one
# and save this one until the next iteration
msg_len = last_m.msg_len
binary = last_m.binary
assert binary, "binary empty"
timestamp = getattr(last_m, '_time', 0.0)
msg_type = last_m.msg_type
sender_id = last_m.sender_id
bin_data = bytearray(last_m.binary[SBR1_DATASTART:SBR1_DATASTART+msg_len])
last_m = m
elif last_m and last_m.get_type() == "SBR2" and m.get_type() == "SBR1":
# just save current message as the last_m.
# We wait until next message received to know whether it is a complete message
last_m = m
else:
# This should only happen on our first iteration
if last_m:
assert num_msgs == 0, "This branch is expected to execute on first message" \
" only. This is a serious logical error in the decoder."
last_m = m
if bin_data != None:
if len(bin_data) != msg_len:
print "Length of SBP message inconsitent for msg_type {0}.".format(msg_type)
print "Expected Length {0}, Actual Length {1}".format(msg_len, len(bin_data))
extracted_data.append((timestamp, msg_type, sender_id, msg_len, bin_data))
num_msgs += 1
print "extracted {0} messages".format(num_msgs)
return extracted_data
def extractSBP(filename):
extracted_data = []
log = DFReader_binary(filename)
last_m = None
num_msgs = 0
while True:
# we use mavlinks recv_match function to iterate through logs
# and give us the SBR1 or SBR2 message
# SBR1 msgs are the first 64 bytes of any sbp message, or the entire message
# if the message is smaller than 64 bytes
# SBR2 msgs are the next n bytes if the original message is longer than 64 bytes
m = log.recv_match(type=['SBR1', 'SBR2'])
if m is None:
break
bin_data = None
timestamp = None
msg_type = None
sender_id = None
msg_len = None
if last_m != None and last_m.get_type() == 'SBR1' and m.get_type(
) == 'SBR2':
# If the last message was an SBR1 and the current message is SBR2
# we combine the two into one SBP message
msg_len = last_m.msg_len
timestamp = getattr(last_m, '_time', 0.0)
binary = last_m.binary
bin_data = bytearray(
last_m.binary[SBR1_DATASTART:SBR1_DATASTART + 64] +
m.binary[SBR2_DATASTART:SBR2_DATASTART + msg_len - 64])
assert len(bin_data) == msg_len, "Length of binary data decoded \
from dataflash does not match msg_len in header"
msg_type = last_m.msg_type
sender_id = last_m.sender_id
last_m = m
elif last_m and last_m.get_type() == 'SBR1' and m.get_type() == 'SBR1':
# If the last message was SBR1 and this one is SBR1, we extract the last one
# and save this one until the next iteration
msg_len = last_m.msg_len
binary = last_m.binary
assert binary, "binary empty"
timestamp = getattr(last_m, '_time', 0.0)
msg_type = last_m.msg_type
sender_id = last_m.sender_id
bin_data = bytearray(last_m.binary[SBR1_DATASTART:SBR1_DATASTART +
msg_len])
last_m = m
elif last_m and last_m.get_type() == "SBR2" and m.get_type() == "SBR1":
# just save current message as the last_m.
# We wait until next message received to know whether it is a complete message
last_m = m
else:
# This should only happen on our first iteration
if last_m:
assert num_msgs == 0, "This branch is expected to execute on first message" \
" only. This is a serious logical error in the decoder."
last_m = m
if bin_data != None:
if len(bin_data) != msg_len:
print "Length of SBP message inconsitent for msg_type {0}.".format(
msg_type)
print "Expected Length {0}, Actual Length {1}".format(
msg_len, len(bin_data))
extracted_data.append(
(timestamp, msg_type, sender_id, msg_len, bin_data))
num_msgs += 1
print "extracted {0} messages".format(num_msgs)
return extracted_data
