def split_logs(filename, seconds2split, prefix=None, verbose=False): """ This splits an ArduPilot binary dataflash log into each flight segment Parameters ---------- filename : str Name of the file to split. seconds2split: int number of seconds gap in the file which will cause a file split """ log = DFReader_binary(filename) # save the ehader header = '\n'.join(log.data.split('\n', 10)[0:9]) + '\n' first = True previous_ts = 0 starting_offset = 0 part = 0 if prefix: outfile = prefix + filename else: outfile = filename # receive messages and check for large gaps while True: m = log.recv_msg() # if m is none we hit the end of the file so we write out the last segment if m is None: with open(outfile + "." + str(part), 'w') as fd: fd.write(header + log.data[starting_offset:]) print(("split {0} into {1} segments".format(filename, part + 1))) break elif not first: timediff = m._timestamp - previous_ts if timediff > seconds2split: if verbose: print(("Breaking at {0} for type{1}".format( m._timestamp, m.get_type()))) with open(outfile + "." + str(part), 'w') as fd: before_message = log.offset - len(m.binary) # if we are in the first message, no need to repeat the header if part == 0: fd.write(log.data[starting_offset:before_message]) # repeat the header for other segments else: fd.write(header + log.data[starting_offset:before_message]) starting_offset = before_message part += 1 else: first = False # each timestamp store the previous timestamp previous_ts = m._timestamp
def split_logs(filename, seconds2split, prefix=None, verbose=False): """ This splits an ArduPilot binary dataflash log into each flight segment Parameters ---------- filename : str Name of the file to split. seconds2split: int number of seconds gap in the file which will cause a file split """ log = DFReader_binary(filename) # save the ehader header = '\n'.join(log.data.split('\n', 10)[0:9]) + '\n' first = True previous_ts = 0 starting_offset = 0 part = 0 if prefix: outfile = prefix + filename else: outfile = filename # receive messages and check for large gaps while True: m = log.recv_msg() # if m is none we hit the end of the file so we write out the last segment if m is None: with open(outfile + "."+str(part), 'w') as fd : fd.write(header + log.data[starting_offset:]) print "split {0} into {1} segments".format(filename, part+1) break elif not first: timediff = m._timestamp - previous_ts if timediff > seconds2split: if verbose: print "Breaking at {0} for type{1}".format(m._timestamp, m.get_type()) with open(outfile+"."+str(part), 'w') as fd : before_message = log.offset-len(m.binary) # if we are in the first message, no need to repeat the header if part == 0: fd.write(log.data[starting_offset:before_message]) # repeat the header for other segments else: fd.write(header + log.data[starting_offset:before_message]) starting_offset = before_message part += 1 else: first = False # each timestamp store the previous timestamp previous_ts = m._timestamp
def extractMAVLINK(filename, outfile, msg_types_to_save): """ From dataflash file save an HDF5 store of PANDAS dataframes for each msg Parameters ---------- filename : str Name of the file to split. outfile: str Name of the output file msg_types_to_save: list list of string identifiers of Mavlink Messages to put in Pandas Frame """ log = DFReader_binary(filename) last_m = None num_msgs = 0 out_dict = {} first = True init_time = 0 while True: # we use mavlinks recv_match function to iterate through logs m = log.recv_match(type=msg_types_to_save) if m: timestamp = m._timestamp if first: init_time = timestamp delta = timestamp - init_time dt = datetime.datetime.utcfromtimestamp(timestamp + NUMLEAPSECONDS) msg_timestamp_dict = out_dict.get(m.get_type(), {}) msg_timestamp_dict[dt] = m.to_dict() out_dict[m.get_type()] = msg_timestamp_dict else: if os.path.exists(outfile): print "Unlinking %s, which already exists!" % outfile os.unlink(outfile) f = pd.HDFStore(outfile, mode='w') try: tabs = [key for key in out_dict.iterkeys()] for tab in tabs: attr = out_dict[tab] f.put(tab, pd.DataFrame(attr)) if f.get(tab).empty: warnings.warn('%s is empty.' % tab) finally: f.close() break
def extractMAVLINK(filename, outfile, msg_types_to_save): """ From dataflash file save an HDF5 store of PANDAS dataframes for each msg Parameters ---------- filename : str Name of the file to split. outfile: str Name of the output file msg_types_to_save: list list of string identifiers of Mavlink Messages to put in Pandas Frame """ log = DFReader_binary(filename) last_m = None num_msgs = 0 out_dict = {} first = True init_time = 0 while True: # we use mavlinks recv_match function to iterate through logs m = log.recv_match(type=msg_types_to_save) if m: timestamp = m._timestamp if first: init_time = timestamp delta = timestamp - init_time dt = datetime.datetime.utcfromtimestamp(timestamp + NUMLEAPSECONDS) msg_timestamp_dict = out_dict.get(m.get_type(), {}) msg_timestamp_dict[dt] = m.to_dict() out_dict[m.get_type()] = msg_timestamp_dict else: if os.path.exists(outfile): print("Unlinking %s, which already exists!" % outfile) os.unlink(outfile) f = pd.HDFStore(outfile, mode='w') try: tabs = [key for key in out_dict.iterkeys()] for tab in tabs: attr = out_dict[tab] f.put(tab, pd.DataFrame(attr)) if f.get(tab).empty: warnings.warn('%s is empty.' % tab) finally: f.close() break
def extractSBP(filename): extracted_data = [] log = DFReader_binary(filename) last_m = None num_msgs = 0 while True: # we use mavlinks recv_match function to iterate through logs # and give us the SBR1 or SBR2 message # SBR1 msgs are the first 64 bytes of any sbp message, or the entire message # if the message is smaller than 64 bytes # SBR2 msgs are the next n bytes if the original message is longer than 64 bytes m = log.recv_match(type=['SBR1', 'SBR2']) if m is None: break bin_data = None timestamp = None msg_type = None sender_id = None msg_len = None if last_m != None and last_m.get_type() == 'SBR1' and m.get_type() == 'SBR2': # If the last message was an SBR1 and the current message is SBR2 # we combine the two into one SBP message msg_len = last_m.msg_len timestamp = getattr(last_m, '_time', 0.0) binary = last_m.binary bin_data = bytearray(last_m.binary[SBR1_DATASTART:SBR1_DATASTART+64] + m.binary[SBR2_DATASTART:SBR2_DATASTART+msg_len-64]) assert len(bin_data) == msg_len, "Length of binary data decoded \ from dataflash does not match msg_len in header" msg_type = last_m.msg_type sender_id = last_m.sender_id last_m = m elif last_m and last_m.get_type() == 'SBR1' and m.get_type() == 'SBR1': # If the last message was SBR1 and this one is SBR1, we extract the last one # and save this one until the next iteration msg_len = last_m.msg_len binary = last_m.binary assert binary, "binary empty" timestamp = getattr(last_m, '_time', 0.0) msg_type = last_m.msg_type sender_id = last_m.sender_id bin_data = bytearray(last_m.binary[SBR1_DATASTART:SBR1_DATASTART+msg_len]) last_m = m elif last_m and last_m.get_type() == "SBR2" and m.get_type() == "SBR1": # just save current message as the last_m. # We wait until next message received to know whether it is a complete message last_m = m else: # This should only happen on our first iteration if last_m: assert num_msgs == 0, "This branch is expected to execute on first message" \ " only. This is a serious logical error in the decoder." last_m = m if bin_data != None: if len(bin_data) != msg_len: print "Length of SBP message inconsitent for msg_type {0}.".format(msg_type) print "Expected Length {0}, Actual Length {1}".format(msg_len, len(bin_data)) extracted_data.append((timestamp, msg_type, sender_id, msg_len, bin_data)) num_msgs += 1 print "extracted {0} messages".format(num_msgs) return extracted_data
def extractSBP(filename): extracted_data = [] log = DFReader_binary(filename) last_m = None num_msgs = 0 while True: # we use mavlinks recv_match function to iterate through logs # and give us the SBR1 or SBR2 message # SBR1 msgs are the first 64 bytes of any sbp message, or the entire message # if the message is smaller than 64 bytes # SBR2 msgs are the next n bytes if the original message is longer than 64 bytes m = log.recv_match(type=['SBR1', 'SBR2']) if m is None: break bin_data = None timestamp = None msg_type = None sender_id = None msg_len = None if last_m != None and last_m.get_type() == 'SBR1' and m.get_type( ) == 'SBR2': # If the last message was an SBR1 and the current message is SBR2 # we combine the two into one SBP message msg_len = last_m.msg_len timestamp = getattr(last_m, '_time', 0.0) binary = last_m.binary bin_data = bytearray( last_m.binary[SBR1_DATASTART:SBR1_DATASTART + 64] + m.binary[SBR2_DATASTART:SBR2_DATASTART + msg_len - 64]) assert len(bin_data) == msg_len, "Length of binary data decoded \ from dataflash does not match msg_len in header" msg_type = last_m.msg_type sender_id = last_m.sender_id last_m = m elif last_m and last_m.get_type() == 'SBR1' and m.get_type() == 'SBR1': # If the last message was SBR1 and this one is SBR1, we extract the last one # and save this one until the next iteration msg_len = last_m.msg_len binary = last_m.binary assert binary, "binary empty" timestamp = getattr(last_m, '_time', 0.0) msg_type = last_m.msg_type sender_id = last_m.sender_id bin_data = bytearray(last_m.binary[SBR1_DATASTART:SBR1_DATASTART + msg_len]) last_m = m elif last_m and last_m.get_type() == "SBR2" and m.get_type() == "SBR1": # just save current message as the last_m. # We wait until next message received to know whether it is a complete message last_m = m else: # This should only happen on our first iteration if last_m: assert num_msgs == 0, "This branch is expected to execute on first message" \ " only. This is a serious logical error in the decoder." last_m = m if bin_data != None: if len(bin_data) != msg_len: print "Length of SBP message inconsitent for msg_type {0}.".format( msg_type) print "Expected Length {0}, Actual Length {1}".format( msg_len, len(bin_data)) extracted_data.append( (timestamp, msg_type, sender_id, msg_len, bin_data)) num_msgs += 1 print "extracted {0} messages".format(num_msgs) return extracted_data