def testWriteRead(): packets = 'When a packet hits a pocket on a socket on a port.'.split() with pcap.open(TmpFilename, 'w') as stream: for p in packets: stream.write(p) with pcap.open(TmpFilename, 'r') as stream: index = 0 prev_ts = 0 for header, packet in stream: assert header.ts >= prev_ts assert header.incl_len == len(packets[index]) assert header.orig_len == len(packets[index]) assert packet == packets[index] index += 1 prev_ts = header.ts assert index == len(packets) header, packet = stream.read() assert header.incomplete() assert packet is None os.unlink(TmpFilename)
def testReadLittleEndian(): bytes = 'Hello World!' ts = int(time.time()) # Write pcap file with open(TmpFilename, 'wb') as stream: stream.write( struct.pack('<IHHiIII', 0xA1B2C3D4, 2, 4, 0, 0, 65535, 147)) stream.write(struct.pack('<IIII', ts, 0, len(bytes), len(bytes))) stream.write(bytes) # Read pcap using API with pcap.open(TmpFilename, 'r') as stream: assert stream.header.magic_number == 0xA1B2C3D4 assert stream.header.version_major == 2 assert stream.header.version_minor == 4 assert stream.header.thiszone == 0 assert stream.header.sigfigs == 0 assert stream.header.snaplen == 65535 assert stream.header.network == 147 header, packet = stream.read() assert header.ts == ts assert header.incl_len == len(bytes) assert header.orig_len == len(bytes) assert packet == bytes header, packet = stream.read() assert header.incomplete() assert packet is None os.unlink(TmpFilename)
def send(self, command, *args, **kwargs): """Creates, validates, and sends the given command as a UDP packet to the destination (host, port) specified when this CmdAPI was created. Returns True if the command was created, valid, and sent, False otherwise. """ status = False cmdobj = CMD_API._cmddict.create(command, *args, **kwargs) messages = [] if not cmdobj.validate(messages): for msg in messages: log.error(msg) else: encoded = cmdobj.encode() if CMD_API._verbose: size = len(cmdobj.name) pad = (size - len(cmdobj.name) + 1) * ' ' gds.hexdump(encoded, preamble=cmdobj.name + ':' + pad) try: self.publish(encoded) status = True with pcap.open(CMD_API.CMD_HIST_FILE, 'a') as output: output.write(str(cmdobj)) except IOError as e: log.error(e.message) return status
def _get_logger(self, handler): ''' Initialize a PCAP stream for logging data ''' log_file = self._get_log_file(handler) if not os.path.isdir(os.path.dirname(log_file)): os.makedirs(os.path.dirname(log_file)) handler['log_rot_time'] = time.gmtime() return pcap.open(log_file, mode='a')
def send(self, command, *args, **kwargs): """Creates, validates, and sends the given command as a UDP packet to the destination (host, port) specified when this CmdAPI was created. Returns True if the command was created, valid, and sent, False otherwise. """ status = False cmdobj = self._cmddict.create(command, *args, **kwargs) messages = [] if not cmdobj.validate(messages): for msg in messages: log.error(msg) else: encoded = cmdobj.encode() if self._verbose: size = len(cmdobj.name) pad = (size - len(cmdobj.name) + 1) * " " gds.hexdump(encoded, preamble=cmdobj.name + ":" + pad) try: # Send to either UDP socket or ZMQ publish socket if self._udp_socket: values = (self._host, self._port, str(cmdobj)) log.command("Sending to %s:%d: %s" % values) self._udp_socket.sendto(encoded, (self._host, self._port)) status = True elif self._pub_socket: values = (self._pub_topic, str(cmdobj)) log.command("Sending via publisher: %s %s" % values) msg = serv_utils.encode_message(self._pub_topic, encoded) if msg is None: log.error( "CmdAPI unable to encode cmd message " f"({self._pub_topic}, {encoded}) for send" ) status = False else: self._pub_socket.send_multipart(msg) status = True # Only add to history file if success status is True if status: with pcap.open(self.CMD_HIST_FILE, "a") as output: output.write(str(cmdobj)) except socket.error as e: log.error(e.message) except IOError as e: log.error(e.message) return status
def testQuery(): TmpRes = "test_pcap_res.pcap" TmpFilename = "test_pcap_file.pcap" packets = "This is a nice little sentence".split() start = datetime.datetime.now() with pcap.open(TmpFilename, 'w') as stream: for p in packets: stream.write(p) end = datetime.datetime.max pcap.query(start, end, TmpRes, (TmpFilename)) with pcap.open(TmpFilename, 'r') as stream1: with pcap.open(TmpRes, 'r') as stream2: header1, packet1 = stream1.read() header2, packet2 = stream2.read() assert str(header1) == str(header2) assert packet1 == packet2 os.remove(TmpRes) os.remove(TmpFilename)
def testTimes(): packets = "This is a nice little sentence".split() with pcap.open(TmpFilename, 'w') as stream: for p in packets: stream.write(p) with pcap.open(TmpFilename, 'r') as stream: i = 0 for header, packet in stream: if i is 0: exp_start = header.timestamp if i is 5: exp_end = header.timestamp i += 1 times = pcap.times(TmpFilename) start = times[TmpFilename][0][0] stop = times[TmpFilename][0][1] assert len(times[TmpFilename]) == 1 assert start == exp_start assert stop == exp_end # test when we have 2 separate time segments with pcap.open(TmpFilename, 'w') as stream: for p in packets: stream.write(p) time.sleep(3) for p in packets: stream.write(p) times = pcap.times(TmpFilename, 2) assert len(times[TmpFilename]) == 2 os.remove(TmpFilename)
def testSegmentBytes(log_info): try: with pcap.open(TmpFilename, 'w') as output: for p in range(10): output.write(str(p)) pcap.segment(TmpFilename, 'foo.pcap', nbytes=41, dryrun=True) expected = 'Would write 41 bytes, 1 packets, 1 seconds to foo.pcap.' assert len(log_info.call_args_list) == 10 for call in log_info.call_args_list: assert call[0][0] == expected finally: os.unlink(TmpFilename)
def testSegmentPackets(log_info): try: with pcap.open(TmpFilename, "w") as output: for p in range(10): output.write(str(p)) pcap.segment(TmpFilename, "foo.pcap", npackets=5, dryrun=True) expected = "Would write 109 bytes, 5 packets, 1 seconds to foo.pcap." assert len(log_info.call_args_list) == 2 for call in log_info.call_args_list: assert call[0][0] == expected finally: os.unlink(TmpFilename)
def testSegmentSeconds(log_info): try: header = pcap.PCapPacketHeader(orig_len=1) with pcap.open(TmpFilename, 'w') as output: for p in range(10): header.ts_sec = p output.write(str(p), header) pcap.segment(TmpFilename, 'foo.pcap', nseconds=2, dryrun=True) expected = 'Would write 58 bytes, 2 packets, 2 seconds to foo.pcap.' assert len(log_info.call_args_list) == 5 for call in log_info.call_args_list: assert call[0][0] == expected finally: os.unlink(TmpFilename)
def handle(): """Return sent command history **Example Response**: .. sourcecode: json [ "NO_OP", "SEQ_START 3423" ] If you set the **detailed** query string flag the JSON returned will include timestamp information. **Example Detailed Response** .. sourcecode: json [ { "timestamp": "2017-08-01 15:41:13.117805", "command": "NO_OP" }, { "timestamp": "2017-08-01 15:40:23.339886", "command": "NO_OP" } ] """ cmds = [] try: with pcap.open(CMD_API.CMD_HIST_FILE, 'r') as stream: if 'detailed' in bottle.request.query: cmds = [ { 'timestamp': str(header.timestamp), 'command': cmdname.decode('utf-8') } for (header, cmdname) in stream ] return json.dumps(list(reversed(cmds))) else: cmds = [cmdname.decode('utf-8') for (header, cmdname) in stream] return json.dumps(list(set(cmds))) except IOError: pass
def main(): log.begin() arguments = { "filename": { "metavar": "</path/to/cmdhist.pcap>", "help": "command history pcap", } } ap = argparse.ArgumentParser(description=__doc__) for name, params in arguments.items(): ap.add_argument(name, **params) args = ap.parse_args() with pcap.open(args.filename) as stream: for header, data in stream: print(header.timestamp.strftime("%Y-%m-%d %H:%M:%S") + "\t" + data.decode()) log.end()
def main(): log.begin() description = """ Query all commands from a Command History PCAP """ arguments = {} arguments['filename'] = { 'type' : str, 'metavar' : '</path/to/cmdhist.pcap>', 'help' : 'command history pcap' } args = gds.arg_parse(arguments, description) with pcap.open(args.filename) as stream: for header, data in stream: print(header.timestamp.strftime('%Y-%m-%d %H:%M:%S') + '\t' + data) log.end()
def testWrite(): bytes = 'Hello World!' ts = time.time() # Write pcap using API with pcap.open(TmpFilename, 'w') as stream: assert stream.write(bytes) == len(bytes) # Read pcap file with open(TmpFilename, 'rb') as stream: header = struct.unpack('IHHiIII', stream.read(24)) assert header == (0xA1B2C3D4, 2, 4, 0, 0, 65535, 147) header = struct.unpack('IIII', stream.read(16)) assert header[0] - ts <= 1 # write timestamp assert header[2] == len( bytes) # number of octets of packet saved in file assert header[3] == len(bytes) # actual length of packet assert stream.read(header[2]) == bytes assert len(stream.read()) == 0 os.unlink(TmpFilename)
def main(): try: log.begin() parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) # Add required command line arguments parser.add_argument("filename") # Add optional command line arguments parser.add_argument("--port", default=3076, type=int) parser.add_argument("--verbose", action="store_true", default=False) # Get command line arguments args = vars(parser.parse_args()) filename = args["filename"] host = "localhost" port = args["port"] sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) verbose = args["verbose"] if not verbose: log.info("Will only report every 10 telemetry packets") log.info("Will only report long telemetry send delays") with pcap.open(filename, "r") as stream: npackets = 0 prev_ts = None for header, packet in stream: if prev_ts is None: prev_ts = header.ts delay = header.ts - prev_ts if delay >= 2: log.info("Next telemetry in %1.2f seconds" % delay) time.sleep(delay) nbytes = len(packet) if npackets == 0: log.info("Sent first telemetry packet (%d bytes)" % nbytes) elif verbose: log.info("Sent telemetry (%d bytes)" % nbytes) elif npackets % 10 == 0: log.info("Sent 10 telemetry packets") sock.sendto(packet, (host, port)) npackets += 1 prev_ts = header.ts except KeyboardInterrupt: log.info("Received Ctrl-C. Stopping telemetry stream.") except Exception as e: log.error("TLM send error: %s" % str(e)) log.end()
def main(): tlmdict = tlm.getDefaultDict() pnames = tlmdict.keys() ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) arguments = { '--packet': { 'type': str, 'choices': pnames, 'default': pnames[0] if len(pnames) > 0 else None, 'help': 'Type of packets (!Packet name in tlm.yaml) in file', 'required': len(pnames) > 1, }, '--database': { 'default': ait.config.get('database.name'), 'help': ('Name of database in which to insert packets (may ' 'also be specified in config.yaml database.name)'), 'required': ait.config.get('database.name') is None }, '--backend': { 'default': 'sqlite', 'choices': ['sqlite', 'influx'], 'action': 'store', 'help': ('Name of database in which to insert packets (may ' 'also be specified in config.yaml database.name)') }, '--use-current-time': { 'action': 'store_true', 'help': ('Use current time stamps when insert packets instead ' 'of ground receipt time (or the time written in the ' 'PCAP header).') }, 'file': { 'nargs': '+', 'help': 'File(s) containing telemetry packets' } } for name, params in arguments.items(): ap.add_argument(name, **params) args = ap.parse_args() log.begin() try: npackets = 0 dbconn = None defn = tlm.getDefaultDict()[args.packet] nbytes = defn.nbytes if args.backend == 'sqlite': dbconn = db.SQLiteBackend() elif args.backend == 'influx': dbconn = db.InfluxDBBackend() if args.backend == 'sqlite' and (args.database == ':memory:' or not os.path.exists(args.database)): dbconn.create(database=args.database) else: dbconn.connect(database=args.database) for filename in args.file: log.info('Processing %s' % filename) with pcap.open(filename) as stream: for header, pkt_data in stream: try: packet = tlm.Packet(defn, pkt_data) time = header.timestamp if args.use_current_time: time = None dbconn.insert(packet, time=time) npackets += 1 except struct.error: log.error( "Unable to unpack data into packet. Skipping ...") except KeyboardInterrupt: log.info('Received Ctrl-C. Stopping database insert.') except IOError as e: log.error(str(e)) finally: dbconn.close() values = npackets, args.packet, args.database log.info('Inserted %d %s packets into database %s.' % values) log.end()
def main(): log.begin() parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--all", action="store_true", help="output all fields/values") parser.add_argument( "--csv", default="output.csv", metavar="</path/to/output/csv>", help="Output as CSV with filename", ) parser.add_argument( "--fields", metavar="</path/to/fields/file>", help="path to the file containing all fields to query, separated by newline.", ) parser.add_argument( "--packet", required=True, help="Packet name from telemetry dictionary specified in config file." ) parser.add_argument( "--time_field", help=( "Time field to use for time range comparisons. Ground receipt time " "will be used if nothing is specified." ), ) parser.add_argument( "--stime", help=( "Datetime in file to start collecting the data values. Defaults to " "beginning of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ" ), ) parser.add_argument( "--etime", help=( "Datetime in file to end collecting the data values. Defaults to end " "of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ" ), ) parser.add_argument( "pcap", nargs="*", help=("PCAP file(s) containing telemetry packets") ) args = parser.parse_args() args.ground_time = True if args.time_field is not None: args.ground_time = False tlmdict = tlm.getDefaultDict() defn = None try: if tlmdict is not None: defn = tlmdict[args.packet] except KeyError: log.error('Packet "%s" not defined in telemetry dictionary.' % args.packet) log.end() sys.exit(2) if not args.all and args.fields is None: log.error( "Must provide fields file with --fields or specify that all fields should be queried with --all" ) log.end() sys.exit(2) if args.all: fields = [flddefn.name for flddefn in defn.fields] else: # Parse the fields file into a list with open(args.fields, "r") as stream: fields = [fldname.strip() for fldname in stream.readlines()] not_found = False # TODO Rework this into the CSV generation. Not here. # Duplicating effort for fldname in fields: raw = fldname.split(".") if fldname not in defn.fieldmap and ( len(raw) == 2 and raw[0] != "raw" or raw[1] not in defn.fieldmap ): not_found = True log.error('No telemetry point named "%s"' % fldname) if not_found: log.end() sys.exit(2) if args.stime: start = datetime.strptime(args.stime, dmc.ISO_8601_Format) else: start = dmc.GPS_Epoch if args.etime: stop = datetime.strptime(args.etime, dmc.ISO_8601_Format) else: stop = datetime.utcnow() # Append time to beginning of each row if not args.ground_time: fields.insert(0, args.time_field) else: fields.insert(0, "Ground Receipt Time") csv_file = None csv_writer = None npackets = 0 if args.csv: csv_file = open(args.csv, "w") csv_writer = csv.writer(csv_file) output(csv_writer, fields) # If we're comparing off ground receipt time we need to drop the header label to avoid # indexing errors when processing the fields. if args.ground_time: fields = fields[1:] rowcnt = 0 for filename in args.pcap: log.debug("Processing %s" % filename) with pcap.open(filename, "rb") as stream: header, data = stream.read() while data: packet = tlm.Packet(defn, data) comp_time = ( header.timestamp if args.ground_time else getattr(packet, args.time_field) ) if start < comp_time < stop: row = [] for field in fields: try: # check if raw value requested _raw = False names = field.split(".") if len(names) == 2 and names[0] == "raw": field = names[1] _raw = True field_val = packet._getattr(field, raw=_raw) if hasattr(field_val, "name"): field_val = field_val.name else: field_val = str(field_val) except KeyError: log.debug("%s not found in Packet" % field) field_val = None except ValueError: # enumeration not found. just get the raw value field_val = packet._getattr(field, raw=True) row.append(field_val) if args.ground_time: row = [comp_time] + row rowcnt += 1 output(csv_writer, row) npackets += 1 header, data = stream.read() log.debug("Parsed %s packets." % npackets) csv_file.close() if rowcnt == 0: os.remove(args.csv) log.end()
def main(): tlmdict = tlm.getDefaultDict() pnames = list(tlmdict.keys()) ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) arguments = { "--packet": { "type": str, "choices": pnames, "default": pnames[0] if len(pnames) > 0 else None, "help": "Type of packets (!Packet name in tlm.yaml) in file", "required": len(pnames) > 1, }, "--database": { "default": ait.config.get("database.dbname"), "help": ("Name of database in which to insert packets (may " "also be specified in config.yaml database.name)"), "required": ait.config.get("database.dbname") is None, }, "--backend": { "default": "sqlite", "choices": ["sqlite", "influx"], "action": "store", "help": ("Name of database in which to insert packets (may " "also be specified in config.yaml database.name)"), }, "--use-current-time": { "action": "store_true", "help": ("Use current time stamps when insert packets instead " "of ground receipt time (or the time written in the " "PCAP header)."), }, "file": { "nargs": "+", "help": "File(s) containing telemetry packets" }, } for name, params in arguments.items(): ap.add_argument(name, **params) args = ap.parse_args() log.begin() try: npackets = 0 dbconn = None defn = tlm.getDefaultDict()[args.packet] if args.backend == "sqlite": dbconn = db.SQLiteBackend() elif args.backend == "influx": dbconn = db.InfluxDBBackend() if args.backend == "sqlite" and (args.database == ":memory:" or not os.path.exists(args.database)): dbconn.create(database=args.database) else: dbconn.connect(database=args.database) for filename in args.file: log.info("Processing %s" % filename) with pcap.open(filename) as stream: for header, pkt_data in stream: try: packet = tlm.Packet(defn, pkt_data) time = header.timestamp if args.use_current_time: time = None dbconn.insert(packet, time=time) npackets += 1 except struct.error: log.error( "Unable to unpack data into packet. Skipping ...") except KeyboardInterrupt: log.info("Received Ctrl-C. Stopping database insert.") except IOError as e: log.error(str(e)) finally: dbconn.close() values = npackets, args.packet, args.database log.info("Inserted %d %s packets into database %s." % values) log.end()
def main(): log.begin() description = """Parses 1553 telemetry into CSV file.""" arguments = { '--all': { 'action': 'store_true', 'help': 'output all fields/values', }, '--csv': { 'type': str, 'default': 'output.csv', 'metavar': '</path/to/output/csv>', 'help': 'Output as CSV with filename' }, '--fields': { 'type': str, 'metavar': '</path/to/fields/file>', 'help': 'file containing all fields to query, separated by newline.' }, '--packet': { 'type': str, 'required': True, 'help': 'field names to query, separated by space' }, '--time_field': { 'type': str, 'help': 'Time field to use for time range comparisons. Ground receipt time will be used if nothing is specified.' }, '--stime': { 'type': str, 'help': 'Datetime in file to start collecting the data values. Defaults to beginning of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ' }, '--etime': { 'type': str, 'help': 'Datetime in file to end collecting the data values. Defaults to end of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ' } } arguments['pcap'] = { 'nargs': '*', 'help': 'PCAP file(s) containing telemetry packets' } args = gds.arg_parse(arguments, description) args.ground_time = True if args.time_field is not None: args.ground_time = False tlmdict = tlm.getDefaultDict() defn = None try: if tlmdict is not None: defn = tlmdict[args.packet] except KeyError: log.error('Packet "%s" not defined in telemetry dictionary.' % args.packet) gds.exit(2) if not args.all and args.fields is None: log.error( 'Must provide fields file with --fields or specify that all fields should be queried with --all' ) gds.exit(2) if args.all: fields = [flddefn.name for flddefn in defn.fields] else: # Parse the fields file into a list with open(args.fields, 'rb') as stream: fields = [fldname.strip() for fldname in stream.readlines()] not_found = False # TODO Rework this into the CSV generation. Not here. # Duplicating effort for fldname in fields: raw = fldname.split('.') if fldname not in defn.fieldmap and (len(raw) == 2 and raw[0] != 'raw' or raw[1] not in defn.fieldmap): not_found = True log.error('No telemetry point named "%s"' % fldname) if not_found: gds.exit(2) if args.stime: start = datetime.strptime(args.stime, dmc.ISO_8601_Format) else: start = dmc.GPS_Epoch if args.etime: stop = datetime.strptime(args.etime, dmc.ISO_8601_Format) else: stop = datetime.utcnow() # Append time to beginning of each row if not args.ground_time: fields.insert(0, args.time_field) else: fields.insert(0, 'Ground Receipt Time') csv_file = None csv_writer = None npackets = 0 if args.csv: csv_file = open(args.csv, 'wb') csv_writer = csv.writer(csv_file) output(csv_writer, fields) # If we're comparing off ground receipt time we need to drop the header label to avoid # indexing errors when processing the fields. if args.ground_time: fields = fields[1:] rowcnt = 0 for filename in args.pcap: log.debug('Processing %s' % filename) with pcap.open(filename, 'rb') as stream: header, data = stream.read() while data: packet = tlm.Packet(defn, data) comp_time = header.timestamp if args.ground_time else getattr( packet, args.time_field) if start < comp_time < stop: row = [] for field in fields: try: # check if raw value requested _raw = False names = field.split('.') if len(names) == 2 and names[0] == 'raw': field = names[1] _raw = True fieldVal = packet._getattr(field, raw=_raw) if hasattr(fieldVal, 'name'): fieldVal = fieldVal.name else: fieldVal = str(fieldVal) except KeyError: log.debug('%s not found in Packet' % field) fieldVal = None except ValueError: # enumeration not found. just get the raw value fieldVal = packet._getattr(field, raw=True) row.append(fieldVal) if args.ground_time: row = [comp_time] + row rowcnt += 1 output(csv_writer, row) npackets += 1 header, data = stream.read() log.debug('Parsed %s packets.' % npackets) csv_file.close() if rowcnt == 0: os.remove(args.csv) log.end()