def producer(args):
# connect to kafka
producer = kafka.KafkaProducer(
bootstrap_servers=args.kafka_brokers.split(","),
partitioner=partitioner)
# initialize packet capture
capture = pcapy.open_live(args.interface, 65535, True, 3000)
packet_count = 0
# start packet capture
while True:
(pkt_hdr, pkt_raw) = capture.next()
if pkt_hdr is not None:
# send packet to kafka
pkt_ts = timestamp(pkt_hdr)
producer.send(args.topic, key=pack_ts(pkt_ts), value=pkt_raw)
# debug messages, if needed
packet_count += 1
if args.debug > 0 and packet_count % args.debug == 0:
print 'Sent Packet: count=%s dt=%s topic=%s' % (
packet_count, to_date(pkt_ts), args.topic)
print to_hex(pkt_raw)
def consumer(args):
# connect to kafka
brokers = args.kafka_brokers.split(",")
kafka_consumer = kafka.KafkaConsumer(args.topic, bootstrap_servers=brokers)
# if debug not set, write libpcap global header
if args.debug == 0:
sys.stdout.write(global_header())
# start packet capture
packet_count = 0
for msg in kafka_consumer:
# if debug not set, write the packet header and packet
if args.debug == 0:
sys.stdout.write(packet_header(msg.value, msg.key))
sys.stdout.write(msg.value)
elif packet_count % args.debug == 0:
print 'Packet: count=%s dt=%s topic=%s' % (
packet_count, to_date(unpack_ts(msg.key)), args.topic)
print to_hex(msg.value)
packet_count += 1
if args.packet_count > 0 and packet_count >= args.packet_count:
break
def consumer(args):
# connect to kafka
brokers = args.kafka_brokers.split(",")
kafka_consumer = kafka.KafkaConsumer(args.topic, bootstrap_servers=brokers)
# if debug not set, write libpcap global header
if args.debug == 0:
sys.stdout.write(global_header())
# start packet capture
packet_count = 0
for msg in kafka_consumer:
# if debug not set, write the packet header and packet
if args.debug == 0:
sys.stdout.write(packet_header(msg.value, msg.key))
sys.stdout.write(msg.value)
elif packet_count % args.debug == 0:
print "Packet: count=%s dt=%s topic=%s" % (packet_count, to_date(unpack_ts(msg.key)), args.topic)
print to_hex(msg.value)
packet_count += 1
if args.packet_count > 0 and packet_count >= args.packet_count:
break
def consumer(args, poll_timeout=3.0):
""" Consumes packets from a Kafka topic. """
# setup the signal handler
signal.signal(signal.SIGINT, signal_handler)
# connect to kafka
logging.debug("Connecting to Kafka; %s", args.kafka_configs)
kafka_consumer = Consumer(args.kafka_configs)
kafka_consumer.subscribe([args.kafka_topic])
# if 'pretty-print' not set, write libpcap global header
if args.pretty_print == 0:
sys.stdout.write(global_header(args))
sys.stdout.flush()
try:
pkts_in = 0
while not finished.is_set() and (args.max_packets <= 0 or pkts_in < args.max_packets):
# consume a message from kafka
msg = kafka_consumer.poll(timeout=poll_timeout)
if msg is None:
# no message received
continue;
elif msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
logging.debug("reached end of topar: topic=%s, partition=%d, offset=%s", msg.topic(), msg.partition(), msg.offset())
elif msg.error():
raise KafkaException(msg.error())
else:
pkts_in += 1
logging.debug("Packet received: pkts_in=%d", pkts_in)
if args.pretty_print == 0:
# write the packet header and packet
sys.stdout.write(packet_header(msg))
sys.stdout.write(msg.value())
sys.stdout.flush()
elif pkts_in % args.pretty_print == 0:
# pretty print
print 'Packet: count=%s date=%s topic=%s' % (
pkts_in, to_date(unpack_ts(msg.key())), args.kafka_topic)
print to_hex(msg.value())
finally:
sys.stdout.close()
kafka_consumer.close()
def test_api_tcp():
'''
Test transmitting TCP messages using API frames.
'''
print("Launching API tcp frames test.")
ser = serial.Serial(port='COM7',
baudrate=115200,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=1)
tx = TXRequest(
payload=
b"GET / HTTP/1.1\r\nHost: api.ipify.org\r\nUser-Agent: curl/7.64.0\r\nAccept: */*\r\n\r\n",
dst="23.21.121.219")
ser.write(tx.packed)
print("TX: %s" % tx)
for i in range(5):
time.sleep(1)
rx = ser.read(1024)
print("RX: NotYetParsed(bytes=%s, hex=%s)" % (rx, to_hex(rx)))
ser.close()
def producer(args, sniff_timeout_ms=500, sniff_promisc=True):
""" Captures packets from a network interface and sends them to a Kafka topic. """
# setup the signal handler
signal.signal(signal.SIGINT, signal_handler)
# connect to kafka
logging.info("Connecting to Kafka; %s", args.kafka_configs)
kafka_producer = Producer(args.kafka_configs)
# initialize packet capture
logging.info("Starting packet capture")
capture = pcapy.open_live(args.interface, args.snaplen, sniff_promisc, sniff_timeout_ms)
pkts_in = 0
try:
while not finished.is_set() and (args.max_packets <= 0 or pkts_in < args.max_packets):
# capture a packet
(pkt_hdr, pkt_raw) = capture.next()
if pkt_hdr is not None:
logging.debug("Packet received: pkts_in=%d, pkt_len=%s", pkts_in, pkt_hdr.getlen())
pkts_in += 1
pkt_ts = timestamp(pkt_hdr)
kafka_producer.produce(args.kafka_topic, key=pack_ts(pkt_ts), value=pkt_raw, callback=delivery_callback)
# debug messages, if needed
if args.pretty_print > 0 and pkts_in % args.pretty_print == 0:
print '{} packet(s) received'.format(pkts_in)
print to_hex(pkt_raw)
# serve the callback queue
kafka_producer.poll(0)
finally:
# flush all messages
logging.info("Waiting for '%d' message(s) to flush", len(kafka_producer))
kafka_producer.flush()
# pkts_out may not be initialized if the callback was never executed
pkts_out = 0
if hasattr(delivery_callback, "pkts_out"):
pkts_out = delivery_callback.pkts_out
logging.info("'%d' packet(s) in, '%d' packet(s) out", pkts_in, pkts_out)
def test_api_tls():
'''
Test transmitting TCP messages with SSL/TLS using API frames.
Note that this also contains the commands to set up the TLS profiles.
Note that if you have a typo in the certs filename, you'll get this: "hex=7E 00 03 89 01 86 EF" where the 0x86 means "Invalid TLS configuration (missing file, etc.)".
If you try to connect to a server that doesn't exist, you'll get "hex=7E 00 03 89 01 82 F3" is a "TX Status" message and the 0x82 means "No Server".
If you try to connect to the wrong port / invalid port on a server that exists, you'll get "hex=7E 00 03 89 01 80 F5" where the 0x89 is the "TX Status" message type and the 0x80 means "Connection Refused" .
For more details on the codes returned by the "TX Status" API Frame message, see: https://www.digi.com/resources/documentation/Digidocs/90002258/#Reference/r_frame_0x89_cell.htm)
'''
print("Launching the API frames SSL/TLS test.")
ser = serial.Serial(port='COM7',
baudrate=115200,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=1)
print("Using the following XBee TLS Profile 0 configuration:")
tx = ATCommand(
b"$0", b""
).packed # @todo what string needs to go in here to set up TLS???
ser.write(tx)
print("TX: %s" % tx)
rx = ser.read(1024)
print("RX: %s" % rx)
print("Now performing the HTTPS request.")
tx = TXRequest(
payload=
b"GET / HTTP/1.1\r\nHost: api.ipify.org\r\nUser-Agent: curl/7.64.0\r\nAccept: */*\r\n\r\n",
dst="23.23.243.154",
tls_profile=b'\x00')
ser.write(tx.packed)
print("TX: %s" % tx)
for i in range(5):
time.sleep(1)
rx = ser.read(1024)
print("RX: NotYetParsed(bytes=%s, hex=%s)" % (rx, to_hex(rx)))
ser.close()
def test_api_AT_Command():
print("Launching API AT Command test.")
ser = serial.Serial(port='COM7',
baudrate=115200,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=1)
tx = ATCommand(b"AP", b"")
ser.write(tx.packed)
print("TX: %s" % tx)
time.sleep(1)
rx = ser.read(100)
print("RX: ATResponse(bytes=%s, hex=%s)" % (rx, to_hex(rx)))
ser.close()
def test_api_aws_connection():
'''
Test transmitting SSL/TLS-secured TCP messages to AWS using API frames.
Assumes XBee operating mode is API Frames With Escapes (ATAP2).
'''
print("Launching the AWS API frames test.")
with serial.Serial(port='COM7',
baudrate=115200,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
timeout=1) as ser:
DO_CONFIG = False
DO_DNS_LOOKUP = True
aws_endpoint = b'ak87jc7d58d2m.iot.us-east-2.amazonaws.com'
#aws_endpoint = b'ak87jc7d58d2m-ats.iot.us-east-2.amazonaws.com'
thing_type = b'XBee3Cellular'
thing_name = b'DanXBee'
if DO_CONFIG:
print("Configuring the XBee TLS settings.")
# Set IP protocol to SSL over TCP communication.
# UPDATE: I believe this setting applies only to Transparent Mode, but we'll set it just in case.
tx = ATCommand(b"IP", b"\x04").packed
ser.write(tx)
print("TX: %s" % tx)
rx = ser.read(1024)
print("RX: %s" % rx)
# Set up TLS v1.2.
tx = ATCommand(b"TL", b"\x03").packed
ser.write(tx)
print("TX: %s" % tx)
rx = ser.read(1024)
print("RX: %s" % rx)
# Set up paths to certs.
tx = ATCommand(
b"$0",
b"/flash/cert/aws.ca;/flash/cert/aws.crt;/flash/cert/aws.key"
).packed # Set up paths to certs
ser.write(tx)
print("TX: %s" % tx)
rx = ser.read(1024)
print("RX: %s" % rx)
else:
print("Skipping config.")
if DO_DNS_LOOKUP:
print("Now performing the DNS lookup.")
tx = ATCommand(b"LA", aws_endpoint).packed
ser.write(tx)
print("TX: %s" % tx)
rx = ser.read(1024)
print("Got: %s" % rx)
if len(rx) < 13:
raise Exception("DNS lookup failed.")
ip_addr = ".".join(
["%i" % i for i in rx[8:12]]
) # This converts something like b"~\x00\t\x88\tLA\x00\x12\xd9\xee\xa9_" to something like "18.221.142.15"
print("Got: %s" % ip_addr)
else:
print("Skipping DNS lookup; using hard-coded IP address.")
ip_addr = "18.224.102.166"
print("Now performing the HTTPS request.")
payload = b"GET /things/%s/shadow HTTP/1.0\r\nHost: %s\r\n\r\n" % (
thing_name, aws_endpoint)
tx = TXRequest(payload=payload,
dst=ip_addr,
tls_profile=b'\x00',
tls_port=8443)
ser.write(tx.packed)
print("TX: %s" % tx)
for i in range(10):
time.sleep(1)
rx = ser.read(1024)
print("RX: NotYetParsed(bytes=%s, hex=%s)" % (rx, to_hex(rx)))