class RdpcapSource(Source):
"""Read packets from a PCAP file send them to low exit.
+----------+
>>-| |->>
| |
>-| [pcap]--|->
+----------+
"""
def __init__(self, fname, name=None):
Source.__init__(self, name=name)
self.fname = fname
self.f = PcapReader(self.fname)
def start(self):
print "start"
self.f = PcapReader(self.fname)
self.is_exhausted = False
def stop(self):
print "stop"
self.f.close()
def fileno(self):
return self.f.fileno()
def deliver(self):
p = self.f.recv()
print "deliver %r" % p
if p is None:
self.is_exhausted = True
else:
self._send(p)
def sniff(count=0, store=1, offline=None, prn = None, lfilter=None, L2socket=None, timeout=None, *arg, **karg):
"""Sniff packets
sniff([count=0,] [prn=None,] [store=1,] [offline=None,] [lfilter=None,] + L2ListenSocket args) -> list of packets
Select interface to sniff by setting conf.iface. Use show_interfaces() to see interface names.
count: number of packets to capture. 0 means infinity
store: wether to store sniffed packets or discard them
prn: function to apply to each packet. If something is returned,
it is displayed. Ex:
ex: prn = lambda x: x.summary()
lfilter: python function applied to each packet to determine
if further action may be done
ex: lfilter = lambda x: x.haslayer(Padding)
offline: pcap file to read packets from, instead of sniffing them
timeout: stop sniffing after a given time (default: None)
L2socket: use the provided L2socket
"""
c = 0
if offline is None:
log_runtime.info('Sniffing on %s' % conf.iface)
if L2socket is None:
L2socket = conf.L2listen
s = L2socket(type=ETH_P_ALL, *arg, **karg)
else:
s = PcapReader(offline)
lst = []
if timeout is not None:
stoptime = time.time()+timeout
remain = None
while 1:
try:
if timeout is not None:
remain = stoptime-time.time()
if remain <= 0:
break
try:
p = s.recv(MTU)
except PcapTimeoutElapsed:
continue
if p is None:
break
if lfilter and not lfilter(p):
continue
if store:
lst.append(p)
c += 1
if prn:
r = prn(p)
if r is not None:
print(r)
if count > 0 and c >= count:
break
except KeyboardInterrupt:
break
s.close()
return plist.PacketList(lst,"Sniffed")
def read(file_name, start, count):
'''
read packets from pcap according to the start packet number and total count
'''
reader = PcapReader(file_name)
if start > 0:
reader.read_all(start)
if count > 0:
return reader.read_all(count)
else:
return reader.read_all(-1)
def wait_for_packet(self, timeout, filter_out_fn=is_ipv6_misc):
"""
Wait for next packet captured with a timeout
:param timeout: How long to wait for the packet
:returns: Captured packet if no packet arrived within timeout
:raises Exception: if no packet arrives within timeout
"""
deadline = time.time() + timeout
if self._pcap_reader is None:
if not self.wait_for_capture_file(timeout):
raise CaptureTimeoutError("Capture file %s did not appear "
"within timeout" % self.out_path)
while time.time() < deadline:
try:
self._pcap_reader = PcapReader(self.out_path)
break
except:
self.test.logger.debug(
"Exception in scapy.PcapReader(%s): %s" %
(self.out_path, format_exc()))
if not self._pcap_reader:
raise CaptureTimeoutError("Capture file %s did not appear within "
"timeout" % self.out_path)
poll = False
if timeout > 0:
self.test.logger.debug("Waiting for packet")
else:
poll = True
self.test.logger.debug("Polling for packet")
while time.time() < deadline or poll:
if not self.verify_enough_packet_data_in_pcap():
self._test.sleep(0) # yield
poll = False
continue
p = self._pcap_reader.recv()
if p is not None:
if filter_out_fn is not None and filter_out_fn(p):
self.test.logger.debug(
"Packet received after %ss was filtered out" %
(time.time() - (deadline - timeout)))
else:
self.test.logger.debug(
"Packet received after %fs" %
(time.time() - (deadline - timeout)))
return p
self._test.sleep(0) # yield
poll = False
self.test.logger.debug("Timeout - no packets received")
raise CaptureTimeoutError("Packet didn't arrive within timeout")
def __init__(self, iface=None, *args, **karg):
_usbpcap_check()
if iface is None:
warning("Available interfaces: [%s]" %
" ".join(x[0] for x in get_usbpcap_interfaces()))
raise NameError("No interface specified !"
" See get_usbpcap_interfaces()")
self.outs = None
args = ['-d', iface, '-b', '134217728', '-A', '-o', '-']
self.usbpcap_proc = subprocess.Popen([conf.prog.usbpcapcmd] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.ins = PcapReader(self.usbpcap_proc.stdout)
def modify_traffic(self, output_filename, fake_ttl=False,
src_mac=None, dst_mac=None, src_ip_addr=None, dst_ip_addr=None):
if fake_ttl is True and len(self.src_hosts_with_fake_ttl) == 0:
print("Please extract ip2hc table before modify traffic with fake ttl.")
# show modify request
request = "Generating " + output_filename + " with\n"
if fake_ttl:
request += " fake ttl\n"
if src_mac is not None:
request += " src mac:" + src_mac + "\n"
if dst_mac is not None:
request += " dst mac:" + dst_mac + "\n"
if src_ip_addr is not None:
request += " src ip addr:" + src_ip_addr + "\n"
if dst_ip_addr is not None:
request += " dst ip addr:" + dst_ip_addr + "\n"
print(request + "\n")
pcap_reader = PcapReader(self.pcap_filename)
pcap_writer = PcapWriter(output_filename)
counter = 0
while True:
pkt = pcap_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer('IP'):
continue
# ipv4 packets
counter += 1
ip_int = self.__ip_str2int(pkt['IP'].src)
if fake_ttl:
pkt['IP'].ttl = self.src_hosts_with_fake_ttl[ip_int]
if src_mac is not None:
pkt['Ethernet'].src = src_mac
if dst_mac is not None:
pkt['Ethernet'].dst = dst_mac
if src_ip_addr is not None:
pkt['IP'].src = src_ip_addr
if dst_ip_addr is not None:
pkt['IP'].dst = dst_ip_addr
pcap_writer.write(pkt)
if counter % 10000 == 0:
print("%d packets have been processed\n" % counter)
pcap_writer.flush()
pcap_writer.close()
pcap_reader.close()
def mysniff(self, *arg, **karg):
c = 0
if self.opened_socket is not None:
s = self.opened_socket
else:
if self.offline is None:
if self.L2socket is None:
self.L2socket = conf.L2listen
s = self.L2socket(type=ETH_P_ALL, *arg, **karg)
else:
s = PcapReader(self.offline)
lst = []
if self.timeout is not None:
stoptime = time.time() + self.timeout
remain = None
while 1:
if not self.running:
break
try:
if self.timeout is not None:
remain = stoptime - time.time()
if remain <= 0:
break
sel = select([s], [], [], remain)
if s in sel[0]:
p = s.recv(MTU)
if p is None:
break
if self.lfilter and not self.lfilter(p):
continue
if self.store:
lst.append(p)
c += 1
if self.prn:
r = self.prn(p)
if r is not None:
print r
if self.stop_filter and self.stop_filter(p):
break
if 0 < self.count <= c:
break
except KeyboardInterrupt:
break
if self.opened_socket is None:
s.close()
return plist.PacketList(lst, "Sniffed")
def mysniff(self, *arg, **karg):
c = 0
if self.opened_socket is not None:
s = self.opened_socket
else:
if self.offline is None:
if self.L2socket is None:
self.L2socket = conf.L2listen
s = self.L2socket(type=ETH_P_ALL, *arg, **karg)
else:
s = PcapReader(self.offline)
lst = []
if self.timeout is not None:
stoptime = time.time() + self.timeout
remain = None
while 1:
if not self.running:
break
try:
if self.timeout is not None:
remain = stoptime - time.time()
if remain <= 0:
break
sel = select([s], [], [], remain)
if s in sel[0]:
p = s.recv(MTU)
if p is None:
break
if self.lfilter and not self.lfilter(p):
continue
if self.store:
lst.append(p)
c += 1
if self.prn:
r = self.prn(p)
if r is not None:
print r
if self.stop_filter and self.stop_filter(p):
break
if 0 < self.count <= c:
break
except KeyboardInterrupt:
break
if self.opened_socket is None:
s.close()
return plist.PacketList(lst, "Sniffed")
class USBpcapSocket(SuperSocket):
"""
Read packets at layer 2 using USBPcapCMD
"""
def __init__(self, iface=None, *args, **karg):
_usbpcap_check()
if iface is None:
warning("Available interfaces: [%s]" %
" ".join(x[0] for x in get_usbpcap_interfaces()))
raise NameError("No interface specified !"
" See get_usbpcap_interfaces()")
self.outs = None
args = ['-d', iface, '-b', '134217728', '-A', '-o', '-']
self.usbpcap_proc = subprocess.Popen(
[conf.prog.usbpcapcmd] + args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
self.ins = PcapReader(self.usbpcap_proc.stdout)
def recv(self, x=MTU):
return self.ins.recv(x)
def close(self):
SuperSocket.close(self)
self.usbpcap_proc.kill()
class L2ListenTcpdump(SuperSocket):
desc = "read packets at layer 2 using tcpdump"
def __init__(self, iface=None, promisc=None, filter=None, nofilter=False,
prog=None, *arg, **karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is not None:
if WINDOWS:
try:
args.extend(['-i', iface.pcap_name])
except AttributeError:
args.extend(['-i', iface])
else:
args.extend(['-i', iface])
elif WINDOWS or DARWIN:
args.extend(['-i', conf.iface.pcap_name if WINDOWS else conf.iface])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.tcpdump_proc = tcpdump(None, prog=prog, args=args, getproc=True)
self.ins = PcapReader(self.tcpdump_proc.stdout)
def recv(self, x=MTU):
return self.ins.recv(x)
def close(self):
SuperSocket.close(self)
self.tcpdump_proc.kill()
class L2ListenTcpdump(SuperSocket):
desc = "read packets at layer 2 using tcpdump"
def __init__(self,
iface=None,
promisc=None,
filter=None,
nofilter=False,
prog=None,
*arg,
**karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is not None:
args.extend(['-i', iface])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.ins = PcapReader(tcpdump(None, prog=prog, args=args, getfd=True))
def recv(self, x=MTU):
return self.ins.recv(x)
def crypt2plain(self, pcapFile, encType, key):
"""Converts an encrypted stream to unencrypted stream
Returns the unencrypted stream input as a scapy PacketList object
Future plans involve offering a yield parameter so that pcapList,
instead returns as a generated object; should save memory this way.
Does not have the capability to diff between multiple keys encTypes
Possible workaround for this is taking the try and using except,
creating a return to let the user know which objs to retry on
For now, skipping.
"""
## Use the generator of PcapReader for memory purposes
pObj = PcapReader(pcapFile)
pcapList = []
## Deal with WEP
if encType == 'WEP':
for i in pObj:
try:
pkt, iv = pyDot11.wepDecrypt(i, key)
except:
pkt = i
pcapList.append(pkt)
## Return the stream like a normal Scapy PacketList
return PacketList(res=pcapList)
class USBpcapSocket(SuperSocket):
"""
Read packets at layer 2 using USBPcapCMD
"""
nonblocking_socket = True
@staticmethod
def select(sockets, remain=None):
return sockets
def __init__(self, iface=None, *args, **karg):
_usbpcap_check()
if iface is None:
warning("Available interfaces: [%s]",
" ".join(x[0] for x in get_usbpcap_interfaces()))
raise NameError("No interface specified !"
" See get_usbpcap_interfaces()")
iface = network_name(iface)
self.outs = None
args = ['-d', iface, '-b', '134217728', '-A', '-o', '-']
self.usbpcap_proc = subprocess.Popen([conf.prog.usbpcapcmd] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.ins = PcapReader(self.usbpcap_proc.stdout)
def recv(self, x=MTU):
return self.ins.recv(x)
def close(self):
SuperSocket.close(self)
self.usbpcap_proc.kill()
def __init__(self, filename, reader_id):
if filename.endswith(".pcap"):
self.reader = PcapReader(filename)
if filename.endswith(".pcapng"):
self.reader = PcapNgReader(filename)
self.reader_id = reader_id
self.current_packet = self.reader.read_packet()
def process_file(in_file, out_file):
"""Read through in_file pcap and convert to csv out_file."""
with PcapReader(in_file) as pcap_reader, open(out_file, 'w') as csvfile:
csv_writer = None
state = {}
for pkt in pcap_reader:
if TCP in pkt and pkt[TCP].dport == 515:
buf = pkt[TCP].load
offset = 0
state['time'] = pkt.time
while offset < len(buf):
block = cms_decode.cms_read_block_from_bytes(buf, offset)
if hasattr(block, 'values'):
state.update(block.values)
if hasattr(block, 'leads'):
state.update(block.leads)
if block.block_type == 0x47:
# write row
if not csv_writer:
fields = [
k for k in sorted(state.keys())
if 'alarm' not in k
]
csv_writer = csv.DictWriter(csvfile,
fields,
extrasaction='ignore')
csv_writer.writeheader()
csv_writer.writerow(state)
offset += 4 + block.length
def __init__(self, iface=None, promisc=None, filter=None, nofilter=False,
prog=None, *arg, **karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is not None:
if WINDOWS:
try:
args.extend(['-i', iface.pcap_name])
except AttributeError:
args.extend(['-i', iface])
else:
args.extend(['-i', iface])
elif WINDOWS or DARWIN:
args.extend(['-i', conf.iface.pcap_name if WINDOWS else conf.iface]) # noqa: E501
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.tcpdump_proc = tcpdump(None, prog=prog, args=args, getproc=True)
self.ins = PcapReader(self.tcpdump_proc.stdout)
def main():
if len(sys.argv) != 2:
sys.stderr.write("Usage: %s <filename.pcap>\n\n" % sys.argv[0])
filename = sys.argv[1]
messages = [seg for seg in PcapReader(filename) if "TCP" in seg]
segments = []
last_ack = {}
for m in messages:
segment = {}
segment["src"] = m.sprintf("%IP.src%:%TCP.sport%")
segment["dst"] = m.sprintf("%IP.dst%:%TCP.dport%")
flags = m.sprintf("%TCP.flags%")
segment["syn"] = "S" in flags
segment["fin"] = "F" in flags
segment["rst"] = "R" in flags
# Acks are special. Not only the A bit must be set, but also the seq number should increase
if "A" in flags:
conn_key = (segment["src"], segment["dst"])
if conn_key in last_ack:
old_ack = last_ack[conn_key]
else:
old_ack = m["TCP"].ack - 1
last_ack[conn_key] = m["TCP"].ack
segment["ack"] = old_ack != m["TCP"].ack
else:
segment["ack"] = False
segments.append(segment)
sys.stdout.write("[\n ")
sys.stdout.write(",\n ".join(str(seg) for seg in segments))
sys.stdout.write("\n]")
def arp_analyse(self, files_path):
# count = 0
for i in range(0, len(files_path)):
try:
packets = PcapReader(files_path[i])
while True:
packet = packets.read_packet()
if packet is None:
break
else:
# count = count+1
print(repr(packet))
packets.close()
except Scapy_Exception as e:
print(e)
def process_pcap_file(self, filename):
"""Processes a pcap file
"""
for pkt in PcapReader(filename):
self.mhp.process(pkt, rawpkt=pkt)
stats = self.mhp.get_stats()
print("Processed", stats['packets'], "packets")
def __init__(self, iface=None, promisc=None, filter=None, nofilter=False,
prog=None, *arg, **karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is not None:
args.extend(['-i', iface])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.ins = PcapReader(tcpdump(None, prog=prog, args=args, getfd=True))
def read_pcap_file(self, iface, filename):
try:
pcap_file = os.path.join(self.config['pcap_dir'], filename)
for pkt in PcapReader(pcap_file):
self.fprn(pkt, iface)
except (IOError, error.Scapy_Exception) as exc:
self.logger.exception('Cant read pcap file %s, err: %s', pcap_file,
str(exc))
def read_pcap():
if chinses_mode:
print("请输入pcap文件名:")
else:
file_name = input(
"Enter the pcap file name like 2019_11_02_16_55_22.pcap:"
) #输入pcap文件名
file_name = "sniff_data/" + file_name #组合文件路径
try:
reader = PcapReader(file_name) #用scapy打开pcap文件
except FileNotFoundError:
if chinses_mode:
print("找不到文件")
else:
print("Can nod find the file")
return
packets = reader.read_all(-1) #读取所有储存的数据包
for i in packets: #循环数据包列表
i.show() #打印数据包
def wait_for_packet(self, timeout, filter_out_fn=is_ipv6_misc):
"""
Wait for next packet captured with a timeout
:param timeout: How long to wait for the packet
:returns: Captured packet if no packet arrived within timeout
:raises Exception: if no packet arrives within timeout
"""
deadline = time.time() + timeout
if self._pcap_reader is None:
if not self.wait_for_capture_file(timeout):
raise CaptureTimeoutError("Capture file %s did not appear "
"within timeout" % self.out_path)
while time.time() < deadline:
try:
self._pcap_reader = PcapReader(self.out_path)
break
except:
self.test.logger.debug(
"Exception in scapy.PcapReader(%s): %s" %
(self.out_path, format_exc()))
if not self._pcap_reader:
raise CaptureTimeoutError("Capture file %s did not appear within "
"timeout" % self.out_path)
poll = False
if timeout > 0:
self.test.logger.debug("Waiting for packet")
else:
poll = True
self.test.logger.debug("Polling for packet")
while time.time() < deadline or poll:
if not self.verify_enough_packet_data_in_pcap():
self._test.sleep(0) # yield
poll = False
continue
p = self._pcap_reader.recv()
if p is not None:
if filter_out_fn is not None and filter_out_fn(p):
self.test.logger.debug(
"Packet received after %ss was filtered out" %
(time.time() - (deadline - timeout)))
else:
self.test.logger.debug(
"Packet received after %fs" %
(time.time() - (deadline - timeout)))
return p
self._test.sleep(0) # yield
poll = False
self.test.logger.debug("Timeout - no packets received")
raise CaptureTimeoutError("Packet didn't arrive within timeout")
def rdpcap(filename, count=-1):
"""
Tries to read file in nanosecond format and falls back to regular format.
"""
reader = None
try:
reader=NSPcapReader(filename)
except Scapy_Exception as e:
if 'Not a pcap capture file (bad magic)' == str(e):
reader = PcapReader(filename)
else:
raise e
return reader.read_all(count=count)
def PCAPread(file_name, start, count):
reader = PcapReader(file_name)
if start > 0:
reader.read_all(start)
if count > 0:
return reader.read_all(count)
else:
return reader.read_all(-1)
def __init__(self, iface=None, *args, **karg):
_usbpcap_check()
if iface is None:
warning("Available interfaces: [%s]" %
" ".join(x[0] for x in get_usbpcap_interfaces()))
raise NameError("No interface specified !"
" See get_usbpcap_interfaces()")
self.outs = None
args = ['-d', iface, '-b', '134217728', '-A', '-o', '-']
self.usbpcap_proc = subprocess.Popen(
[conf.prog.usbpcapcmd] + args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
self.ins = PcapReader(self.usbpcap_proc.stdout)
def analyse(self, files_path):
count = 0
for i in range(0, len(files_path)):
try:
packets = PcapReader(files_path[i])
while True:
packet = packets.read_packet()
if packet is None:
break
else:
count = count + 1
print(repr(packet))
# print(packet['DNS'].id)
# for packet in packets:
# print(repr(packet))
# #print(type(packet))
# #print(packet['CookedLinux'].src)
# break
print(count)
packets.close()
except Scapy_Exception as e:
print(e)
class L2ListenTcpdump(SuperSocket):
desc = "read packets at layer 2 using tcpdump"
def __init__(
self,
iface=None, # type: Optional[_GlobInterfaceType]
promisc=False, # type: bool
filter=None, # type: Optional[str]
nofilter=False, # type: bool
prog=None, # type: Optional[str]
*arg, # type: Any
**karg # type: Any
):
# type: (...) -> None
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is None and (WINDOWS or DARWIN):
iface = conf.iface
self.iface = iface
if iface is not None:
args.extend(['-i', network_name(iface)])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.tcpdump_proc = tcpdump(None, prog=prog, args=args, getproc=True)
self.reader = PcapReader(self.tcpdump_proc.stdout)
self.ins = self.reader # type: ignore
def recv(self, x=MTU):
# type: (int) -> Optional[Packet]
return self.reader.recv(x)
def close(self):
# type: () -> None
SuperSocket.close(self)
self.tcpdump_proc.kill()
@staticmethod
def select(sockets, remain=None):
# type: (List[SuperSocket], Optional[float]) -> List[SuperSocket]
if (WINDOWS or DARWIN):
return sockets
return SuperSocket.select(sockets, remain=remain)
def extract_ip2hc_table(self):
self.counter = 0
print("Reading pcap file: " + self.pcap_filename)
# src_hosts is a dict
pcap_reader = PcapReader(self.pcap_filename)
while True:
pkt = pcap_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer('IP'):
continue
self.counter += 1
# progress indicator
if self.counter % 10000 == 0:
print(str(self.counter) + "packets have been processed")
# ipv4 packets
ip_int = self.__ip_str2int(pkt['IP'].src)
if ip_int in self.src_hosts:
continue
# unrecorded source host
hc = self.__ttl2hc(pkt['IP'].ttl)
fake_ttl = self.__gen_fake_ttl(pkt['IP'].ttl)
if hc is not None and fake_ttl is not None:
self.src_hosts[ip_int] = hc
self.src_hosts_with_fake_ttl[ip_int] = fake_ttl
pcap_reader.close()
print(str(len(self.src_hosts)) + " source hosts extracted")
print("Writing ip,hc dict into " + self.ip_hc_filename + "...")
with open(self.ip_hc_filename, "w") as f:
json.dump(self.src_hosts, f)
f.close()
class RdpcapSource(Source):
"""Read packets from a PCAP file send them to low exit.
.. code::
+----------+
>>-| |->>
| |
>-| [pcap]--|->
+----------+
"""
def __init__(self, fname, name=None):
# type: (str, Optional[Any]) -> None
Source.__init__(self, name=name)
self.fname = fname
self.f = PcapReader(self.fname)
def start(self):
# type: () -> None
self.f = PcapReader(self.fname)
self.is_exhausted = False
def stop(self):
# type: () -> None
self.f.close()
def fileno(self):
# type: () -> int
return self.f.fileno()
def deliver(self):
# type: () -> None
try:
p = self.f.recv()
self._send(p)
except EOFError:
self.is_exhausted = True
def run(self):
while True:
self.pcap = PcapReader(self.f)
self.lasttime = 0
for p in self.pcap:
if (not (self.lasttime == 0)):
diff = (p.time - self.lasttime)
time.sleep(diff * self.time_scale)
self.lasttime = p.time
if (self.done):
return
v = pmt.to_pmt(numpy.fromstring(str(p), dtype=numpy.uint8))
meta = pmt.make_dict()
self.message_port_pub(pmt.intern("pdus"), pmt.cons(meta, v))
class RdpcapSource(Source):
"""Read packets from a PCAP file send them to low exit.
+----------+
>>-| |->>
| |
>-| [pcap]--|->
+----------+
"""
def __init__(self, fname, name=None):
Source.__init__(self, name=name)
self.fname = fname
self.f = PcapReader(self.fname)
def start(self):
print("start")
self.f = PcapReader(self.fname)
self.is_exhausted = False
def stop(self):
print("stop")
self.f.close()
def fileno(self):
return self.f.fileno()
def check_recv(self):
return True
def deliver(self):
p = self.f.recv()
print("deliver %r" % p)
if p is None:
self.is_exhausted = True
else:
self._send(p)
def wait_for_packet(self, timeout):
"""
Wait for next packet captured with a timeout
:param timeout: How long to wait for the packet
:returns: Captured packet if no packet arrived within timeout
:raises Exception: if no packet arrives within timeout
"""
limit = time.time() + timeout
if self._pcap_reader is None:
self.wait_for_capture_file(timeout)
self._pcap_reader = PcapReader(self.out_path)
self.test.logger.debug("Waiting for packet")
while time.time() < limit:
p = self._pcap_reader.recv()
if p is not None:
self.test.logger.debug("Packet received after %fs",
(time.time() - (limit - timeout)))
return p
time.sleep(0) # yield
self.test.logger.debug("Timeout - no packets received")
raise Exception("Packet didn't arrive within timeout")
class L2ListenTcpdump(SuperSocket):
desc = "read packets at layer 2 using tcpdump"
def __init__(self,
iface=None,
promisc=None,
filter=None,
nofilter=False,
prog=None,
*arg,
**karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is None and (WINDOWS or DARWIN):
iface = conf.iface
if WINDOWS:
try:
iface = iface.pcap_name
except AttributeError:
pass
self.iface = iface
if iface is not None:
args.extend(['-i', self.iface])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.tcpdump_proc = tcpdump(None, prog=prog, args=args, getproc=True)
self.ins = PcapReader(self.tcpdump_proc.stdout)
def recv(self, x=MTU):
return self.ins.recv(x)
def close(self):
SuperSocket.close(self)
self.tcpdump_proc.kill()
@staticmethod
def select(sockets, remain=None):
if (WINDOWS or DARWIN):
return sockets, None
return SuperSocket.select(sockets, remain=remain)
def pcap2flows(pth_pcap, num_pkt_thresh=2, verbose=True):
'''Reads pcap and divides packets into 5-tuple flows (arrival times and sizes)
Arguments:
pcap_file (string) = path to pcap file
num_pkt_thresh (int) = discards flows with fewer packets than max(2, thresh)
Returns:
flows (list) = [(fid, arrival times list, packet sizes list)]
'''
print(f'pcap_file: {pth_pcap}')
sessions = OrderedDict() # key order of fid by time
num_pkts = 0
pkt_sizes = []
"filter pcap_file only contains the special srcIP "
i = 0
FRAME_LEN = 1500 + 14
cnt = 0
try:
# sessions= rdpcap(pcap_file).sessions()
# res = PcapReader(pcap_file).read_all(count=-1)
# from scapy import plist
# sessions = plist.PacketList(res, name=os.path.basename(pcap_file)).sessions()
for i, pkt in enumerate(
PcapReader(pth_pcap)): # iteratively get packet from the pcap
if i % 10000 == 0:
print(f'i_pkt: {i}')
# sess_key = session_extractor(pkt) # this function treats bidirection as two sessions.
# if ('TCP' in sess_key) or ('UDP' in sess_key) or (6 in sess_key) or (17 in sess_key):
if (TCP in pkt) or (UDP in pkt):
if len(pkt) > FRAME_LEN:
cnt += 1
print(
f'i: {i}, pkt.wirelen: {pkt.wirelen}, len(pkt): {len(pkt)}, pkt.payload.len: {pkt.payload.len}'
)
pkt_sizes.append(len(pkt))
except Exception as e:
print('Error', e)
print(
f'tot_pkts: {i + 1}, {cnt} packets have wirelen > FRAME_LEN({FRAME_LEN})'
)
print(f'len(sessions) {len(sessions.keys())}')
return pkt_sizes
def __init__(self, iface=None, promisc=None, filter=None, nofilter=False,
prog=None, *arg, **karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is not None:
args.extend(['-i', iface])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.ins = PcapReader(tcpdump(None, prog=prog, args=args, getfd=True))
def parse_pcap_files(self, pcapFiles, quite=True):
"""
Take one more more (list, or tuple) of pcap files and parse them
into the engine.
"""
if not hasattr(pcapFiles, '__iter__'):
if isinstance(pcapFiles, str):
pcapFiles = [pcapFiles]
else:
return
for i in range(0, len(pcapFiles)):
pcap = pcapFiles[i]
pcapName = os.path.split(pcap)[1]
if not quite:
sys.stdout.write("Reading PCap File: {0}\r".format(pcapName))
sys.stdout.flush()
if not os.path.isfile(pcap):
if not quite:
sys.stdout.write("Skipping File {0}: File Not Found\n".format(pcap))
sys.stdout.flush()
continue
elif not os.access(pcap, os.R_OK):
if not quite:
sys.stdout.write("Skipping File {0}: Permissions Issue\n".format(pcap))
sys.stdout.flush()
continue
pcapr = PcapReader(pcap) # pylint: disable=no-value-for-parameter
packet = pcapr.read_packet()
i = 1
try:
while packet:
if not quite:
sys.stdout.write('Parsing File: ' + pcap + ' Packets Done: ' + str(i) + '\r')
sys.stdout.flush()
self.parse_wireless_packet(packet)
packet = pcapr.read_packet()
i += 1
i -= 1
if not quite:
sys.stdout.write((' ' * len('Parsing File: ' + pcap + ' Packets Done: ' + str(i))) + '\r')
sys.stdout.write('Done With File: ' + pcap + ' Read ' + str(i) + ' Packets\n')
sys.stdout.flush()
except KeyboardInterrupt:
if not quite:
sys.stdout.write("Skipping File {0} Due To Ctl+C\n".format(pcap))
sys.stdout.flush()
except: # pylint: disable=bare-except
if not quite:
sys.stdout.write("Skipping File {0} Due To Scapy Exception\n".format(pcap))
sys.stdout.flush()
self.fragment_buffer = {}
pcapr.close()
def _read_pcap(self):
self._say("reading file {name}".format(name=self.pcap_file))
start_time = time.time()
try:
fdesc = PcapReader(self.pcap_file)
except (IOError, Scapy_Exception) as err:
self._say("rdpcap: error: {}".format(err), file=sys.stderr)
self._app_shutdown()
except NameError as err:
self._say("rdpcap error: not a pcap file ({})".format(err),
file=sys.stderr)
self._app_shutdown()
except KeyboardInterrupt:
self._app_shutdown()
self._read_pkts(fdesc)
read_time = time.time()
self._say("took {0:.3f} seconds to read and parse"\
.format(read_time - start_time))
def dst_protocol_analysis(pid, d_mac, result_list):
result = []
for f in filenames[pid]:
for p in PcapReader(f):
packet_len = len(p)
p_ip, snd_rcv = get_pak_ip(p, d_mac)
if p_ip != 'non-ip' and p_ip in dst_info:
p_protocol = get_pak_protocol(packet=p, d_mac=d_mac)
host = dst_info[p_ip]
if p_protocol in protocol_info:
prot = protocol_info[p_protocol]
else:
prot = ProtocolPort.ProtocolPort(p_protocol, '-1', '-1',
'-1', '-1')
index = 0
is_old = False
for dst_pro in result:
if host == dst_pro.host and prot == dst_pro.protocol_port:
is_old = True
break
index += 1
if is_old:
if snd_rcv == 'snd':
result[index].add_snd(packet_len)
result[index].add_ps(1)
else:
result[index].add_rcv(packet_len)
result[index].add_pr(1)
else:
current: DestinationPro.DestinationPro
current = DestinationPro.DestinationPro(host, prot)
if snd_rcv == 'snd':
current.add_snd(packet_len)
current.add_ps(1)
else:
current.add_rcv(packet_len)
current.add_pr(1)
result.append(current)
result_list[pid] = result
def parse_file( tup ):
path, content = tup
ftmp = tempfile.NamedTemporaryFile(delete=True)
ftmp.write(content)
ftmp.flush()
pcap_reader = PcapReader(ftmp.name)
count=0
for packet in pcap_reader:
count+=1
try:
row = parse_packet(packet)
if row is not None:
yield row
except:
pass
ftmp.close()
class L2ListenTcpdump(SuperSocket):
desc = "read packets at layer 2 using tcpdump"
def __init__(self, iface=None, promisc=None, filter=None, nofilter=False,
prog=None, *arg, **karg):
self.outs = None
args = ['-w', '-', '-s', '65535']
if iface is not None:
args.extend(['-i', iface])
if not promisc:
args.append('-p')
if not nofilter:
if conf.except_filter:
if filter:
filter = "(%s) and not (%s)" % (filter, conf.except_filter)
else:
filter = "not (%s)" % conf.except_filter
if filter is not None:
args.append(filter)
self.ins = PcapReader(tcpdump(None, prog=prog, args=args, getfd=True))
def recv(self, x=MTU):
return self.ins.recv(x)
def start(self):
print("start")
self.f = PcapReader(self.fname)
self.is_exhausted = False
def __init__(self, fname, name=None):
Source.__init__(self, name=name)
self.fname = fname
self.f = PcapReader(self.fname)
class VppPGInterface(VppInterface):
"""
VPP packet-generator interface
"""
@property
def pg_index(self):
"""packet-generator interface index assigned by VPP"""
return self._pg_index
@property
def out_path(self):
"""pcap file path - captured packets"""
return self._out_path
@property
def in_path(self):
""" pcap file path - injected packets"""
return self._in_path
@property
def capture_cli(self):
"""CLI string to start capture on this interface"""
return self._capture_cli
@property
def cap_name(self):
"""capture name for this interface"""
return self._cap_name
@property
def input_cli(self):
"""CLI string to load the injected packets"""
return self._input_cli
@property
def in_history_counter(self):
"""Self-incrementing counter used when renaming old pcap files"""
v = self._in_history_counter
self._in_history_counter += 1
return v
@property
def out_history_counter(self):
"""Self-incrementing counter used when renaming old pcap files"""
v = self._out_history_counter
self._out_history_counter += 1
return v
def __init__(self, test, pg_index):
""" Create VPP packet-generator interface """
super(VppPGInterface, self).__init__(test)
r = test.vapi.pg_create_interface(pg_index)
self.set_sw_if_index(r.sw_if_index)
self._in_history_counter = 0
self._out_history_counter = 0
self._out_assert_counter = 0
self._pg_index = pg_index
self._out_file = "pg%u_out.pcap" % self.pg_index
self._out_path = self.test.tempdir + "/" + self._out_file
self._in_file = "pg%u_in.pcap" % self.pg_index
self._in_path = self.test.tempdir + "/" + self._in_file
self._capture_cli = "packet-generator capture pg%u pcap %s" % (
self.pg_index, self.out_path)
self._cap_name = "pcap%u" % self.sw_if_index
self._input_cli = \
"packet-generator new pcap %s source pg%u name %s" % (
self.in_path, self.pg_index, self.cap_name)
def enable_capture(self):
""" Enable capture on this packet-generator interface"""
try:
if os.path.isfile(self.out_path):
name = "%s/history.[timestamp:%f].[%s-counter:%04d].%s" % \
(self.test.tempdir,
time.time(),
self.name,
self.out_history_counter,
self._out_file)
self.test.logger.debug("Renaming %s->%s" %
(self.out_path, name))
os.rename(self.out_path, name)
except:
pass
# FIXME this should be an API, but no such exists atm
self.test.vapi.cli(self.capture_cli)
self._pcap_reader = None
def add_stream(self, pkts):
"""
Add a stream of packets to this packet-generator
:param pkts: iterable packets
"""
try:
if os.path.isfile(self.in_path):
name = "%s/history.[timestamp:%f].[%s-counter:%04d].%s" %\
(self.test.tempdir,
time.time(),
self.name,
self.in_history_counter,
self._in_file)
self.test.logger.debug("Renaming %s->%s" %
(self.in_path, name))
os.rename(self.in_path, name)
except:
pass
wrpcap(self.in_path, pkts)
self.test.register_capture(self.cap_name)
# FIXME this should be an API, but no such exists atm
self.test.vapi.cli(self.input_cli)
def generate_debug_aid(self, kind):
""" Create a hardlink to the out file with a counter and a file
containing stack trace to ease debugging in case of multiple capture
files present. """
self.test.logger.debug("Generating debug aid for %s on %s" %
(kind, self._name))
link_path, stack_path = ["%s/debug_%s_%s_%s.%s" %
(self.test.tempdir, self._name,
self._out_assert_counter, kind, suffix)
for suffix in ["pcap", "stack"]
]
os.link(self.out_path, link_path)
with open(stack_path, "w") as f:
f.writelines(format_stack())
self._out_assert_counter += 1
def _get_capture(self, timeout, filter_out_fn=is_ipv6_misc):
""" Helper method to get capture and filter it """
try:
if not self.wait_for_capture_file(timeout):
return None
output = rdpcap(self.out_path)
self.test.logger.debug("Capture has %s packets" % len(output.res))
except:
self.test.logger.debug("Exception in scapy.rdpcap (%s): %s" %
(self.out_path, format_exc()))
return None
before = len(output.res)
if filter_out_fn:
output.res = [p for p in output.res if not filter_out_fn(p)]
removed = before - len(output.res)
if removed:
self.test.logger.debug(
"Filtered out %s packets from capture (returning %s)" %
(removed, len(output.res)))
return output
def get_capture(self, expected_count=None, remark=None, timeout=1,
filter_out_fn=is_ipv6_misc):
""" Get captured packets
:param expected_count: expected number of packets to capture, if None,
then self.test.packet_count_for_dst_pg_idx is
used to lookup the expected count
:param remark: remark printed into debug logs
:param timeout: how long to wait for packets
:param filter_out_fn: filter applied to each packet, packets for which
the filter returns True are removed from capture
:returns: iterable packets
"""
remaining_time = timeout
capture = None
name = self.name if remark is None else "%s (%s)" % (self.name, remark)
based_on = "based on provided argument"
if expected_count is None:
expected_count = \
self.test.get_packet_count_for_if_idx(self.sw_if_index)
based_on = "based on stored packet_infos"
if expected_count == 0:
raise Exception(
"Internal error, expected packet count for %s is 0!" %
name)
self.test.logger.debug("Expecting to capture %s (%s) packets on %s" % (
expected_count, based_on, name))
while remaining_time > 0:
before = time.time()
capture = self._get_capture(remaining_time, filter_out_fn)
elapsed_time = time.time() - before
if capture:
if len(capture.res) == expected_count:
# bingo, got the packets we expected
return capture
elif len(capture.res) > expected_count:
self.test.logger.error(
ppc("Unexpected packets captured:", capture))
break
else:
self.test.logger.debug("Partial capture containing %s "
"packets doesn't match expected "
"count %s (yet?)" %
(len(capture.res), expected_count))
elif expected_count == 0:
# bingo, got None as we expected - return empty capture
return PacketList()
remaining_time -= elapsed_time
if capture:
self.generate_debug_aid("count-mismatch")
raise Exception("Captured packets mismatch, captured %s packets, "
"expected %s packets on %s" %
(len(capture.res), expected_count, name))
else:
raise Exception("No packets captured on %s" % name)
def assert_nothing_captured(self, remark=None, filter_out_fn=is_ipv6_misc):
""" Assert that nothing unfiltered was captured on interface
:param remark: remark printed into debug logs
:param filter_out_fn: filter applied to each packet, packets for which
the filter returns True are removed from capture
"""
if os.path.isfile(self.out_path):
try:
capture = self.get_capture(
0, remark=remark, filter_out_fn=filter_out_fn)
if not capture or len(capture.res) == 0:
# junk filtered out, we're good
return
except:
pass
self.generate_debug_aid("empty-assert")
if remark:
raise AssertionError(
"Non-empty capture file present for interface %s (%s)" %
(self.name, remark))
else:
raise AssertionError("Capture file present for interface %s" %
self.name)
def wait_for_capture_file(self, timeout=1):
"""
Wait until pcap capture file appears
:param timeout: How long to wait for the packet (default 1s)
:returns: True/False if the file is present or appears within timeout
"""
deadline = time.time() + timeout
if not os.path.isfile(self.out_path):
self.test.logger.debug("Waiting for capture file %s to appear, "
"timeout is %ss" % (self.out_path, timeout))
else:
self.test.logger.debug("Capture file %s already exists" %
self.out_path)
return True
while time.time() < deadline:
if os.path.isfile(self.out_path):
break
self._test.sleep(0) # yield
if os.path.isfile(self.out_path):
self.test.logger.debug("Capture file appeared after %fs" %
(time.time() - (deadline - timeout)))
else:
self.test.logger.debug("Timeout - capture file still nowhere")
return False
return True
def verify_enough_packet_data_in_pcap(self):
"""
Check if enough data is available in file handled by internal pcap
reader so that a whole packet can be read.
:returns: True if enough data present, else False
"""
orig_pos = self._pcap_reader.f.tell() # save file position
enough_data = False
# read packet header from pcap
packet_header_size = 16
caplen = None
end_pos = None
hdr = self._pcap_reader.f.read(packet_header_size)
if len(hdr) == packet_header_size:
# parse the capture length - caplen
sec, usec, caplen, wirelen = struct.unpack(
self._pcap_reader.endian + "IIII", hdr)
self._pcap_reader.f.seek(0, 2) # seek to end of file
end_pos = self._pcap_reader.f.tell() # get position at end
if end_pos >= orig_pos + len(hdr) + caplen:
enough_data = True # yay, we have enough data
self._pcap_reader.f.seek(orig_pos, 0) # restore original position
return enough_data
def wait_for_packet(self, timeout, filter_out_fn=is_ipv6_misc):
"""
Wait for next packet captured with a timeout
:param timeout: How long to wait for the packet
:returns: Captured packet if no packet arrived within timeout
:raises Exception: if no packet arrives within timeout
"""
deadline = time.time() + timeout
if self._pcap_reader is None:
if not self.wait_for_capture_file(timeout):
raise CaptureTimeoutError("Capture file %s did not appear "
"within timeout" % self.out_path)
while time.time() < deadline:
try:
self._pcap_reader = PcapReader(self.out_path)
break
except:
self.test.logger.debug(
"Exception in scapy.PcapReader(%s): %s" %
(self.out_path, format_exc()))
if not self._pcap_reader:
raise CaptureTimeoutError("Capture file %s did not appear within "
"timeout" % self.out_path)
poll = False
if timeout > 0:
self.test.logger.debug("Waiting for packet")
else:
poll = True
self.test.logger.debug("Polling for packet")
while time.time() < deadline or poll:
if not self.verify_enough_packet_data_in_pcap():
self._test.sleep(0) # yield
poll = False
continue
p = self._pcap_reader.recv()
if p is not None:
if filter_out_fn is not None and filter_out_fn(p):
self.test.logger.debug(
"Packet received after %ss was filtered out" %
(time.time() - (deadline - timeout)))
else:
self.test.logger.debug(
"Packet received after %fs" %
(time.time() - (deadline - timeout)))
return p
self._test.sleep(0) # yield
poll = False
self.test.logger.debug("Timeout - no packets received")
raise CaptureTimeoutError("Packet didn't arrive within timeout")
def create_arp_req(self):
"""Create ARP request applicable for this interface"""
return (Ether(dst="ff:ff:ff:ff:ff:ff", src=self.remote_mac) /
ARP(op=ARP.who_has, pdst=self.local_ip4,
psrc=self.remote_ip4, hwsrc=self.remote_mac))
def create_ndp_req(self):
"""Create NDP - NS applicable for this interface"""
nsma = in6_getnsma(inet_pton(socket.AF_INET6, self.local_ip6))
d = inet_ntop(socket.AF_INET6, nsma)
return (Ether(dst=in6_getnsmac(nsma)) /
IPv6(dst=d, src=self.remote_ip6) /
ICMPv6ND_NS(tgt=self.local_ip6) /
ICMPv6NDOptSrcLLAddr(lladdr=self.remote_mac))
def resolve_arp(self, pg_interface=None):
"""Resolve ARP using provided packet-generator interface
:param pg_interface: interface used to resolve, if None then this
interface is used
"""
if pg_interface is None:
pg_interface = self
self.test.logger.info("Sending ARP request for %s on port %s" %
(self.local_ip4, pg_interface.name))
arp_req = self.create_arp_req()
pg_interface.add_stream(arp_req)
pg_interface.enable_capture()
self.test.pg_start()
self.test.logger.info(self.test.vapi.cli("show trace"))
try:
captured_packet = pg_interface.wait_for_packet(1)
except:
self.test.logger.info("No ARP received on port %s" %
pg_interface.name)
return
arp_reply = captured_packet.copy() # keep original for exception
# Make Dot1AD packet content recognizable to scapy
if arp_reply.type == 0x88a8:
arp_reply.type = 0x8100
arp_reply = Ether(scapy.compat.raw(arp_reply))
try:
if arp_reply[ARP].op == ARP.is_at:
self.test.logger.info("VPP %s MAC address is %s " %
(self.name, arp_reply[ARP].hwsrc))
self._local_mac = arp_reply[ARP].hwsrc
else:
self.test.logger.info("No ARP received on port %s" %
pg_interface.name)
except:
self.test.logger.error(
ppp("Unexpected response to ARP request:", captured_packet))
raise
def resolve_ndp(self, pg_interface=None, timeout=1):
"""Resolve NDP using provided packet-generator interface
:param pg_interface: interface used to resolve, if None then this
interface is used
:param timeout: how long to wait for response before giving up
"""
if pg_interface is None:
pg_interface = self
self.test.logger.info("Sending NDP request for %s on port %s" %
(self.local_ip6, pg_interface.name))
ndp_req = self.create_ndp_req()
pg_interface.add_stream(ndp_req)
pg_interface.enable_capture()
self.test.pg_start()
now = time.time()
deadline = now + timeout
# Enabling IPv6 on an interface can generate more than the
# ND reply we are looking for (namely MLD). So loop through
# the replies to look for want we want.
while now < deadline:
try:
captured_packet = pg_interface.wait_for_packet(
deadline - now, filter_out_fn=None)
except:
self.test.logger.error(
"Timeout while waiting for NDP response")
raise
ndp_reply = captured_packet.copy() # keep original for exception
# Make Dot1AD packet content recognizable to scapy
if ndp_reply.type == 0x88a8:
self._test.logger.info(
"Replacing EtherType: 0x88a8 with "
"0x8100 and regenerating Ethernet header. ")
ndp_reply.type = 0x8100
ndp_reply = Ether(scapy.compat.raw(ndp_reply))
try:
ndp_na = ndp_reply[ICMPv6ND_NA]
opt = ndp_na[ICMPv6NDOptDstLLAddr]
self.test.logger.info("VPP %s MAC address is %s " %
(self.name, opt.lladdr))
self._local_mac = opt.lladdr
self.test.logger.debug(self.test.vapi.cli("show trace"))
# we now have the MAC we've been after
return
except:
self.test.logger.info(
ppp("Unexpected response to NDP request:",
captured_packet))
now = time.time()
self.test.logger.debug(self.test.vapi.cli("show trace"))
raise Exception("Timeout while waiting for NDP response")