def get_whole_packet_array(pcap_file, proto):
labels = []
arr = get_array_i(pcap_file, proto)
arra = []
pcapa = pcap.pcap(pcap_file)
i = 0
for ts, pkt in pcapa:
i = i + 1
if i not in arr:
continue
try:
ether = Ethernet(pkt)
ip = ether.data
if isinstance(ip, str):
ip = IP(ether.data)
except Exception as e:
try:
ip = IP(pkt)
except:
continue
tcp = ip.data
if isinstance(ip.data, str):
tcp = TCP(ip.data)
#tmparr = []
arra.append(str(pkt).encode("hex"))
return arra
def _packet_parse(self, pktdata):
"""
Parses the protocols in the given packet data and returns the
resulting packet (here, as a dict indexed by the protocol layers
in form of dpkt classes).
"""
layer = Ethernet(pktdata)
pkt = {}
if isinstance(layer.data, IP):
pkt[IP] = layer = layer.data
elif isinstance(layer.data, IP6):
# XXX This does not correctly skip IPv6 extension headers
pkt[IP6] = layer = layer.data
else:
return pkt
if isinstance(layer.data, ICMP):
pkt[ICMP] = layer.data
elif isinstance(layer.data, ICMP6):
pkt[ICMP6] = layer.data
elif isinstance(layer.data, TCP):
pkt[TCP] = layer.data
elif isinstance(layer.data, UDP):
pkt[UDP] = layer.data
elif isinstance(layer.data, SCTP):
pkt[SCTP] = layer.data
return pkt
def get_array(pcap_file, proto):
labels = []
arr = get_array_i(pcap_file, proto)
arra = []
pcapa = pcap.pcap(pcap_file)
i = 0
for ts, pkt in pcapa:
i = i + 1
if i not in arr:
continue
try:
ether = Ethernet(pkt)
ip = ether.data
if isinstance(ip, str):
ip = IP(ether.data)
except Exception as e:
try:
ip = IP(pkt)
except:
continue
tcp = ip.data
if isinstance(ip.data, str):
tcp = TCP(ip.data)
#tmparr = []
arra.append(conv(tcp.data))
labels.append(proto)
return (arra, labels)
def extract_arp(self, packet, timestamp):
'''
Extracts an ARP record from a packet
'''
raw_packet = packet
ethernet_frame = Ethernet(raw_packet)
thetime = time.strftime("%Y%m%d%H%M%S", time.gmtime(timestamp))
arp = ethernet_frame.data
src_mac = hexlify(arp.sha)
src_ip_encoded = hexlify(arp.spa)
dst_ip_encoded = hexlify(arp.tpa)
src_ip = '.'.join(
str(int(i, 16)) for i in ([
src_ip_encoded[i:i + 2]
for i in range(0, len(src_ip_encoded), 2)
]))
dst_ip = '.'.join(
str(int(i, 16)) for i in ([
dst_ip_encoded[i:i + 2]
for i in range(0, len(dst_ip_encoded), 2)
]))
if (self._pba.cfg('debug')):
print src_mac
print src_ip
print dst_ip
record = PBARecordARPRequest(raw_packet, timestamp)
record.assign(thetime, src_mac, src_ip)
return record
def getIp():
global error, totalN
# 取默认网卡
# name = pcap.findalldevs()
try:
dataPack = pcap.pcap(name=NAME, promisc=True, immediate=True)
# dataPack.setfilter('udp port 9991')
# dataPack.setfilter('tcp')
logger.info('连接网卡->%s，开始抓包', NAME)
except Exception as e:
logger.error('连接网卡->%s失败，强制退出，错误信息->%s', NAME, e)
error = True
sys.exit(1)
else:
for ptime, pdata in dataPack:
totalN += 1
# 解包，获得数据链路层包
Ethernet_pack = Ethernet(pdata)
# 扩展dpkt解析ERSPAN数据
Ethernet.set_type(ETH_TYPE_ERSPAN1, Ethernet)
try:
parseTCP(Ethernet_pack)
# dataBase.insert(tags, fields)
except Exception as e:
pass
dataPack.close()
def get_test_array(pcap_file, proto):
labels = []
actuals = []
arra = []
pcapa = pcap.pcap(pcap_file)
i = 0
for ts, pkt in pcapa:
i = i + 1
try:
ether = Ethernet(pkt)
ip = ether.data
if isinstance(ip, str):
ip = IP(ether.data)
except Exception as e:
try:
ip = IP(pkt)
except:
continue
if isinstance(ip.data, str):
tcp = TCP(ip.data)
else:
tcp = ip.data
tmpstr = ""
#tmparr = []
arra.append(conv(tcp.data))
labels.append(proto)
actuals.append(i)
return (arra, labels, actuals)
def shark(file):
"""
Parse pcap (tcpdump) to extract networking info and CA commands.
This function is also accessible via a CLI installed with caproto. Example::
sudo tcpdump -w - | caproto-shark
Parameters
----------
file : buffer
Yields
------
command_context : SimpleNamespace
Contains timestamp, ethernet, src, dst, ip, transport, and command.
"""
banned = set()
for timestamp, buffer in Reader(file):
ethernet = Ethernet(buffer)
ip = ethernet.data
if not isinstance(ip, IP):
continue
transport = ip.data
if not isinstance(transport, (TCP, UDP)):
continue
try:
src = inet_ntoa(ip.src)
dst = inet_ntoa(ip.dst)
if isinstance(transport, TCP):
if (ip.src, transport.sport) in banned:
continue
data = bytearray(transport.data)
while True:
data, command, _ = read_from_bytestream(data)
if command is NEED_DATA:
break
yield SimpleNamespace(timestamp=timestamp,
ethernet=ethernet,
src=src,
dst=dst,
ip=ip,
transport=transport,
command=command)
elif isinstance(transport, UDP):
if (ip.src, transport.sport) in banned:
continue
address = inet_ntoa(ip.src)
for command in read_datagram(transport.data, address):
yield SimpleNamespace(timestamp=timestamp,
ethernet=ethernet,
src=src,
dst=dst,
ip=ip,
transport=transport,
command=command)
except ValidationError:
banned.add((ip.src, transport.sport))
def read_payloads_from_file(file, tokens: list[str]):
"""Read the given pcap file and yield src, dst, and result."""
pcap = dpkt.pcap.Reader(file)
stats: defaultdict[str, Counter] = defaultdict(Counter)
for _ts, pkt in pcap:
eth = Ethernet(pkt)
if eth.type != ETH_TYPE_IP:
continue
ip = eth.ip
if ip.p != 17:
continue
transport = ip.udp
if transport.dport != 54321 and transport.sport != 54321:
continue
data = transport.data
src_addr = str(ip_address(ip.src))
dst_addr = str(ip_address(ip.dst))
decrypted = None
for token in tokens:
try:
decrypted = Message.parse(data, token=bytes.fromhex(token))
break
except BaseException:
continue
if decrypted is None:
continue
stats["stats"]["miio_packets"] += 1
if decrypted.data.length == 0:
stats["stats"]["empty_packets"] += 1
continue
stats["dst_addr"][dst_addr] += 1
stats["src_addr"][src_addr] += 1
payload = decrypted.data.value
if "result" in payload:
stats["stats"]["results"] += 1
if "method" in payload:
method = payload["method"]
stats["commands"][method] += 1
yield src_addr, dst_addr, payload
print(stats) # noqa: T201
def printPcap(pcap):
# 遍历[timestamp, packet]记录的数组
for (ts, buf) in pcap:
try:
# 获取以太网部分数据
Ethernet_pack = Ethernet(buf)
Ethernet.set_type(ETH_TYPE_ERSPAN1, Ethernet)
# v = dpkt.gre.GRE(Ethernet_pack.data.data.data)
parseTCP(Ethernet_pack)
except:
print('出错')
pass
def process(self, name):
self.logger.debug("Snort alert queue contains: " +
str(snort_alert_queue.qsize()) + " entries")
palerts = []
ws = 'http://' + self.args[
1] + ':' + self.options.remote + '/flower/analysis/AlertsService?wsdl'
self.logger.debug('Connecting to web service: ' + ws +
" for Snort alerts")
try:
client = Client(ws)
while (not snort_alert_queue.empty()):
(msg, ts_sec, ts_usec, caplen, pktlen, dlthdr, nethdr,
transhdr, data, val,
pkt) = struct.unpack(fmt, snort_alert_queue.get_nowait())
ethernet = Ethernet(pkt)
ip = ethernet.data
alert = msg.rstrip("\0")
sport = None
dport = None
if (ip.p == 6 or ip.p == 17):
sport = ip.data.sport
dport = ip.data.dport
packet = base64.b64encode(re.sub(r'\x00*$', '', pkt))
palert = client.factory.create("snortAlert")
palert.date = ts_sec
palert.usec = ts_usec
palert.sourceAddress = self.parse_address(ip.src)
palert.destinationAddress = self.parse_address(ip.dst)
palert.sourcePort = sport
palert.destinationPort = dport
palert.alert = alert
palert.packet = packet
palerts.append(palert)
if (client.service.addSnortAlerts(palerts) != len(palerts)):
self.logger.error(
'Number of alerts reported as received by server does not match number sent!'
)
except URLError:
self.logger.error('Failed to connect to Snort alert web service')
if (not self.stop_flag):
self.tasks.enter(15, 1, self.process, ('process', ))
def get_ip_ethernet(buf):
"""Unpack the data within the Ethernet frame (the IP packet)
Pulling out src, dst, length, fragment info, TTL, and Protocol
:param buf: A string buffer containing the entire packet
:type buf: str
:return: ip of type (dpkt.IP)
"""
lp = Ethernet(buf)
lp.unpack(buf)
ip = lp.data
return ip
def iterate(pktlen, pkt, timestamp):
eth = Ethernet(pkt)
if eth.type == ETH_TYPE_IP:
ip = IP(str(eth.data))
if ip.p == IP_PROTO_TCP:
tcp = TCP(str(ip.data))
f = {
'src': ip.src,
'sport': tcp.sport,
'dst': ip.dst,
'dport': tcp.dport
}
if not f in self.streams:
self.streams.append(f)
def read_pcap(filepath):
with open(filepath, "rb") as pcap_file:
filename, file_extension = os.path.splitext(filepath)
if file_extension == ".pcap":
pcap_data = pcap.Reader(pcap_file)
elif file_extension == ".pcapng":
pcap_data = pcapng.Reader(pcap_file)
else:
raise ValueError("pcap and pcapng is supported!")
for ts, buf in pcap_data:
eth = Ethernet(buf)
ip = eth.data
tcp = ip.data
def send_arp(fp):
arp = ARP(op=ARP_OP_REPLY,
sha='\xac\xf7\xf3\x00\x00\x00',
spa=inet_aton('10.12.0.253'),
tha='\xff\xff\xff\xff\xff\xff',
tpa=inet_aton('10.12.255.255'))
eth = Ethernet(src='\xac\xf7\xf3\x00\x00\x00',
dst='\xff\xff\xff\xff\xff\xff',
type=ETH_TYPE_ARP,
data=arp)
packet = str(eth)
if (pcap_sendpacket(fp, build_packet(packet), len(packet)) != 0):
print("Error sending the packet:\n %s" % pcap_geterr(fp))
def send_packet(fp, msg):
http = u"HTTP/1.1 %s\r\n\r\njust for fun" % msg
tcp = TCP(sport=80,
dport=random.uniform(0, 65535), data=http)
ip = IP(src=inet_aton('19.89.6.4'), dst=inet_aton('20.13.9.27'),
p=IP_PROTO_TCP, data=tcp, len=20 + len(str(tcp)))
eth = Ethernet(src='\xac\xf7\xf3\x00\x00\x00',
dst='\x00\x00\x00\x00\x00\x00',
data=ip)
packet = str(eth)
if (pcap_sendpacket(fp, build_packet(packet), len(packet)) != 0):
print("Error sending the packet:\n %s" % pcap_geterr(fp))
def generate_mapped_pcap(self, ip_mappings, output_pcap_file_path):
"""
:param ip_mappings: A dictionary where keys are IP address strings desired to be mapped to value strings
:param mapped_pcap_file_path: The path of output PCAP file.
:return: None
"""
pcap_reader = dpkt.pcap.Reader(open(self.input_pcap_file_path, "rb"))
l2_type = pcap_reader.datalink()
pcap_writer = dpkt.pcap.Writer(open(output_pcap_file_path, "w"),
linktype=l2_type)
for ts, buf in pcap_reader:
l2 = None
# Unpack the data within the Ethernet frame (the IP packet)
# Pulling out src, dst, length, fragment info, TTL, and Protocol
if l2_type == dpkt.pcap.DLT_NULL:
l2 = Loopback(buf)
elif l2_type == dpkt.pcap.DLT_LINUX_SLL:
l2 = SLL(buf)
else:
l2 = Ethernet(buf)
l2.unpack(buf)
modified_ip = l2.data
src_ip_str, dst_ip_str = get_src_dst_ip_str(l2.data)
print 'IP: %s -> %s ' % (src_ip_str, dst_ip_str)
if src_ip_str in ip_mappings:
modified_ip.src = str_to_inet(ip_mappings[src_ip_str])
if dst_ip_str in ip_mappings:
modified_ip.dst = str_to_inet(ip_mappings[dst_ip_str])
modified_ip.len = len(modified_ip)
modified_ip.sum = 0x0000
l2.data = str(modified_ip)
pcap_writer.writepkt(str(l2), ts)
pcap_writer.close()
def extract_nbds(self, packet, timestamp):
'''
Extracts NBDS record from a packet
'''
raw_packet = packet
ethernet_frame = Ethernet(raw_packet)
ethernet_data = ethernet_frame.data
udp_packet = ethernet_data.data
src_mac = hexlify(ethernet_frame['src'])
src_ip = hexlify(ethernet_data['src'])
dst_ip = hexlify(ethernet_data['dst'])
srcip = str(int(src_ip[0:2], 16))
srcip = srcip + '.' + str(int(src_ip[2:4], 16))
srcip = srcip + '.' + str(int(src_ip[4:6], 16))
srcip = srcip + '.' + str(int(src_ip[6:8], 16))
dstip = str(int(dst_ip[0:2], 16))
dstip = dstip + '.' + str(int(dst_ip[2:4], 16))
dstip = dstip + '.' + str(int(dst_ip[4:6], 16))
dstip = dstip + '.' + str(int(dst_ip[6:8], 16))
offset = 0
snn1 = 15 + offset
snn2 = 47 + offset
thetime = time.strftime("%Y%m%d%H%M%S", time.gmtime(timestamp))
nbname = dpkt.netbios.decode_name(udp_packet.data[snn1:snn2])
snbname = udp_packet.data[snn1:snn2]
hnbname = hexlify(udp_packet.data[snn1:snn2])
dnn1 = 49 + offset
dnn2 = 81 + offset
dnbname = dpkt.netbios.decode_name(udp_packet.data[dnn1:dnn2])
sdnbname = udp_packet.data[dnn1:dnn2]
hdnbname = hexlify(udp_packet.data[dnn1:dnn2])
if (self._pba.cfg('debug')):
print hexlify(udp_packet.data)
print hexlify(udp_packet.data[0:14])
print hexlify(udp_packet.data[snn1:snn2])
print hexlify(udp_packet.data[dnn1:dnn2])
print nbname
print dnbname
print snbname
print sdnbname
record = PBARecordNBDS(raw_packet, timestamp)
record.assign(thetime, src_mac, srcip, dstip, nbname, hnbname,
snbname[0:30], dnbname, hdnbname, sdnbname[0:30])
return record
def iterate(pktlen, pkt, timestamp):
eth = Ethernet(pkt)
if eth.type == ETH_TYPE_IP:
ip = IP(str(eth.data))
if ip.p == IP_PROTO_TCP:
tcp = TCP(str(ip.data))
if len(tcp.data) > 0:
if (outgoing is not None
and ip.src == self.stream['src']
and tcp.sport == self.stream['sport']
and ip.dst == self.stream['dst']
and tcp.dport == self.stream['dport']):
outgoing(pktlen, pkt, timestamp)
if (incoming is not None
and ip.src == self.stream['dst']
and tcp.sport == self.stream['dport']
and ip.dst == self.stream['src']
and tcp.dport == self.stream['sport']):
incoming(pktlen, pkt, timestamp)
def read_payloads_from_file(file):
"""Read the given pcap file and yield json payloads."""
pcap = dpkt.pcap.Reader(file)
for ts, pkt in pcap:
eth = Ethernet(pkt)
if eth.type != ETH_TYPE_IP:
continue
ip = eth.ip
if ip.p == 6:
transport = ip.tcp
elif ip == 17:
transport = ip.udp
else:
continue
if transport.sport != 9999 and transport.dport != 9999:
continue
data = transport.data
try:
decrypted = TPLinkSmartHomeProtocol.decrypt(data[4:])
except Exception as ex:
click.echo(
click.style(f"Unable to decrypt the data, ignoring: {ex}", fg="red")
)
continue
try:
json_payload = json.loads(decrypted)
except Exception as ex:
click.echo(
click.style(f"Unable to parse payload, ignoring: {ex}", fg="red")
)
continue
if not json_payload: # ignore empty payloads
click.echo(click.style("Got empty payload, ignoring", fg="red"))
continue
yield json_payload
def get_sender_ip(self, flow):
senderIp = None
for dumpPath in [self.senderDumps[flow], self.receiverDumps[flow]]:
try:
with open(dumpPath, 'rb') as dumpFile:
reader = Reader(dumpFile)
try:
packet = next(iter(reader))[1]
ip = Ethernet(packet).data
leftIp = min(ip.src, ip.dst)
rightIp = max(ip.src, ip.dst)
senderIp = leftIp if self.directions[
flow] == RIGHTWARD else rightIp
break # switch to the next flow
except StopIteration:
pass
except IOError as error:
raise AnalysisError("Failed to read dump %s:\n%s" %
(dumpPath, error))
return senderIp
def analysis_normal(packet, uid):
try:
proto_data = ProtoData()
proto_data.uid = uid
if isinstance(packet, tuple):
p_header, p_data = packet
analy_ts(proto_data, p_header) # 解析时间
p = Ethernet(p_data)
if isinstance(p.data, dpkt.ip.IP):
ip_data = p.data
analy_ip(proto_data, ip_data)
if isinstance(ip_data.data, dpkt.tcp.TCP):
tcp_data = ip_data.data
if tcp_data.data[:3] == "GET" or tcp_data.data[:
4] == "POST":
analy_http(proto_data, tcp_data)
if isinstance(ip_data.data, dpkt.udp.UDP):
udp_data = ip_data.data
if udp_data.dport == 53 or udp_data.sport == 53:
analy_dns(proto_data, udp_data)
return proto_data
except Exception as error:
raise Exception("{}".format(error))
def xDPIsession(scan_path):
data = []
try:
from datetime import datetime
from dpkt.pcap import Reader
from dpkt.ethernet import Ethernet
from socket import inet_ntoa
with open(scan_path, 'rb') as pf:
dpkt_file_object = False
try:
dpkt_file_object = Reader(pf)
except Exception as err:
dpkt_file_object = False
#print("[-] pcap corruption detected : {}".format(pcap_path))
if dpkt_file_object:
#print("[+] pcap's health fine : {}".format(pcap_path))
for ts, payload in dpkt_file_object:
t1, p = ts, payload
t = datetime.fromtimestamp(t1).strftime(
"%Y-%m-%d %H:%M:%S")
eth = False
try:
eth = Ethernet(payload)
except:
eth = False
if eth:
if eth.type == 2048:
ip = eth.data
src_ip = inet_ntoa(ip.src)
dst_ip = inet_ntoa(ip.dst)
data.append({
'src_ip': src_ip,
'dst_ip': dst_ip,
})
except Exception as err:
print(err)
return data
def parse(self, packet, timestamp):
'''
It parses filtered broadcast packets and encapsulates them in well formed
records
Implements two type of records:
- ARP
- NBDS
'''
self._packet = packet
record = None
raw_packet = packet
ethernet_frame = Ethernet(raw_packet)
ethernet_data = ethernet_frame.data
udp_packet = ethernet_data.data
print ethernet_frame.__dict__
if ('arp' in ethernet_frame.__dict__):
record = self.extract_arp(packet, timestamp)
else:
dst_mac = hexlify(ethernet_frame['dst'])
if (udp_packet['sport'] == 138):
if dst_mac == 'ffffffffffff':
record = self.extract_nbds(packet, timestamp)
return record
def get_ip_ethernet(buf):
lp = Ethernet(buf)
lp.unpack(buf)
ip = lp.data
return ip
def __init__(self, packet: Packet, ts, packet_number):
self.packet = packet
self.dict = {'timestamp': ts, 'num': packet_number}
eth = Ethernet(self.packet)
self.dict['smac'] = str(eth.src)
self.dict['dmac'] = str(eth.dst)
if isinstance(eth.data, IP):
ip = eth.data
self.dict['sip'] = str(ip.src)
self.dict['dip'] = str(ip.dst)
self.dict['ttl'] = ip.ttl
self.dict['ipid'] = ip.id
self.dict['ipv'] = ip.v
self.dict['prot'] = ip.p
self.dict['dfrag'] = ip.df
if isinstance(ip.data, TCP):
tcp = ip.data
self.dict['sport'] = tcp.sport
self.dict['dport'] = tcp.dport
self.dict['tcpwin'] = tcp.win
self.dict['tcpopts'] = str(tcp.opts)
# Parse options:
# self.dict['tcpopts'] = [(opt[0], str(opt[1])) for opt in parse_opts(tcp.opts)]
self.dict['tcpflags'] = tcp.flags
self.dict['tcpackn'] = tcp.ack
self.dict['tcpseq'] = tcp.seq
self.dict['tcpoff'] = tcp.off
if self.dict['dport'] == 80:
try:
http = HTTPRequest(tcp.data)
try:
self.dict['host'] = http.headers['host']
except (AttributeError, KeyError):
pass
try:
self.dict['user-agent'] = http.headers['user-agent']
except (AttributeError, KeyError):
pass
try:
self.dict['cookie'] = http.headers['cookie']
except (AttributeError, KeyError):
pass
try:
self.dict['uri'] = http.uri
except AttributeError:
pass
except (UnpackError, NeedData):
pass
elif self.dict['sport'] == 80:
try:
http = HTTPResponse(tcp.data)
try:
self.dict['setcookie'] = http.headers['set-cookie']
except (AttributeError, KeyError):
pass
except (UnpackError, NeedData):
pass
elif isinstance(ip.data, UDP):
udp = ip.data
self.dict['sport'] = udp.sport
self.dict['dport'] = udp.dport
def passiveSession(pcap_path):
data = {'healthy_sessions': [], 'corrupt_sessions': []}
sessions = {}
complete = []
with open(pcap_path, 'rb') as pf:
pcap_file_name = pcap_path
dpkt_file_object = False
try:
dpkt_file_object = Reader(pf)
except Exception as err:
dpkt_file_object = False
print("[-] pcap corruption detected : {}".format(pcap_path))
if dpkt_file_object:
print("[+] pcap's health fine : {}".format(pcap_path))
for ts, payload in dpkt_file_object:
t1, p = ts, payload
t = datetime.fromtimestamp(t1).strftime("%Y-%m-%d %H:%M:%S")
eth = False
try:
eth = Ethernet(payload)
except:
eth = False
if eth:
if eth.type == 2048:
ip = eth.data
src_ip = inet_ntoa(ip.src)
dst_ip = inet_ntoa(ip.dst)
if ip.p == 6:
tcp_pkt_header = False
tcp = ip.data
try:
tcp_pkt_header = tcp.__hdr__
except:
tcp_pkt_header = False
if tcp_pkt_header:
tcp_packet_data = {}
tcp_packet_data[
'pcap_file_name'] = pcap_path.split(
os.sep)[-1]
tcp_packet_data['src_ip'], tcp_packet_data[
'dst_ip'], tcp_packet_data[
'pkts_num'] = src_ip, dst_ip, 1
tcp_src_port, tcp_dst_port = tcp.sport, tcp.dport
tcp_packet_data['src_port'], tcp_packet_data[
'dst_port'] = tcp_src_port, tcp_dst_port
flags = flagScanner(tcp)
tcp_packet_data[
'pkts_size'] = tcp.data.__len__()
uni_key = '{}{}{}{}'.format(
tcp_packet_data['src_ip'],
tcp_packet_data['src_port'],
tcp_packet_data['dst_ip'],
tcp_packet_data['dst_port'])
if 'syn' in flags:
if uni_key in sessions:
del sessions[uni_key]
tcp_packet_data['start_time'] = t
tcp_packet_data['end_time'] = t
tcp_packet_data['session'] = False
tcp_packet_data['dns_data'] = False
#if tcp_packet_data['src_ip'] in domains:
# tcp_packet_data['dns_data'] = domains[tcp_packet_data['src_ip']]
#if tcp_packet_data['dst_ip'] in domains:
# tcp_packet_data['dns_data'] = domains[tcp_packet_data['dst_ip']]
sessions[uni_key] = tcp_packet_data
elif 'fin' in flags:
if uni_key in sessions:
sessions[uni_key][
'pkts_num'] += tcp_packet_data[
'pkts_num']
sessions[uni_key][
'pkts_size'] += tcp_packet_data[
'pkts_size']
sessions[uni_key]['session'] = True
sessions[uni_key]['end_time'] = t
complete_session = sessions[uni_key]
data["healthy_sessions"].append(
complete_session)
del sessions[uni_key]
else:
if uni_key in sessions:
sessions[uni_key][
'pkts_num'] += tcp_packet_data[
'pkts_num']
sessions[uni_key][
'pkts_size'] += tcp_packet_data[
'pkts_size']
sessions[uni_key]['end_time'] = t
for session in sessions:
data['corrupt_sessions'].append(sessions[session])
return data
def passiveDns(pcap_path):
data = {'dns_requests': [], 'dns_responses': []}
sessions = {}
complete = []
with open(pcap_path, 'rb') as pf:
pcap_file_name = pcap_path
dpkt_file_object = False
try:
dpkt_file_object = Reader(pf)
except Exception as err:
dpkt_file_object = False
print("[-] pcap corruption detected : {}".format(pcap_path))
if dpkt_file_object:
print("[+] pcap's health fine : {}".format(pcap_path))
for ts, payload in dpkt_file_object:
t1, p = ts, payload
t = datetime.fromtimestamp(t1).strftime("%Y-%m-%d %H:%M:%S")
eth = False
try:
eth = Ethernet(payload)
except:
eth = False
if eth:
if eth.type == 2048:
ip = eth.data
src_ip = inet_ntoa(ip.src)
dst_ip = inet_ntoa(ip.dst)
if ip.p == 17:
udp_pkt_header = False
udp = ip.data
try:
udp_pkt_header = udp.__hdr__
except:
udp_pkt_header = False
if udp_pkt_header:
udp_src_port, udp_dst_port = udp.sport, udp.dport
if udp_src_port == 53:
dns_response_data = DnsResponseParser(udp)
dns_response_data[
'src_ip'], dns_response_data[
'dst_ip'] = src_ip, dst_ip
dns_response_data[
'src_port'], dns_response_data[
'dst_port'] = udp_src_port, udp_dst_port
dns_response_data['dns_time'] = t
#dns_response_data['upload_id'] = upload_id
data['dns_responses'].append(
dns_response_data)
elif udp_dst_port == 53:
dns_request_data = DnsRequestParser(udp)
dns_request_data[
'src_ip'], dns_request_data[
'dst_ip'] = src_ip, dst_ip
dns_request_data[
'src_port'], dns_request_data[
'dst_port'] = udp_src_port, udp_dst_port
dns_request_data['dns_time'] = t
#dns_request_data['upload_id'] = upload_id
data['dns_requests'].append(
dns_request_data)
return data
def worker(pcap, filter, index, label, out):
f = open(pcap, "rb")
mime = magic.from_file(pcap, mime=True)
if mime == PCAP:
pkts = dpkt.pcap.Reader(f)
elif mime == PCAPNG:
pkts = dpkt.pcapng.Reader(f)
else:
raise ValueError("Argument pcap must be pcap or pcapng")
id_to_index = {IP_PROTO_TCP: {}, IP_PROTO_UDP: {}}
flows = {IP_PROTO_TCP: {}, IP_PROTO_UDP: {}}
data = [Feature.col]
length = 0
for i, (ts, buf) in enumerate(pkts):
eth = Ethernet(buf)
if eth.type != ETH_TYPE_IP:
continue
ip = eth.data
if ip.p == IP_PROTO_TCP: # Handle TCP
tcp = ip.data
# Create new id to index mapping
id = get_pkt_id(ip.src, tcp.sport, ip.dst, tcp.dport, ip.p, ip.v)
if id not in id_to_index[ip.p]:
id_to_index[ip.p][id] = length
length += 1
# Check requirement
# if index["tcp"] is not None and idx not in index["tcp"]:
# continue
# Create new flow
idx = id_to_index[ip.p][id]
if idx not in flows[ip.p]:
flows[ip.p][idx] = TCPFlow(id, ts, tcp, i)
else:
if tcp.flags == 2: # [TCP Port numbers reused]
if index["tcp"] is None or idx in index["tcp"]:
source = "{}-{}-{}".format(pcap, ip.p, idx)
data.append(flows[ip.p][idx].to_list(source, label))
flows[ip.p].pop(idx, None)
id_to_index[ip.p][id] = length
flows[ip.p][length] = TCPFlow(id, ts, tcp, i)
length += 1
else:
flows[ip.p][idx].upd_flow(ts, tcp, i)
elif ip.p == IP_PROTO_UDP: # Handle UDP
udp = ip.data
# TODO
f.close()
print("{} prepare to write".format(pcap))
for p in [IP_PROTO_TCP, IP_PROTO_UDP]:
for idx, flow in flows[p].items():
if index["tcp"] is None or idx in index["tcp"]:
source = "{}-{}-{}".format(pcap, p, idx)
data.append(flow.to_list(source, label))
with open(out, "w", newline="") as f:
writer = csv.writer(f)
writer.writerows(data)
print("{}: {} lines".format(pcap, len(data) - 1))
return out
def PacketDiessector(pcap_path):
def flagScanner(tcp):
result = []
if (tcp.flags & TH_FIN) != 0:
result.append('fin')
if (tcp.flags & TH_SYN) != 0:
result.append('syn')
if (tcp.flags & TH_RST) != 0:
result.append('rst')
if (tcp.flags & TH_PUSH) != 0:
result.append('psh')
if (tcp.flags & TH_ACK) != 0:
result.append('ack')
if (tcp.flags & TH_URG) != 0:
result.append('urg')
if (tcp.flags & TH_ECE) != 0:
result.append('ece')
if (tcp.flags & TH_CWR) != 0:
result.append('cwr')
return result
print "[+] scanning : {}".format(pcap_path)
sessions = {}
d = {}
for ts, payload in Reader(open(pcap_path, 'rb')):
t, p = ts, payload
eth = Ethernet(payload.__str__())
packet = {}
if eth.type == ETH_TYPE_IP:
ip = eth.data
packet['source_ip'] = inet_ntoa(ip.src)
packet['destination_ip'] = inet_ntoa(ip.dst)
if ip.p == IP_PROTO_TCP:
tcp = ip.data
if tcp.dport == 25:
packet['source_port'] = tcp.sport
packet['destination_port'] = tcp.dport
packet['flags'] = flagScanner(tcp)
packet['pkts_num'] = 1
packet['pkts_size'] = tcp.data.__len__()
packet['data'] = tcp.data
uni_key = '{}:{}-->{}:{}'.format(
packet['source_ip'], packet['source_port'],
packet['destination_ip'], packet['destination_port'])
if 'syn' in packet['flags']:
if uni_key in sessions:
del sessions[uni_key]
sessions[uni_key] = packet
elif 'fin' in packet['flags']:
if uni_key in sessions:
#sessions[uni_key]['flags'].extend(packet['flags'])
#sessions[uni_key]['pkts_num']+=packet['pkts_num']
#sessions[uni_key]['pkts_size']+=packet['pkts_size']
sessions[uni_key]['source_port'] += packet[
'source_port']
sessions[uni_key]['destination_port'] += packet[
'destination_port']
sessions[uni_key]['data'] += packet['data']
complete_session = sessions[uni_key]
body = complete_session['data']
port_source = complete_session['source_port']
port_dest = complete_session['destination_port']
sd = str(inet_ntoa(ip.src)) + ':' + str(
port_source) + '-->' + inet_ntoa(
ip.dst) + ': ' + str(port_dest)
if 'AUTH PLAIN' in complete_session['data']:
pass
j = complete_session['data'][
complete_session['data'].find('AUTH PLAIN'
):200]
dump(j, sd)
#print '*'*50
if 'AUTH LOGIN' in complete_session['data']:
pass
k = complete_session['data'][
complete_session['data'].find('AUTH LOGIN'
):200]
#print k
#print k
dump(k, sd)
#print '*'*50
else:
if uni_key in sessions:
#sessions[uni_key]['flags'].extend(packet['flags'])
#sessions[uni_key]['pkts_num']+=packet['pkts_num']
#sessions[uni_key]['pkts_size']+=packet['pkts_size']
sessions[uni_key]['source_port'] += packet[
'source_port']
sessions[uni_key]['destination_port'] += packet[
'destination_port']
sessions[uni_key]['data'] += packet['data']
def PacketDiessector(pcap_path, upload_id):
def flagScanner(tcp):
result = []
if (tcp.flags & TH_FIN) != 0:
result.append('fin')
if (tcp.flags & TH_SYN) != 0:
result.append('syn')
if (tcp.flags & TH_RST) != 0:
result.append('rst')
if (tcp.flags & TH_PUSH) != 0:
result.append('psh')
if (tcp.flags & TH_ACK) != 0:
result.append('ack')
if (tcp.flags & TH_URG) != 0:
result.append('urg')
if (tcp.flags & TH_ECE) != 0:
result.append('ece')
if (tcp.flags & TH_CWR) != 0:
result.append('cwr')
return result
#print("[+] scanning : {}".format(pcap_path))
domains = {}
sessions = {}
complete = []
#incomplete = []
with open(pcap_path, 'rb') as pf:
pcap_file_name = pcap_path
dpkt_file_object = False
try:
dpkt_file_object = Reader(pf)
except Exception as err:
dpkt_file_object = False
#print("[-] pcap corruption detected : {}".format(pcap_path))
if dpkt_file_object:
#print("[+] pcap's health fine : {}".format(pcap_path))
for ts, payload in dpkt_file_object:
t1, p = ts, payload
t = datetime.fromtimestamp(t1).strftime("%Y-%m-%d %H:%M:%S")
eth = False
try:
eth = Ethernet(payload)
except:
eth = False
if eth:
if eth.type == 2048:
ip = eth.data
src_ip = inet_ntoa(ip.src)
dst_ip = inet_ntoa(ip.dst)
if ip.p == 17:
udp_pkt_header = False
udp = ip.data
try:
udp_pkt_header = udp.__hdr__
except:
udp_pkt_header = False
if udp_pkt_header:
udp_src_port, udp_dst_port = udp.sport, udp.dport
if udp_src_port == 53:
dns_response_data = DnsResponseParser(udp)
dns_response_data[
'src_ip'], dns_response_data[
'dst_ip'] = src_ip, dst_ip
dns_response_data[
'src_port'], dns_response_data[
'dst_port'] = udp_src_port, udp_dst_port
dns_response_data['dns_time'] = t
dns_response_data['upload_id'] = upload_id
domains[dst_ip] = dns_response_data
elif udp_dst_port == 53:
dns_request_data = DnsRequestParser(udp)
dns_request_data[
'src_ip'], dns_request_data[
'dst_ip'] = src_ip, dst_ip
dns_request_data[
'src_port'], dns_request_data[
'dst_port'] = udp_src_port, udp_dst_port
dns_request_data['dns_time'] = t
dns_request_data['upload_id'] = upload_id
domains[src_ip] = dns_request_data
elif ip.p == 6:
tcp_pkt_header = False
tcp = ip.data
try:
tcp_pkt_header = udp.__hdr__
except:
tcp_pkt_header = False
if tcp_pkt_header:
tcp_packet_data = {}
tcp_packet_data['upload_id'] = upload_id
tcp_packet_data[
'pcap_file_path'] = pcap_file_name.split(
'media')[-1]
tcp_packet_data['src_ip'], tcp_packet_data[
'dst_ip'], tcp_packet_data[
'pkts_num'] = src_ip, dst_ip, 1
tcp_src_port, tcp_dst_port = tcp.sport, tcp.dport
tcp_packet_data['src_port'], tcp_packet_data[
'dst_port'] = tcp_src_port, tcp_dst_port
flags = flagScanner(tcp)
tcp_packet_data[
'pkts_size'] = tcp.data.__len__()
uni_key = '{}{}{}{}'.format(
tcp_packet_data['src_ip'],
tcp_packet_data['src_port'],
tcp_packet_data['dst_ip'],
tcp_packet_data['dst_port'])
if 'syn' in flags:
if uni_key in sessions:
del sessions[uni_key]
tcp_packet_data['start_time'] = t
tcp_packet_data['end_time'] = t
tcp_packet_data['session'] = False
tcp_packet_data['dns_data'] = False
if tcp_packet_data['src_ip'] in domains:
tcp_packet_data['dns_data'] = domains[
tcp_packet_data['src_ip']]
if tcp_packet_data['dst_ip'] in domains:
tcp_packet_data['dns_data'] = domains[
tcp_packet_data['dst_ip']]
sessions[uni_key] = tcp_packet_data
elif 'fin' in flags:
if uni_key in sessions:
sessions[uni_key][
'pkts_num'] += tcp_packet_data[
'pkts_num']
sessions[uni_key][
'pkts_size'] += tcp_packet_data[
'pkts_size']
sessions[uni_key]['session'] = True
sessions[uni_key]['end_time'] = t
complete_session = sessions[uni_key]
complete.append(complete_session)
del sessions[uni_key]
else:
if uni_key in sessions:
sessions[uni_key][
'pkts_num'] += tcp_packet_data[
'pkts_num']
sessions[uni_key][
'pkts_size'] += tcp_packet_data[
'pkts_size']
sessions[uni_key]['end_time'] = t
for session in sessions:
complete.append(sessions[session])
for sess in complete:
# phase one scan
sess['phase_one_scan'] = False
detected_subnets = []
src_subnets = all_matching_cidrs(sess['src_ip'], subnets)
dst_subnets = all_matching_cidrs(sess['dst_ip'], subnets)
if src_subnets: detected_subnets.append(str(src_subnets[0]))
if dst_subnets: detected_subnets.append(str(dst_subnets[0]))
if detected_subnets:
detected_subnet = detected_subnets[0]
sess['phase_one_scan'] = subnet_config.loc[
subnet_config['subnet'] == detected_subnet].to_dict()
ddf = subnet_config.loc[subnet_config['subnet'] ==
detected_subnet].to_dict()
sess['phase_one_scan'] = {
'subnet': list(ddf['subnet'].values())[0].strip(),
'application': list(ddf['application'].values())[0].strip(),
'activity': list(ddf['activity'].values())[0].strip(),
'category': list(ddf['category'].values())[0].strip()
}
# phase two scan
sess['phase_two_scan'] = False
if sess['dns_data']:
sess['phase_two_scan'] = False
detected_dnss = []
if 'domain' in sess['dns_data']:
domain = sess['dns_data']['domain']
for item in dns_list:
if item in domain:
detected_dnss.append(item.strip())
if detected_dnss:
df = dns_config.loc[dns_config['domain'] ==
detected_dnss[0]].to_dict()
sess['phase_two_scan'] = {
'domain': list(df['domain'].values())[0].strip(),
'application':
list(df['application'].values())[0].strip(),
'activity': list(df['activity'].values())[0].strip(),
'category': list(df['category'].values())[0].strip()
}
sess['hash'] = hashlib.md5(str(
sess.values()).encode('utf-8')).hexdigest()
return complete