def do_test_scaled_ep_basic(test, client, server, backend):
"""
This function validates the functionality of the scaled endpoint.
1. Tcpdumps to a pcap file with a timeout of 10 seconds.
2. Test HTTP, ICMP, TCP, and UDP between client and scaled endpoint.
3. Validate that the client has sent packets with the 4 protocol layers.
4. Validate that the backend has recieved packets with the 4 protocol layers.
"""
logger.info("{} Testing basic scaled endpoint! {}".format(
'=' * 20, '=' * 20))
do_start_pcap_scaled_ep(test, client, server, backend, 10)
do_ping_test(test, client, server, False)
do_start_backend_servers(test, backend)
do_start_clients(test, client, server)
sleep(10) # Wait for tcpdump to timeout
backend_packets = {}
client_pkts = rdpcap("test/trn_func_tests/output/" + client.host.ip + "_" +
client.host.agent_pcap_file[client.veth_peer] +
"_dump.pcap")
for ep in backend:
backend_packets[ep.host.ip] = (
rdpcap("test/trn_func_tests/output/" + ep.host.ip + "_" +
ep.host.agent_pcap_file[ep.veth_peer] + "_dump.pcap"))
test.assertEqual(do_check_proto(test, {client.host.ip: client_pkts}), 4)
test.assertEqual(do_check_proto(test, backend_packets), 4)
def comparePcaps(src, dst):
pcap1 = scapy.rdpcap(open(src, 'rb'))
pcap2 = scapy.rdpcap(open(dst, 'rb'))
pkts2 = []
for pkt in pcap2:
if scapy.IP in pkt:
pkt[scapy.IP].chksum = 0
pkts2.append(str(pkt))
fail = False
for input in pcap1:
if scapy.IP in input:
input[scapy.IP].chksum = 0
if str(input) in pkts2:
del pkts2[pkts2.index(str(input))]
else:
print("FAIL: Packet %s is not in dst" % (input.show2(dump=True)))
fail = True
if not fail:
print("Success! All packets from src in dst")
exit(0)
exit(-1)
async def post(self):
db = self.settings["db"]
fsdb = MotorGridFSBucket(db)
success = False
"""Gather file data."""
file_info = self.request.files['file'][0]
file_name = file_info['filename']
file_content = file_info['body']
file_checksum = util.checksum(file_content)
"""Check if file exists already."""
document = await db["fs.files"].find_one(
{"metadata.checksum": file_checksum})
if document:
document_id = str(document["_id"])
uploaded = False
else:
"""Esure that the uploaded file is parsable as PCAP."""
try:
ftmp = tempfile.NamedTemporaryFile(delete=True)
ftmp.write(file_content)
ftmp.flush()
rdpcap(ftmp.name)
ftmp.close()
except Scapy_Exception as e:
self.write(
dict(success=False,
data=dict(
message=
"Submitted file does not qualify as a PCAP file.",
exception=str(e))))
self.set_status(500)
return
"""File does not exist, proceed with upload."""
async with fsdb.open_upload_stream(file_name,
metadata=dict(
sha256=file_checksum,
analyzed=False,
)) as grid_in:
document_id = str(grid_in._id)
uploaded = True
await grid_in.write(file_content)
await db.tasks.insert_one(
dict(type="pcap_inserted",
data=grid_in._id,
date=datetime.utcnow()))
"""Send success=true due to upload success. also refer to sha256."""
success = True
"""Send success=false due to the upload failure, however, send the sha256 in return."""
self.write(
dict(success=success,
data=dict(
filename=file_name,
id=document_id,
sha256=file_checksum,
uploaded=uploaded,
)))
return
def concatenate_files(self, pcap1, pcap2):
'''Appends second file's packets onto the first. For creating pcap files used in
unit testing.'''
pkts1 = rdpcap(pcap1)
pkts1_writer = PcapWriter(pcap1, append=True, sync=True)
pkts2 = rdpcap(pcap2)
for pkt in pkts2:
pkts1_writer.write(pkt)
pkts1 = rdpcap(pcap1)
def test_split_by_round_robin(self, tmpdir):
divide_by = 10
args = {
'-f': os.path.join('pcap_examples', 'many-protocols.pcap'),
'-o': tmpdir,
'-m': 'round-robin',
'-p': str(divide_by)
}
self.run_example(args=args)
num_of_packets_per_file = int(len(rdpcap(os.path.join('pcap_examples', 'many-protocols.pcap'))) / divide_by)
assert len(os.listdir(tmpdir)) == divide_by
for filename in os.listdir(tmpdir):
assert num_of_packets_per_file <= len(rdpcap(os.path.join(tmpdir, filename))) <= num_of_packets_per_file + 1
def read_dump(pcap_file):
"""
Read PCAP file
Return dict of packets with serialized flat packet as key
"""
dump = defaultdict(list)
packs = []
count = 0
if not be_quite:
sys.stdout.write("Reading file " + pcap_file + "\n")
sys.stdout.flush()
for packet in rdpcap(pcap_file):
if not be_quite:
sys.stdout.write(":")
sys.stdout.flush()
count += 1
ser = serialize(packet)
dump[ser].append(packet)
packs.append(packet)
if not be_quite:
sys.stdout.write("\nFound " + str(count) + " packets\n\n")
return (dump, packs)
def sepaByIP(phonename, value):
ipdict = {}
pcapfiles = glob.glob('/tmp/' + phonename + '/' + str(value) + '/*.pcap')
for p in pcapfiles:
pcaplength = os.path.getsize(p)
if pcaplength > 150 and pcaplength < 2e+5:
pkgs = scapy.rdpcap(p)
srcip = repr(pkgs[0]['IP'].src).split('.')
dstip = repr(pkgs[0]['IP'].dst).split('.')
srckey = srcip[0][1:] + '.' + srcip[1]
dstkey = dstip[0][1:] + '.' + dstip[1]
ipkey = ''
if srckey not in appset.localiplist:
ipkey = srckey
elif dstkey not in appset.localiplist:
ipkey = dstkey
if ipkey == '':
continue
if ipkey not in ipdict:
ipdict[ipkey] = [p]
else:
ipdict[ipkey].append(p)
return ipdict
def read(self, pcap_path_read: str = '/tmp/test.pcap') -> None:
"""
Read encrypted 802.11 packets from pcap file
:param pcap_path_read: Path to PCAP file for read encrypted packets (example: '/tmp/test.pcap')
:return: None
"""
try:
# Check pcap file for reading exists
assert isfile(
pcap_path_read), 'Pcap file: ' + pcap_path_read + ' not found!'
self.print_info('Read packets from: ', pcap_path_read, ' ....')
# Reading encrypted packets from pcap file
encrypted_packets = rdpcap(args.pcap_path_read)
self.print_info(
'All packets are read, packet analysis is in progress ....')
# Analyze encrypted packets from pcap file
for encrypted_packet in encrypted_packets:
self.analyze_packet(packet=encrypted_packet)
# Check the number of kr00k packets with NULL TK
assert self.number_kr00k_packets > 0, 'Not found kr00k packets'
self.print_success(
'Found ', str(kr00k.number_kr00k_packets),
' kr00ked packets and decrypted packets save in: ',
args.pcap_path_result)
except AssertionError as Error:
self.print_error(Error.args[0])
exit(1)
def LoadPcap(self, PcapFileDirectory, PktList):
if self.LivePcap is None:
self.LivePcap = rdpcap(PcapFileDirectory)
frame = 1
for i in self.LivePcap:
Packet(i, frame, PktList, True)
frame += 1
def read_pcap(filename):
"""
@param filename: Filesystem path to the pcap.
Returns:
[{"client": "\x17\x52\x15"}, {"server": "\x17\x15\x13"}]
"""
from scapy.all import IP, Raw, rdpcap
packets = rdpcap(filename)
checking_first_packet = True
client_ip_addr = None
server_ip_addr = None
ssl_packets = []
messages = []
"""
pcap assumptions:
pcap only contains packets exchanged between a Tor client and a Tor
server. (This assumption makes sure that there are only two IP addresses
in the pcap file)
The first packet of the pcap is sent from the client to the server. (This
assumption is used to get the IP address of the client.)
All captured packets are TLS packets: that is TCP session
establishment/teardown packets should be filtered out (no SYN/SYN+ACK)
"""
"""
Minimally validate the pcap and also find out what's the client
and server IP addresses.
"""
for packet in packets:
if checking_first_packet:
client_ip_addr = packet[IP].src
checking_first_packet = False
else:
if packet[IP].src != client_ip_addr:
server_ip_addr = packet[IP].src
try:
if (packet[Raw]):
ssl_packets.append(packet)
except IndexError:
pass
"""Form our list."""
for packet in ssl_packets:
if packet[IP].src == client_ip_addr:
messages.append({"client": str(packet[Raw])})
elif packet[IP].src == server_ip_addr:
messages.append({"server": str(packet[Raw])})
else:
raise("Detected third IP address! pcap is corrupted.")
return messages
def seq_analysis(self, pcapfile):
"""
this method act as an interface for the dissect() method.
and to represents the data in the required format.
@param pcapfile: path to a pcap/cap library
"""
packetslist = rdpcap(pcapfile)
pktsfields = []
protocols = []
entry = {}
recognized = False
for pkt in packetslist:
firstlayer = True
if pkt:
if firstlayer:
firstlayer = False
self.packet = pkt
fields = self.dissect(self.packet)
load = pkt
while load.payload:
load = load.payload
self.packet = load
fields = self.dissect(self.packet)
if fields[0]:
if fields[0] == "NoPayload":
break
def test_recv_many_packets_out_of_order():
"""
We should be able to put multiple packets together, if they come
out of order and are repeated.
"""
packet_log = rdpcap("test/inputs/wget-36000-nums.pcap")
listener, conn = create_session(packet_log)
_, syn_ack, _, push_ack = packet_log[:4]
listener.dispatch(syn_ack)
payload = get_payload(push_ack)
conn.send(payload)
p1, p2, p3 = packet_log[5], packet_log[7], packet_log[8]
# Send the packets out of order and repeated
listener.dispatch(p2)
listener.dispatch(p3)
listener.dispatch(p2)
listener.dispatch(p1) # Right
listener.dispatch(p3)
listener.dispatch(p2) # Right
listener.dispatch(p3) # Right
# Check that the contents of the packet is right
# This is a good test because one of the packets starts with a 6 or
# something
conn.state = "CLOSED"
recv = conn.recv(38000)
assert recv[-36001:-1] == "1234567890" * 3600
def _test_dhcp_change():
fd = open(ipaddr_log, 'r')
run_dhcp_change = False
dhcp_change_ip = None
for line in fd:
if run_dhcp_change:
if ip_notification in line:
dhcp_parts = line.split(ip_notification + ' ')[1].rstrip()
dhcp_change_ip = dhcp_parts.split()[0]
break
if running_dhcp_change in line:
run_dhcp_change = True
fd.close()
if not running_dhcp_change:
return 'skip', 'DHCP change test did not run.'
if not ip_notification:
return 'fail', 'No ip change found.'
print('dhcp_change looking for ping src IP %s' % dhcp_change_ip)
capture = rdpcap(scan_file)
for packet in capture:
if ICMP in packet:
print('ping from src %s' % packet[IP].src)
if packet[IP].src == dhcp_change_ip:
return 'pass', 'Device has received new IP address.'
return 'fail', 'Device has not received new IP address.'
def read_capture(filename):
packets = scapy.rdpcap(filename)
# Hold the TCP packet fields
tcp = []
# Hold the UDP packet fields
udp = []
for p in packets:
if p.haslayer(scapy.TCP) == 1:
p[scapy.TCP].fields.update({"src": p[scapy.IP].fields["src"]})
p[scapy.TCP].fields.update({"dst": p[scapy.IP].fields["dst"]})
if p.haslayer(scapy.Raw) == 1:
p[scapy.TCP].fields.update({"Raw": (p[scapy.Raw].load)})
else:
p[scapy.TCP].fields.update({"Raw": ""})
tcp.append(p[scapy.TCP].fields)
elif p.haslayer(scapy.UDP) == 1:
p[scapy.UDP].fields.update({"src": p[scapy.IP].fields["src"]})
p[scapy.UDP].fields.update({"dst": p[scapy.IP].fields["dst"]})
if p.haslayer(scapy.Raw) == 1:
p[scapy.UDP].fields.update({"Raw": (p[scapy.Raw].load)})
else:
p[scapy.UDP].fields.update({"Raw": ""})
udp.append(p[scapy.UDP].fields)
# Order the packets by sequence in case the stream needs to be reconstructed
tcp = sorted(tcp, key=lambda paq: paq["seq"])
packets = {}
packets["tcp"] = tcp
packets["udp"] = udp
return packets
def upload():
filepath = app.config['UPLOAD_FOLDER']
upload = Upload()
if request.method == 'GET':
return render_template('./upload/upload.html')
elif request.method == 'POST':
pcap = upload.pcap.data
if upload.validate_on_submit():
pcapname = pcap.filename
if allowed_file(pcapname):
name1 = random_name()
name2 = get_filetype(pcapname)
global PCAP_NAME, PCAPS
PCAP_NAME = name1 + name2
try:
pcap.save(os.path.join(filepath, PCAP_NAME))
PCAPS = rdpcap(os.path.join(filepath, PCAP_NAME))
os.system('rm -rf ' + filepath + '*')
flash('恭喜你,上传成功！')
return render_template('./upload/upload.html')
except Exception as e:
flash('上传错误,错误信息:' +str(e))
return render_template('./upload/upload.html')
else:
flash('上传失败,请上传允许的数据包格式!')
return render_template('./upload/upload.html')
else:
return render_template('./upload/upload.html')
def testBasic(self):
iface = self.tap.name
record = PacketRecord()
# start capture
process = Popen([APP, iface, CAPTURE_FILE], stdout=DEV_NULL, stderr=DEV_NULL)
# send packets
for i in range(PACKET_COUNT):
packet = IP(dst="www.google.com")/ICMP()
sendp(packet, iface=iface, verbose=False)
record.add_sent(packet)
# wait for stragglers
time.sleep(1)
# stop capture
process.terminate()
# hack: send one more packet to make sure capture closes immediately
sendp(IP(), iface=iface, verbose=False)
process.poll()
# verify capture file
for packet in rdpcap(CAPTURE_FILE):
record.add_received(packet)
self.assertTrue(record.verify())
def analyze(self, data):
'''
start analyzing pcap logic, add descriptions and get words and wordsstripped from the file
'''
data["PCAP"] = deepcopy(self.datastruct)
packets = scapy.rdpcap(data["Location"]["File"])
all, ports, ips, rarp, rdns, http, urlshttp, domains = self.read_all_packets(
packets)
data["PCAP"]["Domains"] = domains
data["PCAP"]["URLs"] = urlshttp
data["PCAP"]["ARP"] = rarp
data["PCAP"]["DNS"] = rdns
data["PCAP"]["HTTP"] = http
data["PCAP"]["ALL"] = all
data["PCAP"]["PORTS"] = ports
data["PCAP"]["IP4S"] = ips
self.waf.analyze(data["PCAP"]["HTTP"], data["PCAP"]["WAF"], "waf.json")
add_description("Ports", data["PCAP"]["ALL"], "SourcePort")
add_description("Ports", data["PCAP"]["ALL"], "DestinationPort")
add_description("Ports", data["PCAP"]["PORTS"], "Port")
add_description("DNSServers", data["PCAP"]["IP4S"], "IP")
add_description("ReservedIP", data["PCAP"]["IP4S"], "IP")
add_description("CountriesIPs", data["PCAP"]["IP4S"], "IP")
get_words(data, data["Location"]["File"])
def en_code(self,d_file):
'''主操作，加载参数过滤并展示信息'''
global coun_num
#增加数据库计数器
regx=r'[A-Z]{3,4}.*?\ HTTP'
d_packet=scapy.rdpcap(d_file)
one_regx=regx_raw()
for i in xrange(len(d_packet)):
#获取每一个数据包信息
try:
if d_packet[i]['Raw'].load.startswith('GET'):
vlue_s,snffer_url=one_regx.attack_url(d_packet[i]['Raw'].load)
shost=one_regx.regx_host(d_packet[i]['Raw'].load)
sscrip=d_packet[i]['IP'].src
if vlue_s=='100' or vlue_s=='200':
print '警告',sscrip,snffer_url
coun_num=coun_num+1
dd.insurl(coun_num,snffer_url,'GET',sscrip,one_regx.regx_host(d_packet[i]['Raw'].load),'1','A')
else:
print '正常：',sscrip,snffer_url
pass
'''
if d_packet[i]['Raw'].load.startswith('GET') or d_packet[i]['Raw'].load.startswith('POST'):
print d_packet[i]['IP'].src,'==>',d_packet[i]['IP'].dst,re.findall(regx,d_packet[i]['Raw'].load)[0]
print d_packet[i]['Raw'].load
else:
pass
'''
except:
pass
def pkgLengthDisturb(feature2nd):
fea2nddict = {}
for k, v in feature2nd.items():
fea2nd = [0 for x in range(17)]
# pkgsum = 0
for add in v:
pkgs = scapy.rdpcap(add)
# pkgsum += len(pkgs)
for p in pkgs:
# print(repr(p))
if 'TCP' in p:
ploadlen = len(p['TCP'].payload)
print(ploadlen)
for i in range(17):
if 2**i - 1 <= ploadlen < 2**(i + 1) - 1:
fea2nd[i] += 1
if sum(fea2nd) != 0:
fea2ndratio = [float(y) / float(sum(fea2nd)) for y in fea2nd]
print(fea2nd)
fea2nddict[k] = fea2ndratio
else:
fea2nddict[k] = [0.0 for x in range(17)]
return fea2nddict
def scapy_sniff(self, sniff_timeout=180, sniff_filter=''):
"""
@summary: PTF runner - runs a sniffer in PTF container.
Running sniffer in sonic-mgmt container has missing SOCKET problem
and permission issues (scapy and tcpdump require root user)
The remote function listens on all intfs. Once found, all packets
are dumped to local pcap file, and all packets are saved to
self.all_packets as scapy type.
Args:
sniff_timeout (int): Duration in seconds to sniff the traffic
sniff_filter (str): Filter that Scapy will use to collect only relevant packets
"""
capture_pcap = '/tmp/capture.pcap'
capture_log = '/tmp/capture.log'
self.ptfhost.copy(src='scripts/dual_tor_sniffer.py', dest=self.ptf_sniffer)
self.ptfhost.command(
'python {} -f "{}" -p {} -l {} -t {}'.format(
self.ptf_sniffer, sniff_filter, capture_pcap, capture_log, sniff_timeout
)
)
logger.info('Fetching pcap file from ptf')
self.ptfhost.fetch(src=capture_pcap, dest='/tmp/', flat=True, fail_on_missing=False)
self.all_packets = scapyall.rdpcap(capture_pcap)
logger.info("Number of all packets captured: {}".format(len(self.all_packets)))
def upload():
filepath = app.config['UPLOAD_FOLDER']
upload = Upload()
if request.method == 'GET':
return render_template('./upload/upload.html')
elif request.method == 'POST':
pcap = upload.pcap.data
if upload.validate_on_submit():
pcapname = pcap.filename
if allowed_file(pcapname):
name1 = random_name()
name2 = get_filetype(pcapname)
global PCAP_NAME, PCAPS
PCAP_NAME = name1 + name2
try:
pcap.save(os.path.join(filepath, PCAP_NAME))
PCAPS = rdpcap(os.path.join(filepath, PCAP_NAME))
flash('恭喜你,上传成功！')
return render_template('./upload/upload.html')
except Exception as e:
flash('上传错误,错误信息:' + str(e))
return render_template('./upload/upload.html')
else:
flash('上传失败,请上传允许的数据包格式!')
return render_template('./upload/upload.html')
else:
return render_template('./upload/upload.html')
def _test_ip_change():
fd = open(ipaddr_log, 'r')
run_ip_change = False
ip_change_ip = None
for line in fd:
if run_ip_change:
if ip_notification in line:
ip_change_ip = line.split(ip_notification +
' ')[1].rstrip()
break
if running_ip_change in line:
run_ip_change = True
fd.close()
if ip_change_ip is None:
return 'skip', 'IP change test did not run.'
capture = rdpcap(scan_file)
pingFound = False
for packet in capture:
if ICMP in packet and packet[IP].src == ip_change_ip:
pingFound = True
if pingFound:
return 'pass', 'Ping response received after IP change.'
return 'fail', 'No ping response received after IP change.'
def read_all_pcap_files_once(pcap_dir_in):
pcap_data = dict()
raw_data = dict()
fnames = sorted([
fn for fn in os.listdir(pcap_dir_in)
if "line_" in fn and len(fn) == 24 and "pcap" in fn
])
for pcap_file_in in fnames:
packets = rdpcap(os.path.join(pcap_dir_in, pcap_file_in))
pcap_data[pcap_file_in] = packets
with open(os.path.join(pcap_dir_in, pcap_file_in), "rb") as fh:
raw_pcap = fh.read()
#24 bytes = global header, 16+42 bytes = packet header, 1200 bytes = vlp returns, 2 bytes = vlp factory bytes
#raw_data[pcap_file_in] = [np.frombuffer(raw_pcap, dtype = np.uint8)[24:].reshape((len(raw_pcap)//1264,1264))[:,16+42:]]
raw_data[pcap_file_in] = np.frombuffer(
raw_pcap, dtype=np.uint8)[24:].reshape(
(len(raw_pcap) // 1264, 1264))[:, 16 + 42:].flatten()
raw_data = pd.DataFrame.from_dict(raw_data)
raw_data = raw_data.to_records(index=False)
timestamps = {k: [float(val[0].time)] for k, val in pcap_data.items()}
timestamps = pd.DataFrame.from_dict(timestamps, dtype=np.float64)
timestamps = timestamps.to_records(index=False)
nr_packs = {k: [len(val)] for k, val in pcap_data.items()}
nr_packs = pd.DataFrame.from_dict(nr_packs)
nr_packs = nr_packs.to_records(index=False)
return timestamps, nr_packs, raw_data
def process_data():
pcap_file = scapy.rdpcap("data.pcap")
# Parse pcap
time_start = 0
trip_time = -1
for i in range(len(pcap_file.res)):
ether_packet = pcap_file.res[i]
ether_time = ether_packet.time
try:
tcp_payload = ether_packet.payload["TCP"].load.decode()[:-1]
# Last Packet
if tcp_payload in ["Correct", "Incorrect"] and time_start != 0:
trip_time = ether_time - time_start
else:
time_start = ether_time
except:
continue
# Remove Files
os.remove("tcp_out.txt")
os.remove("tcp_err.txt")
os.remove("data.pcap")
if trip_time > 0:
# Convert time to milliseconds
trip_time = trip_time * 1000
return trip_time
return "Error"
def packet_count(pcap):
conf.verb = 0
try:
pkts = rdpcap(pcap)
return len(pkts)
except Exception as e:
return str(e)
def pcap2packets(input_file='.pcap or pcapng'):
"""
"transform pcap to packets"
:param input_file: pcap or pcapng
:return: a list of packets.
"""
pkts_lst = []
data = rdpcap(input_file)
print('%s info is ', data)
ab_pkts = {'non_Ether_pkts': 0, 'non_IPv4_pkts': 0, 'non_TCP_UDP_pkts': 0}
print('packet info:"srcIP:srcPort-dstIP:dstPort-prtcl" + IP_payload')
cnt = 0
for pkt in data:
if pkt.name == "Ethernet":
if pkt.payload.name.upper() in ['IP', 'IPV4']:
if pkt.payload.payload.name.upper() in ["TCP", "UDP"]:
if cnt == 0:
print('packet info: "%s:%d-%s:%d-%s"+%s' %
(pkt.payload.src, pkt.payload.payload.sport,
pkt.payload.dst, pkt.payload.payload.dport,
pkt.payload.payload.name, pkt.payload.payload))
pkts_lst.append(
pkt.payload) # only include "IPv4+IPv4_payload"
else:
ab_pkts['non_TCP_UDP_pkts'] += 1
else:
ab_pkts['non_IPv4_pkts'] += 1
else:
ab_pkts['non_Ether_pkts'] += 1
print('Number of packets in %s is %d.' % (input_file, len(pkts_lst)))
print('Abnormal packets in %s is %s' % (input_file, ab_pkts))
return pkts_lst
def process_data():
pcap_file = scapy.rdpcap("data.pcap")
# Parse pcap
time_start = 0
time_stop = 0
for i in range(len(pcap_file.res)):
ether_packet = pcap_file.res[i]
ether_time = ether_packet.time
try:
tcp_payload = ether_packet.payload["TCP"].load.decode()[:-1]
if tcp_payload == "z":
time_start = ether_time
elif tcp_payload == "Done" and time_start != 0:
time_stop = ether_time
break
except:
continue
delta = time_stop - time_start
# Remove Files
os.remove("tcp_out.txt")
os.remove("tcp_err.txt")
os.remove("data.pcap")
if all(i != 0 for i in [time_start, time_stop]):
return {"Time": delta}
return "Error"
def extractLengths(streamfile, lengthfile):
packets=rdpcap(streamfile)
lengths=[]
for packet in packets:
if 'IP' in packet:
try:
l=packet['IP'].fields['len']
except:
print('IP packet has no length')
continue
elif 'IPv6' in packet:
try:
l=packet['IPv6'].fields['len']
except:
print('IPv6 packet has no length')
continue
else:
print('Non-IP packet: '+str(packet))
continue
lengths.append(l)
maxlen=max(lengths)
lengthCount=[0]*(maxlen+1)
for l in lengths:
lengthCount[l]=lengthCount[l]+1
f=open(lengthfile, 'wb')
for count in lengthCount:
f.write(str(count)+"\n")
f.close()
def extractCorpus(streamfile, dict, corpus):
from gensim.corpora.dictionary import Dictionary
from gensim.corpora import MmCorpus
print('extractCorpus '+str(streamfile))
packets=rdpcap(streamfile)
buff=b''
maxSize=4
words=[]
# Limit to the first 100 packets
for x in range(100):
if x<len(packets):
if 'Raw' in packets[x]:
buff=buff+bytes(packets[x]['Raw'])
for size in range(1, maxSize):
for x in range(len(buff)):
sub=buff[x:x+size]
if len(sub)==size:
words.append(sub)
corpus.append(dict.doc2bow(words))
return corpus
def step1(pcap, request=0, response=0):
pcap = scapy.rdpcap(pcap)
for p in pcap:
if p.haslayer(request):
print("*********request******")
http_header = p[request].fields
headers = http_header['Headers']
Method = p[request].Method
Path = p[request].Path
version = http_header['Http-Version']
request_line = Method + " " + Path + " " + version
if 'Raw' in p:
request_body = p['Raw'].load
return request_line, headers, request_body
else:
return request_line, headers, None
if p.haslayer(response):
print("*********response******")
http_header = p[response].fields
headers = http_header['Headers']
Status_Line = http_header['Status-Line']
if 'Raw' in p:
response_body = p['Raw'].load
return Status_Line, headers, response_body
else:
return Status_Line, headers, None
def main(pcapFileNames):
print(info('Started [at] {}\n'.format(fetchFormatedTime())))
try:
for pcapFileName in pcapFileNames:
print(info('Opening File -> {}'.format(pcapFileName)))
packets = rdpcap(pcapFileName)
print(good('File read sequence compleated.'))
data = b''
print(
info(
'Reading and, merging data from provided sniffed packets .'
))
for packet in packets[Raw]:
data += packet.load
print(
good('Data Successfully Extracted -> {} bytes'.format(
len(data))))
print(
info('Now, writing the Data to -> {}'.format(pcapFileName +
'.out')))
with open(pcapFileName + '.out', 'wb') as outFile:
outFile.write(data)
outFile.close()
print(good('Done!\n'))
except KeyboardInterrupt:
print(bad('SIGINT recieved, terminating.'))
coolExit(0)
except Exception as exception:
print(bad('Ugh! Error -> {}'.format(exception)))
coolExit(1)
coolExit(0)
def packet_count(pcap):
conf.verb = 0
try:
pkts = rdpcap(pcap)
return len(pkts)
except Exception as e:
return str(e)
def analyse_pcap(pcap_file):
really_old_seq = quite_old_seq = old_seq = current_seq = 0
print "x, y"
packets = scapy.rdpcap(pcap_file)
for packet in packets:
# Weed out SYN retransmissions.
if is_retransmission(packet):
continue
really_old_seq = quite_old_seq
quite_old_seq = old_seq
old_seq = current_seq
current_seq = packet[scapy.TCP].seq
x = current_seq - old_seq
y = old_seq - quite_old_seq
z = quite_old_seq - really_old_seq
if x < 0:
x = SEQ_MAX - abs(x)
if y < 0:
y = SEQ_MAX - abs(y)
if z < 0:
z = SEQ_MAX - abs(z)
print "%d, %d" % (x, y)
def testBasic(self):
iface = self.tap.name
record = PacketRecord()
# start capture
process = Popen([APP, iface, CAPTURE_FILE],
stdout=DEV_NULL,
stderr=DEV_NULL)
# send packets
for i in range(PACKET_COUNT):
packet = IP(dst="www.google.com") / ICMP()
sendp(packet, iface=iface, verbose=False)
record.add_sent(packet)
# wait for stragglers
time.sleep(1)
# stop capture
process.terminate()
# hack: send one more packet to make sure capture closes immediately
sendp(IP(), iface=iface, verbose=False)
process.poll()
# verify capture file
for packet in rdpcap(CAPTURE_FILE):
record.add_received(packet)
self.assertTrue(record.verify())
def parse_pcap(filename):
logging.info("api: parsing %s ..." % filename)
net_id = os.path.basename(filename).replace('.pcap', '')
if '_' in net_id:
# /root/handshakes/ESSID_BSSID.pcap
essid, bssid = net_id.split('_')
else:
# /root/handshakes/BSSID.pcap
essid, bssid = '', net_id
it = iter(bssid)
bssid = ':'.join([a + b for a, b in zip(it, it)])
info = {'essid': essid, 'bssid': bssid}
try:
from scapy.all import rdpcap
for pkt in rdpcap(filename):
info = parse_packet(pkt, info)
except Exception as e:
logging.error("api: %s" % e)
return info['essid'], info['bssid']
def single_thread_pcap_ana(assigned_file_list, out_q, q_writer_lock):
for file in assigned_file_list:
try:
with scapy.rdpcap(file) as packets:
if packets is None:
break
else:
for packet in packets:
if packet.haslayer('IP'):
value = packet.time, \
packet.src, packet.dst, \
packet["IP"].src, packet["IP"].dst, \
packet.sprintf("%IP.proto%")
if packet.haslayer('TCP'):
value += packet["TCP"].sport, packet[
"TCP"].dport
elif packet.haslayer('UDP'):
value += packet["UDP"].sport, packet[
"UDP"].dport
with q_writer_lock:
out_q.put(value)
elif packet.haslayer('IPv6'):
pass
except IOError:
break
def main(count, input_cap, output_txt, mac_filter):
mac_counts = {}
pkt_count = 0
seq['last'] = 0
seq['init'] = False
capture = s.rdpcap(input_cap)
foutput = open(output_txt, 'wt')
for pkt in capture:
n = 0;
pkt_count += 1
try:
if pkt[s.Ether].dst in mac_counts:
mac_counts[pkt[s.Ether].dst] = mac_counts[pkt[s.Ether].dst] + 1
else:
mac_counts[pkt[s.Ether].dst] = 1
# Deal with AVTP packets
if pkt[s.Ether].type == 0x22f0:
# look for the requested MAC
if pkt[s.Ether].dst == mac_filter:
n = pkt_avtp(pkt, foutput, pkt_count)
except IndexError:
print "Unknown ethernet type"
count = count - n
if count == 0:
break
foutput.close();
if count != 0:
print "Could not find the specified MAC, or MAC count"
print "Mac counts"
print mac_counts
print "Complete"
def mymain():
packets=scapy.rdpcap('login.pcap')
mylist=[]
for p in packets:
#p.show()
#sys.stdout=open("my", 'w')
Get_http(p)
def read_capture(filename):
packets = scapy.rdpcap(filename)
#Hold the TCP packet fields
tcp = []
#Hold the UDP packet fields
udp = []
for p in packets:
if p.haslayer(scapy.TCP) == 1:
p[scapy.TCP].fields.update({'src': p[scapy.IP].fields['src']})
p[scapy.TCP].fields.update({'dst': p[scapy.IP].fields['dst']})
if p.haslayer(scapy.Raw) == 1:
p[scapy.TCP].fields.update({'Raw': (p[scapy.Raw].load)})
else:
p[scapy.TCP].fields.update({'Raw': ''})
tcp.append(p[scapy.TCP].fields)
elif p.haslayer(scapy.UDP) == 1:
p[scapy.UDP].fields.update({'src': p[scapy.IP].fields['src']})
p[scapy.UDP].fields.update({'dst': p[scapy.IP].fields['dst']})
if p.haslayer(scapy.Raw) == 1:
p[scapy.UDP].fields.update({'Raw': (p[scapy.Raw].load)})
else:
p[scapy.UDP].fields.update({'Raw': ''})
udp.append(p[scapy.UDP].fields)
#Order the packets by sequence in case the stream needs to be reconstructed
tcp = sorted(tcp, key=lambda paq: paq['seq'])
packets = {}
packets['tcp'] = tcp
packets['udp'] = udp
return packets
def replace_pw(FILENAME, password):
index_list = []
k = 1
found = False
## read and edit file
pkts = rdpcap(FILENAME)
# loop to find password indices
for i in range(0, len(pkts)):
try:
if password[0] in str(pkts[i].load) and len(pkts[i].load) == 1:
index_list.append(i)
for j in range(1, len(password)):
if password[j] in str(pkts[i+j+k].load) and len(pkts[i+j+k].load) == 1:
index_list.append(i+j+k)
k += 1
else:
index_list = []
k = 1
if len(index_list) == len(password):
found = True
break
except AttributeError:
pass
# replace password per packet
for i in index_list:
pkts[i].load = 'X'
if found:
wrpcap("mod.pcap", pkts)
def pcap2flows(input_f, output_dir='../2_flows_data'):
"""
flow is based on five tuple, howerver, there is direction between srcIP and dstIP.
srcIP->dstIP and dstIP->srcIP are recognized as different flow.
:param input_f:
:param output_dir:
:return:
"""
if not os.path.exists(output_dir):
os.makedirs(output_dir)
file_prefix = os.path.split(input_f)[-1].split('.')[0]
data = rdpcap(input_f)
data.stats
sess = data.sessions() # can achieve flows in default, not sessions
others_pkts = 0
for k, v in sess.items():
# print(k,v)
sess_tmp = b''
for vv in v:
payload = vv.payload.payload.payload.original
sess_tmp += payload
if sess_tmp == b'':
print('\'%s\' is not a flow.' % k)
others_pkts += 1
continue
k = os.path.join(output_dir,
file_prefix + '|' + k.replace(' ', '_') + '.png')
print(k, sess_tmp)
output_file = save_png(k, sess_tmp)
print('others_pkts = %d' % others_pkts)
return output_dir
def _test_ip_change():
fd = open(ipaddr_log, 'r')
run_ip_change = False
ip_change_ip = None
for line in fd:
if run_ip_change:
if ip_notification in line:
ip_parts = line.split(ip_notification + ' ')[1].rstrip()
ip_change_ip = ip_parts.split()[0]
break
if running_ip_change in line:
run_ip_change = True
fd.close()
if not run_ip_change:
return 'skip', 'IP change test did not run.'
if not ip_change_ip:
return 'fail', 'No ip change found.'
print('ip_change looking for ping src IP %s' % ip_change_ip)
capture = rdpcap(scan_file)
pingFound = False
for packet in capture:
if ICMP in packet:
print('ping from src %s' % packet[IP].src)
if packet[IP].src == ip_change_ip:
return 'pass', 'Ping response received after IP change.'
return 'fail', 'No ping response received after IP change.'
def __len__(self):
'''
'''
nr = 0
for pkt in rdpcap(self.pkts_src):
nr += 1
return nr
def _check_pcap(dummy_ip_a, dummy_ip_b, filepath):
is_ok_a = False
is_ok_b = False
if dummy_ip_a is None:
is_ok_a = True
if dummy_ip_b is None:
is_ok_b = True
packets = rdpcap(filepath)
for data in packets:
proto = "IPv6" if "IPv6" in data else "IP"
if is_ok_a is False and data[proto].dst == dummy_ip_a:
is_ok_a = True
if is_ok_b is False and data[proto].dst == dummy_ip_b:
is_ok_b = True
if is_ok_a and is_ok_b:
return True
missed_ip = []
if not is_ok_a:
missed_ip.append(dummy_ip_a)
if not is_ok_b:
missed_ip.append(dummy_ip_b)
logger.error("Pcap file doesn't contain dummy syslog ips: ({})".format(", ".join(missed_ip)))
return False
def main(count, input_cap):
mac_data = {}
n_avtp_streams = 0
foutput = []
pkt_count = 0
ts_good = 0
capture = s.rdpcap(input_cap)
for pkt in capture:
n = 0;
pkt_count += 1
try:
# Deal with AVTP packets
if (pkt[s.Ether].type == 0x8100 and pkt[s.Dot1Q].type == 0x22f0):
avtp = pkt[AVTP]
if avtp.controlData == 0x0:
if pkt[s.Ether].dst in mac_data:
mac_data[pkt[s.Ether].dst]['avtp_count'] = mac_data[pkt[s.Ether].dst]['avtp_count'] + 1
else:
print "Packet %d, found AVTP stream with destination MAC %s" % (pkt_count, pkt[s.Ether].dst)
mac_data[pkt[s.Ether].dst] = {}
mac_data[pkt[s.Ether].dst]['this_mac'] = pkt[s.Ether].dst
mac_data[pkt[s.Ether].dst]['avtp_count'] = 1
mac_data[pkt[s.Ether].dst]['fname'] = 'seq%d.csv' % n_avtp_streams
mac_data[pkt[s.Ether].dst]['fout'] = open(mac_data[pkt[s.Ether].dst]['fname'], 'wt')
mac_data[pkt[s.Ether].dst]['seq'] = {}
mac_data[pkt[s.Ether].dst]['seq']['last'] = 0
mac_data[pkt[s.Ether].dst]['seq']['init'] = False
mac_data[pkt[s.Ether].dst]['wraps'] = 0
mac_data[pkt[s.Ether].dst]['prev_pkt_ts'] = 0
mac_data[pkt[s.Ether].dst]['ts_count'] = 0
mac_data[pkt[s.Ether].dst]['ts_accum'] = 0
mac_data[pkt[s.Ether].dst]['ts_uncertain_count'] = 0
n_avtp_streams = n_avtp_streams + 1
if avtp.timestampUncertain:
ts_good = 0
mac_data[pkt[s.Ether].dst]['ts_uncertain_count'] = mac_data[pkt[s.Ether].dst]['ts_uncertain_count'] + 1
else:
ts_good = ts_good + 1
# when we have 2 MACs, process the packet
if n_avtp_streams == 2 and ts_good > 2:
if mac_data[pkt[s.Ether].dst]['ts_count'] == 0:
print "At packet %d start unpacking AVTP dest MAC %s" % (pkt_count, pkt[s.Ether].dst)
n = pkt_avtp(pkt, mac_data[pkt[s.Ether].dst]['fout'], pkt_count, mac_data[pkt[s.Ether].dst])
except IndexError:
print "Unknown ethernet type packet %d" % pkt_count
count = count - n
if count == 0:
break
for k, v in mac_data.items():
v['fout'].close();
print "MAC %s %d AVTP timestamps stored to %s" % (k, v['ts_count'], v['fname'])
print " Timestamp uncertain count: %d" % v['ts_uncertain_count']
if count != 0:
print "Could not find the specified packets counts"
print "Complete"
def sniff_responses():
packets = rdpcap("./74db9d6b62579fea4525d40e6848433f-net03.pcap")
for packet in packets:
if DNSRR in packet:
data = packet[DNSRR].rdata
# first byte is a length byte as per TXT record RFC
data = b64decode(str(correct_base64_padding(data[1:])))
print(" ".join("{0:02x}".format(c) for c in data))
def get_MPTCP_ack(i, dstIP):
try:
tf = tempfile.NamedTemporaryFile()
execCommand("sudo tcpdump -c 1 -w " + tf.name + ".cap -i " + i + " \"tcp[tcpflags] & (tcp-ack) != 0 and tcp[tcpflags] & (tcp-syn) == 0 and dst net " + dstIP + "\" 2>/dev/null", shell = True)
scan = rdpcap("" + tf.name + ".cap")
finally:
execCommand("rm -f " + tf.name + ".cap", shell = True)
return scan[0]
def get_MPTCP_syn(i):
try:
tf = tempfile.NamedTemporaryFile()
execCommand("sudo tcpdump -c 1 -w " + tf.name + ".cap -i " + i + " \"tcp[tcpflags] & tcp-syn != 0\" 2>/dev/null", shell = True)
scan = rdpcap("" + tf.name + ".cap")
finally:
execCommand("rm -f " + tf.name + ".cap", shell = True)
return scan[0]
def sniff_ackseq(i, srcIP):
try:
tf = tempfile.NamedTemporaryFile()
execCommand("sudo tcpdump -c 1 -w " + tf.name + ".cap -i " + i + " \"src net " + srcIP + "\" 2>/dev/null", shell = True)
scan = rdpcap("" + tf.name + ".cap")
finally:
execCommand("rm -f " + tf.name + ".cap", shell = True)
return scan[0]
def get_time(pcap):
try:
p = rdpcap(pcap)
c = len(p)
start = datetime.datetime.fromtimestamp(p[0].time).strftime('%Y-%m-%d %H:%M:%S.%f')
end = datetime.datetime.fromtimestamp(p[c -1].time).strftime('%Y-%m-%d %H:%M:%S.%f')
return [start, end]
except Exception as e:
return str(e)
def doTest(self): #redirect stdout to file sys.stdout = open(self.outputFile, 'w') pkts = rdpcap(self.inputFile) pkts.show() print '\n', '-' * 20, 'Packets Details:', '\n' for p in pkts: p.show() print '=' * 70
def run(self):
"""
Run the client and perform all the operations:
* Connect to the server.
* Receive video while sniffing packets.
* Close connection.
* Process data and extract information.
* Run meters.
:returns: A list of measures (see :attr:`VideoTester.measures.core.Meter.measures`) and the path to the temporary directory plus files prefix: ``<path-to-tempdir>/<prefix>``.
:rtype: tuple
"""
VTLOG.info("Client running!")
VTLOG.info("XMLRPC Server at " + self.conf['ip'] + ':' + self.conf['port'])
VTLOG.info("Evaluating: " + self.conf['video'] + " + " + self.conf['codec'] + " at " + self.conf['bitrate'] + " kbps and " + self.conf['framerate'] + " fps under " + self.conf['protocols'])
from xmlrpclib import ServerProxy
from scapy.all import rdpcap
from multiprocessing import Process, Queue
from VideoTester.gstreamer import RTSPclient
from VideoTester.sniffer import Sniffer
from VideoTester.measures.qos import QoSmeter
from VideoTester.measures.bs import BSmeter
from VideoTester.measures.vq import VQmeter
try:
server = ServerProxy('http://' + self.conf['ip'] + ':' + self.conf['port'])
self.conf['rtspport'] = str(server.run(self.conf['bitrate'], self.conf['framerate']))
except:
VTLOG.error("Bad IP or port")
exit()
sniffer = Sniffer(self.conf)
rtspclient = RTSPclient(self.conf, self.video)
q = Queue()
child = Process(target=sniffer.run, args=(q,))
try:
child.start()
self.__ping()
rtspclient.receiver()
sniffer.cap = rdpcap(q.get())
child.join()
except KeyboardInterrupt:
VTLOG.warning("Keyboard interrupt!")
server.stop(self.conf['bitrate'], self.conf['framerate'])
child.terminate()
child.join()
exit()
server.stop(self.conf['bitrate'], self.conf['framerate'])
videodata, size = rtspclient.reference()
conf = {'codec':self.conf['codec'], 'bitrate':float(self.conf['bitrate']), 'framerate':float(self.conf['framerate']), 'size':size}
packetdata = sniffer.parsePkts()
codecdata, rawdata = self.__loadData(videodata, size, self.conf['codec'])
qosm = QoSmeter(self.conf['qos'], packetdata).run()
bsm = BSmeter(self.conf['bs'], codecdata).run()
vqm = VQmeter(self.conf['vq'], (conf, rawdata, codecdata, packetdata)).run()
self.__saveMeasures(qosm + bsm + vqm)
VTLOG.info("Client stopped!")
return qosm + bsm + vqm, self.conf['tempdir'] + self.conf['num']
def do_q4(ifile, ofile):
pckts = rdpcap(ifile)
flags, chksums = {}, {}
for i, p in enumerate(pckts):
if TCP in p:
ip = p[IP]
tcp = ip[TCP]
if (ip.src, ip.dst) in flags:
if tcp.seq < flags[(ip.src, ip.dst)]:
print "Suspected frame %d with sequence number %s" % (i+1, tcp.seq)
flags[(ip.dst, ip.src)] = tcp.ack
def sniff_queries():
packets = rdpcap("./74db9d6b62579fea4525d40e6848433f-net03.pcap")
for packet in packets:
if DNSRR in packet:
data = packet[DNS].qd.qname
# Since DNS queries are case insensitive, it's base32 rather than
# base64 data and the format of the query name is [base32 data]-[16
# characters].badguy.com
data = data.decode().split("-")[0].replace(".", "")
data = b32decode(str(data), True)
print(" ".join("{0:02x}".format(c) for c in data))
def read_pcap_file(fname):
"""Reads a pcap file and returns the first packet in the file.
Args:
fname (str): Name of the file located in packet_files/.
Returns:
Packet: The first packet found in the file, as a scapy packet.
"""
p = scapy.rdpcap(PACKET_DIR + '%s' % fname)
return p[0]
def get_packets_from_pcap(pcap_path):
"""
receives the packets from the given file path
:param pcap_path: the path to the pcap file to load the data from
:return: list of DataVectors holding the data in the pcap file
"""
packets = [] # the packets of the given pcap file as a DataVector
pcap_data = rdpcap(pcap_path)
for p in pcap_data:
packets.append(DataVector(p))
return packets
def extractFirstStrings(streamfile, stringfile):
print('extractFirstStrings '+str(streamfile)+' '+str(stringfile))
packets=rdpcap(streamfile)
for packet in packets:
if 'Raw' in packet: # First packet with contents
buff=bytes(packet['Raw'])
f=open(stringfile, 'wb')
f.write(buff)
f.close()
break
def extractFirstStrings(streamfile, stringfile):
print ("extractFirstStrings " + str(streamfile) + " " + str(stringfile))
packets = rdpcap(streamfile)
for packet in packets:
if "Raw" in packet: # First packet with contents
buff = bytes(packet["Raw"])
f = open(stringfile, "wb")
f.write(buff)
f.close()
break
def try_decompress():
packets = rdpcap("./74db9d6b62579fea4525d40e6848433f-net03.pcap")
for packet in packets:
if DNSRR in packet:
data = packet[DNS].qd.qname
data = data.decode().split("-")[0].replace(".", "")
data = b32decode(data, True)
decode(data, "client-server")
data = packet[DNSRR].rdata
data = b64decode(data)
decode(data, "server-client")
def __init__(self):
self.summary_string = str(random.randint(1000000000, 9999999999))
# fileHandle = open('fake_data/' + 'packet1279077254.72')
# temp_data = pickle.load(fileHandle)
temp_data = ["Bogus"]
if rdpcap_available:
try:
temp_data = rdpcap("fake_data/" + "http.pcap")
except:
pass
self.timestamp = 1279077254.72
self.actual_packet = Packet(self.timestamp, temp_data[0], None)