def ReadPcap(file):
try:
# scapy.utils.PcapReader
print('Reading: %s' % file)
pcap = PcapReader(file)
except:
# yes, logging.exception should be used here, but it doesn't add any value
print('Not a valid pcap file: %s' % file)
raise
# Build a list of streams that match the search regex
num = 0
while pcap:
try:
packet = pcap.read_packet()
if not 'IP' in packet:
continue;
srcip = packet['IP'].src
dstip = packet['IP'].dst
sport = packet[2].sport
dport = packet[2].dport
length = packet[2].len
print srcip, dstip, sport, dport, length
#print packet.summary();
#print binascii.hexlify(packet[2].payload['Raw'].load)
num+=1
if num%1000 == 0:
print num
except TypeError:
print 'exception'
break
pcap.close()
del pcap
def extract_attributes(pcap_file,
attr_list,
filter_attributes=None,
filter_=None):
packets_reader = PcapReader(pcap_file)
try:
attr_name_list = map(format_attr_name, attr_list)
while 1:
packet = packets_reader.read_packet()
if not packet:
break
attributes = dict()
for i, attr in enumerate(attr_list):
attributes[attr_name_list[i]] = extract_attr(packet, attr)
ok = True
if filter_attributes:
for filter_attr in filter_attributes:
if isinstance(filter_attr, types.StringTypes):
filter_attr = format_attr_name(filter_attr)
if filter_attr not in attributes or not attributes[
filter_attr]:
ok = False
break
if ok and filter_:
ok = filter_(attributes)
if ok:
yield attributes
finally:
packets_reader.close()
def scapy_io(f_in,f_out):
f = PcapReader(f_in)
o = PcapWriter(f_out)
pkt = f.read_packet()
while pkt is not None:
o.write(pkt)
pkt = f.read_packet()
f.close()
o.close()
def scapy_io(f_in, f_out):
f = PcapReader(f_in)
o = PcapWriter(f_out)
pkt = f.read_packet()
while pkt is not None:
o.write(pkt)
pkt = f.read_packet()
f.close()
o.close()
def foo(in_filename, out_filename):
# open the input file for reading
f = PcapReader(in_filename)
# open the output file for writing
o = PcapWriter(out_filename)
# read the first packet from the input file
p = f.read_packet()
# while we haven't processed the last packet
while p:
layer = p.firstlayer()
while not isinstance(layer, NoPayload):
if (type(layer) is IPv6):
new_layer = IP()
del new_layer.ihl
new_layer.ttl = layer.hlim
new_layer.proto = layer.nh
new_layer.src = ".".join(map(str, six2four(layer.src)))
new_layer.dst = ".".join(map(str, six2four(layer.dst)))
new_layer.add_payload(layer.payload)
prev_layer = layer.underlayer
del layer
prev_layer.remove_payload()
prev_layer.add_payload(new_layer)
if type(prev_layer) is Ether:
prev_layer.type = ETH_P_IP
layer = new_layer
if layer.default_fields.has_key('chksum'):
del layer.chksum
if layer.default_fields.has_key('len'):
del layer.len
# advance to the next layer
layer = layer.payload
# write the packet we just dissected into the output file
o.write(p)
# read the next packet
p = f.read_packet()
# close the input file
f.close()
# close the output file
o.close()
def pcap_file(filepath, msg_queue):
"""Parse all packets in the provided pcap file."""
# Keep a copy of each dictionary that may change after parsing packets
init_network_keys = config.network_keys.copy()
init_link_keys = config.link_keys.copy()
init_networks = config.networks.copy()
init_devices = config.devices.copy()
init_addresses = config.addresses.copy()
init_pairs = config.pairs.copy()
# Reset all data entries in the dictionary
config.reset_entries()
# Collect data that are common for all the packets of the pcap file
head, tail = os.path.split(os.path.abspath(filepath))
config.entry["pcap_directory"] = head
config.entry["pcap_filename"] = tail
# Parse the packets of the pcap file
msg_queue.put((config.INFO_MSG, "Reading packets from the \"{}\" file..."
"".format(filepath)))
config.entry["pkt_num"] = 0
pcap_reader = PcapReader(filepath)
for pkt in pcap_reader:
# Collect some data about the packet
config.entry["pkt_num"] += 1
config.entry["pkt_time"] = float(pkt.time)
config.entry["pkt_bytes"] = bytes(pkt).hex()
config.entry["pkt_show"] = pkt.show(dump=True)
# Collect more data about the packet from the PHY layer and onward
phy_fields(pkt, msg_queue)
# Derive additional information from the parsed packet
if config.entry["error_msg"] is None:
derive_info()
# Send a copy of the collected data to the main process
msg_queue.put((config.PKT_MSG, config.entry.copy()))
# Reset only the data entries that the next packet may change
config.reset_entries(
keep=["pcap_directory", "pcap_filename", "pkt_num"])
pcap_reader.close()
# Log the number of parsed packets from this pcap file
msg_queue.put((config.INFO_MSG, "Parsed {} packets from the \"{}\" file"
"".format(config.entry["pkt_num"], filepath)))
# Send a copy of each dictionary that changed after parsing packets
if config.network_keys != init_network_keys:
msg_queue.put((config.NETWORK_KEYS_MSG, config.network_keys.copy()))
if config.link_keys != init_link_keys:
msg_queue.put((config.LINK_KEYS_MSG, config.link_keys.copy()))
if config.networks != init_networks:
msg_queue.put((config.NETWORKS_MSG, config.networks.copy()))
if config.devices != init_devices:
msg_queue.put((config.DEVICES_MSG, config.devices.copy()))
if config.addresses != init_addresses:
msg_queue.put((config.ADDRESSES_MSG, config.addresses.copy()))
if config.pairs != init_pairs:
msg_queue.put((config.PAIRS_MSG, config.pairs.copy()))
def sniff(
count=0,
store=1,
offline=None,
prn=None,
lfilter=None,
L2socket=None,
timeout=None,
opened_socket=None,
stop_filter=None,
var_stop=False,
*arg,
**karg
):
"""Sniff packets
sniff([count=0,] [prn=None,] [store=1,] [offline=None,] [lfilter=None,] + L2ListenSocket args) -> list of packets
count: number of packets to capture. 0 means infinity
store: wether to store sniffed packets or discard them
prn: function to apply to each packet. If something is returned,
it is displayed. Ex:
ex: prn = lambda x: x.summary()
lfilter: python function applied to each packet to determine
if further action may be done
ex: lfilter = lambda x: x.haslayer(Padding)
offline: pcap file to read packets from, instead of sniffing them
timeout: stop sniffing after a given time (default: None)
L2socket: use the provided L2socket
opened_socket: provide an object ready to use .recv() on
stop_filter: python function applied to each packet to determine
if we have to stop the capture after this packet
ex: stop_filter = lambda x: x.haslayer(TCP)
"""
c = 0
if opened_socket is not None:
s = opened_socket
else:
if offline is None:
if L2socket is None:
L2socket = conf.L2listen
s = L2socket(type=ETH_P_ALL, *arg, **karg)
else:
s = PcapReader(offline)
lst = []
if timeout is not None:
stoptime = time.time() + timeout
remain = None
while 1 and not var_stop:
try:
if timeout is not None:
remain = stoptime - time.time()
if remain <= 0:
break
sel = select.select([s], [], [], remain)
if s in sel[0]:
p = s.recv(MTU)
if p is None:
break
if lfilter and not lfilter(p):
continue
if store:
lst.append(p)
c += 1
if prn:
r = prn(p)
if r is not None:
print r
if stop_filter and stop_filter(p):
break
if count > 0 and c >= count:
break
except KeyboardInterrupt:
var_stop = True
break
if opened_socket is None:
s.close()
def sniff(
store=False,
prn=None,
lfilter=None,
stop_event=None,
refresh=0.1,
offline=None,
*args,
**kwargs
):
"""Sniff packets
sniff([count=0,] [prn=None,] [store=1,] [offline=None,] [lfilter=None,] + L2ListenSocket args)
Modified version of scapy.all.sniff
store : bool
wether to store sniffed packets or discard them
prn : None or callable
function to apply to each packet. If something is returned,
it is displayed.
ex: prn = lambda x: x.summary()
lfilter : None or callable
function applied to each packet to determine
if further action may be done
ex: lfilter = lambda x: x.haslayer(Padding)
stop_event : None or Event
Event that stops the function when set
refresh : float
check stop_event.set() every `refresh` seconds
"""
logger.debug("Setting up sniffer...")
if offline is None:
L2socket = conf.L2listen
s = L2socket(type=ETH_P_ALL, *args, **kwargs)
else:
s = PcapReader(offline)
# on Windows, it is not possible to select a L2socket
if WINDOWS:
from scapy.arch.pcapdnet import PcapTimeoutElapsed
read_allowed_exceptions = (PcapTimeoutElapsed,)
def _select(sockets):
return sockets
else:
read_allowed_exceptions = ()
def _select(sockets):
try:
return select(sockets, [], [], refresh)[0]
except OSError as exc:
# Catch 'Interrupted system call' errors
if exc.errno == errno.EINTR:
return []
raise
lst = []
try:
logger.debug("Started Sniffing")
while True:
if stop_event and stop_event.is_set():
break
sel = _select([s])
if s in sel:
try:
p = s.recv(MTU)
except read_allowed_exceptions:
# could add a sleep(refresh) if the CPU usage
# is too much on windows
continue
if p is None:
break
if lfilter and not lfilter(p):
continue
if store:
lst.append(p)
if prn:
r = prn(p)
if r is not None:
print(r)
except KeyboardInterrupt:
pass
finally:
logger.debug("Stopped sniffing.")
s.close()
return plist.PacketList(lst, "Sniffed")
if time > 1313743991 and src in infected_ip:
label = 1
else:
label = 0
elif data[2].name == 'ICMP':
# 从IP层拿到ip地址
src = data[1].src
dst = data[1].dst
sport = None
dport = None
protocol = data[2].name
time = data[0].time
origin = data[0].original
if time > 1313743991 and src in infected_ip:
label = 1
else:
label = 0
else:
continue
# 对origin进行处理
num += 1
with open('input.csv', 'a') as f:
f.write(
str(time) + ',' + str(protocol) + ',' + str(src) + ',' +
str(sport) + ',' + str(dst) + ',' + str(dport) + ',' +
str(label) + '\n')
if num % 1000 == 0:
print(num)
except EOFError:
s1.close()
def MatchPcap(self, regex, file, negated=False, ports=[]):
'''MatchPcap(regex, file, negated=False, ports=[])
Matches the given regex with the given pcap file. Returns a matchMap dictionary of sessions that matched.
regex: Any ol' valid regex will do.
file: A libpcap file.
negated: If true, finds sessions that do NOT match the regex.
ports: A list of ports to restrict the matching to.
'''
try:
regex = re.compile(r'(?s)%s' % regex)
except:
self.error('Invalid regular expression: %s' % regex)
raise Exception('Invalid regular expression.')
try:
# scapy.utils.PcapReader
self.debug('Reading: %s' % file)
pcap = PcapReader(file)
except:
# yes, logging.exception should be used here, but it doesn't add any value
self.error('Not a valid pcap file: %s' % file)
raise
# matchMap format:
# {3: {'proto': 6, 'host1': ('1.2.3.4',1024), 'host2': ('9.8.7.6',80)}}
matchMap = {}
newid = 1
# Build a list of streams that match the search regex
while pcap:
try:
packet = pcap.read_packet()
match = {}
matchedStream = False
# Skip if the session's ports aren't in the allowed port list (-p).
try:
if ports and not (packet[2].sport in ports or packet[2].dport in ports):
continue
except AttributeError:
# Continue; weren't any ports at all (ip.proto not in (6,17))
pass
except IndexError:
# Wasn't even IP, skip it
pass
# Perform match
try:
rawpacket = packet[3].build()
# for some reason, re.match doesn't work, yet re.findall does.
if regex.findall(rawpacket):
matchedStream = True
#most verbose: self.debug('matched\n%s' % str(rawpacket))
if matchedStream or negated:
# Run the list backwards in hope of matching early rather than matching at the end of the entire list.
ids = matchMap.keys()
ids.reverse()
unknownStream = True
for id in ids:
try:
# Assuming we'll never see a packet with same src and dst
# TCP,UDP layers referred to by index offset for code simplicity
# would do this as one if statement, but seperating it helps exit early and save cpu cycles
if (packet['IP'].src,packet[2].sport) in (matchMap[id]['host1'],matchMap[id]['host2']):
if (packet['IP'].dst,packet[2].dport) in (matchMap[id]['host1'],matchMap[id]['host2']):
unknownStream = False
# This avoids source port reuse problems, causing session collisions
# unknownStream is True if its a known session yet tcp syn flag is set.
if packet['IP'].proto == 6 and packet['TCP'].sprintf('%flags%') == 'S':
unknownStream = True
break
except AttributeError:
# most likely the session isn't tcp/udp so scapy throws AttributeError if no sport/dport exists. Try without it instead.
if matchMap[id]['proto'] == packet['IP'].proto:
if packet['IP'].src in (matchMap[id]['host1'], matchMap[id]['host2']):
if packet['IP'].dst in (matchMap[id]['host1'], matchMap[id]['host2']):
unknownStream = False
break
# if its not negated and its a newly matched stream, OR negated and an unknown, add it to matchMap. if its negated and matched later, it gets deleted before the end
if (matchedStream and unknownStream and not negated) or (negated and unknownStream and not matchedStream):
matchMap[newid] = {}
# Personal preference of mine: printing matches here rather than when the function finishes gives the user a feeling things are happening, rather than get the messages all at once at the end of the call.
# This is doubly so when dealing with massive 1g+ pcap files
try:
matchMap[newid] = {'proto': packet['IP'].proto, 'host1': (packet['IP'].src,packet[2].sport), 'host2': (packet['IP'].dst,packet[2].dport)}
self.info('Match #%d: Proto %d, IPs %s:%d, %s:%d' % (newid,matchMap[newid]['proto'],matchMap[newid]['host1'][0],matchMap[newid]['host1'][1],matchMap[newid]['host2'][0],matchMap[newid]['host2'][1]))
except AttributeError:
matchMap[newid] = {'proto': packet['IP'].proto, 'host1': packet['IP'].src, 'host2': packet['IP'].dst}
self.info('Match #%d: Proto %d, IPs %s, %s' % (newid,matchMap[newid]['proto'],matchMap[newid]['host1'],matchMap[newid]['host2']))
newid += 1
elif matchedStream and negated and not unknownStream:
# Flag the session as matching regex to NOT keep.
# If deleted now, it would just come back from the next related packet
matchMap[id]['delete'] = True
except IndexError:
pass # no raw layer, nothing to search
except TypeError:
break
if negated:
for id in matchMap.keys():
try:
if matchMap[id]['delete']:
del matchMap[id]
self.info('Match #%d matched, removed from result.' % id)
except KeyError:
pass
# rebuilding the sequential id's here might get confusing with the prior-printed messages. probably best to avoid it.
pcap.close()
del pcap
return matchMap
def sniff(count=0,
store=1,
offline=None,
prn=None,
lfilter=None,
L2socket=None,
timeout=None,
opened_socket=None,
stop_filter=None,
var_stop=False,
*arg,
**karg):
"""Sniff packets
sniff([count=0,] [prn=None,] [store=1,] [offline=None,] [lfilter=None,] + L2ListenSocket args) -> list of packets
count: number of packets to capture. 0 means infinity
store: wether to store sniffed packets or discard them
prn: function to apply to each packet. If something is returned,
it is displayed. Ex:
ex: prn = lambda x: x.summary()
lfilter: python function applied to each packet to determine
if further action may be done
ex: lfilter = lambda x: x.haslayer(Padding)
offline: pcap file to read packets from, instead of sniffing them
timeout: stop sniffing after a given time (default: None)
L2socket: use the provided L2socket
opened_socket: provide an object ready to use .recv() on
stop_filter: python function applied to each packet to determine
if we have to stop the capture after this packet
ex: stop_filter = lambda x: x.haslayer(TCP)
"""
c = 0
if opened_socket is not None:
s = opened_socket
else:
if offline is None:
if L2socket is None:
L2socket = conf.L2listen
s = L2socket(type=ETH_P_ALL, *arg, **karg)
else:
s = PcapReader(offline)
lst = []
if timeout is not None:
stoptime = time.time() + timeout
remain = None
while 1 and not var_stop:
try:
if timeout is not None:
remain = stoptime - time.time()
if remain <= 0:
break
sel = select.select([s], [], [], remain)
if s in sel[0]:
p = s.recv(MTU)
if p is None:
break
if lfilter and not lfilter(p):
continue
if store:
lst.append(p)
c += 1
if prn:
r = prn(p)
if r is not None:
print r
if stop_filter and stop_filter(p):
break
if count > 0 and c >= count:
break
except KeyboardInterrupt:
var_stop = True
break
if opened_socket is None:
s.close()
def sniff(store=False, prn=None, lfilter=None,
stop_event=None, refresh=.1, offline=None, *args, **kwargs):
"""Sniff packets
sniff([count=0,] [prn=None,] [store=1,] [offline=None,] [lfilter=None,] + L2ListenSocket args)
Modified version of scapy.all.sniff
store : bool
wether to store sniffed packets or discard them
prn : None or callable
function to apply to each packet. If something is returned,
it is displayed.
ex: prn = lambda x: x.summary()
lfilter : None or callable
function applied to each packet to determine
if further action may be done
ex: lfilter = lambda x: x.haslayer(Padding)
stop_event : None or Event
Event that stops the function when set
refresh : float
check stop_event.set() every `refresh` seconds
"""
logger.debug("Setting up sniffer...")
if offline is None:
L2socket = conf.L2listen
s = L2socket(type=ETH_P_ALL, *args, **kwargs)
else:
s = PcapReader(offline)
# on Windows, it is not possible to select a L2socket
if WINDOWS:
from scapy.arch.pcapdnet import PcapTimeoutElapsed
read_allowed_exceptions = (PcapTimeoutElapsed,)
def _select(sockets):
return sockets
else:
read_allowed_exceptions = ()
def _select(sockets):
try:
return select(sockets, [], [], refresh)[0]
except select_error as exc:
# Catch 'Interrupted system call' errors
if exc[0] == errno.EINTR:
return []
raise
lst = []
try:
logger.debug("Started Sniffing")
while True:
if stop_event and stop_event.is_set():
break
sel = _select([s])
if s in sel:
try:
p = s.recv(MTU)
except read_allowed_exceptions:
# could add a sleep(refresh) if the CPU usage
# is too much on windows
continue
if p is None:
break
if lfilter and not lfilter(p):
continue
if store:
lst.append(p)
if prn:
r = prn(p)
if r is not None:
print(r)
except KeyboardInterrupt:
pass
finally:
logger.debug("Stopped sniffing.")
s.close()
return plist.PacketList(lst, "Sniffed")