def validateFlow(self, flow_path):
# We check that the file to be opened exists.
if os.path.exists(flow_path):
try:
Flow_Set = flowtools.FlowSet(flow_path)
except:
return -1
return Flow_Set
return -1
def buildFlowToolsTrie(infile, trie):
global nodes
flows = flowtools.FlowSet(infile)
laststat = 0
for flow in flows:
flip = 0
if (laststat == 0):
laststat = flow.last
if (flow.first - laststat > PERIOD):
printFlows(trie, flow.last)
laststat = flow.last
if flow.dstport in WELL_KNOWN_PORTS and flow.srcport not in WELL_KNOWN_PORTS:
src = flow.srcaddr
dst = flow.dstaddr
sport = flow.srcport
dport = flow.dstport
elif flow.dstport not in WELL_KNOWN_PORTS and flow.srcport in WELL_KNOWN_PORTS:
src = flow.dstaddr
dst = flow.srcaddr
sport = flow.dstport
dport = flow.srcport
flip = 1
else:
continue
if flow.tcp_flags > 63:
flags = bitarray('{0:06b}'.format(flow.tcp_flags & 63))
else:
flags = bitarray('{0:06b}'.format(flow.tcp_flags))
sc = 0
uc = 0
if (flow.prot == 6):
if (flags & SYN_ACK_PSH == SYN_ACK_PSH):
sc = 1
else:
uc = 1
elif (flow.prot == 17):
if (flip == 1):
sc = 1
else:
uc = 1
else:
continue
dst = str(dst) + ":" + str(dport)
if (dst not in trie):
trie[dst] = DestInfo(ls=flow.last)
nodes += 1
trie[dst].sc += sc
trie[dst].uc += uc
def getFlows(infile):
flows = flowtools.FlowSet(infile)
laststat = 0
for flow in flows:
flip = 0
if (laststat == 0):
laststat = flow.last
if flow.dstport in WELL_KNOWN_PORTS and flow.srcport not in WELL_KNOWN_PORTS:
src = flow.srcaddr
dst = flow.dstaddr
sport = flow.srcport
dport = flow.dstport
elif flow.dstport not in WELL_KNOWN_PORTS and flow.srcport in WELL_KNOWN_PORTS:
src = flow.dstaddr
dst = flow.srcaddr
sport = flow.dstport
dport = flow.srcport
flip = 1
else:
continue
if flow.tcp_flags > 63:
flags = bitarray('{0:06b}'.format(flow.tcp_flags & 63))
else:
flags = bitarray('{0:06b}'.format(flow.tcp_flags))
sc = 0
uc = 0
if (flow.prot == 6):
if (flags & SYN_ACK_PSH == SYN_ACK_PSH):
sc = 1
else:
uc = 1
elif (flow.prot == 17):
if (flip == 1):
sc = 1
else:
uc = 1
else:
continue
dst = str(dst) + ":" + str(dport)
src = str(src) + ":" + str(sport)
char = " -> "
if (flip):
char = " <- "
if (dst == "198.108.0.0:53"):
print str(src) + char + str(dst) + " " + str(
flow.dPkts) + " " + str(flow.dOctets) + " " + str(sc)
def buildFlowToolsTrie(infile, trie):
flows = flowtools.FlowSet(infile)
laststat = 0
for flow in flows:
if (laststat == 0):
laststat = flow.first
if (flow.first - laststat > PERIOD):
printFlows(trie)
laststat = flow.first
if flow.prot != 6 or flow.dstport not in WELL_KNOWN_PORTS:
continue
if flow.last - flow.first < 15:
continue
if flow.tcp_flags > 63:
flags = bitarray('{0:06b}'.format(flow.tcp_flags & 63))
else:
flags = bitarray('{0:06b}'.format(flow.tcp_flags))
sc = 0
uc = 0
if (flags & SYN_ACK_PSH == SYN_ACK_PSH) or (flags & SYN_ACK_PSH_FIN == SYN_ACK_PSH_FIN)\
or (flags & SYN_ACK_FIN == SYN_ACK_FIN) or (flags & SYN_ACK_URG == SYN_ACK_URG)\
or (flags & SYN_ACK_URG_FIN == SYN_ACK_URG_FIN):
sc = 1
else:
uc = 1
time = float(flow.last - flow.first)
dst = str(flow.dstaddr)
src = str(flow.srcaddr)
if (dst not in trie and src not in trie):
trie[dst] = DestInfo(ls=flow.last,
sc=sc,
uc=uc,
mb=float(flow.dOctets) / 1000000,
dur=float(flow.last - flow.first))
else:
if (src in trie):
dst = src
trie[dst].last_seen = flow.last
trie[dst].successful_cnxns += sc
trie[dst].unsuccessful_cnxns += uc
trie[dst].mega_bytes += float(flow.dOctets) / 1000000
trie[dst].duration += float(flow.last - flow.first)
def buildFlowToolsTrie(infile, trie):
flows = flowtools.FlowSet(infile)
for flow in flows:
if flow.prot != 6 or flow.dstport not in WELL_KNOWN_PORTS:
continue
if flow.last - flow.first < 15:
continue
if flow.tcp_flags > 63:
flags = bitarray('{0:06b}'.format(flow.tcp_flags & 63))
else:
flags = bitarray('{0:06b}'.format(flow.tcp_flags))
sc = 0
uc = 0
if (flags & SYN_ACK_PSH == SYN_ACK_PSH) or (flags & SYN_ACK_PSH_FIN == SYN_ACK_PSH_FIN)\
or (flags & SYN_ACK_FIN == SYN_ACK_FIN) or (flags & SYN_ACK_URG == SYN_ACK_URG)\
or (flags & SYN_ACK_URG_FIN == SYN_ACK_URG_FIN):
sc = 1
else:
uc = 1
time = float(flow.last - flow.first)
dst = str(flow.dstaddr)
src = str(flow.srcaddr)
if (dst not in trie and src not in trie):
trie[dst] = DestInfo(ls=flow.last,
sc=sc,
uc=uc,
mb=float(flow.dOctets) / 1000000,
dur=float(flow.last - flow.first))
else:
if (src in trie):
dst = src
trie[dst].last_seen = flow.last
trie[dst].successful_cnxns += sc
trie[dst].unsuccessful_cnxns += uc
trie[dst].mega_bytes += float(flow.dOctets) / 1000000
trie[dst].duration += float(flow.last - flow.first)
percent_uc = float(trie[dst].unsuccessful_cnxns) / float(
trie[dst].unsuccessful_cnxns + trie[dst].successful_cnxns)
if (trie[dst].unsuccessful_cnxns +
trie[dst].successful_cnxns) > 100 and percent_uc > 0.1:
print "Destination Address: " + dst + "\t Successful Connections " + str(trie[dst].successful_cnxns) \
+ " Unsuccessful Connections: " + str(trie[dst].unsuccessful_cnxns) + "\t Percentage Unsuccessful "+str(percent_uc*100) + " %" #" \t Bytes per second: " + str(float(trie[dst].mega_bytes)/float(trie[dst].duration))
def main(argv):
global dt_min, dt_max
root = argv[0]
# 1. collect data
for f in os.listdir(root):
flowset = flowtools.FlowSet(os.path.join(root, f))
for flow in flowset:
dt = long(flow.last)
if (dt < dt_min):
dt_min = dt
elif (dt > dt_max):
dt_max = dt
dt = datetime.datetime.fromtimestamp(long(flow.last))
# 2. print
dt_min = datetime.datetime.fromtimestamp(dt_min)
dt_max = datetime.datetime.fromtimestamp(dt_max)
tz = 0
print dt_min, '..', dt_max, ", TZ =", tz
def main(argv):
global data
root = argv[0]
stamp = int(root.replace('-', ''))
# 1. collect data
for f in os.listdir(root):
flowset = flowtools.FlowSet(os.path.join(root, f))
for flow in flowset:
TZ = TZ1 if (stamp > TZmete) else 0
dt = datetime.datetime.fromtimestamp(long(flow.last) + TZ)
ymd, hour = (int(dt.date().isoformat().replace('-', '')), dt.hour)
proto = flow.prot
if (flow.dstaddr_raw & netmask) == net: # inbound
iip = flow.dstaddr_raw & hostmask
iport = flow.dstport
oip = flow.srcaddr_raw
oport = flow.srcport
ibytes = flow.dOctets
obytes = 0
elif (flow.srcaddr_raw & netmask) == net: #outbound
iip = flow.srcaddr_raw & hostmask
iport = flow.srcport
oip = flow.dstaddr_raw
oport = flow.dstport
ibytes = 0
obytes = flow.dOctets
key = (ymd, hour, proto, iip, iport, oip, oport)
r = data.get(key, None)
if r == None:
data[key] = [ibytes, obytes]
else:
data[key] = [r[0] + ibytes, r[1] + obytes]
# 2. print
print "BEGIN;"
print "DELETE FROM data WHERE stamp = %d;" % stamp
for k, v in data.iteritems():
print tpl % (stamp, k[0], k[1], k[2], k[3], k[4], k[5], k[6], v[0],
v[1])
print "COMMIT;"
def main():
nodes = 0
parser = argparse.ArgumentParser(description="Detect heavy hitters from traces")
parser.add_argument('-f', '--format', dest='file_format', nargs=1, default='None', choices=['nfdump', 'flow-tools'],
required=True, help='Trace format i.e. flow-tools or nfdump')
parser.add_argument('infile', nargs='?', default=sys.stdin,
help='File path to read from. If no path is specified then defaults to stdin')
args = parser.parse_args()
trie = dict()
"""t.StringTrie(separator='.')
"""
if (args.file_format[0] == "flow-tools"):
# getFlows(args.infile)
flow_dir = args.infile.split("/")[6]
# Create a TCP/IP socket
server_address = ('localhost', 4242)
flows = flowtools.FlowSet(args.infile)
client_socket = Client(server_address, flow_dir, flows)
try:
asyncore.loop(timeout=1)
except asyncore.ExitNow, e:
pass
import flowtools
set = flowtools.FlowSet(
"/Users/zorro/PycharmProjects/SENSS/ft-v05.2015-07-22.000000-0400")
for flow in set:
print "%s" % (flow.prot)
#! /usr/bin/env python
import flowtools
set = flowtools.FlowSet("-") # Read from stdin
for flow in set:
print "%s %s" % (flow.srcaddr, flow.dstaddr)
print " ", repr(flow.getID())
print " ", repr(flow.getID(1))
def main():
signal.signal(signal.SIGINT, signal_handler)
args = deal_with_arguments()
packets = {}
Bytes = {}
for proto in (TCP, UDP, ICMP):
packets[proto] = 0
Bytes[proto] = 0
print("Press Ctrl+C to exit.")
print("Packets processed:\t")
# Set up our storage.
try:
outputfile = open(args.db_name, 'w')
tcp_flows = FlowStorage(filename=args.db_name)
udp_flows = FlowStorage(filename=args.db_name)
icmp_flows = FlowStorage(filename=args.db_name)
dests = DestStorage(filename=args.db_name)
except Exception as e:
print("Problem setting up databases:\n\t%s" % e)
exit()
# Try opening our trace.
if ("nfdump:" in args.input):
records = search_file(args.input[7:])
parse_nfdump(records, packets, Bytes, args.interval, tcp_flows,
udp_flows, icmp_flows)
elif ("flow-tools:" in args.input):
records = flowtools.FlowSet(args.input[11:])
parse_flowtools(records, packets, Bytes, args.interval, tcp_flows,
udp_flows, icmp_flows)
else:
try:
t = plt.trace(args.input)
except Exception as e:
print("Trouble opening trace URI/device:\n\t%s" % e)
exit()
# Try setting up our filter if given one.
try:
if args.filter != None or args.target != None:
if args.filter != None and args.target != None:
args.filter = args.filter + " and "
elif args.filter == None:
args.filter = ""
if args.target != None:
args.filter = args.filter + "dst "
if '/' in args.target:
args.filter = args.filter + "net "
args.filter = args.filter + args.target
f = plt.filter(args.filter)
print("Applying filter \"%s\"" % args.filter)
t.conf_filter(f)
except Exception as e:
print("Trouble applying bpf filter: \'%s\'\n\t%s" %
(args.filter, e))
exit()
try:
t.start()
except Exception as e:
print(e)
exit()
parse_pcap(t, packets, Bytes, args.interval, tcp_flows, udp_flows,
icmp_flows)
print("\n************ OVERALL STATS ******************\n")
print(
"TCP packets\t%s\tBytes\t%s\nUDP packets\t%s\tBytes\t%s\nICMP packets\t%s\tBytes\t%s\n"
% (packets[TCP], Bytes[TCP], packets[UDP], Bytes[UDP], packets[ICMP],
Bytes[ICMP]))
print >> outputfile, "ALL %s %s %s %s %s %s" % (packets[TCP], packets[UDP],
packets[ICMP], Bytes[TCP],
Bytes[UDP], Bytes[ICMP])
dests.print_stats(tcp_flows, 'TCP', outputfile)
dests.print_stats(udp_flows, 'UDP', outputfile)
dests.print_stats(icmp_flows, 'ICMP', outputfile)
outputfile.close()
def sort_flows(infile):
flows = flowtools.FlowSet(infile)
replies = 0
requests = 0
total_flows = 0
fh = open("all_flows/13-14/" + infile.split('/')[6] + "_recheck.txt", "a")
for flow in flows:
total_flows += 1
flip = False
if flow.dstport in WELL_KNOWN_PORTS and flow.srcport not in WELL_KNOWN_PORTS:
src = flow.srcaddr
dst = flow.dstaddr
sport = flow.srcport
dport = flow.dstport
elif flow.dstport not in WELL_KNOWN_PORTS and flow.srcport in WELL_KNOWN_PORTS:
src = flow.dstaddr
dst = flow.srcaddr
sport = flow.dstport
dport = flow.srcport
flip = True
else:
continue
dst = str(dst) + ":" + str(dport)
src = str(src) + ":" + str(sport)
if dst != "207.75.112.0:53":
continue
if flow.tcp_flags > 63:
flags = bitarray('{0:06b}'.format(flow.tcp_flags & 63))
else:
flags = bitarray('{0:06b}'.format(flow.tcp_flags))
success_count = 0
unsuccessful_count = 0
if flow.prot == 6:
flow_count = 1
if flags & SYN_ACK_PSH == SYN_ACK_PSH:
success_count = 1
else:
unsuccessful_count = 1
elif flow.prot == 17:
flow_count = flow.dPkts
if flip:
success_count = 1
else:
unsuccessful_count = 1
else:
continue
#fh.write(str(int(flow.last)) + "\t" + src + "\t" + dst + "\t" + str(flow_count) + "\n")
fh.write(str(int(flow.last)) + "\n")
if success_count:
replies += flow_count
else:
requests += flow_count
# fh = open("chi-600e_all_times", "a")
# fh.write(infile.split('/')[10] + "\t" + str(requests) + "\t" + str(replies) + "\n")
fh.write("\n" + str(total_flows))
fh.close()