def process_IN_MOVED_TO(self, event): portList = [''] portList.append(self.port) d=pynfdump.Dumper(self.basedir, sources=portList) d.set_where(dirfiles="") records = pynfdump.search_file(event.pathname,"") self.logRecords(records,self.syslogserverlist,Util.str2facility(self.syslogfacility),Util.str2priority(self.syslogpriority),self.timerotation)
def main():
signal.signal(signal.SIGINT, signal_handler)
args = deal_with_arguments()
packets = {}
Bytes = {}
for proto in (TCP, UDP, ICMP):
packets[proto] = 0
Bytes[proto] = 0
print("Press Ctrl+C to exit.")
print("Packets processed:\t")
# Set up our storage.
try:
outputfile = open(args.db_name, 'w')
tcp_flows = FlowStorage(filename=args.db_name)
udp_flows = FlowStorage(filename=args.db_name)
icmp_flows = FlowStorage(filename=args.db_name)
dests = DestStorage(filename=args.db_name)
except Exception as e:
print("Problem setting up databases:\n\t%s" % e)
exit()
# Try opening our trace.
if ("nfdump:" in args.input):
records = search_file(args.input[7:])
parse_nfdump(records, packets, Bytes, args.interval, tcp_flows,
udp_flows, icmp_flows)
elif ("flow-tools:" in args.input):
records = flowtools.FlowSet(args.input[11:])
parse_flowtools(records, packets, Bytes, args.interval, tcp_flows,
udp_flows, icmp_flows)
else:
try:
t = plt.trace(args.input)
except Exception as e:
print("Trouble opening trace URI/device:\n\t%s" % e)
exit()
# Try setting up our filter if given one.
try:
if args.filter != None or args.target != None:
if args.filter != None and args.target != None:
args.filter = args.filter + " and "
elif args.filter == None:
args.filter = ""
if args.target != None:
args.filter = args.filter + "dst "
if '/' in args.target:
args.filter = args.filter + "net "
args.filter = args.filter + args.target
f = plt.filter(args.filter)
print("Applying filter \"%s\"" % args.filter)
t.conf_filter(f)
except Exception as e:
print("Trouble applying bpf filter: \'%s\'\n\t%s" %
(args.filter, e))
exit()
try:
t.start()
except Exception as e:
print(e)
exit()
parse_pcap(t, packets, Bytes, args.interval, tcp_flows, udp_flows,
icmp_flows)
print("\n************ OVERALL STATS ******************\n")
print(
"TCP packets\t%s\tBytes\t%s\nUDP packets\t%s\tBytes\t%s\nICMP packets\t%s\tBytes\t%s\n"
% (packets[TCP], Bytes[TCP], packets[UDP], Bytes[UDP], packets[ICMP],
Bytes[ICMP]))
print >> outputfile, "ALL %s %s %s %s %s %s" % (packets[TCP], packets[UDP],
packets[ICMP], Bytes[TCP],
Bytes[UDP], Bytes[ICMP])
dests.print_stats(tcp_flows, 'TCP', outputfile)
dests.print_stats(udp_flows, 'UDP', outputfile)
dests.print_stats(icmp_flows, 'ICMP', outputfile)
outputfile.close()
def test_file_not_found():
for x in pynfdump.search_file("this file isn't here"):
print x
def nfquery():
logging.info('Script started')
f = open('/etc/netflow-alerting.yaml')
data = yaml.load(f)
f.close()
s = shelve.open('/tmp/netflow-alerting.db')
rootpath = data["netflowpath"]
queries = data["queries"]
# start time is -5 minutes rounded to the the previous 5 minutes
now = datetime.datetime.now()
rounded = now - timedelta(minutes=now.minute % 5 + 5,
seconds=now.second,
microseconds=now.microsecond)
starttime = rounded.strftime('%Y-%m-%d %H:%M')
filename = "nfcapd." + rounded.strftime('%Y%m%d%H%M')
filepath = rootpath + rounded.strftime('%Y/%m/%d/') + filename
if GeoIP is not None:
GEOIP_DB_PATH = data["geoip_db_path"]
gi = GeoIP.open(GEOIP_DB_PATH, GeoIP.GEOIP_STANDARD)
for k, v in queries.items():
nfquery = v["query"]
nforderby = v["order"]
stats = v["stats"]
state = v["state"]
if "threshold" in v:
threshold = int(v["threshold"])
if "ipwhitelist" in v:
ipwhitelist = v["ipwhitelist"]
else:
ipwhitelist = None
logging.info('Performing query %s %s %s %s', nfquery, stats, nforderby,
filepath)
search = pynfdump.search_file(filepath,
query=nfquery,
statistics=stats,
statistics_order=nforderby,
limit=500)
for r in search:
if threshold:
if int(r[nforderby]) >= threshold:
item = str(r[stats])
nb = r[nforderby]
whois = ''
if "ip" in stats:
country_code = gi.country_code_by_addr(item)
whois = "Whois: http://whois.domaintools.com/%s" % (
item)
# Check if IP is whitelisted
if ipwhitelist is not None:
ipwhitelistmatch = all_matching_cidrs(
item, ipwhitelist)
if ipwhitelistmatch:
logging.info('IP %s is whitelisted (%s)', item,
ipwhitelistmatch)
continue
txt = "Alert '%s' triggered matching query '%s' with %s %s for %s %s (%s) at time '%s'. Threshold is %s. %s %s" % (
k, nfquery, nb, nforderby, stats, item, country_code,
starttime, threshold, filepath, whois)
service = "netflow-alerting-%s-%s" % (stats, item)
sendalert(txt, service, state)
logging.info('%s', txt)
# We add the service in the persistence DB
s[service] = starttime
else:
break
logging.info('Query completed')
# We remove any entry that have older timestamp and send a riemann ok event for that envent
for k, v in s.iteritems():
if v != starttime:
del s[k]
sendclear(k)
s.close()
logging.info('Script completed')
def test_bogus_options():
for x in pynfdump.search_file("", statistics="foo", aggregate="bar"):
print x
