def processFile(file):
"""
Process a pcap file.
file is the pcap file to parse
This function will run any enabled options for each pcap file
"""
hs = HoneysnapSingleton.getInstance()
options = hs.getOptions()
tmpf, deletetmp = check_pcap_file(file)
options["tmpf"] = tmpf
try:
if options["filename"] is not None:
out = rawPathOutput(options["filename"], mode="a+")
else:
out = outputSTDOUT()
except IOError:
# we have some error opening the file
# there is something at that path. Is it a directory?
if not os.path.isdir(options["output_data_directory"]):
print "Error: output_data_directory exists, but is not a directory."
else:
print "Unknown Error creating output file"
sys.exit(1)
out("\n\nAnalysing file: %s\n\n" % file)
if options["do_pcap"] == "YES":
out("Pcap file information:\n")
pi = pcapInfo(tmpf)
pi.setOutput(out)
pi.getStats()
myout = rawPathOutput(options["output_data_directory"] +"/pcapinfo.txt")
pi.setOutput(myout)
pi.getStats()
out("\n")
if options["do_packets"] == "YES":
out("\nIP packet summary for common ports:\n\n")
out("%-40s %10s\n" % ("Filter", "Packets"))
filters = setFilters(options)
for i in filters:
key, filt = i
out(key+"\n")
for hp in options['honeypots']:
p = pcap.pcap(tmpf)
c = Counter(p)
c.setOutput(out)
f = filt % hp
c.setFilter(f)
c.count()
del p
out("\n")
if options["do_incoming"] == "YES":
for hp in options["honeypots"]:
out("Counting incoming connections for %s\n" % hp)
outdir = options["output_data_directory"] + "/%s/conns" % hp
make_dir(outdir)
p = pcap.pcap(tmpf)
s = Summarize(p)
filt = 'dst host %s' % hp
s.setFilter(filt)
s.start()
fileout = rawPathOutput(outdir+"/incoming.txt", mode="a")
if options["print_verbose"] == "YES":
outputs = (fileout, out)
else:
outputs = (fileout,)
for output in outputs:
s.setOutput(output)
s.doOutput("\nIncoming connections for %s\n" % hp)
s.writeResults(limit=options["flow_count_limit"])
del p
if options["do_outgoing"] == "YES":
for hp in options["honeypots"]:
out("\nCounting outgoing connections for %s\n" % hp)
outdir = options["output_data_directory"] + "/%s/conns" % hp
make_dir(outdir)
p = pcap.pcap(tmpf)
s = Summarize(p)
filt = 'src host %s' % hp
s.setFilter(filt)
s.start()
fileout = rawPathOutput(outdir+"/outgoing.txt", mode="a")
if options["print_verbose"] == "YES":
outputs = (fileout, out)
else:
outputs = (fileout,)
for output in outputs:
s.setOutput(output)
s.doOutput("\nOutgoing connections for %s\n" % hp)
s.writeResults(limit=options["flow_count_limit"])
del p
if options["do_dns"] == "YES":
out("\nExtracting DNS data to file\n\n")
for hp in options["honeypots"]:
#out("\nHoneypot %s\n\n" % hp)
dns = dnsDecode(hp, direction="queried")
dns.setOutdir(options["output_data_directory"] + "/%s/dns" % hp)
dns.setOutput(out)
dns.run()
del dns
dns = dnsDecode(hp, direction="served")
dns.setOutdir(options["output_data_directory"] + "/%s/dns" % hp)
dns.setOutput(out)
dns.run()
del dns
if options["do_telnet"] == "YES":
out("\nExtracting telnet data to file\n")
for hp in options["honeypots"]:
#out("\nHoneypot %s\n\n" % hp)
tel = telnetDecode(hp)
tel.setOutdir(options["output_data_directory"] + "/%s/telnet" % hp)
tel.setOutput(out)
tel.run()
del tel
if options["do_irc"] == "YES":
"""
Here we will use PcapRE to find packets on irc_port with "PRIVMSG"
in the payload.
"""
for hp in options["honeypots"]:
out("\nLooking for packets containing PRIVMSG for %s\n\n" % hp)
p = pcap.pcap(tmpf)
r = pcapReCounter(p)
r.setFilter("host %s and tcp" % hp)
r.setRE('PRIVMSG')
r.setOutput(out)
r.start()
r.writeResults()
for port in r.server_ports(options['irc_ports'][hp]):
if port not in options['irc_ports'][hp]:
if port==80:
out("\nSaw PRIVMSG on port 80, but cowardly not adding it to IRC port list - check manually\n")
else:
out("\nAdding port %s to irc list for %s\n" % (port, hp))
options['irc_ports'][hp].add(port)
del p
del r
out("\nAnalysing IRC\n")
for hp in options["honeypots"]:
outdir = options["output_data_directory"] + "/%s/irc" % hp
for port in options["irc_ports"][hp]:
out("\nHoneypot %s, port %s\n\n" % (hp, port))
hirc = HoneySnapIRC()
hirc.connect(tmpf, "host %s and tcp and port %s" % (hp, port))
hd = ircDecode(hp)
hd.setOutput(out)
hd.setOutdir(outdir)
hd.setOutfile('irclog-%s.txt' % port)
hirc.addHandler("all_events", hd.decodeCB, -1)
hirc.ircobj.add_global_handler("all_events", hd.printLines, -1)
hirc.ircobj.process_once()
hd.printSummary()
del hd
del hirc
if options["all_flows"] == "YES":
out("\nExtracting all flows\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
filt = "host "
filt += " or host ".join(options["honeypots"])
de.setFilter(filt)
de.setOutdir(options["output_data_directory"]+ "/%s/flows")
de.setOutput(out)
de.start()
de.dump_extract()
del de
del p
if options["do_http"] == "YES":
out("\nExtracting from HTTP\n\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
de.setFilter("port 80")
de.setOutdir(options["output_data_directory"]+ "/%s/http")
de.setOutput(out)
decode = httpDecode.httpDecode()
decode.setOutput(out)
de.registerPlugin(decode.decode)
de.start()
de.dump_extract()
decode.print_summary()
del de
del p
if options["do_ftp"] == "YES":
out("\nExtracting from FTP\n\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
de.setFilter("port 20 or port 21")
de.setOutdir(options["output_data_directory"] + "/%s/ftp")
de.setOutput(out)
decode = ftpDecode.ftpDecode()
decode.setOutput(out)
de.registerPlugin(decode.decode)
de.start()
de.dump_extract()
decode.print_summary()
del de
del p
if options["do_smtp"] == "YES":
out("\nExtracting from SMTP\n\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
de.setFilter("port 25")
de.setOutdir(options["output_data_directory"] + "/%s/smtp")
de.setOutput(out)
decode = smtpDecode.smtpDecode()
decode.setOutput(out)
de.registerPlugin(decode.decode)
de.start()
de.dump_extract()
decode.print_summary()
del de
del p
if options["do_sebek"] == "YES":
out("\nExtracting Sebek data\n")
for hp in options["honeypots"]:
out("\nHoneypot %s\n\n" % hp)
sbd = sebekDecode(hp)
sbd.setOutdir(options["output_data_directory"] + "/%s/sebek" % hp)
sbd.setOutput(out)
sbd.run()
sbd.print_summary()
del sbd
if options["do_socks"] == "YES":
out("\nExtracting Socks proxy information:\n")
for hp in options["honeypots"]:
out("\nHoneypot %s\n\n" % hp)
p = pcap.pcap(tmpf)
socks = SocksDecode(p,hp)
socks.setOutdir(options["output_data_directory"] + "/%s/socks" % hp)
socks.setOutput(out)
socks.start()
# delete the tmp file we used to hold unzipped data
if deletetmp:
os.unlink(tmpf)
def processFile(file):
"""
Process a pcap file.
file is the pcap file to parse
This function will run any enabled options for each pcap file
"""
hs = HoneysnapSingleton.getInstance()
options = hs.getOptions()
tmpf, deletetmp = check_pcap_file(file)
options["tmpf"] = tmpf
try:
if options["filename"] is not None:
out = rawPathOutput(options["filename"], mode="a+")
else:
out = outputSTDOUT()
except IOError:
# we have some error opening the file
# there is something at that path. Is it a directory?
if not os.path.isdir(options["output_data_directory"]):
print "Error: output_data_directory exists, but is not a directory."
else:
print "Unknown Error creating output file"
sys.exit(1)
out("\n\nAnalysing file: %s\n\n" % file)
if options["do_pcap"] == "YES":
out("Pcap file information:\n")
pi = pcapInfo(tmpf)
pi.setOutput(out)
pi.getStats()
myout = rawPathOutput(options["output_data_directory"] +
"/pcapinfo.txt")
pi.setOutput(myout)
pi.getStats()
out("\n")
if options["do_packets"] == "YES":
out("\nIP packet summary for common ports:\n\n")
out("%-40s %10s\n" % ("Filter", "Packets"))
filters = setFilters(options)
for i in filters:
key, filt = i
out(key + "\n")
for hp in options['honeypots']:
p = pcap.pcap(tmpf)
c = Counter(p)
c.setOutput(out)
f = filt % hp
c.setFilter(f)
c.count()
del p
out("\n")
if options["do_incoming"] == "YES":
for hp in options["honeypots"]:
out("Counting incoming connections for %s\n" % hp)
outdir = options["output_data_directory"] + "/%s/conns" % hp
make_dir(outdir)
p = pcap.pcap(tmpf)
s = Summarize(p)
filt = 'dst host %s' % hp
s.setFilter(filt)
s.start()
fileout = rawPathOutput(outdir + "/incoming.txt", mode="a")
if options["print_verbose"] == "YES":
outputs = (fileout, out)
else:
outputs = (fileout, )
for output in outputs:
s.setOutput(output)
s.doOutput("\nIncoming connections for %s\n" % hp)
s.writeResults(limit=options["flow_count_limit"])
del p
if options["do_outgoing"] == "YES":
for hp in options["honeypots"]:
out("\nCounting outgoing connections for %s\n" % hp)
outdir = options["output_data_directory"] + "/%s/conns" % hp
make_dir(outdir)
p = pcap.pcap(tmpf)
s = Summarize(p)
filt = 'src host %s' % hp
s.setFilter(filt)
s.start()
fileout = rawPathOutput(outdir + "/outgoing.txt", mode="a")
if options["print_verbose"] == "YES":
outputs = (fileout, out)
else:
outputs = (fileout, )
for output in outputs:
s.setOutput(output)
s.doOutput("\nOutgoing connections for %s\n" % hp)
s.writeResults(limit=options["flow_count_limit"])
del p
if options["do_dns"] == "YES":
out("\nExtracting DNS data to file\n\n")
for hp in options["honeypots"]:
#out("\nHoneypot %s\n\n" % hp)
dns = dnsDecode(hp, direction="queried")
dns.setOutdir(options["output_data_directory"] + "/%s/dns" % hp)
dns.setOutput(out)
dns.run()
del dns
dns = dnsDecode(hp, direction="served")
dns.setOutdir(options["output_data_directory"] + "/%s/dns" % hp)
dns.setOutput(out)
dns.run()
del dns
if options["do_telnet"] == "YES":
out("\nExtracting telnet data to file\n")
for hp in options["honeypots"]:
#out("\nHoneypot %s\n\n" % hp)
tel = telnetDecode(hp)
tel.setOutdir(options["output_data_directory"] + "/%s/telnet" % hp)
tel.setOutput(out)
tel.run()
del tel
if options["do_irc"] == "YES":
"""
Here we will use PcapRE to find packets on irc_port with "PRIVMSG"
in the payload.
"""
for hp in options["honeypots"]:
out("\nLooking for packets containing PRIVMSG for %s\n\n" % hp)
p = pcap.pcap(tmpf)
r = pcapReCounter(p)
r.setFilter("host %s and tcp" % hp)
r.setRE('PRIVMSG')
r.setOutput(out)
r.start()
r.writeResults()
for port in r.server_ports(options['irc_ports'][hp]):
if port not in options['irc_ports'][hp]:
if port == 80:
out("\nSaw PRIVMSG on port 80, but cowardly not adding it to IRC port list - check manually\n"
)
else:
out("\nAdding port %s to irc list for %s\n" %
(port, hp))
options['irc_ports'][hp].add(port)
del p
del r
out("\nAnalysing IRC\n")
for hp in options["honeypots"]:
outdir = options["output_data_directory"] + "/%s/irc" % hp
for port in options["irc_ports"][hp]:
out("\nHoneypot %s, port %s\n\n" % (hp, port))
hirc = HoneySnapIRC()
hirc.connect(tmpf, "host %s and tcp and port %s" % (hp, port))
hd = ircDecode(hp)
hd.setOutput(out)
hd.setOutdir(outdir)
hd.setOutfile('irclog-%s.txt' % port)
hirc.addHandler("all_events", hd.decodeCB, -1)
hirc.ircobj.add_global_handler("all_events", hd.printLines, -1)
hirc.ircobj.process_once()
hd.printSummary()
del hd
del hirc
if options["all_flows"] == "YES":
out("\nExtracting all flows\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
filt = "host "
filt += " or host ".join(options["honeypots"])
de.setFilter(filt)
de.setOutdir(options["output_data_directory"] + "/%s/flows")
de.setOutput(out)
de.start()
de.dump_extract()
del de
del p
if options["do_http"] == "YES":
out("\nExtracting from HTTP\n\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
de.setFilter("port 80")
de.setOutdir(options["output_data_directory"] + "/%s/http")
de.setOutput(out)
decode = httpDecode.httpDecode()
decode.setOutput(out)
de.registerPlugin(decode.decode)
de.start()
de.dump_extract()
decode.print_summary()
del de
del p
if options["do_ftp"] == "YES":
out("\nExtracting from FTP\n\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
de.setFilter("port 20 or port 21")
de.setOutdir(options["output_data_directory"] + "/%s/ftp")
de.setOutput(out)
decode = ftpDecode.ftpDecode()
decode.setOutput(out)
de.registerPlugin(decode.decode)
de.start()
de.dump_extract()
decode.print_summary()
del de
del p
if options["do_smtp"] == "YES":
out("\nExtracting from SMTP\n\n")
p = pcap.pcap(tmpf)
de = tcpflow.tcpFlow(p)
de.setFilter("port 25")
de.setOutdir(options["output_data_directory"] + "/%s/smtp")
de.setOutput(out)
decode = smtpDecode.smtpDecode()
decode.setOutput(out)
de.registerPlugin(decode.decode)
de.start()
de.dump_extract()
decode.print_summary()
del de
del p
if options["do_sebek"] == "YES":
out("\nExtracting Sebek data\n")
for hp in options["honeypots"]:
out("\nHoneypot %s\n\n" % hp)
sbd = sebekDecode(hp)
sbd.setOutdir(options["output_data_directory"] + "/%s/sebek" % hp)
sbd.setOutput(out)
sbd.run()
sbd.print_summary()
del sbd
if options["do_socks"] == "YES":
out("\nExtracting Socks proxy information:\n")
for hp in options["honeypots"]:
out("\nHoneypot %s\n\n" % hp)
p = pcap.pcap(tmpf)
socks = SocksDecode(p, hp)
socks.setOutdir(options["output_data_directory"] +
"/%s/socks" % hp)
socks.setOutput(out)
socks.start()
# delete the tmp file we used to hold unzipped data
if deletetmp:
os.unlink(tmpf)
def extractPassive(self, state, d):
#print "Passive FTP"
username, password = "******", "Unknown"
# repr(port/256), repr(port%256)
# first we have to find the reverse flow/state
# from it we will extract the ip and port info
rflow = freverse(state.flow)
rstate = self.statemgr.find_flow_state(rflow)
if rstate is None:
# no reverse state, bail
return
rstate.open(flags="rb", statemgr=self.statemgr)
dchannel = rstate.fp.readlines()
rstate.close()
lines = d.splitlines()
iterlines = iter(lines)
portlines = []
cmdlines = []
# find all the lines from the server
# that open a data port
# find all the 227 lines in the data channel
for l in dchannel:
m = self._227re.search(l)
if m is not None:
portlines.append(l)
# find all the client lines that use
# a data port
for l in lines:
m = self.userRE.search(l)
if m:
username = m.group(1)
continue
else:
username = "******"
m = self.passRE.search(l)
if m:
password = m.group(1)
continue
else:
password = "******"
w = [i for i in cmds if i in l.split()[0]]
if len(w) == 0:
# this line doesn't contain a data command
continue
cmdlines.append(l)
# zip the 2 lists together
# should give [(227 response, Client CMD),...]
pairs = zip(portlines, cmdlines)
for p in pairs:
if p[1].find("RETR") < 0:
# not a RETR command
continue
m = self.portIPRE.search(p[0])
if m is not None:
# the last 2 items in the RE result are the port info
info = m.group().split(",")
p256 = int(info[-2])
p1 = int(info[-1])
ip = ".".join(info[0:4])
port = 256*p256 + p1
else:
continue
filename = p[1].split(" ")[1]
rflow.sport = port
# passive ftp transactions happen on high ports
# so the stream extractor has not extracted the data
# create a new stream extractor to pull the data
p = pcap.pcap(self.options["tmpf"])
de = tcpflow.tcpFlow(p)
filter = "src host %s and src port %d" % (rflow.src, rflow.sport)
de.setFilter(filter)
de.setOutdir(self.options["output_data_directory"]+ "/%s/ftp")
# run the flow extractor
de.start()
# now find the correct state
flows = [f for f in de.states.getFlows() if f.isSrcSport(ip, port)]
if len(flows) > 0:
if len(flows) > 1:
print "hmmm, got more than 1 flow"
rflow = flows[0]
rstate = de.states.find_flow_state(rflow)
# rename the data file
if rstate is not None:
fn = renameFile(rstate, filename)
id, m5 = self.id.identify(rstate)
output = "%s requested %s from %s (%s, %s) at %s\n\tfile: %s, filetype: %s, md5 sum: %s\n" % (rstate.flow.dst, filename,
rstate.flow.src, username, password, self.tf(rstate.ts), fn, id, m5)
self.add_flow(rstate.ts, rstate.flow.src, rstate.flow.dst, output)