def alert(subject, ip, username, content):
smtpServer = 'smtp.btconnect.com'
sender = '*****@*****.**'
destination = ['*****@*****.**']
debugLevel = False
try:
# Get DNS info
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name']
# WHOIS information
asInfo = ipintellib.ip2asn(ip)
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR
# GeoIP information
geoIP = ipintellib.geo_ip(ip)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP['longitude'] # Calc approx. localtime
latitude = geoIP['latitude']
info = "haxx0r IP : " + ip + "\nuser : "******"\nDNS : " + dnsName + "\n\nAS Number : " + asNum + "\nAS Name : " + asRegisteredCode + "\n\nGeoIP Country : " + countryCode + "\nGeoIP City : " + "\nGeoIP Longitude : " + "%.2f" % longitude + "\nGeoIP Latitude : " + "%.2f" % latitude
# Haxx0r's client stack information
p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22")
if p0fInfo['result'] == True:
p0fStr = "os=" + p0fInfo['genre'] + " hops=" + p0fInfo[
'hops'] + " linktype=" + p0fInfo[
'linktype'] + " up_secs=" + p0fInfo[
'uptime'] + " tos=" + p0fInfo[
'tos'] + " masq=" + p0fInfo[
'masq'] + " fw=" + p0fInfo[
'firewall'] + " NAT=" + p0fInfo[
'nat'] + " realOS=" + p0fInfo['realos']
else:
p0fStr = p0fInfo['errormsg']
# Notify !
alertSubject = "honeypot intrusion! : " + subject
alertContent = info + "\n\np0f : " + p0fStr + "\n\n" + content + "\n\nSent by Kojoney Honeypot\n\n"
print "alert():\nsubject:" + alertSubject + "\ncontent:\n" + alertContent + "\n"
status = mailalert.mailalert(sender, destination, smtpServer,
alertSubject, alertContent, debugLevel)
# Add a record to syslog
a = "Sent alert e-mail, Subject=" + alertSubject + " to " + destination[
0]
syslog.syslog(a)
except Exception, e:
syslog.syslog("kojoney_tail.py : alert() : " + ` e ` + " ip=" + ip)
def blackhole(txnId, sensorId, attacker):
global BlackholedRoutes
global BH_DURATION
try:
# do not blackhole items on whitelist
print "*** kojoney_defend.py : calling hiddenIP() ***"
if kojoney_hiddenip.hiddenIP(attacker) == True:
msg = "kojoney_defend.py : blackhole() : warning : attacker " + attacker + " is on hiddenIP() whitelist and must not be blackholed"
print msg
syslog.syslog(msg)
return None
# blackhole route already exists - issue - should the time be reset to zero ?
#if BlackholedRoutes.has_key(attacker) == True :
# return None
# get hostname of the attacker
dnsInfo = ipintellib.ip2name(attacker)
dnsName = dnsInfo['name']
# inject the blackhole
cmd = "/sbin/ip route add blackhole " + attacker
msg = "Added blackhole (" + BH_DURATION.__str__(
) + "s) on " + time.ctime(
) + " for " + attacker + " " + dnsName.__str__()
#print msg
#syslog.syslog(msg)
# Update Attacker Database
srcIP = attacker
kojoney_attacker_event.generateAttackerEvent(
txnId, srcIP, None, sensorId, "BHOLE", "DEFEND", None,
"Attacker blackholed", None, None, None, None, None)
kojoney_blackhole_idmef.sendBlackholeIDMEF(attacker, "Added blackhole")
pipe = os.popen(cmd, 'r')
tweet = "DEFEND," + msg
#return None # do not Tweet - not interesting to average punter
return tweet
except Exception, e:
msg = "kojoney_defend.py : blackhole() exception caught = " + ` e `
syslog.syslog(msg)
print msg
return None
def ageBlackholedRoutes(bh_duration):
global BlackholedRoutes
try:
agedAttackers = []
now = time.time()
#print now
for attacker in BlackholedRoutes:
blackholeBorn = BlackholedRoutes[attacker]
blackholeAge = now - blackholeBorn
#print " -> blackholeAge for " + attacker + " is " + blackholeAge.__str__() + " seconds"
if blackholeAge > bh_duration:
print attacker + " has now aged out"
agedAttackers.append(attacker)
#del BlackholedRoutes[attacker]
cmd = "/sbin/ip route delete blackhole " + attacker
# get hostname of the attacker
dnsInfo = ipintellib.ip2name(attacker)
dnsName = dnsInfo['name']
msg = "Removed blackhole on " + time.ctime(
) + " for " + attacker + " " + dnsName.__str__()
#print msg
#syslog.syslog(msg)
kojoney_blackhole_idmef.sendBlackholeIDMEF(
attacker, "Removed blackhole")
pipe = os.popen(cmd, 'r')
raw = pipe.read().rstrip("\n")
tweet = "DEFEND," + msg
# Do not Tweet - not very interesting
twitter_funcs.addTweetToQueue(tweet, geoip=True)
for attacker in agedAttackers:
del BlackholedRoutes[attacker]
#print "attacker " + attacker + " removed from list of blackholes"
except Exception, e:
msg = "kojoney_defend.py : ageBlackholedRoutes() exception caught = " + ` e `
syslog.syslog(msg)
print msg
def processURLfoundWget(line):
try:
line = line.strip("\n")
print "processURLfoundWget() : line read : " + line
if line.find("Honeypot wget requests URL") == -1:
return
print "processURLfoundWget() : candidate line read : " + line
fields = line.split()
print fields
url = fields[9]
print "url is " + url
surl = url.replace("http://", "")
surl = surl.split('/')
surl = surl[0]
print "stripped url is " + surl
dnsInfo = ipintellib.ip2name(surl)
ip = dnsInfo['name']
#ip = a.split(">")[1]
#ip = "0.0.0.0" # bug -> extract the IP from here
# fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv','a')
# print >> fpOut,msg
# fpOut.close()
# Download single URL
retrieveSingleURL(url, ip)
except Exception, e:
syslog.syslog(
"kojoney_guru.py : processURLfoundTweet() exception caught = " +
` e ` + " line=" + line)
def checkDoStweet(text, group, victim):
try:
global VERSION
if ("IP" in text.upper() or "TARGET" in text.upper()
or "FIRE" in text.upper()) and ("#ANON" in text.upper()
or "#OP" in text.upper()):
asInfo = ipintellib.ip2asn(victim)
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo[
'registeredCode'] # Short-form e.g. ARCOR
dnsInfo = ipintellib.ip2name(victim)
dnsName = dnsInfo['name']
geoIP = ipintellib.geo_ip(victim)
countryCode = geoIP['countryCode'].__str__()
subject = "Anonymous DDoS Request"
body = "The text in the Tweet below *MAY* indicate that a hactivist-based DDoS attack is being requested against IP " + victim.__str__() + " DNS=" + dnsName.__str__() \
+ " located in BGP AS" + asNum.__str__() + " ISP=" + asRegisteredCode.__str__() + " CC=" + countryCode.__str__()
body = body + " => Tweet="
body = body + '[' + text.replace('\t', " ") + ']'
body = body + " : "
body = body + "This e-mail notification was generated by soclscrpr v" + VERSION + " at " + time.ctime(
)
body = body + " : "
body = body + "For support, contact [email protected]"
kojoney_alert_client.sendAlert(subject, body, True, True)
except Exception, e:
msg = "twitter_streamer.py : checkDoStweet() : exception : " + e.__str__(
) + " : " + text.__str__()
print msg
syslog.syslog(msg)
ip = None
# *****************************************************************************
# Look for 1 x URL i.e. put into JOB Queue for kojoney_anubis to pick out later
# *****************************************************************************
ipUrl = processURLfoundWget(tweet,test)
# Case 1 : Generate Tweets for raw IPs found
# TODO : Need a safety feature to ensure that IPs in range 192.168.1.0/24 are never added to TweetQueue
if len(ips) != 0 :
# Case #1 : Generate Tweets for raw IPs found
for ip in ips:
if ipList.has_key(ip) == False : # not seen this IP before
# Get DNS name
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name'].rstrip('.') + " "
asMsg = ipintellib.prettyAS(ip)
geoIP = ipintellib.geo_ip(ip)
countryCode = geoIP['countryCode']
city = geoIP['city']
if city == "?" :
geoInfo = ""
else:
geoInfo = "," + "city=" + city
gid = gid + 1
#tweet = "GURU," + countryCode + "," + "ip=" + ip + " " + dnsName + asMsg + geoInfo
tweet = "GURU,IP " + ip + " => " + dnsName + asMsg + geoInfo
def processSebekTweet(line):
ftpInfo = {}
line = line.strip("\n")
print "entered processSebekTweet() with line=" + line
try:
filtered = filter_sebek.filterSebek(line)
print "processSebekTweet() : filtered = " + filtered
# missing step is to normalise the sebek line i.e. to remove [BS] etc.
if len(filtered) > 0:
#msg = "Honeypot access:" + filtered
msg = filtered
#print "tweet before compression : " + msg
sendTweet(msg)
# file needs to be touched - is this the daily file ?
fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv', 'a')
print >> fpOut, msg
fpOut.close()
# for sebek lines containing "wget", perform additional analysis on destination URL
if filtered.find("wget") != -1:
print "kojoney_tweet.py : processSebekTweet() : wget found"
url = extract_url.extractURL(filtered) # Normalise URL
o = urlparse(url)
domain = "127.0.0.1"
path = ""
if o.scheme == 'ftp':
ftpInfo = extract_url.extractDomainFTP(url)
domain = ftpInfo['domain']
path = ftpInfo['path']
elif len(url) != 0:
domain = o.netloc
path = o.path
# Tweet additional info if a valid URL was found
if len(url) != 0:
# Get IP from DNS info
dnsInfo = ipintellib.ip2name(domain)
srcIP = dnsInfo['name'].rstrip(
'.') # right-strip the trailing .
# WHOIS
asInfo = ipintellib.ip2asn(srcIP) #
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo[
'registeredCode'] # Short-form e.g.ARCOR
# GeoIP information - faster than WHOIS for looking up Country Code information
geoIP = ipintellib.geo_ip(srcIP)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP[
'longitude'] # Used to calc approx. localtime
#latitude = geoIP['latitude']
msg = "URL_FOUND," + url + "->" + \
"IP=" + srcIP + \
",WHOIS=" + asNum + " (" + asRegisteredCode + ")" + \
",GeoIP=" + countryCode + " " + city + " " + "%.2f" % longitude + "E"
print msg
sendTweet(msg)
# file needs to be touched - see above
fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv', 'a')
print >> fpOut, msg
fpOut.close()
# for sebek lines containing an IP address, perform additional analysis
# only searches for 1 IP address
pat = r'\d+\.\d+\.\d+\.\d+' # locate a number of IP addresses
ips = re.findall(pat, filtered)
print "first IP address found = " + ` ips[0] `
if len(ips) != 0:
ip = ips[0]
# Get DNS name
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name'].rstrip(
'.') # right-strip the trailing .
# WHOIS
asInfo = ipintellib.ip2asn(ip) #
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo[
'registeredCode'] # Short-form e.g.ARCOR
# GeoIP information - faster than WHOIS for looking up Country Code information
geoIP = ipintellib.geo_ip(ip)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP[
'longitude'] # Used to calc approx. localtime
#latitude = geoIP['latitude']
msg = "IP_ADDR_FOUND," + ip + "->" + \
"DNS=" + dnsName + \
",WHOIS=" + asNum + " (" + asRegisteredCode + ")" + \
",GeoIP=" + countryCode + " " + city + " " + "%.2f" % longitude + "E"
print msg
sendTweet(msg)
# file needs to be touched - see above
fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv', 'a')
print >> fpOut, msg
fpOut.close()
else:
#print "tweet filtered out : " + line
pass
except Exception, e:
syslog.syslog(
"kojoney_tweet.py : processSebekTweet() exception caught = " +
` e ` + " line=" + line)
def processMessages(line):
try:
line = line.strip("\n")
#print "processMessages() : line read from /var/log/messages : " + line
if line.find("from 192.168.1."
) != -1: # do not log legitimate local LAN access
return
if line.find("mars sshd") == -1:
return
if line.find("Accepted password for") == -1:
return
print "processMessages() : candidate line read from /var/log/messages : " + line
# Parse
# -----
pat = r'\d+\.\d+\.\d+\.\d+' # locate a number of IP addresses
ips = re.findall(pat, line)
#fields=line.split(" ")
#print fields # locate username
#username = fields[9]
pat = r'password for (\w+)' # locate a number of IP addresses
username = re.findall(pat, line)[0]
print "kojoney_tweet.py : processMessages() : username is " + username
srcIP = ips[0]
#dstIP = "HPOT"
dstIP = "172.31.0.67"
dstPort = "22"
proto = "6" # Tevent = "user="******"kojoney_tweet.py : processMessages() : p0f input is srcIP=" + srcIP + " dstIP=" + dstIP + " dstPort=" + dstPort
p0fInfo = p0fcmd.getP0fInfo(srcIP, "0", dstIP, dstPort)
# 0 = wildcard the srcPort
if p0fInfo['result'] == True: # p0f data is available
os = p0fInfo['genre']
nat = p0fInfo['nat'][0]
hops = p0fInfo['hops']
else:
os = "?"
nat = "?"
hops = "?"
# Get DNS info
dnsInfo = ipintellib.ip2name(srcIP)
dnsName = dnsInfo['name'].rstrip('.') # right-strip the trailing .
# WHOIS
asInfo = ipintellib.ip2asn(srcIP) #
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR
# GeoIP information - faster than WHOIS for looking up Country Code information
geoIP = ipintellib.geo_ip(srcIP)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP['longitude'] # Used to calc approx. localtime
latitude = geoIP['latitude']
msg = "INTRUSION" + \
",user="******",IP=" + srcIP + \
",p0f=" + os + " NAT=" + nat + " hops=" + hops + \
",DNS=" + dnsName + \
",WHOIS=" + asNum + " (" + asRegisteredCode + ")" + \
",GeoIP=" + countryCode + " " + city + " " + "%.2f" % longitude + "E"
print "tweet before compression : " + msg
sendTweet(msg)
except Exception, e:
syslog.syslog(
"kojoney_tweet.py : processMessages() exception caught = " +
` e ` + " line=" + line)
def processHoneyd(line):
flowEvent = {}
try:
#print "\nEntered blackrain_honeyd.processHoneyd()"
#print line
flowEvent['flowType'] = "FLOW_HONEYD_FLOW"
ip = re.findall("\d+\.\d+\.\d+\.\d+", line)
if len(ip) == 0:
#print "No IP addresses found"
return None
#else :
# print "Found IP addresses..."
if not "END" in line:
#print "No END honeyd flow found, so return None"
return None
sIP = re.findall("(\d+\.\d+\.\d+\.\d+)", line)[0]
#print sIP
sP = re.findall("\d+\.\d+\.\d+\.\d+ (\d+) \d+\.\d+\.\d+\.\d+ \d+",
line)[0]
#print sP
dP = re.findall("\d+\.\d+\.\d+\.\d+ \d+ \d+\.\d+\.\d+\.\d+ (\d+)",
line)[0]
#print dP
flowEvent['flowDirection'] = "in"
flowEvent['flowRemoteIP'] = sIP
flowEvent['flowRemotePort'] = sP
flowEvent['flowHpotPort'] = dP
if "icmp" in line:
flowEvent['flowProto'] = "I"
elif "tcp" in line:
flowEvent['flowProto'] = "T"
elif "udp" in line:
flowEvent['flowProto'] = "UDP"
else:
flowEvent[
'flowProto'] = "unknown" # attention : need to test other protocols
#flowEvent['flowOS'] = "none"
# Optional parameters
# ===================
#flowEvent['flowDuration']
#flowEvent['flowPkts']
#flowEvent['flowTflags'] = re.findall("fl=(\d+)",line)[0]
# Number of bytes received by the honeypot
# This does not include the data part of the TCP handshake
flowEvent['flowBytes'] = re.findall("rx=(\d+)", line)[0]
#print flowEvent
# Data enrichment
# ===============
# DNS info
dnsInfo = ipintellib.ip2name(flowEvent['flowRemoteIP'])
flowEvent['flowDNS'] = dnsInfo['name'].rstrip('.')
#print flowEvent['flowDNS']
# GeoIP info
geoIP = ipintellib.geo_ip(flowEvent['flowRemoteIP'])
flowEvent['flowCC'] = geoIP['countryCode']
flowEvent['flowCountry'] = geoIP['countryName']
flowEvent['flowCity'] = geoIP['city']
flowEvent['flowLat'] = "%.3f" % float(geoIP['latitude'])
flowEvent['flowLong'] = "%.3f" % float(geoIP['longitude'])
#print flowEvent['flowCC']
# WHOIS info
asInfo = ipintellib.ip2asn(flowEvent['flowRemoteIP'])
flowEvent['flowASN'] = asInfo['as'] # AS123
flowEvent['flowISP'] = asInfo[
'registeredCode'] # Short-form e.g. LEVEL3
flowEvent['flowRoute'] = asInfo['netblock']
flowEvent['flowRIR'] = asInfo['registry']
#print flowEvent['flowASN']
#print flowEvent
return flowEvent
except Exception, e:
msg = "Exception : " + e.__str__() + " in line=" + line
print msg
syslog.syslog(msg)
return None
def sendToLoggly(sensorId, line):
try:
global txnId
print "sendToLoggly() : line=" + line
# honeytweeter
#shakey = "e25e6042-e490-4910-a246-94cefbdd11b9"
# honeytweeter-json
shakey = "fe39eb54-7c8f-417c-afc8-5b8db98961d3"
sdata = {}
cifdata = {}
# Event Type
fields = line.split(",")
eventType = fields[0]
sdata['eventType'] = fields[0]
# Do not process obsolete eventTypes
if sdata['eventType'] == 'FW_SNORT' or sdata['eventType'] == 'NIDS_SH':
return None
if sdata['eventType'] == "ANALYST" and "TRACEROUTE" in line: # ANALYST traceroutes don't visualise well
return None
if sdata['eventType'] == "ANALYST" and "NMAP" in line: # ANALYST nmaps don't visualise well
return None
#print sdata['eventType']
# Sensor Name
sdata['sensorId'] = sensorId
# IP address
ips = re.findall("\d+\.\d+\.\d+\.\d+", line)
if len(ips) > 0:
sdata['ip'] = ips[0]
# GeoIP
geoIP = ipintellib.geo_ip(sdata['ip'])
cc = geoIP['countryCode']
if cc != "?":
sdata['cc'] = cc.__str__()
sdata['ipcc'] = sdata['ip'].__str__(
) + " - " + sdata['cc'].__str__()
# Reverse DNS : fixme Extract the TLD and create a sdata field for it
dnsInfo = ipintellib.ip2name(sdata['ip'])
dnsName = dnsInfo['name']
if dnsName != "NoDNS":
sdata['rdns'] = dnsName.rstrip('.').lower()
sdata['tld'] = crudeTld.getTLD(sdata['rdns'])
if "baiduspider" in sdata['rdns']:
return None # this is not an interesting event
if sdata['rdns'] == "pointer":
return None
# ASN info
asInfo = ipintellib.ip2asn(sdata['ip'])
asNum = asInfo['as'] # AS123
if asNum == "AS-none":
asNum = "NO_INFO"
else:
asNum = "AS" + asNum
asRegisteredCode = asInfo['registeredCode'].upper() # e.g. GOOGLE
sdata['asn'] = asNum
sdata['isp'] = asRegisteredCode
# Snort SID
if sdata['eventType'] == "SNORT_NIDS" or sdata[
'eventType'] == "NIDS_SH":
sids = re.findall("SID=(\d+)", line)
if len(sids) > 0:
sdata['sid'] = sids[0]
a = re.findall("P(\d+) SID", line)
if len(a) > 0:
sdata['priority'] = a[0]
# removed the P from the priority - due to ELK issues - fixme : make a Blackrain-wide PRIORITY ? for all events ?
# Clamd malware name - e.g. Exploit.Shellcode.X86-Gen-1
# Jun 27 04:47:39 mars kojoney_tweet_engine[3173]: SENDTWEET = <0>CLAMD,Malware Exploit.Shellcode.X86-Gen-1 in flow from 79.5.203.86 ports={s=4592 d=135}
if sdata['eventType'] == "CLAMD":
a = line.split(",Malware ")[1]
a = a.split(" ")[0]
sdata['malware'] = a
sdata['avendor'] = "ClamAV"
# Threat Report
# submitted=Wed Jun 26 02:54:18 2013 cmd=GEO_IP tweet=REPORT,Threat Level for 203.250.135.20 is 41.1, flags={PR SC PS BH AT GA}
if "flags={" in line and sdata['eventType'] == "REPORT":
a = line.split("flags=")[1]
#a = a.rstrip("}")
sdata['flags'] = a
# VirusTotal
# submitted=Fri May 17 07:48:03 2013 cmd=BASIC tweet=ANALYST,AV eb7656dd256eb414abe092eb0f41ea1f.php => Norman=PhpShell.BL 17/46 VT=http://bit.ly/14vyont
# line=ANALYST,AV 06a940dd7824d6a3a6d5b484bb7ef9d5.php => Unseen by VirusTotal
if sdata[
'eventType'] == "ANALYST" and "AV" in line and "Unseen by" not in line:
a = line.split(" => ")[1]
b = a.split(" ")[0] # Symantec=Trojan.Usuge!gen3
sdata['avendor'] = b.split("=")[0]
sdata['malware'] = b.split("=")[1]
b = a.split("VT=")[1]
sdata['virustotal'] = b
# filename = MD5 name + extension (optional)
c = line.split("AV ")[1]
c = c.split(" ")[0]
#print c
if "." in c:
c = c.split(".")[0] # lose filename extension
sdata['md5'] = c.lower()
# Destination port - make this a string so it is not displayed with commas in Kibana
# ----------------------------------------------------------------------------------
# Snort messages
flow = re.findall("\w+\:(\d+) ", line)
if len(flow) > 0:
sdata['port'] = str(flow[0])
# IPLOG - make a string so it is displayed with commas in Kibana
# --------------------------------------------------------------
if sdata['eventType'] == "IPLOG":
ports = re.findall("port (\d+)", line)
if len(ports) > 0:
sdata['port'] = str(ports[0])
# CLAMD - clsniffer
if sdata['eventType'] == "CLAMD":
ports = re.findall("d=(\d+)", line)
if len(ports) > 0:
sdata['port'] = str(ports[0])
# KIPPO
if sdata['eventType'] == "KIPPO":
#sdata['port'] = 2222
sdata['port'] = "22"
sdata['protocol'] = "tcp"
############
# Glastopf #
############
if "WEB_" in sdata['eventType']:
#sdata['port'] = 18080
sdata['port'] = "80"
sdata['protocol'] = "tcp"
line = line.replace(
'|', '/'
) # pipe character is used in Tweets to avoid Twitter generating shortened URLs
url = line.split('req=')[1]
sdata['url'] = url.lower()
#if sdata['url'] == '/' or sdata['url'] == '//' :
# return None
# Botjuicer cracked scripts
if sdata['eventType'] == "ANALYST" and "BOTJUICER" in line:
#if "UNDETERMINED" in line.upper():
# return None
#print "BOTJUICER log found : " + line
a = re.findall("p=(\d+)", line)
if len(a) > 0:
sdata['port'] = str(a[0])
a = re.findall("ch=(#\w+)", line)
if len(a) > 0:
sdata['irc'] = a[0]
# Protocol - generic
if "TCP" in line.upper():
sdata['protocol'] = "tcp"
if "UDP" in line.upper():
sdata['protocol'] = "udp"
if "ICMP" in line.upper():
sdata['protocol'] = "icmp"
# Timestamp
sdata['datetime'] = time.ctime()
# Message
sdata['msg'] = line.split(",")[1:][0]
# Txnid - increment this last, just before submission to Loggly
txnId = txnId + 1
sdata['txnId'] = txnId
# Splunk
#print "\nkey-value pairs for Splunk :-"
#splunkMsg = "'" + time.ctime() + ' ' + sensorId + ' '
#for i in sdata:
# if i == 'datetime' : # ignore
# continue
# if i == 'msg' and "SNORT" in eventType:
# print "Snort event found"
# # sdata[i] = '"' + sdata[i] + '"'
# # sdata[i] = sdata[i].replace('"','')
# splunkMsg = splunkMsg + ' ' + i.__str__() + '=' + '"' + sdata[i].__str__() + '"'
# #print i,sdata[i]
#splunkMsg = splunkMsg + "'"
# Abandoned - has my API key run out ?
#print "Send to SplunkStorm Cloud SIEM => " + splunkMsg
#log.send(splunkMsg , sourcetype='syslog',host=sensorId)
# Loggly - abandoned in favour of splunk
#body = json.dumps(sdata)
#print "\nsendToLoggly() : JSON body = " + body
# Send to Loggly
#insert_url = "http://logs.loggly.com/inputs/" + shakey
#insert_http = httplib2.Http(timeout=10)
##body = line
#resp, content = insert_http.request(insert_url, "POST", body=body, headers={'content-type':'text/plain'})
##print "Loggly : resp : " + resp.__str__()
##print "Loggly : content : " + content.__str__()
##{'status': '200', 'content-length': '18', 'vary': 'Accept-Encoding', 'server': 'TwistedWeb/12.0.0', 'date': 'Tue, 25 Jun 2013 05:55:58 GMT', 'content-type': 'text/html'}
##{"response": "ok"}
#if "ok" in content.__str__() : # bug -> why can't I look at the "response" field in a structured way
# msg = "Sent to Loggly OK : " + body.__str__()
# print msg
#else:
# msg = "Sent to Loggly FAIL : " + body.__str__() + " error = " + content.__str__()
# print msg
# syslog.syslog(msg)
# crude form of rate-limiter
#time.sleep(0.2)
# Return the JSON structure so it can be used by NoSQL type databases
sdata = json.dumps(sdata)
pprint(sdata)
#writeToSyslogFake(sdata)
return sdata
except Exception, e:
msg = "kojoney_loggly.py : sendToLoggly() : exception : " + e.__str__(
) + " line=" + line
print msg
syslog.syslog(msg)
return None
def getIntelStr(ip1, ip2):
ip = "0.0.0.0"
#print "getintelStr() : ip1= " + ip1 + " ip2=" + ip2
try:
if ip1 == ROUTER:
ip = ip2
if ip2 == ROUTER:
ip = ip1
if ip1 == 'ROUTER':
ip = ip2
if ip2 == 'ROUTER':
ip = ip1
if ip1 == 'HPOT':
ip = ip2
if ip2 == 'HPOT':
ip = ip1
if ip1 == HPOT:
ip = ip2
if ip2 == HPOT:
ip = ip1
#print "Called getIntelStr(): IP to be checked is " + ip
if (ip == ROUTER or ip == HPOT or ip == "ROUTER" or ip == "HPOT"):
intelStr = "*,*,*" # * = masked
else:
# Handle 192.168.1.131 used for testing
#if line.find("192.168.1.") != -1 :
#if ip == "0.0.0.0" : # failed to extract IP from a record
# return "intell=0.0.0.0"
# Get DNS info
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name']
# WHOIS : primary information
asInfo = rch_asn_funcs.ip2asn(
ip) # need to add routes=1 to get ASN - timeout bug at moment
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR
countryCode = asInfo['countryCode']
# WHOIS info gathered from "infos" fields
info = asInfo['info']
purpose = asInfo['purpose']
vodafone = asInfo['vodafone']
# GeoIP information
#geoIP = ipintellib.geo_ip(ip) # getting exceptions still !
#countryCode = geoIP['countryCode']
#city = geoIP['city']
#longitude = geoIP['longitude'] # Used to calc approx. localtime
#intelStr = countryCode + ":" + asNum + ":" + asRegisteredCode + ":" + city + ":" + dnsName
intelStr = countryCode + "," + asNum + "," + asRegisteredCode
# ,long=" + "%.2f" % float(longitude)
#print "***** intelStr for " + ip + " is " + intelStr
return intelStr
except Exception, e:
syslog.syslog("kojoney_tail.py : getIntelStr() exception caught = " +
` e ` + "ip=" + ip)
return "intell=exception!"
def sendTweetCLI(sessionid, username, ip, cli):
global TweetClient
global TweetVersion # bump version of format of Tweet changes
print "Entered sendTweetCLI()"
now = time.time()
try:
# sessionid of -1 indicates that we have no AUTH_OK event and so no username - so don't tweet it
if (int(sessionid) < 0):
print "sessionId < 0 -> no previous AUTH_OK event to get username"
return
p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22")
if p0fInfo['result'] == True: # p0f data is available
hops = p0fInfo['hops']
os = p0fInfo['genre']
fw = p0fInfo['firewall']
nat = p0fInfo['nat']
if p0fInfo['genre'] == "Linux":
uptime = p0fInfo['uptime']
bte = now - int(uptime) # boot time (epoch secs)
hops = p0fInfo['hops']
else:
uptimeHours = 0
bte = 0
else: # p0f data not read OK
hops = 0
os = "?"
fw = "?"
nat = "?"
bte = 0
# get current time
timeTuple = time.localtime(now)
nowStr = time.asctime(timeTuple)
# calc haxx0r bootTime
#timeTuple = time.localtime(bte)
#bootTimeStr = time.asctime(timeTuple)
bteh = int(bte / 3600) # bteh = boot time epoch (hours)
# Get DNS info
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name']
# WHOIS information
asInfo = ipintellib.ip2asn(ip)
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR
# GeoIP information
geoIP = ipintellib.geo_ip(ip)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP['longitude'] # Used to calc approx. localtime
# Compact usual returns
if asNum.find("AS-none") != -1:
asNum = "?"
if asRegisteredCode.find("failed") != -1:
asRegisteredCode = "?"
if countryCode.find("None") != -1:
countryCode = "?"
if city.find("None") != -1:
city = "?"
# Construct Tweet
# todo - if os = windoes, do not add bteh
msg = "sid=" + `sessionid` + ":IP=" + ip + ":" + asNum + "(" + asRegisteredCode + ")" + ":"\
+ countryCode + ":" + city + ":" + "%.2f" % longitude + ":" + os + ":" + hops\
+ ":fw=" + fw + ":nat=" + nat + ":bteh=" + `bteh`\
+ ":" + username + "@hpot $ " + cli
# Send the Tweet - max = 147 ?
syslog.syslog("sendTweetCLI(): msg length=" + ` len(msg) ` + " chars")
print "Tweet=" + msg
# syslog every Tweet attempted to be sent
syslog.syslog("sendTweetCLI(): " + msg)
# *** Disable during testing ***
status = TweetClient.PostUpdate(msg)
time.sleep(5) # crude rate-limit ?
except Exception, e:
syslog.syslog("kojoney_tail.py : sendTweetCLI() exception caught = " +
` e ` + " ip=" + ip)
def sendToLoggly(sensorId,line):
try :
global txnId
print "sendToLoggly() : line=" + line
# honeytweeter
#shakey = "e25e6042-e490-4910-a246-94cefbdd11b9"
# honeytweeter-json
shakey = "fe39eb54-7c8f-417c-afc8-5b8db98961d3"
sdata = {}
# Event Type
fields = line.split(",")
eventType = fields[0]
sdata['eventType'] = fields[0]
#print sdata['eventType']
# Sensor Name
sdata['sensorId'] = sensorId
# IP address
ips = re.findall("\d+\.\d+\.\d+\.\d+",line)
if len(ips) > 0 :
sdata['ip'] = ips[0]
# GeoIP
geoIP = ipintellib.geo_ip(sdata['ip'])
cc = geoIP['countryCode']
if cc != "?" :
sdata['cc'] = cc.__str__()
# Reverse DNS
dnsInfo = ipintellib.ip2name(sdata['ip'])
dnsName = dnsInfo['name']
if dnsName != "NoDNS":
sdata['rdns'] = dnsName.rstrip('.')
# ASN info
asInfo = ipintellib.ip2asn(sdata['ip'])
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'].upper() # e.g. GOOGLE
sdata['asn'] = "AS" + asNum
sdata['isp'] = asRegisteredCode
# Snort SID
if sdata['eventType'] == "SNORT_NIDS" :
sids = re.findall("SID=(\d+)",line)
if len(sids) > 0:
sdata['sid'] = sids[0]
a = re.findall("P(\d+) SID",line)
if len(a) > 0 :
sdata['priority'] = "P" + a[0]
# Clamd malware name - e.g. Exploit.Shellcode.X86-Gen-1
# Jun 27 04:47:39 mars kojoney_tweet_engine[3173]: SENDTWEET = <0>CLAMD,Malware Exploit.Shellcode.X86-Gen-1 in flow from 79.5.203.86 ports={s=4592 d=135}
if sdata['eventType'] == "CLAMD" :
a = line.split(",Malware ")[1]
a = a.split(" ")[0]
sdata['malware'] = a
sdata['avendor'] = "ClamAV"
# Threat Report
# submitted=Wed Jun 26 02:54:18 2013 cmd=GEO_IP tweet=REPORT,Threat Level for 203.250.135.20 is 41.1, flags={PR SC PS BH AT GA}
if "flags={" in line and sdata['eventType'] == "REPORT" :
a = line.split("flags=")[1]
#a = a.rstrip("}")
sdata['flags'] = a
# VirusTotal
# submitted=Fri May 17 07:48:03 2013 cmd=BASIC tweet=ANALYST,AV eb7656dd256eb414abe092eb0f41ea1f.php => Norman=PhpShell.BL 17/46 VT=http://bit.ly/14vyont
# line=ANALYST,AV 06a940dd7824d6a3a6d5b484bb7ef9d5.php => Unseen by VirusTotal
if sdata['eventType'] == "ANALYST" and "AV" in line and "Unseen by" not in line :
a = line.split(" => ")[1]
b = a.split(" ")[0] # Symantec=Trojan.Usuge!gen3
sdata['avendor'] = b.split("=")[0]
sdata['malware'] = b.split("=")[1]
b = a.split("VT=")[1]
sdata['virustotal'] = b
# filename = MD5 name + extension (optional)
c = line.split("AV ")[1]
c = c.split(" ")[0]
#print c
if "." in c:
c = c.split(".")[0] # lose filename extension
sdata['md5'] = c.lower()
# Destination port
# ----------------
# Snort messages
flow = re.findall("\w+\:(\d+) ",line)
if len(flow) > 0 :
sdata['port'] = int(flow[0])
# IPLOG
if sdata['eventType'] == "IPLOG" :
ports = re.findall("port (\d+)",line)
if len(ports) > 0 :
sdata['port'] = int(ports[0])
# CLAMD - clsniffer
if sdata['eventType'] == "CLAMD" :
ports = re.findall("d=(\d+)",line)
if len(ports) > 0 :
sdata['port'] = int(ports[0])
# KIPPO
if sdata['eventType'] == "KIPPO" :
sdata['port'] = 2222
sdata['protocol'] = "tcp"
# Glastopf
if "WEB" in sdata['eventType'] :
sdata['port'] = 18080
sdata['protocol'] = "tcp"
# Botjuicer cracked scripts
if sdata['eventType'] == "ANALYST" and "BOTJUICER" in line :
#if "UNDETERMINED" in line.upper():
# return None
#print "BOTJUICER log found : " + line
a = re.findall("p=(\d+)",line)
if len(a) > 0 :
sdata['port'] = int(a[0])
a = re.findall("ch=(#\w+)",line)
if len(a) > 0 :
sdata['irc'] = a[0]
# Protocol - generic
if "TCP" in line.upper():
sdata['protocol'] = "tcp"
if "UDP" in line.upper():
sdata['protocol'] = "udp"
if "ICMP" in line.upper():
sdata['protocol'] = "icmp"
# Timestamp
sdata['datetime'] = time.ctime()
# Message
sdata['msg'] = line
# Txnid - increment this last, just before submission to Loggly
txnId = txnId + 1
sdata['txnId'] = txnId
body = json.dumps(sdata)
print "sendToLoggly() : JSON body = " + body
# Send to Loggly
insert_url = "http://logs.loggly.com/inputs/" + shakey
insert_http = httplib2.Http(timeout=10)
#body = line
resp, content = insert_http.request(insert_url, "POST", body=body, headers={'content-type':'text/plain'})
#print "Loggly : resp : " + resp.__str__()
#print "Loggly : content : " + content.__str__()
#{'status': '200', 'content-length': '18', 'vary': 'Accept-Encoding', 'server': 'TwistedWeb/12.0.0', 'date': 'Tue, 25 Jun 2013 05:55:58 GMT', 'content-type': 'text/html'}
#{"response": "ok"}
if "ok" in content.__str__() : # bug -> why can't I look at the "response" field in a structured way
msg = "Sent to Loggly OK : " + body.__str__()
print msg
else:
msg = "Sent to Loggly FAIL : " + body.__str__() + " error = " + content.__str__()
print msg
syslog.syslog(msg)
# crude form of rate-limiter
time.sleep(0.5)
except Exception,e:
msg = "kojoney_loggly.py : sendToLoggly() : exception : " + e.__str__() + " line=" + line
print msg
syslog.syslog(msg)
def test():
print "\n\n\n\n"
print "=========================="
print "p0f logfile to Reputation "
print "=========================="
print "Library version : " + ipintellib.getVersion()
print "Hard-coded tests"
print "----------------"
# IP to DNS name
ipStr = "217.41.27.169" # My IP
dnsInfo = ipintellib.ip2name(ipStr) # resolve get DNS name
print ipStr + " resolves to " + dnsInfo['name']
# DNS name to IP
ipStr = "www.openbsd.org"
dnsInfo = ipintellib.ip2name(ipStr) # resolve get DNS name
print ipStr + " resolves to " + dnsInfo['name']
# input file : file of IPs (hand-crafted)
fpIn = open(r'/home/var/log/p0f.log', 'r')
# output file
fpOut0 = open(r'/home/var/log/p0f2reputation.out.csv', 'w')
# Main loop
#logs = {}
asInfo = {}
dnsInfo = {}
# Logging file
#logging.basicConfig(level=logging.INFO,filename='botclientsyslog.py.log')
#logging.info('SYSLOG log file analysis started')
lineCounter = -1
while True:
lineCounter = lineCounter + 1
###############################
# if lineCounter >= 3000:
# sys.exit(0)
###############################
line1 = fpIn.readline()
if not line1: break
if line1[0] == '#': # ignore leading # comment
#print "Ignore # comment"
continue
if line1.find(
"distance") == -1: # all valid lines have 'distance' in them"
continue
print "----------------------------------------------"
fields = line1.split(' ')
print fields
numb = len(fields)
print "numb fields is " + ` numb `
os = fields[8]
# re-write this using regular expressions and make a function so that it can be reused
# !!!!!!!!!!!!!!!!!!!!!!!
if line1.find("Linux") != -1:
if numb == 17:
ipStr, port = fields[12].split(
':') # dest IP and dest TCP port number
elif numb == 24:
ipStr, port = fields[17].split(
':') # dest IP and dest TCP port number
elif line1.find("Windows") != -1:
if numb == 19:
ipStr, port = fields[14].split(
':') # dest IP and dest TCP port number
elif line1.find("Novell") != -1:
continue # ignore Novell for moment
#if numb == 19:
# ipStr,port = fields[13].split(':') # dest IP and dest TCP port number
else:
sys.exit("Failed to parse")
print "line=" + ` lineCounter ` + ": *** haxx0r : ip=" + ipStr + " port=" + port + " os=" + os
dnsInfo = ipintellib.ip2name(ipStr) # resolve get DNS name
asInfo = ipintellib.ip2asn(ipStr) # get AS information from WHOIS whob
reputation = ipintellib.ip2reputation(
ipStr) # does slow things down...
geoIP = ipintellib.geo_ip(ipStr)
result = ipStr + "," + dnsInfo['name'].strip('.') + \
"," + asInfo['as'] + ",ASowner=" + asInfo['registeredCode'] + ",ASnetblock=" + asInfo['netblock'] + ",ASregistry=" + asInfo['registry'] + \
",geoIPcountry=" + geoIP['countryCode'] + ",geoIPcity=" + geoIP['city'] + ",geoIPlat=" + "%.2f" % geoIP['latitude'] + ",geoIPlong=" + "%.2f" % geoIP['longitude'] + \
",zen.spamhaus=" + reputation['zen.spamhaus.org'] + \
",dnsbl.ahbl.org=" + reputation['dnsbl.ahbl.org'] + \
",bl.deadbeef.com=" + reputation['bl.deadbeef.com'] + \
",bogons.cymru.com=" + reputation['bogons.cymru.com'] + \
",zombie.dnsbl.sorbs.net=" + reputation['zombie.dnsbl.sorbs.net'] + \
",bl.spamcop.net=" + reputation['bl.spamcop.net'] + \
",dul.dnsbl.sorbs.net=" + reputation['dul.dnsbl.sorbs.net'] + \
",l2.apews.org=" + reputation['l2.apews.org'] + \
",virus.rbl.msrbl.net=" + reputation['virus.rbl.msrbl.net'] + \
",phishing.rbl.msrbl.net=" + reputation['phishing.rbl.msrbl.net'] + \
",images.rbl.msrbl.net=" + reputation['images.rbl.msrbl.net'] + \
",spam.rbl.msrbl.net=" + reputation['spam.rbl.msrbl.net']
print result
print >> fpOut0, ipStr + "," + "os" + reputation['zen.spamhaus.org']
def processURLfoundWget(line,test=False):
try :
line=line.strip("\n")
print "processURLfoundWget() : line read : " + line
if line.find("GURU,") != -1 :
return # Do not process GURU records
if line.find("http://bit.ly") != -1:
return # probably a Snort IDS xreference
protocol = None
if line.find("http://") != -1 :
protocol = "http://"
if line.find("https://") != -1 :
protocol = "https://"
if line.find("tftp://") != -1 :
protocol = "tftp://"
if line.find("ftp://") != -1 :
protocol = "ftp://"
# Common case for Kippo is wget www.dfsdsd.com/fd.jpg...
if line.find("wget") != -1 and protocol == None :
protocol = "http://"
line = "http://" + line.split("wget ")[1]
#print line
if protocol == None:
return
#print "processURLfoundWget() : candidate line read : " + line
#print "protocol is " + protocol
url = line[line.find(protocol):]
# Experimental - glastopf seems to add ? or ??
url=url.rstrip("?")
url=url.rstrip("?")
print "url is " + url.__str__()
#t = t[:t.find(" ")]
# Write the URL to a file to be picked up by kojoney_analyst for download etc
a = url.split("->")[0]
# remove any data after the URL like "shellcode=plain" etc.
a = a.split(" ")
a = a[0]
msg = "kojoney_guru.py : found url " + a
print msg
# **********************************************************
# Add URL Task to Job Queue for processing by kojoney_anubis
# **********************************************************
addUrlToFile(a,test)
fields = url.split('/')
#print fields
domain = fields[2] # e.g. openbsd.org
#print "url is " + url
#surl = url.replace("http://","")
#surl = surl.split('/')
#surl = surl[0]
#print "stripped url is " + surl
dnsInfo = ipintellib.ip2name(domain)
ip = dnsInfo['name']
if ip == "127.0.0.1" or ip == "0.0.0.0" :
return None
return ip
except Exception,e:
syslog.syslog("kojoney_guru.py : processURLfoundTweet() exception caught = " + `e` + " line=" + line)
def processGlastopf(txnId, sensorId, line):
asMsg = ""
try:
print "-------------\nprocessGlastopf() : line read is " + line
dstIP = "192.168.1.62" # bug - not portable
request = "No URL found"
apacheCLF = "None"
logEntry = line # keep a copy since we are going to modify line
# Normalise Mail attacks
line = line.replace("ail attack found", "ail attack from")
# Looks like a duplication bug so ignore the 'Unknown' variant
if "Unknown POST attack" in line:
return None
# ATTACK : Successful attack
if line.find("attack from") != -1 or line.find(
"Mail attack found from") != -1:
print "WebApp attack detected : " + line
apacheCLF = fakeApacheCLF(line) # fake an Apache CLF record
fields = line.split(" - ")
#print fields
msg = ' '.join(fields[3:])
#msg = ' '.join(fields[3:]).rstrip("?") # does glastopf add trailing ? / ??
msg = "WEB_X," + msg
msg = msg.replace(" with request: ", " req=")
#msg = msg.replace("found from","from") # normalise MAIL versus "other" attacks
msg = msg.replace(
"Mail", "Webmail") # normalise MAIL versus "other" attacks
msg = msg.replace(
"/", "|"
) # fool Twitter into not using t.co URL shortening and ensure users cannot click on malware links
a = re.findall("(GET|POST|Unknown POST|Mail) attack from", line)
if len(a) > 0:
attackType = "WebApp " + a[0].upper(
) + "-based RFI attack" # upper() used to convert Mail to MAIL
else:
attackType = "Unknown WebApp attack"
ips = re.findall("from (\d+\.\d+\.\d+\.\d+)", line)
if len(ips) > 0:
srcIP = ips[0]
if "request" in line:
request = line.split("request: ")[1]
print "Successful WebApp attack : AttackType=" + attackType + " srcIP=" + srcIP + " URLrequest=" + request
if "unknown" in line.lower():
completion = "failed"
else:
completion = "succeeded"
kojoney_glastopf_idmef.sendWebAppIDMEF(attackType, request, "http",
"18080", completion, srcIP,
dstIP, apacheCLF, srcIP,
logEntry)
kojoney_attacker_event.generateAttackerEvent(
txnId, srcIP, None, sensorId, "ATTACKING", "GLASTOPF", None,
attackType, None, None, None, apacheCLF, None)
return msg
# File retrieved
if line.find("successfully opened") != -1:
#apacheCLF=fakeApacheCLF(line) # fake an Apache CLF record
fields = line.split(" - ")
#print fields
msg = ' '.join(fields[3:])
msg = "WEB_OPEN," + msg
urls = re.findall("(\S+) successfully opened", line)
if len(urls) > 0:
url = urls[0]
domain = kojoney_idmef_common.extractDomain(url)
if domain != None:
dnsInfo = ipintellib.ip2name(domain)
dstIP = dnsInfo['name']
else:
dstIP = "0.0.0.0" # error
# IDMEF the honeypot to drop-site flow
kojoney_glastopf_idmef.sendWebAppURLIDMEF(
"WebApp URL opened", url, "http", "192.168.1.62", dstIP,
"80", "succeeded", "None", dstIP, logEntry)
msg = msg.replace(
"/", "|"
) # fool Twitter into not using t.co URL shortening and ensure users cannot click on malware links
return msg
# Googledorks data written to mySQL database - not interesting enough to Tweet
#if line.find("written into local database") != -1 :
# fields = line.split(" - ")
# #print fields
# msg = ' '.join(fields[3:])
# msg = "WEB_GOOGLEDORK," + msg
# return msg
# File saved to disk
if line.find("written to disk") != -1 and line.find("File ") != -1:
fields = line.split(" - ")
#print fields
msg = ' '.join(fields[3:])
msg = msg.replace("File", "Previously unseen PHP malware file")
msg = "WEB_WRITE," + msg
fileMD5 = fields[3].split(" ")[1]
kojoney_glastopf_idmef.sendWebAppFile(
"File retrieved from remote server", fileMD5, logEntry)
return msg
# Scan - i.e. unsuccessful attack
if line.find("No attack found") != -1:
apacheCLF = fakeApacheCLF(line) # fake an Apache CLF record
#print "scan"
ip = kojoney_funcs.findFirstIP(line)
if ip != None:
#print "IP found = " + ip
# WHOIS information
#asInfo = rch_asn_funcs.ip2asn(ip)
asInfo = ipintellib.ip2asn(ip)
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo[
'registeredCode'] # Short-form e.g.ARCOR
asMsg = asRegisteredCode + " (" + asNum + ")"
#print asMsg
fields = line.split(" - ")
#print fields
msg = ' '.join(fields[3:])
if line.find("http"
) != -1: # attacker is trying to test if I am a proxy
msg = msg.replace("No attack found from",
"WEB_PRX,Request from")
attackType = "WebApp proxy scan"
else: # LOC = local
msg = msg.replace("No attack found from", "WEB_SCN,Scan from")
attackType = "WebApp scan"
msg = msg.replace(" with request: ", " req=")
msg = msg.rstrip()
# remove any trailing characters
#msg = msg + " ISP=" + asMsg
kojoney_afterglow.visWebScan(msg)
msg = msg.replace(
"/", "|"
) # fool Twitter into not using t.co URL shortening and ensure users cannot click on malware links
ips = re.findall("from (\d+\.\d+\.\d+\.\d+)", line)
if len(ips) > 0:
srcIP = ips[0]
request = line.split("request: ")[1]
#print attackType + " : srcIP : " + srcIP + " request : " + request
# go for two events ?
if attackType == "WebApp proxy scan":
kojoney_glastopf_idmef.sendWebAppURLIDMEF(
"WebApp proxy scan Request", request, "http", srcIP,
"192.168.1.62", "18080", "failed", apacheCLF, srcIP,
logEntry) # inbound
kojoney_attacker_event.generateAttackerEvent(
txnId, srcIP, None, sensorId, "SCANNING", "GLASTOPF",
None, "WebApp proxy scan Request", None, None, None,
apacheCLF, None)
domain = kojoney_idmef_common.extractDomain(request)
if domain != None:
dnsInfo = ipintellib.ip2name(domain)
dstIP = dnsInfo['name']
else:
dstIP = "0.0.0.0" # error
kojoney_glastopf_idmef.sendWebAppURLIDMEF(
"WebApp proxy scan Retrieval", request, "http",
"192.168.1.62", dstIP, "80", "failed", apacheCLF,
dstIP, logEntry) # outbound
kojoney_attacker_event.generateAttackerEvent(
txnId, srcIP, None, sensorId, "SCANNING", "GLASTOPF",
None, "WebApp proxy scan Retrieval", None, None, None,
apacheCLF, None)
else:
kojoney_glastopf_idmef.sendWebAppURLIDMEF(
attackType, request, "http", srcIP, "192.168.1.62",
"18080", "failed", apacheCLF, srcIP,
logEntry) # inbound
kojoney_attacker_event.generateAttackerEvent(
txnId, srcIP, None, sensorId, "SCANNING", "GLASTOPF",
None, "WebApp scan", None, None, None, apacheCLF, None)
return msg
return None
except Exception, e:
msg = "kojoney_glastopf_parse.py : processGlastopf() : " + ` e ` + " line=" + line
print msg
syslog.syslog(msg)
def processSSH(line):
global PreviousIPs
global CLInum
global SessionId
global Username
if Username == None:
Username = "******"
try:
#print "line read from file\n:" + line
# haxx0r guessed password OK
# Use this one to perform the ipintellib() functions
if line.find("authenticated with password") != -1:
SessionId = SessionId + 1
fields = line.split(',')
#print fields
a = fields[2].split()
ip = a[0].rstrip(']')
Username = a[1]
#print "a is " + `a`
# Get DNS info
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name']
# WHOIS information
asInfo = ipintellib.ip2asn(ip)
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR
# GeoIP information
geoIP = ipintellib.geo_ip(ip)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP['longitude'] # Used to calc approx. localtime
msg = "------------------------------------------------------------------------------------------"
makeMsg(0, ip, msg)
msg = "EXPL:authOK," + Username + "," + countryCode + "," + asNum + "," + asRegisteredCode + "," + city + "," + dnsName + ",long=" + "%.2f" % float(
longitude)
makeMsg(0, ip, msg)
# Write Security Visualisation Data to secViz file
print "Calling writeSecViz3()"
writeSecViz3(ip, Username)
# Compute localtime based on longitude
locTime = calcLocalTime(longitude)
msg = "INTEL:haxx0r localTime(est) is " + locTime
makeMsg(0, ip, msg)
# Haxx0r's client stack information - using p0f 3.0.0, uptime is in seconds not hours
try:
p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22")
if p0fInfo['result'] == True: # p0f data is available
hops = p0fInfo['hops']
os = p0fInfo['genre']
fw = p0fInfo['firewall']
nat = p0fInfo['nat']
if p0fInfo['genre'] == "Linux":
uptime = p0fInfo['uptime'] # measured in secs
else:
uptime = 0
else: # p0f data is not available
hops = 0
os = "?"
fw = "?"
nat = "?"
uptime = 0
#p0fStr = "os=" + os + " hops=" + hops + " lt=" + p0fInfo['linktype'] + " up=" + uptime + " tos=" + p0fInfo['tos'] + " masq=" + p0fInfo['masq'] + " fw=" + p0fInfo['firewall'] + " NAT=" + p0fInfo['nat'] + " realOS=" + p0fInfo['realos']
p0fStr = "os=" + os + " hops=" + hops + " up=" + uptime + " fw=" + fw + " NAT=" + nat
# Compute haxx0r's PC bootTime based on p0f uptime (seconds)
bootTime = calcBootTime(uptime)
bootmsg = "haxx0r PC bootTime(est,UTC) is " + bootTime[
'timeStr']
# p0f info
msg = "p0f,INTEL:" + p0fStr # INT = Intelligence
makeMsg(0, ip, msg)
# haxx0r boot time
msg = "INTEL:bootTime," + bootmsg
makeMsg(0, ip, msg)
# Has this IP been seen before ?
if PreviousIPs.has_key(ip): # IP has been seen before
msg = "INTEL:IP was last seen " + PreviousIPs[ip][
'lastVisitStr'] + " with a bootTime of " + PreviousIPs[
ip]['timeStr']
makeMsg(0, ip, msg)
# Has this PC stack been seen before ?
# i.e. are bootTimes within 30 minutes of each other ?
if abs(
float(PreviousIPs[ip]['epoch']) -
float(bootTime['epoch'])) < 1800.0:
msg = "INTEL:p0f uptime (secs) indicates repeat visit from haxx0r's PC from this ip"
makeMsg(0, ip, msg)
else:
PreviousIPs[
ip] = bootTime # set info to be haxx0r boottime
msg = "INTEL:first time visit from " + ip + ", user " + Username
makeMsg(0, ip, msg)
except Exception, e:
syslog.syslog(
"Exception in processSSH() p0f for authOK section of code "
+ ` e ` + " ip=" + ip)
# Extract haxxor's SSH client
elif line.find("pty request") != -1:
fields = line.split(',')
#print fields
a = fields[2].split()
#print "a is " + `a`
ip = a[0].rstrip(']')
pty = a[3]
#print "pty is " + `pty`
msg = "INTEL:client=" + pty
makeMsg(0, ip, msg)
def processAmunExploit(line):
try:
#print "processAmunExploit() : line read is " + line
# Ignore if do not find shellcode in the exploit
#if line.find("Shellcode: None") != -1 :
# return None
# First IP is attacker IP
pat = '(\d+\.\d+\.\d+\.\d+)\:(\d+)'
ips = re.findall(pat, line)
#print "srcIP = " + srcIP[0].__str__()
if len(ips) > 0:
#print ips
srcIP = ips[0][0]
srcPort = ips[0][1]
dstIP = ips[1][0]
dstPort = ips[1][1]
#print srcIP + " : " + srcPort
#print dstIP + " : " + dstPort
vulns = re.findall('(\w+) Vulnerability', line)
#print vulns
# This is actually the callback URL
info = re.findall('Vulnerability\: (\S+)', line)
#vulns[0] = vulns[0].rstrip(')')
info[0] = info[0].rstrip(')')
#print info
shellcode = re.findall('Shellcode: (\w+)', line)
#print shellcode
# IDMEF #1 : send info about the attacker to Prelude SIEM
kojoney_amun_idmef.exploitIDMEF(srcIP, srcPort, dstIP, dstPort,
vulns[0], info[0], shellcode[0], line)
# IDMEF #2 : send info about the drop-site to Prelude SIEM
domain = kojoney_idmef_common.extractDomain(info[0])
if domain != None:
a = re.findall("(\d+\.\d+\.\d+\.\d+)", domain)
if len(a) > 0:
dstIP = domain
else:
dnsInfo = ipintellib.ip2name(domain)
dstIP = dnsInfo['name']
#else:
# dstIP = "0.0.0.0" # error
kojoney_amun_idmef.exploitDropsiteIDMEF("192.168.1.66", dstIP,
srcIP, info[0], line)
# Ignore if do not find any shellcode in the exploit
if line.find("Shellcode: None") != -1:
return None
# some filenames have these brackets around the last part for some reason
line = line.replace("['", "")
line = line.replace("']", "")
line = line.replace("(", "")
line = line.replace(")", "")
fields = line.split(",")
#print fields
msg = fields[1].split(" ")
#print msg
msg = ' '.join(msg[5:])
msg = srcIP[0].__str__() + "->" + msg
msg = msg.replace("192.168.1.66", "AMUN")
msg = msg.replace("Vulnerability:", "vulnerability")
msg = msg.replace("Shellcode: ", "shellcode=")
msg = "AMUN_X," + msg
return msg
except Exception, e:
error = "kojoney_amun_parse.py : processAmunExploit() : " + ` e ` + " line=" + line
print error
syslog.syslog(error)
def hiddenIP(ipAddress, lanAllowed=False):
try:
#debug = True
debug = False
ipList = []
HPOTS = [
"192.168.1.50", "192.168.1.60", "192.168.1.61", "192.168.1.62",
"192.168.1.63", "192.168.1.64", "192.168.1.65", "192.168.1.66",
"192.168.1.68", "192.168.1.69"
]
if isinstance(ipAddress, str):
ipList.append(ipAddress)
#print "hiddenIP() : passed a STRING type"
else:
ipList = ipAddress
#print "hiddenIP() : passed a LIST type"
#print "kojoney_hiddenip.py : hiddenIP() : ipList = " + ipList.__str__()
for ip in ipList:
# check that IP is valid IP
pat = "(\d+\.\d+\.\d+\.\d+)"
a = re.findall(pat, ip)
if len(a) <= 0:
msg = "kojoney_hiddenip.py : hiddenIP() : error : " + ip + " is not a valid IP address, ipAddress=" + ipAddress.__str__(
)
print msg
#syslog.syslog(msg) # THIS IS STILL A BUG
#msg = "type of ipAddress is " + type(ipAddress)
#syslog.syslog(msg)
return False # !!! ideally a third state would exist for errors
#else:
# print "hiddenIP() : lanAllowed : " + lanAllowed.__str__() + ", IP : " + ip.__str__()
if ip == "127.0.0.1":
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE since ip is 127.0.0.1")
return True
# Google DNS
if lanAllowed == False and ip == "8.8.8.8":
return True
# BlackRain itself
if lanAllowed == False and ip == "192.168.1.67":
return True
# mail
if lanAllowed == False and ip == "192.168.1.70":
return True
# BRX
if lanAllowed == False and ip == "192.168.1.90":
return True
# Node called 'prelude' running VirtualBox
if lanAllowed == False and ip == "192.168.1.73":
return True
# Prelude SIEM
if lanAllowed == False and ip == "192.168.1.74":
return True
# Spade/Snort external sensor
if lanAllowed == False and ip == "192.168.1.76":
return True
# DSL node
if lanAllowed == False and ip == "192.168.1.254":
return True
# Cloud node itself
if lanAllowed == False and ip == "192.168.1.93":
return True
if ip in HPOTS:
continue
#print "slow processing code entered for IP = " + ip.__str__()
# Sometimes LAN IPs are allowed, othertimes not - so cater for both cases
#if lanAllowed == False and ip.find("192.168.1.") != -1 :
# if debug == True :
# syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE at LAN check")
# print ip + " should be hidden"
# return True
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name'].rstrip('.')
#print "kojoney_hiddenip.py : " + dnsName.__str__()
# ignore traffic from Internet test hosts
#if ip.find("173.203.89.") != -1 : # GSOC slice server roy.doesntexist.org
# if debug == True :
# syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = slice server doesntexist.org")
# return True
# ignore traffic from Google DNS
if ip.find("8.8.8.8") != -1:
#if debug == True :
#syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = slice server doesntexist.org")
return True
# ignore traffic associated with Twitter
if dnsName.find("twttr.com") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , DNS = twttr.com")
return True
# ignore traffic associated with updating Slackware patches
if dnsName.find("slackware.mirrors") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , DNS = slackware.mirrors")
return
# ignore traffic associated with malware analysis by Anubis at Univ California Santa Barbara
if dnsName.find("anubis.cs.ucsb.edu") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , DNS = anubis.cs.ucsb.edu")
return
# ignore traffic associated with honeyd updating it's site
if dnsName.find("honeyd.provos.org") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , DNS = honeyd.provos.org")
return True
# ignore traffic associated with Team Cymru
if dnsName.find("cymru.com") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , DNS = cymru.com")
return True
# Filter based on AS number / name - do last since expensive lookup
asInfo = ipintellib.ip2asn(ip)
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'].upper() # e.g. GOOGLE
#print "kojoney_hiddenip.py : " + asRegisteredCode.__str__()
if asRegisteredCode.find("GOOGLE") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , AS = GOOGLE")
return True
if asRegisteredCode.find("TWITTER-NETWORK") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , AS = TWITTER-NETWORK")
return True
if asRegisteredCode.find("FACEBOOK") != -1:
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," +
lanAllowed.__str__() +
") -> TRUE , AS = FACEBOOK")
return True
# ip is an attacker
if debug == True:
syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() +
") -> FALSE, ip is an attacker")
#return False
return False
except Exception, e:
msg = "kojoney_hiddenip.py : hiddenIP() : exception " + ` e ` + " ip=" + ipAddress.__str__(
)
print msg
syslog.syslog(msg)
return None
def sendFiledownloadIDMEF(url,fullFilename,filename,fileMD5,completion,logEntry):
try:
if fileMD5 != None:
cymruHash = kojoney_cymru_hash.cymruHash(fileMD5)
print "cymruHash : " + cymruHash
else:
cymruHash = "0"
# Extract IP from URL
domain = kojoney_idmef_common.extractDomain(url)
if domain != None :
a = re.findall("(\d+\.\d+\.\d+\.\d+)",domain)
if len(a) > 0 :
dstIP = domain
else:
dnsInfo = ipintellib.ip2name(domain)
dstIP = dnsInfo['name']
else:
dstIP = "0.0.0.0"
print "kojoney_anubis_idmef.py : sendFiledownloadIDMEF() : dstIP = " + dstIP.__str__()
# Create a new Prelude client
client = PreludeEasy.ClientEasy("blackrain")
client.Start()
# Create the IDMEF message
idmef = PreludeEasy.IDMEF()
# Sensor
fieldsSet = kojoney_idmef_common.setIDMEFcommon(idmef,"Analyst Honeypot","02DEBE56",None,dstIP,None,dstIP,logEntry)
# Classification
if cymruHash == "0" :
cymruHash = "None"
idmef.Set("alert.classification.text","File identified by URL-snarf method")
idmef.Set("alert.assessment.impact.severity", "low")
else:
idmef.Set("alert.classification.text","Malware file identified by URL-snarf method" + " contains malware")
idmef.Set("alert.assessment.impact.severity", "high")
idmef.Set("alert.target(0).node.address(0).address", dstIP)
idmef.Set("alert.target(0).file(0).name", filename)
idmef.Set("alert.target(0).file(0).path", fullFilename)
# Assessment
idmef.Set("alert.assessment.impact.completion", completion)
if completion == "succeeded" :
idmef.Set("alert.assessment.impact.description", "File downloaded OK")
else:
idmef.Set("alert.assessment.impact.description", "File download failed")
idmef.Set("alert.assessment.impact.type", "file")
if url == '/':
url = "None"
idmef.Set("alert.target(0).service.web_service.url", url)
# Additional Data
fieldsOffset = fieldsSet
#print "fieldsOffset = " + fieldsOffset.__str__()
idmef.Set("alert.additional_data(" + fieldsOffset.__str__() + ").type", "string")
idmef.Set("alert.additional_data(" + fieldsOffset.__str__() + ").meaning", "Team Cymru MHA % of AV triggered")
idmef.Set("alert.additional_data(" + fieldsOffset.__str__() + ").data", cymruHash)
fieldsOffset = fieldsOffset + 1
if fileMD5 != None:
idmef.Set("alert.additional_data(" + fieldsOffset.__str__() + ").type", "string")
idmef.Set("alert.additional_data(" + fieldsOffset.__str__() + ").meaning", "MD5")
idmef.Set("alert.additional_data(" + fieldsOffset.__str__() + ").data", fileMD5)
client.SendIDMEF(idmef)
return
except Exception,e:
msg = "kojoney_anubis_idmef.py : sendFiledownloadIDMEF() : exception : " + e.__str__()
print msg
syslog.syslog(msg)
return
def printAttackerOIDs():
# need to add logic to fail if the shelf file does not exist
database = shelve.open('/home/var/log/attacker_shelf.dat')
if len(database) == 0: # contains no entries
sys.exit("Failed to locate shelf file")
print " "
print "Attackers"
print "---------"
print "NUM_IP : " + database['NUM_IP'].__str__()
print "NUM_SLASH_24 : " + database['NUM_SLASH_24'].__str__()
print "NUM_CC : " + database['NUM_CC'].__str__()
print "NUM_CITY : " + database['NUM_CITY'].__str__()
print "NUM_ASN : " + database['NUM_ASN'].__str__()
print " "
print "Malware Captured"
print "----------------"
print "AMUN_FILES_BIN : " + database['AMUN_FILES_BIN'].__str__()
print "AMUN_FILES_HEX : " + database['AMUN_FILES_HEX'].__str__()
print "NEPENTHES_FILES : " + database['NEPENTHES_FILES'].__str__()
print "KIPPO_FILES : " + database['KIPPO_FILES'].__str__()
print "GLASTOPF_FILES_GET : " + database['GLASTOPF_FILES_GET'].__str__()
print "GLASTOPF_FILES_POST : " + database['GLASTOPF_FILES_POST'].__str__()
print "ANALYST_FILES_ALL : " + database['ANALYST_FILES_ALL'].__str__()
print "ANALYST_FILES_EXE : " + database['ANALYST_FILES_EXE'].__str__()
print "ANALYST_FILES_PHP : " + database['ANALYST_FILES_PHP'].__str__()
print "ANALYST_FILES_TXT : " + database['ANALYST_FILES_TXT'].__str__()
print "ANALYST_FILES_GIF : " + database['ANALYST_FILES_GIF'].__str__()
print "ANALYST_FILES_JPG : " + database['ANALYST_FILES_JPG'].__str__()
print "ANALYST_FILES_PNG : " + database['ANALYST_FILES_PNG'].__str__()
print "ANALYST_FILES_TGZ : " + database['ANALYST_FILES_TGZ'].__str__()
print " "
print "Attackers"
outfile = "blackrain_attackers-report.csv"
fp = open(outfile, 'w')
for ip in database['IP']:
asInfo = ipintellib.ip2asn(ip) #
asNum = asInfo['as'] # AS123
asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR
geoIP = ipintellib.geo_ip(ip)
countryCode = geoIP['countryCode']
city = geoIP['city']
longitude = geoIP['longitude'] # Used to calc approx. localtime
latitude = geoIP['latitude']
dnsInfo = ipintellib.ip2name(ip)
dnsName = dnsInfo['name'].rstrip('.')
msg = ip + "," + "AS" + asNum.__str__(
) + "," + countryCode + "," + dnsName.__str__(
) + "," + asRegisteredCode.__str__() + "," + countryCode.__str__(
) + "," + city.__str__()
print msg
print >> fp, msg
fp.close()
