def writeSecViz1(ip, username, countryCode, commandStr): try: p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22") if p0fInfo['result'] == True: p0fStr = "os:" + p0fInfo['genre'] + ":hops=" + p0fInfo['hops'] else: p0fStr = p0fInfo['errormsg'] #raise Exception # test code #print i # force exception for testing asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR fpOut = open(r'/var/log/kojoney_tail_secviz_cmds.csv', 'a') msg = ip + ":" + asNum + ":" + asRegisteredCode + ":" + p0fStr + "," + username + "," + '"' + commandStr + '"' + "," + countryCode print "writeSecViz1():" + msg print >> fpOut, msg fpOut.close() except Exception, e: syslog.syslog("kojoney_tail.py : writeSecViz1() exception caught = " + ` e ` + " ip=" + ip)
def traceroute(sensorId,ip,hops,fpDebug=None): asn = [] asnUnique = [] print "----------------------" if fpDebug != None: print >> fpDebug,"-------------" if hops != 0: #cmdLine = "/usr/local/bin/traceroute -p 53 -A -m " + hops.__str__() + " " + ip cmdLine = "/usr/local/bin/paris-traceroute -n -m " + hops.__str__() + " " + ip else: #cmdLine = "/usr/local/bin/traceroute -p 53 -A " + ip cmdLine = "/usr/local/bin/traceroute -n " + ip print cmdLine pipe = os.popen(cmdLine,'r') if pipe == None : syslog.syslog("getP0fInfo() os.popen() returned None for " + cmdLine) raw = pipe.read().strip() if fpDebug != None: print >> fpDebug,raw print "raw = " + raw.__str__() #asn = re.findall("AS(\d+)",raw) ipList = re.findall("(\d+\.\d+\.\d+\.\d+)",raw) print "traceroute() : list of IPs = " + ipList.__str__() for ip in ipList[3:] : a = ipintellib.ip2asn(ip) #asn.append(a['as'] + ":" + a['registeredCode']) asn.append(a['registeredCode']) print "asn = " + asn.__str__() #sys.exit() # Pre-pend sensorId to start of AS Path asnUnique.append(sensorId) asnUnique = asnUnique + asn # Create list of unique AS numbers by filtering out duplicates # asnUnique.append("AS" + asn[0]) #for i in range(0,len(asn)-1): # if asn[i+1] != asn[i] : # if asn[i+1] != "28513" and asn[i+1] != "8151": # IP is private IP ? # asnUnique.append("AS" + asn[i+1]) # Append the countryCode to the target IP geoIP = ipintellib.geo_ip(ip) countryCode = geoIP['countryCode'] city = geoIP['city'] #asnUnique.append(countryCode.__str__() + ":" + ip + ":" + city.__str__()) asnUnique.append(countryCode.__str__() + ":" + ip) # Eliminate duplicates asnUnique = eliminateASpathDuplicates(asnUnique) print asnUnique if fpDebug != None: print >> fpDebug,"traceroute(out) : " + asnUnique.__str__() return asnUnique
def alert(subject, ip, username, content): smtpServer = 'smtp.btconnect.com' sender = '*****@*****.**' destination = ['*****@*****.**'] debugLevel = False try: # Get DNS info dnsInfo = ipintellib.ip2name(ip) dnsName = dnsInfo['name'] # WHOIS information asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR # GeoIP information geoIP = ipintellib.geo_ip(ip) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP['longitude'] # Calc approx. localtime latitude = geoIP['latitude'] info = "haxx0r IP : " + ip + "\nuser : "******"\nDNS : " + dnsName + "\n\nAS Number : " + asNum + "\nAS Name : " + asRegisteredCode + "\n\nGeoIP Country : " + countryCode + "\nGeoIP City : " + "\nGeoIP Longitude : " + "%.2f" % longitude + "\nGeoIP Latitude : " + "%.2f" % latitude # Haxx0r's client stack information p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22") if p0fInfo['result'] == True: p0fStr = "os=" + p0fInfo['genre'] + " hops=" + p0fInfo[ 'hops'] + " linktype=" + p0fInfo[ 'linktype'] + " up_secs=" + p0fInfo[ 'uptime'] + " tos=" + p0fInfo[ 'tos'] + " masq=" + p0fInfo[ 'masq'] + " fw=" + p0fInfo[ 'firewall'] + " NAT=" + p0fInfo[ 'nat'] + " realOS=" + p0fInfo['realos'] else: p0fStr = p0fInfo['errormsg'] # Notify ! alertSubject = "honeypot intrusion! : " + subject alertContent = info + "\n\np0f : " + p0fStr + "\n\n" + content + "\n\nSent by Kojoney Honeypot\n\n" print "alert():\nsubject:" + alertSubject + "\ncontent:\n" + alertContent + "\n" status = mailalert.mailalert(sender, destination, smtpServer, alertSubject, alertContent, debugLevel) # Add a record to syslog a = "Sent alert e-mail, Subject=" + alertSubject + " to " + destination[ 0] syslog.syslog(a) except Exception, e: syslog.syslog("kojoney_tail.py : alert() : " + ` e ` + " ip=" + ip)
def prettyAS(ip): asMsg = "" if ip != None: #print "prettyAS() : IP found = " + ip # WHOIS information asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR asMsg = asRegisteredCode + " (" + asNum + ")" #print "AS info : " + asMsg return asMsg
def writeSecViz3(ip, username): #print "entered writeSecViz3()" now = time.time() try: p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22") if p0fInfo['result'] == True: # p0f data is available hops = p0fInfo['hops'] os = p0fInfo['genre'] fw = p0fInfo['firewall'] nat = p0fInfo['nat'] if p0fInfo['genre'] == "Linux": uptime = p0fInfo['uptime'] bte = now - int(uptime) hops = p0fInfo['hops'] else: uptimeHours = 0 bte = 0 else: # p0f data not read OK hops = 0 os = "?" fw = "?" nat = "?" bte = 0 # get current time timeTuple = time.localtime(now) nowStr = time.asctime(timeTuple) # calc haxx0r bootTime timeTuple = time.localtime(bte) bootTimeStr = time.asctime(timeTuple) bootTimeEpochHours = int(bte / 3600) #print "bootTimeEpochHours:" + `bootTimeEpochHours` asInfo = ipintellib.ip2asn(ip) #asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR msg = ip + "," + `bootTimeEpochHours` + "," + ip + ":" + Username + "," + "os=" + os + ",hops=" + hops + ","\ + asRegisteredCode + ",now=" + nowStr + "," + `now` + ",bootTime=" + bootTimeStr + "," + `bte` + ",fw=" + fw + ",nat=" + nat print "WriteSecViz3() = " + msg fpOut = open(r'/var/log/kojoney_tail_secviz3_uptime.csv', 'a') print >> fpOut, msg fpOut.close() except Exception, e: syslog.syslog("kojoney_tail.py : writeSecViz3() exception caught = " + ` e ` + " ip=" + ip)
def checkDoStweet(text, group, victim): try: global VERSION if ("IP" in text.upper() or "TARGET" in text.upper() or "FIRE" in text.upper()) and ("#ANON" in text.upper() or "#OP" in text.upper()): asInfo = ipintellib.ip2asn(victim) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo[ 'registeredCode'] # Short-form e.g. ARCOR dnsInfo = ipintellib.ip2name(victim) dnsName = dnsInfo['name'] geoIP = ipintellib.geo_ip(victim) countryCode = geoIP['countryCode'].__str__() subject = "Anonymous DDoS Request" body = "The text in the Tweet below *MAY* indicate that a hactivist-based DDoS attack is being requested against IP " + victim.__str__() + " DNS=" + dnsName.__str__() \ + " located in BGP AS" + asNum.__str__() + " ISP=" + asRegisteredCode.__str__() + " CC=" + countryCode.__str__() body = body + " => Tweet=" body = body + '[' + text.replace('\t', " ") + ']' body = body + " : " body = body + "This e-mail notification was generated by soclscrpr v" + VERSION + " at " + time.ctime( ) body = body + " : " body = body + "For support, contact [email protected]" kojoney_alert_client.sendAlert(subject, body, True, True) except Exception, e: msg = "twitter_streamer.py : checkDoStweet() : exception : " + e.__str__( ) + " : " + text.__str__() print msg syslog.syslog(msg)
def processSSH(line): global PreviousIPs global CLInum global SessionId global Username if Username == None: Username = "******" try: #print "line read from file\n:" + line # haxx0r guessed password OK # Use this one to perform the ipintellib() functions if line.find("authenticated with password") != -1: SessionId = SessionId + 1 fields = line.split(',') #print fields a = fields[2].split() ip = a[0].rstrip(']') Username = a[1] #print "a is " + `a` # Get DNS info dnsInfo = ipintellib.ip2name(ip) dnsName = dnsInfo['name'] # WHOIS information asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR # GeoIP information geoIP = ipintellib.geo_ip(ip) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP['longitude'] # Used to calc approx. localtime msg = "------------------------------------------------------------------------------------------" makeMsg(0, ip, msg) msg = "EXPL:authOK," + Username + "," + countryCode + "," + asNum + "," + asRegisteredCode + "," + city + "," + dnsName + ",long=" + "%.2f" % float( longitude) makeMsg(0, ip, msg) # Write Security Visualisation Data to secViz file print "Calling writeSecViz3()" writeSecViz3(ip, Username) # Compute localtime based on longitude locTime = calcLocalTime(longitude) msg = "INTEL:haxx0r localTime(est) is " + locTime makeMsg(0, ip, msg) # Haxx0r's client stack information - using p0f 3.0.0, uptime is in seconds not hours try: p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22") if p0fInfo['result'] == True: # p0f data is available hops = p0fInfo['hops'] os = p0fInfo['genre'] fw = p0fInfo['firewall'] nat = p0fInfo['nat'] if p0fInfo['genre'] == "Linux": uptime = p0fInfo['uptime'] # measured in secs else: uptime = 0 else: # p0f data is not available hops = 0 os = "?" fw = "?" nat = "?" uptime = 0 #p0fStr = "os=" + os + " hops=" + hops + " lt=" + p0fInfo['linktype'] + " up=" + uptime + " tos=" + p0fInfo['tos'] + " masq=" + p0fInfo['masq'] + " fw=" + p0fInfo['firewall'] + " NAT=" + p0fInfo['nat'] + " realOS=" + p0fInfo['realos'] p0fStr = "os=" + os + " hops=" + hops + " up=" + uptime + " fw=" + fw + " NAT=" + nat # Compute haxx0r's PC bootTime based on p0f uptime (seconds) bootTime = calcBootTime(uptime) bootmsg = "haxx0r PC bootTime(est,UTC) is " + bootTime[ 'timeStr'] # p0f info msg = "p0f,INTEL:" + p0fStr # INT = Intelligence makeMsg(0, ip, msg) # haxx0r boot time msg = "INTEL:bootTime," + bootmsg makeMsg(0, ip, msg) # Has this IP been seen before ? if PreviousIPs.has_key(ip): # IP has been seen before msg = "INTEL:IP was last seen " + PreviousIPs[ip][ 'lastVisitStr'] + " with a bootTime of " + PreviousIPs[ ip]['timeStr'] makeMsg(0, ip, msg) # Has this PC stack been seen before ? # i.e. are bootTimes within 30 minutes of each other ? if abs( float(PreviousIPs[ip]['epoch']) - float(bootTime['epoch'])) < 1800.0: msg = "INTEL:p0f uptime (secs) indicates repeat visit from haxx0r's PC from this ip" makeMsg(0, ip, msg) else: PreviousIPs[ ip] = bootTime # set info to be haxx0r boottime msg = "INTEL:first time visit from " + ip + ", user " + Username makeMsg(0, ip, msg) except Exception, e: syslog.syslog( "Exception in processSSH() p0f for authOK section of code " + ` e ` + " ip=" + ip) # Extract haxxor's SSH client elif line.find("pty request") != -1: fields = line.split(',') #print fields a = fields[2].split() #print "a is " + `a` ip = a[0].rstrip(']') pty = a[3] #print "pty is " + `pty` msg = "INTEL:client=" + pty makeMsg(0, ip, msg)
def sendToLoggly(sensorId, line): try: global txnId print "sendToLoggly() : line=" + line # honeytweeter #shakey = "e25e6042-e490-4910-a246-94cefbdd11b9" # honeytweeter-json shakey = "fe39eb54-7c8f-417c-afc8-5b8db98961d3" sdata = {} cifdata = {} # Event Type fields = line.split(",") eventType = fields[0] sdata['eventType'] = fields[0] # Do not process obsolete eventTypes if sdata['eventType'] == 'FW_SNORT' or sdata['eventType'] == 'NIDS_SH': return None if sdata['eventType'] == "ANALYST" and "TRACEROUTE" in line: # ANALYST traceroutes don't visualise well return None if sdata['eventType'] == "ANALYST" and "NMAP" in line: # ANALYST nmaps don't visualise well return None #print sdata['eventType'] # Sensor Name sdata['sensorId'] = sensorId # IP address ips = re.findall("\d+\.\d+\.\d+\.\d+", line) if len(ips) > 0: sdata['ip'] = ips[0] # GeoIP geoIP = ipintellib.geo_ip(sdata['ip']) cc = geoIP['countryCode'] if cc != "?": sdata['cc'] = cc.__str__() sdata['ipcc'] = sdata['ip'].__str__( ) + " - " + sdata['cc'].__str__() # Reverse DNS : fixme Extract the TLD and create a sdata field for it dnsInfo = ipintellib.ip2name(sdata['ip']) dnsName = dnsInfo['name'] if dnsName != "NoDNS": sdata['rdns'] = dnsName.rstrip('.').lower() sdata['tld'] = crudeTld.getTLD(sdata['rdns']) if "baiduspider" in sdata['rdns']: return None # this is not an interesting event if sdata['rdns'] == "pointer": return None # ASN info asInfo = ipintellib.ip2asn(sdata['ip']) asNum = asInfo['as'] # AS123 if asNum == "AS-none": asNum = "NO_INFO" else: asNum = "AS" + asNum asRegisteredCode = asInfo['registeredCode'].upper() # e.g. GOOGLE sdata['asn'] = asNum sdata['isp'] = asRegisteredCode # Snort SID if sdata['eventType'] == "SNORT_NIDS" or sdata[ 'eventType'] == "NIDS_SH": sids = re.findall("SID=(\d+)", line) if len(sids) > 0: sdata['sid'] = sids[0] a = re.findall("P(\d+) SID", line) if len(a) > 0: sdata['priority'] = a[0] # removed the P from the priority - due to ELK issues - fixme : make a Blackrain-wide PRIORITY ? for all events ? # Clamd malware name - e.g. Exploit.Shellcode.X86-Gen-1 # Jun 27 04:47:39 mars kojoney_tweet_engine[3173]: SENDTWEET = <0>CLAMD,Malware Exploit.Shellcode.X86-Gen-1 in flow from 79.5.203.86 ports={s=4592 d=135} if sdata['eventType'] == "CLAMD": a = line.split(",Malware ")[1] a = a.split(" ")[0] sdata['malware'] = a sdata['avendor'] = "ClamAV" # Threat Report # submitted=Wed Jun 26 02:54:18 2013 cmd=GEO_IP tweet=REPORT,Threat Level for 203.250.135.20 is 41.1, flags={PR SC PS BH AT GA} if "flags={" in line and sdata['eventType'] == "REPORT": a = line.split("flags=")[1] #a = a.rstrip("}") sdata['flags'] = a # VirusTotal # submitted=Fri May 17 07:48:03 2013 cmd=BASIC tweet=ANALYST,AV eb7656dd256eb414abe092eb0f41ea1f.php => Norman=PhpShell.BL 17/46 VT=http://bit.ly/14vyont # line=ANALYST,AV 06a940dd7824d6a3a6d5b484bb7ef9d5.php => Unseen by VirusTotal if sdata[ 'eventType'] == "ANALYST" and "AV" in line and "Unseen by" not in line: a = line.split(" => ")[1] b = a.split(" ")[0] # Symantec=Trojan.Usuge!gen3 sdata['avendor'] = b.split("=")[0] sdata['malware'] = b.split("=")[1] b = a.split("VT=")[1] sdata['virustotal'] = b # filename = MD5 name + extension (optional) c = line.split("AV ")[1] c = c.split(" ")[0] #print c if "." in c: c = c.split(".")[0] # lose filename extension sdata['md5'] = c.lower() # Destination port - make this a string so it is not displayed with commas in Kibana # ---------------------------------------------------------------------------------- # Snort messages flow = re.findall("\w+\:(\d+) ", line) if len(flow) > 0: sdata['port'] = str(flow[0]) # IPLOG - make a string so it is displayed with commas in Kibana # -------------------------------------------------------------- if sdata['eventType'] == "IPLOG": ports = re.findall("port (\d+)", line) if len(ports) > 0: sdata['port'] = str(ports[0]) # CLAMD - clsniffer if sdata['eventType'] == "CLAMD": ports = re.findall("d=(\d+)", line) if len(ports) > 0: sdata['port'] = str(ports[0]) # KIPPO if sdata['eventType'] == "KIPPO": #sdata['port'] = 2222 sdata['port'] = "22" sdata['protocol'] = "tcp" ############ # Glastopf # ############ if "WEB_" in sdata['eventType']: #sdata['port'] = 18080 sdata['port'] = "80" sdata['protocol'] = "tcp" line = line.replace( '|', '/' ) # pipe character is used in Tweets to avoid Twitter generating shortened URLs url = line.split('req=')[1] sdata['url'] = url.lower() #if sdata['url'] == '/' or sdata['url'] == '//' : # return None # Botjuicer cracked scripts if sdata['eventType'] == "ANALYST" and "BOTJUICER" in line: #if "UNDETERMINED" in line.upper(): # return None #print "BOTJUICER log found : " + line a = re.findall("p=(\d+)", line) if len(a) > 0: sdata['port'] = str(a[0]) a = re.findall("ch=(#\w+)", line) if len(a) > 0: sdata['irc'] = a[0] # Protocol - generic if "TCP" in line.upper(): sdata['protocol'] = "tcp" if "UDP" in line.upper(): sdata['protocol'] = "udp" if "ICMP" in line.upper(): sdata['protocol'] = "icmp" # Timestamp sdata['datetime'] = time.ctime() # Message sdata['msg'] = line.split(",")[1:][0] # Txnid - increment this last, just before submission to Loggly txnId = txnId + 1 sdata['txnId'] = txnId # Splunk #print "\nkey-value pairs for Splunk :-" #splunkMsg = "'" + time.ctime() + ' ' + sensorId + ' ' #for i in sdata: # if i == 'datetime' : # ignore # continue # if i == 'msg' and "SNORT" in eventType: # print "Snort event found" # # sdata[i] = '"' + sdata[i] + '"' # # sdata[i] = sdata[i].replace('"','') # splunkMsg = splunkMsg + ' ' + i.__str__() + '=' + '"' + sdata[i].__str__() + '"' # #print i,sdata[i] #splunkMsg = splunkMsg + "'" # Abandoned - has my API key run out ? #print "Send to SplunkStorm Cloud SIEM => " + splunkMsg #log.send(splunkMsg , sourcetype='syslog',host=sensorId) # Loggly - abandoned in favour of splunk #body = json.dumps(sdata) #print "\nsendToLoggly() : JSON body = " + body # Send to Loggly #insert_url = "http://logs.loggly.com/inputs/" + shakey #insert_http = httplib2.Http(timeout=10) ##body = line #resp, content = insert_http.request(insert_url, "POST", body=body, headers={'content-type':'text/plain'}) ##print "Loggly : resp : " + resp.__str__() ##print "Loggly : content : " + content.__str__() ##{'status': '200', 'content-length': '18', 'vary': 'Accept-Encoding', 'server': 'TwistedWeb/12.0.0', 'date': 'Tue, 25 Jun 2013 05:55:58 GMT', 'content-type': 'text/html'} ##{"response": "ok"} #if "ok" in content.__str__() : # bug -> why can't I look at the "response" field in a structured way # msg = "Sent to Loggly OK : " + body.__str__() # print msg #else: # msg = "Sent to Loggly FAIL : " + body.__str__() + " error = " + content.__str__() # print msg # syslog.syslog(msg) # crude form of rate-limiter #time.sleep(0.2) # Return the JSON structure so it can be used by NoSQL type databases sdata = json.dumps(sdata) pprint(sdata) #writeToSyslogFake(sdata) return sdata except Exception, e: msg = "kojoney_loggly.py : sendToLoggly() : exception : " + e.__str__( ) + " line=" + line print msg syslog.syslog(msg) return None
def processGlastopf(txnId, sensorId, line): asMsg = "" try: print "-------------\nprocessGlastopf() : line read is " + line dstIP = "192.168.1.62" # bug - not portable request = "No URL found" apacheCLF = "None" logEntry = line # keep a copy since we are going to modify line # Normalise Mail attacks line = line.replace("ail attack found", "ail attack from") # Looks like a duplication bug so ignore the 'Unknown' variant if "Unknown POST attack" in line: return None # ATTACK : Successful attack if line.find("attack from") != -1 or line.find( "Mail attack found from") != -1: print "WebApp attack detected : " + line apacheCLF = fakeApacheCLF(line) # fake an Apache CLF record fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) #msg = ' '.join(fields[3:]).rstrip("?") # does glastopf add trailing ? / ?? msg = "WEB_X," + msg msg = msg.replace(" with request: ", " req=") #msg = msg.replace("found from","from") # normalise MAIL versus "other" attacks msg = msg.replace( "Mail", "Webmail") # normalise MAIL versus "other" attacks msg = msg.replace( "/", "|" ) # fool Twitter into not using t.co URL shortening and ensure users cannot click on malware links a = re.findall("(GET|POST|Unknown POST|Mail) attack from", line) if len(a) > 0: attackType = "WebApp " + a[0].upper( ) + "-based RFI attack" # upper() used to convert Mail to MAIL else: attackType = "Unknown WebApp attack" ips = re.findall("from (\d+\.\d+\.\d+\.\d+)", line) if len(ips) > 0: srcIP = ips[0] if "request" in line: request = line.split("request: ")[1] print "Successful WebApp attack : AttackType=" + attackType + " srcIP=" + srcIP + " URLrequest=" + request if "unknown" in line.lower(): completion = "failed" else: completion = "succeeded" kojoney_glastopf_idmef.sendWebAppIDMEF(attackType, request, "http", "18080", completion, srcIP, dstIP, apacheCLF, srcIP, logEntry) kojoney_attacker_event.generateAttackerEvent( txnId, srcIP, None, sensorId, "ATTACKING", "GLASTOPF", None, attackType, None, None, None, apacheCLF, None) return msg # File retrieved if line.find("successfully opened") != -1: #apacheCLF=fakeApacheCLF(line) # fake an Apache CLF record fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) msg = "WEB_OPEN," + msg urls = re.findall("(\S+) successfully opened", line) if len(urls) > 0: url = urls[0] domain = kojoney_idmef_common.extractDomain(url) if domain != None: dnsInfo = ipintellib.ip2name(domain) dstIP = dnsInfo['name'] else: dstIP = "0.0.0.0" # error # IDMEF the honeypot to drop-site flow kojoney_glastopf_idmef.sendWebAppURLIDMEF( "WebApp URL opened", url, "http", "192.168.1.62", dstIP, "80", "succeeded", "None", dstIP, logEntry) msg = msg.replace( "/", "|" ) # fool Twitter into not using t.co URL shortening and ensure users cannot click on malware links return msg # Googledorks data written to mySQL database - not interesting enough to Tweet #if line.find("written into local database") != -1 : # fields = line.split(" - ") # #print fields # msg = ' '.join(fields[3:]) # msg = "WEB_GOOGLEDORK," + msg # return msg # File saved to disk if line.find("written to disk") != -1 and line.find("File ") != -1: fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) msg = msg.replace("File", "Previously unseen PHP malware file") msg = "WEB_WRITE," + msg fileMD5 = fields[3].split(" ")[1] kojoney_glastopf_idmef.sendWebAppFile( "File retrieved from remote server", fileMD5, logEntry) return msg # Scan - i.e. unsuccessful attack if line.find("No attack found") != -1: apacheCLF = fakeApacheCLF(line) # fake an Apache CLF record #print "scan" ip = kojoney_funcs.findFirstIP(line) if ip != None: #print "IP found = " + ip # WHOIS information #asInfo = rch_asn_funcs.ip2asn(ip) asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo[ 'registeredCode'] # Short-form e.g.ARCOR asMsg = asRegisteredCode + " (" + asNum + ")" #print asMsg fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) if line.find("http" ) != -1: # attacker is trying to test if I am a proxy msg = msg.replace("No attack found from", "WEB_PRX,Request from") attackType = "WebApp proxy scan" else: # LOC = local msg = msg.replace("No attack found from", "WEB_SCN,Scan from") attackType = "WebApp scan" msg = msg.replace(" with request: ", " req=") msg = msg.rstrip() # remove any trailing characters #msg = msg + " ISP=" + asMsg kojoney_afterglow.visWebScan(msg) msg = msg.replace( "/", "|" ) # fool Twitter into not using t.co URL shortening and ensure users cannot click on malware links ips = re.findall("from (\d+\.\d+\.\d+\.\d+)", line) if len(ips) > 0: srcIP = ips[0] request = line.split("request: ")[1] #print attackType + " : srcIP : " + srcIP + " request : " + request # go for two events ? if attackType == "WebApp proxy scan": kojoney_glastopf_idmef.sendWebAppURLIDMEF( "WebApp proxy scan Request", request, "http", srcIP, "192.168.1.62", "18080", "failed", apacheCLF, srcIP, logEntry) # inbound kojoney_attacker_event.generateAttackerEvent( txnId, srcIP, None, sensorId, "SCANNING", "GLASTOPF", None, "WebApp proxy scan Request", None, None, None, apacheCLF, None) domain = kojoney_idmef_common.extractDomain(request) if domain != None: dnsInfo = ipintellib.ip2name(domain) dstIP = dnsInfo['name'] else: dstIP = "0.0.0.0" # error kojoney_glastopf_idmef.sendWebAppURLIDMEF( "WebApp proxy scan Retrieval", request, "http", "192.168.1.62", dstIP, "80", "failed", apacheCLF, dstIP, logEntry) # outbound kojoney_attacker_event.generateAttackerEvent( txnId, srcIP, None, sensorId, "SCANNING", "GLASTOPF", None, "WebApp proxy scan Retrieval", None, None, None, apacheCLF, None) else: kojoney_glastopf_idmef.sendWebAppURLIDMEF( attackType, request, "http", srcIP, "192.168.1.62", "18080", "failed", apacheCLF, srcIP, logEntry) # inbound kojoney_attacker_event.generateAttackerEvent( txnId, srcIP, None, sensorId, "SCANNING", "GLASTOPF", None, "WebApp scan", None, None, None, apacheCLF, None) return msg return None except Exception, e: msg = "kojoney_glastopf_parse.py : processGlastopf() : " + ` e ` + " line=" + line print msg syslog.syslog(msg)
def processSebekTweet(line): ftpInfo = {} line = line.strip("\n") print "entered processSebekTweet() with line=" + line try: filtered = filter_sebek.filterSebek(line) print "processSebekTweet() : filtered = " + filtered # missing step is to normalise the sebek line i.e. to remove [BS] etc. if len(filtered) > 0: #msg = "Honeypot access:" + filtered msg = filtered #print "tweet before compression : " + msg sendTweet(msg) # file needs to be touched - is this the daily file ? fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv', 'a') print >> fpOut, msg fpOut.close() # for sebek lines containing "wget", perform additional analysis on destination URL if filtered.find("wget") != -1: print "kojoney_tweet.py : processSebekTweet() : wget found" url = extract_url.extractURL(filtered) # Normalise URL o = urlparse(url) domain = "127.0.0.1" path = "" if o.scheme == 'ftp': ftpInfo = extract_url.extractDomainFTP(url) domain = ftpInfo['domain'] path = ftpInfo['path'] elif len(url) != 0: domain = o.netloc path = o.path # Tweet additional info if a valid URL was found if len(url) != 0: # Get IP from DNS info dnsInfo = ipintellib.ip2name(domain) srcIP = dnsInfo['name'].rstrip( '.') # right-strip the trailing . # WHOIS asInfo = ipintellib.ip2asn(srcIP) # asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo[ 'registeredCode'] # Short-form e.g.ARCOR # GeoIP information - faster than WHOIS for looking up Country Code information geoIP = ipintellib.geo_ip(srcIP) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP[ 'longitude'] # Used to calc approx. localtime #latitude = geoIP['latitude'] msg = "URL_FOUND," + url + "->" + \ "IP=" + srcIP + \ ",WHOIS=" + asNum + " (" + asRegisteredCode + ")" + \ ",GeoIP=" + countryCode + " " + city + " " + "%.2f" % longitude + "E" print msg sendTweet(msg) # file needs to be touched - see above fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv', 'a') print >> fpOut, msg fpOut.close() # for sebek lines containing an IP address, perform additional analysis # only searches for 1 IP address pat = r'\d+\.\d+\.\d+\.\d+' # locate a number of IP addresses ips = re.findall(pat, filtered) print "first IP address found = " + ` ips[0] ` if len(ips) != 0: ip = ips[0] # Get DNS name dnsInfo = ipintellib.ip2name(ip) dnsName = dnsInfo['name'].rstrip( '.') # right-strip the trailing . # WHOIS asInfo = ipintellib.ip2asn(ip) # asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo[ 'registeredCode'] # Short-form e.g.ARCOR # GeoIP information - faster than WHOIS for looking up Country Code information geoIP = ipintellib.geo_ip(ip) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP[ 'longitude'] # Used to calc approx. localtime #latitude = geoIP['latitude'] msg = "IP_ADDR_FOUND," + ip + "->" + \ "DNS=" + dnsName + \ ",WHOIS=" + asNum + " (" + asRegisteredCode + ")" + \ ",GeoIP=" + countryCode + " " + city + " " + "%.2f" % longitude + "E" print msg sendTweet(msg) # file needs to be touched - see above fpOut = open(r'/home/var/log/kojoney_tail_tweets.csv', 'a') print >> fpOut, msg fpOut.close() else: #print "tweet filtered out : " + line pass except Exception, e: syslog.syslog( "kojoney_tweet.py : processSebekTweet() exception caught = " + ` e ` + " line=" + line)
def processMessages(line): try: line = line.strip("\n") #print "processMessages() : line read from /var/log/messages : " + line if line.find("from 192.168.1." ) != -1: # do not log legitimate local LAN access return if line.find("mars sshd") == -1: return if line.find("Accepted password for") == -1: return print "processMessages() : candidate line read from /var/log/messages : " + line # Parse # ----- pat = r'\d+\.\d+\.\d+\.\d+' # locate a number of IP addresses ips = re.findall(pat, line) #fields=line.split(" ") #print fields # locate username #username = fields[9] pat = r'password for (\w+)' # locate a number of IP addresses username = re.findall(pat, line)[0] print "kojoney_tweet.py : processMessages() : username is " + username srcIP = ips[0] #dstIP = "HPOT" dstIP = "172.31.0.67" dstPort = "22" proto = "6" # Tevent = "user="******"kojoney_tweet.py : processMessages() : p0f input is srcIP=" + srcIP + " dstIP=" + dstIP + " dstPort=" + dstPort p0fInfo = p0fcmd.getP0fInfo(srcIP, "0", dstIP, dstPort) # 0 = wildcard the srcPort if p0fInfo['result'] == True: # p0f data is available os = p0fInfo['genre'] nat = p0fInfo['nat'][0] hops = p0fInfo['hops'] else: os = "?" nat = "?" hops = "?" # Get DNS info dnsInfo = ipintellib.ip2name(srcIP) dnsName = dnsInfo['name'].rstrip('.') # right-strip the trailing . # WHOIS asInfo = ipintellib.ip2asn(srcIP) # asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR # GeoIP information - faster than WHOIS for looking up Country Code information geoIP = ipintellib.geo_ip(srcIP) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP['longitude'] # Used to calc approx. localtime latitude = geoIP['latitude'] msg = "INTRUSION" + \ ",user="******",IP=" + srcIP + \ ",p0f=" + os + " NAT=" + nat + " hops=" + hops + \ ",DNS=" + dnsName + \ ",WHOIS=" + asNum + " (" + asRegisteredCode + ")" + \ ",GeoIP=" + countryCode + " " + city + " " + "%.2f" % longitude + "E" print "tweet before compression : " + msg sendTweet(msg) except Exception, e: syslog.syslog( "kojoney_tweet.py : processMessages() exception caught = " + ` e ` + " line=" + line)
def hiddenIP(ipAddress, lanAllowed=False): try: #debug = True debug = False ipList = [] HPOTS = [ "192.168.1.50", "192.168.1.60", "192.168.1.61", "192.168.1.62", "192.168.1.63", "192.168.1.64", "192.168.1.65", "192.168.1.66", "192.168.1.68", "192.168.1.69" ] if isinstance(ipAddress, str): ipList.append(ipAddress) #print "hiddenIP() : passed a STRING type" else: ipList = ipAddress #print "hiddenIP() : passed a LIST type" #print "kojoney_hiddenip.py : hiddenIP() : ipList = " + ipList.__str__() for ip in ipList: # check that IP is valid IP pat = "(\d+\.\d+\.\d+\.\d+)" a = re.findall(pat, ip) if len(a) <= 0: msg = "kojoney_hiddenip.py : hiddenIP() : error : " + ip + " is not a valid IP address, ipAddress=" + ipAddress.__str__( ) print msg #syslog.syslog(msg) # THIS IS STILL A BUG #msg = "type of ipAddress is " + type(ipAddress) #syslog.syslog(msg) return False # !!! ideally a third state would exist for errors #else: # print "hiddenIP() : lanAllowed : " + lanAllowed.__str__() + ", IP : " + ip.__str__() if ip == "127.0.0.1": if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE since ip is 127.0.0.1") return True # Google DNS if lanAllowed == False and ip == "8.8.8.8": return True # BlackRain itself if lanAllowed == False and ip == "192.168.1.67": return True # mail if lanAllowed == False and ip == "192.168.1.70": return True # BRX if lanAllowed == False and ip == "192.168.1.90": return True # Node called 'prelude' running VirtualBox if lanAllowed == False and ip == "192.168.1.73": return True # Prelude SIEM if lanAllowed == False and ip == "192.168.1.74": return True # Spade/Snort external sensor if lanAllowed == False and ip == "192.168.1.76": return True # DSL node if lanAllowed == False and ip == "192.168.1.254": return True # Cloud node itself if lanAllowed == False and ip == "192.168.1.93": return True if ip in HPOTS: continue #print "slow processing code entered for IP = " + ip.__str__() # Sometimes LAN IPs are allowed, othertimes not - so cater for both cases #if lanAllowed == False and ip.find("192.168.1.") != -1 : # if debug == True : # syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE at LAN check") # print ip + " should be hidden" # return True dnsInfo = ipintellib.ip2name(ip) dnsName = dnsInfo['name'].rstrip('.') #print "kojoney_hiddenip.py : " + dnsName.__str__() # ignore traffic from Internet test hosts #if ip.find("173.203.89.") != -1 : # GSOC slice server roy.doesntexist.org # if debug == True : # syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = slice server doesntexist.org") # return True # ignore traffic from Google DNS if ip.find("8.8.8.8") != -1: #if debug == True : #syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = slice server doesntexist.org") return True # ignore traffic associated with Twitter if dnsName.find("twttr.com") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = twttr.com") return True # ignore traffic associated with updating Slackware patches if dnsName.find("slackware.mirrors") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = slackware.mirrors") return # ignore traffic associated with malware analysis by Anubis at Univ California Santa Barbara if dnsName.find("anubis.cs.ucsb.edu") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = anubis.cs.ucsb.edu") return # ignore traffic associated with honeyd updating it's site if dnsName.find("honeyd.provos.org") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = honeyd.provos.org") return True # ignore traffic associated with Team Cymru if dnsName.find("cymru.com") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , DNS = cymru.com") return True # Filter based on AS number / name - do last since expensive lookup asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'].upper() # e.g. GOOGLE #print "kojoney_hiddenip.py : " + asRegisteredCode.__str__() if asRegisteredCode.find("GOOGLE") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , AS = GOOGLE") return True if asRegisteredCode.find("TWITTER-NETWORK") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , AS = TWITTER-NETWORK") return True if asRegisteredCode.find("FACEBOOK") != -1: if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> TRUE , AS = FACEBOOK") return True # ip is an attacker if debug == True: syslog.syslog("hiddenIP(" + ip + "," + lanAllowed.__str__() + ") -> FALSE, ip is an attacker") #return False return False except Exception, e: msg = "kojoney_hiddenip.py : hiddenIP() : exception " + ` e ` + " ip=" + ipAddress.__str__( ) print msg syslog.syslog(msg) return None
def setIDMEFcommon(idmef, analyserClass, sensorId, srcIP, dstIP, dstPort, attackerIP, logEntry): try: print "kojoney_idmef_common.py : setIDMEFcommon() : srcIP = " + srcIP.__str__( ) print "kojoney_idmef_common.py : setIDMEFcommon() : dstIP = " + dstIP.__str__( ) #print "kojoney_idmef_common.py : setIDMEFcommon() : attackerIP = " + attackerIP.__str__() idmef.Set("alert.analyzer(0).model", "Blackrain") idmef.Set("alert.analyzer(0).name", "blackrain-" + sensorId.upper()) idmef.Set("alert.analyzer(0).manufacturer", "Blackrain Technologies") idmef.Set("alert.analyzer(0).class", analyserClass) idmef.Set("alert.analyzer(0).version", "1.0rc1") idmef.Set("alert.analyzer(0).ostype", "Linux") idmef.Set("alert.analyzer(0).osversion", "2.6.21.5") if logEntry != None: logEntry = logEntry.rstrip() else: logEntry = "None" idmef.Set("alert.additional_data(0).type", "string") idmef.Set("alert.additional_data(0).meaning", "Original log entry") idmef.Set("alert.additional_data(0).data", logEntry) print "kojoney_idmef_common.py : setIDMEFcommon() : logEntry = " + logEntry.__str__( ) fieldsSet = 1 if attackerIP != None: # GeoIP enhancement geoIP = ipintellib.geo_ip(attackerIP) #print geoIP.__str__() countryCode = geoIP['countryCode'].__str__() city = geoIP['city'].__str__() # Prewikka has a "spare" pie-chart used for source users - so abuse it for Country Code pie-chart idmef.Set("alert.source(0).user.user_id(0).name", "haxx0r-" + countryCode) idmef.Set("alert.source(0).user.user_id(0).tty", "UserId Name is faked") if attackerIP == srcIP: idmef.Set("alert.source(0).node.location", countryCode) addInfoLabel = "Source(0) " elif attackerIP == dstIP: idmef.Set("alert.target(0).node.location", countryCode) addInfoLabel = "Target(0) " latitude = "%.2f" % geoIP['latitude'] + " N" longitude = "%.2f" % geoIP['longitude'] + " E" # AS enhancement asInfo = ipintellib.ip2asn(attackerIP) #print asInfo.__str__() asNum = asInfo['as'].__str__() asRegisteredCode = asInfo['registeredCode'].__str__() asRegisteredName = asInfo['registeredName'].__str__() asNetblock = asInfo['netblock'].__str__() asRegistry = asInfo['registry'].__str__() idmef.Set("alert.additional_data(1).type", "string") idmef.Set("alert.additional_data(1).meaning", addInfoLabel + "MaxMind GeoIP City") idmef.Set("alert.additional_data(1).data", city) idmef.Set("alert.additional_data(2).type", "string") idmef.Set("alert.additional_data(2).meaning", addInfoLabel + "MaxMind GeoIP Latitude") idmef.Set("alert.additional_data(2).data", latitude) idmef.Set("alert.additional_data(3).type", "string") idmef.Set("alert.additional_data(3).meaning", addInfoLabel + "MaxMind GeoIP Longitude") idmef.Set("alert.additional_data(3).data", longitude) idmef.Set("alert.additional_data(4).type", "string") idmef.Set("alert.additional_data(4).meaning", addInfoLabel + "BGP ASN") idmef.Set("alert.additional_data(4).data", "AS" + asNum) idmef.Set("alert.additional_data(5).type", "string") idmef.Set("alert.additional_data(5).meaning", addInfoLabel + "ISP Code") idmef.Set("alert.additional_data(5).data", asRegisteredCode) idmef.Set("alert.additional_data(6).type", "string") idmef.Set("alert.additional_data(6).meaning", addInfoLabel + "ISP Name") idmef.Set("alert.additional_data(6).data", asRegisteredName) idmef.Set("alert.additional_data(7).type", "string") idmef.Set("alert.additional_data(7).meaning", addInfoLabel + "Internet prefix/route") idmef.Set("alert.additional_data(7).data", asNetblock) idmef.Set("alert.additional_data(8).type", "string") idmef.Set("alert.additional_data(8).meaning", addInfoLabel + "Internet Registry") idmef.Set("alert.additional_data(8).data", asRegistry) fieldsSet = fieldsSet + 8 # p0f enhancement - optional if srcIP != None and dstIP != None and dstPort != None: p0fInfo = p0fcmd.getP0fInfo(srcIP, "0", dstIP, dstPort) # 0 = wildcard the srcPort if p0fInfo['result'] == True: # p0f data is available os = p0fInfo['genre'] nat = p0fInfo['nat'][0] hops = p0fInfo['hops'] idmef.Set("alert.additional_data(9).type", "string") idmef.Set("alert.additional_data(9).meaning", "p0f : OS") idmef.Set("alert.additional_data(9).data", os.__str__()) idmef.Set("alert.additional_data(10).type", "string") idmef.Set("alert.additional_data(10).meaning", "p0f : NAT detected") idmef.Set("alert.additional_data(10).data", nat.__str__()) idmef.Set("alert.additional_data(11).type", "string") idmef.Set("alert.additional_data(11).meaning", "p0f : IP hops") idmef.Set("alert.additional_data(11).data", hops.__str__()) fieldsSet = fieldsSet + 3 #print "fieldsSet = " + fieldsSet.__str__() return fieldsSet except Exception, e: msg = "kojoney_idmef_common.py : setIDMEFcommon() : exception : " + e.__str__( ) print msg syslog.syslog(msg) return None
def sendTweetCLI(sessionid, username, ip, cli): global TweetClient global TweetVersion # bump version of format of Tweet changes print "Entered sendTweetCLI()" now = time.time() try: # sessionid of -1 indicates that we have no AUTH_OK event and so no username - so don't tweet it if (int(sessionid) < 0): print "sessionId < 0 -> no previous AUTH_OK event to get username" return p0fInfo = p0fcmd.getP0fInfo(ip, "0", "172.31.0.67", "22") if p0fInfo['result'] == True: # p0f data is available hops = p0fInfo['hops'] os = p0fInfo['genre'] fw = p0fInfo['firewall'] nat = p0fInfo['nat'] if p0fInfo['genre'] == "Linux": uptime = p0fInfo['uptime'] bte = now - int(uptime) # boot time (epoch secs) hops = p0fInfo['hops'] else: uptimeHours = 0 bte = 0 else: # p0f data not read OK hops = 0 os = "?" fw = "?" nat = "?" bte = 0 # get current time timeTuple = time.localtime(now) nowStr = time.asctime(timeTuple) # calc haxx0r bootTime #timeTuple = time.localtime(bte) #bootTimeStr = time.asctime(timeTuple) bteh = int(bte / 3600) # bteh = boot time epoch (hours) # Get DNS info dnsInfo = ipintellib.ip2name(ip) dnsName = dnsInfo['name'] # WHOIS information asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR # GeoIP information geoIP = ipintellib.geo_ip(ip) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP['longitude'] # Used to calc approx. localtime # Compact usual returns if asNum.find("AS-none") != -1: asNum = "?" if asRegisteredCode.find("failed") != -1: asRegisteredCode = "?" if countryCode.find("None") != -1: countryCode = "?" if city.find("None") != -1: city = "?" # Construct Tweet # todo - if os = windoes, do not add bteh msg = "sid=" + `sessionid` + ":IP=" + ip + ":" + asNum + "(" + asRegisteredCode + ")" + ":"\ + countryCode + ":" + city + ":" + "%.2f" % longitude + ":" + os + ":" + hops\ + ":fw=" + fw + ":nat=" + nat + ":bteh=" + `bteh`\ + ":" + username + "@hpot $ " + cli # Send the Tweet - max = 147 ? syslog.syslog("sendTweetCLI(): msg length=" + ` len(msg) ` + " chars") print "Tweet=" + msg # syslog every Tweet attempted to be sent syslog.syslog("sendTweetCLI(): " + msg) # *** Disable during testing *** status = TweetClient.PostUpdate(msg) time.sleep(5) # crude rate-limit ? except Exception, e: syslog.syslog("kojoney_tail.py : sendTweetCLI() exception caught = " + ` e ` + " ip=" + ip)
def processHoneyd(line): flowEvent = {} try: #print "\nEntered blackrain_honeyd.processHoneyd()" #print line flowEvent['flowType'] = "FLOW_HONEYD_FLOW" ip = re.findall("\d+\.\d+\.\d+\.\d+", line) if len(ip) == 0: #print "No IP addresses found" return None #else : # print "Found IP addresses..." if not "END" in line: #print "No END honeyd flow found, so return None" return None sIP = re.findall("(\d+\.\d+\.\d+\.\d+)", line)[0] #print sIP sP = re.findall("\d+\.\d+\.\d+\.\d+ (\d+) \d+\.\d+\.\d+\.\d+ \d+", line)[0] #print sP dP = re.findall("\d+\.\d+\.\d+\.\d+ \d+ \d+\.\d+\.\d+\.\d+ (\d+)", line)[0] #print dP flowEvent['flowDirection'] = "in" flowEvent['flowRemoteIP'] = sIP flowEvent['flowRemotePort'] = sP flowEvent['flowHpotPort'] = dP if "icmp" in line: flowEvent['flowProto'] = "I" elif "tcp" in line: flowEvent['flowProto'] = "T" elif "udp" in line: flowEvent['flowProto'] = "UDP" else: flowEvent[ 'flowProto'] = "unknown" # attention : need to test other protocols #flowEvent['flowOS'] = "none" # Optional parameters # =================== #flowEvent['flowDuration'] #flowEvent['flowPkts'] #flowEvent['flowTflags'] = re.findall("fl=(\d+)",line)[0] # Number of bytes received by the honeypot # This does not include the data part of the TCP handshake flowEvent['flowBytes'] = re.findall("rx=(\d+)", line)[0] #print flowEvent # Data enrichment # =============== # DNS info dnsInfo = ipintellib.ip2name(flowEvent['flowRemoteIP']) flowEvent['flowDNS'] = dnsInfo['name'].rstrip('.') #print flowEvent['flowDNS'] # GeoIP info geoIP = ipintellib.geo_ip(flowEvent['flowRemoteIP']) flowEvent['flowCC'] = geoIP['countryCode'] flowEvent['flowCountry'] = geoIP['countryName'] flowEvent['flowCity'] = geoIP['city'] flowEvent['flowLat'] = "%.3f" % float(geoIP['latitude']) flowEvent['flowLong'] = "%.3f" % float(geoIP['longitude']) #print flowEvent['flowCC'] # WHOIS info asInfo = ipintellib.ip2asn(flowEvent['flowRemoteIP']) flowEvent['flowASN'] = asInfo['as'] # AS123 flowEvent['flowISP'] = asInfo[ 'registeredCode'] # Short-form e.g. LEVEL3 flowEvent['flowRoute'] = asInfo['netblock'] flowEvent['flowRIR'] = asInfo['registry'] #print flowEvent['flowASN'] #print flowEvent return flowEvent except Exception, e: msg = "Exception : " + e.__str__() + " in line=" + line print msg syslog.syslog(msg) return None
def sendToLoggly(sensorId,line): try : global txnId print "sendToLoggly() : line=" + line # honeytweeter #shakey = "e25e6042-e490-4910-a246-94cefbdd11b9" # honeytweeter-json shakey = "fe39eb54-7c8f-417c-afc8-5b8db98961d3" sdata = {} # Event Type fields = line.split(",") eventType = fields[0] sdata['eventType'] = fields[0] #print sdata['eventType'] # Sensor Name sdata['sensorId'] = sensorId # IP address ips = re.findall("\d+\.\d+\.\d+\.\d+",line) if len(ips) > 0 : sdata['ip'] = ips[0] # GeoIP geoIP = ipintellib.geo_ip(sdata['ip']) cc = geoIP['countryCode'] if cc != "?" : sdata['cc'] = cc.__str__() # Reverse DNS dnsInfo = ipintellib.ip2name(sdata['ip']) dnsName = dnsInfo['name'] if dnsName != "NoDNS": sdata['rdns'] = dnsName.rstrip('.') # ASN info asInfo = ipintellib.ip2asn(sdata['ip']) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'].upper() # e.g. GOOGLE sdata['asn'] = "AS" + asNum sdata['isp'] = asRegisteredCode # Snort SID if sdata['eventType'] == "SNORT_NIDS" : sids = re.findall("SID=(\d+)",line) if len(sids) > 0: sdata['sid'] = sids[0] a = re.findall("P(\d+) SID",line) if len(a) > 0 : sdata['priority'] = "P" + a[0] # Clamd malware name - e.g. Exploit.Shellcode.X86-Gen-1 # Jun 27 04:47:39 mars kojoney_tweet_engine[3173]: SENDTWEET = <0>CLAMD,Malware Exploit.Shellcode.X86-Gen-1 in flow from 79.5.203.86 ports={s=4592 d=135} if sdata['eventType'] == "CLAMD" : a = line.split(",Malware ")[1] a = a.split(" ")[0] sdata['malware'] = a sdata['avendor'] = "ClamAV" # Threat Report # submitted=Wed Jun 26 02:54:18 2013 cmd=GEO_IP tweet=REPORT,Threat Level for 203.250.135.20 is 41.1, flags={PR SC PS BH AT GA} if "flags={" in line and sdata['eventType'] == "REPORT" : a = line.split("flags=")[1] #a = a.rstrip("}") sdata['flags'] = a # VirusTotal # submitted=Fri May 17 07:48:03 2013 cmd=BASIC tweet=ANALYST,AV eb7656dd256eb414abe092eb0f41ea1f.php => Norman=PhpShell.BL 17/46 VT=http://bit.ly/14vyont # line=ANALYST,AV 06a940dd7824d6a3a6d5b484bb7ef9d5.php => Unseen by VirusTotal if sdata['eventType'] == "ANALYST" and "AV" in line and "Unseen by" not in line : a = line.split(" => ")[1] b = a.split(" ")[0] # Symantec=Trojan.Usuge!gen3 sdata['avendor'] = b.split("=")[0] sdata['malware'] = b.split("=")[1] b = a.split("VT=")[1] sdata['virustotal'] = b # filename = MD5 name + extension (optional) c = line.split("AV ")[1] c = c.split(" ")[0] #print c if "." in c: c = c.split(".")[0] # lose filename extension sdata['md5'] = c.lower() # Destination port # ---------------- # Snort messages flow = re.findall("\w+\:(\d+) ",line) if len(flow) > 0 : sdata['port'] = int(flow[0]) # IPLOG if sdata['eventType'] == "IPLOG" : ports = re.findall("port (\d+)",line) if len(ports) > 0 : sdata['port'] = int(ports[0]) # CLAMD - clsniffer if sdata['eventType'] == "CLAMD" : ports = re.findall("d=(\d+)",line) if len(ports) > 0 : sdata['port'] = int(ports[0]) # KIPPO if sdata['eventType'] == "KIPPO" : sdata['port'] = 2222 sdata['protocol'] = "tcp" # Glastopf if "WEB" in sdata['eventType'] : sdata['port'] = 18080 sdata['protocol'] = "tcp" # Botjuicer cracked scripts if sdata['eventType'] == "ANALYST" and "BOTJUICER" in line : #if "UNDETERMINED" in line.upper(): # return None #print "BOTJUICER log found : " + line a = re.findall("p=(\d+)",line) if len(a) > 0 : sdata['port'] = int(a[0]) a = re.findall("ch=(#\w+)",line) if len(a) > 0 : sdata['irc'] = a[0] # Protocol - generic if "TCP" in line.upper(): sdata['protocol'] = "tcp" if "UDP" in line.upper(): sdata['protocol'] = "udp" if "ICMP" in line.upper(): sdata['protocol'] = "icmp" # Timestamp sdata['datetime'] = time.ctime() # Message sdata['msg'] = line # Txnid - increment this last, just before submission to Loggly txnId = txnId + 1 sdata['txnId'] = txnId body = json.dumps(sdata) print "sendToLoggly() : JSON body = " + body # Send to Loggly insert_url = "http://logs.loggly.com/inputs/" + shakey insert_http = httplib2.Http(timeout=10) #body = line resp, content = insert_http.request(insert_url, "POST", body=body, headers={'content-type':'text/plain'}) #print "Loggly : resp : " + resp.__str__() #print "Loggly : content : " + content.__str__() #{'status': '200', 'content-length': '18', 'vary': 'Accept-Encoding', 'server': 'TwistedWeb/12.0.0', 'date': 'Tue, 25 Jun 2013 05:55:58 GMT', 'content-type': 'text/html'} #{"response": "ok"} if "ok" in content.__str__() : # bug -> why can't I look at the "response" field in a structured way msg = "Sent to Loggly OK : " + body.__str__() print msg else: msg = "Sent to Loggly FAIL : " + body.__str__() + " error = " + content.__str__() print msg syslog.syslog(msg) # crude form of rate-limiter time.sleep(0.5) except Exception,e: msg = "kojoney_loggly.py : sendToLoggly() : exception : " + e.__str__() + " line=" + line print msg syslog.syslog(msg)
def test(): print "\n\n\n\n" print "==========================" print "p0f logfile to Reputation " print "==========================" print "Library version : " + ipintellib.getVersion() print "Hard-coded tests" print "----------------" # IP to DNS name ipStr = "217.41.27.169" # My IP dnsInfo = ipintellib.ip2name(ipStr) # resolve get DNS name print ipStr + " resolves to " + dnsInfo['name'] # DNS name to IP ipStr = "www.openbsd.org" dnsInfo = ipintellib.ip2name(ipStr) # resolve get DNS name print ipStr + " resolves to " + dnsInfo['name'] # input file : file of IPs (hand-crafted) fpIn = open(r'/home/var/log/p0f.log', 'r') # output file fpOut0 = open(r'/home/var/log/p0f2reputation.out.csv', 'w') # Main loop #logs = {} asInfo = {} dnsInfo = {} # Logging file #logging.basicConfig(level=logging.INFO,filename='botclientsyslog.py.log') #logging.info('SYSLOG log file analysis started') lineCounter = -1 while True: lineCounter = lineCounter + 1 ############################### # if lineCounter >= 3000: # sys.exit(0) ############################### line1 = fpIn.readline() if not line1: break if line1[0] == '#': # ignore leading # comment #print "Ignore # comment" continue if line1.find( "distance") == -1: # all valid lines have 'distance' in them" continue print "----------------------------------------------" fields = line1.split(' ') print fields numb = len(fields) print "numb fields is " + ` numb ` os = fields[8] # re-write this using regular expressions and make a function so that it can be reused # !!!!!!!!!!!!!!!!!!!!!!! if line1.find("Linux") != -1: if numb == 17: ipStr, port = fields[12].split( ':') # dest IP and dest TCP port number elif numb == 24: ipStr, port = fields[17].split( ':') # dest IP and dest TCP port number elif line1.find("Windows") != -1: if numb == 19: ipStr, port = fields[14].split( ':') # dest IP and dest TCP port number elif line1.find("Novell") != -1: continue # ignore Novell for moment #if numb == 19: # ipStr,port = fields[13].split(':') # dest IP and dest TCP port number else: sys.exit("Failed to parse") print "line=" + ` lineCounter ` + ": *** haxx0r : ip=" + ipStr + " port=" + port + " os=" + os dnsInfo = ipintellib.ip2name(ipStr) # resolve get DNS name asInfo = ipintellib.ip2asn(ipStr) # get AS information from WHOIS whob reputation = ipintellib.ip2reputation( ipStr) # does slow things down... geoIP = ipintellib.geo_ip(ipStr) result = ipStr + "," + dnsInfo['name'].strip('.') + \ "," + asInfo['as'] + ",ASowner=" + asInfo['registeredCode'] + ",ASnetblock=" + asInfo['netblock'] + ",ASregistry=" + asInfo['registry'] + \ ",geoIPcountry=" + geoIP['countryCode'] + ",geoIPcity=" + geoIP['city'] + ",geoIPlat=" + "%.2f" % geoIP['latitude'] + ",geoIPlong=" + "%.2f" % geoIP['longitude'] + \ ",zen.spamhaus=" + reputation['zen.spamhaus.org'] + \ ",dnsbl.ahbl.org=" + reputation['dnsbl.ahbl.org'] + \ ",bl.deadbeef.com=" + reputation['bl.deadbeef.com'] + \ ",bogons.cymru.com=" + reputation['bogons.cymru.com'] + \ ",zombie.dnsbl.sorbs.net=" + reputation['zombie.dnsbl.sorbs.net'] + \ ",bl.spamcop.net=" + reputation['bl.spamcop.net'] + \ ",dul.dnsbl.sorbs.net=" + reputation['dul.dnsbl.sorbs.net'] + \ ",l2.apews.org=" + reputation['l2.apews.org'] + \ ",virus.rbl.msrbl.net=" + reputation['virus.rbl.msrbl.net'] + \ ",phishing.rbl.msrbl.net=" + reputation['phishing.rbl.msrbl.net'] + \ ",images.rbl.msrbl.net=" + reputation['images.rbl.msrbl.net'] + \ ",spam.rbl.msrbl.net=" + reputation['spam.rbl.msrbl.net'] print result print >> fpOut0, ipStr + "," + "os" + reputation['zen.spamhaus.org']
def updateAttackerOIDsNetflow(attacker,line) : try : #print "attacker['IP'] is " + attacker['IP'].__str__() if line.find("dir=in") == -1 : return print "----" #print "attacker_statd.py : updateAttackerOIDsNetflow() : INCOMING : line=" + line[0:120] + "..." pat1 = 'dIP=(\d+\.\d+\.\d+\.\d+)' pat2 = 'sIP=(\d+\.\d+\.\d+\.\d+)' dip = re.findall(pat1,line) sip = re.findall(pat2,line) if len(dip) < 1 or len(sip) < 1 : # could not find both a source IP and a dest IP return #print "attacker_statd.py : sip = " + sip.__str__() #print "attacker_statd.py : dip = " + dip.__str__() if dip[0] in honeypotIPs : #print "destination IP : " + dip[0] + " is a honeynet IP" #print "source IP : found attacker sIP : " + sip[0] # case 1 : update list of unique source IPs srcIP = sip[0] #print srcIP # Weed out comms with Twitter, Google etc. print "attacker_statd.py : called hiddenIP() with IP = " + srcIP if kojoney_hiddenip.hiddenIP(srcIP) == True : print "attacker_statd.py : srcIP " + srcIP + " is not a real attacker since hiddenIP()=True, so ignore" return else: print "attacker_statd.py : srcIP " + srcIP + " is an attacker since hiddenIP()=False, so update SNMP stats" if srcIP not in attacker['IP'] : #print srcIP + " not seen before" temp = attacker['IP'] temp.append(srcIP) attacker['IP'] = temp #print "attacker['IP'] updated to : " + attacker['IP'].__str__() # increment number of unique IPs found num = len(attacker['IP']) attacker['NUM_IP'] = num msg = "attacker['NUM_IP'] updated to " + attacker['NUM_IP'].__str__() + " by adding " + srcIP #print msg #syslog.syslog(msg) # case 2 : update list of unique country codes and cities geoIP = ipintellib.geo_ip(srcIP) countryCode = geoIP['countryCode'] city = geoIP['city'] if countryCode not in attacker['CC'] : #print "attacker countryCode " + countryCode + " not seen before" temp = attacker['CC'] temp.append(countryCode) attacker['CC'] = temp msg = "attacker['CC'] updated to : " + attacker['CC'].__str__() #print msg # increment number of unique CCs found num = len(attacker['CC']) attacker['NUM_CC'] = num msg = "attacker['NUM_CC'] updated to " + attacker['NUM_CC'].__str__() + " by adding " + countryCode #print msg #syslog.syslog(msg) if city not in attacker['CITY'] : #print "attacker city " + city + " not seen before" temp = attacker['CITY'] temp.append(city) attacker['CITY'] = temp msg = "attacker['CITY'] updated to : " + attacker['CITY'].__str__() #print msg # increment number of unique cities found num = len(attacker['CITY']) attacker['NUM_CITY'] = num msg = "attacker['NUM_CITY'] updated to " + attacker['NUM_CITY'].__str__() + " by adding " + city #print msg #syslog.syslog(msg) # case 3 : update list of unique AS numbers asInfo = ipintellib.ip2asn(srcIP) asn = asInfo['as'] if asn not in attacker['ASN'] : #print "attacker ASN " + asn + " not seen before" temp = attacker['ASN'] temp.append(asn) attacker['ASN'] = temp msg = "attacker['ASN'] updated to : " + attacker['ASN'].__str__() #print msg # increment number of unique AS numbers num = len(attacker['ASN']) attacker['NUM_ASN'] = num msg = "attacker['NUM_ASN'] updated to " + attacker['NUM_ASN'].__str__() #print msg #syslog.syslog(msg) # case 4 : update list of unique /24 slash24 = ipintellib.getSlash24(srcIP) if slash24 not in attacker['SLASH_24'] : #print "attacker /24 " + slash24 + " not seen before" temp = attacker['SLASH_24'] temp.append(slash24) attacker['SLASH_24'] = temp msg = "attacker['SLASH_24'] updated to : " + attacker['SLASH_24'].__str__() #print msg # increment number of unique AS numbers num = len(attacker['SLASH_24']) attacker['NUM_SLASH_24'] = num msg = "attacker['NUM_SLASH_24'] updated to " + attacker['NUM_SLASH_24'].__str__() + " by adding " + slash24 #print msg #syslog.syslog(msg) else: print "attacker_statd.py : dstIP " + dip[0] + " not found in honeypotList=" + honeypotIPs.__str__() + ", so no need to update attacker SNMP stats" # All done so return return except Exception,e: msg = "attacker_statd.py : updateAttackerOIDsNetflow() : exception caught = " + `e` + " line=" + line print msg syslog.syslog(msg)
def processGlastopf(line): asMsg = "" try: #print "processGlastopf() : line read is " + line # Successful attack if line.find("attack from") != -1 or line.find( "Mail attack found from") != -1: fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) #msg = ' '.join(fields[3:]).rstrip("?") # does glastopf add trailing ? / ?? msg = "WEB_X," + msg msg = msg.replace(" with request: ", " req=") msg = msg.replace("found from", "from") # normalise MAIL versus "other" attacks msg = msg.replace( "Mail", "Webmail") # normalise MAIL versus "other" attacks return msg # File retrieved if line.find("successfully opened") != -1: fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) msg = "WEB_OPEN," + msg return msg # Googledorks data written to mySQL database - not interesting enough to Tweet #if line.find("written into local database") != -1 : # fields = line.split(" - ") # #print fields # msg = ' '.join(fields[3:]) # msg = "WEB_GOOGLEDORK," + msg # return msg # File saved to disk if line.find("written to disk") != -1 and line.find("File ") != -1: fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) msg = msg.replace("File", "Previously unseen PHP malware file") msg = "WEB_WRITE," + msg return msg # Scan - i.e. unsuccessful attack if line.find("No attack found") != -1: #print "scan" ip = kojoney_funcs.findFirstIP(line) if ip != None: #print "IP found = " + ip # WHOIS information #asInfo = rch_asn_funcs.ip2asn(ip) asInfo = ipintellib.ip2asn(ip) asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo[ 'registeredCode'] # Short-form e.g.ARCOR asMsg = asRegisteredCode + " (" + asNum + ")" #print asMsg fields = line.split(" - ") #print fields msg = ' '.join(fields[3:]) if line.find("http" ) != -1: # attacker is trying to test if I am a proxy msg = msg.replace("No attack found from", "WEB_PRX,Request from") else: # LOC = local msg = msg.replace("No attack found from", "WEB_SCN,Scan from") msg = msg.replace(" with request: ", " req=") msg = msg.rstrip() # remove any trailing characters #msg = msg + " ISP=" + asMsg kojoney_afterglow.visWebScan(msg) return msg return None except Exception, e: syslog.syslog("kojoney_glastopf_parse.py : processGlastopf() : " + ` e ` + " line=" + line)
def printAttackerOIDs(): # need to add logic to fail if the shelf file does not exist database = shelve.open('/home/var/log/attacker_shelf.dat') if len(database) == 0: # contains no entries sys.exit("Failed to locate shelf file") print " " print "Attackers" print "---------" print "NUM_IP : " + database['NUM_IP'].__str__() print "NUM_SLASH_24 : " + database['NUM_SLASH_24'].__str__() print "NUM_CC : " + database['NUM_CC'].__str__() print "NUM_CITY : " + database['NUM_CITY'].__str__() print "NUM_ASN : " + database['NUM_ASN'].__str__() print " " print "Malware Captured" print "----------------" print "AMUN_FILES_BIN : " + database['AMUN_FILES_BIN'].__str__() print "AMUN_FILES_HEX : " + database['AMUN_FILES_HEX'].__str__() print "NEPENTHES_FILES : " + database['NEPENTHES_FILES'].__str__() print "KIPPO_FILES : " + database['KIPPO_FILES'].__str__() print "GLASTOPF_FILES_GET : " + database['GLASTOPF_FILES_GET'].__str__() print "GLASTOPF_FILES_POST : " + database['GLASTOPF_FILES_POST'].__str__() print "ANALYST_FILES_ALL : " + database['ANALYST_FILES_ALL'].__str__() print "ANALYST_FILES_EXE : " + database['ANALYST_FILES_EXE'].__str__() print "ANALYST_FILES_PHP : " + database['ANALYST_FILES_PHP'].__str__() print "ANALYST_FILES_TXT : " + database['ANALYST_FILES_TXT'].__str__() print "ANALYST_FILES_GIF : " + database['ANALYST_FILES_GIF'].__str__() print "ANALYST_FILES_JPG : " + database['ANALYST_FILES_JPG'].__str__() print "ANALYST_FILES_PNG : " + database['ANALYST_FILES_PNG'].__str__() print "ANALYST_FILES_TGZ : " + database['ANALYST_FILES_TGZ'].__str__() print " " print "Attackers" outfile = "blackrain_attackers-report.csv" fp = open(outfile, 'w') for ip in database['IP']: asInfo = ipintellib.ip2asn(ip) # asNum = asInfo['as'] # AS123 asRegisteredCode = asInfo['registeredCode'] # Short-form e.g.ARCOR geoIP = ipintellib.geo_ip(ip) countryCode = geoIP['countryCode'] city = geoIP['city'] longitude = geoIP['longitude'] # Used to calc approx. localtime latitude = geoIP['latitude'] dnsInfo = ipintellib.ip2name(ip) dnsName = dnsInfo['name'].rstrip('.') msg = ip + "," + "AS" + asNum.__str__( ) + "," + countryCode + "," + dnsName.__str__( ) + "," + asRegisteredCode.__str__() + "," + countryCode.__str__( ) + "," + city.__str__() print msg print >> fp, msg fp.close()