def config_db(argv): c = mongo_client() db = c.config db.external.remove() if len(sys.argv) < 3: web_dir = '/var/www/html' else: web_dir = sys.argv[2] print web_dir db.external.insert_one({ 'SELF_SERVE_PATH_SKIPFISH': web_dir + '/reports/selfServeOutputSkipfish/', 'SELF_SERVE_DATABASE': 'selfTestingExternal', 'PATH_WAPITI': web_dir + '/reports/outputWapiti/', 'DATABASE': 'testingExternal', 'SELF_SERVE_PATH_WAPITI': web_dir + '/reports/selfServeOutputWapiti/', 'TIMEOUT_VUL': '180', 'TIMEOUT_TECH': '30', 'PATH_LOGFILE': 'external.log', 'PATH_SKIPFISH': web_dir + '/reports/outputSkipfish/', 'SELF_SERVE_PATH_LOGFILE': 'selfServe.log' }) return web_dir
def start_scan(): # config database client = mongo_client() db = client.config # select configurations cursor = db.external.find() if not cursor or cursor.count() == 0: print "[!] Please configure the DB first by running below command." print "\t$ docker run -t -i --rm --link <your mongo container name>:mongodb watchdog -c" sys.exit(1) logFile = cursor[0]['PATH_LOGFILE'] database = cursor[0]['DATABASE'] db = client[database] # log file logging.basicConfig(filename = logFile,filemode = 'w',format='%(levelname)s:%(message)s',level=logging.DEBUG) # ips cursor = db.ipInventory.find() print "Total IPs in database: "+str(cursor.count()) # each iteration corresponds to one ip for document in cursor: ip = document['ip'] domain = document['domain'] if domain == "": domain = "null" serv = db.services #os.system("python findPorts.py "+str(ip)+" "+str(domain)+" org") # call findPorts print "starting portscan" portScan(str(ip),str(domain),"org") #checking whether 80 or 443 is open if serv.find({"ip":ip,"80":{"$exists":True}}).count() > 0 or serv.find({"ip":ip,"443":{"$exists":True}}).count() > 0: logging.info("Finding Vulnerability") # p = Popen("sudo python findVulnerability.py "+str(ip)+" "+str(domain)+" org",shell = True) # call findVulnerability vulnerabilityScan(str(ip),str(domain),"org") technologyScan(str(ip),str(domain),"org") #os.system("python findTechnology.py "+str(ip)+" "+str(domain)+" org") # call findTechnology else: logging.info("Both 80 and 443 are closed corresponding to ip: "+str(ip))
def config_db(argv): c = mongo_client() db = c.config db.external.remove() if len(sys.argv) < 3: web_dir = '/var/www/html' else: web_dir = sys.argv[2] print web_dir db.external.insert_one({ 'PATH_WAPITI': web_dir+'/reports/outputWapiti/', 'DATABASE': 'testingExternal', 'TIMEOUT_VUL': '180', 'TIMEOUT_TECH': '30', 'PATH_LOGFILE': 'external.log', 'PATH_SKIPFISH': web_dir+'/reports/outputSkipfish/', }) db.internal.insert_one({ 'REPO_PATH':"", 'SCALE_CRITICAL_VUL_CVE_EXTERNAL':8.3, 'SELF_SERVE_DATABASE':'Self_Serve_Internal_Testing', 'SCALE_MEDIUM_VUL_CVE_EXTERNAL':5.0, 'DATABASE':'Internal_Testing', 'ORGANISATION':'<>', 'GIT_TOKEN':'<>:<>', 'SCALE_MEDIUM_VUL_CVE_INTERNAL':5.0, 'DEPENDENCY_PATH_LOGFILE':'dependency_log.log', 'SELF_SERVE_SOURCE_PATH_LOGFILE':'self_serve_source_log.log', 'SELF_SERVE_DEPENDENCY_PATH_LOGFILE':'self_serve_dependency_log.log', 'SOURCE_PATH_LOGFILE':'source_log.log', 'SELF_SERVE_TIME_OUT':30, 'TIME_OUT':30, 'SCALE_HIGH_VUL_CVE_INTERNAL':7.5, 'SCALE_HIGH_VUL_CVE_EXTERNAL':7.5, 'SCALE_CRITICAL_VUL_CVE_INTERNAL':8.3, 'SCALE_HIGH_VUL_REPO':8.3 }) return web_dir
def fill_Inventory(filename, action=None): client = mongo_client() db = client.testingExternal if not filename: filename = 'subdomains.txt' lines = "" with open(filename) as f: lines = f.readlines() if f: f.close() if action == "replace": db.ipInventory.remove() i = 1 for line in lines: if line.strip() != "": domain = {} try: ip = socket.gethostbyname(line.strip()) domain['ip'] = ip domain['domain'] = line.strip() print str(i) + ". ", print domain, count = db.ipInventory.count({ 'domain': line.strip(), 'ip': ip }) if count == 0: print db.ipInventory.insert(domain) else: print "Already in DB" i = i + 1 except Exception as ae: print ae pass
def countVulns(ip, domain, type): cwd = os.getcwd() c = mongo_client() Configure = c.config.external.find()[0] #ip = sys.argv[1] #domain = sys.argv[2] #type = sys.argv[3] if (type == "selfServe"): DBName = Configure['SELF_SERVE_DATABASE'] else: DBName = Configure['DATABASE'] collection = c[DBName].vulnerabilities newCollection = [] serverAddress = "http://127.0.0.1" data = collection.find_one({"ip": ip}) # print collection.find().count() # print collection.find()[0] url = data['skipfish'] fileUrl = serverAddress + str(url).split("html")[1] + "/index.html" command = "phantomjs loadspeed.js " + fileUrl + " > temp/" + ip.replace( ".", "-") + ".html" run_tool(cmd=command) command = "python parser.py temp/" + ip.replace( ".", "-") + ".html > temp/" + ip.replace(".", "-") + ".txt" run_tool(cmd=command) f = open("temp/" + ip.replace(".", "-") + ".txt") number = f.read().split("\n")[0] c[DBName].vulnerabilities.update({"ip": ip}, {"$set": {"count": number}}) f.close()
def technologyScan(ip, domain, type): # config database client = mongo_client() db = client.config cursor = db.external.find() #type = sys.argv[3] # checking for selfServe of org scan and setting the paramter if (type == "selfServe"): logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE'] database = cursor[0]['SELF_SERVE_DATABASE'] else: logFile = cursor[0]['PATH_LOGFILE'] database = cursor[0]['DATABASE'] TIMEOUT = int(cursor[0]['TIMEOUT_TECH']) db = client[database] # log file logging.basicConfig(filename=logFile, format='%(levelname)s:%(message)s', level=logging.DEBUG) # timeout def signal_handler(signum, frame): raise Exception("Timed Out!") signal.signal(signal.SIGALRM, signal_handler) #ip = sys.argv[1] #domain = sys.argv[2] w = Wappalyzer() serv = db.services if domain != "null": host = domain # host is the parameter to be passed else: host = ip if domain == "null": domain = "" # checking whether to scan through 80 or 443 if serv.find({"ip": ip, "443": {"$exists": True}}).count() > 0: prefix = "https://" elif serv.find({"ip": ip, "80": {"$exists": True}}).count() > 0: prefix = "http://" component = {} # every 3rd party tools is scanning 6 times, if it finds the technology than it stops # wappalyzer count = 6 while (count): if count <= 3: host = ip # host is changed to ip after 3 scan count -= 1 logging.info("Wappalyzer working on " + host) signal.alarm(TIMEOUT) try: # calling wappalyzer wapp = w.analyze(prefix + host) except Exception as e: logging.error("Issues with wappalyzer: " + str(e)) signal.alarm(0) continue signal.alarm(0) logging.info(wapp) if len(wapp) == 0: # checking for output logging.info("No output.") if count != 0: logging.info("Sleeping for 10 seconds.") time.sleep(10) continue for key in wapp: component[key.lower()] = wapp[key][unicode('version')] break # builtwith if domain != "": host = domain else: host = ip count = 6 while (count): if count <= 3: host = ip count -= 1 logging.info("Builtwith working on " + host) signal.alarm(TIMEOUT) try: # builtwith working bw = builtwith(prefix + host) except Exception as e: logging.error("Issues with builtwith: " + str(e)) signal.alarm(0) continue signal.alarm(0) logging.info(bw) if len(bw) == 0: logging.info("No output.") if count != 0: logging.info("Sleeping for 10 seconds.") time.sleep(10) continue for keys in bw: # checking for output for key in bw[keys]: if key not in component.keys(): component[key.lower()] = "" break # phantalyzer if domain != "": host = domain else: host = ip count = 6 while (count): if count <= 3: host = ip count -= 1 logging.info("Phantalyzer working on " + host) signal.alarm(TIMEOUT) try: phanta = run_tool(name="phantomjs", prefix=prefix, domain=host) except Exception as e: logging.error("Issue with phantalyzer: " + str(e)) signal.alarm(0) try: phanta = phanta[1] phanta = phanta.strip() logging.info(phanta) if phanta == "": logging.info("No output.") if count != 0: logging.info("Sleeping for 10 seconds.") time.sleep(10) continue phanta = phanta.split("\n") phanta[0] = phanta[0].strip() phanta = phanta[0].split(":")[1] if phanta == "" or phanta.strip() == '160': logging.info("No output.") if count != 0: logging.info("Sleeping for 10 seconds.") time.sleep(10) continue phanta = phanta.split("|") for te in phanta: te = te.strip() if te not in component.keys() and te != "": component[te.lower()] = "" break except Exception as e: logging.error("Issue with phantalyzer: " + str(e)) # wappalyzer extension if domain != "": host = domain else: host = ip count = 6 while (count): if count <= 3: host = ip count -= 1 logging.info("Wappalyzer extension working on " + host) signal.alarm(TIMEOUT) try: cmd = "phantomjs src/drivers/phantomjs/driver.js " + prefix + host phantjs = run_tool(cmd=cmd) except Exception as e: logging.error("Issue with phantomjs code: " + str(e)) signal.alarm(0) try: logging.info(phantjs[1].strip()) if phantjs[1].strip() == "": logging.info("No output.") if count != 0: logging.info("Sleeping for 20 seconds.") time.sleep(2) continue phantjs = json.loads(phantjs[1]) phantjs = phantjs['applications'] if len(phantjs) == 0: logging.info("No output.") if count != 0: logging.info("Sleeping for 20 seconds.") time.sleep(20) continue for i in range(len(phantjs)): if (phantjs[i][unicode('name')] ).lower() not in component.keys(): component[(phantjs[i][unicode('name')] ).lower()] = phantjs[i][unicode('version')] elif component[(phantjs[i][unicode('name')]).lower()] == "": component[(phantjs[i][unicode('name')] ).lower()] = phantjs[i][unicode('version')] break except Exception as e: logging.error("Phantomjs code not working. Issues: " + str(e)) # finding cves try: for key in component: temp = {} temp['version'] = component[key] allCve = [] if component[key] == "": temp['cves'] = allCve temp['false_positive'] = "0" component[key] = temp continue cmd = "python3 Tools/cve-search-master/bin/search.py -p " + str( key).lower().replace(" js", ".js").replace(" ", "_").replace( "apache", "apache:http_server") + ":" + str( component[key]) + " -o json" cves = run_tool(cmd=cmd) cves = cves[1] size = len(cves.split("\n")) if size == 1 and cves == "": temp['cves'] = allCve temp['false_positive'] = "0" component[key] = temp continue for j in range(size): cve = {} tt = json.loads(cves.split("\n")[j]) cve['id'] = tt['id'] cve['cvss'] = tt['cvss'] allCve.append(cve) temp['cves'] = allCve temp['false_positive'] = "0" component[key] = temp except Exception as e: logging.error("Issues with finding cves. Issues: " + str(e)) technologies = db.technologies checking = technologies.find_one({"ip": ip}) if technologies.find({"ip": ip}).count() > 0: technologies.remove({"ip": ip}) technology = {"ip": ip, "domain": domain} technologies.insert_one(technology) for key in component: try: for ch in checking: if key.replace(".", " ") == ch.encode( 'ascii', 'ignore') and component[key][ 'version'] == checking[ch]['version'].encode( 'ascii', 'ignore'): component[key]['false_positive'] = checking[ch][ 'false_positive'] except Exception as e: print "Issues with updating false positive: " + str(e) technologies.update( {"ip": ip}, {"$set": { str(key.replace(".", " ")): component[key] }}) print key + " with version " + str(component[key])
def portScan(ip, domain, type): # config file client = mongo_client() db = client.config cursor = db.external.find() # check for the selfServe or org scan if (type == "selfServe"): logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE'] database = cursor[0]['SELF_SERVE_DATABASE'] else: logFile = cursor[0]['PATH_LOGFILE'] database = cursor[0]['DATABASE'] db = client[database] services = db.services #log file logging.basicConfig(filename=logFile, format='%(levelname)s:%(message)s', level=logging.DEBUG) logging.info("finding Open Ports") host = ip # to be passed as a parameter ports = [] version = [] false_positive = [] try: services = run_tool(name='nmap', ip=host) # nmap command logging.info(services) services = services[1].split(":")[2] services = services.split(",") for i in range( len(services)): # each iteration will correspond to one port services[i] = services[i].strip() ports.append(services[i].split("/")[0]) version.append(services[i].split("//")[2].split("/")[0]) except Exception as e: logging.error("Some issue with Nmap. Issue: " + str(e)) if len(ports) != 0: # calculating md5 sum and updating the database md5 = ''.join(sorted(ports)) md5 = hashlib.md5(md5).hexdigest() if domain == "null": domain = "" services = db.services if services.find({"ip": ip, "md5": md5}).count() == 0: for cnt in range(len(ports)): false_positive.append("") checking = services.find_one({"ip": ip}) if services.find({ "ip": ip }).count( ) > 0: # if ports have changed then delete that entry from database logging.info("Ports have changed.") services.remove({"ip": ip}) else: logging.info("This ip is scanned for the first time.") service = { "ip": ip, "domain": domain, "md5": md5 } # insert port details in database services.insert_one(service) for i in range(len(ports)): # setting false_positive if checking != None: for ch in checking: if ports[i] == ch.encode( 'ascii', 'ignore' ) and version[i] == checking[ch]['version'].encode( 'ascii', 'ignore'): false_positive[i] = checking[ch]['false_positive'] if false_positive[i] == "": false_positive[i] = "0" services.update({"ip": ip}, { "$set": { ports[i]: { "version": version[i], "false_positive": false_positive[i] } } }) logging.info(str(ports[i]) + " has version " + str(version[i])) else: # updating domain if it is not present before if services.find_one({"ip": ip})['domain'].encode('ascii', 'ignore') == "": services.update({"ip": ip}, {"$set": {"domain": domain}}) logging.info("Same ports as before") else: # if no port is identified services = db.services if services.find({"ip": ip}).count() == 0: md5 = hashlib.md5('').hexdigest() serv = {"ip": ip, "domain": domain, "md5": md5} services.insert_one(serv)
import os from subprocess import Popen from mongo_connection import mongo_client import logging import time, datetime client = mongo_client() db = client.config cursor = db.external.find() logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE'] database = cursor[0]['SELF_SERVE_DATABASE'] db = client[database] logging.basicConfig(filename=logFile, filemode='w', format='%(levelname)s:%(message)s', level=logging.DEBUG) cursor = db.onGoingIp.find() for document in cursor: ip = document['ip'] domain = document['domain'] if domain == "": domain = "null" serv = db.services os.system("python findPorts.py " + str(ip) + " " + str(domain) + " selfServe") ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') db.services.update({"ip": ip}, {"$set": {"time": st}}) if serv.find({
def vulnerabilityScan(ip,domain,type): cwd = os.getcwd() client = mongo_client() db = client.config cursor = db.external.find() # type = sys.argv[3] if(type == "selfServe"): logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE'] database = cursor[0]['SELF_SERVE_DATABASE'] skipfishPath = cursor[0]['SELF_SERVE_PATH_SKIPFISH'] wapitiPath = cursor[0]['SELF_SERVE_PATH_WAPITI'] else: logFile = cursor[0]['PATH_LOGFILE'] database = cursor[0]['DATABASE'] skipfishPath = cursor[0]['PATH_SKIPFISH'] wapitiPath = cursor[0]['PATH_WAPITI'] TIMEOUT = int(cursor[0]['TIMEOUT_VUL']) db = client[database] logging.basicConfig(filename = logFile,format='%(levelname)s:%(message)s',level=logging.DEBUG) logging.info("Finding Vulnerability") serv = db.services # ip = sys.argv[1] # domain = sys.argv[2] host = ip if domain == "null": domain = "" if serv.find({"ip":ip,"80":{"$exists":True}}).count() > 0: prefix = "http://" elif serv.find({"ip":ip,"443":{"$exists":True}}).count() > 0: prefix = "https://" try: if os.path.exists(wapitiPath+"wapiti"+host): print "deleting "+wapitiPath+"wapiti"+host cmd = "sudo rm -r "+wapitiPath+"wapiti"+host run_tool(cmd=cmd) except: pass try: if os.path.exists(skipfishPath+"skipfish"+host): print "deleting "+skipfishPath+"skipfish"+host cmd = "sudo rm -r "+skipfishPath+"skipfish"+host run_tool(cmd=cmd) except: pass try: if os.path.exists(cwd+"/Tools/skipfish/new_dict_"+host+".wl"): print cwd+"Tools/skipfish/new_dict_"+host+".wl" cmd="sudo rm -r "+cwd+"Tools/skipfish/new_dict_"+host+".wl" run_tool(cmd=cmd) except: pass logging.info("Wapiti web scanner started for: "+str(host)) try: print "running wapiti on "+prefix+host try: p = run_tool(process=True,name="wapiti",prefix=prefix,domain=host,path=wapitiPath+"wapiti"+host+"/",timeout=TIMEOUT) except: pass except: pass logging.info("Skipfish web scanner started for: "+str(host)) try: cmd="touch "+cwd+"/Tools/skipfish/new_dict_"+host+".wl" run_tool(cmd=cmd) except: pass try: print "running skipfish on "+prefix+host p1 = run_tool(process=True,name="skipfish",prefix=prefix,domain=host,path=skipfishPath+"skipfish"+host+"/",timeout=TIMEOUT) except Exception as e: logging.error("Skipfish not working. Issue: "+str(e)) st = timeit.default_timer() flag = True flag1 = True while 1: cur = timeit.default_timer() try: if flag and (p.poll() is not None or cur-st > TIMEOUT/8): flag = False count = 7 while count and p.poll() is None: count -= 1 time.sleep(TIMEOUT/8) try: p.send_signal(signal.SIGINT) except: pass vuln = db.vulnerabilities if vuln.find({"ip":ip}).count() == 0: vul = {"ip":ip,"domain":domain} vuln.insert_one(vul) vuln.update({"ip":ip},{"$set":{"wapiti":wapitiPath+"wapiti"+str(host)}}) logging.info("*************************Wapiti Finished*******************************") logging.info("Time Taken = "+str(cur-st)) except Exception as e: print "Issues with wapiti: "+str(e) flag = false if flag1 and p1.poll() is not None: flag1 = False vuln = db.vulnerabilities if vuln.find({"ip":ip}).count() == 0: vul = {"ip":ip,"domain":domain} vuln.insert_one(vul) vuln.update({"ip":ip},{"$set":{"skipfish":skipfishPath+"skipfish"+str(host)}}) logging.info("*************************Skipfish Finished******************************") logging.info("Time Taken = "+str(cur-st)) if flag == False and flag1 == False: break time.sleep(10) #update wapiti file try: if os.path.exists(wapitiPath+"wapiti"+str(host)+"/index.html"): f = open(wapitiPath+"wapiti"+str(host)+"/index.html","r") tt = f.readlines() f.close() f = open(wapitiPath+"wapiti"+str(host)+"/index.html","w") for i in range(len(tt)): if tt[i].strip() == "var nb_vulns = report_data[flaw_name][vuln_name].length;": f.write(tt[i]) f.write("if(nb_vulns == 0){\ncount++;\ncontinue;\n}\n") elif tt[i].strip() == "kube_tabs_js = document.createElement('script');": f.write("if(count == 11){\nvar newDiv = document.createElement('h1');\nvar vul = document.createElement('b');\nvar newContent = document.createTextNode('woohoo! you are free from this vulnerabilities');\nnewDiv.appendChild(newContent);\nnewDiv.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Cross Site Scripting'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Htaccess Bypass'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Backup file'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('SQL Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Blind SQL Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('File Handling'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Potentially dangerous file'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('CRLF Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Commands execution'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Resource consumption'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Internal Server Error'));\nvul.appendChild(document.createElement('br'));\nsummary.appendChild(newDiv);\nsummary.appendChild(vul);\n}\n") f.write(tt[i]) elif tt[i].strip() == "var vuln_count = 0;": f.write(tt[i]) f.write("var count = 0;\n") else: f.write(tt[i]) f.close() else: print "WAPITI FILE "+wapitiPath+"wapiti"+str(host)+"/index.html does not exisit" except Exception as e: print "Issues with wapiti file: "+str(e) if domain == "": domain = "null" countVulns(ip,domain,type)