def countVulns(ip, domain, type):
cwd = os.getcwd()
c = mongo_client()
Configure = c.config.external.find()[0]
#ip = sys.argv[1]
#domain = sys.argv[2]
#type = sys.argv[3]
if (type == "selfServe"):
DBName = Configure['SELF_SERVE_DATABASE']
else:
DBName = Configure['DATABASE']
collection = c[DBName].vulnerabilities
newCollection = []
serverAddress = "http://127.0.0.1"
data = collection.find_one({"ip": ip})
# print collection.find().count()
# print collection.find()[0]
url = data['skipfish']
fileUrl = serverAddress + str(url).split("html")[1] + "/index.html"
command = "phantomjs loadspeed.js " + fileUrl + " > temp/" + ip.replace(
".", "-") + ".html"
run_tool(cmd=command)
command = "python parser.py temp/" + ip.replace(
".", "-") + ".html > temp/" + ip.replace(".", "-") + ".txt"
run_tool(cmd=command)
f = open("temp/" + ip.replace(".", "-") + ".txt")
number = f.read().split("\n")[0]
c[DBName].vulnerabilities.update({"ip": ip}, {"$set": {"count": number}})
f.close()
def main(argv):
os.environ["LC_ALL"] = "en_US.UTF-8";
parser = argparse.ArgumentParser()
parser.add_argument("-c","--config",help="to configure db structure",action="store_true")
parser.add_argument("-iA","--inventory-append",help="to append target to IP Inventory")
parser.add_argument("-iR","--inventory-replace",help="to replace targets in IP Inventory")
parser.add_argument("-u","--updateCVEs",help="to configure or update CVE database", choices=['install','map','update'])
parser.add_argument("-s","--start",help="to start scaning engine",action="store_true")
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
if args.inventory_append:
fill_Inventory(args.inventory_append, action="append")
if args.inventory_replace:
fill_Inventory(args.inventory_replace,action="replace")
if args.config:
config_db()
if args.updateCVEs:
if args.updateCVEs == "install":
cmd = "python3 ./Tools/cve-search-master/sbin/db_mgmt.py -p"
run_tool(cmd,verbose=True)
if args.updateCVEs == "map":
cmd = "python3 Tools/cve-search-master/sbin/db_mgmt_cpe_dictionary.py"
run_tool(cmd,verbose=True)
if args.updateCVEs == "update":
cmd = "python3 Tools/cve-search-master/sbin/db_updater.py"
run_tool(cmd,verbose=True)
if args.start:
start_scan()
def technologyScan(ip, domain, type):
# config database
client = mongo_client()
db = client.config
cursor = db.external.find()
#type = sys.argv[3]
# checking for selfServe of org scan and setting the paramter
if (type == "selfServe"):
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
else:
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
TIMEOUT = int(cursor[0]['TIMEOUT_TECH'])
db = client[database]
# log file
logging.basicConfig(filename=logFile,
format='%(levelname)s:%(message)s',
level=logging.DEBUG)
# timeout
def signal_handler(signum, frame):
raise Exception("Timed Out!")
signal.signal(signal.SIGALRM, signal_handler)
#ip = sys.argv[1]
#domain = sys.argv[2]
w = Wappalyzer()
serv = db.services
if domain != "null":
host = domain # host is the parameter to be passed
else:
host = ip
if domain == "null":
domain = ""
# checking whether to scan through 80 or 443
if serv.find({"ip": ip, "443": {"$exists": True}}).count() > 0:
prefix = "https://"
elif serv.find({"ip": ip, "80": {"$exists": True}}).count() > 0:
prefix = "http://"
component = {}
# every 3rd party tools is scanning 6 times, if it finds the technology than it stops
# wappalyzer
count = 6
while (count):
if count <= 3:
host = ip # host is changed to ip after 3 scan
count -= 1
logging.info("Wappalyzer working on " + host)
signal.alarm(TIMEOUT)
try: # calling wappalyzer
wapp = w.analyze(prefix + host)
except Exception as e:
logging.error("Issues with wappalyzer: " + str(e))
signal.alarm(0)
continue
signal.alarm(0)
logging.info(wapp)
if len(wapp) == 0: # checking for output
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
for key in wapp:
component[key.lower()] = wapp[key][unicode('version')]
break
# builtwith
if domain != "":
host = domain
else:
host = ip
count = 6
while (count):
if count <= 3:
host = ip
count -= 1
logging.info("Builtwith working on " + host)
signal.alarm(TIMEOUT)
try: # builtwith working
bw = builtwith(prefix + host)
except Exception as e:
logging.error("Issues with builtwith: " + str(e))
signal.alarm(0)
continue
signal.alarm(0)
logging.info(bw)
if len(bw) == 0:
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
for keys in bw: # checking for output
for key in bw[keys]:
if key not in component.keys():
component[key.lower()] = ""
break
# phantalyzer
if domain != "":
host = domain
else:
host = ip
count = 6
while (count):
if count <= 3:
host = ip
count -= 1
logging.info("Phantalyzer working on " + host)
signal.alarm(TIMEOUT)
try:
phanta = run_tool(name="phantomjs", prefix=prefix, domain=host)
except Exception as e:
logging.error("Issue with phantalyzer: " + str(e))
signal.alarm(0)
try:
phanta = phanta[1]
phanta = phanta.strip()
logging.info(phanta)
if phanta == "":
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
phanta = phanta.split("\n")
phanta[0] = phanta[0].strip()
phanta = phanta[0].split(":")[1]
if phanta == "" or phanta.strip() == '160':
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
phanta = phanta.split("|")
for te in phanta:
te = te.strip()
if te not in component.keys() and te != "":
component[te.lower()] = ""
break
except Exception as e:
logging.error("Issue with phantalyzer: " + str(e))
# wappalyzer extension
if domain != "":
host = domain
else:
host = ip
count = 6
while (count):
if count <= 3:
host = ip
count -= 1
logging.info("Wappalyzer extension working on " + host)
signal.alarm(TIMEOUT)
try:
cmd = "phantomjs src/drivers/phantomjs/driver.js " + prefix + host
phantjs = run_tool(cmd=cmd)
except Exception as e:
logging.error("Issue with phantomjs code: " + str(e))
signal.alarm(0)
try:
logging.info(phantjs[1].strip())
if phantjs[1].strip() == "":
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 20 seconds.")
time.sleep(2)
continue
phantjs = json.loads(phantjs[1])
phantjs = phantjs['applications']
if len(phantjs) == 0:
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 20 seconds.")
time.sleep(20)
continue
for i in range(len(phantjs)):
if (phantjs[i][unicode('name')]
).lower() not in component.keys():
component[(phantjs[i][unicode('name')]
).lower()] = phantjs[i][unicode('version')]
elif component[(phantjs[i][unicode('name')]).lower()] == "":
component[(phantjs[i][unicode('name')]
).lower()] = phantjs[i][unicode('version')]
break
except Exception as e:
logging.error("Phantomjs code not working. Issues: " + str(e))
# finding cves
try:
for key in component:
temp = {}
temp['version'] = component[key]
allCve = []
if component[key] == "":
temp['cves'] = allCve
temp['false_positive'] = "0"
component[key] = temp
continue
cmd = "python3 Tools/cve-search-master/bin/search.py -p " + str(
key).lower().replace(" js", ".js").replace(" ", "_").replace(
"apache", "apache:http_server") + ":" + str(
component[key]) + " -o json"
cves = run_tool(cmd=cmd)
cves = cves[1]
size = len(cves.split("\n"))
if size == 1 and cves == "":
temp['cves'] = allCve
temp['false_positive'] = "0"
component[key] = temp
continue
for j in range(size):
cve = {}
tt = json.loads(cves.split("\n")[j])
cve['id'] = tt['id']
cve['cvss'] = tt['cvss']
allCve.append(cve)
temp['cves'] = allCve
temp['false_positive'] = "0"
component[key] = temp
except Exception as e:
logging.error("Issues with finding cves. Issues: " + str(e))
technologies = db.technologies
checking = technologies.find_one({"ip": ip})
if technologies.find({"ip": ip}).count() > 0:
technologies.remove({"ip": ip})
technology = {"ip": ip, "domain": domain}
technologies.insert_one(technology)
for key in component:
try:
for ch in checking:
if key.replace(".", " ") == ch.encode(
'ascii', 'ignore') and component[key][
'version'] == checking[ch]['version'].encode(
'ascii', 'ignore'):
component[key]['false_positive'] = checking[ch][
'false_positive']
except Exception as e:
print "Issues with updating false positive: " + str(e)
technologies.update(
{"ip": ip}, {"$set": {
str(key.replace(".", " ")): component[key]
}})
print key + " with version " + str(component[key])
def portScan(ip, domain, type):
# config file
client = mongo_client()
db = client.config
cursor = db.external.find()
# check for the selfServe or org scan
if (type == "selfServe"):
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
else:
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
db = client[database]
services = db.services
#log file
logging.basicConfig(filename=logFile,
format='%(levelname)s:%(message)s',
level=logging.DEBUG)
logging.info("finding Open Ports")
host = ip # to be passed as a parameter
ports = []
version = []
false_positive = []
try:
services = run_tool(name='nmap', ip=host) # nmap command
logging.info(services)
services = services[1].split(":")[2]
services = services.split(",")
for i in range(
len(services)): # each iteration will correspond to one port
services[i] = services[i].strip()
ports.append(services[i].split("/")[0])
version.append(services[i].split("//")[2].split("/")[0])
except Exception as e:
logging.error("Some issue with Nmap. Issue: " + str(e))
if len(ports) != 0: # calculating md5 sum and updating the database
md5 = ''.join(sorted(ports))
md5 = hashlib.md5(md5).hexdigest()
if domain == "null":
domain = ""
services = db.services
if services.find({"ip": ip, "md5": md5}).count() == 0:
for cnt in range(len(ports)):
false_positive.append("")
checking = services.find_one({"ip": ip})
if services.find({
"ip": ip
}).count(
) > 0: # if ports have changed then delete that entry from database
logging.info("Ports have changed.")
services.remove({"ip": ip})
else:
logging.info("This ip is scanned for the first time.")
service = {
"ip": ip,
"domain": domain,
"md5": md5
} # insert port details in database
services.insert_one(service)
for i in range(len(ports)): # setting false_positive
if checking != None:
for ch in checking:
if ports[i] == ch.encode(
'ascii', 'ignore'
) and version[i] == checking[ch]['version'].encode(
'ascii', 'ignore'):
false_positive[i] = checking[ch]['false_positive']
if false_positive[i] == "":
false_positive[i] = "0"
services.update({"ip": ip}, {
"$set": {
ports[i]: {
"version": version[i],
"false_positive": false_positive[i]
}
}
})
logging.info(str(ports[i]) + " has version " + str(version[i]))
else: # updating domain if it is not present before
if services.find_one({"ip":
ip})['domain'].encode('ascii',
'ignore') == "":
services.update({"ip": ip}, {"$set": {"domain": domain}})
logging.info("Same ports as before")
else: # if no port is identified
services = db.services
if services.find({"ip": ip}).count() == 0:
md5 = hashlib.md5('').hexdigest()
serv = {"ip": ip, "domain": domain, "md5": md5}
services.insert_one(serv)
def vulnerabilityScan(ip,domain,type):
cwd = os.getcwd()
client = mongo_client()
db = client.config
cursor = db.external.find()
# type = sys.argv[3]
if(type == "selfServe"):
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
skipfishPath = cursor[0]['SELF_SERVE_PATH_SKIPFISH']
wapitiPath = cursor[0]['SELF_SERVE_PATH_WAPITI']
else:
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
skipfishPath = cursor[0]['PATH_SKIPFISH']
wapitiPath = cursor[0]['PATH_WAPITI']
TIMEOUT = int(cursor[0]['TIMEOUT_VUL'])
db = client[database]
logging.basicConfig(filename = logFile,format='%(levelname)s:%(message)s',level=logging.DEBUG)
logging.info("Finding Vulnerability")
serv = db.services
# ip = sys.argv[1]
# domain = sys.argv[2]
host = ip
if domain == "null":
domain = ""
if serv.find({"ip":ip,"80":{"$exists":True}}).count() > 0:
prefix = "http://"
elif serv.find({"ip":ip,"443":{"$exists":True}}).count() > 0:
prefix = "https://"
try:
if os.path.exists(wapitiPath+"wapiti"+host):
print "deleting "+wapitiPath+"wapiti"+host
cmd = "sudo rm -r "+wapitiPath+"wapiti"+host
run_tool(cmd=cmd)
except:
pass
try:
if os.path.exists(skipfishPath+"skipfish"+host):
print "deleting "+skipfishPath+"skipfish"+host
cmd = "sudo rm -r "+skipfishPath+"skipfish"+host
run_tool(cmd=cmd)
except:
pass
try:
if os.path.exists(cwd+"/Tools/skipfish/new_dict_"+host+".wl"):
print cwd+"Tools/skipfish/new_dict_"+host+".wl"
cmd="sudo rm -r "+cwd+"Tools/skipfish/new_dict_"+host+".wl"
run_tool(cmd=cmd)
except:
pass
logging.info("Wapiti web scanner started for: "+str(host))
try:
print "running wapiti on "+prefix+host
try:
p = run_tool(process=True,name="wapiti",prefix=prefix,domain=host,path=wapitiPath+"wapiti"+host+"/",timeout=TIMEOUT)
except:
pass
except:
pass
logging.info("Skipfish web scanner started for: "+str(host))
try:
cmd="touch "+cwd+"/Tools/skipfish/new_dict_"+host+".wl"
run_tool(cmd=cmd)
except:
pass
try:
print "running skipfish on "+prefix+host
p1 = run_tool(process=True,name="skipfish",prefix=prefix,domain=host,path=skipfishPath+"skipfish"+host+"/",timeout=TIMEOUT)
except Exception as e:
logging.error("Skipfish not working. Issue: "+str(e))
st = timeit.default_timer()
flag = True
flag1 = True
while 1:
cur = timeit.default_timer()
try:
if flag and (p.poll() is not None or cur-st > TIMEOUT/8):
flag = False
count = 7
while count and p.poll() is None:
count -= 1
time.sleep(TIMEOUT/8)
try:
p.send_signal(signal.SIGINT)
except:
pass
vuln = db.vulnerabilities
if vuln.find({"ip":ip}).count() == 0:
vul = {"ip":ip,"domain":domain}
vuln.insert_one(vul)
vuln.update({"ip":ip},{"$set":{"wapiti":wapitiPath+"wapiti"+str(host)}})
logging.info("*************************Wapiti Finished*******************************")
logging.info("Time Taken = "+str(cur-st))
except Exception as e:
print "Issues with wapiti: "+str(e)
flag = false
if flag1 and p1.poll() is not None:
flag1 = False
vuln = db.vulnerabilities
if vuln.find({"ip":ip}).count() == 0:
vul = {"ip":ip,"domain":domain}
vuln.insert_one(vul)
vuln.update({"ip":ip},{"$set":{"skipfish":skipfishPath+"skipfish"+str(host)}})
logging.info("*************************Skipfish Finished******************************")
logging.info("Time Taken = "+str(cur-st))
if flag == False and flag1 == False:
break
time.sleep(10)
#update wapiti file
try:
if os.path.exists(wapitiPath+"wapiti"+str(host)+"/index.html"):
f = open(wapitiPath+"wapiti"+str(host)+"/index.html","r")
tt = f.readlines()
f.close()
f = open(wapitiPath+"wapiti"+str(host)+"/index.html","w")
for i in range(len(tt)):
if tt[i].strip() == "var nb_vulns = report_data[flaw_name][vuln_name].length;":
f.write(tt[i])
f.write("if(nb_vulns == 0){\ncount++;\ncontinue;\n}\n")
elif tt[i].strip() == "kube_tabs_js = document.createElement('script');":
f.write("if(count == 11){\nvar newDiv = document.createElement('h1');\nvar vul = document.createElement('b');\nvar newContent = document.createTextNode('woohoo! you are free from this vulnerabilities');\nnewDiv.appendChild(newContent);\nnewDiv.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Cross Site Scripting'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Htaccess Bypass'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Backup file'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('SQL Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Blind SQL Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('File Handling'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Potentially dangerous file'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('CRLF Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Commands execution'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Resource consumption'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Internal Server Error'));\nvul.appendChild(document.createElement('br'));\nsummary.appendChild(newDiv);\nsummary.appendChild(vul);\n}\n")
f.write(tt[i])
elif tt[i].strip() == "var vuln_count = 0;":
f.write(tt[i])
f.write("var count = 0;\n")
else:
f.write(tt[i])
f.close()
else:
print "WAPITI FILE "+wapitiPath+"wapiti"+str(host)+"/index.html does not exisit"
except Exception as e:
print "Issues with wapiti file: "+str(e)
if domain == "":
domain = "null"
countVulns(ip,domain,type)