def config_db(argv):
c = mongo_client()
db = c.config
db.external.remove()
if len(sys.argv) < 3:
web_dir = '/var/www/html'
else:
web_dir = sys.argv[2]
print web_dir
db.external.insert_one({
'SELF_SERVE_PATH_SKIPFISH':
web_dir + '/reports/selfServeOutputSkipfish/',
'SELF_SERVE_DATABASE': 'selfTestingExternal',
'PATH_WAPITI': web_dir + '/reports/outputWapiti/',
'DATABASE': 'testingExternal',
'SELF_SERVE_PATH_WAPITI': web_dir + '/reports/selfServeOutputWapiti/',
'TIMEOUT_VUL': '180',
'TIMEOUT_TECH': '30',
'PATH_LOGFILE': 'external.log',
'PATH_SKIPFISH': web_dir + '/reports/outputSkipfish/',
'SELF_SERVE_PATH_LOGFILE': 'selfServe.log'
})
return web_dir
def start_scan():
# config database
client = mongo_client()
db = client.config
# select configurations
cursor = db.external.find()
if not cursor or cursor.count() == 0:
print "[!] Please configure the DB first by running below command."
print "\t$ docker run -t -i --rm --link <your mongo container name>:mongodb watchdog -c"
sys.exit(1)
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
db = client[database]
# log file
logging.basicConfig(filename = logFile,filemode = 'w',format='%(levelname)s:%(message)s',level=logging.DEBUG)
# ips
cursor = db.ipInventory.find()
print "Total IPs in database: "+str(cursor.count())
# each iteration corresponds to one ip
for document in cursor:
ip = document['ip']
domain = document['domain']
if domain == "":
domain = "null"
serv = db.services
#os.system("python findPorts.py "+str(ip)+" "+str(domain)+" org") # call findPorts
print "starting portscan"
portScan(str(ip),str(domain),"org")
#checking whether 80 or 443 is open
if serv.find({"ip":ip,"80":{"$exists":True}}).count() > 0 or serv.find({"ip":ip,"443":{"$exists":True}}).count() > 0:
logging.info("Finding Vulnerability")
# p = Popen("sudo python findVulnerability.py "+str(ip)+" "+str(domain)+" org",shell = True) # call findVulnerability
vulnerabilityScan(str(ip),str(domain),"org")
technologyScan(str(ip),str(domain),"org")
#os.system("python findTechnology.py "+str(ip)+" "+str(domain)+" org") # call findTechnology
else:
logging.info("Both 80 and 443 are closed corresponding to ip: "+str(ip))
def config_db(argv):
c = mongo_client()
db = c.config
db.external.remove()
if len(sys.argv) < 3:
web_dir = '/var/www/html'
else:
web_dir = sys.argv[2]
print web_dir
db.external.insert_one({
'PATH_WAPITI': web_dir+'/reports/outputWapiti/',
'DATABASE': 'testingExternal',
'TIMEOUT_VUL': '180',
'TIMEOUT_TECH': '30',
'PATH_LOGFILE': 'external.log',
'PATH_SKIPFISH': web_dir+'/reports/outputSkipfish/',
})
db.internal.insert_one({
'REPO_PATH':"",
'SCALE_CRITICAL_VUL_CVE_EXTERNAL':8.3,
'SELF_SERVE_DATABASE':'Self_Serve_Internal_Testing',
'SCALE_MEDIUM_VUL_CVE_EXTERNAL':5.0,
'DATABASE':'Internal_Testing',
'ORGANISATION':'<>',
'GIT_TOKEN':'<>:<>',
'SCALE_MEDIUM_VUL_CVE_INTERNAL':5.0,
'DEPENDENCY_PATH_LOGFILE':'dependency_log.log',
'SELF_SERVE_SOURCE_PATH_LOGFILE':'self_serve_source_log.log',
'SELF_SERVE_DEPENDENCY_PATH_LOGFILE':'self_serve_dependency_log.log',
'SOURCE_PATH_LOGFILE':'source_log.log',
'SELF_SERVE_TIME_OUT':30,
'TIME_OUT':30,
'SCALE_HIGH_VUL_CVE_INTERNAL':7.5,
'SCALE_HIGH_VUL_CVE_EXTERNAL':7.5,
'SCALE_CRITICAL_VUL_CVE_INTERNAL':8.3,
'SCALE_HIGH_VUL_REPO':8.3
})
return web_dir
def fill_Inventory(filename, action=None):
client = mongo_client()
db = client.testingExternal
if not filename:
filename = 'subdomains.txt'
lines = ""
with open(filename) as f:
lines = f.readlines()
if f:
f.close()
if action == "replace":
db.ipInventory.remove()
i = 1
for line in lines:
if line.strip() != "":
domain = {}
try:
ip = socket.gethostbyname(line.strip())
domain['ip'] = ip
domain['domain'] = line.strip()
print str(i) + ". ",
print domain,
count = db.ipInventory.count({
'domain': line.strip(),
'ip': ip
})
if count == 0:
print db.ipInventory.insert(domain)
else:
print "Already in DB"
i = i + 1
except Exception as ae:
print ae
pass
def countVulns(ip, domain, type):
cwd = os.getcwd()
c = mongo_client()
Configure = c.config.external.find()[0]
#ip = sys.argv[1]
#domain = sys.argv[2]
#type = sys.argv[3]
if (type == "selfServe"):
DBName = Configure['SELF_SERVE_DATABASE']
else:
DBName = Configure['DATABASE']
collection = c[DBName].vulnerabilities
newCollection = []
serverAddress = "http://127.0.0.1"
data = collection.find_one({"ip": ip})
# print collection.find().count()
# print collection.find()[0]
url = data['skipfish']
fileUrl = serverAddress + str(url).split("html")[1] + "/index.html"
command = "phantomjs loadspeed.js " + fileUrl + " > temp/" + ip.replace(
".", "-") + ".html"
run_tool(cmd=command)
command = "python parser.py temp/" + ip.replace(
".", "-") + ".html > temp/" + ip.replace(".", "-") + ".txt"
run_tool(cmd=command)
f = open("temp/" + ip.replace(".", "-") + ".txt")
number = f.read().split("\n")[0]
c[DBName].vulnerabilities.update({"ip": ip}, {"$set": {"count": number}})
f.close()
def technologyScan(ip, domain, type):
# config database
client = mongo_client()
db = client.config
cursor = db.external.find()
#type = sys.argv[3]
# checking for selfServe of org scan and setting the paramter
if (type == "selfServe"):
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
else:
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
TIMEOUT = int(cursor[0]['TIMEOUT_TECH'])
db = client[database]
# log file
logging.basicConfig(filename=logFile,
format='%(levelname)s:%(message)s',
level=logging.DEBUG)
# timeout
def signal_handler(signum, frame):
raise Exception("Timed Out!")
signal.signal(signal.SIGALRM, signal_handler)
#ip = sys.argv[1]
#domain = sys.argv[2]
w = Wappalyzer()
serv = db.services
if domain != "null":
host = domain # host is the parameter to be passed
else:
host = ip
if domain == "null":
domain = ""
# checking whether to scan through 80 or 443
if serv.find({"ip": ip, "443": {"$exists": True}}).count() > 0:
prefix = "https://"
elif serv.find({"ip": ip, "80": {"$exists": True}}).count() > 0:
prefix = "http://"
component = {}
# every 3rd party tools is scanning 6 times, if it finds the technology than it stops
# wappalyzer
count = 6
while (count):
if count <= 3:
host = ip # host is changed to ip after 3 scan
count -= 1
logging.info("Wappalyzer working on " + host)
signal.alarm(TIMEOUT)
try: # calling wappalyzer
wapp = w.analyze(prefix + host)
except Exception as e:
logging.error("Issues with wappalyzer: " + str(e))
signal.alarm(0)
continue
signal.alarm(0)
logging.info(wapp)
if len(wapp) == 0: # checking for output
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
for key in wapp:
component[key.lower()] = wapp[key][unicode('version')]
break
# builtwith
if domain != "":
host = domain
else:
host = ip
count = 6
while (count):
if count <= 3:
host = ip
count -= 1
logging.info("Builtwith working on " + host)
signal.alarm(TIMEOUT)
try: # builtwith working
bw = builtwith(prefix + host)
except Exception as e:
logging.error("Issues with builtwith: " + str(e))
signal.alarm(0)
continue
signal.alarm(0)
logging.info(bw)
if len(bw) == 0:
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
for keys in bw: # checking for output
for key in bw[keys]:
if key not in component.keys():
component[key.lower()] = ""
break
# phantalyzer
if domain != "":
host = domain
else:
host = ip
count = 6
while (count):
if count <= 3:
host = ip
count -= 1
logging.info("Phantalyzer working on " + host)
signal.alarm(TIMEOUT)
try:
phanta = run_tool(name="phantomjs", prefix=prefix, domain=host)
except Exception as e:
logging.error("Issue with phantalyzer: " + str(e))
signal.alarm(0)
try:
phanta = phanta[1]
phanta = phanta.strip()
logging.info(phanta)
if phanta == "":
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
phanta = phanta.split("\n")
phanta[0] = phanta[0].strip()
phanta = phanta[0].split(":")[1]
if phanta == "" or phanta.strip() == '160':
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 10 seconds.")
time.sleep(10)
continue
phanta = phanta.split("|")
for te in phanta:
te = te.strip()
if te not in component.keys() and te != "":
component[te.lower()] = ""
break
except Exception as e:
logging.error("Issue with phantalyzer: " + str(e))
# wappalyzer extension
if domain != "":
host = domain
else:
host = ip
count = 6
while (count):
if count <= 3:
host = ip
count -= 1
logging.info("Wappalyzer extension working on " + host)
signal.alarm(TIMEOUT)
try:
cmd = "phantomjs src/drivers/phantomjs/driver.js " + prefix + host
phantjs = run_tool(cmd=cmd)
except Exception as e:
logging.error("Issue with phantomjs code: " + str(e))
signal.alarm(0)
try:
logging.info(phantjs[1].strip())
if phantjs[1].strip() == "":
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 20 seconds.")
time.sleep(2)
continue
phantjs = json.loads(phantjs[1])
phantjs = phantjs['applications']
if len(phantjs) == 0:
logging.info("No output.")
if count != 0:
logging.info("Sleeping for 20 seconds.")
time.sleep(20)
continue
for i in range(len(phantjs)):
if (phantjs[i][unicode('name')]
).lower() not in component.keys():
component[(phantjs[i][unicode('name')]
).lower()] = phantjs[i][unicode('version')]
elif component[(phantjs[i][unicode('name')]).lower()] == "":
component[(phantjs[i][unicode('name')]
).lower()] = phantjs[i][unicode('version')]
break
except Exception as e:
logging.error("Phantomjs code not working. Issues: " + str(e))
# finding cves
try:
for key in component:
temp = {}
temp['version'] = component[key]
allCve = []
if component[key] == "":
temp['cves'] = allCve
temp['false_positive'] = "0"
component[key] = temp
continue
cmd = "python3 Tools/cve-search-master/bin/search.py -p " + str(
key).lower().replace(" js", ".js").replace(" ", "_").replace(
"apache", "apache:http_server") + ":" + str(
component[key]) + " -o json"
cves = run_tool(cmd=cmd)
cves = cves[1]
size = len(cves.split("\n"))
if size == 1 and cves == "":
temp['cves'] = allCve
temp['false_positive'] = "0"
component[key] = temp
continue
for j in range(size):
cve = {}
tt = json.loads(cves.split("\n")[j])
cve['id'] = tt['id']
cve['cvss'] = tt['cvss']
allCve.append(cve)
temp['cves'] = allCve
temp['false_positive'] = "0"
component[key] = temp
except Exception as e:
logging.error("Issues with finding cves. Issues: " + str(e))
technologies = db.technologies
checking = technologies.find_one({"ip": ip})
if technologies.find({"ip": ip}).count() > 0:
technologies.remove({"ip": ip})
technology = {"ip": ip, "domain": domain}
technologies.insert_one(technology)
for key in component:
try:
for ch in checking:
if key.replace(".", " ") == ch.encode(
'ascii', 'ignore') and component[key][
'version'] == checking[ch]['version'].encode(
'ascii', 'ignore'):
component[key]['false_positive'] = checking[ch][
'false_positive']
except Exception as e:
print "Issues with updating false positive: " + str(e)
technologies.update(
{"ip": ip}, {"$set": {
str(key.replace(".", " ")): component[key]
}})
print key + " with version " + str(component[key])
def portScan(ip, domain, type):
# config file
client = mongo_client()
db = client.config
cursor = db.external.find()
# check for the selfServe or org scan
if (type == "selfServe"):
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
else:
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
db = client[database]
services = db.services
#log file
logging.basicConfig(filename=logFile,
format='%(levelname)s:%(message)s',
level=logging.DEBUG)
logging.info("finding Open Ports")
host = ip # to be passed as a parameter
ports = []
version = []
false_positive = []
try:
services = run_tool(name='nmap', ip=host) # nmap command
logging.info(services)
services = services[1].split(":")[2]
services = services.split(",")
for i in range(
len(services)): # each iteration will correspond to one port
services[i] = services[i].strip()
ports.append(services[i].split("/")[0])
version.append(services[i].split("//")[2].split("/")[0])
except Exception as e:
logging.error("Some issue with Nmap. Issue: " + str(e))
if len(ports) != 0: # calculating md5 sum and updating the database
md5 = ''.join(sorted(ports))
md5 = hashlib.md5(md5).hexdigest()
if domain == "null":
domain = ""
services = db.services
if services.find({"ip": ip, "md5": md5}).count() == 0:
for cnt in range(len(ports)):
false_positive.append("")
checking = services.find_one({"ip": ip})
if services.find({
"ip": ip
}).count(
) > 0: # if ports have changed then delete that entry from database
logging.info("Ports have changed.")
services.remove({"ip": ip})
else:
logging.info("This ip is scanned for the first time.")
service = {
"ip": ip,
"domain": domain,
"md5": md5
} # insert port details in database
services.insert_one(service)
for i in range(len(ports)): # setting false_positive
if checking != None:
for ch in checking:
if ports[i] == ch.encode(
'ascii', 'ignore'
) and version[i] == checking[ch]['version'].encode(
'ascii', 'ignore'):
false_positive[i] = checking[ch]['false_positive']
if false_positive[i] == "":
false_positive[i] = "0"
services.update({"ip": ip}, {
"$set": {
ports[i]: {
"version": version[i],
"false_positive": false_positive[i]
}
}
})
logging.info(str(ports[i]) + " has version " + str(version[i]))
else: # updating domain if it is not present before
if services.find_one({"ip":
ip})['domain'].encode('ascii',
'ignore') == "":
services.update({"ip": ip}, {"$set": {"domain": domain}})
logging.info("Same ports as before")
else: # if no port is identified
services = db.services
if services.find({"ip": ip}).count() == 0:
md5 = hashlib.md5('').hexdigest()
serv = {"ip": ip, "domain": domain, "md5": md5}
services.insert_one(serv)
import os
from subprocess import Popen
from mongo_connection import mongo_client
import logging
import time, datetime
client = mongo_client()
db = client.config
cursor = db.external.find()
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
db = client[database]
logging.basicConfig(filename=logFile,
filemode='w',
format='%(levelname)s:%(message)s',
level=logging.DEBUG)
cursor = db.onGoingIp.find()
for document in cursor:
ip = document['ip']
domain = document['domain']
if domain == "":
domain = "null"
serv = db.services
os.system("python findPorts.py " + str(ip) + " " + str(domain) +
" selfServe")
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
db.services.update({"ip": ip}, {"$set": {"time": st}})
if serv.find({
def vulnerabilityScan(ip,domain,type):
cwd = os.getcwd()
client = mongo_client()
db = client.config
cursor = db.external.find()
# type = sys.argv[3]
if(type == "selfServe"):
logFile = cursor[0]['SELF_SERVE_PATH_LOGFILE']
database = cursor[0]['SELF_SERVE_DATABASE']
skipfishPath = cursor[0]['SELF_SERVE_PATH_SKIPFISH']
wapitiPath = cursor[0]['SELF_SERVE_PATH_WAPITI']
else:
logFile = cursor[0]['PATH_LOGFILE']
database = cursor[0]['DATABASE']
skipfishPath = cursor[0]['PATH_SKIPFISH']
wapitiPath = cursor[0]['PATH_WAPITI']
TIMEOUT = int(cursor[0]['TIMEOUT_VUL'])
db = client[database]
logging.basicConfig(filename = logFile,format='%(levelname)s:%(message)s',level=logging.DEBUG)
logging.info("Finding Vulnerability")
serv = db.services
# ip = sys.argv[1]
# domain = sys.argv[2]
host = ip
if domain == "null":
domain = ""
if serv.find({"ip":ip,"80":{"$exists":True}}).count() > 0:
prefix = "http://"
elif serv.find({"ip":ip,"443":{"$exists":True}}).count() > 0:
prefix = "https://"
try:
if os.path.exists(wapitiPath+"wapiti"+host):
print "deleting "+wapitiPath+"wapiti"+host
cmd = "sudo rm -r "+wapitiPath+"wapiti"+host
run_tool(cmd=cmd)
except:
pass
try:
if os.path.exists(skipfishPath+"skipfish"+host):
print "deleting "+skipfishPath+"skipfish"+host
cmd = "sudo rm -r "+skipfishPath+"skipfish"+host
run_tool(cmd=cmd)
except:
pass
try:
if os.path.exists(cwd+"/Tools/skipfish/new_dict_"+host+".wl"):
print cwd+"Tools/skipfish/new_dict_"+host+".wl"
cmd="sudo rm -r "+cwd+"Tools/skipfish/new_dict_"+host+".wl"
run_tool(cmd=cmd)
except:
pass
logging.info("Wapiti web scanner started for: "+str(host))
try:
print "running wapiti on "+prefix+host
try:
p = run_tool(process=True,name="wapiti",prefix=prefix,domain=host,path=wapitiPath+"wapiti"+host+"/",timeout=TIMEOUT)
except:
pass
except:
pass
logging.info("Skipfish web scanner started for: "+str(host))
try:
cmd="touch "+cwd+"/Tools/skipfish/new_dict_"+host+".wl"
run_tool(cmd=cmd)
except:
pass
try:
print "running skipfish on "+prefix+host
p1 = run_tool(process=True,name="skipfish",prefix=prefix,domain=host,path=skipfishPath+"skipfish"+host+"/",timeout=TIMEOUT)
except Exception as e:
logging.error("Skipfish not working. Issue: "+str(e))
st = timeit.default_timer()
flag = True
flag1 = True
while 1:
cur = timeit.default_timer()
try:
if flag and (p.poll() is not None or cur-st > TIMEOUT/8):
flag = False
count = 7
while count and p.poll() is None:
count -= 1
time.sleep(TIMEOUT/8)
try:
p.send_signal(signal.SIGINT)
except:
pass
vuln = db.vulnerabilities
if vuln.find({"ip":ip}).count() == 0:
vul = {"ip":ip,"domain":domain}
vuln.insert_one(vul)
vuln.update({"ip":ip},{"$set":{"wapiti":wapitiPath+"wapiti"+str(host)}})
logging.info("*************************Wapiti Finished*******************************")
logging.info("Time Taken = "+str(cur-st))
except Exception as e:
print "Issues with wapiti: "+str(e)
flag = false
if flag1 and p1.poll() is not None:
flag1 = False
vuln = db.vulnerabilities
if vuln.find({"ip":ip}).count() == 0:
vul = {"ip":ip,"domain":domain}
vuln.insert_one(vul)
vuln.update({"ip":ip},{"$set":{"skipfish":skipfishPath+"skipfish"+str(host)}})
logging.info("*************************Skipfish Finished******************************")
logging.info("Time Taken = "+str(cur-st))
if flag == False and flag1 == False:
break
time.sleep(10)
#update wapiti file
try:
if os.path.exists(wapitiPath+"wapiti"+str(host)+"/index.html"):
f = open(wapitiPath+"wapiti"+str(host)+"/index.html","r")
tt = f.readlines()
f.close()
f = open(wapitiPath+"wapiti"+str(host)+"/index.html","w")
for i in range(len(tt)):
if tt[i].strip() == "var nb_vulns = report_data[flaw_name][vuln_name].length;":
f.write(tt[i])
f.write("if(nb_vulns == 0){\ncount++;\ncontinue;\n}\n")
elif tt[i].strip() == "kube_tabs_js = document.createElement('script');":
f.write("if(count == 11){\nvar newDiv = document.createElement('h1');\nvar vul = document.createElement('b');\nvar newContent = document.createTextNode('woohoo! you are free from this vulnerabilities');\nnewDiv.appendChild(newContent);\nnewDiv.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Cross Site Scripting'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Htaccess Bypass'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Backup file'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('SQL Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Blind SQL Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('File Handling'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Potentially dangerous file'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('CRLF Injection'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Commands execution'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Resource consumption'));\nvul.appendChild(document.createElement('br'));\nvul.appendChild(document.createTextNode('Internal Server Error'));\nvul.appendChild(document.createElement('br'));\nsummary.appendChild(newDiv);\nsummary.appendChild(vul);\n}\n")
f.write(tt[i])
elif tt[i].strip() == "var vuln_count = 0;":
f.write(tt[i])
f.write("var count = 0;\n")
else:
f.write(tt[i])
f.close()
else:
print "WAPITI FILE "+wapitiPath+"wapiti"+str(host)+"/index.html does not exisit"
except Exception as e:
print "Issues with wapiti file: "+str(e)
if domain == "":
domain = "null"
countVulns(ip,domain,type)
