def process_nmap_data2(nmap_report, workspace, target=None):
for scanned_host in nmap_report.hosts:
#print(scanned_host)
ip = scanned_host.id
#print(ip)
if (IPAddress(ip) == target) or (target is None):
#has_vhost_been_scanned = db.get_unique_inscope_vhosts_for_ip(ip,workspace)
has_vhost_been_scanned = db.get_inscope_submitted_vhosts_for_ip(
ip, workspace)
if has_vhost_been_scanned:
answer = raw_input(
"[!] {0} has already been scanned. Scan again? [Y\\n] ".
format(ip))
if (answer == "Y") or (answer == "y") or (answer == ""):
db.update_vhosts_submitted(ip, ip, workspace, 0)
else:
db_vhost = (ip, ip, 1, 0, workspace
) # in this mode all vhosts are in scope
#print(db_vhost)
db.create_vhost(db_vhost)
for scanned_service_item in scanned_host.services:
if scanned_service_item.state == "open":
scanned_service_port = scanned_service_item.port
scanned_service_name = scanned_service_item.service
scanned_service_protocol = scanned_service_item.protocol
if scanned_service_item.tunnel == 'ssl':
scanned_service_name = 'https'
db_service = db.get_service(ip, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
db_string = (ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
db.create_service(db_string)
else:
db.update_service(ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
#Not using this yet, but I'd like to do send this to searchsploit
try:
scanned_service_product = scanned_service_item.service_dict[
'product']
except:
scanned_service_product = ''
try:
scanned_service_version = scanned_service_item.service_dict[
'version']
except:
scanned_service_version = ''
try:
scanned_service_extrainfo = scanned_service_item.service_dict[
'extrainfo']
except:
scanned_service_extrainfo = ''
def determine_if_domains_are_in_scope(vhosts, process_domain_tuple):
command_name, populated_command, output_base_dir, workspace, domain, simulation, celery_path, scan_mode = process_domain_tuple
config, supported_services = config_parser.read_config_ini()
vhosts = vhosts.splitlines()
# from https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
for vhost in vhosts:
#print("raw:\t" + vhost)
vhost = ansi_escape.sub('', vhost)
#print("escaped:\t" + vhost)
if re.match(r'\w', vhost):
in_scope, ip = utils.domain_scope_checker(vhost, workspace)
if in_scope == 1:
print("Found subdomain (in scope):\t" + vhost)
db_vhost = (ip, vhost, 1, 0, 0, workspace)
db.create_vhost(db_vhost)
else:
print("Found subdomain (out of scope):\t" + vhost)
db_vhost = (ip, vhost, 0, 0, 0, workspace)
db.create_vhost(db_vhost)
def post_process_domains_bb(vhosts, command_name, populated_command, output_base_dir, workspace, simulation,
celery_path,out_of_scope_hosts):
config, supported_services = config_parser.read_config_ini()
vhosts = vhosts.splitlines()
# from https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
for vhost in vhosts:
# print("raw:\t" + vhost)
vhost = ansi_escape.sub('', vhost)
# print("escaped:\t" + vhost)
if re.match(r'\w', vhost):
try:
ip = socket.gethostbyname(vhost)
if vhost not in out_of_scope_hosts:
print("Found subdomain (in scope):\t" + vhost)
db_vhost = (ip, vhost, 1, 0, workspace)
db.create_vhost(db_vhost)
else:
print("Found subdomain (out of scope):\t" + vhost)
db_vhost = (ip, vhost, 0, 0, workspace)
db.create_vhost(db_vhost)
except:
print("1There was an issue running the nmap scan against {0}.").format(vhost)
ip = ""
db_vhost = (ip, vhost, 0, 0, workspace) # not in scope if no IP
print(db_vhost)
db.create_vhost(db_vhost)
# pull all in scope vhosts that have not been submitted
inscope_vhosts = db.get_inscope_unsubmitted_vhosts(workspace)
for scannable_vhost in inscope_vhosts:
scannable_vhost = scannable_vhost[0]
ip = db.get_vhost_ip(scannable_vhost, workspace)
ip = ip[0][0]
print("I'm going to scan: " + scannable_vhost + ":" + ip)
db_scanned_services = db.get_all_services_for_ip(ip, workspace)
for (
id, ip, scanned_service_port, scanned_service_protocol, scanned_service_name, workspace) in db_scanned_services:
# run chain on each one and then update db as submitted
scan_output_base_file_name = output_base_dir + "/" + ip + "/celerystalkOutput/" + scannable_vhost + "_" + str(
scanned_service_port) + "_" + scanned_service_protocol + "_"
host_dir = output_base_dir + "/" + ip
# TODO: This def might introduce a bug - same code as parse config submit jobs to celery. need to just call that function here
for section in config.sections():
if (section == "http") or (section == "https"):
if section == scanned_service_name:
for (cmd_name, cmd) in config.items(section):
outfile = scan_output_base_file_name + cmd_name
populated_command = cmd.replace("[TARGET]", scannable_vhost)\
.replace("[PORT]",str(scanned_service_port))\
.replace("[OUTPUT]", outfile) \
.replace("[PATH]", "")
if simulation:
# debug - sends jobs to celery, but with a # in front of every one.
populated_command = "#" + populated_command
# Grab a UUID from celery.utils so that i can assign it to my task at init, which is amazing because
# that allows me to pass it to all of the tasks in the chain.
task_id = uuid()
result = chain(
cel_create_task.subtask(args=(cmd_name,populated_command, scannable_vhost, outfile + ".txt",workspace, task_id)),
run_cmd.si(cmd_name, populated_command, celery_path, task_id).set(task_id=task_id),
post_process.si(cmd_name, populated_command, output_base_dir, workspace, scannable_vhost,
host_dir,
simulation,
scanned_service_port, scanned_service_name,
scanned_service_protocol, celery_path),
)()
host_audit_log = host_dir + "/" + "{0}_executed_commands.txt".format(ip)
f = open(host_audit_log, 'a')
f.write(populated_command + "\n\n")
f.close()
db.update_vhosts_submitted(ip, scannable_vhost, workspace, 1)
def post_process_domains(vhosts,command_name,populated_command,output_base_dir,workspace,domain,simulation,celery_path,scan_mode):
config,supported_services = config_parser.read_config_ini()
vhosts = vhosts.splitlines()
# from https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
for vhost in vhosts:
#print("raw:\t" + vhost)
vhost = ansi_escape.sub('', vhost)
#print("escaped:\t" + vhost)
if re.match(r'\w', vhost):
in_scope,ip = utils.domain_scope_checker(vhost,workspace)
if in_scope == 1:
print("Found subdomain (in scope):\t" + vhost)
db_vhost = (ip,vhost,1, 0,workspace)
db.create_vhost(db_vhost)
else:
print("Found subdomain (out of scope):\t" + vhost)
db_vhost = (ip, vhost, 0, 0, workspace)
db.create_vhost(db_vhost)
# elif scan_mode == "BB":
#
# cmd_name, cmd = config['nmap-bug-bounty_mode']
#
# utils.
#
# db_vhost = ("", vhost, 1, 0, workspace)
# db.create_vhost(db_vhost)
#pull all in scope vhosts that have not been submitted
inscope_vhosts = db.get_inscope_unsubmitted_vhosts(workspace)
for scannable_vhost in inscope_vhosts:
scannable_vhost = scannable_vhost[0]
ip = db.get_vhost_ip(scannable_vhost,workspace)
ip = ip[0][0]
db_scanned_services = db.get_all_services_for_ip(ip, workspace)
for (id,ip,scanned_service_port,scanned_service_protocol,scanned_service_name,workspace) in db_scanned_services:
#run chain on each one and then update db as submitted
scan_output_base_file_name = output_base_dir + "/" + ip + "/celerystalkOutput/" + scannable_vhost + "_" + str(scanned_service_port) + "_" + scanned_service_protocol + "_"
host_dir = output_base_dir + "/" + ip
#TODO: This def might introduce a bug - same code as parse config submit jobs to celery. need to just call that function here
for section in config.sections():
if (section == "http") or (section == "https"):
if section == scanned_service_name:
for (cmd_name, cmd) in config.items(section):
outfile = scan_output_base_file_name + cmd_name
populated_command = cmd.replace("[TARGET]", scannable_vhost).replace("[PORT]",
str(scanned_service_port)).replace("[OUTPUT]", outfile).replace("[PATH]", "")
if simulation:
# debug - sends jobs to celery, but with a # in front of every one.
populated_command = "#" + populated_command
# Grab a UUID from celery.utils so that i can assign it to my task at init, which is amazing because
# that allows me to pass it to all of the tasks in the chain.
task_id = uuid()
result = chain(
# insert a row into the database to mark the task as submitted. a subtask does not get tracked
# in celery the same way a task does, for instance, you can't find it in flower
cel_create_task.subtask(args=(cmd_name,populated_command, scannable_vhost, outfile + ".txt", workspace, task_id)),
# run the command. run_task takes care of marking the task as started and then completed.
# The si tells run_cmd to ignore the data returned from a previous task
run_cmd.si(cmd_name, populated_command, celery_path, task_id).set(task_id=task_id),
# right now, every executed command gets sent to a generic post_process task that can do
# additinoal stuff based on the command that just ran.
post_process.si(cmd_name, populated_command, output_base_dir, workspace, scannable_vhost, host_dir,
simulation,
scanned_service_port, scanned_service_name,
scanned_service_protocol,celery_path),
)() # .apply_async()
#task_id_list.append(result.task_id)
host_audit_log = host_dir + "/" + "{0}_executed_commands.txt".format(ip)
f = open(host_audit_log, 'a')
f.write(populated_command + "\n\n")
f.close()
db.update_vhosts_submitted(ip,scannable_vhost,workspace,1)
def process_url(url, workspace, output_dir, arguments):
celery_path = sys.path[0]
config, supported_services = config_parser.read_config_ini()
task_id_list = []
urls_to_screenshot = []
simulation = arguments["--simulation"]
try:
parsed_url = urlparse.urlparse(url)
scheme = parsed_url[0]
if not scheme:
print(
"\n[!] URL parameter (-u) requires that you specify the scheme (http:// or https://)\n"
)
exit()
if ":" in parsed_url[1]:
vhost, port = parsed_url[1].split(':')
else:
vhost = parsed_url[1]
if scheme == "http":
port = 80
elif scheme == "https":
port = 443
path = parsed_url[2]
except:
if not scheme:
exit()
#get ip from db, or if not in db, get it through dns
db_ip = lib.db.get_vhost_ip(vhost, workspace)
if db_ip:
ip = db_ip[0][0]
else:
try:
ip = socket.gethostbyname(vhost)
except:
print("Error getting IP")
proto = "tcp"
if ip == vhost:
scan_output_base_file_dir = output_dir + "/" + ip + "/celerystalkOutput/" + ip + "_" + str(
port) + "_" + proto + "_"
else:
scan_output_base_file_dir = output_dir + "/" + ip + "/celerystalkOutput/" + vhost + "_" + str(
port) + "_" + proto + "_"
host_dir = output_dir + "/" + ip
host_data_dir = host_dir + "/celerystalkOutput/"
# Creates something like /pentest/10.0.0.1, /pentest/10.0.0.2, etc.
utils.create_dir_structure(ip, host_dir)
# Next two lines create the file that will contain each command that was executed. This is not the audit log,
# but a log of commands that can easily be copy/pasted if you need to run them again.
summary_file_name = host_data_dir + "ScanSummary.log"
summary_file = open(summary_file_name, 'a')
db_vhost = (ip, vhost, 1, 0, 1, workspace
) # in this mode all vhosts are in scope
# print(db_vhost)
#create it if it doesnt exist (if it does, doing this doesnt change anything)
db.create_vhost(db_vhost)
# mark this host as in scope now
lib.db.update_vhosts_in_scope(ip, vhost, workspace, 1)
#only mark it as submitted if it is not in scope.
if not simulation:
lib.db.update_vhosts_submitted(ip, vhost, workspace, 1)
# Insert port/service combo into services table if it doesnt exist
db_service = db.get_service(ip, port, proto, workspace)
if not db_service:
db_string = (ip, port, proto, scheme, workspace)
db.create_service(db_string)
#mark this host as in scope now
if not simulation:
db.update_vhosts_submitted(vhost, vhost, workspace, 1)
# I might want to keep this, but i think it is redundant if we have gobuster and photon screenshots...
# Insert url into paths table and take screenshot
# db_path = db.get_path(path, workspace)
# if not db_path:
# url_screenshot_filename = scan_output_base_file_dir + url.replace("http", "").replace("https", "") \
# .replace("/", "_") \
# .replace("\\", "") \
# .replace(":", "_") + ".png"
# url_screenshot_filename = url_screenshot_filename.replace("__", "")
# db_path = (ip, port, url, 0, url_screenshot_filename, workspace)
# db.insert_new_path(db_path)
# # print("Found Url: " + str(url))
# urls_to_screenshot.append((url, url_screenshot_filename))
# if not simulation:
# task_id = uuid()
# command_name = "Screenshots"
# populated_command = "firefox-esr URL mode screenshot | {0} | {1}".format(vhost,scan_output_base_file_dir)
# utils.create_task(command_name, populated_command, vhost, scan_output_base_file_dir, workspace, task_id)
# result = tasks.cel_take_screenshot.delay(urls_to_screenshot,task_id,vhost,scan_output_base_file_dir, workspace,command_name,populated_command)
# # print(result)
# TODO: This def might introduce a bug - same code as parse config submit jobs to celery. need to just call that function here
for section in config.sections():
if (section == "http") or (section == "https"):
if section == scheme:
for (cmd_name, cmd) in config.items(section):
outfile = scan_output_base_file_dir + cmd_name
populated_command = cmd.replace("[TARGET]", vhost).replace(
"[PORT]",
str(port)).replace("[OUTPUT]",
outfile).replace("[PATH]", path)
if simulation:
# debug - sends jobs to celery, but with a # in front of every one.
populated_command = "#" + populated_command
# Grab a UUID from celery.utils so that i can assign it to my task at init, which is amazing because
# that allows me to pass it to all of the tasks in the chain.
task_id = uuid()
utils.create_task(cmd_name, populated_command, vhost,
outfile + ".txt", workspace, task_id)
result = chain(
# insert a row into the database to mark the task as submitted. a subtask does not get tracked
# in celery the same way a task does, for instance, you can't find it in flower
# tasks.cel_create_task.subtask(args=(cmd_name,populated_command, vhost, outfile + ".txt", workspace, task_id)),
# run the command. run_task takes care of marking the task as started and then completed.
# The si tells run_cmd to ignore the data returned from a previous task
tasks.run_cmd.si(cmd_name, populated_command,
celery_path,
task_id).set(task_id=task_id),
# right now, every executed command gets sent to a generic post_process task that can do
# additinoal stuff based on the command that just ran.
tasks.post_process.si(cmd_name, populated_command,
output_dir, workspace, vhost,
host_dir, simulation, port,
scheme, proto, celery_path),
)() # .apply_async()
task_id_list.append(result.task_id)
host_audit_log = host_dir + "/" + "{0}_executed_commands.txt".format(
ip)
f = open(host_audit_log, 'a')
f.write(populated_command + "\n\n")
f.close()
print("[+] Submitted {0} tasks to queue.\n".format(len(task_id_list)))
def process_nmap_data(nmap_report, workspace, target=None):
workspace_mode = lib.db.get_workspace_mode(workspace)[0][0]
services_file = open('/etc/services', mode='r')
services_file_data = services_file.readlines()
services_file.close()
for scanned_host in nmap_report.hosts:
ip = scanned_host.id
unique_db_ips = lib.db.is_vhost_in_db(
ip, workspace) #Returns data if IP is in database
#print(unique_db_ips)
vhosts = scanned_host.hostnames
#print("process_nmap_data: " + str(vhosts))
for vhost in vhosts:
print("process_nmap_data: " + vhost)
vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(
vhost, workspace)
if not vhost_explicitly_out_of_scope: # if the vhost is not explicitly out of scope, add it to db
is_vhost_in_db = lib.db.is_vhost_in_db(
vhost, workspace) # Returns data if IP is in database
if not is_vhost_in_db:
db_vhost = (ip, vhost, 1, 0, 0, workspace)
lib.db.create_vhost(db_vhost)
else:
if not lib.db.get_in_scope_ip(
ip, workspace
): # if it is in the DB but not in scope...
print(
"[+] IP is in the DB, but not in scope. Adding to scope:\t[{0}]"
.format(ip))
lib.db.update_vhosts_in_scope(
ip, vhost, workspace, 1
) # update the host to add it to scope, if it was already in scope, do nothing
else:
print(
"[!] {0} is explicitly marked as out of scope. Skipping..."
.format(ip))
if unique_db_ips: #If this IP was in the db...
vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(
ip, workspace)
if not vhost_explicitly_out_of_scope: #and if the vhost is not explicitly out of scope
if not lib.db.get_in_scope_ip(
ip, workspace): # and if it is not in scope...
print(
"[+] IP is in the DB, but not in scope. Adding to scope:\t[{0}]"
.format(ip))
lib.db.update_vhosts_in_scope(
ip, ip, workspace, 1
) # update the host to add it to scope, if it was already in scope, do nothing
# else:
# print("[+] [{0}] is already in the DB and considered in scope".format(ip))
else:
print(
"[!] {0} is explicitly marked as out of scope. Skipping..."
.format(ip))
else: #if this ip was not already in the db, create a new host and mark it as in scope
#note to self: i dont need to check to see if it is out of scope beccause i already know its not in db at all...
print("[+] IP not in DB. Adding it to DB and to scope:\t [{0}]".
format(ip))
db_vhost = (ip, ip, 1, 0, 0, workspace)
db.create_vhost(db_vhost)
for scanned_service_item in scanned_host.services:
if scanned_service_item.state == "open":
scanned_service_port = scanned_service_item.port
scanned_service_name = scanned_service_item.service
scanned_service_protocol = scanned_service_item.protocol
#print(str(scanned_service_port))
if scanned_service_item.tunnel == 'ssl':
scanned_service_name = 'https'
if scanned_service_name == "tcpwrapped":
try:
port_proto = "\t" + str(
scanned_service_port) + "/" + str(
scanned_service_protocol) + "\t"
for line in services_file_data:
if port_proto in line:
scanned_service_name = line.split("\t")[0]
except:
pass
#Not using this yet, but I'd like to do send this to searchsploit
try:
scanned_service_product = scanned_service_item.service_dict[
'product']
except:
scanned_service_product = ''
try:
scanned_service_version = scanned_service_item.service_dict[
'version']
except:
scanned_service_version = ''
try:
scanned_service_extrainfo = scanned_service_item.service_dict[
'extrainfo']
except:
scanned_service_extrainfo = ''
#print "Port: {0}\tService: {1}\tProduct & Version: {3} {4} {5}".format(scanned_service_port,scanned_service_name,scanned_service_product,scanned_service_version,scanned_service_extrainfo)
db_service = db.get_service(ip, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
db_string = (ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, scanned_service_product,
scanned_service_version,
scanned_service_extrainfo, workspace)
db.create_service(db_string)
else:
db.update_service(ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
for vhost in vhosts:
#print("process_nmap_data - add service: " + vhost)
db_service = db.get_service(vhost, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
#print("service didnt exist, adding: " + vhost + str(scanned_service_port))
db_string = (vhost, scanned_service_port,
scanned_service_protocol,
scanned_service_name,
scanned_service_product,
scanned_service_version,
scanned_service_extrainfo, workspace)
db.create_service(db_string)
else:
#print("service does exist, updating: " + vhost + str(scanned_service_port))
db.update_service(vhost, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
def process_qualys_data(qualys_port_services, workspace, target=None):
with open(qualys_port_services) as f:
for i, line in enumerate(csv.reader(f, delimiter=','), 1):
# Skip the header sections. The first entry is on the 7th line
if i > 6:
ip = line[0]
vhost = line[1]
service = line[2]
protocol = line[3]
port = line[4]
default_service = line[5]
date_first_seen = line[6]
date_last_seen = line[7]
unique_db_ips = lib.db.is_vhost_in_db(
ip, workspace) # Returns data if IP is in database
# print(unique_db_ips)
# print("process_nmap_data: " + str(vhosts))
vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(
vhost, workspace)
if not vhost_explicitly_out_of_scope: # if the vhost is not explicitly out of scope, add it to db
is_vhost_in_db = lib.db.is_vhost_in_db(
vhost, workspace) # Returns data if IP is in database
if not is_vhost_in_db:
db_vhost = (ip, vhost, 1, 0, 0, workspace)
lib.db.create_vhost(db_vhost)
else:
if not lib.db.get_in_scope_ip(
ip, workspace
): # if it is in the DB but not in scope...
print(
"[+] IP is in the DB, but not in scope. Adding to scope:\t[{0}]"
.format(ip))
lib.db.update_vhosts_in_scope(
ip, vhost, workspace, 1
) # update the host to add it to scope, if it was already in scope, do nothing
else:
print(
"[!] {0} is explicitly marked as out of scope. Skipping..."
.format(ip))
if unique_db_ips: # If this IP was in the db...
vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(
ip, workspace)
if not vhost_explicitly_out_of_scope: # and if the vhost is not explicitly out of scope
if not lib.db.get_in_scope_ip(
ip, workspace): # and if it is not in scope...
print(
"[+] IP is in the DB, but not in scope. Adding to scope:\t[{0}]"
.format(ip))
lib.db.update_vhosts_in_scope(
ip, ip, workspace, 1
) # update the host to add it to scope, if it was already in scope, do nothing
# else:
# print("[+] [{0}] is already in the DB and considered in scope".format(ip))
else:
print(
"[!] {0} is explicitly marked as out of scope. Skipping..."
.format(ip))
else: # if this ip was not already in the db, create a new host and mark it as in scope
# note to self: i dont need to check to see if it is out of scope beccause i already know its not in db at all...
print(
"[+] IP not in DB. Adding it to DB and to scope:\t [{0}]"
.format(ip))
db_vhost = (ip, ip, 1, 0, 0, workspace)
db.create_vhost(db_vhost)
if not service:
service = default_service
db_service = db.get_service(ip, port, protocol, workspace)
if not db_service:
db_string = (ip, port, protocol, service, "", "", "",
workspace)
db.create_service(db_string)
else:
db.update_service(ip, port, protocol, service, workspace)
if vhost:
db_service = db.get_service(vhost, port, protocol,
workspace)
if not db_service:
db_string = (vhost, port, protocol, service, "", "",
"", workspace)
db.create_service(db_string)
else:
db.update_service(vhost, port, protocol, service,
workspace)
def process_nessus_data(nessus_report, workspace, target=None):
for scanned_host in nessus_report.hosts:
#print scanned_host.address
ip = scanned_host.address
unique_db_ips = lib.db.is_vhost_in_db(
ip, workspace) #Returns data if IP is in database
#print(unique_db_ips)
if unique_db_ips: #If this IP was in the db...
vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(
ip, workspace)
if not vhost_explicitly_out_of_scope: # and if the vhost is not explicitly out of scope
if not lib.db.get_in_scope_ip(
ip, workspace): # but if it is not in scope...
print(
"[+] IP is in the DB, but not in scope. Adding to scope:\t[{0}]"
.format(ip))
lib.db.update_vhosts_in_scope(
ip, ip, workspace, 1
) # update the host to add it to scope, if it was already in scope, do nothing
# else:
# print("[+] [{0}] is already in the DB and considered in scope".format(ip))
else:
print(
"[!] {0} is explicitly marked as out of scope. Skipping..."
.format(ip))
else: #if this ip was not already in the db, create a new host and mark it as in scope
#note to self: i dont need to check to see if it is explicitly out of scope beccause i already know its not in db at all...
print("[+] IP not in DB. Adding it to DB and to scope:\t [{0}]".
format(ip))
db_vhost = (ip, ip, 1, 0, 0, workspace)
db.create_vhost(db_vhost)
# Step 1: pull all report items in the port scanner family to get every port. The services names are IANA
# default as this point, which is why we need the next two loops.
for report_item in scanned_host.get_report_items:
if report_item.plugin_family == "Port scanners":
if report_item.port != "0":
scanned_service_port = report_item.port
scanned_service_protocol = report_item.protocol
scanned_service_name = report_item.service
db_service = db.get_service(ip, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
db_string = (ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, '', '', '',
workspace)
db.create_service(db_string)
# Step 2: Cycle through the service detection items and update the services where we have a better idea of
# the real running service on the port. These are a subset of open ports which is why we need loop 1
for report_item in scanned_host.get_report_items:
if report_item.plugin_family == "Service detection":
scanned_service_port = report_item.port
scanned_service_protocol = report_item.protocol
scanned_service_name = report_item.service
db_service = db.get_service(ip, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
db_string = (ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, '', '', '', workspace)
db.create_service(db_string)
#print("new service2: " + ip,scanned_service_port,scanned_service_name)
else:
db.update_service(ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
#print("updating service servicename2: " + ip, scanned_service_port, scanned_service_name)
#print("old service servicename2 : " + ip, scanned_service_port,str(db_service[0][4]))
# Step 3: This is needed to split up HTTPS from HTTP
for report_item in scanned_host.get_report_items:
if (report_item.plugin_name
== "TLS Version 1.0 Protocol Detection"
or report_item.plugin_name == "OpenSSL Detection"
or report_item.plugin_name
== "SSL Version 2 and 3 Protocol Detection"):
scanned_service_port = report_item.port
scanned_service_protocol = report_item.protocol
scanned_service_name = 'https'
try:
db.update_service(ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
except:
print(
"if this errors that means there was no service to update as https which is a bigger problem"
)
def process_nessus_data2(nessus_report, workspace, target=None):
for scanned_host in nessus_report.hosts:
#print scanned_host.address
ip = scanned_host.address
# this if takes care of only acting on the targets specififed at hte command line, if the target
# this if takes care of only acting on the targets specififed at hte command line, if the target
# param is used. This is a very simple comparison now. In the future, i'd like to be able to use
# the target splitter function and be able to handle ranges and cidr's in the target option
if (IPAddress(ip) == target) or (target is None):
has_vhost_been_scanned = db.get_inscope_submitted_vhosts_for_ip(
ip, workspace)
if has_vhost_been_scanned:
answer = raw_input(
"[!] {0} has already been scanned. Scan again? [Y\\n] ".
format(ip))
if (answer == "Y") or (answer == "y") or (answer == ""):
db.update_vhosts_submitted(ip, ip, workspace, 0)
else:
db_vhost = (ip, ip, 1, 0, workspace
) # in this mode all vhosts are in scope
#print(db_vhost)
db.create_vhost(db_vhost)
# Step 1: pull all report items in the port scanner family to get every port. The services names are IANA
# default as this point, which is why we need the next two loops.
for report_item in scanned_host.get_report_items:
if report_item.plugin_family == "Port scanners":
scanned_service_port = report_item.port
scanned_service_protocol = report_item.protocol
scanned_service_name = report_item.service
db_service = db.get_service(ip, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
db_string = (ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
db.create_service(db_string)
# Step 2: Cycle through the service detection items and update the services where we have a better idea of
# the real running service on the port. These are a subset of open ports which is why we need loop 1
for report_item in scanned_host.get_report_items:
if report_item.plugin_family == "Service detection":
scanned_service_port = report_item.port
scanned_service_protocol = report_item.protocol
scanned_service_name = report_item.service
db_service = db.get_service(ip, scanned_service_port,
scanned_service_protocol,
workspace)
if not db_service:
db_string = (ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
db.create_service(db_string)
#print("new service2: " + ip,scanned_service_port,scanned_service_name)
else:
db.update_service(ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
#print("updating service servicename2: " + ip, scanned_service_port, scanned_service_name)
#print("old service servicename2 : " + ip, scanned_service_port,str(db_service[0][4]))
# Step 3: This is needed to split up HTTPS from HTTP
for report_item in scanned_host.get_report_items:
if (report_item.plugin_name
== "TLS Version 1.0 Protocol Detection"
or report_item.plugin_name == "OpenSSL Detection"
or report_item.plugin_name
== "SSL Version 2 and 3 Protocol Detection"):
scanned_service_port = report_item.port
scanned_service_protocol = report_item.protocol
scanned_service_name = 'https'
try:
db.update_service(ip, scanned_service_port,
scanned_service_protocol,
scanned_service_name, workspace)
except:
print(
"if this errors that means there was no service to update as https which is a bigger problem"
)
