class BaseScanner:
def __init__(self, db_file, num_threads, request_types, process_timeout, scanner_exe, display_progress, scanner_argv):
self.scan_start_time = int(time.time())
self.threads = []
self._th_lock = threading.Lock()
self._th_lock_db = threading.Lock()
self.performed_requests = 0
self._urlpatterns = []
self._exitcode = 0
self.scanner_name = self.__class__.__name__.lower()
self._running = False
self.settings = self.get_settings()
#override default settings
if num_threads: self.settings['num_threads'] = num_threads
if request_types: self.settings['request_types'] = request_types
if process_timeout: self.settings['process_timeout'] = process_timeout
if scanner_exe: self.settings['scanner_exe'] = scanner_exe
self.settings['scanner_exe'] = self.settings['scanner_exe'].split(" ")
self.db = Database(db_file)
self.id_assessment = self.db.create_assessment(self.scanner_name, int(time.time()))
self.pending_requests = self.db.get_requests(self.settings['request_types'])
self.tot_requests = len(self.pending_requests)
self._duplicated_requests = []
urlpatterns = []
for req in self.pending_requests:
patt = RequestPattern(req).pattern
if patt in urlpatterns:
self._duplicated_requests.append(req.db_id)
else:
urlpatterns.append(patt)
init = self.init(scanner_argv if scanner_argv else [])
self._running = True
print "Scanner %s started with %d threads" % (self.scanner_name, self.settings['num_threads'])
for n in range(0, self.settings['num_threads']):
thread = self.Executor(self)
self.threads.append(thread)
thread.start()
try:
self.wait_executor(self.threads, display_progress)
except KeyboardInterrupt:
print "\nTerminated by user"
self.kill_threads()
self.save_assessment()
sys.exit(self._exitcode)
def get_settings(self):
return dict(
request_types = "xhr,link,redirect,form,json",
num_threads = 10,
process_timeout = 120,
scanner_exe = ""
)
def get_cmd(self, url, outfile):
cmd = []
return cmd
def scanner_executed(self, id_parent, out, err, out_file):
return
def wait_executor(self, threads, display_progress):
executor_done = False
while not executor_done:
executor_done = True
for th in threads:
if th.isAlive():
executor_done = False
th.join(1)
if display_progress:
self._th_lock.acquire()
scanned = self.performed_requests
pending = len(self.pending_requests)
tot = self.tot_requests
self._th_lock.release()
print_progressbar(tot, scanned, self.scan_start_time, "requests scanned")
if display_progress:
print ""
def kill_threads(self):
self._th_lock.acquire()
for th in self.threads:
if th.isAlive(): th.exit = True
self._th_lock.release()
def exit(self, code):
if self._running:
self._th_lock.acquire()
self._exitcode = code
self._th_lock.release()
self.kill_threads()
print "kill thread"
print ""
else :
sys.exit(code)
def save_vulnerability(self, request, type, description):
self._th_lock_db.acquire()
self.db.insert_vulnerability(self.id_assessment, request.db_id, type, description)
self._th_lock_db.release()
def save_assessment(self):
self._th_lock_db.acquire()
self.db.save_assessment(self.id_assessment, int(time.time()))
self._th_lock_db.release()
def is_request_duplicated(self, request):
return request.db_id in self._duplicated_requests
class Executor(threading.Thread):
def __init__(self, scanner):
threading.Thread.__init__(self)
self.scanner = scanner
self.exit = False
self.thread_uuid = uuid.uuid4()
self.tmp_dir = "%s%shtcap_tempdir-%s" % (tempfile.gettempdir(), os.sep, self.thread_uuid)
os.makedirs(self.tmp_dir, 0700)
def inc_counter(self):
self.scanner._th_lock.acquire()
self.scanner.performed_requests += 1
self.scanner._th_lock.release()
def run(self):
req = None
while True:
self.scanner._th_lock.acquire()
if self.exit == True or len(self.scanner.pending_requests) == 0:
self.scanner._th_lock.release()
shutil.rmtree(self.tmp_dir)
return
req = self.scanner.pending_requests.pop()
self.scanner._th_lock.release()
cmd_options = self.scanner.get_cmd(req, self.tmp_dir)
if cmd_options == False:
self.inc_counter()
continue
cmd = self.scanner.settings['scanner_exe'] + cmd_options
exe = CommandExecutor(cmd, True)
out, err = exe.execute(self.scanner.settings['process_timeout'])
# if err: print "\nError: \n%s\n%s\n%s\n" % (err," ".join(cmd),out)
self.inc_counter()
self.scanner.scanner_executed(req, out,err, self.tmp_dir, cmd)
class BaseScanner:
def __init__(self, db_file, num_threads, request_types, display_progress,
scanner_argv, proxy, cookies, user_agent, extra_headers):
self.scan_start_time = int(time.time())
self.threads = []
self.lock = threading.Lock()
self._th_lock = threading.Lock()
self._th_lock_db = threading.Lock()
self._th_lock_stdout = threading.Lock()
self.performed_requests = 0
self._urlpatterns = []
self._exitcode = 0
self._commands = []
self.scanner_name = self.__class__.__name__.lower()
self._running = False
self.settings = self.get_settings()
#self._type = self.settings['scanner_type'] if 'scanner_type' in self.settings else "external"
self.exit_requested = False
self.pause_requested = False
self._print_queue = {}
self.display_progress = display_progress
#override default settings
if num_threads: self.settings['num_threads'] = num_threads
if request_types: self.settings['request_types'] = request_types
#if process_timeout: self.settings['process_timeout'] = process_timeout
#if scanner_exe: self.settings['scanner_exe'] = scanner_exe
# if self._type == "external":
# self.settings['scanner_exe'] = self.settings['scanner_exe'].split(" ")
self._db = Database(db_file)
self.id_assessment = self._db.create_assessment(
self.scanner_name, int(time.time()))
self.pending_requests = self._db.get_requests(
self.settings['request_types'])
self.tot_requests = len(self.pending_requests)
self._duplicated_requests = []
self.proxy = proxy
self.cookies = cookies
self.user_agent = user_agent
self.extra_headers = extra_headers
self.utils = ScannerUtils(self)
urlpatterns = []
for req in self.pending_requests:
patt = RequestPattern(req).pattern
if patt in urlpatterns:
self._duplicated_requests.append(req.db_id)
else:
urlpatterns.append(patt)
init = self.init(scanner_argv if scanner_argv else [])
# if self._type == "external" and not os.path.isfile(self.settings['scanner_exe'][0]):
# raise Exception("scanner_exe not found")
self._running = True
print(
"Scanner %s started with %d threads (^C to pause or change verbosity)"
% (self.scanner_name, self.settings['num_threads']))
for n in range(0, self.settings['num_threads']):
thread = self.Executor(self)
self.threads.append(thread)
thread.start()
self.wait_executor(self.threads)
if not self.wait_threads_exit():
self._th_lock.acquire()
for cmd in self._commands:
if cmd:
cmd.kill()
self._th_lock.release()
os._exit(1)
self.end()
self.save_assessment()
def end(self):
pass
def get_settings(self):
return dict(
request_types="xhr,fetch,link,redirect,form,json",
num_threads=10,
#process_timeout = 120,
#scanner_exe = ""
)
def wait_executor(self, threads):
executor_done = False
pb = Progressbar(self.scan_start_time, "requests scanned")
while not executor_done:
try:
executor_done = True
for th in threads:
if self.display_progress:
self._th_lock.acquire()
scanned = self.performed_requests
pending = len(self.pending_requests)
tot = self.tot_requests
self._th_lock.release()
pb.out(tot, scanned)
else:
self._th_lock_stdout.acquire()
for id in self._print_queue:
for out in self._print_queue[id]:
print(out)
self._print_queue = {}
self._th_lock_stdout.release()
if th.isAlive():
executor_done = False
th.join(1)
except KeyboardInterrupt:
try:
self._th_lock.release()
self._th_lock_stdout.release()
except:
pass
self.pause_threads(True)
if not self.get_runtime_command():
print("Exiting . . .")
self.request_exit()
return
print("Scan is running")
self.pause_threads(False)
if self.display_progress:
print("")
def get_runtime_command(self):
while True:
print("\nScan is paused. Choose what to do:\n"
" r resume scan\n"
" v verbose mode\n"
" p show progress bar\n"
"Hit ctrl-c again to exit\n")
try:
ui = input("> ").strip()
except KeyboardInterrupt:
print("")
return False
if ui == "r":
break
elif ui == "v":
self.display_progress = False
break
elif ui == "p":
self.display_progress = True
break
print(" ")
return True
def request_exit(self):
self.exit_requested = True
self.pause_requested = False
self._th_lock.acquire()
for th in self.threads:
if th.isAlive():
th.exit = True
th.pause = False
for cmd in self._commands:
if cmd:
cmd.terminate()
self._th_lock.release()
def wait_threads_exit(self):
waittime = 0.0
msg = ""
while True:
try:
at = 0
for th in self.threads:
if th.isAlive(): at += 1
if at == 0:
break
if waittime > 2:
stdoutw("\b" * len(msg))
msg = "Waiting %d requests to be completed" % at
stdoutw(msg)
waittime += 0.1
time.sleep(0.1)
except KeyboardInterrupt:
try:
die = input("\nForce exit? [y/N] ").strip()
if die == "y":
return False
except KeyboardInterrupt:
return True
print("")
return True
def pause_threads(self, pause):
self.pause_requested = pause
self._th_lock.acquire()
for th in self.threads:
if th.isAlive(): th.pause = pause
self._th_lock.release()
def _sprint(self, id, str):
self._th_lock_stdout.acquire()
if not id in self._print_queue or not self._print_queue[id]:
self._print_queue[id] = []
self._print_queue[id].append(str)
self._th_lock_stdout.release()
def exit(self, code):
if self._running:
self._th_lock.acquire()
self._exitcode = code
self._th_lock.release()
self.kill_threads()
print("kill thread")
print("")
else:
sys.exit(code)
def db(self, method, params):
self._th_lock_db.acquire()
m = getattr(self._db, method)
ret = m(*params)
self._th_lock_db.release()
return ret
def save_vulnerability(self, request, type, description):
self._th_lock_db.acquire()
self._db.insert_vulnerability(self.id_assessment, request.db_id, type,
description)
self._th_lock_db.release()
def save_vulnerabilities(self, request, vulnerabilities):
self._th_lock_db.acquire()
self._db.insert_vulnerabilities(self.id_assessment, request.db_id,
vulnerabilities)
self._th_lock_db.release()
def save_assessment(self):
self._th_lock_db.acquire()
self._db.save_assessment(self.id_assessment, int(time.time()))
self._th_lock_db.release()
def is_request_duplicated(self, request):
return request.db_id in self._duplicated_requests
# class utils:
# @staticmethod
# def send(url, method=None, data=None, cookies=None, user_agent=None, proxy=None, extra_headers=None, req_timeout=5, ignore_errors=False):
# if not method:
# method = METHOD_GET
# req = Request(REQTYPE_LINK, method, url)
# http = HttpGet(req, req_timeout, proxy=proxy, useragent=user_agent, extra_headers=extra_headers)
# return http.send_request(method=method, url=url, data=data, cookies=cookies, ignore_errors=ignore_errors)
# @staticmethod
# def strip_html_tags(html):
# return strip_html_tags(html)
# @staticmethod
# def execmd(cmd, params=None, timeout=None):
# return execmd(cmd, params, timeout)
class Executor(threading.Thread):
def __init__(self, scanner):
threading.Thread.__init__(self)
self.scanner = scanner
self.exit = False
self.pause = False
self.thread_uuid = uuid.uuid4()
self.tmp_dir = "%s%shtcap_tempdir-%s" % (tempfile.gettempdir(),
os.sep, self.thread_uuid)
os.makedirs(self.tmp_dir, 0o700)
def inc_counter(self):
self.scanner._th_lock.acquire()
self.scanner.performed_requests += 1
self.scanner._th_lock.release()
def run(self):
req = None
while True:
self.scanner._th_lock.acquire()
if self.exit == True or len(
self.scanner.pending_requests) == 0:
self.scanner._th_lock.release()
shutil.rmtree(self.tmp_dir)
return
if self.pause == True:
self.scanner._th_lock.release()
time.sleep(1)
continue
req = self.scanner.pending_requests.pop()
self.scanner._th_lock.release()
#if self.scanner._type == "native":
sc = self.scanner.Scan(self, req)
sc.run()
self.inc_counter()
class BaseScanner:
def __init__(self, db_file, num_threads, request_types, process_timeout,
scanner_exe, display_progress, scanner_argv):
self.scan_start_time = int(time.time())
self.threads = []
self._th_lock = threading.Lock()
self._th_lock_db = threading.Lock()
self.performed_requests = 0
self._urlpatterns = []
self._exitcode = 0
self.scanner_name = self.__class__.__name__.lower()
self._running = False
self.settings = self.get_settings()
#override default settings
if num_threads: self.settings['num_threads'] = num_threads
if request_types: self.settings['request_types'] = request_types
if process_timeout: self.settings['process_timeout'] = process_timeout
if scanner_exe: self.settings['scanner_exe'] = scanner_exe
self.settings['scanner_exe'] = self.settings['scanner_exe'].split(" ")
self.db = Database(db_file)
self.id_assessment = self.db.create_assessment(self.scanner_name,
int(time.time()))
self.pending_requests = self.db.get_requests(
self.settings['request_types'])
self.tot_requests = len(self.pending_requests)
self._duplicated_requests = []
urlpatterns = []
for req in self.pending_requests:
patt = RequestPattern(req).pattern
if patt in urlpatterns:
self._duplicated_requests.append(req.db_id)
else:
urlpatterns.append(patt)
init = self.init(scanner_argv if scanner_argv else [])
self._running = True
print "Scanner %s started with %d threads" % (
self.scanner_name, self.settings['num_threads'])
for n in range(0, self.settings['num_threads']):
thread = self.Executor(self)
self.threads.append(thread)
thread.start()
try:
self.wait_executor(self.threads, display_progress)
except KeyboardInterrupt:
print "\nTerminated by user"
self.kill_threads()
self.save_assessment()
sys.exit(self._exitcode)
def get_settings(self):
return dict(request_types="xhr,link,redirect,form,json",
num_threads=10,
process_timeout=120,
scanner_exe="")
def get_cmd(self, url, outfile):
cmd = []
return cmd
def scanner_executed(self, id_parent, out, err, out_file):
return
def wait_executor(self, threads, display_progress):
executor_done = False
while not executor_done:
executor_done = True
for th in threads:
if th.isAlive():
executor_done = False
th.join(1)
if display_progress:
self._th_lock.acquire()
scanned = self.performed_requests
pending = len(self.pending_requests)
tot = self.tot_requests
self._th_lock.release()
print_progressbar(tot, scanned, self.scan_start_time,
"requests scanned")
if display_progress:
print ""
def kill_threads(self):
self._th_lock.acquire()
for th in self.threads:
if th.isAlive(): th.exit = True
self._th_lock.release()
def exit(self, code):
if self._running:
self._th_lock.acquire()
self._exitcode = code
self._th_lock.release()
self.kill_threads()
print "kill thread"
print ""
else:
sys.exit(code)
def save_vulnerability(self, request, type, description):
self._th_lock_db.acquire()
self.db.insert_vulnerability(self.id_assessment, request.db_id, type,
description)
self._th_lock_db.release()
def save_assessment(self):
self._th_lock_db.acquire()
self.db.save_assessment(self.id_assessment, int(time.time()))
self._th_lock_db.release()
def is_request_duplicated(self, request):
return request.db_id in self._duplicated_requests
class Executor(threading.Thread):
def __init__(self, scanner):
threading.Thread.__init__(self)
self.scanner = scanner
self.exit = False
self.thread_uuid = uuid.uuid4()
self.tmp_dir = "%s%shtcap_tempdir-%s" % (tempfile.gettempdir(),
os.sep, self.thread_uuid)
os.makedirs(self.tmp_dir, 0700)
def inc_counter(self):
self.scanner._th_lock.acquire()
self.scanner.performed_requests += 1
self.scanner._th_lock.release()
def run(self):
req = None
while True:
self.scanner._th_lock.acquire()
if self.exit == True or len(
self.scanner.pending_requests) == 0:
self.scanner._th_lock.release()
shutil.rmtree(self.tmp_dir)
return
req = self.scanner.pending_requests.pop()
self.scanner._th_lock.release()
cmd_options = self.scanner.get_cmd(req, self.tmp_dir)
if cmd_options == False:
self.inc_counter()
continue
cmd = self.scanner.settings['scanner_exe'] + cmd_options
exe = CommandExecutor(cmd, True)
out, err = exe.execute(
self.scanner.settings['process_timeout'])
# if err: print "\nError: \n%s\n%s\n%s\n" % (err," ".join(cmd),out)
self.inc_counter()
self.scanner.scanner_executed(req, out, err, self.tmp_dir, cmd)