def run(self):
req = None
while True:
self.scanner._th_lock.acquire()
if self.exit == True or len(
self.scanner.pending_requests) == 0:
self.scanner._th_lock.release()
shutil.rmtree(self.tmp_dir)
return
req = self.scanner.pending_requests.pop()
self.scanner._th_lock.release()
cmd_options = self.scanner.get_cmd(req, self.tmp_dir)
if cmd_options == False:
self.inc_counter()
continue
cmd = self.scanner.settings['scanner_exe'] + cmd_options
exe = CommandExecutor(cmd, True)
out, err = exe.execute(
self.scanner.settings['process_timeout'])
# if err: print "\nError: \n%s\n%s\n%s\n" % (err," ".join(cmd),out)
self.inc_counter()
self.scanner.scanner_executed(req, out, err, self.tmp_dir, cmd)
def run(self): req = None while True: self.scanner._th_lock.acquire() if self.exit == True or len(self.scanner.pending_requests) == 0: self.scanner._th_lock.release() shutil.rmtree(self.tmp_dir) return req = self.scanner.pending_requests.pop() self.scanner._th_lock.release() cmd_options = self.scanner.get_cmd(req, self.tmp_dir) if cmd_options == False: self.inc_counter() continue cmd = self.scanner.settings['scanner_exe'] + cmd_options exe = CommandExecutor(cmd, True) out, err = exe.execute(self.scanner.settings['process_timeout']) # if err: print "\nError: \n%s\n%s\n%s\n" % (err," ".join(cmd),out) self.inc_counter() self.scanner.scanner_executed(req, out,err, self.tmp_dir, cmd)
def test_command_timeout_with_errors(self, mock_comm):
mock_comm.return_value = (None, "error")
cmd = ['sleep', '10']
executor = CommandExecutor(cmd, stderr=True)
result = executor.execute(1)
self.assertEquals("error", result[1])
def execmd(cmd, params=None, timeout=None):
"""
Execute a shell command
Parameters
----------
cmd: string
Path to executable or executable name (if in $PATH)
params: array_like, optional
List of command arguments
timeout: int, optional
Execution timeout in seconds
Returns
-------
dict
Dictionary with fields:
- out str: command output (stdout)
- err str: command error or error string if timeout
- returncode int: command return code
Raises
------
"""
if not re.search(r'^\.?/', cmd):
cmd = get_cmd_path(cmd)
if not cmd or not os.path.isfile(cmd):
raise Exception("Command not found")
cmd = [cmd] + params if params else [cmd]
exe = CommandExecutor(cmd, stderr=True)
out, err = exe.execute(timeout)
return {"out": out, "err": err, "returncode": exe.returncode}
def test_command_responds(self, mock_process):
mock_process.Popen.side_effect = time.sleep(1)
mock_process.Popen().communicate.return_value = ('hurray', 'err')
executor = CommandExecutor(['cmd'])
result = executor.execute(2)
self.assertEqual(result, "hurray")
def check_user_script_syntax(self, probe_cmd, user_script):
try:
exe = CommandExecutor(probe_cmd + ["-u", user_script, "-v"] , False)
out = exe.execute(5)
if out:
print "\n* USER_SCRIPT error: %s" % out
sys.exit(1)
stdoutw(". ")
except KeyboardInterrupt:
print "\nAborted"
sys.exit(0)
def check_user_script_syntax(self, probe_cmd, user_script):
try:
exe = CommandExecutor(probe_cmd + ["-u", user_script, "-v"], False)
out = exe.execute(5)
if out:
print "\n* USER_SCRIPT error: %s" % out
sys.exit(1)
stdoutw(". ")
except KeyboardInterrupt:
print "\nAborted"
sys.exit(0)
def scanner_executed(self, request, out, err, tmp_dir, cmd):
out_file = tmp_dir + "/report"
if not os.path.isfile(out_file):
return
json_file = tmp_dir + "/report.json"
cmd = [
self.reporter, "--reporter",
"json:outfile=%s" % json_file, out_file
]
exe = CommandExecutor(cmd, True)
out, err = exe.execute(30)
if err:
print ">>> error exporting arachni to json: %s %s" % (err,
request.url)
return
if not os.path.isfile(json_file):
return
with open(json_file, 'r') as fil:
jsn = fil.read()
report = []
try:
report = json.loads(jsn)
except Exception as e:
print err
issues = report['issues']
for i in issues:
ref = i['references']['OWASP'] if i['references'] and 'OWASP' in i[
'references'] else "N/A"
req = "N/A"
req = None
if 'request' in i:
req = i['request']
elif 'variations' in i and len(i['variations']) > 0:
req = i['variations'][0]['request']
fields = (i['name'], ref, i['severity'],
req['headers_string'] if req else "N/A")
descr = "D E T A I L S\n\nName: %s\nReference: %s\nSeverity: %s\n\n\nR E Q U E S T\n\n%s" % fields
if req and req['method'] == "post":
descr += "%s" % urllib.urlencode(req['body'])
self.save_vulnerability(request, i['check']['shortname'], descr)
def scanner_executed(self, request, out, err, tmp_dir, cmd): out_file = tmp_dir + "/report" if not os.path.isfile(out_file): return json_file = tmp_dir + "/report.json" cmd = [self.reporter, "--reporter", "json:outfile=%s" % json_file, out_file] exe = CommandExecutor(cmd, True) out, err = exe.execute(30) if err: print ">>> error exporting arachni to json: %s %s" % (err, request.url) return if not os.path.isfile(json_file): return with open(json_file,'r') as fil: jsn = fil.read() report = [] try: report = json.loads(jsn) except Exception as e: print err issues = report['issues'] for i in issues: ref = i['references']['OWASP'] if i['references'] and 'OWASP' in i['references'] else "N/A" req = "N/A" req = None if 'request' in i: req = i['request'] elif 'variations' in i and len(i['variations']) > 0: req = i['variations'][0]['request'] fields = (i['name'], ref, i['severity'], req['headers_string'] if req else "N/A") descr = "D E T A I L S\n\nName: %s\nReference: %s\nSeverity: %s\n\n\nR E Q U E S T\n\n%s" % fields if req and req['method'] == "post": descr += "%s" % urllib.urlencode(req['body']) self.save_vulnerability(request, i['check']['shortname'], descr)
def execmd(self, cmd, params=None, timeout=None):
if not re.search(r'^\.?/', cmd):
cmd = get_cmd_path(cmd)
if not cmd or not os.path.isfile(cmd):
raise Exception("Command not found")
cmd = [cmd] + params if params else [cmd]
exe = CommandExecutor(cmd, stderr=True)
self.scanner._th_lock.acquire()
self.scanner._commands.append(exe)
self.scanner._th_lock.release()
out, err = exe.execute(timeout)
ret = {"out": out, "err": err, "returncode": exe.returncode}
self.scanner._th_lock.acquire()
self.scanner._commands.remove(exe)
self.scanner._th_lock.release()
return ret
def _send_probe(self, request, errors):
probe = None
retries = CrawlerThread._PROCESS_RETRIES
params = self._set_probe_params(request)
while retries:
# DEBUG:
# print("### INPUT: %s" % repr(Shared.probe_cmd + params))
cmd = CommandExecutor(Shared.probe_cmd + params)
jsn = cmd.execute(Shared.options['process_timeout'] + 2)
# DEBUG:
# print("### OUTPUT: %s" % repr(jsn))
if jsn is None:
errors.append(ERROR_PROBEKILLED)
sleep(CrawlerThread._PROCESS_RETRIES_INTERVAL) # ... ???
retries -= 1
continue
else:
probe_array = self._load_probe_json(jsn)
if probe_array:
probe = Probe(probe_array, request)
if probe.status == "ok":
break
errors.append(probe.errcode)
if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO,
ERROR_FORCE_STOP):
break
sleep(CrawlerThread._PROCESS_RETRIES_INTERVAL)
retries -= 1
return probe
def send_probe(self, request, errors):
url = request.url
jsn = None
probe = None
retries = self.process_retries
params = []
cookies = []
if request.method == "POST":
params.append("-P")
if request.data:
params.extend(("-D", request.data))
if len(request.cookies) > 0:
for cookie in request.cookies:
cookies.append(cookie.get_dict())
with open(self.cookie_file,'w') as fil:
fil.write(json.dumps(cookies))
params.extend(("-c", self.cookie_file))
if request.http_auth:
params.extend(("-p" ,request.http_auth))
if Shared.options['set_referer'] and request.referer:
params.extend(("-r", request.referer))
params.extend(("-i", str(request.db_id)))
params.append(url)
while retries:
#while False:
# print cmd_to_str(Shared.probe_cmd + params)
# print ""
cmd = CommandExecutor(Shared.probe_cmd + params)
jsn = cmd.execute(Shared.options['process_timeout'] + 2)
if jsn == None:
errors.append(ERROR_PROBEKILLED)
time.sleep(self.process_retries_interval) # ... ???
retries -= 1
continue
# try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
try:
if jsn and type(jsn) is not str:
jsn = jsn[0]
probeArray = self.load_probe_json(jsn)
except Exception as e:
raise
if probeArray:
probe = Probe(probeArray, request)
if probe.status == "ok":
break
errors.append(probe.errcode)
if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
break
time.sleep(self.process_retries_interval)
retries -= 1
return probe
def main(self, args, opts):
passw = None
format = None
out_cookies = True
out_logouts = True
for o, v in opts:
if o == "-h":
print self.usage()
sys.exit(0)
elif o == "-p":
passw = v
elif o == "-c":
out_cookies = False
elif o == "-l":
out_logouts = False
elif o in ("-H", "-J", "-A"):
format = o
if not passw:
print "The password is hidden here BUT it will be passed to phantomjs via commandline ..."
try:
passw = getpass.getpass()
except KeyboardInterrupt:
print "\nAbort..."
sys.exit(0)
jspath = "%s%s%s%s" % (getrealdir(__file__), "login", os.sep,
"login.js")
cmd = get_phantomjs_cmd() + [jspath, args[0], args[1], passw]
if len(args) > 2: cmd.append(args[2])
#print cmd_to_str(cmd)
exe = CommandExecutor(cmd, True)
out, err = exe.execute(20)
if err:
print "Unable to login"
sys.exit(1)
try:
ret = json.loads(out)
except ValueError as e:
print e
sys.exit(1)
allcookies, logouts = ret
cookies = []
if out_cookies:
for c in reversed(allcookies):
cookie = Cookie(c)
if not cookie in cookies: cookies.append(cookie)
if not out_logouts:
logouts = []
if not format:
print "Cookies:"
for c in cookies:
print " %s=%s" % (c.name, c.value)
print "Logout urls:"
for u in logouts:
print " %s" % u
elif format == "-A":
for c in cookies:
print cmd_to_str([c.name, c.value])
for u in logouts:
print cmd_to_str([u])
elif format == "-H":
args = []
if len(cookies) > 0:
args = [
"-c",
";".join(["%s=%s" % (c.name, c.value) for c in cookies])
]
if len(logouts) > 0:
args.extend(["-x", ",".join(logouts)])
if len(args) > 0:
print cmd_to_str(args)
elif format == "-J":
cd = []
for c in cookies:
cd.append(c.get_dict())
if out_cookies:
print json.dumps(cd)
def main(self, args, opts):
passw = None
format = None
out_cookies = True
out_logouts = True
for o,v in opts:
if o == "-h":
print self.usage()
sys.exit(0)
elif o == "-p":
passw = v
elif o == "-c":
out_cookies = False
elif o == "-l":
out_logouts = False
elif o in ("-H", "-J", "-A"):
format = o
if not passw:
print "The password is hidden here BUT it will be passed to phantomjs via commandline ..."
try:
passw = getpass.getpass()
except KeyboardInterrupt:
print "\nAbort..."
sys.exit(0)
jspath = "%s%s%s%s" % (getrealdir(__file__), "login", os.sep, "login.js")
cmd = get_phantomjs_cmd() + [jspath, args[0], args[1], passw]
if len(args) > 2: cmd.append(args[2])
#print cmd_to_str(cmd)
exe = CommandExecutor(cmd, True)
out, err = exe.execute(20)
if err:
print "Unable to login"
sys.exit(1)
try:
ret = json.loads(out)
except ValueError as e:
print e
sys.exit(1)
allcookies, logouts = ret
cookies = []
if out_cookies:
for c in reversed(allcookies):
cookie = Cookie(c)
if not cookie in cookies: cookies.append(cookie)
if not out_logouts:
logouts = []
if not format:
print "Cookies:"
for c in cookies:
print " %s=%s" % (c.name, c.value)
print "Logout urls:"
for u in logouts:
print " %s" % u
elif format == "-A":
for c in cookies:
print cmd_to_str([c.name, c.value])
for u in logouts:
print cmd_to_str([u])
elif format == "-H":
args = []
if len(cookies) > 0:
args = ["-c", ";".join(["%s=%s" % (c.name, c.value) for c in cookies])]
if len(logouts) > 0:
args.extend(["-x", ",".join(logouts)])
if len(args) > 0:
print cmd_to_str(args)
elif format == "-J":
cd = []
for c in cookies:
cd.append(c.get_dict())
if out_cookies:
print json.dumps(cd)
def test_command_timeout_with_results(self):
cmd = ['tail', '-f', os.path.realpath(__file__)]
executor = CommandExecutor(cmd, stderr=True)
result = executor.execute(1)
self.assertIn("result", result[0])
def send_probe(self, request, errors):
url = request.url
jsn = None
probe = None
retries = self.process_retries
params = []
cookies = []
if request.method == "POST":
params.append("-P")
if request.data:
params.extend(("-D", request.data))
if len(request.cookies) > 0:
for cookie in request.cookies:
c = cookie.get_dict()
if not c['domain']:
purl = urlsplit(request.url)
c['domain'] = purl.netloc.split(":")[0]
cookies.append(c)
with open(self.cookie_file, 'w') as fil:
fil.write(json.dumps(cookies))
params.extend(("-c", self.cookie_file))
if request.http_auth:
params.extend(("-p", request.http_auth))
if Shared.options['set_referer'] and request.referer:
params.extend(("-r", request.referer))
params.extend(("-i", str(request.db_id)))
params.extend(("-J", self.out_file))
params.append(url)
while retries:
#while False:
# print cmd_to_str(Shared.probe_cmd + params)
# print ""
jsn = None
cmd = CommandExecutor(Shared.probe_cmd + params, True)
out, err = cmd.execute(Shared.options['process_timeout'] + 10)
if os.path.isfile(self.out_file):
with open(self.out_file, "r") as f:
jsn = f.read()
os.unlink(self.out_file)
if err or not jsn:
errors.append(ERROR_PROBEKILLED)
if not jsn:
break
# try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
try:
if jsn and type(jsn) is not str:
jsn = jsn[0]
probeArray = self.load_probe_json(jsn)
except Exception as e:
raise
if probeArray:
probe = Probe(probeArray, request)
if probe.status == "ok":
break
errors.append(probe.errcode)
if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
break
time.sleep(self.process_retries_interval)
retries -= 1
return probe
def send_probe(self, request, errors):
url = request.url
jsn = None
probe = None
retries = self.process_retries
params = []
cookies = []
if request.method == "POST":
params.append("-P")
if request.data:
params.extend(("-D", request.data))
if len(request.cookies) > 0:
for cookie in request.cookies:
cookies.append(cookie.get_dict())
with open(self.cookie_file, 'w') as fil:
fil.write(json.dumps(cookies))
params.extend(("-c", self.cookie_file))
if request.http_auth:
params.extend(("-p", request.http_auth))
if Shared.options['set_referer'] and request.referer:
params.extend(("-r", request.referer))
params.append(url)
while retries:
#while False:
# print cmd_to_str(Shared.probe_cmd + params)
# print ""
cmd = CommandExecutor(Shared.probe_cmd + params)
jsn = cmd.execute(Shared.options['process_timeout'] + 2)
if jsn == None:
errors.append(ERROR_PROBEKILLED)
time.sleep(self.process_retries_interval) # ... ???
retries -= 1
continue
# try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
try:
if jsn and type(jsn) is not str:
jsn = jsn[0]
probeArray = self.load_probe_json(jsn)
except Exception as e:
raise
if probeArray:
probe = Probe(probeArray, request)
if probe.status == "ok":
break
errors.append(probe.errcode)
if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
break
time.sleep(self.process_retries_interval)
retries -= 1
return probe
class CrawlerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.thread_uuid = uuid.uuid4()
self.process_retries = 2
self.process_retries_interval = 0.5
self.status = THSTAT_RUNNING
self.exit = False
self.pause = False
self.cookie_file = "%s%shtcap_cookiefile-%s.json" % (
tempfile.gettempdir(), os.sep, self.thread_uuid)
self.out_file = "%s%shtcap_output-%s.json" % (tempfile.gettempdir(),
os.sep, self.thread_uuid)
self.cmd = None
def run(self):
self.crawl()
def wait_request(self):
request = None
Shared.th_condition.acquire()
while True:
if self.exit == True:
Shared.th_condition.notifyAll()
Shared.th_condition.release()
raise ThreadExitRequestException("exit request received")
if Shared.requests_index >= len(Shared.requests):
self.status = THSTAT_WAITING
Shared.th_condition.wait(
) # The wait method releases the lock, blocks the current thread until another thread calls notify
continue
request = Shared.requests[Shared.requests_index]
Shared.requests_index += 1
break
Shared.th_condition.release()
self.status = THSTAT_RUNNING
return request
def load_probe_json(self, jsn):
jsn = jsn.strip()
if not jsn: jsn = "["
if jsn[-1] != "]":
jsn += '{"status":"ok", "partialcontent":true}]'
try:
return json.loads(jsn)
except Exception:
print "-- JSON DECODE ERROR %s" % jsn
raise
def send_probe(self, request, errors):
url = request.url
jsn = None
probe = None
retries = self.process_retries
params = []
cookies = []
if request.method == "POST":
params.append("-P")
if request.data:
params.extend(("-D", request.data))
if len(request.cookies) > 0:
for cookie in request.cookies:
c = cookie.get_dict()
if not c['domain']:
purl = urlsplit(request.url)
c['domain'] = purl.netloc.split(":")[0]
cookies.append(c)
with open(self.cookie_file, 'w') as fil:
fil.write(json.dumps(cookies))
params.extend(("-c", self.cookie_file))
if request.http_auth:
params.extend(("-p", request.http_auth))
if Shared.options['set_referer'] and request.referer:
params.extend(("-r", request.referer))
params.extend(("-i", str(request.db_id)))
params.extend(("-J", self.out_file))
params.append(url)
while retries:
#while False:
# print cmd_to_str(Shared.probe_cmd + params)
# print ""
jsn = None
self.cmd = CommandExecutor(Shared.probe_cmd + params, True)
out, err = self.cmd.execute(Shared.options['process_timeout'] + 10)
if os.path.isfile(self.out_file):
with open(self.out_file, "r") as f:
jsn = f.read()
os.unlink(self.out_file)
if err or not jsn:
errors.append(ERROR_PROBEKILLED)
if not jsn:
break
# try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
try:
if jsn and type(jsn) is not str:
jsn = jsn[0]
probeArray = self.load_probe_json(jsn)
except Exception as e:
raise
if probeArray:
probe = Probe(probeArray, request)
if probe.status == "ok":
break
errors.append(probe.errcode)
if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
break
time.sleep(self.process_retries_interval)
retries -= 1
return probe
def wait_pause(self):
while True:
Shared.th_condition.acquire()
paused = self.pause
Shared.th_condition.release()
if not paused:
break
time.sleep(0.5)
def crawl(self):
while True:
url = None
cookies = []
requests = []
requests_to_crawl = []
redirects = 0
errors = []
try:
request = self.wait_request()
except ThreadExitRequestException:
if os.path.exists(self.cookie_file):
os.remove(self.cookie_file)
return
except Exception as e:
print "-->" + str(e)
continue
url = request.url
purl = urlsplit(url)
probe = None
probe = self.send_probe(request, errors)
if probe:
if probe.status == "ok" or probe.errcode == ERROR_PROBE_TO:
requests = probe.requests
if probe.html:
request.html = probe.html
if len(probe.user_output) > 0:
request.user_output = probe.user_output
else:
errors.append(ERROR_PROBEFAILURE)
# get urls with python to continue crawling
if Shared.options['use_urllib_onerror'] == False:
continue
try:
hr = HttpGet(request, Shared.options['process_timeout'],
self.process_retries,
Shared.options['useragent'],
Shared.options['proxy'],
Shared.options['extra_headers'])
requests = hr.get_requests()
except Exception as e:
errors.append(str(e))
# set out_of_scope, apply user-supplied filters to urls (ie group_qs)
adjust_requests(requests)
Shared.main_condition.acquire()
res = CrawlResult(request, requests, errors,
probe.page_hash if probe else "")
Shared.crawl_results.append(res)
Shared.main_condition.notify()
Shared.main_condition.release()
self.wait_pause()