def printProgress():
msg = '%s success | %s remaining | %s scanned in %.2f seconds' % (
out.count(), KB["task_queue"].qsize(), KB["finished"],
time.time() - KB['start_time'])
_ = '\r' + ' ' * (KB['console_width'][0] - len(msg)) + msg
Share.dataToStdout(_)
def execute(self, request: Request, response: Response):
self.target = ''
self.requests = request
self.response = response
output = None
try:
output = self.audit()
except NotImplementedError:
msg = 'Plugin: {0} not defined "{1} mode'.format(
self.name, 'audit')
Share.dataToStdout(Share.dataToStdout('\r' + msg + '\n\r'))
except (ConnectTimeout, requests.exceptions.ReadTimeout,
urllib3.exceptions.ReadTimeoutError, socket.timeout):
retry = conf["retry"]
while retry > 0:
msg = 'Plugin: {0} timeout, start it over.'.format(self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
try:
output = self.audit()
break
except (ConnectTimeout, requests.exceptions.ReadTimeout,
urllib3.exceptions.ReadTimeoutError, socket.timeout):
# msg = 'Plugin: {0} time-out retry failed!'.format(self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
retry -= 1
else:
msg = "connect target '{0}' failed!".format(self.target)
# Share.dataToStdout('\r' + msg + '\n\r')
except HTTPError as e:
msg = 'Plugin: {0} HTTPError occurs, start it over.'.format(
self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
except ConnectionError:
msg = "connect target '{0}' failed!".format(self.target)
# Share.dataToStdout('\r' + msg + '\n\r')
except TooManyRedirects as e:
# Share.dataToStdout('\r' + str(e) + '\n\r')
pass
except RemoteDisconnected as e:
pass
except NewConnectionError as ex:
pass
except PoolError as ex:
pass
except Exception as e:
if conf["is_debug"]:
Share.dataToStdout('\r' + "[x]{} report:".format(self.name) +
str(e) + '\n\r')
traceback.print_exc()
return output
def log(self, msg, color=Fore.YELLOW):
width = KB["console_width"][0]
outputs = []
msgs = msg.split('\n')
for i in msgs:
line = i
while len(line) >= width:
_ = line[:width]
outputs.append(_)
# Share.dataToStdout('\r' + _ + ' ' * (width - len(msg)) + '\n\r')
line = line[width:]
outputs.append(line)
for i in outputs:
Share.dataToStdout('\r' + color + i + ' ' * (width - len(i)) + '\n\r')
def run_threads(num_threads, thread_function, args: tuple = ()):
threads = []
try:
info_msg = "Staring {0} threads".format(num_threads)
logger.info(info_msg)
# Start the threads
for num_threads in range(num_threads):
thread = threading.Thread(target=exception_handled_function,
name=str(num_threads),
args=(thread_function, args))
thread.setDaemon(True)
try:
thread.start()
except Exception as ex:
err_msg = "error occurred while starting new thread ('{0}')".format(
str(ex))
logger.critical(err_msg)
break
threads.append(thread)
# And wait for them to all finish
alive = True
while alive:
alive = False
for thread in threads:
if thread.isAlive():
alive = True
time.sleep(0.1)
except KeyboardInterrupt as ex:
KB['continue'] = False
if num_threads > 1:
logger.info("waiting for threads to finish{0}".format(
" (Ctrl+C was pressed)" if isinstance(ex, KeyboardInterrupt
) else ""))
try:
while threading.activeCount() > 1:
pass
except KeyboardInterrupt:
raise
except Exception as ex:
logger.error("thread {0}: {1}".format(
threading.currentThread().getName(), str(ex)))
traceback.print_exc()
finally:
Share.dataToStdout('\n')
def execute(self, request: Request, response: Response):
self.target = ''
self.requests = request
self.response = response
output = None
try:
output = self.audit()
except NotImplementedError:
msg = 'Plugin: {0} not defined "{1} mode'.format(
self.name, 'audit')
Share.dataToStdout(Share.dataToStdout('\r' + msg + '\n\r'))
except (ConnectTimeout, requests.exceptions.ReadTimeout,
urllib3.exceptions.ReadTimeoutError, socket.timeout):
retry = conf["retry"]
while retry > 0:
msg = 'Plugin: {0} timeout, start it over.'.format(self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
try:
output = self.audit()
break
except (ConnectTimeout, requests.exceptions.ReadTimeout,
urllib3.exceptions.ReadTimeoutError, socket.timeout):
# msg = 'Plugin: {0} time-out retry failed!'.format(self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
retry -= 1
except:
raise
else:
msg = "connect target '{0}' failed!".format(self.target)
# Share.dataToStdout('\r' + msg + '\n\r')
except HTTPError as e:
msg = 'Plugin: {0} HTTPError occurs, start it over.'.format(
self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
except ConnectionError:
msg = "connect target '{0}' failed!".format(self.target)
# Share.dataToStdout('\r' + msg + '\n\r')
except TooManyRedirects as e:
# Share.dataToStdout('\r' + str(e) + '\n\r')
pass
except RemoteDisconnected as e:
pass
except NewConnectionError as ex:
pass
except PoolError as ex:
pass
except:
errMsg = "W13scan plugin traceback:\n"
errMsg += "Running version: {}\n".format(VERSION)
errMsg += "Python version: {}\n".format(sys.version.split()[0])
errMsg += "Operating system: {}\n".format(platform.platform())
errMsg += "Threads: {}".format(conf["threads"])
excMsg = traceback.format_exc()
Share.lock.acquire()
if createGithubIssue(errMsg, excMsg):
Share.dataToStdout('\r' + "[x] a issue has reported" + '\n\r')
Share.lock.release()
return output
def do_GET(self):
'''
处理GET请求
:return:
'''
if self.path == 'http://baseproxy.ca/' or self.path == 'http://w13scan.ca/':
self._send_ca()
return
try:
if not self.is_connected:
# 如果不是https，需要连接http服务器
try:
self._proxy_to_dst()
except Exception as e:
try:
self.send_error(
500, '{} connect fail because of "{}"'.format(
self.hostname, str(e)))
except BrokenPipeError:
pass
finally:
return
else:
self._target = self.ssl_host + self.path
# 这里就是代理发送请求，并接收响应信息
request = Request(self)
if request:
if self.is_connected:
request.set_https(True)
self._proxy_sock.sendall(request.to_data())
# 将响应信息返回给客户端
try:
response = Response(request, self._proxy_sock)
except ConnectionResetError:
response = None
if response:
try:
self.request.sendall(response.to_data())
except BrokenPipeError:
pass
except OSError:
pass
else:
self.send_error(404, 'response is None')
if not self._is_replay() and response:
KB['task_queue'].put(('loader', request, response))
else:
self.send_error(404, 'request is None')
except:
errMsg = "W13scan baseproxy get request traceback:\n"
errMsg += "Running version: {}\n".format(VERSION)
errMsg += "Python version: {}\n".format(sys.version.split()[0])
errMsg += "Operating system: {}\n".format(platform.platform())
errMsg += "Threads: {}".format(conf["threads"])
excMsg = traceback.format_exc()
Share.lock.acquire()
if createGithubIssue(errMsg, excMsg):
Share.dataToStdout('\r' + "[x] a issue has reported" + '\n\r')
Share.lock.release()
def execute(self, request: Request, response: Response):
self.target = ''
self.requests = request
self.response = response
output = None
try:
output = self.audit()
except NotImplementedError:
msg = 'Plugin: {0} not defined "{1} mode'.format(
self.name, 'audit')
Share.dataToStdout(Share.dataToStdout('\r' + msg + '\n\r'))
except (ConnectTimeout, requests.exceptions.ReadTimeout,
urllib3.exceptions.ReadTimeoutError, socket.timeout):
retry = conf["retry"]
while retry > 0:
msg = 'Plugin: {0} timeout, start it over.'.format(self.name)
if conf["is_debug"]:
dataToStdout('\r' + msg + '\n\r')
# Share.dataToStdout('\r' + msg + '\n\r')
try:
output = self.audit()
break
except (ConnectTimeout, requests.exceptions.ReadTimeout,
urllib3.exceptions.ReadTimeoutError, socket.timeout):
retry -= 1
except Exception:
return
else:
msg = "connect target '{0}' failed!".format(self.target)
# Share.dataToStdout('\r' + msg + '\n\r')
except HTTPError as e:
msg = 'Plugin: {0} HTTPError occurs, start it over.'.format(
self.name)
# Share.dataToStdout('\r' + msg + '\n\r')
except ConnectionError:
msg = "connect target '{0}' failed!".format(self.target)
# Share.dataToStdout('\r' + msg + '\n\r')
except requests.exceptions.ChunkedEncodingError:
pass
except ConnectionResetError:
pass
except TooManyRedirects as e:
# Share.dataToStdout('\r' + str(e) + '\n\r')
pass
except NewConnectionError as ex:
pass
except PoolError as ex:
pass
except UnicodeDecodeError:
# 这是由于request redirect没有处理编码问题，导致一些网站编码转换被报错,又不能hook其中的关键函数
# 暂时先pass这个错误
# refer：https://github.com/boy-hack/w13scan/labels/Requests%20UnicodeDecodeError
pass
except (requests.exceptions.InvalidURL,
requests.exceptions.InvalidSchema,
requests.exceptions.ContentDecodingError):
# 出现在跳转上的一个奇葩错误，一些网站会在收到敏感操作后跳转到不符合规范的网址，request跟进时就会抛出这个异常
# refer: https://github.com/boy-hack/w13scan/labels/requests.exceptions.InvalidURL
# 奇葩的ContentDecodingError
# refer:https://github.com/boy-hack/w13scan/issues?q=label%3Arequests.exceptions.ContentDecodingError
pass
except KeyboardInterrupt:
raise
except Exception:
errMsg = "W13scan plugin traceback:\n"
errMsg += "Running version: {}\n".format(VERSION)
errMsg += "Python version: {}\n".format(sys.version.split()[0])
errMsg += "Operating system: {}\n".format(platform.platform())
errMsg += "Threads: {}".format(conf["threads"])
if request:
errMsg += '\n\nrequest raw:\n'
errMsg += request.to_data().decode()
excMsg = traceback.format_exc()
Share.lock.acquire()
if conf["is_debug"]:
dataToStdout('\r' + errMsg + '\n\r')
if createGithubIssue(errMsg, excMsg):
dataToStdout('\r' + "[x] a issue has reported" + '\n\r')
Share.lock.release()
return output