def handle(self, response, request_url, items_size, total_size, ignore_list):
"""
Response handler
:param urllib3.response.HTTPResponse response: response object
:param str request_url: url from request
:param int items_size: current items sizes
:param int total_size: response object
:param list ignore_list: ignore list
:raise ResponseError
:return: dict
"""
if hasattr(response, 'status'):
if self.HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_response(response.headers.items())
try:
status = super(Response, self).detect(request_url, response)
redirect_uri = None
if status in ['redirect']:
redirect_uri = self._get_redirect_url(request_url, response)
check_for_ignored = helper.parse_url(redirect_uri).path[1:]
if check_for_ignored in ignore_list:
status = 'failed'
if self.__config.SUBDOMAINS_SCAN == self.__config.scan: # subdomains
ips = Socket.get_ips_addresses(helper.parse_url(request_url).hostname)
url = request_url = '{0} {1}'.format(request_url, ips)
else:
url = redirect_uri
else:
if self.__config.SUBDOMAINS_SCAN == self.__config.scan: # subdomains
ips = Socket.get_ips_addresses(helper.parse_url(request_url).hostname)
url = request_url = '{0} {1}'.format(request_url, ips)
else:
url = request_url
self.__debug.debug_request_uri(status=status,
request_uri=request_url,
redirect_uri=redirect_uri,
items_size=items_size,
total_size=total_size,
content_size=ResponseProvider._get_content_size(response)
)
return status, url
except Exception as error:
raise ResponseError(str(error))
elif 'subdomains' in self._cfg.scan:
status = 'failed'
self.__debug.debug_request_uri(status=status,
request_uri=request_url,
items_size=items_size,
total_size=total_size,
content_size=ResponseProvider._get_content_size(response)
)
return status, request_url
else:
pass
def request(self, url):
"""
Client request using Proxy
:param str url: request uri
:return: urllib3.HTTPResponse
"""
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_request(self._headers, url, self.__cfg.method)
try:
response = self.__pool_request(url)
return response
except MaxRetryError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='proxy_max_retry_error',
url=helper.parse_url(url).path,
proxy=self.__server)
# Retrying request
return self.__pool_request(url)
except ReadTimeoutError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='read_timeout_error',
url=helper.parse_url(url).path)
def request(self, url):
"""
Client request using Proxy
:param str url: request uri
:return: urllib3.HTTPResponse
"""
pool = self.__proxy_pool()
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_request(self._headers, url, self.__cfg.method)
try:
response = pool.request(self.__cfg.method, url, headers=self._headers, retries=self.__cfg.retries,
redirect=False)
self.cookies_middleware(is_accept=self.__cfg.accept_cookies, response=response)
return response
except MaxRetryError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='proxy_max_retry_error', url=helper.parse_url(url).path, proxy=self.__server)
except ReadTimeoutError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='read_timeout_error', url=helper.parse_url(url).path)
def handle(self, response, request_url, items_size, total_size,
ignore_list):
"""
Response handler
:param urllib3.response.HTTPResponse response: response object
:param str request_url: url from request
:param int items_size: current items sizes
:param int total_size: response object
:param list ignore_list: ignore list
:raise ResponseError
:return: dict
"""
if hasattr(response, 'status'):
if self.HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_response(response.headers.items())
try:
status = super(Response, self).detect(request_url, response)
redirect_uri = None
if status in ['redirect']:
redirect_uri = self._get_redirect_url(
request_url, response)
check_for_ignored = helper.parse_url(redirect_uri).path[1:]
if check_for_ignored in ignore_list:
status = 'failed'
url = redirect_uri
else:
if self.__config.DEFAULT_SCAN is not self.__config.scan: # subdomains
ips = Socket.get_ips_addresses(
helper.parse_url(request_url).hostname)
url = request_url = '{0} {1}'.format(request_url, ips)
else:
url = request_url
self.__debug.debug_request_uri(
status=status,
request_uri=request_url,
redirect_uri=redirect_uri,
items_size=items_size,
total_size=total_size,
content_size=ResponseProvider._get_content_size(response))
return status, url
except Exception as error:
raise ResponseError(str(error))
elif 'subdomains' in self._cfg.scan:
status = 'failed'
self.__debug.debug_request_uri(
status=status,
request_uri=request_url,
items_size=items_size,
total_size=total_size,
content_size=ResponseProvider._get_content_size(response))
return status, request_url
else:
pass
def request(self, url):
"""
Client request SSL
:param str url: request uri
:return: urllib3.HTTPResponse
"""
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_request(self._headers, url, self.__cfg.method)
try:
disable_warnings(InsecureRequestWarning)
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: # directories requests
response = self.__pool.request(self.__cfg.method,
helper.parse_url(url).path,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False)
self.cookies_middleware(is_accept=self.__cfg.accept_cookies,
response=response)
else: # subdomains
response = PoolManager().request(self.__cfg.method,
url,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False)
return response
except MaxRetryError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='max_retry_error',
url=helper.parse_url(url).path)
except HostChangedError as error:
self.__tpl.warning(key='host_changed_error', details=error)
pass
except ReadTimeoutError:
self.__tpl.warning(key='read_timeout_error', url=url)
pass
except ConnectTimeoutError:
self.__tpl.warning(key='connection_timeout_error', url=url)
pass
except SSLError:
if self.__cfg.DEFAULT_SCAN != self.__cfg.scan:
return self._provide_ssl_auth_required()
def request(self, url):
"""
Client request SSL
:param str url: request uri
:return: urllib3.HTTPResponse
"""
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_request(self._headers, url, self.__cfg.method)
try:
disable_warnings(InsecureRequestWarning)
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: # directories requests
response = self.__pool.request(self.__cfg.method,
helper.parse_url(url).path,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False)
self.cookies_middleware(is_accept=self.__cfg.accept_cookies, response=response)
else: # subdomains
response = PoolManager().request(self.__cfg.method, url,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False)
return response
except MaxRetryError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='max_retry_error', url=helper.parse_url(url).path)
except HostChangedError as error:
self.__tpl.warning(key='host_changed_error', details=error)
pass
except ReadTimeoutError:
self.__tpl.warning(key='read_timeout_error', url=url)
pass
except ConnectTimeoutError:
self.__tpl.warning(key='connection_timeout_error', url=url)
pass
except SSLError:
if self.__cfg.DEFAULT_SCAN != self.__cfg.scan:
return self._provide_ssl_auth_required()
def request(self, url):
"""
Client request HTTP
:param str url: request uri
:return: urllib3.HTTPResponse
"""
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_request(self._headers, url, self.__cfg.method)
try:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
response = self.__pool.request(self.__cfg.method,
helper.parse_url(url).path,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=True,
redirect=False)
self.cookies_middleware(is_accept=self.__cfg.accept_cookies,
response=response)
else:
response = PoolManager().request(
self.__cfg.method,
url,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False,
timeout=Timeout(connect=self.__cfg.timeout,
read=self.__cfg.timeout))
return response
except MaxRetryError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='max_retry_error',
url=helper.parse_url(url).path)
pass
except HostChangedError as error:
self.__tpl.warning(key='host_changed_error', details=error)
pass
except ReadTimeoutError:
self.__tpl.warning(key='read_timeout_error', url=url)
pass
except ConnectTimeoutError:
self.__tpl.warning(key='connection_timeout_error', url=url)
pass
def host(hostname):
"""
Input `host` param filter
:param str hostname: input hostname
:raise FilterError
:return: str
"""
if not re.search('http', hostname, re.IGNORECASE):
if re.search('https', hostname, re.IGNORECASE):
hostname = "https://" + hostname
else:
hostname = "http://" + hostname
hostname = helper.parse_url(hostname).netloc
regex = re.compile(r"" + Filter.URL_REGEX + "")
if not regex.match(hostname):
try:
hostname = helper.decode_hostname(hostname)
except UnicodeError as error:
raise FilterError("\"{0}\" is invalid host. {1}".format(hostname, str(error)))
if not regex.match(hostname):
raise FilterError("\"{0}\" is invalid host. Use ip, http(s) or just hostname".format(hostname))
return hostname
def _add_urls(self, urllist):
"""
Add recieved urllist to threadpool
:param dict urllist: read from dictionary
:raise KeyboardInterrupt
:return: None
"""
try:
for url in urllist:
if False is self.__is_ignored(url):
self.__pool.add(self.__http_request, url)
else:
self.__catch_report_data('ignored', url)
tpl.warning(key='ignored_item',
current='{0:0{l}d}'.format(
0,
l=len(str(abs(
self.__reader.total_lines)))),
total=self.__reader.total_lines,
item=helper.parse_url(url).path)
self.__pool.join()
except (SystemExit, KeyboardInterrupt):
raise KeyboardInterrupt
def host(hostname):
"""
Input `host` param filter
:param str hostname: input hostname
:raise FilterError
:return: str
"""
if not re.search('http', hostname, re.IGNORECASE):
if re.search('https', hostname, re.IGNORECASE):
hostname = "https://" + hostname
else:
hostname = "http://" + hostname
hostname = helper.parse_url(hostname).netloc
regex = re.compile(r"" + Filter.URL_REGEX + "")
if not regex.match(hostname):
try:
hostname = helper.decode_hostname(hostname)
except UnicodeError as error:
raise FilterError("\"{0}\" is invalid host. {1}".format(
hostname, str(error)))
if not regex.match(hostname):
raise FilterError(
"\"{0}\" is invalid host. Use ip, http(s) or just hostname"
.format(hostname))
return hostname
def detect(self, request_url, response):
"""
Detect response by status code
:param str request_url: request url
:param urllib3.response.HTTPResponse response: response object
:raise Exception
:return: str
"""
if response.status in self.DEFAULT_HTTP_SUCCESS_STATUSES:
if 'Content-Length' in response.headers:
if self.DEFAULT_SOURCE_DETECT_MIN_SIZE <= int(
response.headers['Content-Length']):
return 'file'
if True is self._cfg.is_indexof:
if True is self.is_indexof(response.data):
return 'indexof'
return 'success'
elif response.status in self.DEFAULT_HTTP_FAILED_STATUSES:
return 'failed'
elif response.status in self.DEFAULT_SSL_CERT_REQUIRED_STATUSES:
return 'certificat'
elif response.status in self.DEFAULT_HTTP_REDIRECT_STATUSES:
location = response.get_redirect_location()
if location is not False and location is not None:
urlfrag = helper.parse_url(request_url)
redirect_url = location.rstrip('/')
redirectfrag = helper.parse_url(redirect_url)
url = "{0}://{1}".format(urlfrag.scheme, urlfrag.netloc)
if url == redirect_url \
or (0 < len(redirectfrag.query) and redirectfrag.query in urlfrag.path):
return 'failed'
return 'redirect'
else:
return 'failed'
elif response.status in self.DEFAULT_HTTP_BAD_REQUEST_STATUSES:
return 'bad'
elif response.status in self.DEFAULT_HTTP_FORBIDDEN_STATUSES:
return 'forbidden'
elif response.status in self.DEFAULT_HTTP_AUTH_STATUSES:
return 'auth'
else:
raise Exception('Unknown response status : `{0}`'.format(
response.status))
def __is_ignored(self, url):
"""
Check if path will be ignored
:param str url: recieved url
:return: bool
"""
path = helper.parse_url(url).path.strip("/")
return path in self.__reader.get_ignored_list()
def __is_ignored(self, url):
"""
Check if path will be ignored
:param str url: recieved url
:return: bool
"""
path = helper.parse_url(url).path.strip("/")
return path in self.__reader.get_ignored_list()
def scheme(hostname):
"""
Get `host` scheme from input
:param str hostname: input hostname
:return: str
"""
scheme = helper.parse_url(hostname).scheme
if not scheme:
scheme = 'http'
return scheme + "://"
def debug_request_uri(self, status, request_uri, **kwargs):
"""
Debug request_uri
:param str status: response status
:param str request_uri: http request uri
:param mixed kwargs:
:return: bool
"""
percentage = tpl.line(
msg=helper.percent(kwargs.get('items_size', 0), kwargs.get('total_size', 1)),
color='cyan')
total_len = len(str(abs(kwargs.get('total_size', 1))))
if self.__cfg.DEFAULT_SCAN is self.__cfg.scan:
urlpath = helper.parse_url(request_uri).path
else:
urlpath = request_uri
if status in ['success', 'file', 'indexof', 'certificat', 'auth']:
request_uri = tpl.line(key=status, color='green', url=urlpath)
elif status in ['bad', 'forbidden']:
request_uri = tpl.line(key='forbidden', color='yellow', url=urlpath)
elif status in ['redirect']:
request_uri = tpl.line(key='redirect', color='blue', url=urlpath,
rurl=kwargs.get('redirect_uri'))
self.__clear = True if self.__catched else False
if status in ['success', 'file', 'bad', 'forbidden', 'redirect', 'indexof', 'certificat', 'auth']:
sys.writels("", flush=True)
tpl.info(key='get_item',
clear=self.__clear,
percent=percentage,
current='{0:0{l}d}'.format(kwargs.get('items_size', 0), l=total_len),
total=kwargs.get('total_size', 1),
item=request_uri,
size=kwargs.get('content_size')
)
self.__catched = True
else:
tpl.line_log(key='get_item',
percent=percentage,
current='{0:0{l}d}'.format(kwargs.get('items_size', 0), l=total_len),
total=kwargs.get('total_size', 1),
item=request_uri,
size=kwargs.get('content_size'),
)
self.__catched = False
if kwargs.get('items_size', 0) is kwargs.get('total_size', 1):
sys.writels("", flush=True)
return True
def scheme(hostname):
"""
Get `host` scheme from input
:param str hostname: input hostname
:return: str
"""
scheme = helper.parse_url(hostname).scheme
if not scheme:
scheme = 'http'
return scheme + "://"
def __is_redirect(response, request_url):
"""
Return redirect status
:param str request_url: request url
:param urllib3.response.HTTPResponse response: response object
:return: str
"""
location = response.get_redirect_location()
if location is not False and location is not None:
urlfrag = helper.parse_url(request_url)
redirect_url = location.rstrip('/')
redirectfrag = helper.parse_url(redirect_url)
url = "{0}://{1}".format(urlfrag.scheme, urlfrag.netloc)
if url == redirect_url \
or (0 < len(redirectfrag.query) and redirectfrag.query in urlfrag.path):
return 'failed'
return 'redirect'
else:
return 'failed'
def __is_redirect(response, request_url):
"""
Return redirect status
:param str request_url: request url
:param urllib3.response.HTTPResponse response: response object
:return: str
"""
location = response.get_redirect_location()
if location is not False and location is not None:
urlfrag = helper.parse_url(request_url)
redirect_url = location.rstrip('/')
redirectfrag = helper.parse_url(redirect_url)
url = "{0}://{1}".format(urlfrag.scheme, urlfrag.netloc)
if url == redirect_url \
or (0 < len(redirectfrag.query) and redirectfrag.query in urlfrag.path):
return 'failed'
return 'redirect'
else:
return 'failed'
def proxy(proxyaddress):
"""
Input `proxy` param filter
:param str proxyaddress: input proxy server address
:raise FilterError
:return: str
"""
proxy = helper.parse_url(proxyaddress)
if proxy.scheme not in ['http', 'https', 'socks4', 'socks5'] or None is proxy.port:
raise FilterError("\"{0}\" is invalid proxy in --proxy. Use scheme:ip:port format".format(proxyaddress))
return proxyaddress
def proxy(proxyaddress):
"""
Input `proxy` param filter
:param str proxyaddress: input proxy server address
:raise FilterError
:return: str
"""
proxy = helper.parse_url(proxyaddress)
if proxy.scheme not in ['http', 'https', 'socks4', 'socks5'] or None is proxy.port:
raise FilterError("\"{0}\" is invalid proxy in --proxy. Use scheme:ip:port format".format(proxyaddress))
return proxyaddress
def _get_redirect_url(cls, url, response):
"""
Get redirect url
:param str url: redirect url
:param urllib3.response.HTTPResponse response: response object
:return: str
"""
redirect_url = None
location = response.get_redirect_location()
if location is not False:
matches = re.search("(?P<url>https?://[^\s]+)", location)
if matches is not None:
redirect_url = matches.group("url")
else:
urlp = helper.parse_url(url)
location = location if True is location.startswith('/') else ''.join(('/', location))
redirect_url = urlp.scheme + '://' + urlp.netloc + location
return redirect_url
def _get_redirect_url(cls, url, response):
"""
Get redirect url
:param str url: redirect url
:param urllib3.response.HTTPResponse response: response object
:return: str
"""
redirect_url = None
location = response.get_redirect_location()
if location is not False:
matches = re.search("(?P<url>https?://[^\s]+)", location)
if matches is not None:
redirect_url = matches.group("url")
else:
urlp = helper.parse_url(url)
location = location if True is location.startswith('/') else ''.join(('/', location))
redirect_url = urlp.scheme + '://' + urlp.netloc + location
return redirect_url
def _add_urls(self, urllist):
"""
Add received urllist to threadpool
:param dict urllist: read from dictionary
:raise KeyboardInterrupt
:return: None
"""
try:
for url in urllist:
if False is self.__is_ignored(url):
self.__pool.add(self.__http_request, url)
else:
self.__catch_report_data('ignored', url)
tpl.warning(
key='ignored_item',
current='{0:0{l}d}'.format(0, l=len(str(abs(self.__reader.total_lines)))),
total=self.__reader.total_lines,
item=helper.parse_url(url).path
)
self.__pool.join()
except (SystemExit, KeyboardInterrupt):
raise KeyboardInterrupt
