def handle(self, response, request_url, items_size, total_size, ignore_list): """ Response handler :param urllib3.response.HTTPResponse response: response object :param str request_url: url from request :param int items_size: current items sizes :param int total_size: response object :param list ignore_list: ignore list :raise ResponseError :return: dict """ if hasattr(response, 'status'): if self.HTTP_DBG_LEVEL <= self.__debug.level: self.__debug.debug_response(response.headers.items()) try: status = super(Response, self).detect(request_url, response) redirect_uri = None if status in ['redirect']: redirect_uri = self._get_redirect_url(request_url, response) check_for_ignored = helper.parse_url(redirect_uri).path[1:] if check_for_ignored in ignore_list: status = 'failed' if self.__config.SUBDOMAINS_SCAN == self.__config.scan: # subdomains ips = Socket.get_ips_addresses(helper.parse_url(request_url).hostname) url = request_url = '{0} {1}'.format(request_url, ips) else: url = redirect_uri else: if self.__config.SUBDOMAINS_SCAN == self.__config.scan: # subdomains ips = Socket.get_ips_addresses(helper.parse_url(request_url).hostname) url = request_url = '{0} {1}'.format(request_url, ips) else: url = request_url self.__debug.debug_request_uri(status=status, request_uri=request_url, redirect_uri=redirect_uri, items_size=items_size, total_size=total_size, content_size=ResponseProvider._get_content_size(response) ) return status, url except Exception as error: raise ResponseError(str(error)) elif 'subdomains' in self._cfg.scan: status = 'failed' self.__debug.debug_request_uri(status=status, request_uri=request_url, items_size=items_size, total_size=total_size, content_size=ResponseProvider._get_content_size(response) ) return status, request_url else: pass
def request(self, url): """ Client request using Proxy :param str url: request uri :return: urllib3.HTTPResponse """ if self._HTTP_DBG_LEVEL <= self.__debug.level: self.__debug.debug_request(self._headers, url, self.__cfg.method) try: response = self.__pool_request(url) return response except MaxRetryError: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: self.__tpl.warning(key='proxy_max_retry_error', url=helper.parse_url(url).path, proxy=self.__server) # Retrying request return self.__pool_request(url) except ReadTimeoutError: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: self.__tpl.warning(key='read_timeout_error', url=helper.parse_url(url).path)
def request(self, url): """ Client request using Proxy :param str url: request uri :return: urllib3.HTTPResponse """ pool = self.__proxy_pool() if self._HTTP_DBG_LEVEL <= self.__debug.level: self.__debug.debug_request(self._headers, url, self.__cfg.method) try: response = pool.request(self.__cfg.method, url, headers=self._headers, retries=self.__cfg.retries, redirect=False) self.cookies_middleware(is_accept=self.__cfg.accept_cookies, response=response) return response except MaxRetryError: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: self.__tpl.warning(key='proxy_max_retry_error', url=helper.parse_url(url).path, proxy=self.__server) except ReadTimeoutError: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: self.__tpl.warning(key='read_timeout_error', url=helper.parse_url(url).path)
def handle(self, response, request_url, items_size, total_size, ignore_list): """ Response handler :param urllib3.response.HTTPResponse response: response object :param str request_url: url from request :param int items_size: current items sizes :param int total_size: response object :param list ignore_list: ignore list :raise ResponseError :return: dict """ if hasattr(response, 'status'): if self.HTTP_DBG_LEVEL <= self.__debug.level: self.__debug.debug_response(response.headers.items()) try: status = super(Response, self).detect(request_url, response) redirect_uri = None if status in ['redirect']: redirect_uri = self._get_redirect_url( request_url, response) check_for_ignored = helper.parse_url(redirect_uri).path[1:] if check_for_ignored in ignore_list: status = 'failed' url = redirect_uri else: if self.__config.DEFAULT_SCAN is not self.__config.scan: # subdomains ips = Socket.get_ips_addresses( helper.parse_url(request_url).hostname) url = request_url = '{0} {1}'.format(request_url, ips) else: url = request_url self.__debug.debug_request_uri( status=status, request_uri=request_url, redirect_uri=redirect_uri, items_size=items_size, total_size=total_size, content_size=ResponseProvider._get_content_size(response)) return status, url except Exception as error: raise ResponseError(str(error)) elif 'subdomains' in self._cfg.scan: status = 'failed' self.__debug.debug_request_uri( status=status, request_uri=request_url, items_size=items_size, total_size=total_size, content_size=ResponseProvider._get_content_size(response)) return status, request_url else: pass
def request(self, url): """ Client request SSL :param str url: request uri :return: urllib3.HTTPResponse """ if self._HTTP_DBG_LEVEL <= self.__debug.level: self.__debug.debug_request(self._headers, url, self.__cfg.method) try: disable_warnings(InsecureRequestWarning) if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: # directories requests response = self.__pool.request(self.__cfg.method, helper.parse_url(url).path, headers=self._headers, retries=self.__cfg.retries, assert_same_host=False, redirect=False) self.cookies_middleware(is_accept=self.__cfg.accept_cookies, response=response) else: # subdomains response = PoolManager().request(self.__cfg.method, url, headers=self._headers, retries=self.__cfg.retries, assert_same_host=False, redirect=False) return response except MaxRetryError: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: self.__tpl.warning(key='max_retry_error', url=helper.parse_url(url).path) except HostChangedError as error: self.__tpl.warning(key='host_changed_error', details=error) pass except ReadTimeoutError: self.__tpl.warning(key='read_timeout_error', url=url) pass except ConnectTimeoutError: self.__tpl.warning(key='connection_timeout_error', url=url) pass except SSLError: if self.__cfg.DEFAULT_SCAN != self.__cfg.scan: return self._provide_ssl_auth_required()
def request(self, url): """ Client request HTTP :param str url: request uri :return: urllib3.HTTPResponse """ if self._HTTP_DBG_LEVEL <= self.__debug.level: self.__debug.debug_request(self._headers, url, self.__cfg.method) try: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: response = self.__pool.request(self.__cfg.method, helper.parse_url(url).path, headers=self._headers, retries=self.__cfg.retries, assert_same_host=True, redirect=False) self.cookies_middleware(is_accept=self.__cfg.accept_cookies, response=response) else: response = PoolManager().request( self.__cfg.method, url, headers=self._headers, retries=self.__cfg.retries, assert_same_host=False, redirect=False, timeout=Timeout(connect=self.__cfg.timeout, read=self.__cfg.timeout)) return response except MaxRetryError: if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: self.__tpl.warning(key='max_retry_error', url=helper.parse_url(url).path) pass except HostChangedError as error: self.__tpl.warning(key='host_changed_error', details=error) pass except ReadTimeoutError: self.__tpl.warning(key='read_timeout_error', url=url) pass except ConnectTimeoutError: self.__tpl.warning(key='connection_timeout_error', url=url) pass
def host(hostname): """ Input `host` param filter :param str hostname: input hostname :raise FilterError :return: str """ if not re.search('http', hostname, re.IGNORECASE): if re.search('https', hostname, re.IGNORECASE): hostname = "https://" + hostname else: hostname = "http://" + hostname hostname = helper.parse_url(hostname).netloc regex = re.compile(r"" + Filter.URL_REGEX + "") if not regex.match(hostname): try: hostname = helper.decode_hostname(hostname) except UnicodeError as error: raise FilterError("\"{0}\" is invalid host. {1}".format(hostname, str(error))) if not regex.match(hostname): raise FilterError("\"{0}\" is invalid host. Use ip, http(s) or just hostname".format(hostname)) return hostname
def _add_urls(self, urllist): """ Add recieved urllist to threadpool :param dict urllist: read from dictionary :raise KeyboardInterrupt :return: None """ try: for url in urllist: if False is self.__is_ignored(url): self.__pool.add(self.__http_request, url) else: self.__catch_report_data('ignored', url) tpl.warning(key='ignored_item', current='{0:0{l}d}'.format( 0, l=len(str(abs( self.__reader.total_lines)))), total=self.__reader.total_lines, item=helper.parse_url(url).path) self.__pool.join() except (SystemExit, KeyboardInterrupt): raise KeyboardInterrupt
def host(hostname): """ Input `host` param filter :param str hostname: input hostname :raise FilterError :return: str """ if not re.search('http', hostname, re.IGNORECASE): if re.search('https', hostname, re.IGNORECASE): hostname = "https://" + hostname else: hostname = "http://" + hostname hostname = helper.parse_url(hostname).netloc regex = re.compile(r"" + Filter.URL_REGEX + "") if not regex.match(hostname): try: hostname = helper.decode_hostname(hostname) except UnicodeError as error: raise FilterError("\"{0}\" is invalid host. {1}".format( hostname, str(error))) if not regex.match(hostname): raise FilterError( "\"{0}\" is invalid host. Use ip, http(s) or just hostname" .format(hostname)) return hostname
def detect(self, request_url, response): """ Detect response by status code :param str request_url: request url :param urllib3.response.HTTPResponse response: response object :raise Exception :return: str """ if response.status in self.DEFAULT_HTTP_SUCCESS_STATUSES: if 'Content-Length' in response.headers: if self.DEFAULT_SOURCE_DETECT_MIN_SIZE <= int( response.headers['Content-Length']): return 'file' if True is self._cfg.is_indexof: if True is self.is_indexof(response.data): return 'indexof' return 'success' elif response.status in self.DEFAULT_HTTP_FAILED_STATUSES: return 'failed' elif response.status in self.DEFAULT_SSL_CERT_REQUIRED_STATUSES: return 'certificat' elif response.status in self.DEFAULT_HTTP_REDIRECT_STATUSES: location = response.get_redirect_location() if location is not False and location is not None: urlfrag = helper.parse_url(request_url) redirect_url = location.rstrip('/') redirectfrag = helper.parse_url(redirect_url) url = "{0}://{1}".format(urlfrag.scheme, urlfrag.netloc) if url == redirect_url \ or (0 < len(redirectfrag.query) and redirectfrag.query in urlfrag.path): return 'failed' return 'redirect' else: return 'failed' elif response.status in self.DEFAULT_HTTP_BAD_REQUEST_STATUSES: return 'bad' elif response.status in self.DEFAULT_HTTP_FORBIDDEN_STATUSES: return 'forbidden' elif response.status in self.DEFAULT_HTTP_AUTH_STATUSES: return 'auth' else: raise Exception('Unknown response status : `{0}`'.format( response.status))
def __is_ignored(self, url): """ Check if path will be ignored :param str url: recieved url :return: bool """ path = helper.parse_url(url).path.strip("/") return path in self.__reader.get_ignored_list()
def scheme(hostname): """ Get `host` scheme from input :param str hostname: input hostname :return: str """ scheme = helper.parse_url(hostname).scheme if not scheme: scheme = 'http' return scheme + "://"
def debug_request_uri(self, status, request_uri, **kwargs): """ Debug request_uri :param str status: response status :param str request_uri: http request uri :param mixed kwargs: :return: bool """ percentage = tpl.line( msg=helper.percent(kwargs.get('items_size', 0), kwargs.get('total_size', 1)), color='cyan') total_len = len(str(abs(kwargs.get('total_size', 1)))) if self.__cfg.DEFAULT_SCAN is self.__cfg.scan: urlpath = helper.parse_url(request_uri).path else: urlpath = request_uri if status in ['success', 'file', 'indexof', 'certificat', 'auth']: request_uri = tpl.line(key=status, color='green', url=urlpath) elif status in ['bad', 'forbidden']: request_uri = tpl.line(key='forbidden', color='yellow', url=urlpath) elif status in ['redirect']: request_uri = tpl.line(key='redirect', color='blue', url=urlpath, rurl=kwargs.get('redirect_uri')) self.__clear = True if self.__catched else False if status in ['success', 'file', 'bad', 'forbidden', 'redirect', 'indexof', 'certificat', 'auth']: sys.writels("", flush=True) tpl.info(key='get_item', clear=self.__clear, percent=percentage, current='{0:0{l}d}'.format(kwargs.get('items_size', 0), l=total_len), total=kwargs.get('total_size', 1), item=request_uri, size=kwargs.get('content_size') ) self.__catched = True else: tpl.line_log(key='get_item', percent=percentage, current='{0:0{l}d}'.format(kwargs.get('items_size', 0), l=total_len), total=kwargs.get('total_size', 1), item=request_uri, size=kwargs.get('content_size'), ) self.__catched = False if kwargs.get('items_size', 0) is kwargs.get('total_size', 1): sys.writels("", flush=True) return True
def __is_redirect(response, request_url): """ Return redirect status :param str request_url: request url :param urllib3.response.HTTPResponse response: response object :return: str """ location = response.get_redirect_location() if location is not False and location is not None: urlfrag = helper.parse_url(request_url) redirect_url = location.rstrip('/') redirectfrag = helper.parse_url(redirect_url) url = "{0}://{1}".format(urlfrag.scheme, urlfrag.netloc) if url == redirect_url \ or (0 < len(redirectfrag.query) and redirectfrag.query in urlfrag.path): return 'failed' return 'redirect' else: return 'failed'
def proxy(proxyaddress): """ Input `proxy` param filter :param str proxyaddress: input proxy server address :raise FilterError :return: str """ proxy = helper.parse_url(proxyaddress) if proxy.scheme not in ['http', 'https', 'socks4', 'socks5'] or None is proxy.port: raise FilterError("\"{0}\" is invalid proxy in --proxy. Use scheme:ip:port format".format(proxyaddress)) return proxyaddress
def _get_redirect_url(cls, url, response): """ Get redirect url :param str url: redirect url :param urllib3.response.HTTPResponse response: response object :return: str """ redirect_url = None location = response.get_redirect_location() if location is not False: matches = re.search("(?P<url>https?://[^\s]+)", location) if matches is not None: redirect_url = matches.group("url") else: urlp = helper.parse_url(url) location = location if True is location.startswith('/') else ''.join(('/', location)) redirect_url = urlp.scheme + '://' + urlp.netloc + location return redirect_url
def _add_urls(self, urllist): """ Add received urllist to threadpool :param dict urllist: read from dictionary :raise KeyboardInterrupt :return: None """ try: for url in urllist: if False is self.__is_ignored(url): self.__pool.add(self.__http_request, url) else: self.__catch_report_data('ignored', url) tpl.warning( key='ignored_item', current='{0:0{l}d}'.format(0, l=len(str(abs(self.__reader.total_lines)))), total=self.__reader.total_lines, item=helper.parse_url(url).path ) self.__pool.join() except (SystemExit, KeyboardInterrupt): raise KeyboardInterrupt