def _on_request(self, adapter, request, **kwargs): match = self._find_match(request) # TODO(dcramer): find the correct class for this if match is None: error_msg = 'Connection refused: {0} {1}'.format(request.method, request.url) response = ConnectionError(error_msg) response.request = request self._calls.add(request, response) raise response if 'body' in match and isinstance(match['body'], Exception): self._calls.add(request, match['body']) raise match['body'] headers = {} if match['content_type'] is not None: headers['Content-Type'] = match['content_type'] if 'callback' in match: # use callback status, r_headers, body = match['callback'](request) if isinstance(body, six.text_type): body = body.encode('utf-8') body = BufferIO(body) headers.update(r_headers) elif 'body' in match: if match['adding_headers']: headers.update(match['adding_headers']) status = match['status'] body = BufferIO(match['body']) response = HTTPResponse( status=status, reason=six.moves.http_client.responses[status], body=body, headers=headers, preload_content=False, # Need to not decode_content to mimic requests decode_content=False, ) response = adapter.build_response(request, response) if not match.get('stream'): response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict(dict( (v.name, v.value) for _, v in resp_cookies.items() )) except (KeyError, TypeError): pass self._calls.add(request, response) return response
def __init__(self, args): super(HttpScan, self).__init__(args) self.session = requesocks.session() adapters.DEFAULT_RETRIES = self.args.max_retries self.tor = None if self.args.tor: self.out.log("Enabling TOR") self.tor = Torify() self.session.proxies = {'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050'} if self.args.check_tor: # Check TOR self.out.log("Checking IP via TOR") rip, tip = self.tor.check_ip(verbose=True) if tip is None: self.out.log('TOR is not working properly!', logging.ERROR) exit(-1) if self.args.cookies is not None: if path.exists(self.args.cookies) and path.isfile(self.args.cookies): self.cookies = MozillaCookieJar(self.args.cookies) self.cookies.load() else: # self.out.log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR) self.cookies = Cookies.from_request(self.args.cookies) else: self.cookies = None self.ua = UserAgent() if self.args.user_agent is None else self.args.user_agent
def _on_request(self, adapter, request, **kwargs): match = self._find_match(request) # TODO(dcramer): find the correct class for this if match is None: error_msg = 'Connection refused: {0} {1}'.format(request.method, request.url) response = ConnectionError(error_msg) response.request = request self._calls.add(request, response) raise response if 'body' in match and isinstance(match['body'], Exception): self._calls.add(request, match['body']) raise match['body'] headers = {} if match['content_type'] is not None: headers['Content-Type'] = match['content_type'] if 'callback' in match: # use callback status, r_headers, body = match['callback'](request) if isinstance(body, six.text_type): body = body.encode('utf-8') body = BufferIO(body) headers.update(r_headers) elif 'body' in match: if match['adding_headers']: headers.update(match['adding_headers']) status = match['status'] body = BufferIO(match['body']) response = HTTPResponse( status=status, reason=six.moves.http_client.responses[status], body=body, headers=headers, preload_content=False, ) response = adapter.build_response(request, response) if not match.get('stream'): response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict(dict( (v.name, v.value) for _, v in resp_cookies.items() )) except (KeyError, TypeError): pass self._calls.add(request, response) return response
def _cookies_from_headers(headers): if sys.version_info[:2] < (3, 4): from cookies import Cookies resp_cookies = Cookies.from_request(headers["set-cookie"]) cookies_dict = {v.name: v.value for _, v in resp_cookies.items()} else: import biscuits cookies_dict = biscuits.parse(headers["set-cookie"]) return cookiejar_from_dict(cookies_dict)
def _cookies_from_headers(headers): try: import http.cookies as cookies resp_cookie = cookies.SimpleCookie() resp_cookie.load(headers["set-cookie"]) cookies_dict = {name: v.value for name, v in resp_cookie.items()} except ImportError: from cookies import Cookies resp_cookies = Cookies.from_request(headers["set-cookie"]) cookies_dict = {v.name: v.value for _, v in resp_cookies.items()} return cookiejar_from_dict(cookies_dict)
def obtain_session(self, env): cookie_str = env.get('HTTP_COOKIE', None) session = None if cookie_str: cookie = Cookies.from_request( cookie_str, ignore_bad_cookies=True, ).get('session', None) if cookie and cookie.value: if cookie.value in self.sessions: # Session found session = self.sessions[cookie.value] if session.is_dead(): session = None return session
def _on_request(self, adapter, request, **kwargs): match = self._find_match(request) resp_callback = self.response_callback if self.allow_external_requests: logger.info('request.allowed-external', extra={ 'url': request.url, }) return _real_send(adapter, request) if match is None: if request.url.startswith(self.passthru_prefixes): logger.info('request.allowed-passthru', extra={ 'url': request.url, }) return _real_send(adapter, request) response = self._no_match(request, resp_callback) raise response try: response = adapter.build_response( request, match.get_response(request), ) except Exception as response: match.call_count += 1 self._calls.add(request, response) response = resp_callback(response) if resp_callback else response raise if not match.stream: response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict( dict((v.name, v.value) for _, v in resp_cookies.items())) except (KeyError, TypeError): pass response = resp_callback(response) if resp_callback else response match.call_count += 1 self._calls.add(request, response) return response
def __init__(self, cnxn, method, url, headers, body): self.cnxn = cnxn self.method = method self.url = url self.headers = headers self.body = body # A counter to help troubleshoot. self._id = Request._next_id Request._next_id = (Request._next_id + 1) % 1000000 self.form = self.parse_form() c = headers.get('cookie', None) if c: self.cookies = Cookies.from_request(c) else: self.cookies = Cookies()
def _on_request(self, adapter, request, **kwargs): match = self._find_match(request) # TODO(dcramer): find the correct class for this if match is None: error_msg = "Connection refused: {0} {1}".format(request.method, request.url) response = ConnectionError(error_msg) self._calls.add(request, response) raise response if "body" in match and isinstance(match["body"], Exception): self._calls.add(request, match["body"]) raise match["body"] headers = {"Content-Type": match["content_type"]} if "callback" in match: # use callback status, r_headers, body = match["callback"](request) if isinstance(body, six.text_type): body = body.encode("utf-8") body = BufferIO(body) headers.update(r_headers) elif "body" in match: if match["adding_headers"]: headers.update(match["adding_headers"]) status = match["status"] body = BufferIO(match["body"]) response = HTTPResponse(status=status, body=body, headers=headers, preload_content=False) response = adapter.build_response(request, response) if not match.get("stream"): response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers["set-cookie"]) response.cookies = cookiejar_from_dict(dict((v.name, v.value) for _, v in resp_cookies.items())) except (KeyError, TypeError): pass self._calls.add(request, response) return response
def _on_request(self, adapter, request, **kwargs): match = self._find_match(request) resp_callback = self.response_callback if match is None: if request.url.startswith(self.passthru_prefixes): logger.info("request.allowed-passthru", extra={"url": request.url}) return _real_send(adapter, request, **kwargs) error_msg = "Connection refused: {0} {1}".format( request.method, request.url ) response = ConnectionError(error_msg) response.request = request self._calls.add(request, response) response = resp_callback(response) if resp_callback else response raise response try: response = adapter.build_response(request, match.get_response(request)) except Exception as response: match.call_count += 1 self._calls.add(request, response) response = resp_callback(response) if resp_callback else response raise if not match.stream: response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers["set-cookie"]) response.cookies = cookiejar_from_dict( dict((v.name, v.value) for _, v in resp_cookies.items()) ) except (KeyError, TypeError): pass response = resp_callback(response) if resp_callback else response match.call_count += 1 self._calls.add(request, response) return response
def _on_request(self, adapter, request, **kwargs): match = self._find_match(request) resp_callback = self.response_callback if match is None: error_msg = 'Connection refused: {0} {1}'.format( request.method, request.url) response = ConnectionError(error_msg) response.request = request self._calls.add(request, response) response = resp_callback(response) if resp_callback else response raise response try: response = adapter.build_response( request, match.get_response(request), ) except Exception as response: match.call_count += 1 self._calls.add(request, response) response = resp_callback(response) if resp_callback else response raise if not match.stream: response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict( dict((v.name, v.value) for _, v in resp_cookies.items())) except (KeyError, TypeError): pass response = resp_callback(response) if resp_callback else response match.call_count += 1 self._calls.add(request, response) return response
def _on_urlopen(self, pool, method, url, body=None, headers=None, **kwargs): built_url = "{0}://{1}{2}".format(pool.scheme, pool.host, url) request = Request(method, built_url, body, headers, pool.scheme, pool.host, pool.port) match = self._find_match(request) resp_callback = self.response_callback if self.allow_external_requests: logger.info('request.allowed-external', extra={ 'url': request.url, }) return _real_urlopen(pool, method, built_url, body=body, headers=headers, **kwargs) if match is None: if request.url.startswith(self.passthru_prefixes): logger.info('request.allowed-passthru', extra={ 'url': request.url, }) return _real_urlopen(pool, method, built_url, body=body, headers=headers, **kwargs) response = self._on_no_match(request, resp_callback) raise response try: response = match.get_response(request) # match.get_response(request), ) except Exception as response: match.call_count += 1 self._calls.add(request, response) response = resp_callback(response) if resp_callback else response raise try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict( dict((v.name, v.value) for _, v in resp_cookies.items())) except (KeyError, TypeError): pass response = resp_callback(response) if resp_callback else response match.call_count += 1 self._calls.add(request, response) return response
def _on_request(self, session, request, **kwargs): match = self._find_match(request) # TODO(dcramer): find the correct class for this if match is None: error_msg = 'Connection refused: {0} {1}'.format( request.method, request.url) response = ConnectionError(error_msg) self._calls.add(request, response) raise response if 'body' in match and isinstance(match['body'], Exception): self._calls.add(request, match['body']) raise match['body'] headers = { 'Content-Type': match['content_type'], } if 'callback' in match: # use callback status, r_headers, body = match['callback'](request) if isinstance(body, six.text_type): body = body.encode('utf-8') body = BufferIO(body) headers.update(r_headers) elif 'body' in match: if match['adding_headers']: headers.update(match['adding_headers']) status = match['status'] body = BufferIO(match['body']) response = HTTPResponse( status=status, body=body, headers=headers, preload_content=False, ) adapter = session.get_adapter(request.url) response = adapter.build_response(request, response) if not match.get('stream'): response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict( dict((v.name, v.value) for _, v in resp_cookies.items())) session.cookies = response.cookies except (KeyError, TypeError): pass self._calls.add(request, response) if kwargs.get('allow_redirects') and response.is_redirect: # include redirect resolving logic from requests.sessions.Session keep_kws = ('stream', 'timeout', 'cert', 'proxies') resolve_kwargs = dict([(k, v) for (k, v) in kwargs.items() if k in keep_kws]) # this recurses if response.is_redirect, # but limited by session.max_redirects gen = session.resolve_redirects(response, request, **resolve_kwargs) history = [resp for resp in gen] # Shuffle things around if there's history. if history: # Insert the first (original) request at the start history.insert(0, response) # Get the last request made response = history.pop() response.history = history return response
def _init_scan_options(self): # Session self.session = session() self.session.timeout = self.args.timeout self.session.verify = False # TODO: debug and check # self.session.mount("http://", HTTPAdapter(max_retries=self.args.max_retries)) # self.session.mount("https://", HTTPAdapter(max_retries=self.args.max_retries)) # http://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request # Max retries adapters.DEFAULT_RETRIES = self.args.max_retries # TOR if self.args.tor: self.output.write_log("TOR usage detected. Making some checks.") self.session.proxies = { 'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050' } url = 'http://ifconfig.me/ip' real_ip, tor_ip = None, None # Ger real IP address try: real_ip = get(url).text.strip() except Exception as exception: self.output.print_and_log("Couldn't get real IP address. Check yout internet connection.", logging.ERROR) self.output.write_log(str(exception), logging.ERROR) exit(-1) # Get TOR IP address try: tor_ip = self.session.get(url).text.strip() except Exception as exception: self.output.print_and_log("TOR socks proxy doesn't seem to be working.", logging.ERROR) self.output.write_log(str(exception), logging.ERROR) exit(-1) # Show IP addresses self.output.print_and_log('Real IP: %s TOR IP: %s' % (real_ip, tor_ip)) if real_ip == tor_ip: self.output.print_and_log("TOR doesn't work! Stop to be secure.", logging.ERROR) exit(-1) # Proxy if self.args.proxy is not None: self.session.proxies = {"https": self.args.proxy, "http": self.args.proxy} # Auth if self.args.auth is not None: items = self.args.auth.split(':') self.session.auth = (items[0], items[1]) # Cookies self.cookies = {} if self.args.cookies is not None: self.cookies = Cookies.from_request(self.args.cookies) # Cookies from file if self.args.load_cookies is not None: if not path.exists(self.args.load_cookies) or not path.isfile(self.args.load_cookies): self.output.print_and_log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR) exit(-1) self.cookies = MozillaCookieJar(self.args.load_cookies) self.cookies.load() self.session.cookies = self.cookies # User-Agent self.ua = UserAgent() if self.args.random_agent else None
def _on_request(self, session, request, **kwargs): match = self._find_match(request) # TODO(dcramer): find the correct class for this if match is None: error_msg = 'Connection refused: {0} {1}'.format(request.method, request.url) response = ConnectionError(error_msg) self._calls.add(request, response) raise response if 'body' in match and isinstance(match['body'], Exception): self._calls.add(request, match['body']) raise match['body'] headers = { 'Content-Type': match['content_type'], } if 'callback' in match: # use callback status, r_headers, body = match['callback'](request) if isinstance(body, six.text_type): body = body.encode('utf-8') body = BufferIO(body) headers.update(r_headers) elif 'body' in match: if match['adding_headers']: headers.update(match['adding_headers']) status = match['status'] body = BufferIO(match['body']) response = HTTPResponse( status=status, body=body, headers=headers, preload_content=False, ) adapter = session.get_adapter(request.url) response = adapter.build_response(request, response) if not match.get('stream'): response.content # NOQA try: resp_cookies = Cookies.from_request(response.headers['set-cookie']) response.cookies = cookiejar_from_dict(dict( (v.name, v.value) for _, v in resp_cookies.items() )) session.cookies = response.cookies except (KeyError, TypeError): pass self._calls.add(request, response) if kwargs.get('allow_redirects') and response.is_redirect: # include redirect resolving logic from requests.sessions.Session keep_kws = ('stream', 'timeout', 'cert', 'proxies') resolve_kwargs = dict([(k, v) for (k, v) in kwargs.items() if k in keep_kws]) # this recurses if response.is_redirect, # but limited by session.max_redirects gen = session.resolve_redirects(response, request, **resolve_kwargs) history = [resp for resp in gen] # Shuffle things around if there's history. if history: # Insert the first (original) request at the start history.insert(0, response) # Get the last request made response = history.pop() response.history = history return response
def _init_scan_options(self): # Session self.session = session() self.session.timeout = self.args.timeout self.session.verify = False # TODO: debug and check # self.session.mount("http://", HTTPAdapter(max_retries=self.args.max_retries)) # self.session.mount("https://", HTTPAdapter(max_retries=self.args.max_retries)) # http://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request # Max retries adapters.DEFAULT_RETRIES = self.args.max_retries # TOR if self.args.tor: self.output.write_log("TOR usage detected. Making some checks.") self.session.proxies = { 'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050' } url = 'http://ifconfig.me/ip' real_ip, tor_ip = None, None # Ger real IP address try: real_ip = get(url).text.strip() except Exception as exception: self.output.print_and_log( "Couldn't get real IP address. Check yout internet connection.", logging.ERROR) self.output.write_log(str(exception), logging.ERROR) exit(-1) # Get TOR IP address try: tor_ip = self.session.get(url).text.strip() except Exception as exception: self.output.print_and_log( "TOR socks proxy doesn't seem to be working.", logging.ERROR) self.output.write_log(str(exception), logging.ERROR) exit(-1) # Show IP addresses self.output.print_and_log('Real IP: %s TOR IP: %s' % (real_ip, tor_ip)) if real_ip == tor_ip: self.output.print_and_log( "TOR doesn't work! Stop to be secure.", logging.ERROR) exit(-1) # Proxy if self.args.proxy is not None: self.session.proxies = { "https": self.args.proxy, "http": self.args.proxy } # Auth if self.args.auth is not None: items = self.args.auth.split(':') self.session.auth = (items[0], items[1]) # Cookies self.cookies = {} if self.args.cookies is not None: self.cookies = Cookies.from_request(self.args.cookies) # Cookies from file if self.args.load_cookies is not None: if not path.exists(self.args.load_cookies) or not path.isfile( self.args.load_cookies): self.output.print_and_log( 'Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR) exit(-1) self.cookies = MozillaCookieJar(self.args.load_cookies) self.cookies.load() self.session.cookies = self.cookies # User-Agent self.ua = UserAgent() if self.args.random_agent else None