def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)
# TODO(dcramer): find the correct class for this
if match is None:
error_msg = 'Connection refused: {0} {1}'.format(request.method,
request.url)
response = ConnectionError(error_msg)
response.request = request
self._calls.add(request, response)
raise response
if 'body' in match and isinstance(match['body'], Exception):
self._calls.add(request, match['body'])
raise match['body']
headers = {}
if match['content_type'] is not None:
headers['Content-Type'] = match['content_type']
if 'callback' in match: # use callback
status, r_headers, body = match['callback'](request)
if isinstance(body, six.text_type):
body = body.encode('utf-8')
body = BufferIO(body)
headers.update(r_headers)
elif 'body' in match:
if match['adding_headers']:
headers.update(match['adding_headers'])
status = match['status']
body = BufferIO(match['body'])
response = HTTPResponse(
status=status,
reason=six.moves.http_client.responses[status],
body=body,
headers=headers,
preload_content=False,
# Need to not decode_content to mimic requests
decode_content=False,
)
response = adapter.build_response(request, response)
if not match.get('stream'):
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(dict(
(v.name, v.value)
for _, v
in resp_cookies.items()
))
except (KeyError, TypeError):
pass
self._calls.add(request, response)
return response
def __init__(self, args):
super(HttpScan, self).__init__(args)
self.session = requesocks.session()
adapters.DEFAULT_RETRIES = self.args.max_retries
self.tor = None
if self.args.tor:
self.out.log("Enabling TOR")
self.tor = Torify()
self.session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
if self.args.check_tor:
# Check TOR
self.out.log("Checking IP via TOR")
rip, tip = self.tor.check_ip(verbose=True)
if tip is None:
self.out.log('TOR is not working properly!', logging.ERROR)
exit(-1)
if self.args.cookies is not None:
if path.exists(self.args.cookies) and path.isfile(self.args.cookies):
self.cookies = MozillaCookieJar(self.args.cookies)
self.cookies.load()
else:
# self.out.log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR)
self.cookies = Cookies.from_request(self.args.cookies)
else:
self.cookies = None
self.ua = UserAgent() if self.args.user_agent is None else self.args.user_agent
def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)
# TODO(dcramer): find the correct class for this
if match is None:
error_msg = 'Connection refused: {0} {1}'.format(request.method,
request.url)
response = ConnectionError(error_msg)
response.request = request
self._calls.add(request, response)
raise response
if 'body' in match and isinstance(match['body'], Exception):
self._calls.add(request, match['body'])
raise match['body']
headers = {}
if match['content_type'] is not None:
headers['Content-Type'] = match['content_type']
if 'callback' in match: # use callback
status, r_headers, body = match['callback'](request)
if isinstance(body, six.text_type):
body = body.encode('utf-8')
body = BufferIO(body)
headers.update(r_headers)
elif 'body' in match:
if match['adding_headers']:
headers.update(match['adding_headers'])
status = match['status']
body = BufferIO(match['body'])
response = HTTPResponse(
status=status,
reason=six.moves.http_client.responses[status],
body=body,
headers=headers,
preload_content=False,
)
response = adapter.build_response(request, response)
if not match.get('stream'):
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(dict(
(v.name, v.value)
for _, v
in resp_cookies.items()
))
except (KeyError, TypeError):
pass
self._calls.add(request, response)
return response
def _cookies_from_headers(headers):
if sys.version_info[:2] < (3, 4):
from cookies import Cookies
resp_cookies = Cookies.from_request(headers["set-cookie"])
cookies_dict = {v.name: v.value for _, v in resp_cookies.items()}
else:
import biscuits
cookies_dict = biscuits.parse(headers["set-cookie"])
return cookiejar_from_dict(cookies_dict)
def _cookies_from_headers(headers):
try:
import http.cookies as cookies
resp_cookie = cookies.SimpleCookie()
resp_cookie.load(headers["set-cookie"])
cookies_dict = {name: v.value for name, v in resp_cookie.items()}
except ImportError:
from cookies import Cookies
resp_cookies = Cookies.from_request(headers["set-cookie"])
cookies_dict = {v.name: v.value for _, v in resp_cookies.items()}
return cookiejar_from_dict(cookies_dict)
def _cookies_from_headers(headers):
try:
import http.cookies as cookies
resp_cookie = cookies.SimpleCookie()
resp_cookie.load(headers["set-cookie"])
cookies_dict = {name: v.value for name, v in resp_cookie.items()}
except ImportError:
from cookies import Cookies
resp_cookies = Cookies.from_request(headers["set-cookie"])
cookies_dict = {v.name: v.value for _, v in resp_cookies.items()}
return cookiejar_from_dict(cookies_dict)
def obtain_session(self, env):
cookie_str = env.get('HTTP_COOKIE', None)
session = None
if cookie_str:
cookie = Cookies.from_request(
cookie_str,
ignore_bad_cookies=True,
).get('session', None)
if cookie and cookie.value:
if cookie.value in self.sessions:
# Session found
session = self.sessions[cookie.value]
if session.is_dead():
session = None
return session
def obtain_session(self, env):
cookie_str = env.get('HTTP_COOKIE', None)
session = None
if cookie_str:
cookie = Cookies.from_request(
cookie_str,
ignore_bad_cookies=True,
).get('session', None)
if cookie and cookie.value:
if cookie.value in self.sessions:
# Session found
session = self.sessions[cookie.value]
if session.is_dead():
session = None
return session
def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)
resp_callback = self.response_callback
if self.allow_external_requests:
logger.info('request.allowed-external',
extra={
'url': request.url,
})
return _real_send(adapter, request)
if match is None:
if request.url.startswith(self.passthru_prefixes):
logger.info('request.allowed-passthru',
extra={
'url': request.url,
})
return _real_send(adapter, request)
response = self._no_match(request, resp_callback)
raise response
try:
response = adapter.build_response(
request,
match.get_response(request),
)
except Exception as response:
match.call_count += 1
self._calls.add(request, response)
response = resp_callback(response) if resp_callback else response
raise
if not match.stream:
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(
dict((v.name, v.value) for _, v in resp_cookies.items()))
except (KeyError, TypeError):
pass
response = resp_callback(response) if resp_callback else response
match.call_count += 1
self._calls.add(request, response)
return response
def __init__(self, cnxn, method, url, headers, body):
self.cnxn = cnxn
self.method = method
self.url = url
self.headers = headers
self.body = body
# A counter to help troubleshoot.
self._id = Request._next_id
Request._next_id = (Request._next_id + 1) % 1000000
self.form = self.parse_form()
c = headers.get('cookie', None)
if c:
self.cookies = Cookies.from_request(c)
else:
self.cookies = Cookies()
def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)
# TODO(dcramer): find the correct class for this
if match is None:
error_msg = "Connection refused: {0} {1}".format(request.method, request.url)
response = ConnectionError(error_msg)
self._calls.add(request, response)
raise response
if "body" in match and isinstance(match["body"], Exception):
self._calls.add(request, match["body"])
raise match["body"]
headers = {"Content-Type": match["content_type"]}
if "callback" in match: # use callback
status, r_headers, body = match["callback"](request)
if isinstance(body, six.text_type):
body = body.encode("utf-8")
body = BufferIO(body)
headers.update(r_headers)
elif "body" in match:
if match["adding_headers"]:
headers.update(match["adding_headers"])
status = match["status"]
body = BufferIO(match["body"])
response = HTTPResponse(status=status, body=body, headers=headers, preload_content=False)
response = adapter.build_response(request, response)
if not match.get("stream"):
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers["set-cookie"])
response.cookies = cookiejar_from_dict(dict((v.name, v.value) for _, v in resp_cookies.items()))
except (KeyError, TypeError):
pass
self._calls.add(request, response)
return response
def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)
resp_callback = self.response_callback
if match is None:
if request.url.startswith(self.passthru_prefixes):
logger.info("request.allowed-passthru", extra={"url": request.url})
return _real_send(adapter, request, **kwargs)
error_msg = "Connection refused: {0} {1}".format(
request.method, request.url
)
response = ConnectionError(error_msg)
response.request = request
self._calls.add(request, response)
response = resp_callback(response) if resp_callback else response
raise response
try:
response = adapter.build_response(request, match.get_response(request))
except Exception as response:
match.call_count += 1
self._calls.add(request, response)
response = resp_callback(response) if resp_callback else response
raise
if not match.stream:
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers["set-cookie"])
response.cookies = cookiejar_from_dict(
dict((v.name, v.value) for _, v in resp_cookies.items())
)
except (KeyError, TypeError):
pass
response = resp_callback(response) if resp_callback else response
match.call_count += 1
self._calls.add(request, response)
return response
def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)
resp_callback = self.response_callback
if match is None:
error_msg = 'Connection refused: {0} {1}'.format(
request.method, request.url)
response = ConnectionError(error_msg)
response.request = request
self._calls.add(request, response)
response = resp_callback(response) if resp_callback else response
raise response
try:
response = adapter.build_response(
request,
match.get_response(request),
)
except Exception as response:
match.call_count += 1
self._calls.add(request, response)
response = resp_callback(response) if resp_callback else response
raise
if not match.stream:
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(
dict((v.name, v.value) for _, v in resp_cookies.items()))
except (KeyError, TypeError):
pass
response = resp_callback(response) if resp_callback else response
match.call_count += 1
self._calls.add(request, response)
return response
def _on_urlopen(self,
pool,
method,
url,
body=None,
headers=None,
**kwargs):
built_url = "{0}://{1}{2}".format(pool.scheme, pool.host, url)
request = Request(method, built_url, body, headers, pool.scheme,
pool.host, pool.port)
match = self._find_match(request)
resp_callback = self.response_callback
if self.allow_external_requests:
logger.info('request.allowed-external',
extra={
'url': request.url,
})
return _real_urlopen(pool,
method,
built_url,
body=body,
headers=headers,
**kwargs)
if match is None:
if request.url.startswith(self.passthru_prefixes):
logger.info('request.allowed-passthru',
extra={
'url': request.url,
})
return _real_urlopen(pool,
method,
built_url,
body=body,
headers=headers,
**kwargs)
response = self._on_no_match(request, resp_callback)
raise response
try:
response = match.get_response(request)
# match.get_response(request), )
except Exception as response:
match.call_count += 1
self._calls.add(request, response)
response = resp_callback(response) if resp_callback else response
raise
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(
dict((v.name, v.value) for _, v in resp_cookies.items()))
except (KeyError, TypeError):
pass
response = resp_callback(response) if resp_callback else response
match.call_count += 1
self._calls.add(request, response)
return response
def _on_request(self, session, request, **kwargs):
match = self._find_match(request)
# TODO(dcramer): find the correct class for this
if match is None:
error_msg = 'Connection refused: {0} {1}'.format(
request.method, request.url)
response = ConnectionError(error_msg)
self._calls.add(request, response)
raise response
if 'body' in match and isinstance(match['body'], Exception):
self._calls.add(request, match['body'])
raise match['body']
headers = {
'Content-Type': match['content_type'],
}
if 'callback' in match: # use callback
status, r_headers, body = match['callback'](request)
if isinstance(body, six.text_type):
body = body.encode('utf-8')
body = BufferIO(body)
headers.update(r_headers)
elif 'body' in match:
if match['adding_headers']:
headers.update(match['adding_headers'])
status = match['status']
body = BufferIO(match['body'])
response = HTTPResponse(
status=status,
body=body,
headers=headers,
preload_content=False,
)
adapter = session.get_adapter(request.url)
response = adapter.build_response(request, response)
if not match.get('stream'):
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(
dict((v.name, v.value) for _, v in resp_cookies.items()))
session.cookies = response.cookies
except (KeyError, TypeError):
pass
self._calls.add(request, response)
if kwargs.get('allow_redirects') and response.is_redirect:
# include redirect resolving logic from requests.sessions.Session
keep_kws = ('stream', 'timeout', 'cert', 'proxies')
resolve_kwargs = dict([(k, v) for (k, v) in kwargs.items()
if k in keep_kws])
# this recurses if response.is_redirect,
# but limited by session.max_redirects
gen = session.resolve_redirects(response, request,
**resolve_kwargs)
history = [resp for resp in gen]
# Shuffle things around if there's history.
if history:
# Insert the first (original) request at the start
history.insert(0, response)
# Get the last request made
response = history.pop()
response.history = history
return response
def _init_scan_options(self):
# Session
self.session = session()
self.session.timeout = self.args.timeout
self.session.verify = False
# TODO: debug and check
# self.session.mount("http://", HTTPAdapter(max_retries=self.args.max_retries))
# self.session.mount("https://", HTTPAdapter(max_retries=self.args.max_retries))
# http://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
# Max retries
adapters.DEFAULT_RETRIES = self.args.max_retries
# TOR
if self.args.tor:
self.output.write_log("TOR usage detected. Making some checks.")
self.session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
url = 'http://ifconfig.me/ip'
real_ip, tor_ip = None, None
# Ger real IP address
try:
real_ip = get(url).text.strip()
except Exception as exception:
self.output.print_and_log("Couldn't get real IP address. Check yout internet connection.",
logging.ERROR)
self.output.write_log(str(exception), logging.ERROR)
exit(-1)
# Get TOR IP address
try:
tor_ip = self.session.get(url).text.strip()
except Exception as exception:
self.output.print_and_log("TOR socks proxy doesn't seem to be working.", logging.ERROR)
self.output.write_log(str(exception), logging.ERROR)
exit(-1)
# Show IP addresses
self.output.print_and_log('Real IP: %s TOR IP: %s' % (real_ip, tor_ip))
if real_ip == tor_ip:
self.output.print_and_log("TOR doesn't work! Stop to be secure.", logging.ERROR)
exit(-1)
# Proxy
if self.args.proxy is not None:
self.session.proxies = {"https": self.args.proxy,
"http": self.args.proxy}
# Auth
if self.args.auth is not None:
items = self.args.auth.split(':')
self.session.auth = (items[0], items[1])
# Cookies
self.cookies = {}
if self.args.cookies is not None:
self.cookies = Cookies.from_request(self.args.cookies)
# Cookies from file
if self.args.load_cookies is not None:
if not path.exists(self.args.load_cookies) or not path.isfile(self.args.load_cookies):
self.output.print_and_log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR)
exit(-1)
self.cookies = MozillaCookieJar(self.args.load_cookies)
self.cookies.load()
self.session.cookies = self.cookies
# User-Agent
self.ua = UserAgent() if self.args.random_agent else None
def _on_request(self, session, request, **kwargs):
match = self._find_match(request)
# TODO(dcramer): find the correct class for this
if match is None:
error_msg = 'Connection refused: {0} {1}'.format(request.method,
request.url)
response = ConnectionError(error_msg)
self._calls.add(request, response)
raise response
if 'body' in match and isinstance(match['body'], Exception):
self._calls.add(request, match['body'])
raise match['body']
headers = {
'Content-Type': match['content_type'],
}
if 'callback' in match: # use callback
status, r_headers, body = match['callback'](request)
if isinstance(body, six.text_type):
body = body.encode('utf-8')
body = BufferIO(body)
headers.update(r_headers)
elif 'body' in match:
if match['adding_headers']:
headers.update(match['adding_headers'])
status = match['status']
body = BufferIO(match['body'])
response = HTTPResponse(
status=status,
body=body,
headers=headers,
preload_content=False,
)
adapter = session.get_adapter(request.url)
response = adapter.build_response(request, response)
if not match.get('stream'):
response.content # NOQA
try:
resp_cookies = Cookies.from_request(response.headers['set-cookie'])
response.cookies = cookiejar_from_dict(dict(
(v.name, v.value)
for _, v
in resp_cookies.items()
))
session.cookies = response.cookies
except (KeyError, TypeError):
pass
self._calls.add(request, response)
if kwargs.get('allow_redirects') and response.is_redirect:
# include redirect resolving logic from requests.sessions.Session
keep_kws = ('stream', 'timeout', 'cert', 'proxies')
resolve_kwargs = dict([(k, v) for (k, v) in kwargs.items() if
k in keep_kws])
# this recurses if response.is_redirect,
# but limited by session.max_redirects
gen = session.resolve_redirects(response, request,
**resolve_kwargs)
history = [resp for resp in gen]
# Shuffle things around if there's history.
if history:
# Insert the first (original) request at the start
history.insert(0, response)
# Get the last request made
response = history.pop()
response.history = history
return response
def _init_scan_options(self):
# Session
self.session = session()
self.session.timeout = self.args.timeout
self.session.verify = False
# TODO: debug and check
# self.session.mount("http://", HTTPAdapter(max_retries=self.args.max_retries))
# self.session.mount("https://", HTTPAdapter(max_retries=self.args.max_retries))
# http://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
# Max retries
adapters.DEFAULT_RETRIES = self.args.max_retries
# TOR
if self.args.tor:
self.output.write_log("TOR usage detected. Making some checks.")
self.session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
url = 'http://ifconfig.me/ip'
real_ip, tor_ip = None, None
# Ger real IP address
try:
real_ip = get(url).text.strip()
except Exception as exception:
self.output.print_and_log(
"Couldn't get real IP address. Check yout internet connection.",
logging.ERROR)
self.output.write_log(str(exception), logging.ERROR)
exit(-1)
# Get TOR IP address
try:
tor_ip = self.session.get(url).text.strip()
except Exception as exception:
self.output.print_and_log(
"TOR socks proxy doesn't seem to be working.",
logging.ERROR)
self.output.write_log(str(exception), logging.ERROR)
exit(-1)
# Show IP addresses
self.output.print_and_log('Real IP: %s TOR IP: %s' %
(real_ip, tor_ip))
if real_ip == tor_ip:
self.output.print_and_log(
"TOR doesn't work! Stop to be secure.", logging.ERROR)
exit(-1)
# Proxy
if self.args.proxy is not None:
self.session.proxies = {
"https": self.args.proxy,
"http": self.args.proxy
}
# Auth
if self.args.auth is not None:
items = self.args.auth.split(':')
self.session.auth = (items[0], items[1])
# Cookies
self.cookies = {}
if self.args.cookies is not None:
self.cookies = Cookies.from_request(self.args.cookies)
# Cookies from file
if self.args.load_cookies is not None:
if not path.exists(self.args.load_cookies) or not path.isfile(
self.args.load_cookies):
self.output.print_and_log(
'Could not find cookie file: %s' % self.args.load_cookies,
logging.ERROR)
exit(-1)
self.cookies = MozillaCookieJar(self.args.load_cookies)
self.cookies.load()
self.session.cookies = self.cookies
# User-Agent
self.ua = UserAgent() if self.args.random_agent else None
