def is_present(mpkt):
hdrs = mpkt.cfields.get('dissector.http.headers', None)
if not hdrs:
return False
if 'cookie' in hdrs:
for cstr in hdrs['cookie']:
cookie = SimpleCookie()
cookie.load(cstr)
for k, mar in cookie.items():
cookies[k].append(mar.value)
return True
elif 'set-cookie' in hdrs:
for cstr in hdrs['set-cookie']:
cookie = SimpleCookie()
cookie.load(cstr)
for k, mar in cookie.items():
cookies[k].append(mar.value)
return True
return False
def is_present(mpkt):
hdrs = mpkt.cfields.get('dissector.http.headers', None)
if not hdrs:
return False
if 'cookie' in hdrs:
for cstr in hdrs['cookie']:
cookie = SimpleCookie()
cookie.load(cstr)
for k, mar in cookie.items():
cookies[k].append(mar.value)
return True
elif 'set-cookie' in hdrs:
for cstr in hdrs['set-cookie']:
cookie = SimpleCookie()
cookie.load(cstr)
for k, mar in cookie.items():
cookies[k].append(mar.value)
return True
return False
def parseStrToCookies(self, cookieStr):
SC = SimpleCookie()
SC.load(cookieStr)
cookies = {}
for key, morsel in SC.items():
cookies[key] = morsel.value
return cookies
def cookies(self):
"""Returns a dictionary mapping cookie names to their values."""
if self._cookies is None:
c = SimpleCookie(self.environ.get('HTTP_COOKIE'))
self._cookies = dict([(k.decode('utf-8'), v.value.decode('utf-8'))
for k, v in c.items()])
return self._cookies
def get_cookies(str):
cookie = SimpleCookie()
cookie.load(str)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def get_cookies(str):
cookie = SimpleCookie()
cookie.load(str)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def cookies(self, rawdata):
"""Set Response Cookies.
:param rawdata: (str or dict). List of `cookielib.Cookie`.
"""
sc = SimpleCookie()
sc.load(rawdata)
self._cookies = sc.items()
def cookies(self, rawdata):
"""Set Response Cookies.
:param rawdata: (str or dict). List of `cookielib.Cookie`.
"""
sc = SimpleCookie()
sc.load(rawdata)
self._cookies = sc.items()
def get_cookies_ignore_origin(response):
cookies = dict_from_cookiejar(response.cookies)
if 0 == len(cookies) and SET_COOKIE_KEY in response.headers:
simple_cookie = SimpleCookie(response.headers[SET_COOKIE_KEY])
for key, morsel in simple_cookie.items():
cookies[key] = morsel.value
return cookies
def __retrieve_cookie_values(self, raw_string):
cookies = {}
cookie = SimpleCookie()
cookie.load(raw_string)
for key, morsel in cookie.items():
if any(name.lower() in key.lower()
for name in Scanner.cookie_keys):
cookies[key] = morsel.value
return cookies
def read_sparo_html(url):
import requests
coockie = {}
cookies = {
'sucuri_cloudproxy_uuid_53a9650e8':
'6c3d2f947083eaa2a62370d6c90b4495',
'laravel_session':
'xC5WAE2yuuU0L8k8pCYUQZGh6YMbsv0vjYNvhGBN',
'XSRF-TOKEN':
'eyJpdiI6IjB3OHhLaUNzTEpSXC83ZUtZYnBQVUtnPT0iLCJ2YWx1ZSI6ImtNQWM4b2ZLSE4yN3l0VHFCNEIyMVNmQ2s5THdNdW8zS21tQmluVFVZNDFSNlpZM3UwRkptd1wvMWxnMHBVT09PM3BlMTU1TVVRdm02S3pCUXZlVjU4Zz09IiwibWFjIjoiYzcxZTM4MDU4ZmMxMGE5NjljMTdiMzBjZDI0ODcyOTEzNDc1NDBjNmU5MGQwN2JiZTZiMmNjNjZkOGUwMjhlMSJ9',
}
headers = {
'Host': Domain,
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'he,he-IL;q=0.8,en-US;q=0.5,en;q=0.3',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
x = requests.get(url, headers=headers).content
if 'var s={}' in x:
coockie2, token = (get_c(x))
cookie = SimpleCookie()
cookie.load(str(coockie2))
# Even though SimpleCookie is dictionary-like, it internally uses a Morsel object
# which is incompatible with requests. Manually construct a dictionary instead.
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
cookies['XSRF-TOKEN'] = str(token)
cookies['max-age'] = '86400'
#cookies['X-CSRF-TOKEN'] = "wYZBYILaf7Dbw0TZDKjJ3udNO8aPUa7Bns3EQcUe"
#cookies['XSRF-TOKEN'] = "eyJpdiI6IkVZSmViYXpCdWZwMThjSmJ6dVwvRUhRPT0iLCJ2YWx1ZSI6IkpPQkE5QW83QWc1NzMxY2VwbGR1NkM3RlRDOTJ3dGpXaFF6Q29GWENlY0NOQ3NzSTBTK2ZRN2xPeDJ6dnh5TVlnNEQwbGgweHkwXC85djllSkZ0WTVOQT09IiwibWFjIjoiNTg5Y2U1OTIzY2E0YTM5ZTYxMTY2NjBhOTI4NjhjN2MyMGU1ZDJmM2U0MmEzNDBkYThmNDQ4YzU1M2NkODY5ZCJ9"
#cookies['laravel_session'] = "xC5WAE2yuuU0L8k8pCYUQZGh6YMbsv0vjYNvhGBN"
x = requests.get(url, headers=headers, cookies=cookies)
x.encoding = 'utf-8'
x = x.content
return x
def test_get_json(yahoo_response, cookies):
response = yahoo_response('v2/groups/groupname/files/a/2?param1=c¶m2=4', {'result': 'returned data'})
yga = YahooGroupsAPI('groupname', cookies)
json = yga.get_json('files', 'a', 2, param1='c', param2=4)
request = response.calls[0].request
request_cookies = SimpleCookie()
request_cookies.load(request.headers['Cookie'])
assert dict(cookies) == {k: v.value for k, v in request_cookies.items()}
assert json == {'result': 'returned data'}
def use_credentials(self, credentials):
fetch_arguments = build_fetch_arguments("/")
credentials(fetch_arguments)
# Selenium doesn't support arbitrary headers (afaik), so we only
# support cookies for now.
if "Cookie" in fetch_arguments.headers:
cookie = SimpleCookie()
cookie.load(fetch_arguments.headers["Cookie"])
for morsel_name, morsel in cookie.items():
self._driver.add_cookie({
"name": morsel_name,
"value": morsel.value
})
def get_dict_cookies(self, rawdata):
'''Returnt dict from cookies raw text'''
try:
from Cookie import SimpleCookie
cookie = SimpleCookie()
cookie.load(rawdata)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
except:
logging.error('Cannot get cookies from raw text', exc_info=True)
cookies = {}
return cookies
def _make_request(self, step, **kwargs):
request_metadata = self._get_step_metadata(step)
log.debug('kwargs are : ' + str(kwargs))
data = None
if not request_metadata:
return None
if 'data' in request_metadata:
data = request_metadata['data']
if 'method' in request_metadata:
method = request_metadata['method']
else:
method = 'GET' if not data else 'POST'
if 'redirect' in request_metadata:
redirect = request_metadata['redirect']
else:
redirect = True
headers = {}
if 'csrf_token' in self.cookies and self.cookies['csrf_token']:
headers['X-CSRF-Token'] = self.cookies['csrf_token']
log.debug('setting csrf header to ' + self.cookies['csrf_token'])
if self.cookies:
cookie_str = "; ".join(
[str(x) + "=" + str(y) for x, y in self.cookies.items()])
headers['Cookie'] = cookie_str
request_params = {
'url': self._get_step_url(step, **kwargs),
'method': method,
'allow_redirects': redirect,
'headers': headers,
'verify': False
}
if (data):
if isinstance(data, dict):
data = {
k: v(self, **kwargs) if callable(v) else v
for k, v in data.items()
}
log.debug('posting data : ' + str(data))
request_params['data'] = json.dumps(data)
headers['Content-Type'] = 'application/json'
else:
request_params['data'] = data
r = web.request(**request_params)
if (r.status_code >= 200 and r.status_code <= 400
and 'set-cookie' in r.headers):
cookies = SimpleCookie()
cookies.load(r.headers['Set-Cookie'])
for key, value in cookies.items():
self.cookies[key] = value.value
return r
def middleware(environ, start_response):
request = Request(environ).copy()
response = request.get_response(django)
if response.status_int != 305:
return response(environ, start_response)
data = json.loads(response.body)
proxy = RemoteProxy([data['base_url']], rewrite_links=True)
request.environ.pop("HTTP_ACCEPT_ENCODING", None)
request.environ['SCRIPT_NAME'] = str(data['script_name']).rstrip("/")
request.environ['PATH_INFO'] = "/" + str(data['path_info'].lstrip("/") )
from Cookie import SimpleCookie as Cookie
orig_cookie = cookie = request.environ.get('HTTP_COOKIE')
if cookie:
cookie = Cookie(cookie)
for key in data['cookie_blacklist']:
cookie.pop(key, None)
cookiestr = []
for key, value in cookie.items():
cookiestr.append(value.OutputString())
cookiestr = "; ".join(cookiestr)
request.environ['HTTP_COOKIE'] = cookiestr
request.environ['HTTP_X_THING_THEME'] = data['theme']
filter = deliverance(proxy)
def per_project_theme(environ):
return "%(wsgi.url_scheme)s://%(HTTP_HOST)s%(HTTP_X_THING_THEME)s" % environ
filter.default_theme = per_project_theme
filter.rule_getter = TemplateRuleGetter(data['deliverance_rules'])
filter.use_internal_subrequest = lambda *args, **kw: False
base_subrequest = filter.build_external_subrequest
def build_external_subrequest(url, orig_req, log):
subreq = base_subrequest(url, orig_req, log)
if url.endswith("/theme/") and orig_cookie:
subreq.environ['HTTP_COOKIE'] = orig_cookie
return subreq
filter.build_external_subrequest = build_external_subrequest
resp = request.get_response(filter)
return resp(request.environ, start_response)
def process_cookie(self, cookie, host):
parsed = SimpleCookie(str(cookie))
for key, c in parsed.items():
if b"domain" in c:
host = c['domain']
url = self.create_url(host)
host_tld = self.tld_parse(url)
self.send_cookie_to_listeners(
CookieData(name=c.key,
value=c.value,
domain="." + host_tld,
url=url,
expiry=self.default_expiry))
def decompose_incoming_envelope(self, ctx, message):
assert message == FlatDictDocument.REQUEST
ctx.transport.itself.decompose_incoming_envelope(self, ctx, message)
if self.parse_cookie:
cookies = ctx.in_header_doc.get('cookie', [])
for cookie_string in cookies:
cookie = SimpleCookie()
cookie.load(cookie_string)
for k,v in cookie.items():
l = ctx.in_header_doc.get(k, [])
l.append(v.coded_value)
ctx.in_header_doc[k] = l
logger.debug('\theader : %r' % (ctx.in_header_doc))
logger.debug('\tbody : %r' % (ctx.in_body_doc))
def decompose_incoming_envelope(self, ctx, message):
assert message == FlatDictDocument.REQUEST
ctx.transport.itself.decompose_incoming_envelope(self, ctx, message)
if self.parse_cookie:
cookies = ctx.in_header_doc.get('cookie', [])
for cookie_string in cookies:
cookie = SimpleCookie()
cookie.load(cookie_string)
for k,v in cookie.items():
l = ctx.in_header_doc.get(k, [])
l.append(v.coded_value)
ctx.in_header_doc[k] = l
logger.debug('\theader : %r' % (ctx.in_header_doc))
logger.debug('\tbody : %r' % (ctx.in_body_doc))
def process_request(self, request, spider):
"""Process a request using the selenium driver if applicable"""
# print 'SeleniumMiddleware process_request: request ' + str(request)
if not request.meta.get('enable_selenium', False):
return None
cookie = SimpleCookie()
cookie.load(request.headers.getlist('Cookie')[0])
driver = self.generate_webdriver()
for cookie_name, morsel in cookie.items():
driver.add_cookie({'name': cookie_name, 'value': morsel.value})
driver.get(request.url)
# wait til page loads
# print driver.find_element_by_tag_name(
# 'title').get_attribute(
# 'innerHTML') + ', ' + request.meta['title']
# print 'compare: ' + str(driver.find_element_by_tag_name(
# 'title').get_attribute(
# 'innerHTML') == request.meta['title'])
# print EC.text_to_be_present_in_element(
# (By.TAG_NAME, 'title'), request.meta['title'])
try:
element = WebDriverWait(
driver,
10).until(lambda x: driver.find_element_by_tag_name('title').
get_attribute('innerHTML') == request.meta['title'])
except TimeoutException:
raise IgnoreRequest('Cannot open: ' + request.url)
# print 'waiting ends: ' + request.meta['title']
body = driver.page_source
# Expose the driver via the "meta" attribute
request.meta.update({'driver': driver})
# print 'send response: ' + str(request)
return HtmlResponse(request.url,
body=body,
encoding='utf-8',
request=request)
def probe_securecookie(self):
"""Something that looks like a securecookie was detected"""
rv = self.make_request(self.config.url)
setcookie = rv.headers.get('set-cookie')
if setcookie is None:
return False
if not '; Path=' in setcookie:
return False
cookie = SimpleCookie(setcookie)
for key, morsel in cookie.items():
if not '?' in morsel.value:
continue
try:
morsel.value.split('?')[0].decode('base64')
return True
except Exception:
return False
return False
def probe_securecookie(self):
"""Something that looks like a securecookie was detected"""
rv = self.make_request(self.config.url)
setcookie = rv.headers.get('set-cookie')
if setcookie is None:
return False
if not '; Path=' in setcookie:
return False
cookie = SimpleCookie(setcookie)
for key, morsel in cookie.items():
if not '?' in morsel.value:
continue
try:
morsel.value.split('?')[0].decode('base64')
return True
except Exception:
return False
return False
def build(resp):
response = Response()
# Pass settings over.
response.config = self.config
if resp:
# Fallback to None if there's no staus_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(
getattr(resp, 'headers', None))
# Start off with our local cookies.
cookies = self.cookies or dict()
# Add new cookies from the server.
if 'set-cookie' in response.headers:
cookie_header = response.headers['set-cookie']
c = SimpleCookie()
c.load(cookie_header)
for k, v in c.items():
cookies.update({k: v.value})
# Save cookies in Response.
response.cookies = cookies
# Save original resopnse for later.
response.raw = resp
if is_error:
response.error = resp
response.url = self.full_url
return response
def build(resp):
response = Response()
# Pass settings over.
response.config = self.config
if resp:
# Fallback to None if there's no staus_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
# Start off with our local cookies.
cookies = self.cookies or dict()
# Add new cookies from the server.
if 'set-cookie' in response.headers:
cookie_header = response.headers['set-cookie']
c = SimpleCookie()
c.load(cookie_header)
for k,v in c.items():
cookies.update({k: v.value})
# Save cookies in Response.
response.cookies = cookies
# Save original resopnse for later.
response.raw = resp
if is_error:
response.error = resp
response.url = self.full_url
return response
class AbstractCookiesTransport:
"""An Transport subclass that retains cookies over its lifetime and has the
ability to store/load cookies in a file.
Taken from http://stackoverflow.com/a/25876504 and modified to implement
cookie file save/load functionality.
"""
def __init__(self, cookiefile=None, **kwargs):
super().__init__(**kwargs)
self.cookiefile = cookiefile
self.cookies = SimpleCookie()
self.load_cookies()
def load_cookies(self, cookiefile=None):
cookiefile = cookiefile or self.cookiefile
if os.path.isfile(cookiefile):
with open(cookiefile) as fd:
self.cookies.load(fd.read())
def save_cookies(self, cookiefile=None):
cookiefile = cookiefile or self.cookiefile
with open(cookiefile, 'w') as fd:
fd.write(self.cookies.output(header="", sep="\n"))
def send_headers(self, connection, headers):
if self.cookies:
cookies = [
"{}={}".format(k,v.value) for k,v in self.cookies.items()
]
cookie_str = "; ".join(cookies)
connection.putheader("Cookie", cookie_str)
super().send_headers(connection, headers)
def parse_response(self, response):
cookies = response.msg.get_all("Set-Cookie") or []
for cookie in cookies:
self.cookies.load(cookie) # Add or replace cookie
self.save_cookies()
return super().parse_response(response)
def requester(target, cookie, user_agent, proxy, match, filename):
try:
url = target + filename
proxy_dict = {}
cookie_dict = {}
if proxy != "":
proxy_dict = {"http": proxy, "https": proxy}
# If the user specified a cookie convert it to a requests acceptable dict
if cookie != "":
cookie_dict = SimpleCookie()
cookie_dict.load(cookie)
cookie = {}
for key, morse1 in cookie_dict.items():
cookie[key] = morse1.value
# If the user want to match a specific string in the response
if match != "":
req = requests.get(url,
cookies=cookie,
headers={"User-Agent": user_agent},
proxies=proxy_dict,
verify=False)
requester.q.put([url, req.status_code, req.content])
else:
req = requests.head(url,
cookies=cookie,
headers={"User-Agent": user_agent},
proxies=proxy_dict,
verify=False)
requester.q.put([url, req.status_code])
except KeyboardInterrupt:
sys.stdout = open(os.devnull, 'w')
#convert cookies
from Cookie import SimpleCookie
Cookie11 = 'Cookie: akavpau_p6=1492510799~id=bad5a5f677b9c92fd22ae7ce69a953d7; search.perf.metric=timerPromiseAll=404ms|timerHeaderAction=53ms|timerSearchAction=404ms|timerFooterAction=26ms|timerPreso=402ms; akavpau_p3=1492510790~id=d7334d155b9d5313f20c6c8b19e356d7; AID=wmlspartner%3D0%3Areflectorid%3D0000000000000000000000%3Alastupd%3D1492510198882; DL=98105%2C47.66059875488281%2C-122.29190063476562%2Cip%2C98105%2CUSA%2CWA; prefper=PREFSTORE~13098~2PREFCITY~1Bellevue~2PREFFULLSTREET~115063%20Main%20St~2PREFSTATE~1WA~2PREFZIP~198105; bstc=ZYcj-sZzb56LzolJAq24WY; NSID=3098.8-5939.8-5678.13-2594.13-2516.13-2317.16-5195.16-3053.18-5272.18-2325.19-5628.21-5073.22-2571.23-2385.25-3794.25-3801.29-2595.29-4137.30-3537.34-3757.34; vtc=ZYcj-sZzb56LzolJAq24WY; exp-ck=vOXNpEqkz0MWIxT; exp=0%2B1492510198%2BZYcj-sZzb56LzolJAq24WY%2B0%2BCcvJB.vOXNp|K8foX.Eqkz0|VxHIY.MWIxT; com.wm.reflector="reflectorid:0000000000000000000000@lastupd:1492510198883@firstcreate:1492510198883"'
output = {'akavpau_p6':'1492510799~id=bad5a5f677b9c92fd22ae7ce69a953d7', 'search.perf.metric':'timerPromiseAll=404ms|timerHeaderAction=53ms|timerSearchAction=404ms|timerFooterAction=26ms|timerPreso=402ms', 'akavpau_p3':'1492510790~id=d7334d155b9d5313f20c6c8b19e356d7', 'AID':'wmlspartner%3D0%3Areflectorid%3D0000000000000000000000%3Alastupd%3D1492510198882', 'DL':'98105%2C47.66059875488281%2C-122.29190063476562%2Cip%2C98105%2CUSA%2CWA', 'prefper':'PREFSTORE~13098~2PREFCITY~1Bellevue~2PREFFULLSTREET~115063%20Main%20St~2PREFSTATE~1WA~2PREFZIP~198105', 'bstc':'ZYcj-sZzb56LzolJAq24WY', 'NSID':'3098.8-5939.8-5678.13-2594.13-2516.13-2317.16-5195.16-3053.18-5272.18-2325.19-5628.21-5073.22-2571.23-2385.25-3794.25-3801.29-2595.29-4137.30-3537.34-3757.34', 'vtc':'ZYcj-sZzb56LzolJAq24WY', 'exp-ck':'vOXNpEqkz0MWIxT', 'exp':'0%2B1492510198%2BZYcj-sZzb56LzolJAq24WY%2B0%2BCcvJB.vOXNp|K8foX.Eqkz0|VxHIY.MWIxT', 'com.wm.reflector':'"reflectorid:0000000000000000000000@lastupd:1492510198883@firstcreate:1492510198883"'}
cookie = SimpleCookie()
cookie.load(Cookie11)
cookies12 = {}
for key, morsel in cookie.items():
cookies12[key] = morsel.value
print cookies12
def check_truth(injection_string):
global data
if config.request_method == "POST":
url = config.injection_url
post_params = deepcopy(config.post_params)
for key in post_params:
if "INJECTION_STRING" in post_params[key]:
post_params[key] = post_params[key].replace('INJECTION_STRING', injection_string)
else:
url = "%s and (%s)" % (config.injection_url, injection_string)
args = {}
if config.cookies:
from Cookie import SimpleCookie
cookie = SimpleCookie()
cookie.load(config.cookies)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
args["cookies"] = cookies
if config.headers:
args["headers"] = config.headers
if config.proxies:
args["proxies"] = config.proxies
if config.request_method == "POST":
args["data"] = post_params
# print injection_string, args['data']
if config.parameter_needs_quote:
url = "%s' and (%s)-- j" % (config.injection_url, injection_string)
if injection_string in data:
return data[injection_string]
try:
if config.request_method == "POST":
response = requests.post(url, allow_redirects=False, **args)
else:
response = requests.get(url, allow_redirects=False, **args)
if config.truth_check == "STATUS_CODE":
if response.status_code == config.truth_status:
lock.acquire()
data[injection_string] = True
add_to_cache(injection_string, True)
lock.release()
return True
elif config.truth_check == "RESPONSE_HEADER" and response.status_code == 302:
if response.headers[config.truth_response_header_name] == config.truth_response_header_value:
lock.acquire()
data[injection_string] = True
add_to_cache(injection_string, True)
lock.release()
return True
elif config.truth_check == "CONTENT_STRING":
content = response.content
if config.truth_string in content:
lock.acquire()
data[injection_string] = True
add_to_cache(injection_string, True)
lock.release()
return True
except:
traceback.print_exc()
raise Exception("Error occurred while requesting the url '''%s'''!" % url)
lock.acquire()
data[injection_string] = False
add_to_cache(injection_string, False)
lock.release()
return False
def do_request(self, req, status, expect_errors):
"""
Executes the given request (``req``), with the expected
``status``. Generally ``.get()`` and ``.post()`` are used
instead.
To use this::
resp = app.do_request(webtest.TestRequest.blank(
'url', ...args...))
Note you can pass any keyword arguments to
``TestRequest.blank()``, which will be set on the request.
These can be arguments like ``content_type``, ``accept``, etc.
"""
__tracebackhide__ = True
errors = StringIO()
req.environ['wsgi.errors'] = errors
if self.cookies:
cookie_header = ''.join([
'%s=%s; ' % (name, cookie_quote(value))
for name, value in self.cookies.items()])
req.environ['HTTP_COOKIE'] = cookie_header
req.environ['paste.testing'] = True
req.environ['paste.testing_variables'] = {}
app = lint.middleware(self.app)
old_stdout = sys.stdout
out = CaptureStdout(old_stdout)
try:
sys.stdout = out
start_time = time.time()
## FIXME: should it be an option to not catch exc_info?
res = req.get_response(app, catch_exc_info=True)
end_time = time.time()
finally:
sys.stdout = old_stdout
res.app = app
res.test_app = self
# We do this to make sure the app_iter is exausted:
res.body
res.errors = errors.getvalue()
total_time = end_time - start_time
for name, value in req.environ['paste.testing_variables'].items():
if hasattr(res, name):
raise ValueError(
"paste.testing_variables contains the variable %r, but "
"the response object already has an attribute by that "
"name" % name)
setattr(res, name, value)
if not expect_errors:
self._check_status(status, res)
self._check_errors(res)
res.cookies_set = {}
for header in res.headers.getall('set-cookie'):
try:
c = SimpleCookie(header)
except CookieError, e:
raise CookieError(
"Could not parse cookie header %r: %s" % (header, e))
for key, morsel in c.items():
self.cookies[key] = morsel.value
res.cookies_set[key] = morsel.value
parser.add_option("-p", "--proxy", dest="proxy")
parser.add_option("-s", "--sleep", dest="seconds_to_sleep")
parser.add_option("-d", "--delay", dest="delay")
parser.add_option("-c", "--cookies", dest="cookies")
options, args = parser.parse_args()
target_url = options.target_url
seconds_to_sleep = float(options.seconds_to_sleep)
delay = float(options.delay)
if options.cookies:
raw_cookies = options.cookies
simple_cookies = SimpleCookie()
simple_cookies.load(raw_cookies)
cookies = {}
for key, morsel in simple_cookies.items():
cookies[key] = morsel.value
else:
cookies = { "1337": "1337" }
if options.proxy:
proxy = {
"http": "%s" % options.proxy,
"https": "%s" % options.proxy
}
else:
proxy = { "http": "None" }
# ASCII character ranges, ordered by probability
upper_underscore = list(range(65,90))
upper_underscore.append(95)
upper_lower_numeric_symbols = list(range(32,126))
class BrowserPythonNetlocData:
def __init__(self, browser, netloc):
# Remove standard port number from netloc string
# Allowed by HTTP W3C and make some web-apps happy
if ':' in netloc:
nl = netloc.split(':')
if nl[1] == '80' or nl[1] == '443':
netloc = nl[0]
self.browser = browser
self.netloc = netloc
self.cookies = SimpleCookie()
self.httpool = {}
self.lock = Lock()
self.header = {}
self.header[
'User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 PYTHON'
self.header['Connection'] = 'keep-alive'
self.header[
'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
self.header['Accept-Language'] = 'en-US,en;q=0.5'
self.header['Content-Type'] = 'application/x-www-form-urlencoded'
self.header['Host'] = self.netloc
def _get_http_pool(self, scheme):
if scheme not in self.httpool:
loc = "%s://%s" % (scheme, self.netloc)
self.browser.log_debug("allocating connection pool to %s" % loc)
self.httpool[scheme] = HTTPPool("%s" % loc)
return self.httpool[scheme]
def __del__(self):
for pool in self.httpool.values():
pool.clear()
def set_cookie(self, url, key, val=None, path=None):
self.browser.log_debug("set cookie: %s, %s=%s" % (url, key, str(val)))
scheme, netloc, path = parse_url(url)
with self.lock:
if type(key) == 'dict':
c = key
self.cookies[c['name']] = c
elif val:
self.cookies[key] = val
if path:
self.cookies[key]["path"] = path
elif path in self.cookies:
del self.cookies[key]
def set_header(self, key, val):
self.browser.log_debug("set header: %s=%s" % (key, val))
with self.lock:
if val is not None:
self.header[key] = val
elif key in self.header:
del self.header[key]
def get_cookies_str(self, url):
"""
get_cookies_str() returns only the cookies applicable to given *url*
"""
_, _, path = parse_url(url)
path_cookies = {}
for key, morsel in self.cookies.items():
p = morsel["path"] if morsel["path"] else "/"
if key not in path_cookies:
path_cookies[key] = {}
path_cookies[key][p] = morsel.value
cookies = {}
for key, paths in path_cookies.items():
# sort by string length: ["/home/path", "/home", "/"]
for p in sorted(paths.keys(), lambda x: len(x), reverse=True):
if path.startswith(p):
cookies[key] = paths[p]
break
return "; ".join("%s=%s" % (key, val) for key, val in cookies.items())
def _get_redirected_request(self, req, response):
new_req = None
if req.status in [301, 302, 303]:
new_req = req.duplicate()
new_req.method = 'GET'
new_req.params = None
new_req.url = response.getheader('location')
elif self.browser.js_redirects:
page_actions = httputils.PageActionsExtractor(req.data)
redirect = page_actions.get_action()
if redirect:
new_req = req.duplicate()
new_req.method = redirect.method
new_req.params = redirect.get_params()
new_req.url = redirect.url
new_req.header[
'Cookie'] = self.browser.browser_get_cookies_str(req.url)
req.page_actions = page_actions
if new_req:
new_req.validator = None
new_req.header['Cookie'] = self.browser.browser_get_cookies_str(
req.url)
if not new_req.url.startswith('http'):
new_req.update_netloc(req.url)
_, new_req.header['Host'], _ = parse_url(new_req.url)
return new_req
return None
def _execute_page_request(self, pool, page, req, path_with_args):
assert (req.method == 'POST'
or not req.params) # only POST request may have params
with pool.borrow() as (conn, is_new):
# conn.set_debuglevel(1)
conn.request(req.method, path_with_args, req.params, req.header)
self.header['Referer'] = req.url
response = conn.getresponse()
req.status = response.status
req.data = response.read()
self.browser.log_debug(" req %s HTTP response: %s, headers: %s" %
(req.id, req.status, response.getheaders()))
# handle new cookies
for cookie in extract_cookies(response):
if sys.version_info[0] < 3 and isinstance(cookie, unicode):
cookie = cookie.encode('ascii', 'ignore')
self.cookies.load(cookie)
# handle redirection
new_req = self._get_redirected_request(req, response)
if new_req:
self.browser.log_debug(" req %s, redirect to %s" %
(new_req.id, new_req.url))
page.add_request(new_req)
# FIXME:
# 1. _get_netloc_data is private method of the browser.
self.browser._get_netloc_data(
new_req.url).execute_page_request(page, new_req)
req.status = new_req.status
req.data = new_req.data
if req.valid_statuses and req.status not in req.valid_statuses:
raise BrowserExc(' req %s, %s %s status %d' (req.id,
req.method,
req.url,
req.status))
if self.browser.validation:
req.validate_response(req.data)
req.complete()
return
def execute_page_request(self, page, req):
scheme, netloc, path_with_args = parse_url(req.url, args=True)
if scheme not in ("http", "https", "ftp"):
msg = "Can't execute request on URL with unsupported scheme: %s, %s" % (
scheme, netloc)
self.browser.log_error(msg)
raise BrowserExc(msg)
req.start()
pool = self._get_http_pool(scheme)
# 10 - is technological retry count to handle closed keep-alive connections
for loop in range(10):
try:
self._execute_page_request(pool, page, req, path_with_args)
return
except pool.temporary_errors as ex:
raise # aandreev
# usually it means server has closed keep-alive connection due to timeout. lets retry
self.browser.log_debug(
"HTTP Exception: %s %s: %s %s, connection closed? retrying"
% (req.method, req.url, type(ex), str(ex)))
req.status = str(type(ex))
time.sleep(0.1 * loop)
except pool.fatal_errors as ex:
req.status = str(type(ex))
req.complete()
self.browser.log_error(
"HTTP Exception: %s %s: %s %s" %
(req.method, req.url, type(ex), str(ex)))
return
req.complete()
self.browser.log_error("HTTPException: %s %s, all retries failed" %
(req.method, req.url))
def execute_page_request_parallel(self, page, reqs, parallel=8):
if not len(reqs):
return
if len(reqs) == 1:
self.execute_page_request(page, reqs[0])
return
def execute(arg):
global data
page, req = arg
try:
self.execute_page_request(page, req)
except RuntimeError:
import traceback
self.browser.log_error("traceback:\n" + traceback.format_exc())
return 0
args = [(page, req) for req in reqs]
if parallel > len(reqs):
parallel = len(reqs)
pool = ThreadPool(parallel)
pool.map(execute, args)
pool.close()
pool.join()
def call_wsgi_app(wsgi_app, request, path_info):
"""
Call the ``wsgi_app`` with ``request`` and return its response.
:param wsgi_app: The WSGI application to be run.
:type wsgi_app: callable
:param request: The Django request.
:type request: :class:`twod.wsgi.handler.TwodWSGIRequest`
:param path_info: The ``PATH_INFO`` to be used by the WSGI application.
:type path: :class:`basestring`
:raises twod.wsgi.exc.ApplicationCallError: If ``path_info`` is not the
last portion of the ``PATH_INFO`` in ``request``.
:return: The response from the WSGI application, turned into a Django
response.
:rtype: :class:`twod.wsgi.TwodResponse`
"""
new_request = request.copy()
# Moving the portion of the path consumed by the current view, from the
# PATH_INTO to the SCRIPT_NAME:
if not request.path_info.endswith(path_info):
raise ApplicationCallError("Path %s is not the last portion of the "
"PATH_INFO in the original request (%s)"
% (path_info, request.path_info))
consumed_path = request.path_info[:-len(path_info)]
new_request.path_info = path_info
new_request.script_name = request.script_name + consumed_path
# If the user has been authenticated in Django, log him in the WSGI app:
if request.user.is_authenticated():
new_request.remote_user = request.user.username
# Cleaning the routing_args, if any. The application should have its own
# arguments, without relying on any arguments from a parent application:
if "wsgiorg.routing_args" in request.environ:
del new_request.environ['wsgiorg.routing_args']
# And the same for the WebOb ad-hoc attributes:
if "webob.adhoc_attrs" in request.environ:
del new_request.environ['webob.adhoc_attrs']
# Calling the WSGI application and getting its response:
(status, headers, body) = new_request.call_application(wsgi_app)
# Turning its response into a Django response:
cookies = SimpleCookie()
django_response = TwodResponse(body, status=status)
for (header, value) in headers:
if header.upper() == "SET-COOKIE":
if isinstance(value, unicode):
# It can't be Unicode:
value = value.encode("us-ascii")
cookies.load(value)
else:
django_response[header] = value
# Setting the cookies from Django:
for (cookie_name, cookie) in cookies.items():
cookie_attributes = {
'key': cookie_name,
'value': cookie.value,
'max_age': cookie.get("max-age"),
'expires': cookie.get("expires"),
'path': cookie.get("path", "/"),
'domain': cookie.get("domain"),
}
django_response.set_cookie(**cookie_attributes)
return django_response
def _extract_response(page, encoding='utf8'):
history = []
set_cookies = []
res = None
try:
for i, url in enumerate(page['history']):
resource = page['resources'].pop(0)
while resource['request']['url'] != url:
resource = page['resources'].pop(0)
if resource['error']:
return resource['error'], None
request = resource['request']
req = PreparedRequest()
req.method = request['method'].encode(encoding)
req.url = request['url'].encode(encoding)
# Set Request Headers
req.headers = CaseInsensitiveDict()
for header in request['headers']:
req.headers[header['name'].encode(encoding)] = header['value'].encode(encoding)
# Set Request Cookies
req._cookies = RequestsCookieJar()
if set_cookies:
if 'Cookie' not in req.headers:
req.headers['Cookie'] = ""
else:
set_cookies.insert(0, '')
req.headers['Cookie'] += "; ".join(set_cookies)
if 'Cookie' in req.headers:
cookies = SimpleCookie()
cookies.load(req.headers['Cookie'])
for key, cookie in cookies.items():
req._cookies.set(key, cookie.value)
req.body = request.get('postData', None)
if req.body:
req.body = req.body.encode(encoding)
response = resource['endReply'] or resource['startReply']
res = Response()
res.encoding = encoding
res.url = response['url'].encode(encoding)
res.status_code = response['status']
for header in response['headers']:
res.headers[header['name'].encode(encoding)] = header['value'].encode(encoding)
if header['name'] == 'Set-Cookie':
set_cookies.append(res.headers[header['name']].rsplit(';', 1)[0])
res.history = list(history)
res.request = req
history.append(res)
res._content = re.sub(
(
'<html><head></head><body>'
'<pre style="word-wrap: break-word; white-space: pre-wrap;">(.*?)</pre>'
'</body></html>'
),
r'\1',
page['content'],
flags=re.DOTALL
).encode(encoding)
return None, res
except IndexError:
return {'errorCode': -1,
'errorString': 'An existing connection was forcibly closed by the remote host'}, None
def call_wsgi_app(wsgi_app, request, path_info):
"""
Call the ``wsgi_app`` with ``request`` and return its response.
:param wsgi_app: The WSGI application to be run.
:type wsgi_app: callable
:param request: The Django request.
:type request: :class:`twod.wsgi.handler.TwodWSGIRequest`
:param path_info: The ``PATH_INFO`` to be used by the WSGI application.
:type path: :class:`basestring`
:raises twod.wsgi.exc.ApplicationCallError: If ``path_info`` is not the
last portion of the ``PATH_INFO`` in ``request``.
:return: The response from the WSGI application, turned into a Django
response.
:rtype: :class:`twod.wsgi.TwodResponse`
"""
new_request = request.copy()
# Moving the portion of the path consumed by the current view, from the
# PATH_INTO to the SCRIPT_NAME:
if not request.path_info.endswith(path_info):
raise ApplicationCallError("Path %s is not the last portion of the "
"PATH_INFO in the original request (%s)" %
(path_info, request.path_info))
consumed_path = request.path_info[:-len(path_info)]
new_request.path_info = path_info
new_request.script_name = request.script_name + consumed_path
# If the user has been authenticated in Django, log him in the WSGI app:
if request.user.is_authenticated():
new_request.remote_user = request.user.username
# Cleaning the routing_args, if any. The application should have its own
# arguments, without relying on any arguments from a parent application:
if "wsgiorg.routing_args" in request.environ:
del new_request.environ['wsgiorg.routing_args']
# And the same for the WebOb ad-hoc attributes:
if "webob.adhoc_attrs" in request.environ:
del new_request.environ['webob.adhoc_attrs']
# Calling the WSGI application and getting its response:
(status, headers, body) = new_request.call_application(wsgi_app)
# Turning its response into a Django response:
cookies = SimpleCookie()
django_response = TwodResponse(body, status=status)
for (header, value) in headers:
if header.upper() == "SET-COOKIE":
if isinstance(value, unicode):
# It can't be Unicode:
value = value.encode("us-ascii")
cookies.load(value)
else:
django_response[header] = value
# Setting the cookies from Django:
for (cookie_name, cookie) in cookies.items():
cookie_attributes = {
'key': cookie_name,
'value': cookie.value,
'max_age': cookie.get("max-age"),
'expires': cookie.get("expires"),
'path': cookie.get("path", "/"),
'domain': cookie.get("domain"),
}
django_response.set_cookie(**cookie_attributes)
return django_response