def __build_url_opener(self, uri):
"""
Build the url opener for managing HTTP Basic Athentication
"""
# Create an OpenerDirector with support
# for Basic HTTP Authentication...
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(realm=None,
uri=uri,
user=self.client_id,
passwd=self.client_secret)
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(handler)
return opener
def post(self,
url,
data=None,
cookies=None,
headers=None,
auth=None,
method='auto',
charset='utf-8'):
self.url = self.app + url
# if this POST form requires a postback do it
if data and '_formname' in data and self.postbacks and \
self.history and self.history[-1][1] != self.url:
# to bypass the web2py CSRF need to get formkey
# before submitting the form
self.get(url, cookies=cookies, headers=headers, auth=auth)
# unless cookies are specified, recycle cookies
if cookies is None:
cookies = self.cookies
cookies = cookies or {}
headers = headers or {}
args = [
urllib2.HTTPCookieProcessor(self.cookiejar),
urllib2.HTTPHandler(debuglevel=0)
]
# if required do basic auth
if auth:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(**auth)
args.append(auth_handler)
opener = urllib2.build_opener(*args)
# copy headers from dict to list of key,value
headers_list = []
for key, value in iteritems(self.default_headers):
if not key in headers:
headers[key] = value
for key, value in iteritems(headers):
if isinstance(value, (list, tuple)):
for v in value:
headers_list.append((key, v))
else:
headers_list.append((key, value))
# move cookies to headers
for key, value in iteritems(cookies):
headers_list.append(('Cookie', '%s=%s' % (key, value)))
# add headers to request
for key, value in headers_list:
opener.addheaders.append((key, str(value)))
# assume everything is ok and make http request
error = None
try:
if isinstance(data, str):
self.method = 'POST' if method == 'auto' else method
elif isinstance(data, dict):
self.method = 'POST' if method == 'auto' else method
# if there is only one form, set _formname automatically
if not '_formname' in data and len(self.forms) == 1:
data['_formname'] = next(iter(
self.forms.keys())) # Use the first key
# if there is no formkey but it is known, set it
if '_formname' in data and not '_formkey' in data and \
data['_formname'] in self.forms:
data['_formkey'] = self.forms[data['_formname']]
# time the POST request
data = urlencode(data, doseq=True)
else:
self.method = 'GET' if method == 'auto' else method
data = None
t0 = time.time()
self.response = opener.open(self.url, to_bytes(data))
self.time = time.time() - t0
except urllib2.HTTPError as er:
error = er
# catch HTTP errors
self.time = time.time() - t0
self.response = er
if hasattr(self.response, 'getcode'):
self.status = self.response.getcode()
else: #python2.5
self.status = None
self.text = self.response.read()
if charset:
if charset == 'auto':
charset = self.response.headers.getparam('charset')
self.text = to_native(self.text, charset)
# In PY3 self.response.headers are case sensitive
self.headers = dict()
for h in self.response.headers:
self.headers[h.lower()] = self.response.headers[h]
# treat web2py tickets as special types of errors
if error is not None:
if 'web2py_error' in self.headers:
raise RuntimeError(self.headers['web2py_error'])
else:
raise error
self._parse_headers_in_cookies()
# check is a new session id has been issued, symptom of broken session
if self.session_regex is not None:
for cookie, value in iteritems(self.cookies):
match = self.session_regex.match(cookie)
if match:
name = match.group('name')
if name in self.sessions and self.sessions[name] != value:
print(RuntimeError('Changed session ID %s' % name))
self.sessions[name] = value
# find all forms and formkeys in page
if charset:
self.forms = {}
for match in FORM_REGEX.finditer(self.text):
self.forms[match.group('formname')] = match.group('formkey')
# log this request
self.history.append((self.method, self.url, self.status, self.time))
def post(self, url, data=None, cookies=None,
headers=None, auth=None, method='auto'):
self.url = self.app + url
# if this POST form requires a postback do it
if data and '_formname' in data and self.postbacks and \
self.history and self.history[-1][1] != self.url:
# to bypass the web2py CSRF need to get formkey
# before submitting the form
self.get(url, cookies=cookies, headers=headers, auth=auth)
# unless cookies are specified, recycle cookies
if cookies is None:
cookies = self.cookies
cookies = cookies or {}
headers = headers or {}
args = [
urllib2.HTTPCookieProcessor(self.cookiejar),
urllib2.HTTPHandler(debuglevel=0)
]
# if required do basic auth
if auth:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(**auth)
args.append(auth_handler)
opener = urllib2.build_opener(*args)
# copy headers from dict to list of key,value
headers_list = []
for key, value in iteritems(self.default_headers):
if not key in headers:
headers[key] = value
for key, value in iteritems(headers):
if isinstance(value, (list, tuple)):
for v in value:
headers_list.append((key, v))
else:
headers_list.append((key, value))
# move cookies to headers
for key, value in iteritems(cookies):
headers_list.append(('Cookie', '%s=%s' % (key, value)))
# add headers to request
for key, value in headers_list:
opener.addheaders.append((key, str(value)))
# assume everything is ok and make http request
error = None
try:
if isinstance(data, str):
self.method = 'POST' if method=='auto' else method
elif isinstance(data, dict):
self.method = 'POST' if method=='auto' else method
# if there is only one form, set _formname automatically
if not '_formname' in data and len(self.forms) == 1:
data['_formname'] = self.forms.keys()[0]
# if there is no formkey but it is known, set it
if '_formname' in data and not '_formkey' in data and \
data['_formname'] in self.forms:
data['_formkey'] = self.forms[data['_formname']]
# time the POST request
data = urlencode(data, doseq=True)
else:
self.method = 'GET' if method=='auto' else method
data = None
t0 = time.time()
self.response = opener.open(self.url, to_bytes(data))
self.time = time.time() - t0
except urllib2.HTTPError as er:
error = er
# catch HTTP errors
self.time = time.time() - t0
self.response = er
if hasattr(self.response, 'getcode'):
self.status = self.response.getcode()
else:#python2.5
self.status = None
self.text = to_native(self.response.read())
# In PY3 self.response.headers are case sensitive
self.headers = dict()
for h in self.response.headers:
self.headers[h.lower()] = self.response.headers[h]
# treat web2py tickets as special types of errors
if error is not None:
if 'web2py_error' in self.headers:
raise RuntimeError(self.headers['web2py_error'])
else:
raise error
self._parse_headers_in_cookies()
# check is a new session id has been issued, symptom of broken session
if self.session_regex is not None:
for cookie, value in iteritems(self.cookies):
match = self.session_regex.match(cookie)
if match:
name = match.group('name')
if name in self.sessions and self.sessions[name] != value:
print(RuntimeError('Changed session ID %s' % name))
self.sessions[name] = value
# find all forms and formkeys in page
self.forms = {}
for match in FORM_REGEX.finditer(to_native(self.text)):
self.forms[match.group('formname')] = match.group('formkey')
# log this request
self.history.append((self.method, self.url, self.status, self.time))