def main(output_dir):
# Create opener.
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
# Iterate over the files in the docs directory and copy them, as appropriate.
for root, dirs, files in os.walk('.'):
for file_name in files:
if file_name.endswith('.soy') and not file_name.startswith('__'):
# Strip the './' prefix, if appropriate.
if root.startswith('./'):
root = root[2:]
# Construct the URL where the .soy file is being served.
soy_file = file_name
html_file = root + '/' + soy_file[:-len('.soy')] + '.html'
url = 'http://localhost:9811/' + html_file
# Fetch url and copy its contents to output_dir.
req = urllib2.Request(url)
res = opener.open(req)
html = res.read()
copy_to_output_dir(html_file, output_dir, html)
elif file_name.endswith('.css') or file_name.endswith('.js'):
# Copy the static resource to output_dir.
relative_path = os.path.join(root, file_name)
with open(relative_path) as resource_file:
resource = resource_file.read()
copy_to_output_dir(relative_path, output_dir, resource)
def __init__(self):
cookie_handler = urllib2.HTTPCookieProcessor()
redirect_handler = urllib2.HTTPRedirectHandler()
http_handler = urllib2.HTTPHandler()
error_handler = urllib2.HTTPErrorProcessor()
self._opener = urllib2.build_opener(cookie_handler, redirect_handler,
http_handler, error_handler)
def __init__(self, debug=False):
'''
'''
super(AfSession, self).__init__()
self.cookiejar = cookielib.CookieJar()
self.urlopener = urllib2.OpenerDirector()
debug_level = 0
if debug:
debug_level = 1
self.debug = debug
#building the urllib2 session opener
try:
self.urlopener.add_handler(
urllib2.HTTPHandler(debuglevel=debug_level))
self.urlopener.add_handler(
urllib2.HTTPSHandler(debuglevel=debug_level))
self.urlopener.add_handler(urllib2.HTTPRedirectHandler())
self.urlopener.add_handler(urllib2.HTTPErrorProcessor())
self.urlopener.add_handler(
urllib2.HTTPCookieProcessor(self.cookiejar))
except Exception, ex:
self.error = 'unexpected error while defining session uri opener' + ex.message
raise Exception(ex.message)
def __init__(self, url, proxy=None, trace=False):
if not url.endswith('/'):
url = url + '/'
self.url = url
if 'APPDATA' in os.environ:
homepath = os.environ["APPDATA"]
elif 'USERPROFILE' in os.environ:
homepath = os.path.join(os.environ["USERPROFILE"],
"Local Settings", "Application Data")
elif 'HOME' in os.environ:
homepath = os.environ["HOME"]
else:
homepath = ''
self.cookie_file = os.path.join(homepath, ".post-review-cookies.txt")
self._cj = cookielib.MozillaCookieJar(self.cookie_file)
self._password_mgr = ReviewBoardHTTPPasswordMgr(self.url)
self._opener = opener = urllib2.build_opener(
urllib2.ProxyHandler(proxy), urllib2.UnknownHandler(),
urllib2.HTTPHandler(), HttpErrorHandler(),
urllib2.HTTPErrorProcessor(),
urllib2.HTTPCookieProcessor(self._cj),
urllib2.HTTPBasicAuthHandler(self._password_mgr),
urllib2.HTTPDigestAuthHandler(self._password_mgr))
urllib2.install_opener(self._opener)
self._trace = trace
def __init__(self, debug=False):
'''
constructor
'''
self.account_number = ''
self.account_name = ''
self.login_name = ''
self.password = ''
self.cookiejar = cookielib.CookieJar()
self.error = ''
self.debug = None
self.urlopener = urllib2.OpenerDirector()
debug_level = 0
if debug:
debug_level = 1
self.debug = debug
#building the urllib2 session opener
try:
self.urlopener.add_handler(
urllib2.HTTPHandler(debuglevel=debug_level))
self.urlopener.add_handler(
urllib2.HTTPSHandler(debuglevel=debug_level))
self.urlopener.add_handler(urllib2.HTTPRedirectHandler())
self.urlopener.add_handler(urllib2.HTTPErrorProcessor())
self.urlopener.add_handler(
urllib2.HTTPCookieProcessor(self.cookiejar))
except Exception, ex:
self.error = 'unexpected error while defining session uri opener' + ex.message
raise Exception(ex.message)
def _is_cache_same(self, dest_file, dist_url):
'''
Checks if the local cache version and the upstream
version is the same or not. If they are the same,
returns True; else False.
'''
if not exists(dest_file):
if self.DEBUG:
stderr.write(
"No file in cache, fetching {0}\n".format(dest_file))
return False
opener = urllib.OpenerDirector()
opener.add_handler(urllib.HTTPHandler())
opener.add_handler(urllib.HTTPSHandler())
opener.add_handler(urllib.HTTPDefaultErrorHandler())
# Extra for handling redirects
opener.add_handler(urllib.HTTPErrorProcessor())
opener.add_handler(urllib.HTTPRedirectHandler())
# Add the header
opener.addheaders = self.hdr2
# Grab the header
try:
res = opener.open(HeadRequest(dist_url))
headers = dict(res.info())
res.close()
remote_ts = headers['last-modified']
except urllib.HTTPError as http_error:
if self.DEBUG:
stderr.write("Cannot send HTTP HEAD request to get \"last-modified\"" \
" attribute of remote content file.\n{0} - {1}\n"
.format(http_error.code, http_error.reason))
return False
except KeyError:
if self.DEBUG:
stderr.write("Response header of HTTP doesn't contain " \
"\"last-modified\" field. Cannot determine version" \
" of remote file \"{0}\"\n".format(dist_url))
return False
# The remote's datetime
remote_dt = datetime.datetime.strptime(remote_ts, self.remote_pattern)
# Get the locals datetime from the file's mtime, converted to UTC
local_dt = datetime.datetime.utcfromtimestamp(
(stat(dest_file)).st_mtime)
# Giving a two second comfort zone
# Else we declare they are different
if (remote_dt - local_dt).seconds > 2:
if self.DEBUG:
stderr.write("Had a local file {0} " \
"but it wasn't new enough\n".format(dest_file))
return False
if self.DEBUG:
stderr.write("File {0} is same as upstream\n".format(dest_file))
return True
def http_error_401(self, req, fp, code, msg, headers):
response = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler.http_error_401(
self, req, fp, code, msg, headers)
# NOTE: problem with how 401 errors are retried, don't utilize the 'chain'
handler = urllib2.HTTPErrorProcessor()
handler.parent = self.parent
tmp = handler.http_response(req, response)
if tmp:
return tmp
return response
def get_opener(cookiejar=None):
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(urllib2.HTTPSHandler())
if cookiejar:
opener.add_handler(urllib2.HTTPCookieProcessor(cookiejar))
return opener
def _GetOpener(self):
# Authentication code needs to know about 302 response.
# So make OpenerDirector without HTTPRedirectHandler.
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar))
return opener
def getheadersonly(url, redirections=True):
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
if redirections:
# HTTPErrorProcessor makes HTTPRedirectHandler work
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(urllib2.HTTPRedirectHandler())
try:
res = opener.open(HeadRequest(url))
except urllib2.HTTPError, res:
pass
def __init__(self, proxy, verbose=0):
self.proxy = proxy
self.verbose = verbose
self.opener = opener = urllib2.OpenerDirector()
if proxy:
opener.add_handler(urllib2.ProxyHandler({'http':self.proxy}))
else:
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
def _GetOpener(self):
"""Returns an OpenerDirector that supports cookies and ignores redirects.
Returns:
A urllib2.OpenerDirector object.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(fancy_urllib.FancyProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(ContentEncodingHandler())
auth_domain = ''
if 'AUTH_DOMAIN' in os.environ:
auth_domain = os.environ['AUTH_DOMAIN'].lower()
if self.save_cookies:
if auth_domain == 'appscale':
cookies_dir = os.path.expanduser(
HttpRpcServer.APPSCALE_COOKIE_DIR)
if not os.path.exists(cookies_dir):
os.mkdir(cookies_dir)
else:
self.cookie_jar.filename = os.path.expanduser(
HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)
if os.path.exists(self.cookie_jar.filename):
try:
self.cookie_jar.load()
self.authenticated = True
logger.debug("Loaded authentication cookies from %s",
self.cookie_jar.filename)
except (OSError, IOError, cookielib.LoadError), e:
logger.debug(
"Could not load authentication cookies; %s: %s",
e.__class__.__name__, e)
self.cookie_jar.filename = None
else:
try:
fd = os.open(self.cookie_jar.filename, os.O_CREAT,
0600)
os.close(fd)
except (OSError, IOError), e:
logger.debug("Could not create authentication cookies file " + \
"; %s: %s" % (e.__class__.__name__, e))
self.cookie_jar.filename = None
def __init__(self, realm, url_root, username, password):
"""
Initializes the handler with the necessary authenticaton
credentials.
"""
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=url_root,
user=username,
passwd=password)
self.url_root = url_root
self.url_opener = urllib2.build_opener(auth_handler,
urllib2.HTTPErrorProcessor())
def test_errors(self):
h = urllib2.HTTPErrorProcessor()
o = h.parent = MockOpener()
url = "http://example.com/"
req = Request(url)
# 200 OK is passed through
r = MockResponse(200, "OK", {}, "", url)
newr = h.http_response(req, r)
self.assert_(r is newr)
self.assert_(not hasattr(o, "proto")) # o.error not called
# anything else calls o.error (and MockOpener returns None, here)
r = MockResponse(201, "Created", {}, "", url)
self.assert_(h.http_response(req, r) is None)
self.assertEqual(o.proto, "http") # o.error called
self.assertEqual(o.args, (req, r, 201, "Created", {}))
def _GetHTTPOpener():
"""Create an http opener used to interact with Google's ClientLogin.
Returns:
An http opener capable of handling anything needed to interact with
Google's ClientLogin.
"""
# Create an http opener capable of handling proxies, http and https.
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(urllib2.HTTPSHandler())
return opener
def _GetOpener(self):
"""Returns an OpenerDirector that supports cookies and ignores redirects.
Returns:
A urllib2.OpenerDirector object.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
return opener
def __init__(self, *args, **kargs):
urllib2.OpenerDirector.__init__(self, *args, **kargs)
#agregando soporte basico
self.add_handler(urllib2.ProxyHandler())
self.add_handler(urllib2.UnknownHandler())
self.add_handler(urllib2.HTTPHandler())
self.add_handler(urllib2.HTTPDefaultErrorHandler())
self.add_handler(urllib2.HTTPRedirectHandler())
self.add_handler(urllib2.FTPHandler())
self.add_handler(urllib2.FileHandler())
self.add_handler(urllib2.HTTPErrorProcessor())
#Agregar soporte para cookies. (en este momento no es necesario,
#pero uno nunca sabe si se puede llegar a nececitar)
self.cj = cookielib.CookieJar()
self.add_handler(urllib2.HTTPCookieProcessor(self.cj))
def _build_opener(self):
# 创建一个opener用于配置请求
opener = urllib2.OpenerDirector()
# disable 本地代理
opener.add_handler(urllib2.ProxyHandler())
# 对URL不符合要求抛出URLError
opener.add_handler(urllib2.UnknownHandler())
# 发送http请求
opener.add_handler(urllib2.HTTPHandler())
# 建错误reponse抛出为HttpEerror
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
# 发送https请求
opener.add_handler(urllib2.HTTPSHandler())
# 对于 http response status code 不属于[200,300)
# 转换为错误response
opener.add_handler(urllib2.HTTPErrorProcessor())
self.opener = opener
def __init__(self, base_url, exc_class=None, logger=None):
"""
@param base_url: The base url to the API.
@param exc_class: An exception class to handle non-200 results.
Creates an HTTP(S) client to connect to the Cloudera Manager API.
"""
self._base_url = base_url.rstrip('/')
self._exc_class = exc_class or RestException
self._logger = logger or LOG
self._headers = { }
# Make a cookie processor
cookiejar = cookielib.CookieJar()
self._opener = urllib2.build_opener(
urllib2.HTTPErrorProcessor(),
urllib2.HTTPCookieProcessor(cookiejar))
def _GetOpener(self):
"""Returns an OpenerDirector that supports cookies and ignores redirects.
Returns:
A urllib2.OpenerDirector object.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(fancy_urllib.FancyProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(fancy_urllib.FancyHTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
opener.add_handler(ContentEncodingHandler())
if self.save_cookies:
self.cookie_jar.filename = os.path.expanduser(
HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)
if os.path.exists(self.cookie_jar.filename):
try:
self.cookie_jar.load()
self.authenticated = True
logger.debug("Loaded authentication cookies from %s",
self.cookie_jar.filename)
except (OSError, IOError, cookielib.LoadError), e:
# Failed to load cookies. The target file path is bad.
logger.debug(
"Could not load authentication cookies; %s: %s",
e.__class__.__name__, e)
self.cookie_jar.filename = None
else:
# Create an empty cookie file. This must be created with the file
# permissions set upfront in order to be secure.
try:
fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
os.close(fd)
except (OSError, IOError), e:
# Failed to create cookie file. Don't try to save cookies.
logger.debug(
"Could not create authentication cookies file; %s: %s",
e.__class__.__name__, e)
self.cookie_jar.filename = None
def __init__(self, url):
self.html_parser = 'html5lib'
self.cj = cookielib.CookieJar()
self.url = url.strip()
# opener
self.opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(),
urllib2.HTTPHandler(debuglevel=0),
urllib2.HTTPSHandler(debuglevel=0),
urllib2.HTTPCookieProcessor(self.cj),
urllib2.HTTPErrorProcessor())
# Fake a regular browser to get right content
self.opener.addheaders = [
('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
('Accept-Language', 'en-US,en;q=0.8')
]
self._get_raw()
def _GetOpener(self):
"""Returns an OpenerDirector that supports cookies and ignores redirects.
Returns:
A urllib2.OpenerDirector object.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(fancy_urllib.FancyProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(fancy_urllib.FancyHTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
if self.save_cookies:
self.cookie_jar.filename = os.path.expanduser(
HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)
if os.path.exists(self.cookie_jar.filename):
try:
self.cookie_jar.load()
self.authenticated = True
logger.info("Loaded authentication cookies from %s",
self.cookie_jar.filename)
except (OSError, IOError, cookielib.LoadError), e:
logger.debug(
"Could not load authentication cookies; %s: %s",
e.__class__.__name__, e)
self.cookie_jar.filename = None
else:
try:
fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
os.close(fd)
except (OSError, IOError), e:
logger.debug(
"Could not create authentication cookies file; %s: %s",
e.__class__.__name__, e)
self.cookie_jar.filename = None
def __init__(self, base_url, exc_class=ex.CMApiException):
"""Init Method
:param base_url: The base url to the API.
:param exc_class: An exception class to handle non-200 results.
Creates an HTTP(S) client to connect to the Cloudera Manager API.
"""
self._base_url = base_url.rstrip('/')
self._exc_class = exc_class
self._headers = {}
# Make a basic auth handler that does nothing. Set credentials later.
self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr)
# Make a cookie processor
cookiejar = cookielib.CookieJar()
self._opener = urllib2.build_opener(
urllib2.HTTPErrorProcessor(),
urllib2.HTTPCookieProcessor(cookiejar), authhandler)
def unshorten(url):
""" Unshortens URLs like in bit.ly, t.co, etc
This function uses an internal cache to avoid reprocessing URL
that already have been unshotened in the current runtime."""
# check if caching dict is available
if not hasattr(unshorten, "urlsyn_dict"):
unshorten.urlsyn_dict = dict()
# return cached value if in cache
elif url in unshorten.urlsyn_dict:
return unshorten.urlsyn_dict[url]
urlsyn = [url]
cj = cookielib.CookieJar()
opener = urllib2.build_opener(HTTPRedirectHandler(urlsyn),
urllib2.HTTPHandler(debuglevel=0),
urllib2.HTTPSHandler(debuglevel=0),
urllib2.HTTPCookieProcessor(cj),
urllib2.HTTPErrorProcessor())
# Fake a regular browser to get right content
opener.addheaders = [
('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
('Accept-Language', 'en-US,en;q=0.8')
]
request = urllib2.Request(url)
opener.open(request)
# caching redirects
final_url = urlsyn.pop()
for syn in urlsyn:
unshorten.urlsyn_dict[syn] = final_url
return final_url
def _GetOpener(self):
"""Returns an OpenerDirector that supports cookies and ignores redirects.
Returns:
A urllib2.OpenerDirector object.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
if self.save_cookies:
self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
if os.path.exists(self.cookie_file):
try:
self.cookie_jar.load()
self.authenticated = True
StatusUpdate("Loaded authentication cookies from %s" %
self.cookie_file)
except (cookielib.LoadError, IOError):
# Failed to load cookies - just ignore them.
pass
else:
# Create an empty cookie file with mode 600
fd = os.open(self.cookie_file, os.O_CREAT, 0600)
os.close(fd)
# Always chmod the cookie file
os.chmod(self.cookie_file, 0600)
else:
# Don't save cookies across runs of update.py.
self.cookie_jar = cookielib.CookieJar()
opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
return opener
def test_errors(self):
h = urllib2.HTTPErrorProcessor()
o = h.parent = MockOpener()
url = "http://example.com/"
req = Request(url)
# all 2xx are passed through
r = MockResponse(200, "OK", {}, "", url)
newr = h.http_response(req, r)
self.assertTrue(r is newr)
self.assertTrue(not hasattr(o, "proto")) # o.error not called
r = MockResponse(202, "Accepted", {}, "", url)
newr = h.http_response(req, r)
self.assertTrue(r is newr)
self.assertTrue(not hasattr(o, "proto")) # o.error not called
r = MockResponse(206, "Partial content", {}, "", url)
newr = h.http_response(req, r)
self.assertTrue(r is newr)
self.assertTrue(not hasattr(o, "proto")) # o.error not called
# anything else calls o.error (and MockOpener returns None, here)
r = MockResponse(502, "Bad gateway", {}, "", url)
self.assertTrue(h.http_response(req, r) is None)
self.assertEqual(o.proto, "http") # o.error called
self.assertEqual(o.args, (req, r, 502, "Bad gateway", {}))
courseFolder = ''
courseName = ''
website = ""
viewedLinks = []
totalFiles = []
num = 0
tempnum = 0
restyleCourse = False
errors = 0
#This creates cookies so that you only have to login once.
cookieJar = cookielib.LWPCookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar),
urllib2.HTTPRedirectHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPErrorProcessor(),
urllib2.HTTPHandler(debuglevel=0))
# opener.addheaders = [('User-agent', "Safari/536.28.10")]
opener.addheaders = [(
'User-agent',
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"
)]
#This gets the user information and logs in to Moodle.
username = raw_input("username:"******"password:"******"username": username, "password": password}
data = urllib.urlencode(forms)
req1 = urllib2.Request(loginPage, data)
opener.open(req1)
"Connection": "close",
"User-Agent": self.user_agent,
}
url = url.encode('utf-8')
broken_error = ValidationError(
_(u'This URL appears to be a broken link.'),
code='invalid_link')
try:
req = urllib2.Request(url, None, headers)
req.get_method = lambda: 'HEAD'
#Create an opener that does not support local file access
opener = urllib2.OpenerDirector()
#Don't follow redirects, but don't treat them as errors either
error_nop = lambda *args, **kwargs: True
http_error_processor = urllib2.HTTPErrorProcessor()
http_error_processor.http_error_301 = error_nop
http_error_processor.http_error_302 = error_nop
http_error_processor.http_error_307 = error_nop
handlers = [
urllib2.UnknownHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPDefaultErrorHandler(),
urllib2.FTPHandler(), http_error_processor
]
try:
import ssl
handlers.append(urllib2.HTTPSHandler())
except:
#Python isn't compiled with SSL support
def _get_opener(self):
opener = urllib2.build_opener(urllib2.HTTPErrorProcessor())
opener.add_handler(HTTPBasicAuthHandler(self.password_mgr))
opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr))
opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar))
return opener
def GetAdminOpener(host,
user=None,
pwd=None,
otp_entry=None,
cookiejar_path=None):
"""Returns an OpenerDirector for retrieving administrative URLs.
Uses stored admin cookies if available, or prompts for authentication
credentials and authenticates with server otherwise.
Based on reitveld codereview script.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
# TODO(spencer): remove the HTTP handler when we move to AsyncHTTPSTestCase.
# This is only for testing currently.
opener.add_handler(urllib2.HTTPHandler())
if cookiejar_path is None:
cookiejar_path = expanduser('~/.viewfinder_admin_cookie')
cookie_jar = cookielib.MozillaCookieJar(cookiejar_path)
if os.path.exists(cookiejar_path):
try:
cookie_jar.load()
logging.info('loaded admin authentication cookies from %s' %
cookiejar_path)
except:
# Otherwise, bad cookies; clear them.
os.unlink(cookiejar_path)
if not os.path.exists(cookiejar_path):
# Create empty file with correct permissions.
fd = os.open(cookiejar_path, os.O_CREAT, 0600)
os.close(fd)
# Always chmod to be sure.
os.chmod(cookiejar_path, 0600)
opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar))
class TornadoXSRFProcessor(urllib2.BaseHandler):
"""Add tornado's xsrf headers to outgoing requests."""
handler_order = urllib2.HTTPCookieProcessor.handler_order + 1
def http_request(self, request):
cookie_header = request.get_header('Cookie')
if cookie_header is not None and '_xsrf=' in cookie_header:
# We have an xsrf cookie in the cookie jar. Copy it into the X-Xsrftoken header.
request.add_unredirected_header(
'X-Xsrftoken',
re.match('_xsrf=([^;]+)', cookie_header).group(1))
else:
# No xsrf cookie, so just make one up. (this is currently the expected case because cookielib
# considers our xsrf cookie to be a "session" cookie and doesn't save it)
request.add_unredirected_header('X-Xsrftoken', 'fake_xsrf')
if cookie_header:
request.add_unredirected_header(
'Cookie', '_xsrf="fake_xsrf"; ' + cookie_header)
else:
request.add_unredirected_header('Cookie',
'_xsrf="fake_xsrf"')
return request
https_request = http_request
opener.add_handler(TornadoXSRFProcessor())
# Look for admin cookie. If it doesn't exist (or is expired), prompt
# and reauthenticate.
if len(cookie_jar) == 0 or \
any([c.is_expired() for c in cookie_jar if c.domain == host]):
if user is None or pwd is None or otp_entry is None:
user, pwd, otp_entry = _PromptForAdminCookie(user, pwd, otp_entry)
from viewfinder.backend.www.admin import admin_api
admin_api.Authenticate(opener, host, user, pwd, otp_entry)
cookie_jar.save()
logging.info('saved admin authentication cookies to %s' %
cookiejar_path)
return opener
