def main(output_dir): # Create opener. opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) # Iterate over the files in the docs directory and copy them, as appropriate. for root, dirs, files in os.walk('.'): for file_name in files: if file_name.endswith('.soy') and not file_name.startswith('__'): # Strip the './' prefix, if appropriate. if root.startswith('./'): root = root[2:] # Construct the URL where the .soy file is being served. soy_file = file_name html_file = root + '/' + soy_file[:-len('.soy')] + '.html' url = 'http://localhost:9811/' + html_file # Fetch url and copy its contents to output_dir. req = urllib2.Request(url) res = opener.open(req) html = res.read() copy_to_output_dir(html_file, output_dir, html) elif file_name.endswith('.css') or file_name.endswith('.js'): # Copy the static resource to output_dir. relative_path = os.path.join(root, file_name) with open(relative_path) as resource_file: resource = resource_file.read() copy_to_output_dir(relative_path, output_dir, resource)
def __init__(self): cookie_handler = urllib2.HTTPCookieProcessor() redirect_handler = urllib2.HTTPRedirectHandler() http_handler = urllib2.HTTPHandler() error_handler = urllib2.HTTPErrorProcessor() self._opener = urllib2.build_opener(cookie_handler, redirect_handler, http_handler, error_handler)
def __init__(self, debug=False): ''' ''' super(AfSession, self).__init__() self.cookiejar = cookielib.CookieJar() self.urlopener = urllib2.OpenerDirector() debug_level = 0 if debug: debug_level = 1 self.debug = debug #building the urllib2 session opener try: self.urlopener.add_handler( urllib2.HTTPHandler(debuglevel=debug_level)) self.urlopener.add_handler( urllib2.HTTPSHandler(debuglevel=debug_level)) self.urlopener.add_handler(urllib2.HTTPRedirectHandler()) self.urlopener.add_handler(urllib2.HTTPErrorProcessor()) self.urlopener.add_handler( urllib2.HTTPCookieProcessor(self.cookiejar)) except Exception, ex: self.error = 'unexpected error while defining session uri opener' + ex.message raise Exception(ex.message)
def __init__(self, url, proxy=None, trace=False): if not url.endswith('/'): url = url + '/' self.url = url if 'APPDATA' in os.environ: homepath = os.environ["APPDATA"] elif 'USERPROFILE' in os.environ: homepath = os.path.join(os.environ["USERPROFILE"], "Local Settings", "Application Data") elif 'HOME' in os.environ: homepath = os.environ["HOME"] else: homepath = '' self.cookie_file = os.path.join(homepath, ".post-review-cookies.txt") self._cj = cookielib.MozillaCookieJar(self.cookie_file) self._password_mgr = ReviewBoardHTTPPasswordMgr(self.url) self._opener = opener = urllib2.build_opener( urllib2.ProxyHandler(proxy), urllib2.UnknownHandler(), urllib2.HTTPHandler(), HttpErrorHandler(), urllib2.HTTPErrorProcessor(), urllib2.HTTPCookieProcessor(self._cj), urllib2.HTTPBasicAuthHandler(self._password_mgr), urllib2.HTTPDigestAuthHandler(self._password_mgr)) urllib2.install_opener(self._opener) self._trace = trace
def __init__(self, debug=False): ''' constructor ''' self.account_number = '' self.account_name = '' self.login_name = '' self.password = '' self.cookiejar = cookielib.CookieJar() self.error = '' self.debug = None self.urlopener = urllib2.OpenerDirector() debug_level = 0 if debug: debug_level = 1 self.debug = debug #building the urllib2 session opener try: self.urlopener.add_handler( urllib2.HTTPHandler(debuglevel=debug_level)) self.urlopener.add_handler( urllib2.HTTPSHandler(debuglevel=debug_level)) self.urlopener.add_handler(urllib2.HTTPRedirectHandler()) self.urlopener.add_handler(urllib2.HTTPErrorProcessor()) self.urlopener.add_handler( urllib2.HTTPCookieProcessor(self.cookiejar)) except Exception, ex: self.error = 'unexpected error while defining session uri opener' + ex.message raise Exception(ex.message)
def _is_cache_same(self, dest_file, dist_url): ''' Checks if the local cache version and the upstream version is the same or not. If they are the same, returns True; else False. ''' if not exists(dest_file): if self.DEBUG: stderr.write( "No file in cache, fetching {0}\n".format(dest_file)) return False opener = urllib.OpenerDirector() opener.add_handler(urllib.HTTPHandler()) opener.add_handler(urllib.HTTPSHandler()) opener.add_handler(urllib.HTTPDefaultErrorHandler()) # Extra for handling redirects opener.add_handler(urllib.HTTPErrorProcessor()) opener.add_handler(urllib.HTTPRedirectHandler()) # Add the header opener.addheaders = self.hdr2 # Grab the header try: res = opener.open(HeadRequest(dist_url)) headers = dict(res.info()) res.close() remote_ts = headers['last-modified'] except urllib.HTTPError as http_error: if self.DEBUG: stderr.write("Cannot send HTTP HEAD request to get \"last-modified\"" \ " attribute of remote content file.\n{0} - {1}\n" .format(http_error.code, http_error.reason)) return False except KeyError: if self.DEBUG: stderr.write("Response header of HTTP doesn't contain " \ "\"last-modified\" field. Cannot determine version" \ " of remote file \"{0}\"\n".format(dist_url)) return False # The remote's datetime remote_dt = datetime.datetime.strptime(remote_ts, self.remote_pattern) # Get the locals datetime from the file's mtime, converted to UTC local_dt = datetime.datetime.utcfromtimestamp( (stat(dest_file)).st_mtime) # Giving a two second comfort zone # Else we declare they are different if (remote_dt - local_dt).seconds > 2: if self.DEBUG: stderr.write("Had a local file {0} " \ "but it wasn't new enough\n".format(dest_file)) return False if self.DEBUG: stderr.write("File {0} is same as upstream\n".format(dest_file)) return True
def http_error_401(self, req, fp, code, msg, headers): response = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler.http_error_401( self, req, fp, code, msg, headers) # NOTE: problem with how 401 errors are retried, don't utilize the 'chain' handler = urllib2.HTTPErrorProcessor() handler.parent = self.parent tmp = handler.http_response(req, response) if tmp: return tmp return response
def get_opener(cookiejar=None): opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(urllib2.HTTPSHandler()) if cookiejar: opener.add_handler(urllib2.HTTPCookieProcessor(cookiejar)) return opener
def _GetOpener(self): # Authentication code needs to know about 302 response. # So make OpenerDirector without HTTPRedirectHandler. opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar)) return opener
def getheadersonly(url, redirections=True): opener = urllib2.OpenerDirector() opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) if redirections: # HTTPErrorProcessor makes HTTPRedirectHandler work opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(urllib2.HTTPRedirectHandler()) try: res = opener.open(HeadRequest(url)) except urllib2.HTTPError, res: pass
def __init__(self, proxy, verbose=0): self.proxy = proxy self.verbose = verbose self.opener = opener = urllib2.OpenerDirector() if proxy: opener.add_handler(urllib2.ProxyHandler({'http':self.proxy})) else: opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor())
def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(fancy_urllib.FancyProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(ContentEncodingHandler()) auth_domain = '' if 'AUTH_DOMAIN' in os.environ: auth_domain = os.environ['AUTH_DOMAIN'].lower() if self.save_cookies: if auth_domain == 'appscale': cookies_dir = os.path.expanduser( HttpRpcServer.APPSCALE_COOKIE_DIR) if not os.path.exists(cookies_dir): os.mkdir(cookies_dir) else: self.cookie_jar.filename = os.path.expanduser( HttpRpcServer.DEFAULT_COOKIE_FILE_PATH) if os.path.exists(self.cookie_jar.filename): try: self.cookie_jar.load() self.authenticated = True logger.debug("Loaded authentication cookies from %s", self.cookie_jar.filename) except (OSError, IOError, cookielib.LoadError), e: logger.debug( "Could not load authentication cookies; %s: %s", e.__class__.__name__, e) self.cookie_jar.filename = None else: try: fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600) os.close(fd) except (OSError, IOError), e: logger.debug("Could not create authentication cookies file " + \ "; %s: %s" % (e.__class__.__name__, e)) self.cookie_jar.filename = None
def __init__(self, realm, url_root, username, password): """ Initializes the handler with the necessary authenticaton credentials. """ auth_handler = urllib2.HTTPBasicAuthHandler() auth_handler.add_password(realm=realm, uri=url_root, user=username, passwd=password) self.url_root = url_root self.url_opener = urllib2.build_opener(auth_handler, urllib2.HTTPErrorProcessor())
def test_errors(self): h = urllib2.HTTPErrorProcessor() o = h.parent = MockOpener() url = "http://example.com/" req = Request(url) # 200 OK is passed through r = MockResponse(200, "OK", {}, "", url) newr = h.http_response(req, r) self.assert_(r is newr) self.assert_(not hasattr(o, "proto")) # o.error not called # anything else calls o.error (and MockOpener returns None, here) r = MockResponse(201, "Created", {}, "", url) self.assert_(h.http_response(req, r) is None) self.assertEqual(o.proto, "http") # o.error called self.assertEqual(o.args, (req, r, 201, "Created", {}))
def _GetHTTPOpener(): """Create an http opener used to interact with Google's ClientLogin. Returns: An http opener capable of handling anything needed to interact with Google's ClientLogin. """ # Create an http opener capable of handling proxies, http and https. opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(urllib2.HTTPSHandler()) return opener
def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) return opener
def __init__(self, *args, **kargs): urllib2.OpenerDirector.__init__(self, *args, **kargs) #agregando soporte basico self.add_handler(urllib2.ProxyHandler()) self.add_handler(urllib2.UnknownHandler()) self.add_handler(urllib2.HTTPHandler()) self.add_handler(urllib2.HTTPDefaultErrorHandler()) self.add_handler(urllib2.HTTPRedirectHandler()) self.add_handler(urllib2.FTPHandler()) self.add_handler(urllib2.FileHandler()) self.add_handler(urllib2.HTTPErrorProcessor()) #Agregar soporte para cookies. (en este momento no es necesario, #pero uno nunca sabe si se puede llegar a nececitar) self.cj = cookielib.CookieJar() self.add_handler(urllib2.HTTPCookieProcessor(self.cj))
def _build_opener(self): # 创建一个opener用于配置请求 opener = urllib2.OpenerDirector() # disable 本地代理 opener.add_handler(urllib2.ProxyHandler()) # 对URL不符合要求抛出URLError opener.add_handler(urllib2.UnknownHandler()) # 发送http请求 opener.add_handler(urllib2.HTTPHandler()) # 建错误reponse抛出为HttpEerror opener.add_handler(urllib2.HTTPDefaultErrorHandler()) # 发送https请求 opener.add_handler(urllib2.HTTPSHandler()) # 对于 http response status code 不属于[200,300) # 转换为错误response opener.add_handler(urllib2.HTTPErrorProcessor()) self.opener = opener
def __init__(self, base_url, exc_class=None, logger=None): """ @param base_url: The base url to the API. @param exc_class: An exception class to handle non-200 results. Creates an HTTP(S) client to connect to the Cloudera Manager API. """ self._base_url = base_url.rstrip('/') self._exc_class = exc_class or RestException self._logger = logger or LOG self._headers = { } # Make a cookie processor cookiejar = cookielib.CookieJar() self._opener = urllib2.build_opener( urllib2.HTTPErrorProcessor(), urllib2.HTTPCookieProcessor(cookiejar))
def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(fancy_urllib.FancyProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(fancy_urllib.FancyHTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) opener.add_handler(ContentEncodingHandler()) if self.save_cookies: self.cookie_jar.filename = os.path.expanduser( HttpRpcServer.DEFAULT_COOKIE_FILE_PATH) if os.path.exists(self.cookie_jar.filename): try: self.cookie_jar.load() self.authenticated = True logger.debug("Loaded authentication cookies from %s", self.cookie_jar.filename) except (OSError, IOError, cookielib.LoadError), e: # Failed to load cookies. The target file path is bad. logger.debug( "Could not load authentication cookies; %s: %s", e.__class__.__name__, e) self.cookie_jar.filename = None else: # Create an empty cookie file. This must be created with the file # permissions set upfront in order to be secure. try: fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600) os.close(fd) except (OSError, IOError), e: # Failed to create cookie file. Don't try to save cookies. logger.debug( "Could not create authentication cookies file; %s: %s", e.__class__.__name__, e) self.cookie_jar.filename = None
def __init__(self, url): self.html_parser = 'html5lib' self.cj = cookielib.CookieJar() self.url = url.strip() # opener self.opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(), urllib2.HTTPHandler(debuglevel=0), urllib2.HTTPSHandler(debuglevel=0), urllib2.HTTPCookieProcessor(self.cj), urllib2.HTTPErrorProcessor()) # Fake a regular browser to get right content self.opener.addheaders = [ ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), ('Accept-Language', 'en-US,en;q=0.8') ] self._get_raw()
def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(fancy_urllib.FancyProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(fancy_urllib.FancyHTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) if self.save_cookies: self.cookie_jar.filename = os.path.expanduser( HttpRpcServer.DEFAULT_COOKIE_FILE_PATH) if os.path.exists(self.cookie_jar.filename): try: self.cookie_jar.load() self.authenticated = True logger.info("Loaded authentication cookies from %s", self.cookie_jar.filename) except (OSError, IOError, cookielib.LoadError), e: logger.debug( "Could not load authentication cookies; %s: %s", e.__class__.__name__, e) self.cookie_jar.filename = None else: try: fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600) os.close(fd) except (OSError, IOError), e: logger.debug( "Could not create authentication cookies file; %s: %s", e.__class__.__name__, e) self.cookie_jar.filename = None
def __init__(self, base_url, exc_class=ex.CMApiException): """Init Method :param base_url: The base url to the API. :param exc_class: An exception class to handle non-200 results. Creates an HTTP(S) client to connect to the Cloudera Manager API. """ self._base_url = base_url.rstrip('/') self._exc_class = exc_class self._headers = {} # Make a basic auth handler that does nothing. Set credentials later. self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr) # Make a cookie processor cookiejar = cookielib.CookieJar() self._opener = urllib2.build_opener( urllib2.HTTPErrorProcessor(), urllib2.HTTPCookieProcessor(cookiejar), authhandler)
def unshorten(url): """ Unshortens URLs like in bit.ly, t.co, etc This function uses an internal cache to avoid reprocessing URL that already have been unshotened in the current runtime.""" # check if caching dict is available if not hasattr(unshorten, "urlsyn_dict"): unshorten.urlsyn_dict = dict() # return cached value if in cache elif url in unshorten.urlsyn_dict: return unshorten.urlsyn_dict[url] urlsyn = [url] cj = cookielib.CookieJar() opener = urllib2.build_opener(HTTPRedirectHandler(urlsyn), urllib2.HTTPHandler(debuglevel=0), urllib2.HTTPSHandler(debuglevel=0), urllib2.HTTPCookieProcessor(cj), urllib2.HTTPErrorProcessor()) # Fake a regular browser to get right content opener.addheaders = [ ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), ('Accept-Language', 'en-US,en;q=0.8') ] request = urllib2.Request(url) opener.open(request) # caching redirects final_url = urlsyn.pop() for syn in urlsyn: unshorten.urlsyn_dict[syn] = final_url return final_url
def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) if self.save_cookies: self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies") self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) if os.path.exists(self.cookie_file): try: self.cookie_jar.load() self.authenticated = True StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file) except (cookielib.LoadError, IOError): # Failed to load cookies - just ignore them. pass else: # Create an empty cookie file with mode 600 fd = os.open(self.cookie_file, os.O_CREAT, 0600) os.close(fd) # Always chmod the cookie file os.chmod(self.cookie_file, 0600) else: # Don't save cookies across runs of update.py. self.cookie_jar = cookielib.CookieJar() opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) return opener
def test_errors(self): h = urllib2.HTTPErrorProcessor() o = h.parent = MockOpener() url = "http://example.com/" req = Request(url) # all 2xx are passed through r = MockResponse(200, "OK", {}, "", url) newr = h.http_response(req, r) self.assertTrue(r is newr) self.assertTrue(not hasattr(o, "proto")) # o.error not called r = MockResponse(202, "Accepted", {}, "", url) newr = h.http_response(req, r) self.assertTrue(r is newr) self.assertTrue(not hasattr(o, "proto")) # o.error not called r = MockResponse(206, "Partial content", {}, "", url) newr = h.http_response(req, r) self.assertTrue(r is newr) self.assertTrue(not hasattr(o, "proto")) # o.error not called # anything else calls o.error (and MockOpener returns None, here) r = MockResponse(502, "Bad gateway", {}, "", url) self.assertTrue(h.http_response(req, r) is None) self.assertEqual(o.proto, "http") # o.error called self.assertEqual(o.args, (req, r, 502, "Bad gateway", {}))
courseFolder = '' courseName = '' website = "" viewedLinks = [] totalFiles = [] num = 0 tempnum = 0 restyleCourse = False errors = 0 #This creates cookies so that you only have to login once. cookieJar = cookielib.LWPCookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar), urllib2.HTTPRedirectHandler(), urllib2.HTTPSHandler(), urllib2.HTTPErrorProcessor(), urllib2.HTTPHandler(debuglevel=0)) # opener.addheaders = [('User-agent', "Safari/536.28.10")] opener.addheaders = [( 'User-agent', "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36" )] #This gets the user information and logs in to Moodle. username = raw_input("username:"******"password:"******"username": username, "password": password} data = urllib.urlencode(forms) req1 = urllib2.Request(loginPage, data) opener.open(req1)
"Connection": "close", "User-Agent": self.user_agent, } url = url.encode('utf-8') broken_error = ValidationError( _(u'This URL appears to be a broken link.'), code='invalid_link') try: req = urllib2.Request(url, None, headers) req.get_method = lambda: 'HEAD' #Create an opener that does not support local file access opener = urllib2.OpenerDirector() #Don't follow redirects, but don't treat them as errors either error_nop = lambda *args, **kwargs: True http_error_processor = urllib2.HTTPErrorProcessor() http_error_processor.http_error_301 = error_nop http_error_processor.http_error_302 = error_nop http_error_processor.http_error_307 = error_nop handlers = [ urllib2.UnknownHandler(), urllib2.HTTPHandler(), urllib2.HTTPDefaultErrorHandler(), urllib2.FTPHandler(), http_error_processor ] try: import ssl handlers.append(urllib2.HTTPSHandler()) except: #Python isn't compiled with SSL support
def _get_opener(self): opener = urllib2.build_opener(urllib2.HTTPErrorProcessor()) opener.add_handler(HTTPBasicAuthHandler(self.password_mgr)) opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr)) opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar)) return opener
def GetAdminOpener(host, user=None, pwd=None, otp_entry=None, cookiejar_path=None): """Returns an OpenerDirector for retrieving administrative URLs. Uses stored admin cookies if available, or prompts for authentication credentials and authenticates with server otherwise. Based on reitveld codereview script. """ opener = urllib2.OpenerDirector() opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) # TODO(spencer): remove the HTTP handler when we move to AsyncHTTPSTestCase. # This is only for testing currently. opener.add_handler(urllib2.HTTPHandler()) if cookiejar_path is None: cookiejar_path = expanduser('~/.viewfinder_admin_cookie') cookie_jar = cookielib.MozillaCookieJar(cookiejar_path) if os.path.exists(cookiejar_path): try: cookie_jar.load() logging.info('loaded admin authentication cookies from %s' % cookiejar_path) except: # Otherwise, bad cookies; clear them. os.unlink(cookiejar_path) if not os.path.exists(cookiejar_path): # Create empty file with correct permissions. fd = os.open(cookiejar_path, os.O_CREAT, 0600) os.close(fd) # Always chmod to be sure. os.chmod(cookiejar_path, 0600) opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar)) class TornadoXSRFProcessor(urllib2.BaseHandler): """Add tornado's xsrf headers to outgoing requests.""" handler_order = urllib2.HTTPCookieProcessor.handler_order + 1 def http_request(self, request): cookie_header = request.get_header('Cookie') if cookie_header is not None and '_xsrf=' in cookie_header: # We have an xsrf cookie in the cookie jar. Copy it into the X-Xsrftoken header. request.add_unredirected_header( 'X-Xsrftoken', re.match('_xsrf=([^;]+)', cookie_header).group(1)) else: # No xsrf cookie, so just make one up. (this is currently the expected case because cookielib # considers our xsrf cookie to be a "session" cookie and doesn't save it) request.add_unredirected_header('X-Xsrftoken', 'fake_xsrf') if cookie_header: request.add_unredirected_header( 'Cookie', '_xsrf="fake_xsrf"; ' + cookie_header) else: request.add_unredirected_header('Cookie', '_xsrf="fake_xsrf"') return request https_request = http_request opener.add_handler(TornadoXSRFProcessor()) # Look for admin cookie. If it doesn't exist (or is expired), prompt # and reauthenticate. if len(cookie_jar) == 0 or \ any([c.is_expired() for c in cookie_jar if c.domain == host]): if user is None or pwd is None or otp_entry is None: user, pwd, otp_entry = _PromptForAdminCookie(user, pwd, otp_entry) from viewfinder.backend.www.admin import admin_api admin_api.Authenticate(opener, host, user, pwd, otp_entry) cookie_jar.save() logging.info('saved admin authentication cookies to %s' % cookiejar_path) return opener