def _GetOpener(self): """Returns an OpenerDirector that supports cookies and ignores redirects. Returns: A urllib2.OpenerDirector object. """ opener = urllib2.OpenerDirector() opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) if self.save_cookies: self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies") self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) if os.path.exists(self.cookie_file): try: self.cookie_jar.load() self.authenticated = True StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file) except (cookielib.LoadError, IOError): # Failed to load cookies - just ignore them. pass else: # Create an empty cookie file with mode 600 fd = os.open(self.cookie_file, os.O_CREAT, 0600) os.close(fd) # Always chmod the cookie file os.chmod(self.cookie_file, 0600) else: # Don't save cookies across runs of update.py. self.cookie_jar = cookielib.CookieJar() opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) return opener
def _twitter_req(self, url, method, parameters): req = oauth2.Request.from_consumer_and_token(self.oauth_consumer, token = self.oauth_token, http_method = \ self.http_method, http_url = url, parameters = parameters) req.sign_request(self.signature_method_hmac_sha1, self.oauth_consumer, self.oauth_token) headers = req.to_header() if (self.http_method == "POST"): encoded_post_data = req.to_postdata() else: encoded_post_data = None url = req.to_url() opener = urllib2.OpenerDirector() opener.add_handler(self.http_handler) opener.add_handler(self.https_handler) response = opener.open(url, encoded_post_data) return response
def sendRequest(self, content, method='GET'): req = urllib2.Request( '%s://%s:%d%s' % (self.options.ssl and 'https' or 'http', self.options.remote[0], self.options.remote[1], self.path), None, self.headers) # not using the build_opener as i want to handle 302s... # opener = urllib2.build_opener() opener = urllib2.OpenerDirector() opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPHandler()) req.add_header( 'Host', '%s:%d' % (self.options.remote[0], self.options.remote[1])) if method not in 'GET': req.get_method = lambda: method if content is not None: resp = opener.open(req, content) else: resp = opener.open(req) info = resp.info() data = resp.read() def sedform(data): for xform in self.options.responseSed: data = xform(data) return data data = sedform(data) headers = idict({k: sedform(v) for k, v in info.items()}) self.logResponse(resp, headers, data) self.sendResponse(resp, headers, data)
def twitterreq(url, method, parameters): req = oauth.Request.from_consumer_and_token(oauth_consumer, token=oauth_token, http_method=http_method, http_url=url, parameters=parameters) req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token) headers = req.to_header() if http_method == "POST": encoded_post_data = req.to_postdata() else: encoded_post_data = None url = req.to_url() opener = urllib.OpenerDirector() opener.add_handler(http_handler) opener.add_handler(https_handler) response = opener.open(url, encoded_post_data) return response
def _FetchUrl(self, url, post_data=None, parameters=None): if post_data: http_method = "POST" else: http_method = "GET" opener = urllib2.OpenerDirector() opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPSHandler()) if not post_data: opener.addheaders.append(('Accept-Encoding', 'gzip')) if http_method == "POST": parameters = post_data.copy() req = oauth.Request.from_consumer_and_token(self._oauth_consumer, token=self._oauth_token, http_method=http_method, http_url=url, parameters=parameters) req.sign_request(self._signature_method_hmac_sha1, self._oauth_consumer, self._oauth_token) if http_method == "POST": encoded_post_data = req.to_postdata() else: encoded_post_data = None url = req.to_url() try: response = opener.open(url, encoded_post_data) url_data = self._UnGzip(response) except urllib2.HTTPError, e: print e
def threadedTwitterRequest(self): """ Method for downloading Twitter stream data in a new Thread. """ if not os.path.isfile(self.TwitterKeysFile): tkMessageBox.showerror( "ERROR", "Twitter API credentials not filled. Use the Set/Update Twitter Credentials button to do so.", parent=self.parent) return # note starting time start_time = time.time() #search_term = TweetSentimentTermEntry.get() # read Twitter credentials with open("Twitter_API_Keys", "r") as twitter_keys_file: twitter_keys = twitter_keys_file.read().split("|") #print twitter_keys api_key = twitter_keys[0] api_secret = twitter_keys[1] access_token_key = twitter_keys[2] access_token_secret = twitter_keys[3] try: _debug = 0 # oauth token objects oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret) oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret) signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() # method = GET http_method = "GET" # handlers for HTTP and HTTPS http_handler = urllib.HTTPHandler(debuglevel=_debug) https_handler = urllib.HTTPSHandler(debuglevel=_debug) ## if search_term == "": ## url = "https://stream.twitter.com/1/statuses/sample.json" ## else: ## url = "https://api.twitter.com/1.1/search/tweets.json?q=" ## url += search_term.strip().split()[0] ## print search_term # API URL url = "https://stream.twitter.com/1/statuses/sample.json" print url # optional parameters parameters = [] # make the request object req = oauth.Request.from_consumer_and_token( oauth_consumer, token=oauth_token, http_method=http_method, http_url=url, parameters=parameters) # sign the request req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token) # get request headers headers = req.to_header() if http_method == "POST": encoded_post_data = req.to_postdata() else: encoded_post_data = None url = req.to_url() # initialize stream opener opener = urllib.OpenerDirector() opener.add_handler(http_handler) opener.add_handler(https_handler) # remove old cached Twitter stream if os.path.isfile(self.TwitterStreamFile): os.remove(self.TwitterStreamFile) # make the request response = opener.open(url, encoded_post_data) print "Updating Twitter Stream ..." # write response to cache file for line in response: self.var.set( "Updating... This process takes 1-2 minutes to complete. Please wait ..." ) # update for some time if abs(time.time() - start_time) >= 60: self.pb.pack_forget() self.var.set("Update complete.") return with open(self.TwitterStreamFile, "a") as twitter_stream_file: twitter_stream_file.write(line.strip() + os.linesep) except: print "EXCEPTION"
def __init__(self, req): Exception.__init__(self, "Redirect") self.req = req class MyRedirectHandler(urllib2.HTTPRedirectHandler): """Throw an exception on redirects instead of continuing. The redirect will be handled in the main thread so it can work with connection pooling.""" def redirect_request(self, req, fp, code, msg, headers, newurl): new_req = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) if new_req: raise Redirect(new_req) # Our handler differs from the Python default in that: # - we don't support file:// URLs # - we don't follow HTTP redirects _my_urlopen = urllib2.OpenerDirector() for klass in [urllib2.ProxyHandler, urllib2.UnknownHandler, urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler, MyRedirectHandler, urllib2.FTPHandler, urllib2.HTTPErrorProcessor, MyHTTPSHandler]: _my_urlopen.add_handler(klass()) def download_in_thread(url, target_file, if_modified_since, notify_done): try: #print "Child downloading", url if url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:'): req = urllib2.Request(url) if url.startswith('http:') and if_modified_since: req.add_header('If-Modified-Since', if_modified_since) src = _my_urlopen.open(req) else: raise Exception(_('Unsupported URL protocol in: %s') % url)
def GetAdminOpener(host, user=None, pwd=None, otp_entry=None, cookiejar_path=None): """Returns an OpenerDirector for retrieving administrative URLs. Uses stored admin cookies if available, or prompts for authentication credentials and authenticates with server otherwise. Based on reitveld codereview script. """ opener = urllib2.OpenerDirector() opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) # TODO(spencer): remove the HTTP handler when we move to AsyncHTTPSTestCase. # This is only for testing currently. opener.add_handler(urllib2.HTTPHandler()) if cookiejar_path is None: cookiejar_path = expanduser('~/.viewfinder_admin_cookie') cookie_jar = cookielib.MozillaCookieJar(cookiejar_path) if os.path.exists(cookiejar_path): try: cookie_jar.load() logging.info('loaded admin authentication cookies from %s' % cookiejar_path) except: # Otherwise, bad cookies; clear them. os.unlink(cookiejar_path) if not os.path.exists(cookiejar_path): # Create empty file with correct permissions. fd = os.open(cookiejar_path, os.O_CREAT, 0600) os.close(fd) # Always chmod to be sure. os.chmod(cookiejar_path, 0600) opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar)) class TornadoXSRFProcessor(urllib2.BaseHandler): """Add tornado's xsrf headers to outgoing requests.""" handler_order = urllib2.HTTPCookieProcessor.handler_order + 1 def http_request(self, request): cookie_header = request.get_header('Cookie') if cookie_header is not None and '_xsrf=' in cookie_header: # We have an xsrf cookie in the cookie jar. Copy it into the X-Xsrftoken header. request.add_unredirected_header( 'X-Xsrftoken', re.match('_xsrf=([^;]+)', cookie_header).group(1)) else: # No xsrf cookie, so just make one up. (this is currently the expected case because cookielib # considers our xsrf cookie to be a "session" cookie and doesn't save it) request.add_unredirected_header('X-Xsrftoken', 'fake_xsrf') if cookie_header: request.add_unredirected_header( 'Cookie', '_xsrf="fake_xsrf"; ' + cookie_header) else: request.add_unredirected_header('Cookie', '_xsrf="fake_xsrf"') return request https_request = http_request opener.add_handler(TornadoXSRFProcessor()) # Look for admin cookie. If it doesn't exist (or is expired), prompt # and reauthenticate. if len(cookie_jar) == 0 or \ any([c.is_expired() for c in cookie_jar if c.domain == host]): if user is None or pwd is None or otp_entry is None: user, pwd, otp_entry = _PromptForAdminCookie(user, pwd, otp_entry) from viewfinder.backend.www.admin import admin_api admin_api.Authenticate(opener, host, user, pwd, otp_entry) cookie_jar.save() logging.info('saved admin authentication cookies to %s' % cookiejar_path) return opener
sends the user to that location with the provided HTTP code. """ dest = cls.format_output_url(dest or "/") response.status_int = code response.headers['Location'] = dest class EmbedHandler(urllib2.BaseHandler, urllib2.HTTPHandler, urllib2.HTTPErrorProcessor, urllib2.HTTPDefaultErrorHandler): def http_redirect(self, req, fp, code, msg, hdrs): to = hdrs['Location'] h = urllib2.HTTPRedirectHandler() r = h.redirect_request(req, fp, code, msg, hdrs, to) return embedopen.open(r) http_error_301 = http_redirect http_error_302 = http_redirect http_error_303 = http_redirect http_error_307 = http_redirect embedopen = urllib2.OpenerDirector() embedopen.add_handler(EmbedHandler()) def proxyurl(url): r = urllib2.Request(url, None, {}) content = embedopen.open(r).read() return content
def __init__(self, proxy=None, noverify=True, follow_redirects=False, headers={}, timeout=5, cadata=None): self.ctx = ssl.create_default_context(cadata=cadata) if noverify: self.ctx.check_hostname = False self.ctx.verify_mode = ssl.CERT_NONE self.proxy = None tproxy = type(proxy) if tproxy in (str, unicode): proxyscheme = urlparse.urlparse(proxy) scheme = proxyscheme.scheme.upper() if scheme == 'SOCKS': scheme = 'SOCKS5' self.proxy = scheme, proxyscheme.hostname+( ':'+str(proxyscheme.port) if proxyscheme.port else ''), \ proxyscheme.username or None, \ proxyscheme.password or None elif proxy in (True, None): self.proxy = find_default_proxy() elif hasattr(proxy, 'as_tuple'): self.proxy = proxy.as_tuple() else: self.proxy = proxy self.noverify = noverify self.timeout = timeout if not self.proxy: handlers = [ StreamingHTTPHandler, StreamingHTTPSHandler(context=self.ctx), TCPReaderHandler(context=self.ctx) ] else: scheme, host, user, password = self.proxy scheme = socks.PROXY_TYPES[scheme] port = socks.DEFAULT_PORTS[scheme] if ':' in host: host, maybe_port = host.split(':') try: port = int(maybe_port) except ValueError: pass sockshandler = SocksiPyHandler( scheme, host, port, user or None, password or None, context=self.ctx if self.noverify else None) handlers = [] if scheme == socks.HTTP: http_proxy = '{}:{}'.format(host, port) if user and password: http_proxy = '{}:{}@{}'.format(user, password, http_proxy) http_proxy = 'http://' + http_proxy handlers.append(urllib2.ProxyHandler({'http': http_proxy})) handlers.append(StreamingHTTPHandler) handlers.append(sockshandler) if not follow_redirects: handlers.append(NoRedirects) handlers.append(UDPReaderHandler) self.opener = urllib2.OpenerDirector() for h in handlers: if isinstance(h, (types.ClassType, type)): h = h() self.opener.add_handler(h) if type(headers) == dict: self.opener.addheaders = [(x, y) for x, y in headers.iteritems()] else: self.opener.addheaders = headers
def check_http_traffic(request): settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1 # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) if settings.SCHEME == 'https': handle = httplib.HTTPSConnection else: handle = httplib.HTTPConnection class do_connection(handle): def request(self, method, url, body, headers): info_msg = "The provided HTTP request headers: " if settings.VERBOSITY_LEVEL >= 2: print settings.print_info_msg(info_msg) if menu.options.traffic_file: logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37) header = method + " " + url if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) for item in headers.items(): header = item[0] + ": " + item[1] if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if body: header = body if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if menu.options.traffic_file: logs.log_traffic("\n\n") if settings.SCHEME == 'https': httplib.HTTPSConnection.request(self, method, url, body, headers) else: httplib.HTTPConnection.request(self, method, url, body, headers) class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler): if settings.SCHEME == 'https': def https_open(self, req): try: return self.do_open(do_connection, req) except Exception as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." if settings.INIT_TEST == True: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" else: if settings.VERBOSITY_LEVEL < 1: print "" print settings.print_critical_msg(error_msg) raise SystemExit() else: def http_open(self, req): try: return self.do_open(do_connection, req) except Exception as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." if settings.INIT_TEST == True: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" else: if settings.VERBOSITY_LEVEL < 1: print "" print settings.print_critical_msg(error_msg) raise SystemExit() if settings.REVERSE_TCP == False and settings.BIND_TCP == False: opener = urllib2.OpenerDirector() opener.add_handler(connection_handler()) response = False current_attempt = 0 while not response and current_attempt <= settings.MAX_RETRIES: try: opener.open(request) response = True if settings.VERBOSITY_LEVEL < 2: if current_attempt != 0: info_msg = "Checking connection to the target URL... " sys.stdout.write(settings.print_info_msg(info_msg)) sys.stdout.flush() if settings.INIT_TEST == True: print "[ " + Fore.GREEN + "SUCCEED" + Style.RESET_ALL + " ]" if not settings.CHECK_INTERNET: settings.INIT_TEST = False except urllib2.URLError, err_msg: if current_attempt == 0: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" try: error_msg = str(err_msg.args[0]).split("] ")[1] + ". " except IndexError: error_msg = "" error_msg += "Please wait while retring the request(s)." print settings.print_critical_msg(error_msg) warn_msg = "In case the provided target URL is valid, try to rerun with" warn_msg += " the switch '--random-agent' and/or proxy switch." print settings.print_warning_msg(warn_msg) if settings.VERBOSITY_LEVEL >= 2 or current_attempt == 1: info_msg = "Please wait while retring the request(s)." print settings.print_info_msg(info_msg) current_attempt = current_attempt + 1 time.sleep(3) except httplib.BadStatusLine, err_msg: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" if len(err_msg.line) > 2: print err_msg.line, err_msg.message raise SystemExit() except ValueError, err: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" err_msg = "Invalid target URL has been given." print settings.print_critical_msg(err_msg) raise SystemExit()
def __call__(self, value): try: super(RelativeURLValidator, self).__call__(value) except ValidationError as e: # Trivial case failed. Try for possible IDN domain if value: value = smart_text(value) scheme, netloc, path, query, fragment = urlparse.urlsplit( value) try: netloc = netloc.encode('idna') # IDN -> ACE except UnicodeError: # invalid domain part raise e url = urlparse.urlunsplit( (scheme, netloc, path, query, fragment)) super(RelativeURLValidator, self).__call__(url) else: raise else: url = value if self.verify_exists: broken_error = ValidationError( _(u'This URL appears to be a broken link.'), code='invalid_link') if url.startswith('http://') or url.startswith('ftp://'): headers = { "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5", "Accept-Language": "en-us,en;q=0.5", "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Connection": "close", "User-Agent": self.user_agent, } url = url.encode('utf-8') try: req = urllib2.Request(url, None, headers) req.get_method = lambda: 'HEAD' #Create an opener that does not support local file access opener = urllib2.OpenerDirector() #Don't follow redirects, but don't treat them as errors either error_nop = lambda *args, **kwargs: True http_error_processor = urllib2.HTTPErrorProcessor() http_error_processor.http_error_301 = error_nop http_error_processor.http_error_302 = error_nop http_error_processor.http_error_307 = error_nop handlers = [ urllib2.UnknownHandler(), urllib2.HTTPHandler(), urllib2.HTTPDefaultErrorHandler(), urllib2.FTPHandler(), http_error_processor ] try: import ssl handlers.append(urllib2.HTTPSHandler()) except: #Python isn't compiled with SSL support pass map(opener.add_handler, handlers) if platform.python_version_tuple() >= (2, 6): opener.open(req, timeout=10) else: opener.open(req) except ValueError: raise ValidationError(_(u'Enter a valid URL.'), code='invalid') except: # urllib2.URLError, httplib.InvalidURL, etc. raise broken_error else: # Resolve the relative URL try: resolve(url) except Http404: raise broken_error
#Default opener, used for the password request opener1 = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(passMan), urllib2.HTTPCookieProcessor(), urllib2.HTTPHandler(debuglevel=options.debug)) #No-redirect opener: Why am I building the opener in this #strange way, instead of using the code above? Because in #older versions of Python, redirects after a POST are not #handled correctly. ELOG responds to a successful post with a #redirect, which then hangs. Since we don't actually care #about the redirection, I build an opener that doesn't include #redirection (or a bunch of other default things). Should #work fine-- ignores request to redirect and terminates. opener2 = urllib2.OpenerDirector() opener2.add_handler(urllib2.HTTPHandler(debuglevel=options.debug)) opener2.add_handler(urllib2.ProxyHandler()) opener2.add_handler(urllib2.HTTPBasicAuthHandler(passMan)) opener2.add_handler(urllib2.HTTPCookieProcessor()) opener2.add_handler(urllib2.HTTPDefaultErrorHandler()) opener2.add_handler(urllib2.HTTPErrorProcessor()) elif options.elog == "shyft": print "Adding SHyFT (JHU) specific post data." #form.add_field('exp','top') form.add_field('Type', 'Progress') form.add_field('Category', 'SHyFT') form.add_field('Subject', 'Automatic plots posting') elogUrl = "http://hep.pha.jhu.edu:8080/top/"
def _is_cache_same(self, local_file, remote_url): """Checks if the local cache version and the upstream version is the same or not. If they are the same, returns True; else False. """ with self.fetch_lock: if not os.path.exists(local_file): logging.debug( "No local file cached, will fetch {0}".format(remote_url)) return False last_checked = self.fetch_last_checked.get(remote_url, 0) now = time.time() if now - last_checked <= self.fetch_timeout: logging.debug( "Checked for fresh version of '%s' just %f seconds ago. " "Will wait %f seconds before checking again.", remote_url, now - last_checked, self.fetch_timeout - now + last_checked) return True opener = urllib.OpenerDirector() opener.add_handler(urllib.HTTPHandler()) opener.add_handler(urllib.HTTPSHandler()) opener.add_handler(urllib.HTTPDefaultErrorHandler()) # Extra for handling redirects opener.add_handler(urllib.HTTPErrorProcessor()) opener.add_handler(urllib.HTTPRedirectHandler()) # Add the header opener.addheaders = self.hdr2 # Grab the header try: res = opener.open(CVEFeedManager.HeadRequest(remote_url)) headers = self._parse_http_headers(res.info()) res.close() remote_ts = headers['last-modified'] except urllib.HTTPError as http_error: logging.debug( "Cannot send HTTP HEAD request to get \"last-modified\" " "attribute of remote content file.\n{0} - {1}".format( http_error.code, http_error.reason)) return False except KeyError: self._print_no_last_modified_warning(remote_url) return False self.fetch_last_checked[remote_url] = time.time() # The remote's datetime remote_dt = datetime.datetime.strptime(remote_ts, self.remote_pattern) # Get the locals datetime from the file's mtime, converted to UTC local_dt = datetime.datetime.utcfromtimestamp( os.stat(local_file).st_mtime) # Giving a two second comfort zone # Else we declare they are different if (remote_dt - local_dt).seconds > 2: logging.info("Had a local version of {0} " "but it wasn't new enough".format(local_file)) return False logging.debug("File {0} is same as upstream".format(local_file)) return True
safe_url_opener = urlreq.build_opener(URLSafetyHandler()) if __name__ == '__main__': import itertools good_urls = [ 'http://www.w3.org/index.html', 'https://w3.org:8080/Overview.html', 'ftp://ftp.w3.org' ] bad_urls = [ 'file:///etc/passwd', 'rsync://w3.org/', 'http://www.w3.org:22/', 'http://localhost/server-status', 'http://localhost:8001/2012/pyRdfa/Overview.html' ] test_opener = urlreq.OpenerDirector() test_opener.add_handler(URLSafetyHandler()) check_funcs = [check_url_safety, test_opener.open] def is_check_passed(call, *args, **kwargs): try: call(*args, **kwargs) except UnsupportedResourceError: return False return True for host in [ '127.0.0.1', '127.254.1.2', '10.1.2.3', '10.254.4.5', '172.16.1.2', '172.31.4.5', '192.168.0.1', '192.168.254.5', 'fe80::1', 'fe80:ffff::ffff', 'localhost', 'ip6-localhost' ]:
def twitterreq(url, method, parameters): """ Send twitter URL request Utility function used by the others in this package Note: calls a function twitter_credentials() contained in a file named twitter_credentials.py which must be provided as follows: api_key = " your credentials " api_secret = " your credentials " access_token_key = " your credentials " access_token_secret = " your credentials " return (api_key,api_secret,access_token_key,access_token_secret) This function is based on a shell provided by Bill Howe University of Washington for the Coursera course Introduction to Data Science Spring/Summer 2014 (which I HIGHLY recommend) """ import oauth2 as oauth import urllib2 as urllib # this is a private function containing my Twitter credentials from twitter_credentials import twitter_credentials api_key, api_secret, access_token_key, access_token_secret = twitter_credentials( ) _debug = 0 oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret) oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret) signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() http_method = "GET" http_handler = urllib.HTTPHandler(debuglevel=_debug) https_handler = urllib.HTTPSHandler(debuglevel=_debug) ''' Construct, sign, and open a twitter request using the hard-coded credentials above. ''' req = oauth.Request.from_consumer_and_token(oauth_consumer, token=oauth_token, http_method=http_method, http_url=url, parameters=parameters) req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token) headers = req.to_header() if http_method == "POST": encoded_post_data = req.to_postdata() else: encoded_post_data = None url = req.to_url() opener = urllib.OpenerDirector() opener.add_handler(http_handler) opener.add_handler(https_handler) response = opener.open(url, encoded_post_data) return response
def do_check(url): """ This functinality is based on Filippo's Valsorda script [1] which uses HEAD requests (with fallback in case of 405) to follow the redirect path up to the real URL. --- [1] https://gist.github.com/FiloSottile/2077115 """ class HeadRequest(urllib2.Request): def get_method(self): return "HEAD" class HEADRedirectHandler(urllib2.HTTPRedirectHandler): """ Subclass the HTTPRedirectHandler to make it use our HeadRequest also on the redirected URL """ def redirect_request(self, req, fp, code, msg, headers, redirected_url): if code in (301, 302, 303, 307): redirected_url = redirected_url.replace(' ', '%20') newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type")) warn_msg = "Got a " + str(code) + " redirection (" + redirected_url + ")." print settings.print_warning_msg(warn_msg) return HeadRequest(redirected_url, headers = newheaders, origin_req_host = req.get_origin_req_host(), unverifiable = True ) else: err_msg = str(urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)).replace(": "," (") print settings.print_critical_msg(err_msg + ").") raise SystemExit() class HTTPMethodFallback(urllib2.BaseHandler): """ Fallback to GET if HEAD is not allowed (405 HTTP error) """ def http_error_405(self, req, fp, code, msg, headers): fp.read() fp.close() newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type")) return self.parent.open(urllib2.Request(req.get_full_url(), headers = newheaders, origin_req_host = req.get_origin_req_host(), unverifiable = True) ) # Build our opener opener = urllib2.OpenerDirector() for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler, HTTPMethodFallback, HEADRedirectHandler, urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]: opener.add_handler(handler()) try: response = opener.open(HeadRequest(url)) redirected_url = response.geturl() if redirected_url != url: while True: if not menu.options.batch: question_msg = "Do you want to follow the identified redirection? [Y/n] > " sys.stdout.write(settings.print_question_msg(question_msg)) redirection_option = sys.stdin.readline().replace("\n","").lower() else: redirection_option = "" if len(redirection_option) == 0 or redirection_option in settings.CHOICE_YES: if menu.options.batch: info_msg = "Following redirection to '" + redirected_url + "'. " print settings.print_info_msg(info_msg) return redirected_url elif redirection_option in settings.CHOICE_NO: return url elif redirection_option in settings.CHOICE_QUIT: sys.exit(0) else: err_msg = "'" + redirection_option + "' is not a valid answer." print settings.print_error_msg(err_msg) pass else: return url except urllib2.HTTPError, err: err_msg = str(err).replace(": "," (") print settings.print_critical_msg(err_msg + ").") raise SystemExit()
def sentiment(responses_dict): data = [] terms = {} intensifier_terms = [] negation_terms = [] positive_tweets = [] negative_tweets = [] neutral_tweets = [] pos_word_set = set() neg_word_set = set() for line in responses_dict: datadict = {} if "text" in line.keys(): datadict["text"] = line["text"] pass if "user" in line.keys(): datadict["user"] = line["user"]["screen_name"] data.append(datadict) continue opener = urllib.OpenerDirector() opener.add_handler(http_handler) response = opener.open( "http://twittersentiment.s3.amazonaws.com/sentiment3.txt") filedata = StringIO(response.read()) for sent_line in filedata: term, score = sent_line.split( "\t") # The file is tab-delimited. "\t" means "tab character" terms[term] = int(score) # Convert the score to an integer continue intensifier_response = opener.open( "http://twittersentiment.s3.amazonaws.com/intensifiers.txt") intensifier_data = StringIO(intensifier_response.read()) for int_line in intensifier_data: int_term = int_line.strip() intensifier_terms.append(int_term) negation_response = opener.open( "http://twittersentiment.s3.amazonaws.com/negation1.txt") negation_data = StringIO(negation_response.read()) for neg_line in negation_data: neg_term = neg_line.strip() negation_terms.append(neg_term) for tweet in data: if "text" in tweet.keys(): tweet_split = tweet["text"].split() score = 0 for i, word in enumerate(tweet_split): new_score, i_term, n_term = rec_unicode_check( 0, i, word, terms, tweet_split, intensifier_terms, negation_terms) if new_score > 0: if i_term != "" or n_term != "": pos_word_set.add(i_term + n_term + " " + word.lower()) else: pos_word_set.add(word.lower()) elif new_score < 0: if i_term != "" or n_term != "": neg_word_set.add(i_term + n_term + " " + word.lower()) else: neg_word_set.add(word.lower()) score += new_score continue print pos_word_set print neg_word_set tweet["score"] = score if score > 0: positive_tweets.append(tweet) elif score < 0: negative_tweets.append(tweet) else: neutral_tweets.append(tweet) final_list = sorted(data, key=lambda k: k['score']) d3data = [len(positive_tweets), len(neutral_tweets), len(negative_tweets)] return final_list, d3data, pos_word_set, neg_word_set
def _make_opener(url): opener = urllib2.OpenerDirector() for handler_class in _DEFAULT_HANDLERS: opener.add_handler(handler_class()) opener.add_handler(VerifiedHTTPSHandler()) return opener
def setUp(self): self.http_mock = Mock(side_effect=[]) opener = urllib2.OpenerDirector() opener.add_handler(TestHandler(self.http_mock)) urllib2.install_opener(opener)
def check_http_traffic(request): # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) class do_connection(httplib.HTTPConnection, httplib.HTTPSConnection): """ Checking the HTTP / HTTPS requests. """ def request(self, method, url, body, headers): info_msg = "The provided HTTP request headers: " if settings.VERBOSITY_LEVEL >= 2: print settings.print_info_msg(info_msg) if menu.options.traffic_file: logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37) header = method + " " + url if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) for item in headers.items(): header = item[0] + ": " + item[1] if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if body: header = body if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if menu.options.traffic_file: logs.log_traffic("\n\n") if settings.PROXY_PROTOCOL == 'https': httplib.HTTPSConnection.request(self, method, url, body, headers) else: httplib.HTTPConnection.request(self, method, url, body, headers) class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler): if settings.PROXY_PROTOCOL == 'https': def https_open(self, req): return self.do_open(do_connection, req) else: def http_open(self, req): return self.do_open(do_connection, req) if settings.REVERSE_TCP == False and settings.BIND_TCP == False: opener = urllib2.OpenerDirector() opener.add_handler(connection_handler()) try: opener.open(request) except: pass response = urllib2.urlopen(request) # Check the HTTP response headers. http_response(response.info()) # Check the HTTP response content. http_response_content(response.read()) return response
def _on_urllib2_urlopen(self, url, data=None, timeout=None, cafile=None, capath=None, cadefault=False, context=None): request = Request(url.get_method(), url.get_full_url(), data, url.headers, url.get_type(), url.get_host(), url.port) match = self._find_match(request) resp_callback = self.response_callback if self.allow_external_requests: logger.info('request.allowed-external', extra={ 'url': request.url, }) return _real_urllib2_urlopen(url) if match is None: if request.url.startswith(self.passthru_prefixes): logger.info('request.allowed-passthru', extra={ 'url': request.url, }) return _real_urllib2_urlopen(url) response = self._on_no_match(request, resp_callback) raise response class MockHTTPSConnection(HTTPSConnection): def getresponse(buffering=False): response = match.get_response(request) response.msg = '' return response class MockHTTPConnection(HTTPConnection): def getresponse(buffering=False): response = match.get_response(request) response.msg = '' return response class MockHTTPSHandler(urllib2.AbstractHTTPHandler): def __init__(self, debuglevel=0, context=None): urllib2.AbstractHTTPHandler.__init__(self, debuglevel) self._context = context def https_open(self, req): return self.do_open(MockHTTPSConnection, req, context=self._context) class MockHTTPHandler(urllib2.AbstractHTTPHandler): def http_open(self, req): return self.do_open(MockHTTPConnection, req) response = match.get_response(request) opener = urllib2.OpenerDirector() opener.add_handler(MockHTTPHandler()) if cafile or cadefault: if context is not None: raise ValueError( "You can't pass both context and any of cafile, capath, and " "cadefault") if not _have_ssl: raise ValueError('SSL support not available') context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH, cafile=cafile, capath=capath) https_handler = MockHTTPSHandler(context=context) opener.add_handler(https_handler) elif context: https_handler = MockHTTPSHandler(context=context) opener = opener.add_handler(https_handler) elif hasattr(httplib, 'HTTPS'): opener.add_handler(MockHTTPSHandler()) self._calls.add(request, response) return opener.open(url, data, timeout)
api_secret = "**" access_token_key = "*-**" access_token_secret = "**" oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret) oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret) signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() ## Send request to url url = "https://stream.twitter.com/1.1/statuses/sample.json" req = oauth.Request.from_consumer_and_token(oauth_consumer, token=oauth_token, http_method="GET", http_url=url) req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token) url = req.to_url() ## Open the connection opener = urllib.OpenerDirector() http_handler = urllib.HTTPHandler(debuglevel=0) https_handler = urllib.HTTPSHandler(debuglevel=0) opener.add_handler(http_handler) opener.add_handler(https_handler) ## Get the response response = opener.open(url) ## Write response to file f = open('tweets.json', 'w') i = 0 for line in response: entry = json.loads(line) ## Keep English tweets only if u'lang' in entry and entry[u'lang'] == 'en' and u'text' in entry:
"""A password manager that grabs credentials from upwards in its call stack. This is for cooperation with urlopenRemote, which defines a name _temp_credentials. If this is non-None, it's supposed to be a pair of user password presented to *any* realm. This means that, at least with http basic auth, password stealing is almost trivial. """ def find_user_password(self, realm, authuri): creds = codetricks.stealVar("_temp_credentials") if creds is not None: return creds _restrictedURLOpener = urllib2.OpenerDirector() _restrictedURLOpener.add_handler(urllib2.HTTPRedirectHandler()) _restrictedURLOpener.add_handler(urllib2.HTTPHandler()) _restrictedURLOpener.add_handler(urllib2.HTTPSHandler()) _restrictedURLOpener.add_handler(urllib2.HTTPErrorProcessor()) _restrictedURLOpener.add_handler( urllib2.HTTPBasicAuthHandler(_UrlopenRemotePasswordMgr())) _restrictedURLOpener.add_handler(urllib2.FTPHandler()) _restrictedURLOpener.add_handler(urllib2.UnknownHandler()) _restrictedURLOpener.addheaders = [("user-agent", "GAVO DaCHS HTTP client")] def urlopenRemote(url, data=None, creds=(None, None)): """works like urllib2.urlopen, except only http, https, and ftp URLs are handled.
def check_http_traffic(request): # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) class do_connection(httplib.HTTPConnection, httplib.HTTPSConnection): """ Checking the HTTP / HTTPS requests. """ def request(self, method, url, body, headers): info_msg = "The provided HTTP request headers: " if settings.VERBOSITY_LEVEL >= 2: print settings.print_info_msg(info_msg) if menu.options.traffic_file: logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37) header = method + " " + url if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) for item in headers.items(): header = item[0] + ": " + item[1] if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if body: header = body if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if menu.options.traffic_file: logs.log_traffic("\n\n") if settings.PROXY_PROTOCOL == 'https': httplib.HTTPSConnection.request(self, method, url, body, headers) else: httplib.HTTPConnection.request(self, method, url, body, headers) class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler): if settings.PROXY_PROTOCOL == 'https': def https_open(self, req): return self.do_open(do_connection, req) else: def http_open(self, req): return self.do_open(do_connection, req) if settings.REVERSE_TCP == False and settings.BIND_TCP == False: opener = urllib2.OpenerDirector() opener.add_handler(connection_handler()) response = False current_attempt = 0 while not response and current_attempt <= settings.MAX_RETRIES: try: opener.open(request) response = True if settings.VERBOSITY_LEVEL < 2: if current_attempt != 0: info_msg = "Checking connection to the target URL... " sys.stdout.write(settings.print_info_msg(info_msg)) sys.stdout.flush() if settings.INIT_TEST == True: print "[ " + Fore.GREEN + "SUCCEED" + Style.RESET_ALL + " ]" if not settings.CHECK_INTERNET: settings.INIT_TEST = False except urllib2.URLError, err_msg: if menu.options.check_internet: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" break if current_attempt == 0: if settings.VERBOSITY_LEVEL < 2: print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]" error_msg = str(err_msg.args[0]).split("] ")[1] + ". " error_msg += "Please wait while retring the request(s)." print settings.print_critical_msg(error_msg) warn_msg = "In case the provided target URL is valid, try to rerun with" warn_msg += " the switch '--random-agent' and/or proxy switch." print settings.print_warning_msg(warn_msg) if settings.VERBOSITY_LEVEL >= 2 or current_attempt == 1: #info_msg = str(err_msg.args[0]).split("] ")[1] + ". " info_msg = "Please wait while retring the request(s)." print settings.print_info_msg(info_msg) current_attempt = current_attempt + 1 time.sleep(3) pass
dictWriter.writeheader() dictWriter.writerows(dict) # print dict if __name__ == "__main__": # main_1() import urllib2 manager = urllib2.HTTPPasswordMgrWithDefaultRealm() manager.add_password(None, 'https://app.streamsend.com/emails', 'login', 'key') handler = urllib2.HTTPBasicAuthHandler(manager) director = urllib2.OpenerDirector() director.add_handler(handler) req = urllib2.Request('https://app.streamsend.com/emails', headers={'Accept': 'application/xml'}) result = director.open(req) # result.read() will contain the data # result.info() will contain the HTTP headers # To get say the content-length header length = result.info()['Content-Length'] """ while ( True ) : if len(sys.argv) > 1:
def do_check(url): """ This functinality is based on Filippo's Valsorda script [1]. --- [1] https://gist.github.com/FiloSottile/2077115 """ class Request(urllib2.Request): def get_method(self): return "GET" class RedirectHandler(urllib2.HTTPRedirectHandler): """ Subclass the HTTPRedirectHandler to make it use our Request also on the redirected URL """ def redirect_request(self, req, fp, code, msg, headers, redirected_url): if code in (301, 302, 303, 307): redirected_url = redirected_url.replace(' ', '%20') newheaders = dict( (k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) warn_msg = "Got a " + str( code) + " redirection (" + redirected_url + ")." print settings.print_warning_msg(warn_msg) return Request(redirected_url, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: err_msg = str( urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)).replace(": ", " (") print settings.print_critical_msg(err_msg + ").") raise SystemExit() class HTTPMethodFallback(urllib2.BaseHandler): """ """ def http_error_405(self, req, fp, code, msg, headers): fp.read() fp.close() newheaders = dict( (k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) return self.parent.open( urllib2.Request(req.get_full_url(), headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True)) # Build our opener opener = urllib2.OpenerDirector() # Check if defined any Host HTTP header. if menu.options.host and settings.HOST_INJECTION == False: opener.addheaders.append(('Host', menu.options.host)) # Check if defined any User-Agent HTTP header. if menu.options.agent: opener.addheaders.append(('User-Agent', menu.options.agent)) # Check if defined any Referer HTTP header. if menu.options.referer and settings.REFERER_INJECTION == False: opener.addheaders.append(('Referer', menu.options.referer)) # Check if defined any Cookie HTTP header. if menu.options.cookie and settings.COOKIE_INJECTION == False: opener.addheaders.append(('Cookie', menu.options.cookie)) for handler in [ urllib2.HTTPHandler, HTTPMethodFallback, RedirectHandler, urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler ]: opener.add_handler(handler()) try: response = opener.open(Request(url)) redirected_url = response.geturl() if redirected_url != url: while True: if not menu.options.batch: question_msg = "Do you want to follow the identified redirection? [Y/n] > " sys.stdout.write(settings.print_question_msg(question_msg)) redirection_option = sys.stdin.readline().replace( "\n", "").lower() else: redirection_option = "" if len(redirection_option ) == 0 or redirection_option in settings.CHOICE_YES: if menu.options.batch: info_msg = "Following redirection to '" + redirected_url + "'. " print settings.print_info_msg(info_msg) return redirected_url elif redirection_option in settings.CHOICE_NO: return url elif redirection_option in settings.CHOICE_QUIT: raise SystemExit() else: err_msg = "'" + redirection_option + "' is not a valid answer." print settings.print_error_msg(err_msg) pass else: return url except AttributeError: pass # Raise exception due to ValueError. except ValueError, err: err_msg = str(err).replace(": ", " (") print settings.print_critical_msg(err_msg + ").") raise SystemExit()
def make_opener(self, address, headers=None): scheme = None proxy_host = None proxy_password_manager = None http_password_manager = OptionalPasswordManager() password_managers = [] if self.proxy == 'wpad': proxy = get_proxy_for_address(address) if proxy: proxy = proxy[0] else: proxy = None else: proxy = self.proxy if not proxy or proxy[0] == 'DIRECT' or self._is_direct(address): handlers = [ StreamingHTTPHandler, StreamingHTTPSHandler(context=self.ctx), TCPReaderHandler(context=self.ctx) ] else: scheme, host, user, password = proxy scheme = PROXY_TYPES[scheme] port = DEFAULT_PORTS[scheme] if ':' in host: host, maybe_port = host.split(':') try: port = int(maybe_port) except ValueError: pass proxy_host = host + ':' + str(port) sockshandler = SocksiPyHandler( scheme, host, port, user or None, password or None, context=self.ctx if self.noverify else None) handlers = [] if scheme == PROXY_SCHEME_HTTP: http_proxy = proxy_host handlers.append( urllib2.ProxyHandler({'http': 'http://' + http_proxy})) proxy_password_manager = ProxyPasswordManager( 'http', host, port, user, password) for handler_klass in (ProxyAuthHandler, urllib2.ProxyBasicAuthHandler, urllib2.ProxyDigestAuthHandler): if handler_klass is None: continue instance = handler_klass(proxy_password_manager) if hasattr(instance, 'set_logger'): instance.set_logger(logger) handlers.append(instance) password_managers.append(proxy_password_manager) handlers.append(StreamingHTTPHandler) handlers.append(sockshandler) if self.follow_redirects: handlers.append(urllib2.HTTPRedirectHandler) else: handlers.append(NoRedirects) handlers.append(UDPReaderHandler) for handler_klass in (urllib2.HTTPBasicAuthHandler, urllib2.HTTPDigestAuthHandler, HTTPAuthHandler): if handler_klass is None: continue instance = handler_klass(http_password_manager) if hasattr(instance, 'set_logger'): instance.set_logger(logger) handlers.append(instance) password_managers.append(http_password_manager) context = HTTPContext.get_default() handlers.append(context) handlers.append(urllib2.HTTPDefaultErrorHandler) handlers.append(urllib2.HTTPErrorProcessor) opener = urllib2.OpenerDirector() for h in handlers: if isinstance(h, (types.ClassType, type)): h = h() opener.add_handler(h) filter_headers = set() if headers: if isinstance(headers, dict): filter_headers = set(headers.keys()) else: filter_headers = set(x for x, _ in headers) if isinstance(self.headers, dict): opener.addheaders = [(x, y) for x, y in self.headers.iteritems() if x not in filter_headers] else: opener.addheaders = self.headers if headers: if isinstance(headers, dict): opener.addheaders.extend([ (x, y) for x, y in self.headers.iteritems() ]) else: opener.addheaders.extend(headers) return opener, scheme, proxy_host, password_managers, context
class WizardTool(BaseTool): """ WizardTool is able to generate custom business templates. """ id = 'portal_wizard' meta_type = 'ERP5 Wizard Tool' portal_type = 'Wizard Tool' isPortalContent = ConstantGetter('isPortalContent', value=True) property_sheets = () security = ClassSecurityInfo() security.declareProtected(Permissions.ManagePortal, 'manage_overview') manage_overview = DTMLFile('explainWizardTool', _dtmldir) # Stop traversing a concatenated path after the proxy method. def __before_publishing_traverse__(self, self2, request): path = request['TraversalRequestNameStack'] if path and path[-1] == 'proxy': subpath = path[:-1] subpath.reverse() request.set('traverse_subpath', subpath) # initialize our root proxy URL which we use for a referer global referer path[:-1] = [] if referer is None: referer = '%s/portal_wizard/proxy/%s/view' % ( self.getPortalObject().absolute_url(), '/'.join( subpath[:3])) def _getProxyURL(self, subpath='', query=''): # Helper method to construct an URL appropriate for proxying a request. # This makes sure that URLs generated by absolute_url at a remote site # will be always towards the proxy method again. # # Note that the code assumes that VirtualHostBase is visible. The setting # of a front-end server must allow this. # # This should generate an URL like this: # # http://remotehost:9080/VirtualHostBase/http/localhost:8080/VirtualHostRoot/_vh_erp5/_vh_portal_wizard/_vh_proxy/erp5/person_module/2 part_list = [] server_url = self.getServerUrl().rstrip('/') part_list.append(server_url) part_list.append('VirtualHostBase') portal_url = self.getPortalObject().absolute_url() scheme, rest = urllib.splittype(portal_url) addr, path = urllib.splithost(rest) host, port = urllib.splitnport(addr, scheme == 'http' and 80 or 443) part_list.append(scheme) part_list.append('%s:%s' % (host, port)) part_list.append('VirtualHostRoot') method_path = self.absolute_url_path() + '/proxy' part_list.extend(('_vh_' + p for p in method_path.split('/') if p)) server_root = self.getServerRoot().strip('/') if isinstance(subpath, (list, tuple)): subpath = '/'.join(subpath) if not subpath.startswith(server_root): part_list.append(server_root) part_list.append(subpath) url = '/'.join((p for p in part_list if p)) if query: url = url + '?' + query return url def _getSubsribedUserAndPassword(self): """Retrieve the username and password for the subscription from the system.""" user = CachingMethod(self.getExpressConfigurationPreference, 'WizardTool_preferred_express_user_id', cache_factory='erp5_content_long')( 'preferred_express_user_id', '') pw = CachingMethod(self.getExpressConfigurationPreference, 'WizardTool_preferred_express_password', cache_factory='erp5_content_long')( 'preferred_express_password', '') return (user, pw) # This is a custom opener director for not handling redirections # and errors automatically. This is necessary because the proxy # should pass all results to a client as they are. simple_opener_director = urllib2.OpenerDirector() for name in ('ProxyHandler', 'UnknownHandler', \ 'HTTPHandler', 'FTPHandler', 'FileHandler', 'HTTPSHandler',): handler = getattr(urllib2, name, None) if handler is not None: simple_opener_director.add_handler(handler()) # add cookie support simple_opener_director.add_handler(urllib2.HTTPCookieProcessor(cookiejar)) security.declareProtected(Permissions.View, 'proxy') def proxy(self, **kw): """Proxy a request to a server.""" global cookiejar, referer, last_loggedin_user_and_password if self.REQUEST['REQUEST_METHOD'] != 'GET': # XXX this depends on the internal of HTTPRequest. pos = self.REQUEST.stdin.tell() self.REQUEST.stdin.seek(0) # XXX if filesize is too big, this might cause a problem. data = self.REQUEST.stdin.read() self.REQUEST.stdin.seek(pos) else: data = None content_type = self.REQUEST.get_header('content-type') # XXX if ":method" trick is used, then remove it from subpath. if self.REQUEST.traverse_subpath: if data is not None: user_input = data else: user_input = self.REQUEST.QUERY_STRING if user_input: mark = ':method' content_type_value = None content_type_dict = None if content_type: content_type_value, content_type_dict = cgi.parse_header( content_type) if content_type_value == 'multipart/form-data': fp = StringIO(user_input) user_input_dict = cgi.parse_multipart( fp, content_type_dict) else: user_input_dict = cgi.parse_qs(user_input) for i in user_input_dict: if i.endswith(mark): method_name = i[:-len(mark)] method_path = method_name.split('/') if self.REQUEST.traverse_subpath[ -len(method_path):] == method_path: del self.REQUEST.traverse_subpath[-len(method_path ):] break url = self._getProxyURL(self.REQUEST.traverse_subpath, self.REQUEST['QUERY_STRING']) # XXX this will send the password unconditionally! # I hope https will be good enough. header_dict = {} user_and_password = self._getSubsribedUserAndPassword() if (len(user_and_password) == 2 and user_and_password[0] and user_and_password[1]): if user_and_password != last_loggedin_user_and_password: # credentials changed we need to renew __ac cookie from server as well cookiejar.clear() # try login to server only once using cookie method if not _isUserAcknowledged(cookiejar): server_url = self.getServerUrl() f = _getAcCookieFromServer('%s/WebSite_login' % server_url, self.simple_opener_director, cookiejar, user_and_password[0], user_and_password[1]) # if server doesn't support cookie authentication try basic # authentication if not _isUserAcknowledged(cookiejar): auth = 'Basic %s' % base64.standard_b64encode( '%s:%s' % user_and_password) header_dict['Authorization'] = auth # save last credentials we passed to server last_loggedin_user_and_password = user_and_password if content_type: header_dict['Content-Type'] = content_type # send locally saved cookies to remote web server if not header_dict.has_key('Cookie'): header_dict['Cookie'] = '' for cookie in cookiejar: # unconditionally send all cookies (no matter if expired or not) as URL # is always the same header_dict['Cookie'] += '%s=%s;' % (cookie.name, cookie.value) # include cookies from local browser (like show/hide tabs) which are set # directly by client JavaScript code (i.e. not sent from server) for cookie_name, cookie_value in self.REQUEST.cookies.items(): header_dict['Cookie'] += '%s=%s;' % (cookie_name, cookie_value) # add HTTP referer (especially useful in Localizer when changing language) header_dict['REFERER'] = self.REQUEST.get('HTTP_REFERER', None) or referer request = urllib2.Request(url, data, header_dict) f = self.simple_opener_director.open(request) try: data = f.read() metadata = f.info() response = self.REQUEST.RESPONSE if f.code > 300 and f.code < 400: # adjust return url which my contain proxy URLs as arguments location = metadata.getheader('location') if location is not None: parsed_url = list(urlparse(location)) local_site_url_prefix = urllib.quote( '%s/portal_wizard/proxy' % self.getPortalObject().absolute_url()) remote_url_parsed = urlparse(self.getServerUrl()) remote_site_url_prefix = '%s://%s/kb' % ( remote_url_parsed[0], remote_url_parsed[1]) # fix arguments for returned location URL parsed_url[4] = parsed_url[4].replace( local_site_url_prefix, remote_site_url_prefix) response['location'] = urlunparse(parsed_url) response.setStatus(f.code, f.msg) response.setHeader('content-type', metadata.getheader('content-type')) # FIXME this list should be confirmed with the RFC 2616. for k in ('uri', 'cache-control', 'last-modified', 'etag', 'if-matched', 'if-none-match', 'if-range', 'content-language', 'content-range' 'content-location', 'content-md5', 'expires', 'content-encoding', 'vary', 'pragma', 'content-disposition', 'content-length', 'age'): if k in metadata: response.setHeader(k, metadata.getheader(k)) return data finally: f.close() def _getRemoteWitchTool(self, url, user_name=None, password=None): """ Return remote portal_witch tool interface. """ handle = self.getPortalObject().portal_web_services.connect( url=url, user_name=user_name, password=password, transport='xml-rpc') return handle.portal_witch def callRemoteProxyMethod(self, distant_method, server_url=None, use_cache=1, ignore_exceptions=1, **kw): """ Call proxy method on server. """ configurator_user_preferred_language = self\ .getConfiguratorUserPreferredLanguage() def wrapper(distant_method, **kw): return self._callRemoteMethod(distant_method, use_proxy=1, ignore_exceptions=ignore_exceptions, **kw)['data'] if use_cache: wrapper = CachingMethod( wrapper, id='callRemoteProxyMethod_%s_%s' % (distant_method, configurator_user_preferred_language), cache_factory='erp5_ui_medium') rc = wrapper(distant_method, **kw) return rc def _callRemoteMethod(self, distant_method, server_url=None, use_proxy=0, ignore_exceptions=1, **kw): """ Call remote method on server and get result. """ result_call = GeneratorCall() user_name = None password = None if server_url is None: # calculate it server_url = self.getServerUrl() + self.getServerRoot() # include authentication if possible user_and_password = self._getSubsribedUserAndPassword() if (len(user_and_password) == 2 and user_and_password[0] and user_and_password[1]): user_name, password = user_and_password witch_tool = self._getRemoteWitchTool(server_url, user_name, password) parameter_dict = self.REQUEST.form.copy() if use_proxy: # add remote method arguments parameter_dict['method_id'] = distant_method parameter_dict['method_kw'] = kw distant_method = 'proxyMethodHandler' ## add client arguments self._updateParameterDictWithServerInfo(parameter_dict) ## handle file upload self._updateParameterDictWithFileUpload(parameter_dict) message = None ## call remote method try: method = getattr(witch_tool, distant_method) html = method(parameter_dict) except socket.error, message: html = _generateErrorXML("""Cannot contact the server: %s. Please check your network settings.""" % server_url) zLOG.LOG('Wizard Tool socket error', zLOG.ERROR, message) result_call.update({ "command": "show", "data": html, "next": None, "previous": None }) except xmlrpclib.ProtocolError, message: html = _generateErrorXML("""The server %s refused to reply. Please contact [email protected]""" % server_url) zLOG.LOG('Wizard Tool xmlrpc protocol error', zLOG.ERROR, message) result_call.update({ "command": "show", "data": html, "next": None, "previous": None })
def check_http_traffic(request): # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) class do_connection(httplib.HTTPConnection, httplib.HTTPSConnection): """ Checking the HTTP / HTTPS requests. """ def request(self, method, url, body, headers): info_msg = "The provided HTTP request headers: " if settings.VERBOSITY_LEVEL >= 2: print settings.print_info_msg(info_msg) if menu.options.traffic_file: logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37) header = method + " " + url if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) for item in headers.items(): header = item[0] + ": " + item[1] if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if body: header = body if settings.VERBOSITY_LEVEL >= 2: print settings.print_traffic(header) if menu.options.traffic_file: logs.log_traffic("\n" + header) if menu.options.traffic_file: logs.log_traffic("\n\n") if settings.PROXY_PROTOCOL == 'https': httplib.HTTPSConnection.request(self, method, url, body, headers) else: httplib.HTTPConnection.request(self, method, url, body, headers) class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler): if settings.PROXY_PROTOCOL == 'https': def https_open(self, req): return self.do_open(do_connection, req) else: def http_open(self, req): return self.do_open(do_connection, req) if settings.REVERSE_TCP == False and settings.BIND_TCP == False: opener = urllib2.OpenerDirector() opener.add_handler(connection_handler()) response = False current_attempt = 0 while not response and current_attempt <= settings.MAX_RETRIES: try: opener.open(request) response = True except urllib2.URLError, err_msg: if settings.VERBOSITY_LEVEL >= 2: info_msg = str(err_msg.args[0]).split("] ")[1] + ", " info_msg += "please wait while retring the request(s)." print settings.print_info_msg(info_msg) current_attempt = current_attempt + 1 time.sleep(3) pass