def request(): uri = options.pop('endpoint', config.endpoint) if '://' not in uri: uri = config.get_endpoint() api_key = json.loads(payload).pop('apiKey', config.get('api_key')) req = Request(uri, payload.encode('utf-8', 'replace'), default_headers(api_key)) if config.proxy_host: proxies = ProxyHandler({ 'https': config.proxy_host, 'http': config.proxy_host }) opener = build_opener(proxies) else: opener = build_opener() resp = opener.open(req) status = resp.getcode() if 'success' in options: success = options['success'] else: success = 200 if status != success: bugsnag.logger.warning('Delivery to %s failed, status %d' % (uri, status))
def set_proxy(proxy, user=None, password=''): """ Set the HTTP proxy for Python to download through. If ``proxy`` is None then tries to set proxy from environment or system settings. :param proxy: The HTTP proxy server to use. For example: 'http://proxy.example.com:3128/' :param user: The username to authenticate with. Use None to disable authentication. :param password: The password to authenticate with. """ from nltk import compat if proxy is None: # Try and find the system proxy settings try: proxy = getproxies()['http'] except KeyError: raise ValueError('Could not detect default proxy settings') # Set up the proxy handler proxy_handler = ProxyHandler({'https': proxy, 'http': proxy}) opener = build_opener(proxy_handler) if user is not None: # Set up basic proxy authentication if provided password_manager = HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(realm=None, uri=proxy, user=user, passwd=password) opener.add_handler(ProxyBasicAuthHandler(password_manager)) opener.add_handler(ProxyDigestAuthHandler(password_manager)) # Overide the existing url opener install_opener(opener)
def _create_opener(): """ Creates an opener for the internet. It also attaches the :class:`CachingRedirectHandler` to the opener and sets its User-agent to ``Mozilla/5.0``. If the Network Proxy settings are set and recognized, it creates the opener and attaches the proxy_handler to it. The opener is tested and returned if the test passes. If the test fails an opener without the proxy settings is created instead and is returned instead. """ use_proxy = False proxy_handler = None if NETWORK_PROXY_TYPE == 'http': use_proxy = True proxyurl = _get_http_proxy_url() proxy_handler = ProxyHandler({'http': proxyurl, 'https': proxyurl}) if use_proxy: openr = build_opener(HTTPHandler(), HTTPSHandler(), proxy_handler, CachingRedirectHandler) else: openr = build_opener(HTTPSHandler(), HTTPSHandler(), CachingRedirectHandler) openr.addheaders = [('User-agent', 'Mozilla/5.0')] global _internet_connected _internet_connected = _test_opener(openr) return openr
def setup_urllib_proxies(): global _urllib_proxies_installed, SYSTEM_PROXIES if _urllib_proxies_installed: return _urllib_proxies_installed = True if not SYSTEM_PROXIES: return proxies = dict( (k, "%s://%s:%s" % (k, SYSTEM_PROXIES[k][0], SYSTEM_PROXIES[k][1])) for k in SYSTEM_PROXIES ) from six.moves.urllib.request import ProxyHandler, build_opener, install_opener proxy_handler = ProxyHandler(proxies) opener = build_opener(proxy_handler) install_opener(opener)
def prepare(self): """ Read options for uploading, check that they're sane """ super(BlazeMeterUploader, self).prepare() self.client.address = self.settings.get("address", self.client.address) self.client.data_address = self.settings.get("data-address", self.client.data_address) self.client.timeout = dehumanize_time(self.settings.get("timeout", self.client.timeout)) self.send_interval = dehumanize_time(self.settings.get("send-interval", self.send_interval)) self.browser_open = self.settings.get("browser-open", self.browser_open) token = self.settings.get("token", "") proxy_settings = self.engine.config.get("settings").get("proxy") if proxy_settings: if proxy_settings.get("address"): proxy_url = urlsplit(proxy_settings.get("address")) username = proxy_settings.get("username") pwd = proxy_settings.get("password") if username and pwd: proxy_uri = "%s://%s:%s@%s" % (proxy_url.scheme, username, pwd, proxy_url.netloc) else: proxy_uri = "%s://%s" % (proxy_url.scheme, proxy_url.netloc) proxy_handler = ProxyHandler({"https": proxy_uri, "http": proxy_uri}) opener = build_opener(proxy_handler) install_opener(opener) if not token: self.log.warning("No BlazeMeter API key provided, will upload anonymously") self.client.token = token self.client.active_session_id = self.parameters.get("session-id", None) self.client.test_id = self.parameters.get("test-id", None) self.client.user_id = self.parameters.get("user-id", None) self.client.data_signature = self.parameters.get("signature", None) if not self.client.test_id: test_name = self.parameters.get("test", "Taurus Test") # TODO: provide a way to put datetime into test name try: self.client.ping() # to check connectivity and auth if token: self.test_id = self.client.test_by_name(test_name, {"type": "external"}) except HTTPError: self.log.error("Cannot reach online results storage, maybe the address/token is wrong") raise
def fixture_httpd_with_proxy_handler(docroot): """Yields a started MozHttpd server for the proxy test.""" httpd = mozhttpd.MozHttpd(port=0, docroot=str(docroot)) httpd.start(block=False) port = httpd.httpd.server_port proxy_support = ProxyHandler({ "http": "http://127.0.0.1:{port:d}".format(port=port), }) install_opener(build_opener(proxy_support)) yield httpd httpd.stop() # Reset proxy opener in case it changed install_opener(None)
def http_emitter(message, log, url): """Send payload """ log.debug('http_emitter: attempting postback to ' + url) # Post back the data partial_payload = [] for measurement in message: partial_payload.append(measurement) payload = json.dumps(partial_payload) if PY3: payload = payload.encode('utf-8') url = "%s/intake" % url headers = post_headers(payload) try: # Make sure no proxy is autodetected for this localhost connection proxy_handler = ProxyHandler({}) # Should this be installed as the default opener and reused? opener = build_opener(proxy_handler) request = Request(url, payload, headers) response = None try: response = opener.open(request) log.debug('http_emitter: postback response: ' + str(response.read())) except Exception as exc: log.error("""Forwarder at {0} is down or not responding... Error is {1} Please restart the monasca-agent.""".format( url, repr(exc))) finally: if response: response.close() except HTTPError as e: if e.code == 202: log.debug("http payload accepted") else: raise
def fixture_httpd_with_proxy_host_dirs(docroot, hosts): for host in hosts: index_file = docroot.mkdir(host).join("index.html") index_file.write(index_contents(host)) httpd = mozhttpd.MozHttpd(port=0, docroot=str(docroot), proxy_host_dirs=True) httpd.start(block=False) port = httpd.httpd.server_port proxy_support = ProxyHandler( {"http": "http://127.0.0.1:{port:d}".format(port=port)}) install_opener(build_opener(proxy_support)) yield httpd httpd.stop() # Reset proxy opener in case it changed install_opener(None)
def request(): endpoint = config.endpoint if '://' not in endpoint: endpoint = config.get_endpoint() req = Request(endpoint, payload.encode('utf-8', 'replace'), {'Content-Type': 'application/json'}) if config.proxy_host: proxies = ProxyHandler({ 'https': config.proxy_host, 'http': config.proxy_host }) opener = build_opener(proxies) else: opener = build_opener() resp = opener.open(req) status = resp.getcode() if status != 200: bugsnag.logger.warning('Notification to %s failed, status %d' % (config.endpoint, status))
def __init__(self, url, cookie_file=None, username=None, password=None, api_token=None, agent=None, session=None, disable_proxy=False, auth_callback=None, otp_token_callback=None, verify_ssl=True, save_cookies=True, ext_auth_cookies=None): if not url.endswith('/'): url += '/' self.url = url + 'api/' self.save_cookies = save_cookies self.ext_auth_cookies = ext_auth_cookies if self.save_cookies: self.cookie_jar, self.cookie_file = create_cookie_jar( cookie_file=cookie_file) try: self.cookie_jar.load(ignore_expires=True) except IOError: pass else: self.cookie_jar = CookieJar() self.cookie_file = None if self.ext_auth_cookies: try: self.cookie_jar.load(ext_auth_cookies, ignore_expires=True) except IOError as e: logging.critical( 'There was an error while loading a ' 'cookie file: %s', e) pass # Get the cookie domain from the url. If the domain # does not contain a '.' (e.g. 'localhost'), we assume # it is a local domain and suffix it (See RFC 2109). parsed_url = urlparse(url) self.domain = parsed_url[1].partition(':')[0] # Remove Port. if self.domain.count('.') < 1: self.domain = '%s.local' % self.domain if session: cookie = Cookie(version=0, name=RB_COOKIE_NAME, value=session, port=None, port_specified=False, domain=self.domain, domain_specified=True, domain_initial_dot=True, path=parsed_url[2], path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={'HttpOnly': None}) self.cookie_jar.set_cookie(cookie) if self.save_cookies: self.cookie_jar.save() if username: # If the username parameter is given, we have to clear the session # cookie manually or it will override the username:password # combination retrieved from the authentication callback. try: self.cookie_jar.clear(self.domain, parsed_url[2], RB_COOKIE_NAME) except KeyError: pass # Set up the HTTP libraries to support all of the features we need. password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password, api_token, auth_callback, otp_token_callback) self.preset_auth_handler = PresetHTTPAuthHandler( self.url, password_mgr) handlers = [] if not verify_ssl: context = ssl._create_unverified_context() handlers.append(HTTPSHandler(context=context)) if disable_proxy: handlers.append(ProxyHandler({})) handlers += [ HTTPCookieProcessor(self.cookie_jar), ReviewBoardHTTPBasicAuthHandler(password_mgr), HTTPDigestAuthHandler(password_mgr), self.preset_auth_handler, ReviewBoardHTTPErrorProcessor(), ] if agent: self.agent = agent else: self.agent = ('RBTools/' + get_package_version()).encode('utf-8') opener = build_opener(*handlers) opener.addheaders = [ (str('User-agent'), str(self.agent)), ] install_opener(opener) self._cache = None self._urlopen = urlopen
def __init__(self, url, cookie_file=None, username=None, password=None, api_token=None, agent=None, session=None, disable_proxy=False, auth_callback=None, otp_token_callback=None): self.url = url if not self.url.endswith('/'): self.url += '/' self.url = self.url + 'api/' self.cookie_jar, self.cookie_file = create_cookie_jar( cookie_file=cookie_file) try: self.cookie_jar.load(ignore_expires=True) except IOError: pass if session: parsed_url = urlparse(url) # Get the cookie domain from the url. If the domain # does not contain a '.' (e.g. 'localhost'), we assume # it is a local domain and suffix it (See RFC 2109). domain = parsed_url[1].partition(':')[0] # Remove Port. if domain.count('.') < 1: domain = '%s.local' % domain cookie = Cookie(version=0, name=RB_COOKIE_NAME, value=session, port=None, port_specified=False, domain=domain, domain_specified=True, domain_initial_dot=True, path=parsed_url[2], path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={'HttpOnly': None}) self.cookie_jar.set_cookie(cookie) self.cookie_jar.save() # Set up the HTTP libraries to support all of the features we need. password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password, api_token, auth_callback, otp_token_callback) self.preset_auth_handler = PresetHTTPAuthHandler( self.url, password_mgr) handlers = [] if disable_proxy: handlers.append(ProxyHandler({})) handlers += [ HTTPCookieProcessor(self.cookie_jar), ReviewBoardHTTPBasicAuthHandler(password_mgr), HTTPDigestAuthHandler(password_mgr), self.preset_auth_handler, ReviewBoardHTTPErrorProcessor(), ] if agent: self.agent = agent else: self.agent = ('RBTools/' + get_package_version()).encode('utf-8') opener = build_opener(*handlers) opener.addheaders = [ (b'User-agent', self.agent), ] install_opener(opener) self._cache = APICache()
def test_proxy(self): docroot = tempfile.mkdtemp() self.addCleanup(mozfile.remove, docroot) hosts = ('mozilla.com', 'mozilla.org') unproxied_host = 'notmozilla.org' def url(host): return 'http://%s/' % host index_filename = 'index.html' def index_contents(host): return '%s index' % host index = open(os.path.join(docroot, index_filename), 'w') index.write(index_contents('*')) index.close() httpd = mozhttpd.MozHttpd(port=0, docroot=docroot) httpd.start(block=False) server_port = httpd.httpd.server_port proxy_support = ProxyHandler({ 'http': 'http://127.0.0.1:%d' % server_port, }) install_opener(build_opener(proxy_support)) for host in hosts: f = urlopen(url(host)) try: self.assertEqual(f.getcode(), 200) except AttributeError: pass # python 2.4 self.assertEqual(f.read(), index_contents('*')) httpd.stop() # test separate directories per host httpd = mozhttpd.MozHttpd(port=0, docroot=docroot, proxy_host_dirs=True) httpd.start(block=False) server_port = httpd.httpd.server_port proxy_support = ProxyHandler({ 'http': 'http://127.0.0.1:%d' % server_port, }) install_opener(build_opener(proxy_support)) # set up dirs for host in hosts: os.mkdir(os.path.join(docroot, host)) open(os.path.join(docroot, host, index_filename), 'w') \ .write(index_contents(host)) for host in hosts: f = urlopen(url(host)) try: self.assertEqual(f.getcode(), 200) except AttributeError: pass # python 2.4 self.assertEqual(f.read(), index_contents(host)) exc = None try: urlopen(url(unproxied_host)) except HTTPError as e: exc = e self.assertNotEqual(exc, None) self.assertEqual(exc.code, 404)