def request():
uri = options.pop('endpoint', config.endpoint)
if '://' not in uri:
uri = config.get_endpoint()
api_key = json.loads(payload).pop('apiKey', config.get('api_key'))
req = Request(uri, payload.encode('utf-8', 'replace'),
default_headers(api_key))
if config.proxy_host:
proxies = ProxyHandler({
'https': config.proxy_host,
'http': config.proxy_host
})
opener = build_opener(proxies)
else:
opener = build_opener()
resp = opener.open(req)
status = resp.getcode()
if 'success' in options:
success = options['success']
else:
success = 200
if status != success:
bugsnag.logger.warning('Delivery to %s failed, status %d' %
(uri, status))
def set_proxy(proxy, user=None, password=''):
"""
Set the HTTP proxy for Python to download through.
If ``proxy`` is None then tries to set proxy from environment or system
settings.
:param proxy: The HTTP proxy server to use. For example:
'http://proxy.example.com:3128/'
:param user: The username to authenticate with. Use None to disable
authentication.
:param password: The password to authenticate with.
"""
from nltk import compat
if proxy is None:
# Try and find the system proxy settings
try:
proxy = getproxies()['http']
except KeyError:
raise ValueError('Could not detect default proxy settings')
# Set up the proxy handler
proxy_handler = ProxyHandler({'https': proxy, 'http': proxy})
opener = build_opener(proxy_handler)
if user is not None:
# Set up basic proxy authentication if provided
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None, uri=proxy, user=user, passwd=password)
opener.add_handler(ProxyBasicAuthHandler(password_manager))
opener.add_handler(ProxyDigestAuthHandler(password_manager))
# Overide the existing url opener
install_opener(opener)
def _create_opener():
"""
Creates an opener for the internet.
It also attaches the :class:`CachingRedirectHandler` to the opener and
sets its User-agent to ``Mozilla/5.0``.
If the Network Proxy settings are set and recognized, it creates the
opener and attaches the proxy_handler to it. The opener is tested and
returned if the test passes.
If the test fails an opener without the proxy settings is created instead
and is returned instead.
"""
use_proxy = False
proxy_handler = None
if NETWORK_PROXY_TYPE == 'http':
use_proxy = True
proxyurl = _get_http_proxy_url()
proxy_handler = ProxyHandler({'http': proxyurl, 'https': proxyurl})
if use_proxy:
openr = build_opener(HTTPHandler(), HTTPSHandler(), proxy_handler,
CachingRedirectHandler)
else:
openr = build_opener(HTTPSHandler(), HTTPSHandler(),
CachingRedirectHandler)
openr.addheaders = [('User-agent', 'Mozilla/5.0')]
global _internet_connected
_internet_connected = _test_opener(openr)
return openr
def setup_urllib_proxies():
global _urllib_proxies_installed, SYSTEM_PROXIES
if _urllib_proxies_installed:
return
_urllib_proxies_installed = True
if not SYSTEM_PROXIES:
return
proxies = dict(
(k, "%s://%s:%s" % (k, SYSTEM_PROXIES[k][0], SYSTEM_PROXIES[k][1])) for k in SYSTEM_PROXIES
)
from six.moves.urllib.request import ProxyHandler, build_opener, install_opener
proxy_handler = ProxyHandler(proxies)
opener = build_opener(proxy_handler)
install_opener(opener)
def prepare(self):
"""
Read options for uploading, check that they're sane
"""
super(BlazeMeterUploader, self).prepare()
self.client.address = self.settings.get("address", self.client.address)
self.client.data_address = self.settings.get("data-address", self.client.data_address)
self.client.timeout = dehumanize_time(self.settings.get("timeout", self.client.timeout))
self.send_interval = dehumanize_time(self.settings.get("send-interval", self.send_interval))
self.browser_open = self.settings.get("browser-open", self.browser_open)
token = self.settings.get("token", "")
proxy_settings = self.engine.config.get("settings").get("proxy")
if proxy_settings:
if proxy_settings.get("address"):
proxy_url = urlsplit(proxy_settings.get("address"))
username = proxy_settings.get("username")
pwd = proxy_settings.get("password")
if username and pwd:
proxy_uri = "%s://%s:%s@%s" % (proxy_url.scheme, username, pwd, proxy_url.netloc)
else:
proxy_uri = "%s://%s" % (proxy_url.scheme, proxy_url.netloc)
proxy_handler = ProxyHandler({"https": proxy_uri, "http": proxy_uri})
opener = build_opener(proxy_handler)
install_opener(opener)
if not token:
self.log.warning("No BlazeMeter API key provided, will upload anonymously")
self.client.token = token
self.client.active_session_id = self.parameters.get("session-id", None)
self.client.test_id = self.parameters.get("test-id", None)
self.client.user_id = self.parameters.get("user-id", None)
self.client.data_signature = self.parameters.get("signature", None)
if not self.client.test_id:
test_name = self.parameters.get("test", "Taurus Test") # TODO: provide a way to put datetime into test name
try:
self.client.ping() # to check connectivity and auth
if token:
self.test_id = self.client.test_by_name(test_name, {"type": "external"})
except HTTPError:
self.log.error("Cannot reach online results storage, maybe the address/token is wrong")
raise
def fixture_httpd_with_proxy_handler(docroot):
"""Yields a started MozHttpd server for the proxy test."""
httpd = mozhttpd.MozHttpd(port=0, docroot=str(docroot))
httpd.start(block=False)
port = httpd.httpd.server_port
proxy_support = ProxyHandler({
"http":
"http://127.0.0.1:{port:d}".format(port=port),
})
install_opener(build_opener(proxy_support))
yield httpd
httpd.stop()
# Reset proxy opener in case it changed
install_opener(None)
def http_emitter(message, log, url):
"""Send payload
"""
log.debug('http_emitter: attempting postback to ' + url)
# Post back the data
partial_payload = []
for measurement in message:
partial_payload.append(measurement)
payload = json.dumps(partial_payload)
if PY3:
payload = payload.encode('utf-8')
url = "%s/intake" % url
headers = post_headers(payload)
try:
# Make sure no proxy is autodetected for this localhost connection
proxy_handler = ProxyHandler({})
# Should this be installed as the default opener and reused?
opener = build_opener(proxy_handler)
request = Request(url, payload, headers)
response = None
try:
response = opener.open(request)
log.debug('http_emitter: postback response: ' +
str(response.read()))
except Exception as exc:
log.error("""Forwarder at {0} is down or not responding...
Error is {1}
Please restart the monasca-agent.""".format(
url, repr(exc)))
finally:
if response:
response.close()
except HTTPError as e:
if e.code == 202:
log.debug("http payload accepted")
else:
raise
def fixture_httpd_with_proxy_host_dirs(docroot, hosts):
for host in hosts:
index_file = docroot.mkdir(host).join("index.html")
index_file.write(index_contents(host))
httpd = mozhttpd.MozHttpd(port=0,
docroot=str(docroot),
proxy_host_dirs=True)
httpd.start(block=False)
port = httpd.httpd.server_port
proxy_support = ProxyHandler(
{"http": "http://127.0.0.1:{port:d}".format(port=port)})
install_opener(build_opener(proxy_support))
yield httpd
httpd.stop()
# Reset proxy opener in case it changed
install_opener(None)
def request():
endpoint = config.endpoint
if '://' not in endpoint:
endpoint = config.get_endpoint()
req = Request(endpoint, payload.encode('utf-8', 'replace'),
{'Content-Type': 'application/json'})
if config.proxy_host:
proxies = ProxyHandler({
'https': config.proxy_host,
'http': config.proxy_host
})
opener = build_opener(proxies)
else:
opener = build_opener()
resp = opener.open(req)
status = resp.getcode()
if status != 200:
bugsnag.logger.warning('Notification to %s failed, status %d' %
(config.endpoint, status))
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None,
verify_ssl=True,
save_cookies=True,
ext_auth_cookies=None):
if not url.endswith('/'):
url += '/'
self.url = url + 'api/'
self.save_cookies = save_cookies
self.ext_auth_cookies = ext_auth_cookies
if self.save_cookies:
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
else:
self.cookie_jar = CookieJar()
self.cookie_file = None
if self.ext_auth_cookies:
try:
self.cookie_jar.load(ext_auth_cookies, ignore_expires=True)
except IOError as e:
logging.critical(
'There was an error while loading a '
'cookie file: %s', e)
pass
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
parsed_url = urlparse(url)
self.domain = parsed_url[1].partition(':')[0] # Remove Port.
if self.domain.count('.') < 1:
self.domain = '%s.local' % self.domain
if session:
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=self.domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
if self.save_cookies:
self.cookie_jar.save()
if username:
# If the username parameter is given, we have to clear the session
# cookie manually or it will override the username:password
# combination retrieved from the authentication callback.
try:
self.cookie_jar.clear(self.domain, parsed_url[2],
RB_COOKIE_NAME)
except KeyError:
pass
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if not verify_ssl:
context = ssl._create_unverified_context()
handlers.append(HTTPSHandler(context=context))
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(str('User-agent'), str(self.agent)),
]
install_opener(opener)
self._cache = None
self._urlopen = urlopen
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None):
self.url = url
if not self.url.endswith('/'):
self.url += '/'
self.url = self.url + 'api/'
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
if session:
parsed_url = urlparse(url)
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
domain = parsed_url[1].partition(':')[0] # Remove Port.
if domain.count('.') < 1:
domain = '%s.local' % domain
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
self.cookie_jar.save()
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(b'User-agent', self.agent),
]
install_opener(opener)
self._cache = APICache()
def test_proxy(self):
docroot = tempfile.mkdtemp()
self.addCleanup(mozfile.remove, docroot)
hosts = ('mozilla.com', 'mozilla.org')
unproxied_host = 'notmozilla.org'
def url(host):
return 'http://%s/' % host
index_filename = 'index.html'
def index_contents(host):
return '%s index' % host
index = open(os.path.join(docroot, index_filename), 'w')
index.write(index_contents('*'))
index.close()
httpd = mozhttpd.MozHttpd(port=0, docroot=docroot)
httpd.start(block=False)
server_port = httpd.httpd.server_port
proxy_support = ProxyHandler({
'http':
'http://127.0.0.1:%d' % server_port,
})
install_opener(build_opener(proxy_support))
for host in hosts:
f = urlopen(url(host))
try:
self.assertEqual(f.getcode(), 200)
except AttributeError:
pass # python 2.4
self.assertEqual(f.read(), index_contents('*'))
httpd.stop()
# test separate directories per host
httpd = mozhttpd.MozHttpd(port=0,
docroot=docroot,
proxy_host_dirs=True)
httpd.start(block=False)
server_port = httpd.httpd.server_port
proxy_support = ProxyHandler({
'http':
'http://127.0.0.1:%d' % server_port,
})
install_opener(build_opener(proxy_support))
# set up dirs
for host in hosts:
os.mkdir(os.path.join(docroot, host))
open(os.path.join(docroot, host, index_filename), 'w') \
.write(index_contents(host))
for host in hosts:
f = urlopen(url(host))
try:
self.assertEqual(f.getcode(), 200)
except AttributeError:
pass # python 2.4
self.assertEqual(f.read(), index_contents(host))
exc = None
try:
urlopen(url(unproxied_host))
except HTTPError as e:
exc = e
self.assertNotEqual(exc, None)
self.assertEqual(exc.code, 404)