def fetch(method, uri, params_prefix=None, **params):
"""Fetch the given uri and return the contents of the response."""
params = urlencode(_prepare_params(params, params_prefix))
binary_params = params.encode('ASCII')
# build the HTTP request
url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri)
req = Request(url, binary_params)
req.get_method = lambda: method
# use basic authentication
user, api_key = get_credentials()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key
)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except HTTPError as e:
if e.code != 422:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def create_url_opener(cookie_file_path, domain):
"""Load username and password from .gitcookies and return a URL opener with
an authentication handler."""
# Load authentication credentials
credentials = load_auth_credentials(cookie_file_path)
username, password = credentials[domain]
# Create URL opener with authentication handler
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(domain, domain, username, password)
return build_opener(auth_handler)
def fetch(method, uri, params_prefix=None, **params):
"""Fetch the given uri and return the contents of the response."""
params = urlencode(_prepare_params(params, params_prefix))
binary_params = params.encode('ASCII')
# build the HTTP request
url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri)
req = Request(url, binary_params)
req.get_method = lambda: method
# use basic authentication
user, api_key = get_credentials()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except HTTPError as e:
if e.code != 422:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def opener_for_url_prefix(url_prefix,
username=None,
password=None,
cache_dict=None):
if cache_dict is not None:
cache_key = (url_prefix, username, password)
try:
return cache_dict[cache_key]
except KeyError:
pass
if username or password:
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm="Open Amiga Game Database",
uri="{0}".format(url_prefix),
user=username,
passwd=password)
auth_handler.add_password(realm="OpenRetro",
uri="{0}".format(url_prefix),
user=username,
passwd=password)
opener = build_opener(auth_handler)
else:
opener = build_opener()
if cache_dict is not None:
cache_key = (url_prefix, username, password)
cache_dict[cache_key] = opener
return opener
def POST(url, args={}, cred=None):
"""do http post
url is the URL you want
args is a dict of cgi args
cred is ( host, realm, username, password )
"""
auth_handler = None
arg_string = ''
if cred is not None:
(host, realm, username, password) = cred
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm, host, username, password)
if auth_handler:
opener = build_opener(cookie_processor, auth_handler)
else:
opener = build_opener(cookie_processor)
install_opener(opener)
print("URL %s" % url)
data = urlencode(args)
req = Request(url, data)
f = urlopen(req)
return f
def get_auth_handler(cls, feed_url, username, password, logger=None):
"""
Create a URL that will perform authentication for the given feed.
Arguments:
feed_url -- The URL of the feed to retrieve (as a string)
username -- The username to use when authenticating
password -- The password to use when authenticating
"""
realm, auth_type = cls.get_realm_and_auth_type(feed_url, username, password, logger)
# Make the associated auth handler
if auth_type == None:
return None
elif auth_type.lower() == "basic":
auth_handler = HTTPBasicAuthHandler()
else:
auth_handler = HTTPDigestAuthHandler()
# Set the password
auth_handler.add_password(realm=realm,
uri=feed_url,
user=username,
passwd=password)
return auth_handler
def opener_for_url_prefix(
url_prefix, username=None, password=None, cache_dict=None
):
if cache_dict is not None:
cache_key = (url_prefix, username, password)
try:
return cache_dict[cache_key]
except KeyError:
pass
if username or password:
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Open Amiga Game Database",
uri="{0}".format(url_prefix),
user=username,
passwd=password,
)
auth_handler.add_password(
realm="OpenRetro",
uri="{0}".format(url_prefix),
user=username,
passwd=password,
)
opener = build_opener(auth_handler)
else:
opener = build_opener()
if cache_dict is not None:
cache_key = (url_prefix, username, password)
cache_dict[cache_key] = opener
return opener
def _register_agent(self):
register_name = self.app.config.get('PSDASH_REGISTER_AS')
if not register_name:
register_name = socket.gethostname()
url_args = {
'name': register_name,
'port': self.app.config.get('PSDASH_PORT', self.DEFAULT_PORT),
}
register_url = '%s/register?%s' % (
self.app.config['PSDASH_REGISTER_TO'], urllib.urlencode(url_args))
if 'PSDASH_AUTH_USERNAME' in self.app.config and 'PSDASH_AUTH_PASSWORD' in self.app.config:
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm='psDash login required',
uri=register_url,
user=self.app.config['PSDASH_AUTH_USERNAME'],
passwd=self.app.config['PSDASH_AUTH_PASSWORD'])
opener = build_opener(auth_handler)
install_opener(opener)
try:
urlopen(register_url)
except HTTPError as e:
logger.error('Failed to register agent to "%s": %s', register_url,
e)
def send_challonge_request(challonge_path, current_user):
data = {} # needed so the Request object is a "POST" request
req = Request(challonge_path, data)
# use basic authentication
user, api_key = current_user.challonge_username, xor_crypt_string(current_user.api_key, decode=True)
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key
)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except URLLibHTTPError as e:
if e.code != 422:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def POST(url, args={}, cred=None):
"""do http post
url is the URL you want
args is a dict of cgi args
cred is ( host, realm, username, password )
"""
auth_handler = None
arg_string = ''
if cred is not None:
(host, realm, username, password) = cred
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm, host, username, password)
if auth_handler:
opener = build_opener(cookie_processor, auth_handler)
else:
opener = build_opener(cookie_processor)
install_opener(opener)
print("URL %s" % url)
data = urlencode(args)
req = Request(url, data)
f = urlopen(req)
return f
def token_request(uri, post_data):
"""
This function encapsulates the code used to make the request
to the (reddit) OAuth2 endpoint that is shared between
obtaining the first, refreshing and revoking the token.
The parameters are the endpoint URI and dictionary with the post data
to be JSONified and sent with the POST request to the URI.
"""
request_data = urlencode(post_data).encode('utf-8')
token_request = client_http_request(uri, method='POST', data=request_data)
token_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
token_request.add_header('User-Agent', rwh.config['APP_USER_AGENT_SERVER'])
authenticator = HTTPBasicAuthHandler()
authenticator.add_password(realm='reddit', uri=uri,
user=rwh.config['APP_ID'], passwd='')
authenticated_opener = build_opener(authenticator)
request_result = authenticated_opener.open(token_request)
status_code = request_result.getcode()
result_body = None
if request_result:
result_body = request_result.read().decode('utf-8')
if result_body and (len(result_body.strip()) > 0):
result_data = parse_json_string(result_body)
else:
result_data = None
return result_data, status_code
def build_opener(apiurl, user, password, cookie_path, debuglevel=0, capath=None, cafile=None, headers=()):
"""build urllib opener for given name/password
it creates
* HTTPSHandler with proper ssl context
* HTTPCookieProcessor with a link to cookiejar
* HTTPBasicAuthHandler with user/password
* proxyhandler which respects no_proxy variable
"""
handlers = list()
if hasattr(ssl, "SSLContext"):
#allow only sslv3 and tlsv1, but not sslv2
ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_SSLv23)
ctx.options |= ssl.OP_NO_SSLv2
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.set_default_verify_paths()
if cafile or capath:
if ctx.load_verify_locations(capath=capath, cafile=cafile) != -1:
raise Exception("load_verify_locations failed for capath={}, cafile={}".format(capath, cafile))
#TODO: debuglevel
httpshandler = HTTPSHandler(debuglevel=debuglevel, context=ctx, check_hostname=True)
handlers.append(httpshandler)
try:
# TODO is this correct?
cookie_file = os.path.expanduser(cookie_path)
cookiejar = LWPCookieJar(cookie_file)
cookiejar.load(ignore_discard=True)
except (OSError, IOError, AttributeError):
try:
os.open(cookie_file, os.O_WRONLY | os.O_CREAT, mode=0o600)
except:
#TODO: log it
cookiejar = CookieJar()
handlers.append(HTTPCookieProcessor(cookiejar))
authhandler = HTTPBasicAuthHandler(
HTTPPasswordMgrWithDefaultRealm())
authhandler.add_password(None, apiurl, bytes(user, "utf-8"), bytes(password, "ascii"))
handlers.append(authhandler)
# proxy handling
if not proxy_bypass(apiurl):
proxyhandler = ProxyHandler()
else:
proxyhandler = ProxyHandler({})
handlers.append(proxyhandler)
opener = _build_opener(*handlers)
from bslib import __version__
opener.addheaders = [("User-agent", "bslib/{}".format(__version__)), ]
for h in headers:
opener.addheaders(h)
return opener
def create_url_opener(cookie_file_path, domain):
"""Load username and password from .gitcookies and return a URL opener with
an authentication handler."""
# Load authentication credentials
credentials = load_auth_credentials(cookie_file_path)
username, password = credentials[domain]
# Create URL opener with authentication handler
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(domain, domain, username, password)
return build_opener(auth_handler)
def download(self, url: str, user: str, password: str, output: str):
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(None, url, user, password)
req = Request(
url,
headers={'Authorization': self.__build_basic_auth(user, password)})
with urlopen(req) as remote_file:
with self.fs.open(output, 'wb', None) as local_file:
copyfileobj(remote_file, local_file)
return True
def download(self, url: str, user: str, password: str, output:str):
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(None, url, user, password)
req = Request(url, headers = {
'Authorization': self.__build_basic_auth(user, password)
})
with urlopen(req) as remote_file:
with self.fs.open(output, 'wb', None) as local_file:
copyfileobj(remote_file, local_file)
return True
def Auth(User, Pass):
"""
This authentication works for the current 2008-2019 physionet authentication
version. It will NOT work for the NEW version of physionet that is being developed.
"""
url='https://archive.physionet.org/works/MIMICIIIClinicalDatabase/files/'
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm='PhysioNetWorks', uri=url, user=User, passwd=Pass)
opener = build_opener(auth_handler)
try:
r = opener.open(url)
return r.getcode()
except HTTPError as e:
return e.getcode()
def fetch(method, uri, params_prefix=None, **params):
"""Fetch the given uri and return the contents of the response."""
# build the HTTP request
is_json = False
if not params.get('json', None):
params = urlencode(_prepare_params(params, params_prefix))
binary_params = params.encode('ASCII')
url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri)
req = Request(url, binary_params)
req.get_method = lambda: method
else:
url = "https://%s/%s.json" % (CHALLONGE_API_URL, uri)
payload = params.get('data', None)
req = Request(url,
data=payload,
headers={'Content-Type': 'application/json'})
req.get_method = lambda: method
is_json = True
# use basic authentication
user, api_key = get_credentials()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except HTTPError as e:
if e.code != 422:
raise
if is_json:
msg = e.readline()
if 'errors' in msg:
raise ChallongeException(msg)
else:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def _bget(my_url, user=None, passw=None):
"""
Get remote auth-protected url *that contains a ~file~*
and pass its binary data straight from remote response
(for instance when retrieving fulltext from ISTEX API)
"""
# /!\ attention le password est en clair ici /!\
# print ("REGARD:", user, passw, file=stderr)
no_contents = False
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm='Authentification sur api.istex.fr',
uri='https://api.istex.fr',
user=user,
passwd=passw)
install_opener(build_opener(auth_handler))
print("GET bin (user:%s)" % user, file=stderr)
# contact
try:
remote_file = urlopen(my_url)
except URLError as url_e:
if url_e.getcode() == 401:
raise AuthWarning("need_auth")
else:
# 404 à gérer *sans quitter* pour les fulltexts en nombre...
no_contents = True
print("api: HTTP ERR no %i (%s) sur '%s'" %
(url_e.getcode(), url_e.msg, my_url),
file=stderr)
# pour + de détail
# print ("ERR.info(): \n %s" % url_e.info(),file=stderr)
if no_contents:
return None
else:
# lecture
contents = remote_file.read()
remote_file.close()
return contents
def _bget(my_url, user=None, passw=None):
"""
Get remote auth-protected url *that contains a ~file~*
and pass its binary data straight from remote response
(for instance when retrieving fulltext from ISTEX API)
"""
# /!\ attention le password est en clair ici /!\
# print ("REGARD:", user, passw, file=stderr)
no_contents = False
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm = 'Authentification sur api.istex.fr',
uri = 'https://api.istex.fr',
user = user,
passwd = passw)
install_opener(build_opener(auth_handler))
print("GET bin (user:%s)" % user, file=stderr)
# contact
try:
remote_file = urlopen(my_url)
except URLError as url_e:
if url_e.getcode() == 401:
raise AuthWarning("need_auth")
else:
# 404 à gérer *sans quitter* pour les fulltexts en nombre...
no_contents = True
print("api: HTTP ERR no %i (%s) sur '%s'" %
(url_e.getcode(),url_e.msg, my_url), file=stderr)
# pour + de détail
# print ("ERR.info(): \n %s" % url_e.info(),file=stderr)
if no_contents:
return None
else:
# lecture
contents = remote_file.read()
remote_file.close()
return contents
def download_archive(username: str, password: str) -> str:
url = "http://brg-files.ai.sri.com/public/dist/meta.tar.gz"
try:
res = urlopen(url)
except HTTPError as err:
res = err
if res.getcode() == 401:
"""
Retry with basic authentication
Find realm in headers:
e.g. Www-authenticate: Basic realm="SRI BRG Restricted access"
"""
header = res.info()["WWW-Authenticate"]
realm = re.match("Basic realm=[\"'](.+?)[\"']", header).group(1)
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=url,
user=username,
passwd=password)
opener = build_opener(auth_handler)
try:
res = opener.open(url)
except HTTPError as err:
res = err
if res.getcode() != 200:
raise HTTPError(res.geturl(), res.getcode(), '', res.info(), None)
fd, filepath = mkstemp()
os.close(fd)
with open(filepath, "wb") as fh:
for chunk in res:
fh.write(chunk)
return filepath
def load_auth(cookie_file_path):
"""Load username and password from .gitcookies and return an
HTTPBasicAuthHandler."""
auth_handler = HTTPBasicAuthHandler()
with open(cookie_file_path, 'r') as cookie_file:
for lineno, line in enumerate(cookie_file, start=1):
if line.startswith('#HttpOnly_'):
line = line[len('#HttpOnly_'):]
if not line or line[0] == '#':
continue
row = line.split('\t')
if len(row) != 7:
continue
domain = row[0]
cookie = row[6]
sep = cookie.find('=')
if sep == -1:
continue
username = cookie[0:sep]
password = cookie[sep + 1:]
auth_handler.add_password(domain, domain, username, password)
return auth_handler
def load_auth(cookie_file_path):
"""Load username and password from .gitcookies and return an
HTTPBasicAuthHandler."""
auth_handler = HTTPBasicAuthHandler()
with open(cookie_file_path, 'r') as cookie_file:
for lineno, line in enumerate(cookie_file, start=1):
if line.startswith('#HttpOnly_'):
line = line[len('#HttpOnly_'):]
if not line or line[0] == '#':
continue
row = line.split('\t')
if len(row) != 7:
continue
domain = row[0]
cookie = row[6]
sep = cookie.find('=')
if sep == -1:
continue
username = cookie[0:sep]
password = cookie[sep + 1:]
auth_handler.add_password(domain, domain, username, password)
return auth_handler
def update(self):
if not self.config['url']:
return None
auth = HTTPBasicAuthHandler()
if self.config['realm'] and self.config['user'] \
and self.config['password']:
auth.add_password(self.config['realm'], self.config['url'],
self.config['user'], self.config['password'])
opener = build_opener(auth)
try:
resp = opener.open(self.config['url'])
except HTTPError:
return False
p = feedparser.parse(resp)
if not p.entries:
return False
if self.config['feed_name']:
author = self.config['feed_name']
else:
author = p.feed.title
for entry in p.entries[::-1]:
if 'link' not in entry or 'title' not in entry \
or self.manager.state.check(self.id, entry.link):
continue
link = entry.link
id = entry.link
title = entry.title
body = None
if 'content' in entry and entry.content:
if entry.content[0].type == 'text/html':
body = recent.markup.html_to_log(entry.content[0].value)
else:
body = entry.content[0].value
self.manager.state.add(self.id, entry.link)
n = Notification(provider=self.name, id=id, title=title,
author=author, link=link, body=body)
self.manager.notify(n)
def handler_version(url):
hdlr = HTTPBasicAuthHandler()
hdlr.add_password(REALM, urlparse(url)[1], LOGIN, PASSWD)
opener = build_opener(hdlr)
install_opener(opener)
return url
def add_auth(self, x, url, username, password):
handler = HTTPBasicAuthHandler()
handler.add_password(x, url, username, password)
self.auth_handler = handler
def build_opener(apiurl,
user,
password,
cookie_path,
debuglevel=0,
capath=None,
cafile=None,
headers=()):
"""build urllib opener for given name/password
it creates
* HTTPSHandler with proper ssl context
* HTTPCookieProcessor with a link to cookiejar
* HTTPBasicAuthHandler with user/password
* proxyhandler which respects no_proxy variable
"""
handlers = list()
if hasattr(ssl, "SSLContext"):
#allow only sslv3 and tlsv1, but not sslv2
ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_SSLv23)
ctx.options |= ssl.OP_NO_SSLv2
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.set_default_verify_paths()
if cafile or capath:
if ctx.load_verify_locations(capath=capath, cafile=cafile) != -1:
raise Exception(
"load_verify_locations failed for capath={}, cafile={}".
format(capath, cafile))
#TODO: debuglevel
httpshandler = HTTPSHandler(debuglevel=debuglevel,
context=ctx,
check_hostname=True)
handlers.append(httpshandler)
try:
# TODO is this correct?
cookie_file = os.path.expanduser(cookie_path)
cookiejar = LWPCookieJar(cookie_file)
cookiejar.load(ignore_discard=True)
except (OSError, IOError, AttributeError):
try:
os.open(cookie_file, os.O_WRONLY | os.O_CREAT, mode=0o600)
except:
#TODO: log it
cookiejar = CookieJar()
handlers.append(HTTPCookieProcessor(cookiejar))
authhandler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
authhandler.add_password(None, apiurl, bytes(user, "utf-8"),
bytes(password, "ascii"))
handlers.append(authhandler)
# proxy handling
if not proxy_bypass(apiurl):
proxyhandler = ProxyHandler()
else:
proxyhandler = ProxyHandler({})
handlers.append(proxyhandler)
opener = _build_opener(*handlers)
from bslib import __version__
opener.addheaders = [
("User-agent", "bslib/{}".format(__version__)),
]
for h in headers:
opener.addheaders(h)
return opener
from urllib.request import HTTPBasicAuthHandler, build_opener
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm='ksh',
user='******',
passwd='shkimadmin',
uri='http://127.0.0.1:8000/auth/')
opener = build_opener(auth_handler)
resp = opener.open('http://127.0.0.1:8000/auth/')
print(resp.read().decode('utf-8'))
from urllib.request import HTTPBasicAuthHandler, build_opener
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="ksh", user="******", passwd="shkimadmin", uri="http://127.0.0.1:8000/auth/"
) # OK
# NOK. auth_handler.add_password(realm='ksh', user='******', passwd='shkimadmin', uri='http://127.0.0.1:8000/')
opener = build_opener(auth_handler)
resp = opener.open("http://127.0.0.1:8000/auth/")
print(resp.read().decode("utf-8"))
def open_url(url, config, data=None, handlers=None):
"""Attempts to open a connection to a specified URL.
@param url: URL to attempt to open
@param config: SSL context configuration
@type config: Configuration
@param data: HTTP POST data
@type data: str
@param handlers: list of custom urllib2 handlers to add to the request
@type handlers: iterable
@return: tuple (
returned HTTP status code or 0 if an error occurred
returned message or error description
response object)
"""
debuglevel = 1 if config.debug else 0
# Set up handlers for URL opener.
if config.cookie:
cj = config.cookie
else:
cj = cookiejar_.CookieJar()
# Use a cookie processor that accumulates cookies when redirects occur so
# that an application can redirect for authentication and retain both any
# cookies for the application and the security system (c.f.,
# urllib2.HTTPCookieProcessor which replaces cookies).
cookie_handler = AccumulatingHTTPCookieProcessor(cj)
if not handlers:
handlers = []
handlers.append(cookie_handler)
if config.debug:
http_handler = HTTPHandler_(debuglevel=debuglevel)
https_handler = HTTPSContextHandler(config.ssl_context,
debuglevel=debuglevel)
handlers.extend([http_handler, https_handler])
if config.http_basicauth:
# currently only supports http basic auth
auth_handler = HTTPBasicAuthHandler_(HTTPPasswordMgrWithDefaultRealm_())
auth_handler.add_password(realm=None, uri=url,
user=config.httpauth[0],
passwd=config.httpauth[1])
handlers.append(auth_handler)
# Explicitly remove proxy handling if the host is one listed in the value of
# the no_proxy environment variable because urllib2 does use proxy settings
# set via http_proxy and https_proxy, but does not take the no_proxy value
# into account.
if not _should_use_proxy(url, config.no_proxy):
handlers.append(ProxyHandler_({}))
log.debug("Not using proxy")
elif config.proxies:
handlers.append(ProxyHandler_(config.proxies))
log.debug("Configuring proxies: %s" % config.proxies)
opener = build_opener(*handlers, ssl_context=config.ssl_context)
headers = config.headers
if headers is None:
headers = {}
request = Request_(url, data, headers)
# Open the URL and check the response.
return_code = 0
return_message = ''
response = None
# FIXME
response = opener.open(request)
try:
response = opener.open(request)
return_message = response.msg
return_code = response.code
if log.isEnabledFor(logging.DEBUG):
for index, cookie in enumerate(cj):
log.debug("%s : %s", index, cookie)
except HTTPError_ as exc:
return_code = exc.code
return_message = "Error: %s" % exc.msg
if log.isEnabledFor(logging.DEBUG):
log.debug("%s %s", exc.code, exc.msg)
except Exception as exc:
return_message = "Error: %s" % exc.__str__()
if log.isEnabledFor(logging.DEBUG):
import traceback
log.debug(traceback.format_exc())
return (return_code, return_message, response)
def open_url(url, config, data=None, handlers=None):
"""Attempts to open a connection to a specified URL.
@param url: URL to attempt to open
@param config: SSL context configuration
@type config: Configuration
@param data: HTTP POST data
@type data: str
@param handlers: list of custom urllib2 handlers to add to the request
@type handlers: iterable
@return: tuple (
returned HTTP status code or 0 if an error occurred
returned message or error description
response object)
"""
debuglevel = 1 if config.debug else 0
# Set up handlers for URL opener.
if config.cookie:
cj = config.cookie
else:
cj = cookiejar_.CookieJar()
# Use a cookie processor that accumulates cookies when redirects occur so
# that an application can redirect for authentication and retain both any
# cookies for the application and the security system (c.f.,
# urllib2.HTTPCookieProcessor which replaces cookies).
cookie_handler = AccumulatingHTTPCookieProcessor(cj)
if not handlers:
handlers = []
handlers.append(cookie_handler)
if config.debug:
http_handler = HTTPHandler_(debuglevel=debuglevel)
https_handler = HTTPSContextHandler(config.ssl_context,
debuglevel=debuglevel)
handlers.extend([http_handler, https_handler])
if config.http_basicauth:
# currently only supports http basic auth
auth_handler = HTTPBasicAuthHandler_(HTTPPasswordMgrWithDefaultRealm_())
auth_handler.add_password(realm=None, uri=url,
user=config.http_basicauth[0],
passwd=config.http_basicauth[1])
handlers.append(auth_handler)
# Explicitly remove proxy handling if the host is one listed in the value of
# the no_proxy environment variable because urllib2 does use proxy settings
# set via http_proxy and https_proxy, but does not take the no_proxy value
# into account.
if not _should_use_proxy(url, config.no_proxy):
handlers.append(ProxyHandler_({}))
log.debug("Not using proxy")
elif config.proxies:
handlers.append(ProxyHandler_(config.proxies))
log.debug("Configuring proxies: %s" % config.proxies)
opener = build_opener(*handlers, ssl_context=config.ssl_context)
headers = config.headers
if headers is None:
headers = {}
request = Request_(url, data, headers)
# Open the URL and check the response.
return_code = 0
return_message = ''
response = None
try:
response = opener.open(request)
return_message = response.msg
return_code = response.code
if log.isEnabledFor(logging.DEBUG):
for index, cookie in enumerate(cj):
log.debug("%s : %s", index, cookie)
except HTTPError_ as exc:
return_code = exc.code
return_message = "Error: %s" % exc.msg
if log.isEnabledFor(logging.DEBUG):
log.debug("%s %s", exc.code, exc.msg)
except Exception as exc:
return_message = "Error: %s" % exc.__str__()
if log.isEnabledFor(logging.DEBUG):
import traceback
log.debug(traceback.format_exc())
return (return_code, return_message, response)
def urlmon(check):
url = check['url']
post = check.get('post', None)
count = check.get('count', 1)
headers = check.get('headers', {})
username = check.get('username', None)
password = check.get('password', None)
realm = check.get('realm', None)
uri = check.get('uri', None)
proxy = check.get('proxy', False)
status = 0
reason = None
body = None
rtt = 0
while True:
count -= 1
start = time.time()
if username and password:
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=uri,
user=username,
passwd=password)
if proxy:
opener = build_opener(auth_handler, ProxyHandler(proxy))
else:
opener = build_opener(auth_handler)
else:
if proxy:
opener = build_opener(ProxyHandler(proxy))
else:
opener = build_opener()
install_opener(opener)
if 'User-agent' not in headers:
headers['User-agent'] = 'alert-urlmon/%s' % (__version__)
try:
if post:
req = Request(url, json.dumps(post), headers=headers)
else:
req = Request(url, headers=headers)
response = urlopen(req, None, MAX_TIMEOUT)
except ValueError as e:
LOG.error('Request failed: %s' % e)
except URLError as e:
if hasattr(e, 'reason'):
reason = str(e.reason)
status = None
elif hasattr(e, 'code'):
reason = None
status = e.code # pylint: disable=no-member
except Exception as e:
LOG.warning('Unexpected error: %s' % e)
else:
status = response.getcode()
body = response.read()
rtt = int((time.time() - start) * 1000) # round-trip time
if status: # return result if any HTTP/S response is received
break
if not count:
break
time.sleep(10)
return status, reason, body, rtt
