def get_opener(self, url, username, password):
"""Return an opener with an installed password manager.
Gerrit's API requires HTTP Digest Auth for its API endpoints. Unlike
HTTP Basic Auth, this method requires a failed request and response to
build the digest header, so we cannot pre-compute the header and send
it along with the initial request.
Args:
url (unicode):
The only part of this required of the URL is the FQDN and the
URL scheme. The location and fragment portions will be ignored.
username (unicode):
The username to use for authentication.
password (unicode):
The password to use for authentication.
Returns:
urllib2.OpenerDirector:
The opener with the password manager installed.
"""
assert username is not None
assert password is not None
result = urlparse(url)
top_level_url = '%s://%s' % (result.scheme, result.netloc)
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, top_level_url, username, password)
return build_opener(HTTPDigestAuthHandler(password_mgr))
def get_opener(self, url, username, password):
"""Return an opener with an installed password manager.
Gerrit's API requires HTTP Digest Auth for its API endpoints. Unlike
HTTP Basic Auth, this method requires a failed request and response to
build the digest header, so we cannot pre-compute the header and send
it along with the initial request.
Args:
url (unicode):
The only part of this required of the URL is the FQDN and the
URL scheme. The location and fragment portions will be ignored.
username (unicode):
The username to use for authentication.
password (unicode):
The password to use for authentication.
Returns:
urllib2.OpenerDirector:
The opener with the password manager installed.
"""
assert username is not None
assert password is not None
result = urlparse(url)
top_level_url = '%s://%s' % (result.scheme, result.netloc)
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, top_level_url, username, password)
return build_opener(HTTPDigestAuthHandler(password_mgr))
def login(self):
login_url = '%sindex/login/' % self.base_url
opener = build_opener(HTTPCookieProcessor())
data = urlencode({
'username': self.username,
'password': self.password
})
opener.open(login_url, data.encode('utf-8'), TIMEOUT)
install_opener(opener)
import json
from django.utils.six.moves.urllib.parse import urlencode
from django.utils.six.moves.urllib.error import URLError
from django.utils.six.moves.urllib.request import (
ProxyHandler,
Request,
urlopen,
build_opener,
install_opener
)
# A custom ProxyHandler that will not auto-detect proxy settings
proxy_support = ProxyHandler({})
opener = build_opener(proxy_support)
install_opener(opener)
class DisqusException(Exception):
"""Exception raised for errors with the DISQUS API."""
pass
class DisqusClient(object):
"""
Client for the DISQUS API.
Example:
>>> client = DisqusClient()
>>> json = client.get_forum_list(user_api_key=DISQUS_API_KEY)
"""
import json
from django.utils.six.moves.urllib.parse import urlencode
from django.utils.six.moves.urllib.error import URLError
from django.utils.six.moves.urllib.request import (ProxyHandler, Request,
urlopen, build_opener,
install_opener)
# A custom ProxyHandler that will not auto-detect proxy settings
proxy_support = ProxyHandler({})
opener = build_opener(proxy_support)
install_opener(opener)
class DisqusException(Exception):
"""Exception raised for errors with the DISQUS API."""
pass
class DisqusClient(object):
"""
Client for the DISQUS API.
Example:
>>> client = DisqusClient()
>>> json = client.get_forum_list(user_api_key=DISQUS_API_KEY)
"""
METHODS = {
'create_post': 'POST',
'get_forum_api_key': 'GET',
'get_forum_list': 'GET',
def _check_external(self, tested_url, external_recheck_interval):
logger.info('checking external link: %s' % tested_url)
external_recheck_datetime = now() - timedelta(minutes=external_recheck_interval)
if self.last_checked and (self.last_checked > external_recheck_datetime):
return self.status
opener = build_opener(RedirectHandler)
# Remove URL fragment identifiers
url = tested_url.rsplit('#')[0]
# Check that non-ascii chars are properly encoded
try:
url.encode('ascii')
except UnicodeEncodeError:
url = iri_to_uri(url)
try:
if tested_url.count('#'):
# We have to get the content so we can check the anchors
response = opener.open(
url,
timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT
)
else:
# Might as well just do a HEAD request
req = HeadRequest(url, headers={'User-Agent' : "http://%s Linkchecker" % settings.SITE_DOMAIN})
try:
response = opener.open(
req,
timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT
)
except (ValueError, HTTPError) as error:
# ...except sometimes it triggers a bug in urllib2
if hasattr(error, 'code') and error.code == METHOD_NOT_ALLOWED:
req = GetRequest(url, headers={'User-Agent' : "http://%s Linkchecker" % settings.SITE_DOMAIN})
else:
req = url
response = opener.open(
req,
timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT
)
self.message = ' '.join([str(response.code), response.msg])
self.status = True
if tested_url.count('#'):
anchor = tested_url.split('#')[1]
from linkcheck import parse_anchors
try:
names = parse_anchors(response.read())
if anchor in names:
self.message = 'Working external hash anchor'
self.status = True
else:
self.message = 'Broken external hash anchor'
self.status = False
except:
# The external web page is mal-formatted #or maybe other parse errors like encoding
# I reckon a broken anchor on an otherwise good URL should count as a pass
self.message = "Page OK but anchor can't be checked"
self.status = True
except http_client.BadStatusLine:
self.message = "Bad Status Line"
except HTTPError as e:
if hasattr(e, 'code') and hasattr(e, 'msg'):
self.message = ' '.join([str(e.code), e.msg])
else:
self.message = "Unknown Error"
except URLError as e:
if hasattr(e, 'reason'):
self.message = 'Unreachable: '+str(e.reason)
elif hasattr(e, 'code') and e.code!=301:
self.message = 'Error: '+str(e.code)
else:
self.message = 'Redirect. Check manually: '+str(e.code)
except Exception as e:
self.message = 'Other Error: %s' % e
else:
if response.getcode() == 301 and response.geturl() != url:
self.redirect_to = response.geturl()
elif self.redirect_to:
self.redirect_to = ''
self.last_checked = now()
self.save()
def dispatch_webhook_event(request, webhook_targets, event, payload):
"""Dispatch the given event and payload to the given WebHook targets."""
encoder = ResourceAPIEncoder()
bodies = {}
for webhook_target in webhook_targets:
if webhook_target.use_custom_content:
try:
body = render_custom_content(webhook_target.custom_content,
payload)
body = body.encode('utf-8')
except Exception as e:
logging.exception('Could not render WebHook payload: %s', e)
continue
else:
encoding = webhook_target.encoding
if encoding not in bodies:
try:
if encoding == webhook_target.ENCODING_JSON:
adapter = JSONEncoderAdapter(encoder)
body = adapter.encode(payload, request=request)
body = body.encode('utf-8')
elif encoding == webhook_target.ENCODING_XML:
adapter = XMLEncoderAdapter(encoder)
body = adapter.encode(payload, request=request)
elif encoding == webhook_target.ENCODING_FORM_DATA:
adapter = JSONEncoderAdapter(encoder)
body = urlencode({
'payload':
adapter.encode(payload, request=request),
})
body = body.encode('utf-8')
else:
logging.error(
'Unexpected WebHookTarget encoding "%s" '
'for ID %s', encoding, webhook_target.pk)
continue
except Exception as e:
logging.exception('Could not encode WebHook payload: %s',
e)
continue
bodies[encoding] = body
else:
body = bodies[encoding]
headers = {
b'X-ReviewBoard-Event':
event.encode('utf-8'),
b'Content-Type':
webhook_target.encoding.encode('utf-8'),
b'Content-Length':
len(body),
b'User-Agent':
('ReviewBoard-WebHook/%s' % get_package_version()).encode('utf-8'),
}
if webhook_target.secret:
signer = hmac.new(webhook_target.secret.encode('utf-8'), body,
hashlib.sha1)
headers[b'X-Hub-Signature'] = \
('sha1=%s' % signer.hexdigest()).encode('utf-8')
logging.info('Dispatching webhook for event %s to %s', event,
webhook_target.url)
try:
url = webhook_target.url
url_parts = urlsplit(url)
if url_parts.username or url_parts.password:
netloc = url_parts.netloc.split('@', 1)[1]
url = urlunsplit((url_parts.scheme, netloc, url_parts.path,
url_parts.params, url_parts.query))
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, url_parts.username,
url_parts.password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
else:
opener = build_opener()
opener.open(Request(url.encode('utf-8'), body, headers))
except Exception as e:
logging.exception('Could not dispatch WebHook to %s: %s',
webhook_target.url, e)
def dispatch_webhook_event(request, webhook_targets, event, payload):
"""Dispatch the given event and payload to the given WebHook targets.
Args:
request (django.http.HttpRequest):
The HTTP request from the client.
webhook_targets (list of
reviewboard.notifications.models.WebHookTarget):
The list of WebHook targets containing endpoint URLs to dispatch
to.
event (unicode):
The name of the event being dispatched.
payload (dict):
The payload data to encode for the WebHook payload.
Raises:
ValueError:
There was an error with the payload format. Details are in the
log and the exception message.
"""
try:
payload = normalize_webhook_payload(payload, request)
except TypeError as e:
logging.exception('WebHook payload passed to dispatch_webhook_event '
'containing invalid data types: %s',
e)
raise ValueError(six.text_type(e))
encoder = BasicAPIEncoder()
bodies = {}
for webhook_target in webhook_targets:
if webhook_target.use_custom_content:
try:
body = render_custom_content(webhook_target.custom_content,
payload)
body = body.encode('utf-8')
except Exception as e:
logging.exception('Could not render WebHook payload: %s', e)
continue
else:
encoding = webhook_target.encoding
if encoding not in bodies:
try:
if encoding == webhook_target.ENCODING_JSON:
adapter = JSONEncoderAdapter(encoder)
body = adapter.encode(payload, request=request)
body = body.encode('utf-8')
elif encoding == webhook_target.ENCODING_XML:
adapter = XMLEncoderAdapter(encoder)
body = adapter.encode(payload, request=request)
elif encoding == webhook_target.ENCODING_FORM_DATA:
adapter = JSONEncoderAdapter(encoder)
body = urlencode({
'payload': adapter.encode(payload,
request=request),
})
body = body.encode('utf-8')
else:
logging.error('Unexpected WebHookTarget encoding "%s" '
'for ID %s',
encoding, webhook_target.pk)
continue
except Exception as e:
logging.exception('Could not encode WebHook payload: %s',
e)
continue
bodies[encoding] = body
else:
body = bodies[encoding]
headers = {
b'X-ReviewBoard-Event': event.encode('utf-8'),
b'Content-Type': webhook_target.encoding.encode('utf-8'),
b'Content-Length': len(body),
b'User-Agent':
('ReviewBoard-WebHook/%s' % get_package_version())
.encode('utf-8'),
}
if webhook_target.secret:
signer = hmac.new(webhook_target.secret.encode('utf-8'), body,
hashlib.sha1)
headers[b'X-Hub-Signature'] = \
('sha1=%s' % signer.hexdigest()).encode('utf-8')
logging.info('Dispatching webhook for event %s to %s',
event, webhook_target.url)
try:
url = webhook_target.url
url_parts = urlsplit(url)
if url_parts.username or url_parts.password:
netloc = url_parts.netloc.split('@', 1)[1]
url = urlunsplit(
(url_parts.scheme, netloc, url_parts.path,
url_parts.params, url_parts.query))
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None, url, url_parts.username, url_parts.password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
else:
opener = build_opener()
opener.open(Request(url.encode('utf-8'), body, headers))
except Exception as e:
logging.exception('Could not dispatch WebHook to %s: %s',
webhook_target.url, e)
def dispatch_webhook_event(request, webhook_targets, event, payload):
"""Dispatch the given event and payload to the given WebHook targets.
Args:
request (django.http.HttpRequest):
The HTTP request from the client.
webhook_targets (list of
reviewboard.notifications.models.WebHookTarget):
The list of WebHook targets containing endpoint URLs to dispatch
to.
event (unicode):
The name of the event being dispatched.
payload (dict):
The payload data to encode for the WebHook payload.
Raises:
ValueError:
There was an error with the payload format. Details are in the
log and the exception message.
"""
try:
payload = normalize_webhook_payload(payload, request)
except TypeError as e:
logging.exception('WebHook payload passed to dispatch_webhook_event '
'containing invalid data types: %s',
e)
raise ValueError(six.text_type(e))
encoder = BasicAPIEncoder()
bodies = {}
for webhook_target in webhook_targets:
if webhook_target.use_custom_content:
try:
body = render_custom_content(webhook_target.custom_content,
payload)
body = body.encode('utf-8')
except Exception as e:
logging.exception('Could not render WebHook payload: %s', e)
continue
else:
encoding = webhook_target.encoding
if encoding not in bodies:
try:
if encoding == webhook_target.ENCODING_JSON:
adapter = JSONEncoderAdapter(encoder)
body = adapter.encode(payload, request=request)
body = body.encode('utf-8')
elif encoding == webhook_target.ENCODING_XML:
adapter = XMLEncoderAdapter(encoder)
body = adapter.encode(payload, request=request)
elif encoding == webhook_target.ENCODING_FORM_DATA:
adapter = JSONEncoderAdapter(encoder)
body = urlencode({
'payload': adapter.encode(payload,
request=request),
})
body = body.encode('utf-8')
else:
logging.error('Unexpected WebHookTarget encoding "%s" '
'for ID %s',
encoding, webhook_target.pk)
continue
except Exception as e:
logging.exception('Could not encode WebHook payload: %s',
e)
continue
bodies[encoding] = body
else:
body = bodies[encoding]
headers = {
b'X-ReviewBoard-Event': event.encode('utf-8'),
b'Content-Type': webhook_target.encoding.encode('utf-8'),
b'Content-Length': len(body),
b'User-Agent':
('ReviewBoard-WebHook/%s' % get_package_version())
.encode('utf-8'),
}
if webhook_target.secret:
signer = hmac.new(webhook_target.secret.encode('utf-8'), body,
hashlib.sha1)
headers[b'X-Hub-Signature'] = \
('sha1=%s' % signer.hexdigest()).encode('utf-8')
logging.info('Dispatching webhook for event %s to %s',
event, webhook_target.url)
try:
url = webhook_target.url
url_parts = urlsplit(url)
if url_parts.username or url_parts.password:
netloc = url_parts.netloc.split('@', 1)[1]
url = urlunsplit(
(url_parts.scheme, netloc, url_parts.path,
url_parts.params, url_parts.query))
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None, url, url_parts.username, url_parts.password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
else:
opener = build_opener()
opener.open(Request(url.encode('utf-8'), body, headers))
except Exception as e:
logging.exception('Could not dispatch WebHook to %s: %s',
webhook_target.url, e)
def _check_external(self, tested_url, external_recheck_interval):
logger.info('checking external link: %s' % tested_url)
external_recheck_datetime = now() - timedelta(
minutes=external_recheck_interval)
if self.last_checked and (self.last_checked >
external_recheck_datetime):
return self.status
opener = build_opener(RedirectHandler)
# Remove URL fragment identifiers
url = tested_url.rsplit('#')[0]
# Check that non-ascii chars are properly encoded
try:
url.encode('ascii')
except UnicodeEncodeError:
url = iri_to_uri(url)
try:
if tested_url.count('#'):
# We have to get the content so we can check the anchors
response = opener.open(
url, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT)
else:
# Might as well just do a HEAD request
req = HeadRequest(url,
headers={
'User-Agent':
"http://%s Linkchecker" %
settings.SITE_DOMAIN
})
try:
response = opener.open(
req, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT)
except URLError as e:
# When we get CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579) error
# we try the link using requests, and ignore SSL verification error.
if hasattr(
e,
'reason') and 'certificate verify failed' in str(
e.reason):
response = requests.head(
url,
verify=False,
timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT)
response.code = response.status_code
response.msg = ''
else:
raise
except (ValueError, HTTPError) as error:
# ...except sometimes it triggers a bug in urllib2
if hasattr(error,
'code') and error.code == METHOD_NOT_ALLOWED:
req = GetRequest(url,
headers={
'User-Agent':
"http://%s Linkchecker" %
settings.SITE_DOMAIN
})
else:
req = url
response = opener.open(
req, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT)
self.message = ' '.join([str(response.code), response.msg])
self.status = True
if tested_url.count('#'):
anchor = tested_url.split('#')[1]
from linkcheck import parse_anchors
try:
names = parse_anchors(response.read())
if anchor in names:
self.message = 'Working external hash anchor'
self.status = True
else:
self.message = 'Broken external hash anchor'
self.status = False
except:
# The external web page is mal-formatted #or maybe other parse errors like encoding
# I reckon a broken anchor on an otherwise good URL should count as a pass
self.message = "Page OK but anchor can't be checked"
self.status = True
except http_client.BadStatusLine:
self.message = "Bad Status Line"
except HTTPError as e:
if hasattr(e, 'code') and hasattr(e, 'msg'):
self.message = ' '.join([str(e.code), e.msg])
else:
self.message = "Unknown Error"
except URLError as e:
if hasattr(e, 'reason'):
self.message = 'Unreachable: ' + str(e.reason)
elif hasattr(e, 'code') and e.code != 301:
self.message = 'Error: ' + str(e.code)
else:
self.message = 'Redirect. Check manually: ' + str(e.code)
except Exception as e:
self.message = 'Other Error: %s' % e
else:
if getattr(response, 'getcode', False) and response.getcode(
) == 301 and response.geturl() != url:
self.redirect_to = response.geturl()
elif self.redirect_to:
self.redirect_to = ''
self.last_checked = now()
self.save()
def dispatch_webhook_event(request, webhook_targets, event, payload):
"""Dispatch the given event and payload to the given WebHook targets.
Args:
request (django.http.HttpRequest):
The HTTP request from the client.
webhook_targets (list of
reviewboard.notifications.models.WebHookTarget):
The list of WebHook targets containing endpoint URLs to dispatch
to.
event (unicode):
The name of the event being dispatched.
payload (dict):
The payload data to encode for the WebHook payload.
Raises:
ValueError:
There was an error with the payload format. Details are in the
log and the exception message.
"""
encoder = BasicAPIEncoder()
bodies = {}
raw_norm_payload = None
json_norm_payload = None
for webhook_target in webhook_targets:
use_custom_content = webhook_target.use_custom_content
encoding = webhook_target.encoding
# See how we need to handle normalizing this payload. If we need
# something JSON-safe, then we need to go the more aggressive route
# and normalize keys to strings.
if raw_norm_payload is None or json_norm_payload is None:
try:
if (raw_norm_payload is None
and (use_custom_content
or encoding == webhook_target.ENCODING_XML)):
# This payload's going to be provided for XML and custom
# templates. We don't want to alter the keys at all.
raw_norm_payload = normalize_webhook_payload(
payload=payload, request=request)
elif (json_norm_payload is None and not use_custom_content
and encoding in (webhook_target.ENCODING_JSON,
webhook_target.ENCODING_FORM_DATA)):
# This payload's going to be provided for JSON or
# form-data. We want to normalize all keys to strings.
json_norm_payload = normalize_webhook_payload(
payload=payload, request=request, use_string_keys=True)
except TypeError as e:
logging.exception(
'WebHook payload passed to '
'dispatch_webhook_event containing invalid '
'data types: %s', e)
raise ValueError(six.text_type(e))
if use_custom_content:
try:
assert raw_norm_payload is not None
body = render_custom_content(webhook_target.custom_content,
raw_norm_payload)
body = force_bytes(body)
except Exception as e:
logging.exception('Could not render WebHook payload: %s', e)
continue
else:
if encoding not in bodies:
try:
if encoding == webhook_target.ENCODING_JSON:
assert json_norm_payload is not None
adapter = JSONEncoderAdapter(encoder)
body = adapter.encode(json_norm_payload,
request=request)
elif encoding == webhook_target.ENCODING_XML:
assert raw_norm_payload is not None
adapter = XMLEncoderAdapter(encoder)
body = adapter.encode(raw_norm_payload,
request=request)
elif encoding == webhook_target.ENCODING_FORM_DATA:
assert json_norm_payload is not None
adapter = JSONEncoderAdapter(encoder)
body = urlencode({
'payload':
adapter.encode(json_norm_payload, request=request),
})
else:
logging.error(
'Unexpected WebHookTarget encoding "%s" '
'for ID %s', encoding, webhook_target.pk)
continue
except Exception as e:
logging.exception('Could not encode WebHook payload: %s',
e)
continue
body = force_bytes(body)
bodies[encoding] = body
else:
body = bodies[encoding]
headers = {
b'X-ReviewBoard-Event':
event.encode('utf-8'),
b'Content-Type':
webhook_target.encoding.encode('utf-8'),
b'Content-Length':
len(body),
b'User-Agent':
('ReviewBoard-WebHook/%s' % get_package_version()).encode('utf-8'),
}
if webhook_target.secret:
signer = hmac.new(webhook_target.secret.encode('utf-8'), body,
hashlib.sha1)
headers[b'X-Hub-Signature'] = \
('sha1=%s' % signer.hexdigest()).encode('utf-8')
logging.info('Dispatching webhook for event %s to %s', event,
webhook_target.url)
try:
url = webhook_target.url
url_parts = urlsplit(url)
if url_parts.username or url_parts.password:
netloc = url_parts.netloc.split('@', 1)[1]
url = urlunsplit((url_parts.scheme, netloc, url_parts.path,
url_parts.params, url_parts.query))
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, url_parts.username,
url_parts.password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
else:
opener = build_opener()
opener.open(Request(url, body, headers))
except Exception as e:
logging.exception('Could not dispatch WebHook to %s: %s',
webhook_target.url, e)
