def get_opener(self, url, username, password): """Return an opener with an installed password manager. Gerrit's API requires HTTP Digest Auth for its API endpoints. Unlike HTTP Basic Auth, this method requires a failed request and response to build the digest header, so we cannot pre-compute the header and send it along with the initial request. Args: url (unicode): The only part of this required of the URL is the FQDN and the URL scheme. The location and fragment portions will be ignored. username (unicode): The username to use for authentication. password (unicode): The password to use for authentication. Returns: urllib2.OpenerDirector: The opener with the password manager installed. """ assert username is not None assert password is not None result = urlparse(url) top_level_url = '%s://%s' % (result.scheme, result.netloc) password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, top_level_url, username, password) return build_opener(HTTPDigestAuthHandler(password_mgr))
def login(self): login_url = '%sindex/login/' % self.base_url opener = build_opener(HTTPCookieProcessor()) data = urlencode({ 'username': self.username, 'password': self.password }) opener.open(login_url, data.encode('utf-8'), TIMEOUT) install_opener(opener)
import json from django.utils.six.moves.urllib.parse import urlencode from django.utils.six.moves.urllib.error import URLError from django.utils.six.moves.urllib.request import ( ProxyHandler, Request, urlopen, build_opener, install_opener ) # A custom ProxyHandler that will not auto-detect proxy settings proxy_support = ProxyHandler({}) opener = build_opener(proxy_support) install_opener(opener) class DisqusException(Exception): """Exception raised for errors with the DISQUS API.""" pass class DisqusClient(object): """ Client for the DISQUS API. Example: >>> client = DisqusClient() >>> json = client.get_forum_list(user_api_key=DISQUS_API_KEY) """
import json from django.utils.six.moves.urllib.parse import urlencode from django.utils.six.moves.urllib.error import URLError from django.utils.six.moves.urllib.request import (ProxyHandler, Request, urlopen, build_opener, install_opener) # A custom ProxyHandler that will not auto-detect proxy settings proxy_support = ProxyHandler({}) opener = build_opener(proxy_support) install_opener(opener) class DisqusException(Exception): """Exception raised for errors with the DISQUS API.""" pass class DisqusClient(object): """ Client for the DISQUS API. Example: >>> client = DisqusClient() >>> json = client.get_forum_list(user_api_key=DISQUS_API_KEY) """ METHODS = { 'create_post': 'POST', 'get_forum_api_key': 'GET', 'get_forum_list': 'GET',
def _check_external(self, tested_url, external_recheck_interval): logger.info('checking external link: %s' % tested_url) external_recheck_datetime = now() - timedelta(minutes=external_recheck_interval) if self.last_checked and (self.last_checked > external_recheck_datetime): return self.status opener = build_opener(RedirectHandler) # Remove URL fragment identifiers url = tested_url.rsplit('#')[0] # Check that non-ascii chars are properly encoded try: url.encode('ascii') except UnicodeEncodeError: url = iri_to_uri(url) try: if tested_url.count('#'): # We have to get the content so we can check the anchors response = opener.open( url, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT ) else: # Might as well just do a HEAD request req = HeadRequest(url, headers={'User-Agent' : "http://%s Linkchecker" % settings.SITE_DOMAIN}) try: response = opener.open( req, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT ) except (ValueError, HTTPError) as error: # ...except sometimes it triggers a bug in urllib2 if hasattr(error, 'code') and error.code == METHOD_NOT_ALLOWED: req = GetRequest(url, headers={'User-Agent' : "http://%s Linkchecker" % settings.SITE_DOMAIN}) else: req = url response = opener.open( req, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT ) self.message = ' '.join([str(response.code), response.msg]) self.status = True if tested_url.count('#'): anchor = tested_url.split('#')[1] from linkcheck import parse_anchors try: names = parse_anchors(response.read()) if anchor in names: self.message = 'Working external hash anchor' self.status = True else: self.message = 'Broken external hash anchor' self.status = False except: # The external web page is mal-formatted #or maybe other parse errors like encoding # I reckon a broken anchor on an otherwise good URL should count as a pass self.message = "Page OK but anchor can't be checked" self.status = True except http_client.BadStatusLine: self.message = "Bad Status Line" except HTTPError as e: if hasattr(e, 'code') and hasattr(e, 'msg'): self.message = ' '.join([str(e.code), e.msg]) else: self.message = "Unknown Error" except URLError as e: if hasattr(e, 'reason'): self.message = 'Unreachable: '+str(e.reason) elif hasattr(e, 'code') and e.code!=301: self.message = 'Error: '+str(e.code) else: self.message = 'Redirect. Check manually: '+str(e.code) except Exception as e: self.message = 'Other Error: %s' % e else: if response.getcode() == 301 and response.geturl() != url: self.redirect_to = response.geturl() elif self.redirect_to: self.redirect_to = '' self.last_checked = now() self.save()
def dispatch_webhook_event(request, webhook_targets, event, payload): """Dispatch the given event and payload to the given WebHook targets.""" encoder = ResourceAPIEncoder() bodies = {} for webhook_target in webhook_targets: if webhook_target.use_custom_content: try: body = render_custom_content(webhook_target.custom_content, payload) body = body.encode('utf-8') except Exception as e: logging.exception('Could not render WebHook payload: %s', e) continue else: encoding = webhook_target.encoding if encoding not in bodies: try: if encoding == webhook_target.ENCODING_JSON: adapter = JSONEncoderAdapter(encoder) body = adapter.encode(payload, request=request) body = body.encode('utf-8') elif encoding == webhook_target.ENCODING_XML: adapter = XMLEncoderAdapter(encoder) body = adapter.encode(payload, request=request) elif encoding == webhook_target.ENCODING_FORM_DATA: adapter = JSONEncoderAdapter(encoder) body = urlencode({ 'payload': adapter.encode(payload, request=request), }) body = body.encode('utf-8') else: logging.error( 'Unexpected WebHookTarget encoding "%s" ' 'for ID %s', encoding, webhook_target.pk) continue except Exception as e: logging.exception('Could not encode WebHook payload: %s', e) continue bodies[encoding] = body else: body = bodies[encoding] headers = { b'X-ReviewBoard-Event': event.encode('utf-8'), b'Content-Type': webhook_target.encoding.encode('utf-8'), b'Content-Length': len(body), b'User-Agent': ('ReviewBoard-WebHook/%s' % get_package_version()).encode('utf-8'), } if webhook_target.secret: signer = hmac.new(webhook_target.secret.encode('utf-8'), body, hashlib.sha1) headers[b'X-Hub-Signature'] = \ ('sha1=%s' % signer.hexdigest()).encode('utf-8') logging.info('Dispatching webhook for event %s to %s', event, webhook_target.url) try: url = webhook_target.url url_parts = urlsplit(url) if url_parts.username or url_parts.password: netloc = url_parts.netloc.split('@', 1)[1] url = urlunsplit((url_parts.scheme, netloc, url_parts.path, url_parts.params, url_parts.query)) password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, url, url_parts.username, url_parts.password) handler = HTTPBasicAuthHandler(password_mgr) opener = build_opener(handler) else: opener = build_opener() opener.open(Request(url.encode('utf-8'), body, headers)) except Exception as e: logging.exception('Could not dispatch WebHook to %s: %s', webhook_target.url, e)
def dispatch_webhook_event(request, webhook_targets, event, payload): """Dispatch the given event and payload to the given WebHook targets. Args: request (django.http.HttpRequest): The HTTP request from the client. webhook_targets (list of reviewboard.notifications.models.WebHookTarget): The list of WebHook targets containing endpoint URLs to dispatch to. event (unicode): The name of the event being dispatched. payload (dict): The payload data to encode for the WebHook payload. Raises: ValueError: There was an error with the payload format. Details are in the log and the exception message. """ try: payload = normalize_webhook_payload(payload, request) except TypeError as e: logging.exception('WebHook payload passed to dispatch_webhook_event ' 'containing invalid data types: %s', e) raise ValueError(six.text_type(e)) encoder = BasicAPIEncoder() bodies = {} for webhook_target in webhook_targets: if webhook_target.use_custom_content: try: body = render_custom_content(webhook_target.custom_content, payload) body = body.encode('utf-8') except Exception as e: logging.exception('Could not render WebHook payload: %s', e) continue else: encoding = webhook_target.encoding if encoding not in bodies: try: if encoding == webhook_target.ENCODING_JSON: adapter = JSONEncoderAdapter(encoder) body = adapter.encode(payload, request=request) body = body.encode('utf-8') elif encoding == webhook_target.ENCODING_XML: adapter = XMLEncoderAdapter(encoder) body = adapter.encode(payload, request=request) elif encoding == webhook_target.ENCODING_FORM_DATA: adapter = JSONEncoderAdapter(encoder) body = urlencode({ 'payload': adapter.encode(payload, request=request), }) body = body.encode('utf-8') else: logging.error('Unexpected WebHookTarget encoding "%s" ' 'for ID %s', encoding, webhook_target.pk) continue except Exception as e: logging.exception('Could not encode WebHook payload: %s', e) continue bodies[encoding] = body else: body = bodies[encoding] headers = { b'X-ReviewBoard-Event': event.encode('utf-8'), b'Content-Type': webhook_target.encoding.encode('utf-8'), b'Content-Length': len(body), b'User-Agent': ('ReviewBoard-WebHook/%s' % get_package_version()) .encode('utf-8'), } if webhook_target.secret: signer = hmac.new(webhook_target.secret.encode('utf-8'), body, hashlib.sha1) headers[b'X-Hub-Signature'] = \ ('sha1=%s' % signer.hexdigest()).encode('utf-8') logging.info('Dispatching webhook for event %s to %s', event, webhook_target.url) try: url = webhook_target.url url_parts = urlsplit(url) if url_parts.username or url_parts.password: netloc = url_parts.netloc.split('@', 1)[1] url = urlunsplit( (url_parts.scheme, netloc, url_parts.path, url_parts.params, url_parts.query)) password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password( None, url, url_parts.username, url_parts.password) handler = HTTPBasicAuthHandler(password_mgr) opener = build_opener(handler) else: opener = build_opener() opener.open(Request(url.encode('utf-8'), body, headers)) except Exception as e: logging.exception('Could not dispatch WebHook to %s: %s', webhook_target.url, e)
def _check_external(self, tested_url, external_recheck_interval): logger.info('checking external link: %s' % tested_url) external_recheck_datetime = now() - timedelta( minutes=external_recheck_interval) if self.last_checked and (self.last_checked > external_recheck_datetime): return self.status opener = build_opener(RedirectHandler) # Remove URL fragment identifiers url = tested_url.rsplit('#')[0] # Check that non-ascii chars are properly encoded try: url.encode('ascii') except UnicodeEncodeError: url = iri_to_uri(url) try: if tested_url.count('#'): # We have to get the content so we can check the anchors response = opener.open( url, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT) else: # Might as well just do a HEAD request req = HeadRequest(url, headers={ 'User-Agent': "http://%s Linkchecker" % settings.SITE_DOMAIN }) try: response = opener.open( req, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT) except URLError as e: # When we get CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579) error # we try the link using requests, and ignore SSL verification error. if hasattr( e, 'reason') and 'certificate verify failed' in str( e.reason): response = requests.head( url, verify=False, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT) response.code = response.status_code response.msg = '' else: raise except (ValueError, HTTPError) as error: # ...except sometimes it triggers a bug in urllib2 if hasattr(error, 'code') and error.code == METHOD_NOT_ALLOWED: req = GetRequest(url, headers={ 'User-Agent': "http://%s Linkchecker" % settings.SITE_DOMAIN }) else: req = url response = opener.open( req, timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT) self.message = ' '.join([str(response.code), response.msg]) self.status = True if tested_url.count('#'): anchor = tested_url.split('#')[1] from linkcheck import parse_anchors try: names = parse_anchors(response.read()) if anchor in names: self.message = 'Working external hash anchor' self.status = True else: self.message = 'Broken external hash anchor' self.status = False except: # The external web page is mal-formatted #or maybe other parse errors like encoding # I reckon a broken anchor on an otherwise good URL should count as a pass self.message = "Page OK but anchor can't be checked" self.status = True except http_client.BadStatusLine: self.message = "Bad Status Line" except HTTPError as e: if hasattr(e, 'code') and hasattr(e, 'msg'): self.message = ' '.join([str(e.code), e.msg]) else: self.message = "Unknown Error" except URLError as e: if hasattr(e, 'reason'): self.message = 'Unreachable: ' + str(e.reason) elif hasattr(e, 'code') and e.code != 301: self.message = 'Error: ' + str(e.code) else: self.message = 'Redirect. Check manually: ' + str(e.code) except Exception as e: self.message = 'Other Error: %s' % e else: if getattr(response, 'getcode', False) and response.getcode( ) == 301 and response.geturl() != url: self.redirect_to = response.geturl() elif self.redirect_to: self.redirect_to = '' self.last_checked = now() self.save()
def dispatch_webhook_event(request, webhook_targets, event, payload): """Dispatch the given event and payload to the given WebHook targets. Args: request (django.http.HttpRequest): The HTTP request from the client. webhook_targets (list of reviewboard.notifications.models.WebHookTarget): The list of WebHook targets containing endpoint URLs to dispatch to. event (unicode): The name of the event being dispatched. payload (dict): The payload data to encode for the WebHook payload. Raises: ValueError: There was an error with the payload format. Details are in the log and the exception message. """ encoder = BasicAPIEncoder() bodies = {} raw_norm_payload = None json_norm_payload = None for webhook_target in webhook_targets: use_custom_content = webhook_target.use_custom_content encoding = webhook_target.encoding # See how we need to handle normalizing this payload. If we need # something JSON-safe, then we need to go the more aggressive route # and normalize keys to strings. if raw_norm_payload is None or json_norm_payload is None: try: if (raw_norm_payload is None and (use_custom_content or encoding == webhook_target.ENCODING_XML)): # This payload's going to be provided for XML and custom # templates. We don't want to alter the keys at all. raw_norm_payload = normalize_webhook_payload( payload=payload, request=request) elif (json_norm_payload is None and not use_custom_content and encoding in (webhook_target.ENCODING_JSON, webhook_target.ENCODING_FORM_DATA)): # This payload's going to be provided for JSON or # form-data. We want to normalize all keys to strings. json_norm_payload = normalize_webhook_payload( payload=payload, request=request, use_string_keys=True) except TypeError as e: logging.exception( 'WebHook payload passed to ' 'dispatch_webhook_event containing invalid ' 'data types: %s', e) raise ValueError(six.text_type(e)) if use_custom_content: try: assert raw_norm_payload is not None body = render_custom_content(webhook_target.custom_content, raw_norm_payload) body = force_bytes(body) except Exception as e: logging.exception('Could not render WebHook payload: %s', e) continue else: if encoding not in bodies: try: if encoding == webhook_target.ENCODING_JSON: assert json_norm_payload is not None adapter = JSONEncoderAdapter(encoder) body = adapter.encode(json_norm_payload, request=request) elif encoding == webhook_target.ENCODING_XML: assert raw_norm_payload is not None adapter = XMLEncoderAdapter(encoder) body = adapter.encode(raw_norm_payload, request=request) elif encoding == webhook_target.ENCODING_FORM_DATA: assert json_norm_payload is not None adapter = JSONEncoderAdapter(encoder) body = urlencode({ 'payload': adapter.encode(json_norm_payload, request=request), }) else: logging.error( 'Unexpected WebHookTarget encoding "%s" ' 'for ID %s', encoding, webhook_target.pk) continue except Exception as e: logging.exception('Could not encode WebHook payload: %s', e) continue body = force_bytes(body) bodies[encoding] = body else: body = bodies[encoding] headers = { b'X-ReviewBoard-Event': event.encode('utf-8'), b'Content-Type': webhook_target.encoding.encode('utf-8'), b'Content-Length': len(body), b'User-Agent': ('ReviewBoard-WebHook/%s' % get_package_version()).encode('utf-8'), } if webhook_target.secret: signer = hmac.new(webhook_target.secret.encode('utf-8'), body, hashlib.sha1) headers[b'X-Hub-Signature'] = \ ('sha1=%s' % signer.hexdigest()).encode('utf-8') logging.info('Dispatching webhook for event %s to %s', event, webhook_target.url) try: url = webhook_target.url url_parts = urlsplit(url) if url_parts.username or url_parts.password: netloc = url_parts.netloc.split('@', 1)[1] url = urlunsplit((url_parts.scheme, netloc, url_parts.path, url_parts.params, url_parts.query)) password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, url, url_parts.username, url_parts.password) handler = HTTPBasicAuthHandler(password_mgr) opener = build_opener(handler) else: opener = build_opener() opener.open(Request(url, body, headers)) except Exception as e: logging.exception('Could not dispatch WebHook to %s: %s', webhook_target.url, e)