def find_links(self):
        if not self.output_path.endswith(".html"):
            return

        self.site.info("Finding links in {}", self)

        self._load_output()

        try:
            root = _XML(self.content)
        except Exception as e:
            self.site.info(str(e))
            return

        assert root is not None, self.content

        links = self._gather_links(root)
        link_targets = self._gather_link_targets(root)

        for link in links:
            if link == "?":
                continue

            scheme, netloc, path, query, fragment = _urlsplit(link)

            if scheme and scheme not in ("file", "http", "https", "ftp"):
                continue

            if netloc in ("issues.apache.org", "bugzilla.redhat.com"):
                continue

            if (fragment and not path) or not path.startswith("/"):
                link = _urljoin(self.url, link)

            self.site.links[link].add(self.url)

        self.site.link_targets.update(link_targets)
Exemple #2
0
def urljoin(base, *fragments):
    for f in fragments:
        base = _urljoin(base, f, allow_fragments=True)
    return base
Exemple #3
0
 def get_relative_url(self, rel: str):
     return URL(_urljoin(str(self), rel))
Exemple #4
0
def urljoin(base, *parts):
    """
    URL join function that makes more sense than Python's standard library version.
    """
    return reduce(lambda base, part: _urljoin(base + "/", str(part).lstrip("/")), parts, base)
Exemple #5
0
def urljoin(*args):
    if len(args) == 1: return args
    url = _urljoin(str(args[0]) + "/", str(args[1]))
    if len(args) == 2: return url
    return urljoin(url, *args[2:])
Exemple #6
0
def _join(head, tail):
    return _urljoin(head.rstrip('/')+'/', tail.lstrip('/'))
Exemple #7
0
def _query(function,
           api_key=None,
           args=None,
           method='GET',
           header_dict=None,
           data=None):
    """
    Slack object method function to construct and execute on the API URL.

    :param api_key:     The Slack api key.
    :param function:    The Slack api function to perform.
    :param method:      The HTTP method, e.g. GET or POST.
    :param data:        The data to be sent for POST method.
    :return:            The json response from the API call or False.
    """
    query_params = {}

    ret = {'message': '', 'res': True}

    if not api_key:
        try:
            options = __mods__['config.option']('slack')
            if not api_key:
                api_key = options.get('api_key')
        except (NameError, KeyError, AttributeError):
            log.error('No Slack api key found.')
            ret['message'] = 'No Slack api key found.'
            ret['res'] = False
            return ret

    base_url = _urljoin(API_URL, '/api/')
    path = SLACK_FUNCTIONS.get(function).get('request')
    url = _urljoin(base_url, path, False)

    if not isinstance(args, dict):
        query_params = {}
    query_params['token'] = api_key

    if header_dict is None:
        header_dict = {}

    if method != 'POST':
        header_dict['Accept'] = 'application/json'

    result = hubblestack.utils.http.query(
        url,
        method,
        params=query_params,
        data=data,
        decode=True,
        status=True,
        header_dict=header_dict,
        opts=__opts__,
    )

    if result.get('status', None) == http.client.OK:
        _result = result['dict']
        response = SLACK_FUNCTIONS.get(function).get('response')
        if 'error' in _result:
            ret['message'] = _result['error']
            ret['res'] = False
            return ret
        ret['message'] = _result.get(response)
        return ret
    elif result.get('status', None) == http.client.NO_CONTENT:
        return True
    log.debug(url)
    log.debug(query_params)
    log.debug(data)
    log.debug(result)
    _result = result['dict']
    if 'error' in _result:
        ret['message'] = _result['error']
        ret['res'] = False
        return ret
    ret['message'] = _result.get('response')
    return ret