def process_image_node(self, node, docname, standalone=False): """ process an image node This method will process an image node for asset tracking. Asset information is tracked in this manager and other helper methods can be used to pull asset information when needed. Args: node: the image node docname: the document's name standalone (optional): ignore hash mappings (defaults to False) Returns: the key, document name and path """ uri = str(node['uri']) if not uri.startswith('data:') and uri.find('://') == -1: logger.verbose('process image node: %s' % uri) path = self._interpret_asset_path(node) if path: return self._handle_entry(path, docname, standalone) return None, None, None
def _interpret_asset_path(self, node): """ find an absolute path for a target assert Returns the absolute path to an assert. For unsupported asset types, this method will return ``None`` values. This method should not be invoked on external assets (i.e. URLs). Args: node: the node to parse Returns: the absolute path """ path = None if isinstance(node, nodes.image): # uri's will be relative to documentation root. path = str(node['uri']) elif isinstance(node, addnodes.download_reference): # reftarget will be a reference to the asset with respect to the # document (refdoc) holding this reference. Use reftarget and refdoc # to find a proper path. docdir = os.path.dirname(node['refdoc']) path = os.path.join(docdir, node['reftarget']) abspath = find_env_abspath(self.env, self.outdir, path) if not abspath: logger.verbose('failed to find path: %s' % path) return abspath
def remove_page(self, page_id): if self.dryrun: self._dryrun('removing page', page_id) return elif self.onlynew: self._onlynew('page removal restricted', page_id) return try: try: self.rest_client.delete('content', page_id) except ConfluenceBadApiError as ex: if str(ex).find('Transaction rolled back') == -1: raise logger.warn('delete failed; retrying...') time.sleep(3) self.rest_client.delete('content', page_id) except ConfluenceBadApiError as ex: # Check if Confluence reports that this content does not exist. If # so, we want to suppress the API error. This is most likely a # result of a Confluence instance reporting a page descendant # identifier which no longer exists (possibly a caching issue). if str(ex).find('No content found with id') == -1: raise logger.verbose('ignore missing delete for page ' 'identifier: {}'.format(page_id)) except ConfluencePermissionError: raise ConfluencePermissionError( """Publish user does not have permission to delete """ """from the configured space.""")
def process_file_node(self, node, docname, standalone=False): """ process an file node This method will process an file node for asset tracking. Asset information is tracked in this manager and other helper methods can be used to pull asset information when needed. Args: node: the file node docname: the document's name standalone (optional): ignore hash mappings (defaults to False) Returns: the key, document name and path """ target = node['reftarget'] if target.find('://') == -1: logger.verbose('process file node: %s' % target) path = self._interpret_asset_path(node) if path: return self._handle_entry(path, docname, standalone) return None, None, None
def register_title(docname, title, config): """ register the title for the provided document name In Confluence, a page is identified by the name/title of a page (at least, from the user's perspective). When processing a series of document names, the title value used for a document is based off the first heading detected. This register method allows a builder to track a document's title name name, so it may provide a document's contents and target title when passed to the publish operation. If a prefix (or postfix) value is provided, it will be added to the beginning (or at the end) of the provided title value. """ try_max = CONFLUENCE_MAX_TITLE_LEN base_tail = '' postfix = None prefix = None if config and (not config.confluence_ignore_titlefix_on_index or docname != config.root_doc): postfix = config.confluence_publish_postfix prefix = config.confluence_publish_prefix if prefix: title = prefix + title if postfix: base_tail += postfix if len(title) + len(base_tail) > try_max: warning = 'document title has been trimmed due to length: %s' % title if len(base_tail) > 0: warning += '; With postfix: %s' % base_tail logger.warn(warning) title = title[0:try_max - len(base_tail)] base_title = title title += base_tail # check if title is already used; if so, append a new value offset = 2 while title.lower() in ConfluenceState.title2doc: if offset == 2: logger.warn('title conflict detected with ' "'{}' and '{}'".format( ConfluenceState.title2doc[title.lower()], docname)) tail = ' ({}){}'.format(offset, base_tail) if len(base_title) + len(tail) > try_max: base_title = base_title[0:(try_max - len(tail))] title = base_title + tail offset += 1 ConfluenceState.doc2title[docname] = title ConfluenceState.title2doc[title.lower()] = docname logger.verbose('mapping %s to title: %s' % (docname, title)) return title
def register_toctree_depth(docname, depth): """ register the toctree-depth for the provided document name Documents using toctree's will only use the first toctree's 'maxdepth' option [1]. This method provides the ability to track the depth of a document before toctree resolution removes any hints at the maximum depth desired. [1]: http://www.sphinx-doc.org/en/stable/markup/toctree.html#id3 """ ConfluenceState.doc2ttd[docname] = depth logger.verbose('track %s toc-depth: %s' % (docname, depth))
def _wrapper(self, *args, **kwargs): # apply any user-set delay on an api request if self.config.confluence_publish_delay: delay = self.config.confluence_publish_delay logger.verbose('user-set api delay set; ' 'waiting {} seconds...'.format(math.ceil(delay))) time.sleep(delay) # if confluence asked us to wait so many seconds before a next # api request, wait a moment if self.next_delay: delay = self.next_delay logger.verbose('rate-limit header detected; ' 'waiting {} seconds...'.format(math.ceil(delay))) time.sleep(delay) self.next_delay = None # if we have imposed some rate-limiting requests where confluence # did not provide retry information, slowly decrease our tracked # delay if requests are going through self.last_retry = max(self.last_retry / 2, 1) attempt = 1 while True: try: return func(self, *args, **kwargs) except ConfluenceRateLimited as e: # if max attempts have been reached, stop any more attempts if attempt > RATE_LIMITED_MAX_RETRIES: raise e # determine the amount of delay to wait again -- either from the # provided delay (if any) or exponential backoff if self.next_delay: delay = self.next_delay self.next_delay = None else: delay = 2 * self.last_retry # cap delay to a maximum delay = min(delay, RATE_LIMITED_MAX_RETRY_DURATION) # add jitter delay += random.uniform(0.3, 1.3) # wait the calculated delay before retrying again logger.warn('rate-limit response detected; ' 'waiting {} seconds...'.format(math.ceil(delay))) time.sleep(delay) self.last_retry = delay attempt += 1
def register_target(refid, target): """ register a reference to a specific (anchor) target When interpreting a reference in reStructuredText, the reference could point to an anchor in the same document, another document or an anchor in another document. In Confluence, the target name is typically dependent on the document's title name (auto-generated targets provided by Confluence; ex. title#header). This register method allows a builder to track the target value to use for a provided reference (so that a writer can properly prepare a link; see also `target`). """ ConfluenceState.refid2target[refid] = target logger.verbose('mapping %s to target: %s' % (refid, target))
def register_upload_id(docname, id_): """ register a page (upload) identifier for a docname When a publisher creates/updates a page on a Confluence instance, the resulting page will have an identifier for it. This state utility class can help track the Confluence page's identifier by invoking this registration method. This method is primarily used to help track/order published documents into a hierarchical fashion (see `registerParentDocname`). It is important to note that the order of published documents will determine if a page's upload identifier is tracked in this state (see also `uploadId`). """ ConfluenceState.doc2uploadId[docname] = id_ logger.verbose("tracking docname %s's upload id: %s" % (docname, id_))
def register_parent_docname(docname, parent_docname): """ register a parent docname for a provided docname When using Sphinx's toctree, documents defined in the tree can be considered child pages (see the configuration option 'confluence_page_hierarchy'). This method helps track a parent document for a provided child document. With the ability to track a parent document and track publish upload identifiers (see `registerUploadId`), the publish operation can help ensure pages are structured in a hierarchical fashion (see also `parentDocname`). [1]: http://www.sphinx-doc.org/en/stable/markup/toctree.html#directive-toctree """ ConfluenceState.doc2parentDoc[docname] = parent_docname logger.verbose('setting parent of %s to: %s' % (docname, parent_docname))
def unknown_visit(self, node): node_name = node.__class__.__name__ ignore_nodes = self.builder.config.confluence_adv_ignore_nodes if node_name in ignore_nodes: ConfluenceLogger.verbose('ignore node {} (conf)'.format(node_name)) raise nodes.SkipNode # allow users to override unknown nodes # # A node handler allows an advanced user to provide implementation to # process a node not supported by this extension. This is to assist in # providing a quick alternative to supporting another third party # extension in this translator (without having to take the time in # building a third extension). handler = self.builder.config.confluence_adv_node_handler if handler and isinstance(handler, dict) and node_name in handler: handler[node_name](self, node) raise nodes.SkipNode raise NotImplementedError('unknown node: ' + node_name)
def build_intersphinx(builder): """ build intersphinx information from the state of the builder Attempt to build a series of entries for an intersphinx inventory resource for Confluence builder generated content. This is only supported after processing a publishing event where page identifiers are cached to build URI entries. Args: builder: the builder """ def escape(string): return re.sub("\\s+", ' ', string) if builder.cloud: pages_part = 'pages/{}/' else: pages_part = 'pages/viewpage.action?pageId={}' with open(path.join(builder.outdir, INVENTORY_FILENAME), 'wb') as f: # header f.write(('# Sphinx inventory version 2\n' '# Project: %s\n' '# Version: %s\n' '# The remainder of this file is compressed using zlib.\n' % (escape(builder.env.config.project), escape(builder.env.config.version))).encode()) # contents compressor = zlib.compressobj(9) for domainname, domain in sorted(builder.env.domains.items()): if domainname == 'std': for name, dispname, typ, docname, raw_anchor, prio in sorted( domain.get_objects()): page_id = ConfluenceState.uploadId(docname) if not page_id: continue target_name = '{}#{}'.format(docname, raw_anchor) target = ConfluenceState.target(target_name) if raw_anchor and target: title = ConfluenceState.title(docname) anchor = 'id-' + title + '-' + target anchor = anchor.replace(' ', '') # confluence will convert quotes to right-quotes for # anchor values; replace and encode the anchor value anchor = anchor.replace('"', '”') anchor = anchor.replace("'", '’') anchor = requests.utils.quote(anchor) else: anchor = '' uri = pages_part.format(page_id) if anchor: uri += '#' + anchor if dispname == name: dispname = '-' entry = ('%s %s:%s %s %s %s\n' % (name, domainname, typ, prio, uri, dispname)) ConfluenceLogger.verbose('(intersphinx) ' + entry.strip()) f.write(compressor.compress(entry.encode('utf-8'))) f.write(compressor.flush())
def storePage(self, page_name, data, parent_id=None): uploaded_page_id = None if self.config.confluence_adv_trace_data: ConfluenceLogger.trace('data', data['content']) if self.dryrun: _, page = self.getPage(page_name, 'version,ancestors') if not page: self._dryrun('adding new page ' + page_name) return None else: misc = '' if parent_id and 'ancestors' in page: if not any(a['id'] == parent_id for a in page['ancestors']): if parent_id in self._name_cache: misc += '[new parent page {} ({})]'.format( self._name_cache[parent_id], parent_id) else: misc += '[new parent page]' self._dryrun('updating existing page', page['id'], misc) return page['id'] can_labels = 'labels' not in self.config.confluence_adv_restricted expand = 'version' if can_labels and self.append_labels: expand += ',metadata.labels' _, page = self.getPage(page_name, expand=expand) if self.onlynew and page: self._onlynew('skipping existing page', page['id']) return page['id'] try: # new page if not page: newPage = { 'type': 'page', 'title': page_name, 'body': { 'storage': { 'representation': 'storage', 'value': data['content'], } }, 'space': { 'key': self.space_name }, } if can_labels: self._populate_labels(newPage, data['labels']) if parent_id: newPage['ancestors'] = [{'id': parent_id}] try: rsp = self.rest_client.post('content', newPage) if 'id' not in rsp: api_err = ('Confluence reports a successful page ' + 'creation; however, provided no ' + 'identifier.\n\n') try: api_err += 'DATA: {}'.format(json.dumps( rsp, indent=2)) except TypeError: api_err += 'DATA: <not-or-invalid-json>' raise ConfluenceBadApiError(api_err) uploaded_page_id = rsp['id'] except ConfluenceBadApiError as ex: # Check if Confluence reports that the new page request # fails, indicating it already exists. This is usually # (outside of possible permission use cases) that the page # name's casing does not match. In this case, attempt to # re-check for the page in a case-insensitive fashion. If # found, attempt to perform an update request instead. if str(ex).find('title already exists') == -1: raise ConfluenceLogger.verbose('title already exists warning ' 'for page {}'.format(page_name)) _, page = self.getPageCaseInsensitive(page_name) if not page: raise if self.onlynew: self._onlynew('skipping existing page', page['id']) return page['id'] # update existing page if page: last_version = int(page['version']['number']) updatePage = { 'id': page['id'], 'type': 'page', 'title': page_name, 'body': { 'storage': { 'representation': 'storage', 'value': data['content'], } }, 'space': { 'key': self.space_name }, 'version': { 'number': last_version + 1 }, } if can_labels: labels = list(data['labels']) if self.append_labels: labels.extend([lbl.get('name') for lbl in page.get('metadata', {}).get( 'labels', {}).get('results', {}) ]) self._populate_labels(updatePage, labels) if not self.notify: updatePage['version']['minorEdit'] = True if parent_id: updatePage['ancestors'] = [{'id': parent_id}] try: self.rest_client.put('content', page['id'], updatePage) except ConfluenceBadApiError as ex: if str(ex).find('unreconciled') != -1: raise ConfluenceUnreconciledPageError( page_name, page['id'], self.server_url, ex) # Confluence Cloud may (rarely) fail to complete a # content request with an OptimisticLockException/ # StaleObjectStateException exception. It is suspected # that this is just an instance timing/processing issue. # If this is observed, wait a moment and retry the # content request. If it happens again, the put request # will fail as it normally would. if str(ex).find('OptimisticLockException') == -1: raise ConfluenceLogger.warn( 'remote page updated failed; retrying...') time.sleep(1) self.rest_client.put('content', page['id'], updatePage) uploaded_page_id = page['id'] except ConfluencePermissionError: raise ConfluencePermissionError( """Publish user does not have permission to add page """ """content to the configured space.""" ) if not self.watch: self.rest_client.delete('user/watch/content', uploaded_page_id) return uploaded_page_id
def storeAttachment(self, page_id, name, data, mimetype, hash, force=False): """ request to store an attachment on a provided page Makes a request to a Confluence instance to either publish a new attachment or update an existing attachment. If the attachment's hash matches the tracked hash (via the comment field) of an existing attachment, this call will assume the attachment is already published and will return (unless forced). Args: page_id: the identifier of the page to attach to name: the attachment name data: the attachment data mimetype: the mime type of this attachment hash: the hash of the attachment force (optional): force publishing if exists (defaults to False) Returns: the attachment identifier """ HASH_KEY = 'SCB_KEY' uploaded_attachment_id = None _, attachment = self.getAttachment(page_id, name) # check if attachment (of same hash) is already published to this page comment = None if attachment and 'metadata' in attachment: metadata = attachment['metadata'] if 'comment' in metadata: comment = metadata['comment'] if not force and comment: parts = comment.split(HASH_KEY + ':') if len(parts) > 1: tracked_hash = parts[1] if hash == tracked_hash: ConfluenceLogger.verbose('attachment ({}) is already ' 'published to document with same hash'.format(name)) return attachment['id'] if self.dryrun: if not attachment: self._dryrun('adding new attachment ' + name) return None else: self._dryrun('updating existing attachment', attachment['id']) return attachment['id'] elif self.onlynew and attachment: self._onlynew('skipping existing attachment', attachment['id']) return attachment['id'] # publish attachment try: data = { 'comment': '{}:{}'.format(HASH_KEY, hash), 'file': (name, data, mimetype), } if not self.notify: # using str over bool to support requests pre-v2.19.0 data['minorEdit'] = 'true' if not attachment: url = 'content/{}/child/attachment'.format(page_id) rsp = self.rest_client.post(url, None, files=data) uploaded_attachment_id = rsp['results'][0]['id'] else: url = 'content/{}/child/attachment/{}/data'.format( page_id, attachment['id']) rsp = self.rest_client.post(url, None, files=data) uploaded_attachment_id = rsp['id'] if not self.watch: self.rest_client.delete('user/watch/content', uploaded_attachment_id) except ConfluencePermissionError: raise ConfluencePermissionError( """Publish user does not have permission to add an """ """attachment to the configured space.""" ) return uploaded_attachment_id
def store_page(self, page_name, data, parent_id=None): """ request to store page information to a confluence instance Performs a request which will attempt to store the provided page information and publish it to either a new page with the provided page name or update an existing page with a matching page name. Pages will be published at the root of a Confluence space unless a provided parent page identifier is provided. Args: page_name: the page title to use on the updated page data: the page data to apply parent_id (optional): the id of the ancestor to use """ uploaded_page_id = None if self.config.confluence_adv_trace_data: logger.trace('data', data['content']) if self.dryrun: _, page = self.get_page(page_name, 'version,ancestors') if not page: self._dryrun('adding new page ' + page_name) return None else: misc = '' if parent_id and 'ancestors' in page: if not any(a['id'] == parent_id for a in page['ancestors']): if parent_id in self._name_cache: misc += '[new parent page {} ({})]'.format( self._name_cache[parent_id], parent_id) else: misc += '[new parent page]' self._dryrun('updating existing page', page['id'], misc) return page['id'] expand = 'version' if self.append_labels: expand += ',metadata.labels' _, page = self.get_page(page_name, expand=expand) if self.onlynew and page: self._onlynew('skipping existing page', page['id']) return page['id'] try: # new page if not page: new_page = self._build_page(page_name, data) self._populate_labels(new_page, data['labels']) if parent_id: new_page['ancestors'] = [{'id': parent_id}] try: rsp = self.rest_client.post('content', new_page) if 'id' not in rsp: api_err = ('Confluence reports a successful page ' + 'creation; however, provided no ' + 'identifier.\n\n') try: api_err += 'DATA: {}'.format( json.dumps(rsp, indent=2)) except TypeError: api_err += 'DATA: <not-or-invalid-json>' raise ConfluenceBadApiError(-1, api_err) uploaded_page_id = rsp['id'] # if we have labels and this is a non-cloud instance, # initial labels need to be applied in their own request labels = new_page['metadata']['labels'] if not self.cloud and labels: url = 'content/{}/label'.format(uploaded_page_id) self.rest_client.post(url, labels) except ConfluenceBadApiError as ex: # Check if Confluence reports that the new page request # fails, indicating it already exists. This is usually # (outside of possible permission use cases) that the page # name's casing does not match. In this case, attempt to # re-check for the page in a case-insensitive fashion. If # found, attempt to perform an update request instead. if str(ex).find('title already exists') == -1: raise logger.verbose('title already exists warning ' 'for page {}'.format(page_name)) _, page = self.get_page_case_insensitive(page_name) if not page: raise if self.onlynew: self._onlynew('skipping existing page', page['id']) return page['id'] # update existing page if page: self._update_page(page, page_name, data, parent_id=parent_id) uploaded_page_id = page['id'] except ConfluencePermissionError: raise ConfluencePermissionError( """Publish user does not have permission to add page """ """content to the configured space.""") if not self.watch: self.rest_client.delete('user/watch/content', uploaded_page_id) return uploaded_page_id
def confluence_supported_svg(builder, node): """ process an image node and ensure confluence-supported svg (if applicable) SVGs have some limitations when being presented on a Confluence instance. The following have been observed issues: 1) If an SVG file does not have an XML declaration, Confluence will fail to render an image. 2) If an `ac:image` macro is applied custom width/height values on an SVG, Confluence Confluence will fail to render the image. This call will process a provided image node and ensure an SVG is in a ready state for publishing. If a node is not an SVG, this method will do nothing. To support custom width/height fields for an SVG image, the image file itself will be modified to an expected lengths. Any hints in the documentation using width/height or scale, the desired width and height fields of an image will calculated and replaced/injected into the SVG image. Any SVG files which do not have an XML declaration will have on injected. Args: builder: the builder node: the image node to check """ uri = node['uri'] # ignore external/embedded images if uri.find('://') != -1 or uri.startswith('data:'): return # invalid uri/path uri_abspath = find_env_abspath(builder.env, builder.outdir, uri) if not uri_abspath: return # ignore non-svgs mimetype = guess_mimetype(uri_abspath) if mimetype != 'image/svg+xml': return try: with open(uri_abspath, 'rb') as f: svg_data = f.read() except (IOError, OSError) as err: builder.warn('error reading svg: %s' % err) return modified = False svg_root = xml_et.fromstring(svg_data) # determine (if possible) the svgs desired width/height svg_height = None if 'height' in svg_root.attrib: svg_height = svg_root.attrib['height'] svg_width = None if 'width' in svg_root.attrib: svg_width = svg_root.attrib['width'] # try to fallback on the viewbox attribute viewbox = False if svg_height is None or svg_width is None: if 'viewBox' in svg_root.attrib: try: _, _, svg_width, svg_height = \ svg_root.attrib['viewBox'].split(' ') viewbox = True except ValueError: pass # if tracking an svg width/height, ensure the sizes are in pixels if svg_height: svg_height, svg_height_units = extract_length(svg_height) svg_height = convert_length(svg_height, svg_height_units, pct=False) if svg_width: svg_width, svg_width_units = extract_length(svg_width) svg_width = convert_length(svg_width, svg_width_units, pct=False) # extract length/scale properties from the node height, hu = extract_length(node.get('height')) scale = node.get('scale') width, wu = extract_length(node.get('width')) # if a percentage is detected, ignore these lengths when attempting to # perform any adjustments; percentage hints for internal images will be # managed with container tags in the translator if hu == '%': height = None hu = None if wu == '%': width = None wu = None # confluence can have difficulty rendering svgs with only a viewbox entry; # if a viewbox is used, use it for the height/width if these options have # not been explicitly configured on the directive if viewbox and not height and not width: height = svg_height width = svg_width # if only one size is set, fetch (and scale) the other if width and not height: if svg_height and svg_width: height = float(width) / svg_width * svg_height else: height = width hu = wu if height and not width: if svg_height and svg_width: width = float(height) / svg_height * svg_width else: width = height wu = hu # if a scale value is provided and a height/width is not set, attempt to # determine the size of the image so that we can apply a scale value on # the detected size values if scale: if not height and svg_height: height = svg_height hu = 'px' if not width and svg_width: width = svg_width wu = 'px' # apply scale factor to height/width fields if scale: if height: height = int(round(float(height) * scale / 100)) if width: width = int(round(float(width) * scale / 100)) # confluence only supports pixel sizes -- adjust any other unit type # (if possible) to a pixel length if height: height = convert_length(height, hu, pct=False) if height is None: builder.warn('unsupported svg unit type for confluence: ' + hu) if width: width = convert_length(width, wu, pct=False) if width is None: builder.warn('unsupported svg unit type for confluence: ' + wu) # if we have a height/width to apply, adjust the svg if height and width: svg_root.attrib['height'] = str(height) svg_root.attrib['width'] = str(width) svg_data = xml_et.tostring(svg_root) modified = True # ensure xml declaration exists if not svg_data.lstrip().startswith(b'<?xml'): svg_data = XML_DEC + b'\n' + svg_data modified = True # ignore svg file if not modifications are needed if not modified: return fname = sha256(svg_data).hexdigest() + '.svg' outfn = os.path.join(builder.outdir, builder.imagedir, 'svgs', fname) # write the new svg file (if needed) if not os.path.isfile(outfn): logger.verbose('generating compatible svg of: %s' % uri) logger.verbose('generating compatible svg to: %s' % outfn) ensuredir(os.path.dirname(outfn)) try: with open(outfn, 'wb') as f: f.write(svg_data) except (IOError, OSError) as err: builder.warn('error writing svg: %s' % err) return # replace the required node attributes node['uri'] = outfn if 'height' in node: del node['height'] if 'scale' in node: del node['scale'] if 'width' in node: del node['width']