def __init__(self, app): super(ConfluenceBuilder, self).__init__(app) self.cache_doctrees = {} self.cloud = False self.file_suffix = '.conf' self.info = ConfluenceLogger.info self.link_suffix = None self.master_doc_page_id = None self.metadata = {} self.nav_next = {} self.nav_prev = {} self.omitted_docnames = [] self.publish_allowlist = [] self.publish_denylist = [] self.publish_docnames = [] self.publisher = ConfluencePublisher() self.secnumbers = {} self.verbose = ConfluenceLogger.verbose self.warn = ConfluenceLogger.warn self._original_get_doctree = None # state tracking is set at initialization (not cleanup) so its content's # can be checked/validated on after the builder has executed (testing) ConfluenceState.reset()
def publish_doc(self, docname, output): conf = self.config title = ConfluenceState.title(docname) parent_id = None if self.config.master_doc and self.config.confluence_page_hierarchy: if self.config.master_doc != docname: parent = ConfluenceState.parentDocname(docname) parent_id = ConfluenceState.uploadId(parent) if not parent_id: parent_id = self.parent_id data = { 'content': output, 'labels': [], } if self.config.confluence_global_labels: data['labels'].extend(self.config.confluence_global_labels) metadata = self.metadata[docname] if 'labels' in metadata: data['labels'].extend([v for v in metadata['labels']]) uploaded_id = self.publisher.storePage(title, data, parent_id) ConfluenceState.registerUploadId(docname, uploaded_id) if self.config.master_doc == docname: self.master_doc_page_id = uploaded_id if conf.confluence_purge and self.legacy_pages is None: if conf.confluence_purge_from_master and self.master_doc_page_id: baseid = self.master_doc_page_id else: baseid = self.parent_id # if no base identifier and dry running, ignore legeacy page # searching as there is no initial master document to reference # against if (conf.confluence_purge_from_master and conf.confluence_publish_dryrun and not baseid): self.legacy_pages = [] elif self.config.confluence_adv_aggressive_search is True: self.legacy_pages = self.publisher.getDescendantsCompat(baseid) else: self.legacy_pages = self.publisher.getDescendants(baseid) # only populate a list of possible legacy assets when a user is # configured to check or push assets to the target space asset_override = conf.confluence_asset_override if asset_override is None or asset_override: for legacy_page in self.legacy_pages: attachments = self.publisher.getAttachments(legacy_page) self.legacy_assets[legacy_page] = attachments if conf.confluence_purge: if uploaded_id in self.legacy_pages: self.legacy_pages.remove(uploaded_id)
def intern_uri_anchor_value(docname, refuri): """ determine the anchor value for an internal uri point This call helps determine the anchor value to use for a link to an anchor for an internal document. The anchor value will be parsed out of the provided URI checked to see if a target entry already exists (e.g. if a header with a preconfigured identifier can be used). If so, the target value will be provided. If not, the parsed/raw anchor value will be returned. Args: docname: the docname of the page to link to refuri: the uri Returns: the encoded text """ anchor_value = None if '#' in refuri: anchor = refuri.split('#')[1] target_name = '{}#{}'.format(docname, anchor) # check if this target is reachable without an anchor; if so, use # the identifier value instead target = ConfluenceState.target(target_name) if target: anchor_value = target else: anchor_value = anchor anchor_value = encode_storage_format(anchor_value) return anchor_value
def _register_doctree_title_targets(self, docname, doctree): """ register title targets for a doctree Compiles a list of title targets which references can link against. This tracked expected targets for sections which are automatically generated in a rendered Confluence instance. Args: docname: the docname of the doctree doctree: the doctree to search for targets """ doc_used_names = {} secnumbers = self.env.toc_secnumbers.get(docname, {}) for node in doctree.traverse(nodes.title): if isinstance(node.parent, nodes.section): section_node = node.parent if 'ids' in section_node: target = ''.join(node.astext().split()) # when confluence has a header that contains a link, the # automatically assigned identifier removes any underscores if node.next_node(addnodes.pending_xref): target = target.replace('_', '') if self.add_secnumbers: anchorname = '#' + target if anchorname not in secnumbers: anchorname = '' secnumber = secnumbers.get(anchorname) if secnumber: target = ('.'.join(map(str, secnumber)) + self.secnumber_suffix + target) section_id = doc_used_names.get(target, 0) doc_used_names[target] = section_id + 1 if section_id > 0: target = '{}.{}'.format(target, section_id) for id in section_node['ids']: id = '{}#{}'.format(docname, id) ConfluenceState.registerTarget(id, target)
def test_parent_registration(self): root_doc = ConfluenceState.parentDocname('toctree') self.assertIsNone(root_doc, 'root toctree has a parent') parent_doc = ConfluenceState.parentDocname('toctree-doc1') self.assertEqual(parent_doc, 'toctree', 'toctree-doc1 parent is not root toctree') parent_doc = ConfluenceState.parentDocname('toctree-doc2') self.assertEqual(parent_doc, 'toctree', 'toctree-doc2 parent is not root toctree') parent_doc = ConfluenceState.parentDocname('toctree-doc3') self.assertEqual(parent_doc, 'toctree', 'toctree-doc3 parent is not root toctree') parent_doc = ConfluenceState.parentDocname('toctree-doc2a') self.assertEqual(parent_doc, 'toctree-doc2', 'toctree-doc2a parent is not toctree-doc2')
def process_tree_structure(self, ordered, docname, traversed, depth=0): omit = False max_depth = self.config.confluence_max_doc_depth if max_depth is not None and depth > max_depth: omit = True self.omitted_docnames.append(docname) if not omit: ordered.append(docname) modified = False doctree = self.env.get_doctree(docname) for toctreenode in doctree.traverse(addnodes.toctree): if not omit and max_depth is not None: if (toctreenode['maxdepth'] == -1 or depth + toctreenode['maxdepth'] > max_depth): new_depth = max_depth - depth assert new_depth >= 0 toctreenode['maxdepth'] = new_depth movednodes = [] for child in toctreenode['includefiles']: if child not in traversed: ConfluenceState.registerParentDocname(child, docname) traversed.append(child) children = self.process_tree_structure( ordered, child, traversed, depth + 1) if children: movednodes.append(children) self._fix_std_labels(child, docname) if movednodes: modified = True toctreenode.replace_self(movednodes) toctreenode.parent['classes'].remove('toctree-wrapper') if omit: container = addnodes.start_of_file(docname=docname) container.children = doctree.children return container elif modified: self.env.resolve_references(doctree, docname, self)
def _register_doctree_title_targets(self, docname, doctree): """ register title targets for a doctree Compiles a list of title targets which references can link against. This tracked expected targets for sections which are automatically generated in a rendered Confluence instance. Args: docname: the docname of the doctree doctree: the doctree to search for targets """ doc_used_names = {} secnumbers = self.env.toc_secnumbers.get(self.config.master_doc, {}) for node in doctree.traverse(nodes.title): if isinstance(node.parent, nodes.section): section_node = node.parent if 'ids' in section_node: title_name = ''.join(node.astext().split()) for id in section_node['ids']: target = title_name anchorname = '%s/#%s' % (docname, id) if anchorname not in secnumbers: anchorname = '%s/' % id if self.add_secnumbers: secnumber = secnumbers.get(anchorname) if secnumber: target = ('.'.join(map(str, secnumber)) + self.secnumber_suffix + target) section_id = doc_used_names.get(target, 0) doc_used_names[target] = section_id + 1 if section_id > 0: target = '{}.{}'.format(target, section_id) ConfluenceState.registerTarget(anchorname, target)
def _process_root_document(self): docname = self.config.master_doc # Extract the title from the root document as it will be used to decide # which Confluence page the document will be published to. if (self.config.confluence_title_overrides and docname in self.config.confluence_title_overrides): doctitle = self.config.confluence_title_overrides[docname] else: doctree = self.env.get_doctree(docname) doctitle = self._parse_doctree_title(docname, doctree) if not doctitle: return None # register the title for the root document (for references, assets, ...) ConfluenceState.registerTitle(docname, doctitle, self.config) # register the root document for publishing self.publish_docnames.append(docname) return doctitle
def publish_asset(self, key, docname, output, type, hash): conf = self.config publisher = self.publisher title = ConfluenceState.title(docname) page_id = ConfluenceState.uploadId(docname) if not page_id: # A page identifier may not be tracked in cases where only a subset # of documents are published and the target page an asset will be # published to was not part of the request. In this case, ask the # Confluence instance what the target page's identifier is. page_id, _ = publisher.getPage(title) if page_id: ConfluenceState.registerUploadId(docname, page_id) else: ConfluenceLogger.warn('cannot publish asset since publishing ' 'point cannot be found ({}): {}'.format( key, docname)) return if conf.confluence_asset_override is None: # "automatic" management -- check if already published; if not, push attachment_id = publisher.storeAttachment(page_id, key, output, type, hash) elif conf.confluence_asset_override: # forced publishing of the asset attachment_id = publisher.storeAttachment(page_id, key, output, type, hash, force=True) if attachment_id and conf.confluence_purge: if page_id in self.legacy_assets: legacy_asset_info = self.legacy_assets[page_id] if attachment_id in legacy_asset_info: legacy_asset_info.pop(attachment_id, None)
def process_doclink(config, refuri): """ process a generated index link entry This call is used to process a index-generated link value -- to translate a docname/anchor value to a final document title/anchor value. This is to better prepare an index for the template to easily work with. Args: config: the active configuration refuri: the uri Returns: the document's title and anchor value """ docname = posixpath.normpath(os.path.splitext(refuri.split('#')[0])[0]) doctitle = ConfluenceState.title(docname) anchor_value = intern_uri_anchor_value(docname, refuri) return doctitle, anchor_value
def test_config_hierarchy_parent_registration(self): ConfluenceState.reset() self.build(self.dataset, relax=True) # root toctree should not have a parent root_doc = ConfluenceState.parent_docname('index') self.assertIsNone(root_doc) # check various documents for expected parents parent_doc = ConfluenceState.parent_docname('toctree-doc1') self.assertEqual(parent_doc, 'index') parent_doc = ConfluenceState.parent_docname('toctree-doc2') self.assertEqual(parent_doc, 'index') parent_doc = ConfluenceState.parent_docname('toctree-doc3') self.assertEqual(parent_doc, 'index') parent_doc = ConfluenceState.parent_docname('toctree-doc2a') self.assertEqual(parent_doc, 'toctree-doc2')
def build_intersphinx(builder): """ build intersphinx information from the state of the builder Attempt to build a series of entries for an intersphinx inventory resource for Confluence builder generated content. This is only supported after processing a publishing event where page identifiers are cached to build URI entries. Args: builder: the builder """ def escape(string): return re.sub("\\s+", ' ', string) if builder.cloud: pages_part = 'pages/{}/' else: pages_part = 'pages/viewpage.action?pageId={}' with open(path.join(builder.outdir, INVENTORY_FILENAME), 'wb') as f: # header f.write(('# Sphinx inventory version 2\n' '# Project: %s\n' '# Version: %s\n' '# The remainder of this file is compressed using zlib.\n' % (escape(builder.env.config.project), escape(builder.env.config.version))).encode()) # contents compressor = zlib.compressobj(9) for domainname, domain in sorted(builder.env.domains.items()): if domainname == 'std': for name, dispname, typ, docname, raw_anchor, prio in sorted( domain.get_objects()): page_id = ConfluenceState.uploadId(docname) if not page_id: continue target_name = '{}#{}'.format(docname, raw_anchor) target = ConfluenceState.target(target_name) if raw_anchor and target: title = ConfluenceState.title(docname) anchor = 'id-' + title + '-' + target anchor = anchor.replace(' ', '') # confluence will convert quotes to right-quotes for # anchor values; replace and encode the anchor value anchor = anchor.replace('"', '”') anchor = anchor.replace("'", '’') anchor = requests.utils.quote(anchor) else: anchor = '' uri = pages_part.format(page_id) if anchor: uri += '#' + anchor if dispname == name: dispname = '-' entry = ('%s %s:%s %s %s %s\n' % (name, domainname, typ, prio, uri, dispname)) ConfluenceLogger.verbose('(intersphinx) ' + entry.strip()) f.write(compressor.compress(entry.encode('utf-8'))) f.write(compressor.flush())
def tearDown(self): ConfluenceState.reset()
def _register_title(self, title): with skip_warningiserror(): return ConfluenceState.register_title('mock', title, self.config)
def setUp(self): ConfluenceState.reset() self.config = MockedConfig()
def prepare_writing(self, docnames): ordered_docnames = [] traversed = [self.config.master_doc] # prepare caching doctree hook # # We'll temporarily override the environment's 'get_doctree' method to # allow this extension to manipulate the doctree for a document inside # the pre-writing stage to also take effect in the writing stage. self._original_get_doctree = self.env.get_doctree self.env.get_doctree = self._get_doctree # process the document structure of the master document, allowing: # - populating a publish order to ensure parent pages are created first # (when using hierarchy mode) # - squash pages which exceed maximum depth (if configured with a max # depth value) self.process_tree_structure(ordered_docnames, self.config.master_doc, traversed) # track relations between accepted documents # # Prepares a relation mapping between each non-orphan documents which # can be used by navigational elements. prevdoc = ordered_docnames[0] if ordered_docnames else None for docname in ordered_docnames[1:]: self.nav_prev[docname] = self.get_relative_uri(docname, prevdoc) self.nav_next[prevdoc] = self.get_relative_uri(prevdoc, docname) prevdoc = docname # add orphans (if any) to the publish list ordered_docnames.extend(x for x in docnames if x not in traversed) for docname in ordered_docnames: doctree = self.env.get_doctree(docname) # acquire title from override (if any), or parse first title entity if (self.config.confluence_title_overrides and docname in self.config.confluence_title_overrides): doctitle = self.config.confluence_title_overrides[docname] else: doctitle = self._parse_doctree_title(docname, doctree) # only register title/track for publishing if there is a title # value that can be applied to this document if doctitle: secnumbers = self.env.toc_secnumbers.get(docname, {}) if self.add_secnumbers and secnumbers.get(''): doctitle = ('.'.join(map(str, secnumbers[''])) + self.secnumber_suffix + doctitle) doctitle = ConfluenceState.registerTitle( docname, doctitle, self.config) # only publish documents that sphinx asked to prepare if docname in docnames: self.publish_docnames.append(docname) # track the toctree depth for a document, which a translator can # use as a hint when dealing with max-depth capabilities toctree = first(doctree.traverse(addnodes.toctree)) if toctree and toctree.get('maxdepth') > 0: ConfluenceState.registerToctreeDepth(docname, toctree.get('maxdepth')) # register title targets for references self._register_doctree_title_targets(docname, doctree) # post-prepare a ready doctree self._prepare_doctree_writing(docname, doctree) # Scan for assets that may exist in the documents to be published. This # will find most if not all assets in the documentation set. The # exception is assets which may be finalized during a document's post # transformation stage (e.g. embedded images are converted into real # images in Sphinx, which is then provided to a translator). Embedded # images are detected during an 'doctree-resolved' hook (see __init__). self.assets.process(ordered_docnames)