def __init__(self, app): super(ConfluenceBuilder, self).__init__(app) self.cache_doctrees = {} self.cloud = False self.file_suffix = '.conf' self.info = ConfluenceLogger.info self.link_suffix = None self.master_doc_page_id = None self.metadata = {} self.nav_next = {} self.nav_prev = {} self.omitted_docnames = [] self.publish_allowlist = [] self.publish_denylist = [] self.publish_docnames = [] self.publisher = ConfluencePublisher() self.secnumbers = {} self.verbose = ConfluenceLogger.verbose self.warn = ConfluenceLogger.warn self._original_get_doctree = None # state tracking is set at initialization (not cleanup) so its content's # can be checked/validated on after the builder has executed (testing) ConfluenceState.reset()
def process_raw_upload(target_sandbox): test_dir = os.path.dirname(os.path.realpath(__file__)) base_dir = os.path.join(test_dir, os.pardir) sandbox_dir = os.path.join(base_dir, target_sandbox) raw_file = os.path.join(sandbox_dir, 'raw.conf') if not os.path.exists(raw_file): print('[sandbox] missing file', raw_file) return doc_dir = prepare_dirs('sandbox-raw') with prepare_sphinx(sandbox_dir, out_dir=doc_dir, relax=True) as app: publisher = ConfluencePublisher() publisher.init(app.config) publisher.connect() while True: data = { 'labels': [], } with io.open(raw_file, 'r', encoding='utf-8') as f: data['content'] = f.read() print('[sandbox] publishing page...') try: publisher.store_page('raw', data) except ConfluenceBadApiError as ex: print('[sandbox] failed to publish content:', ex) print('[sandbox] any key to retry; q to quit') if input().lower() == 'q': break
def process_raw_upload(): test_dir = os.path.dirname(os.path.realpath(__file__)) sandbox_dir = os.path.join(test_dir, 'sandbox') raw_file = os.path.join(sandbox_dir, 'raw.conf') if not os.path.exists(raw_file): print('[sandbox] missing file', raw_file) return doc_dir, doctree_dir = _.prepareDirectories('sandbox-test') with _.prepareSphinx(sandbox_dir, doc_dir, doctree_dir) as app: publisher = ConfluencePublisher() publisher.init(app.config) publisher.connect() while True: with open(raw_file, 'r') as f: data = f.read() print('[sandbox] publishing page...') try: publisher.storePage('raw', data) except ConfluenceBadApiError as ex: print('[sandbox] failed to publish content:', ex) print('[sandbox] any key to retry; q to quit') if input().lower() == 'q': break
class ConfluenceBuilder(Builder): allow_parallel = True name = 'confluence' format = 'confluence_storage' supported_image_types = ConfluenceSupportedImages() supported_remote_images = True def __init__(self, app): super(ConfluenceBuilder, self).__init__(app) self.cache_doctrees = {} self.cloud = False self.file_suffix = '.conf' self.info = ConfluenceLogger.info self.link_suffix = None self.master_doc_page_id = None self.metadata = {} self.nav_next = {} self.nav_prev = {} self.omitted_docnames = [] self.publish_allowlist = [] self.publish_denylist = [] self.publish_docnames = [] self.publisher = ConfluencePublisher() self.secnumbers = {} self.verbose = ConfluenceLogger.verbose self.warn = ConfluenceLogger.warn self._original_get_doctree = None # state tracking is set at initialization (not cleanup) so its content's # can be checked/validated on after the builder has executed (testing) ConfluenceState.reset() def init(self): validate_configuration(self) apply_defaults(self.config) config = self.config self.add_secnumbers = self.config.confluence_add_secnumbers self.secnumber_suffix = self.config.confluence_secnumber_suffix if self.config.confluence_additional_mime_types: for type_ in self.config.confluence_additional_mime_types: self.supported_image_types.register(type_) if 'graphviz_output_format' in self.config: self.graphviz_output_format = self.config['graphviz_output_format'] else: self.graphviz_output_format = 'png' if self.config.confluence_publish: process_ask_configs(self.config) self.assets = ConfluenceAssetManager(config, self.env, self.outdir) self.writer = ConfluenceWriter(self) self.config.sphinx_verbosity = self.app.verbosity self.publisher.init(self.config) old_url = self.config.confluence_server_url new_url = ConfluenceUtil.normalizeBaseUrl(old_url) if old_url != new_url: ConfluenceLogger.warn('normalizing confluence url from ' '{} to {} '.format(old_url, new_url)) self.config.confluence_server_url = new_url # detect if Confluence Cloud if using the Atlassian domain if new_url: self.cloud = new_url.endswith('.atlassian.net/wiki/') if self.config.confluence_file_suffix is not None: self.file_suffix = self.config.confluence_file_suffix if self.config.confluence_link_suffix is not None: self.link_suffix = self.config.confluence_link_suffix elif self.link_suffix is None: self.link_suffix = self.file_suffix # Function to convert the docname to a reST file name. def file_transform(docname): return docname + self.file_suffix # Function to convert the docname to a relative URI. def link_transform(docname): return docname + self.link_suffix self.prev_next_loc = self.config.confluence_prev_next_buttons_location if self.config.confluence_file_transform is not None: self.file_transform = self.config.confluence_file_transform else: self.file_transform = file_transform if self.config.confluence_link_transform is not None: self.link_transform = self.config.confluence_link_transform else: self.link_transform = link_transform if self.config.confluence_lang_transform is not None: self.lang_transform = self.config.confluence_lang_transform else: self.lang_transform = None if self.config.confluence_publish: self.publish = True self.publisher.connect() else: self.publish = False def prepare_subset(option): value = getattr(config, option) if not value: return None # if provided via command line, treat as a list if option in config['overrides'] and isinstance(value, basestring): value = value.split(',') if isinstance(value, basestring): files = extract_strings_from_file(value) else: files = value return set(files) if files else None self.publish_allowlist = prepare_subset('confluence_publish_allowlist') self.publish_denylist = prepare_subset('confluence_publish_denylist') def get_outdated_docs(self): """ Return an iterable of input files that are outdated. """ # This method is taken from TextBuilder.get_outdated_docs() # with minor changes to support :confval:`rst_file_transform`. for docname in self.env.found_docs: if docname not in self.env.all_docs: yield docname continue sourcename = path.join(self.env.srcdir, docname + self.file_suffix) targetname = path.join(self.outdir, self.file_transform(docname)) try: targetmtime = path.getmtime(targetname) except Exception: targetmtime = 0 try: srcmtime = path.getmtime(sourcename) if srcmtime > targetmtime: yield docname except EnvironmentError: # source doesn't exist anymore pass def get_target_uri(self, docname, typ=None): return self.link_transform(docname) def prepare_writing(self, docnames): ordered_docnames = [] traversed = [self.config.master_doc] # prepare caching doctree hook # # We'll temporarily override the environment's 'get_doctree' method to # allow this extension to manipulate the doctree for a document inside # the pre-writing stage to also take effect in the writing stage. self._original_get_doctree = self.env.get_doctree self.env.get_doctree = self._get_doctree # process the document structure of the master document, allowing: # - populating a publish order to ensure parent pages are created first # (when using hierarchy mode) # - squash pages which exceed maximum depth (if configured with a max # depth value) self.process_tree_structure(ordered_docnames, self.config.master_doc, traversed) # track relations between accepted documents # # Prepares a relation mapping between each non-orphan documents which # can be used by navigational elements. prevdoc = ordered_docnames[0] if ordered_docnames else None for docname in ordered_docnames[1:]: self.nav_prev[docname] = self.get_relative_uri(docname, prevdoc) self.nav_next[prevdoc] = self.get_relative_uri(prevdoc, docname) prevdoc = docname # add orphans (if any) to the publish list ordered_docnames.extend(x for x in docnames if x not in traversed) for docname in ordered_docnames: doctree = self.env.get_doctree(docname) # acquire title from override (if any), or parse first title entity if (self.config.confluence_title_overrides and docname in self.config.confluence_title_overrides): doctitle = self.config.confluence_title_overrides[docname] else: doctitle = self._parse_doctree_title(docname, doctree) # only register title/track for publishing if there is a title # value that can be applied to this document if doctitle: secnumbers = self.env.toc_secnumbers.get(docname, {}) if self.add_secnumbers and secnumbers.get(''): doctitle = ('.'.join(map(str, secnumbers[''])) + self.secnumber_suffix + doctitle) doctitle = ConfluenceState.registerTitle( docname, doctitle, self.config) # only publish documents that sphinx asked to prepare if docname in docnames: self.publish_docnames.append(docname) # track the toctree depth for a document, which a translator can # use as a hint when dealing with max-depth capabilities toctree = first(doctree.traverse(addnodes.toctree)) if toctree and toctree.get('maxdepth') > 0: ConfluenceState.registerToctreeDepth(docname, toctree.get('maxdepth')) # register title targets for references self._register_doctree_title_targets(docname, doctree) # post-prepare a ready doctree self._prepare_doctree_writing(docname, doctree) # Scan for assets that may exist in the documents to be published. This # will find most if not all assets in the documentation set. The # exception is assets which may be finalized during a document's post # transformation stage (e.g. embedded images are converted into real # images in Sphinx, which is then provided to a translator). Embedded # images are detected during an 'doctree-resolved' hook (see __init__). self.assets.process(ordered_docnames) def _prepare_doctree_writing(self, docname, doctree): # extract metadata information self._extract_metadata(docname, doctree) # replace inheritance diagram with images # (always invoke before _replace_graphviz_nodes) self._replace_inheritance_diagram(doctree) # replace graphviz nodes with images self._replace_graphviz_nodes(doctree) # replace math blocks with images self._replace_math_blocks(doctree) # for every doctree, pick the best image candidate self.post_process_images(doctree) def process_tree_structure(self, ordered, docname, traversed, depth=0): omit = False max_depth = self.config.confluence_max_doc_depth if max_depth is not None and depth > max_depth: omit = True self.omitted_docnames.append(docname) if not omit: ordered.append(docname) modified = False doctree = self.env.get_doctree(docname) for toctreenode in doctree.traverse(addnodes.toctree): if not omit and max_depth is not None: if (toctreenode['maxdepth'] == -1 or depth + toctreenode['maxdepth'] > max_depth): new_depth = max_depth - depth assert new_depth >= 0 toctreenode['maxdepth'] = new_depth movednodes = [] for child in toctreenode['includefiles']: if child not in traversed: ConfluenceState.registerParentDocname(child, docname) traversed.append(child) children = self.process_tree_structure( ordered, child, traversed, depth + 1) if children: movednodes.append(children) self._fix_std_labels(child, docname) if movednodes: modified = True toctreenode.replace_self(movednodes) toctreenode.parent['classes'].remove('toctree-wrapper') if omit: container = addnodes.start_of_file(docname=docname) container.children = doctree.children return container elif modified: self.env.resolve_references(doctree, docname, self) def write_doc(self, docname, doctree): if docname in self.omitted_docnames: return if self.prev_next_loc in ('top', 'both'): navnode = self._build_navigation_node(docname) if navnode: navnode.top = True doctree.insert(0, navnode) if self.prev_next_loc in ('bottom', 'both'): navnode = self._build_navigation_node(docname) if navnode: navnode.bottom = True doctree.append(navnode) self.secnumbers = self.env.toc_secnumbers.get(docname, {}) self.fignumbers = self.env.toc_fignumbers.get(docname, {}) # remove title from page contents (if any) if self.config.confluence_remove_title: title_element = self._find_title_element(doctree) if title_element: # If the removed title is referenced to from within the same # document (i.e. a local table of contents entry), flag any # references pointing to it as a "top" (anchor) reference. This # can be used later in a translator to hint at what type of link # to build. ids = [] if 'ids' in title_element: ids.extend(title_element['ids']) parent = title_element.parent if isinstance(parent, nodes.section) and 'ids' in parent: ids.extend(parent['ids']) if ids: for node in doctree.traverse(nodes.reference): if 'refid' in node and node['refid']: if node['refid'] in ids: node['top-reference'] = True break title_element.parent.remove(title_element) # This method is taken from TextBuilder.write_doc() # with minor changes to support :confval:`rst_file_transform`. destination = StringOutput(encoding='utf-8') self.writer.write(doctree, destination) outfilename = path.join(self.outdir, self.file_transform(docname)) if self.writer.output is not None: ensuredir(path.dirname(outfilename)) try: with io.open(outfilename, 'w', encoding='utf-8') as file: if self.writer.output: file.write(self.writer.output) except (IOError, OSError) as err: ConfluenceLogger.warn("error writing file " "%s: %s" % (outfilename, err)) def publish_doc(self, docname, output): conf = self.config title = ConfluenceState.title(docname) parent_id = None if self.config.master_doc and self.config.confluence_page_hierarchy: if self.config.master_doc != docname: parent = ConfluenceState.parentDocname(docname) parent_id = ConfluenceState.uploadId(parent) if not parent_id: parent_id = self.parent_id data = { 'content': output, 'labels': [], } if self.config.confluence_global_labels: data['labels'].extend(self.config.confluence_global_labels) metadata = self.metadata[docname] if 'labels' in metadata: data['labels'].extend([v for v in metadata['labels']]) uploaded_id = self.publisher.storePage(title, data, parent_id) ConfluenceState.registerUploadId(docname, uploaded_id) if self.config.master_doc == docname: self.master_doc_page_id = uploaded_id if conf.confluence_purge and self.legacy_pages is None: if conf.confluence_purge_from_master and self.master_doc_page_id: baseid = self.master_doc_page_id else: baseid = self.parent_id # if no base identifier and dry running, ignore legeacy page # searching as there is no initial master document to reference # against if (conf.confluence_purge_from_master and conf.confluence_publish_dryrun and not baseid): self.legacy_pages = [] elif self.config.confluence_adv_aggressive_search is True: self.legacy_pages = self.publisher.getDescendantsCompat(baseid) else: self.legacy_pages = self.publisher.getDescendants(baseid) # only populate a list of possible legacy assets when a user is # configured to check or push assets to the target space asset_override = conf.confluence_asset_override if asset_override is None or asset_override: for legacy_page in self.legacy_pages: attachments = self.publisher.getAttachments(legacy_page) self.legacy_assets[legacy_page] = attachments if conf.confluence_purge: if uploaded_id in self.legacy_pages: self.legacy_pages.remove(uploaded_id) def publish_asset(self, key, docname, output, type, hash): conf = self.config publisher = self.publisher title = ConfluenceState.title(docname) page_id = ConfluenceState.uploadId(docname) if not page_id: # A page identifier may not be tracked in cases where only a subset # of documents are published and the target page an asset will be # published to was not part of the request. In this case, ask the # Confluence instance what the target page's identifier is. page_id, _ = publisher.getPage(title) if page_id: ConfluenceState.registerUploadId(docname, page_id) else: ConfluenceLogger.warn('cannot publish asset since publishing ' 'point cannot be found ({}): {}'.format( key, docname)) return if conf.confluence_asset_override is None: # "automatic" management -- check if already published; if not, push attachment_id = publisher.storeAttachment(page_id, key, output, type, hash) elif conf.confluence_asset_override: # forced publishing of the asset attachment_id = publisher.storeAttachment(page_id, key, output, type, hash, force=True) if attachment_id and conf.confluence_purge: if page_id in self.legacy_assets: legacy_asset_info = self.legacy_assets[page_id] if attachment_id in legacy_asset_info: legacy_asset_info.pop(attachment_id, None) def publish_finalize(self): if self.master_doc_page_id: if self.config.confluence_master_homepage is True: self.info('updating space\'s homepage... ', nonl=True) self.publisher.updateSpaceHome(self.master_doc_page_id) self.info('done\n') if self.cloud: point_url = '{0}spaces/{1}/pages/{2}' else: point_url = '{0}pages/viewpage.action?pageId={2}' self.info('Publish point: ' + point_url.format( self.config.confluence_server_url, self.config.confluence_space_name, self.master_doc_page_id)) def publish_purge(self): if self.config.confluence_purge: if self.publish_allowlist or self.publish_denylist: self.warn( 'confluence_purge disabled due to ' 'confluence_publish_allowlist/confluence_publish_denylist') return if self.legacy_pages: n = len(self.legacy_pages) self.info('removing legacy pages... (total: {}) '.format(n), nonl=True) for legacy_page_id in self.legacy_pages: self.publisher.removePage(legacy_page_id) # remove any pending assets to remove from the page (as they # are already been removed) self.legacy_assets.pop(legacy_page_id, None) self.info('done\n') n = 0 for legacy_asset_info in self.legacy_assets.values(): n += len(legacy_asset_info.keys()) if n > 0: self.info('removing legacy assets... (total: {}) '.format(n), nonl=True) for legacy_asset_info in self.legacy_assets.values(): for id in legacy_asset_info.keys(): self.publisher.removeAttachment(id) self.info('done\n') def finish(self): # restore environment's get_doctree if it was temporarily replaced if self._original_get_doctree: self.env.get_doctree = self._original_get_doctree if self.publish: self.legacy_assets = {} self.legacy_pages = None self.parent_id = self.publisher.getBasePageId() for docname in status_iterator(self.publish_docnames, 'publishing documents... ', length=len(self.publish_docnames), verbosity=self.app.verbosity): if self._check_publish_skip(docname): self.verbose(docname + ' skipped due to configuration') continue docfile = path.join(self.outdir, self.file_transform(docname)) try: with io.open(docfile, 'r', encoding='utf-8') as file: output = file.read() self.publish_doc(docname, output) except (IOError, OSError) as err: ConfluenceLogger.warn("error reading file %s: " "%s" % (docfile, err)) def to_asset_name(asset): return asset[0] assets = self.assets.build() for asset in status_iterator(assets, 'publishing assets... ', length=len(assets), verbosity=self.app.verbosity, stringify_func=to_asset_name): key, absfile, type, hash, docname = asset if self._check_publish_skip(docname): self.verbose(key + ' skipped due to configuration') continue try: with open(absfile, 'rb') as file: output = file.read() self.publish_asset(key, docname, output, type, hash) except (IOError, OSError) as err: ConfluenceLogger.warn("error reading asset %s: " "%s" % (key, err)) self.publish_purge() self.publish_finalize() self.info('building intersphinx... ', nonl=True) build_intersphinx(self) self.info('done\n') def cleanup(self): if self.publish: self.publisher.disconnect() def _build_navigation_node(self, docname): """ build a navigation node for a document Requests to build a navigation node for a provided document name. If this document has no navigational hints to apply, this method will return a `None` value. Args: docname: the document name Returns: returns a navigation node; ``None`` if no nav-node for document """ if docname not in self.nav_prev and docname not in self.nav_next: return None navnode = ConfluenceNavigationNode() if docname in self.nav_prev: prev_label = '← ' + __('Previous') reference = nodes.reference(prev_label, prev_label, internal=True, refuri=self.nav_prev[docname]) reference._navnode = True reference._navnode_next = False reference._navnode_previous = True navnode.append(reference) if docname in self.nav_next: next_label = __('Next') + ' →' reference = nodes.reference(next_label, next_label, internal=True, refuri=self.nav_next[docname]) reference._navnode = True reference._navnode_next = True reference._navnode_previous = False navnode.append(reference) return navnode def _check_publish_skip(self, docname): """ check publishing should be skipped for the provided docname A runner's configuration may have an explicit list of docnames to either allow or deny publishing. Check if the provided docname has been flagged to be skipped. Args: docname: the docname to check """ if self.publish_denylist and docname in self.publish_denylist: return True if self.publish_allowlist and docname not in self.publish_allowlist: return True return False def _extract_metadata(self, docname, doctree): """ extract metadata from a document Documents may define metadata information which can be used during a publication event. When processing a doctree, strip out the metadata information and save it for when a publish event occurs. Args: docname: the document doctree: the doctree to extract metadata from """ metadata = self.metadata.setdefault(docname, {}) for node in doctree.traverse(confluence_metadata): labels = metadata.setdefault('labels', []) labels.extend(node['params']['labels']) node.parent.remove(node) def _find_title_element(self, doctree): """ find (if any) the title element of a document From a provided document's doctree, attempt to extract a possible title value from known information. This call will look for the first section node's title value as the title value for a document. Args: doctree: the document tree to find a title value element on Returns: the title element """ node = doctree.next_node(nodes.section) if isinstance(node, nodes.section): return node.next_node(nodes.title) return None def _fix_std_labels(self, olddocname, newdocname): """ fix standard domain labels for squashed documents When Sphinx resolves references for a doctree ('resolve_references'), the standard domain's internal labels are used to map references to target documents. To support document squashing (aka. max depth pages), this utility method helps override a document's tuple labels so that any squashed page's labels can be moved into a parent document's label set. """ # see also: sphinx/domains/std.py std_domain = self.env.get_domain('std') try: citation_domain = self.env.get_domain('citation') except ExtensionError: citation_domain = None if 'anonlabels' in std_domain.data: anonlabels = std_domain.data['anonlabels'] for key, (fn, _l) in list(anonlabels.items()): if fn == olddocname: data = anonlabels[key] anonlabels[key] = newdocname, data[1] citations = None if 'citations' in std_domain.data: # Sphinx <2.1 citations = std_domain.data['citations'] elif citation_domain: # Sphinx >=2.1 citations = citation_domain.citations if citations: for key, (fn, _l, lineno) in list(citations.items()): if fn == olddocname: data = citations[key] citations[key] = newdocname, data[1], data[2] if 'citation_refs' in std_domain.data: # Sphinx <2.0 citation_refs = std_domain.data['citation_refs'] for key, docnames in list(citation_refs.items()): if fn == olddocname: data = citation_refs[key] citation_refs[key] = newdocname if 'labels' in std_domain.data: labels = std_domain.data['labels'] for key, (fn, _l, _l) in list(labels.items()): if fn == olddocname: data = labels[key] labels[key] = newdocname, data[1], data[2] if 'objects' in std_domain.data: objects = std_domain.data['objects'] for key, (fn, _l) in list(objects.items()): if fn == olddocname: data = objects[key] objects[key] = newdocname, data[1] if 'progoptions' in std_domain.data: progoptions = std_domain.data['progoptions'] for key, (fn, _l) in list(progoptions.items()): if fn == olddocname: data = progoptions[key] progoptions[key] = newdocname, data[1] def _get_doctree(self, docname): """ override 'get_doctree' method To support document squashing (aka. max depth pages), doctree's may be loaded and manipulated before the writing stage. Normally, the writing stage will load target doctree's from their source so there is no way to pre-load and pass a document's doctree into the writing stage. To overcome this, this extension hooks into the environment's 'get_doctree' method and caches loaded document's doctree's into a map. """ if docname not in self.cache_doctrees: self.cache_doctrees[docname] = self._original_get_doctree(docname) return self.cache_doctrees[docname] def _register_doctree_title_targets(self, docname, doctree): """ register title targets for a doctree Compiles a list of title targets which references can link against. This tracked expected targets for sections which are automatically generated in a rendered Confluence instance. Args: docname: the docname of the doctree doctree: the doctree to search for targets """ doc_used_names = {} secnumbers = self.env.toc_secnumbers.get(docname, {}) for node in doctree.traverse(nodes.title): if isinstance(node.parent, nodes.section): section_node = node.parent if 'ids' in section_node: target = ''.join(node.astext().split()) # when confluence has a header that contains a link, the # automatically assigned identifier removes any underscores if node.next_node(addnodes.pending_xref): target = target.replace('_', '') if self.add_secnumbers: anchorname = '#' + target if anchorname not in secnumbers: anchorname = '' secnumber = secnumbers.get(anchorname) if secnumber: target = ('.'.join(map(str, secnumber)) + self.secnumber_suffix + target) section_id = doc_used_names.get(target, 0) doc_used_names[target] = section_id + 1 if section_id > 0: target = '{}.{}'.format(target, section_id) for id in section_node['ids']: id = '{}#{}'.format(docname, id) ConfluenceState.registerTarget(id, target) def _replace_graphviz_nodes(self, doctree): """ replace graphviz nodes with images graphviz nodes are pre-processed and replaced with respective images in the processed documentation set. Typically, the node support from `sphinx.ext.graphviz` would be added to the builder; however, this extension renders graphs during the translation phase (which is not ideal for how assets are managed in this extension). Instead, this implementation just traverses for graphviz nodes, generates renderings and replaces the nodes with image nodes (which in turn will be handled by the existing image-based implementation). Args: doctree: the doctree to replace blocks on """ if graphviz is None: return # graphviz's render_dot call expects a translator to be passed in; mock # a translator tied to our self-builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(self) for node in doctree.traverse(graphviz): try: _, out_filename = render_dot(mock_translator, node['code'], node['options'], self.graphviz_output_format, 'graphviz') if not out_filename: node.parent.remove(node) continue new_node = nodes.image(candidates={'?'}, uri=out_filename) if 'align' in node: new_node['align'] = node['align'] node.replace_self(new_node) except GraphvizError as exc: ConfluenceLogger.warn('dot code {}: {}'.format( node['code'], exc)) node.parent.remove(node) def _replace_inheritance_diagram(self, doctree): """ replace inheritance diagrams with images Inheritance diagrams are pre-processed and replaced with respective images in the processed documentation set. Typically, the node support from `sphinx.ext.inheritance_diagram` would be added to the builder; however, this extension renders graphs during the translation phase (which is not ideal for how assets are managed in this extension). Instead, this implementation just traverses for inheritance diagrams, generates renderings and replaces the nodes with image nodes (which in turn will be handled by the existing image-based implementation). Note that the interactive image map is not handled in this implementation since Confluence does not support image maps (without external extensions). Args: doctree: the doctree to replace blocks on """ if inheritance_diagram is None: return # graphviz's render_dot call expects a translator to be passed in; mock # a translator tied to our self-builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(self) for node in doctree.traverse(inheritance_diagram.inheritance_diagram): graph = node['graph'] graph_hash = inheritance_diagram.get_graph_hash(node) name = 'inheritance%s' % graph_hash dotcode = graph.generate_dot(name, {}, env=self.env) try: _, out_filename = render_dot(mock_translator, dotcode, {}, self.graphviz_output_format, 'inheritance') if not out_filename: node.parent.remove(node) continue new_node = nodes.image(candidates={'?'}, uri=out_filename) if 'align' in node: new_node['align'] = node['align'] node.replace_self(new_node) except GraphvizError as exc: ConfluenceLogger.warn('dot code {}: {}'.format(dotcode, exc)) node.parent.remove(node) def _replace_math_blocks(self, doctree): """ replace math blocks with images Math blocks are pre-processed and replaced with respective images in the list of documents to process. This is to help prepare additional images into the asset management for this extension. Math support will work on systems which have latex/dvipng installed. Args: doctree: the doctree to replace blocks on """ if imgmath is None: return # imgmath's render_math call expects a translator to be passed # in; mock a translator tied to our self-builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(self) for node in itertools.chain(doctree.traverse(nodes.math), doctree.traverse(nodes.math_block)): try: mf, _ = imgmath.render_math(mock_translator, '$' + node.astext() + '$') if not mf: continue new_node = nodes.image(candidates={'?'}, uri=path.join(self.outdir, mf)) if not isinstance(node, nodes.math): new_node['align'] = 'center' node.replace_self(new_node) except imgmath.MathExtError as exc: ConfluenceLogger.warn('inline latex {}: {}'.format( node.astext(), exc)) def _parse_doctree_title(self, docname, doctree): """ parse a doctree for a raw title value Examine a document's doctree value to find a title value from a title section element. If no title is found, a title can be automatically generated (if configuration permits) or a `None` value is returned. """ doctitle = None title_element = self._find_title_element(doctree) if title_element: doctitle = title_element.astext() if not doctitle: if not self.config.confluence_disable_autogen_title: doctitle = "autogen-{}".format(docname) if self.publish: ConfluenceLogger.warn( "document will be published using an " "generated title value: {}".format(docname)) elif self.publish: ConfluenceLogger.warn( "document will not be published since it " "has no title: {}".format(docname)) return doctitle
def report_main(args_parser): """ report mainline The mainline for the 'report' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('--full-config', '-C', action='store_true') args_parser.add_argument('--no-sanitize', action='store_true') args_parser.add_argument('--offline', action='store_true') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) rv = 0 offline = args.offline work_dir = args.work_dir if args.work_dir else os.getcwd() # setup sphinx engine to extract configuration config = {} configuration_load_issue = None confluence_instance_info = None publisher = ConfluencePublisher() try: with temp_dir() as tmp_dir: with docutils_namespace(): print('fetching configuration information...') builder = ConfluenceReportBuilder.name app = Sphinx( work_dir, # document sources work_dir, # directory with configuration tmp_dir, # output for built documents tmp_dir, # output for doctree files builder, # builder to execute status=sys.stdout, # sphinx status output warning=sys.stderr) # sphinx warning output if app.config.confluence_publish: try: process_ask_configs(app.config) except ConfluenceConfigurationError: offline = True # extract configuration information for k, v in app.config.values.items(): raw = getattr(app.config, k) if raw is None: continue if callable(raw): value = '(callable)' else: value = raw prefixes = ( 'confluence_', 'singleconfluence_', ) if not args.full_config and not k.startswith(prefixes): continue # always extract some known builder configurations if args.full_config and k.startswith(IGNORE_BUILDER_CONFS): continue config[k] = value # initialize the publisher (if needed later) publisher.init(app.config) except Exception: sys.stdout.flush() tb_msg = traceback.format_exc() logger.error(tb_msg) if os.path.isfile(os.path.join(work_dir, 'conf.py')): configuration_load_issue = 'unable to load configuration' configuration_load_issue += '\n\n' + tb_msg.strip() else: configuration_load_issue = 'no documentation/missing configuration' rv = 1 # attempt to fetch confluence instance version confluence_publish = config.get('confluence_publish') confluence_server_url = config.get('confluence_server_url') if not offline and confluence_publish and confluence_server_url: base_url = ConfluenceUtil.normalize_base_url(confluence_server_url) info = '' session = None try: print('connecting to confluence instance...') sys.stdout.flush() publisher.connect() info += ' connected: yes\n' session = publisher.rest_client.session except Exception: sys.stdout.flush() logger.error(traceback.format_exc()) info += ' connected: no\n' rv = 1 if session: try: # fetch print('fetching confluence instance information...') manifest_url = base_url + MANIFEST_PATH rsp = session.get(manifest_url) if rsp.status_code == 200: info += ' fetched: yes\n' # extract print('decoding information...') rsp.encoding = 'utf-8' raw_data = rsp.text info += ' decoded: yes\n' # parse print('parsing information...') xml_data = ElementTree.fromstring(raw_data) info += ' parsed: yes\n' root = ElementTree.ElementTree(xml_data) for o in root.findall('typeId'): info += ' type: ' + o.text + '\n' for o in root.findall('version'): info += ' version: ' + o.text + '\n' for o in root.findall('buildNumber'): info += ' build: ' + o.text + '\n' else: logger.error('bad response from server ({})'.format( rsp.status_code)) info += ' fetched: error ({})\n'.format(rsp.status_code) rv = 1 except Exception: sys.stdout.flush() logger.error(traceback.format_exc()) info += 'failure to determine confluence data\n' rv = 1 confluence_instance_info = info def sensitive_config(key): if key in config: if config[key]: config[key] = '(set)' else: config[key] = '(set; empty)' # always sanitize out sensitive information sensitive_config('confluence_client_cert_pass') sensitive_config('confluence_publish_headers') sensitive_config('confluence_publish_token') sensitive_config('confluence_server_pass') # optional sanitization if not args.no_sanitize: sensitive_config('author') sensitive_config('confluence_client_cert') sensitive_config('confluence_global_labels') sensitive_config('confluence_jira_servers') sensitive_config('confluence_mentions') sensitive_config('confluence_parent_page') sensitive_config('confluence_parent_page_id_check') sensitive_config('confluence_proxy') sensitive_config('confluence_publish_root') sensitive_config('confluence_server_auth') sensitive_config('confluence_server_cookies') sensitive_config('confluence_server_user') sensitive_config('project') # remove confluence instance (attempt to keep scheme) if 'confluence_server_url' in config: value = config['confluence_server_url'] parsed = urlparse(value) if parsed.scheme: value = parsed.scheme + '://<removed>' else: value = '(set; no scheme)' if parsed.netloc and parsed.netloc.endswith('atlassian.net'): value += ' (cloud)' config['confluence_server_url'] = value # remove space key, but track casing space_cfgs = [ 'confluence_space_key', 'confluence_space_name', # deprecated ] for space_cfg in space_cfgs: if space_cfg not in config: continue value = config[space_cfg] if value.startswith('~'): value = '(set; user)' elif value.isupper(): value = '(set; upper)' elif value.islower(): value = '(set; lower)' else: value = '(set; mixed)' config[space_cfg] = value print('') print('Confluence builder report has been generated.') print('Please copy the following text for the GitHub issue:') print('') logger.note('------------[ cut here ]------------') print('```') print('(system)') print(' platform:', single_line_version(platform.platform())) print(' python:', single_line_version(sys.version)) print(' sphinx:', single_line_version(sphinx_version)) print(' requests:', single_line_version(requests_version)) print(' builder:', single_line_version(scb_version)) print('') print('(configuration)') if config: for k, v in OrderedDict(sorted(config.items())).items(): print('{}: {}'.format(k, v)) else: print('~default configuration~') if configuration_load_issue: print('') print('(error loading configuration)') print(configuration_load_issue) if confluence_instance_info: print('') print('(confluence instance)') print(confluence_instance_info.rstrip()) print('```') logger.note('------------[ cut here ]------------') return rv
class ConfluenceBuilder(Builder): allow_parallel = True default_translator_class = ConfluenceStorageFormatTranslator name = 'confluence' format = 'confluence_storage' supported_image_types = ConfluenceSupportedImages() supported_remote_images = True def __init__(self, app): super(ConfluenceBuilder, self).__init__(app) self.cache_doctrees = {} self.cloud = False self.domain_indices = {} self.file_suffix = '.conf' self.info = ConfluenceLogger.info self.link_suffix = None self.metadata = defaultdict(dict) self.nav_next = {} self.nav_prev = {} self.omitted_docnames = [] self.publish_allowlist = [] self.publish_denylist = [] self.publish_docnames = [] self.publisher = ConfluencePublisher() self.root_doc_page_id = None self.secnumbers = {} self.state = ConfluenceState self.use_index = None self.use_search = None self.verbose = ConfluenceLogger.verbose self.warn = ConfluenceLogger.warn self._cached_footer_data = None self._cached_header_data = None self._original_get_doctree = None self._verbose = self.app.verbosity # state tracking is set at initialization (not cleanup) so its content's # can be checked/validated on after the builder has executed (testing) self.state.reset() def init(self): validate_configuration(self) apply_defaults(self.config) config = self.config self.add_secnumbers = self.config.confluence_add_secnumbers self.secnumber_suffix = self.config.confluence_secnumber_suffix if self.name != 'singleconfluence': self.use_index = config.confluence_use_index self.use_search = config.confluence_include_search if self.config.confluence_additional_mime_types: for type_ in self.config.confluence_additional_mime_types: self.supported_image_types.register(type_) if 'graphviz_output_format' in self.config: self.graphviz_output_format = self.config['graphviz_output_format'] else: self.graphviz_output_format = 'png' if self.config.confluence_publish: process_ask_configs(self.config) old_url = self.config.confluence_server_url new_url = ConfluenceUtil.normalize_base_url(old_url) if old_url != new_url: self.warn('normalizing confluence url from ' '{} to {} '.format(old_url, new_url)) self.config.confluence_server_url = new_url # detect if Confluence Cloud if using the Atlassian domain if new_url: self.cloud = new_url.endswith('.atlassian.net/wiki/') self.assets = ConfluenceAssetManager(config, self.env, self.outdir) self.writer = ConfluenceWriter(self) self.config.sphinx_verbosity = self._verbose self.publisher.init(self.config, self.cloud) self.create_template_bridge() self.templates.init(self) if self.config.confluence_file_suffix is not None: self.file_suffix = self.config.confluence_file_suffix if self.config.confluence_link_suffix is not None: self.link_suffix = self.config.confluence_link_suffix elif self.link_suffix is None: self.link_suffix = self.file_suffix # Function to convert the docname to a reST file name. def file_transform(docname): return docname + self.file_suffix # Function to convert the docname to a relative URI. def link_transform(docname): return docname + self.link_suffix if self.config.confluence_file_transform is not None: self.file_transform = self.config.confluence_file_transform else: self.file_transform = file_transform if self.config.confluence_link_transform is not None: self.link_transform = self.config.confluence_link_transform else: self.link_transform = link_transform if self.config.confluence_lang_transform is not None: self.lang_transform = self.config.confluence_lang_transform else: self.lang_transform = None if self.config.confluence_publish: self.publish = True self.publisher.connect() else: self.publish = False def prepare_subset(option): value = getattr(config, option) if not value: return None # if provided via command line, treat as a list if option in config['overrides'] and isinstance(value, basestring): value = value.split(',') if isinstance(value, basestring): files = extract_strings_from_file(value) else: files = value return set(files) if files else None self.publish_allowlist = prepare_subset('confluence_publish_allowlist') self.publish_denylist = prepare_subset('confluence_publish_denylist') def get_outdated_docs(self): """ Return an iterable of input files that are outdated. """ # This method is taken from TextBuilder.get_outdated_docs() # with minor changes to support :confval:`rst_file_transform`. for docname in self.env.found_docs: if docname not in self.env.all_docs: yield docname continue sourcename = path.join(self.env.srcdir, docname + self.file_suffix) targetname = path.join(self.outdir, self.file_transform(docname)) try: targetmtime = path.getmtime(targetname) except Exception: targetmtime = 0 try: srcmtime = path.getmtime(sourcename) if srcmtime > targetmtime: yield docname except EnvironmentError: # source doesn't exist anymore pass def get_target_uri(self, docname, typ=None): return self.link_transform(docname) def prepare_writing(self, docnames): ordered_docnames = [] traversed = [self.config.root_doc] # default enable special document names if they are references in the # official docnames list if self.use_index is None and 'genindex' in docnames: self.use_index = True if self.use_search is None and 'search' in docnames: self.use_search = True # generate domain index information self.domain_indices = {} indices_config = self.config.confluence_domain_indices if indices_config and self.name != 'singleconfluence': for domain_name in sorted(self.env.domains): domain = self.env.domains[domain_name] for indexcls in domain.indices: indexname = '%s-%s' % (domain.name, indexcls.name) if isinstance(indices_config, list): if indexname not in indices_config: continue content, _ = indexcls(domain).generate() if content: self.domain_indices[indexname] = (indexcls, content) # prepare caching doctree hook # # We'll temporarily override the environment's 'get_doctree' method to # allow this extension to manipulate the doctree for a document inside # the pre-writing stage to also take effect in the writing stage. self._original_get_doctree = self.env.get_doctree self.env.get_doctree = self._get_doctree # process the document structure of the root document, allowing: # - populating a publish order to ensure parent pages are created first # (when using hierarchy mode) # - squash pages which exceed maximum depth (if configured with a max # depth value) self.process_tree_structure(ordered_docnames, self.config.root_doc, traversed) # add orphans (if any) to the publish list ordered_docnames.extend(x for x in docnames if x not in traversed) for docname in ordered_docnames: doctree = self.env.get_doctree(docname) # acquire title from override (if any), or parse first title entity if (self.config.confluence_title_overrides and docname in self.config.confluence_title_overrides): doctitle = self.config.confluence_title_overrides[docname] else: doctitle = self._parse_doctree_title(docname, doctree) # only register title/track for publishing if there is a title # value that can be applied to this document if doctitle: secnumbers = self.env.toc_secnumbers.get(docname, {}) if self.add_secnumbers and secnumbers.get(''): doctitle = ('.'.join(map(str, secnumbers[''])) + self.secnumber_suffix + doctitle) self.state.register_title(docname, doctitle, self.config) # only publish documents that sphinx asked to prepare if docname in docnames: self.publish_docnames.append(docname) # track the toctree depth for a document, which a translator can # use as a hint when dealing with max-depth capabilities toctree = first(doctree.traverse(addnodes.toctree)) if toctree and toctree.get('maxdepth') > 0: self.state.register_toctree_depth(docname, toctree.get('maxdepth')) # register title targets for references self._register_doctree_title_targets(docname, doctree) # post-prepare a ready doctree self._prepare_doctree_writing(docname, doctree) # register titles for special documents (if needed); if a title is not # already set from a placeholder document, configure a default title if self.use_index and not self.state.title('genindex'): self.state.register_title('genindex', SL('Index'), self.config) if self.use_search and not self.state.title('search'): self.state.register_title('search', SL('Search'), self.config) if self.domain_indices: for indexname, indexdata in self.domain_indices.items(): if self.state.title(indexname): continue indexcls, _ = indexdata title = indexcls.localname self.state.register_title(indexname, title, self.config) # track relations between accepted documents # # Prepares a relation mapping between each non-orphan documents which # can be used by navigational elements. If any documents are special # pages, ignore them for consideration in the navigational bar. nav_docnames = list(ordered_docnames) if self.use_index and 'genindex' in nav_docnames: nav_docnames.remove('genindex') if self.use_search and 'search' in nav_docnames: nav_docnames.remove('search') if self.domain_indices: for indexname, indexdata in self.domain_indices.items(): if indexname in nav_docnames: nav_docnames.remove(indexname) navdocs_transform = self.config.confluence_navdocs_transform if navdocs_transform: nav_docnames = navdocs_transform(self, nav_docnames) prevdoc = nav_docnames[0] if nav_docnames else None for docname in nav_docnames[1:]: self.nav_prev[docname] = self.get_relative_uri(docname, prevdoc) self.nav_next[prevdoc] = self.get_relative_uri(prevdoc, docname) prevdoc = docname # register labels for special documents (if needed) labels = self.env.domaindata['std']['labels'] anonlabels = self.env.domaindata['std']['anonlabels'] if self.use_index: anonlabels['genindex'] = 'genindex', '' labels['genindex'] = 'genindex', '', '' if self.use_search: anonlabels['search'] = 'search', '' labels['search'] = 'search', '', '' if self.domain_indices: for indexname, _ in self.domain_indices.items(): anonlabels[indexname] = indexname, '' labels[indexname] = indexname, '', '' # Scan for assets that may exist in the documents to be published. This # will find most if not all assets in the documentation set. The # exception is assets which may be finalized during a document's post # transformation stage (e.g. embedded images are converted into real # images in Sphinx, which is then provided to a translator). Embedded # images and other late-injected assets are processed in a translator # when needed. if self.name != 'singleconfluence': self.assets.process(ordered_docnames) def _prepare_doctree_writing(self, docname, doctree): # extract metadata information self._extract_metadata(docname, doctree) # convert any desired nodes in a doctree to node types supported by the # translator implementation doctree_transmute(self, doctree) # for every doctree, pick the best image candidate self.post_process_images(doctree) def process_tree_structure(self, ordered, docname, traversed, depth=0): omit = False max_depth = self.config.confluence_max_doc_depth if max_depth is not None and depth > max_depth: omit = True self.omitted_docnames.append(docname) if not omit: ordered.append(docname) modified = False doctree = self.env.get_doctree(docname) for toctreenode in doctree.traverse(addnodes.toctree): if not omit and max_depth is not None: if (toctreenode['maxdepth'] == -1 or depth + toctreenode['maxdepth'] > max_depth): new_depth = max_depth - depth assert new_depth >= 0 toctreenode['maxdepth'] = new_depth movednodes = [] for child in toctreenode['includefiles']: if child not in traversed: self.state.register_parent_docname(child, docname) traversed.append(child) children = self.process_tree_structure( ordered, child, traversed, depth + 1) if children: movednodes.append(children) self._fix_std_labels(child, docname) if movednodes: modified = True toctreenode.replace_self(movednodes) toctreenode.parent['classes'].remove('toctree-wrapper') if omit: container = addnodes.start_of_file(docname=docname) container.children = doctree.children return container elif modified: self.env.resolve_references(doctree, docname, self) def write_doc(self, docname, doctree): if docname in self.omitted_docnames: return self._header_footer_init(docname, doctree) self.secnumbers = self.env.toc_secnumbers.get(docname, {}) self.fignumbers = self.env.toc_fignumbers.get(docname, {}) # remove title from page contents (if any) if self.config.confluence_remove_title: title_element = self._find_title_element(doctree) if title_element: # If the removed title is referenced to from within the same # document (i.e. a local table of contents entry), flag any # references pointing to it as a "top" (anchor) reference. This # can be used later in a translator to hint at what type of link # to build. ids = [] if 'ids' in title_element: ids.extend(title_element['ids']) parent = title_element.parent if isinstance(parent, nodes.section) and 'ids' in parent: ids.extend(parent['ids']) if ids: for node in doctree.traverse(nodes.reference): if 'refid' in node and node['refid']: top_ref = node['refid'] in ids # allow a derived class to hint if this is a #top # reference node if not top_ref: top_ref = self._top_ref_check(node) if top_ref: node['top-reference'] = True break title_element.parent.remove(title_element) # This method is taken from TextBuilder.write_doc() # with minor changes to support :confval:`rst_file_transform`. destination = StringOutput(encoding='utf-8') self.writer.write(doctree, destination) outfilename = path.join(self.outdir, self.file_transform(docname)) if self.writer.output is not None: ensuredir(path.dirname(outfilename)) try: with io.open(outfilename, 'w', encoding='utf-8') as file: if self.writer.output: file.write(self.writer.output) except (IOError, OSError) as err: self.warn('error writing file %s: %s' % (outfilename, err)) def publish_doc(self, docname, output): conf = self.config title = self.state.title(docname) is_root_doc = self.config.root_doc == docname parent_id = None if self.config.root_doc and self.config.confluence_page_hierarchy: if self.config.root_doc != docname: parent = self.state.parent_docname(docname) parent_id = self.state.upload_id(parent) if not parent_id: parent_id = self.parent_id data = { 'content': output, 'labels': [], } if self.config.confluence_global_labels: data['labels'].extend(self.config.confluence_global_labels) metadata = self.metadata[docname] if 'labels' in metadata: data['labels'].extend([v for v in metadata['labels']]) if conf.confluence_publish_root and is_root_doc: uploaded_id = self.publisher.store_page_by_id( title, conf.confluence_publish_root, data) else: uploaded_id = self.publisher.store_page(title, data, parent_id) self.state.register_upload_id(docname, uploaded_id) if self.config.root_doc == docname: self.root_doc_page_id = uploaded_id # populate ancestors to be used to pre-check ancestors assignments # on new pages (`uploaded_id` may not be set if dry run) if uploaded_id: root_ancestors = self.publisher.get_ancestors(uploaded_id) self.publisher.restrict_ancestors(root_ancestors) # if purging is enabled and we have yet to populate a list of legacy # pages to cache, populate pages in our target scope now if conf.confluence_purge and self.legacy_pages is None: if conf.confluence_publish_root: baseid = conf.confluence_publish_root elif conf.confluence_purge_from_root and self.root_doc_page_id: baseid = self.root_doc_page_id else: baseid = self.parent_id # if no base identifier and dry running, ignore legacy page # searching as there is no initial root document to reference # against if (conf.confluence_purge_from_root and conf.confluence_publish_dryrun and not baseid): self.legacy_pages = [] elif self.config.confluence_adv_aggressive_search is True: self.legacy_pages = self.publisher.get_descendants_compat( baseid) else: self.legacy_pages = self.publisher.get_descendants(baseid) # only populate a list of possible legacy assets when a user is # configured to check or push assets to the target space asset_override = conf.confluence_asset_override if asset_override is None or asset_override: for legacy_page in self.legacy_pages: attachments = self.publisher.get_attachments(legacy_page) self.legacy_assets[legacy_page] = attachments if conf.confluence_purge: if uploaded_id in self.legacy_pages: self.legacy_pages.remove(uploaded_id) def publish_asset(self, key, docname, output, type_, hash_): conf = self.config publisher = self.publisher title = self.state.title(docname) page_id = self.state.upload_id(docname) if not page_id and not conf.confluence_publish_dryrun: # A page identifier may not be tracked in cases where only a subset # of documents are published and the target page an asset will be # published to was not part of the request. In this case, ask the # Confluence instance what the target page's identifier is. page_id, _ = publisher.get_page(title) if page_id: self.state.register_upload_id(docname, page_id) else: self.warn('cannot publish asset since publishing ' 'point cannot be found ({}): {}'.format( key, docname)) return attachment_id = None if conf.confluence_asset_override is None: # "automatic" management -- check if already published; if not, push attachment_id = publisher.store_attachment(page_id, key, output, type_, hash_) elif conf.confluence_asset_override: # forced publishing of the asset attachment_id = publisher.store_attachment(page_id, key, output, type_, hash_, force=True) if attachment_id and conf.confluence_purge: if page_id in self.legacy_assets: legacy_asset_info = self.legacy_assets[page_id] if attachment_id in legacy_asset_info: legacy_asset_info.pop(attachment_id, None) def publish_finalize(self): if self.root_doc_page_id: if self.config.confluence_root_homepage is True: self.info('updating space\'s homepage... ', nonl=(not self._verbose)) self.publisher.update_space_home(self.root_doc_page_id) self.info('done') if self.cloud: point_url = '{0}spaces/{1}/pages/{2}' else: point_url = '{0}pages/viewpage.action?pageId={2}' self.info( 'Publish point: ' + point_url.format(self.config.confluence_server_url, self.config .confluence_space_key, self.root_doc_page_id)) def publish_purge(self): if self.config.confluence_purge: if self.publish_allowlist or self.publish_denylist: self.warn( 'confluence_purge disabled due to ' 'confluence_publish_allowlist/confluence_publish_denylist') return if self.legacy_pages: for legacy_page_id in status_iterator( self.legacy_pages, 'removing legacy pages... ', length=len(self.legacy_pages), verbosity=self._verbose): self.publisher.remove_page(legacy_page_id) # remove any pending assets to remove from the page (as they # are already been removed) self.legacy_assets.pop(legacy_page_id, None) legacy_assets = {} for legacy_asset_info in self.legacy_assets.values(): for attachment_id, attachment_name in legacy_asset_info.items( ): legacy_assets[attachment_id] = attachment_name if legacy_assets: def to_asset_name(attachment_id): return legacy_assets[attachment_id] for attachment_id in status_iterator( legacy_assets.keys(), 'removing legacy assets... ', length=len(legacy_assets.keys()), verbosity=self._verbose, stringify_func=to_asset_name): self.publisher.remove_attachment(attachment_id) def finish(self): # restore environment's get_doctree if it was temporarily replaced if self._original_get_doctree: self.env.get_doctree = self._original_get_doctree # build index if self.use_index: self.info('generating index...', nonl=(not self._verbose)) self._generate_special_document('genindex', generate_storage_format_genindex) if not self._verbose: self.info(' done') # build domain indexes if self.domain_indices: for indexname, indexdata in self.domain_indices.items(): self.info('generating index ({})...'.format(indexname), nonl=(not self._verbose)) self._generate_special_document( indexname, generate_storage_format_domainindex) if not self._verbose: self.info(' done') # build search if self.use_search: self.info('generating search...', nonl=(not self._verbose)) self._generate_special_document('search', generate_storage_format_search) if not self._verbose: self.info(' done') # publish generated output (if desired) if self.publish: self.legacy_assets = {} self.legacy_pages = None self.parent_id = self.publisher.get_base_page_id() for docname in status_iterator(self.publish_docnames, 'publishing documents... ', length=len(self.publish_docnames), verbosity=self._verbose): if self._check_publish_skip(docname): self.verbose(docname + ' skipped due to configuration') continue docfile = path.join(self.outdir, self.file_transform(docname)) try: with io.open(docfile, 'r', encoding='utf-8') as file: output = file.read() self.publish_doc(docname, output) except (IOError, OSError) as err: self.warn('error reading file %s: %s' % (docfile, err)) def to_asset_name(asset): return asset[0] assets = self.assets.build() for asset in status_iterator(assets, 'publishing assets... ', length=len(assets), verbosity=self._verbose, stringify_func=to_asset_name): key, absfile, type_, hash_, docname = asset if self._check_publish_skip(docname): self.verbose(key + ' skipped due to configuration') continue try: with open(absfile, 'rb') as file: output = file.read() self.publish_asset(key, docname, output, type_, hash_) except (IOError, OSError) as err: self.warn('error reading asset %s: %s' % (key, err)) self.publish_purge() self.publish_finalize() self.info('building intersphinx... ', nonl=(not self._verbose)) build_intersphinx(self) self.info('done') def cleanup(self): if self.publish: self.publisher.disconnect() def _check_publish_skip(self, docname): """ check publishing should be skipped for the provided docname A runner's configuration may have an explicit list of docnames to either allow or deny publishing. Check if the provided docname has been flagged to be skipped. Args: docname: the docname to check """ if self.publish_denylist and docname in self.publish_denylist: return True if self.publish_allowlist and docname not in self.publish_allowlist: return True return False def _extract_metadata(self, docname, doctree): """ extract metadata from a document Documents may define metadata information which can be used during a publication event. When processing a doctree, strip out the metadata information and save it for when a publish event occurs. Args: docname: the document doctree: the doctree to extract metadata from """ metadata = self.metadata.setdefault(docname, {}) for node in doctree.traverse(confluence_metadata): labels = metadata.setdefault('labels', []) labels.extend(node.params['labels']) node.parent.remove(node) def _find_title_element(self, doctree): """ find (if any) the title element of a document From a provided document's doctree, attempt to extract a possible title value from known information. This call will look for the first section node's title value as the title value for a document. Args: doctree: the document tree to find a title value element on Returns: the title element """ node = doctree.next_node(nodes.section) if isinstance(node, nodes.section): return node.next_node(nodes.title) return None def _fix_std_labels(self, olddocname, newdocname): """ fix standard domain labels for squashed documents When Sphinx resolves references for a doctree ('resolve_references'), the standard domain's internal labels are used to map references to target documents. To support document squashing (aka. max depth pages), this utility method helps override a document's tuple labels so that any squashed page's labels can be moved into a parent document's label set. """ # see also: sphinx/domains/std.py std_domain = self.env.get_domain('std') try: citation_domain = self.env.get_domain('citation') except ExtensionError: citation_domain = None if 'anonlabels' in std_domain.data: anonlabels = std_domain.data['anonlabels'] for key, (fn, _l) in list(anonlabels.items()): if fn == olddocname: data = anonlabels[key] anonlabels[key] = newdocname, data[1] citations = None if 'citations' in std_domain.data: # Sphinx <2.1 citations = std_domain.data['citations'] elif citation_domain: # Sphinx >=2.1 citations = citation_domain.citations if citations: for key, (fn, _l, _) in list(citations.items()): if fn == olddocname: data = citations[key] citations[key] = newdocname, data[1], data[2] if 'citation_refs' in std_domain.data: # Sphinx <2.0 citation_refs = std_domain.data['citation_refs'] for key, _ in list(citation_refs.items()): if fn == olddocname: data = citation_refs[key] citation_refs[key] = newdocname if 'labels' in std_domain.data: labels = std_domain.data['labels'] for key, (fn, _l, _l) in list(labels.items()): if fn == olddocname: data = labels[key] labels[key] = newdocname, data[1], data[2] if 'objects' in std_domain.data: objects = std_domain.data['objects'] for key, (fn, _l) in list(objects.items()): if fn == olddocname: data = objects[key] objects[key] = newdocname, data[1] if 'progoptions' in std_domain.data: progoptions = std_domain.data['progoptions'] for key, (fn, _l) in list(progoptions.items()): if fn == olddocname: data = progoptions[key] progoptions[key] = newdocname, data[1] def _generate_special_document(self, docname, generator): """ generate a special document Provides support to generate the contents of a document for Sphinx "special" documents -- specifically, genindex, search and domain index documents. Args: docname: the docname to generate generator: instance which will generate content for a file """ # register document if its not already registered for publishing # (i.e. placeholder documents) if docname not in self.publish_docnames: self.publish_docnames.append(docname) # cache header data (if any) if self._cached_header_data is None: self._cached_header_data = '' if self.config.confluence_header_file is not None: fname = path.join(self.env.srcdir, self.config.confluence_header_file) try: with io.open(fname, encoding='utf-8') as file: self._cached_header_data = file.read() + '\n' except (IOError, OSError) as err: self.warn('error reading file {}: {}'.format(fname, err)) # cache footer data (if any) if self._cached_footer_data is None: self._cached_footer_data = '' if self.config.confluence_footer_file is not None: fname = path.join(self.env.srcdir, self.config.confluence_footer_file) try: with io.open(fname, encoding='utf-8') as file: self._cached_footer_data = file.read() + '\n' except (IOError, OSError) as err: self.warn('error reading file {}: {}'.format(fname, err)) # generate/replace the document in the output directory fname = path.join(self.outdir, docname + self.file_suffix) try: with io.open(fname, 'w', encoding='utf-8') as f: f.write(self._cached_header_data) generator(self, docname, f) f.write(self._cached_footer_data) except (IOError, OSError) as err: self.warn('error writing file %s: %s', docname, err) def _get_doctree(self, docname): """ override 'get_doctree' method To support document squashing (aka. max depth pages), doctree's may be loaded and manipulated before the writing stage. Normally, the writing stage will load target doctree's from their source so there is no way to pre-load and pass a document's doctree into the writing stage. To overcome this, this extension hooks into the environment's 'get_doctree' method and caches loaded document's doctree's into a map. """ if docname not in self.cache_doctrees: self.cache_doctrees[docname] = self._original_get_doctree(docname) return self.cache_doctrees[docname] def _header_footer_init(self, docname, doctree): """ initialize header/footer nodes (if needed) for a document Generates header/footers nodes and injects them into a provided doctree. Args: docname: the document name doctree: the doctree """ add_header_node = False add_footer_node = False header_node = confluence_header() footer_node = confluence_footer() prev_next_loc = self.config.confluence_prev_next_buttons_location # add source link if self.config.confluence_sourcelink: default_host = '' default_view = 'blob' sourcelink = dict(self.config.confluence_sourcelink) if 'url' not in sourcelink: url_base = '{protocol}://{host}/{owner}/{repo}/' source_type = sourcelink.get('type') if source_type == 'bitbucket': default_host = 'bitbucket.org' default_view = 'view' url = 'src/{version}/{container}{page}{suffix}?mode={view}' elif source_type == 'github': default_host = 'github.com' url = '{view}/{version}/{container}{page}{suffix}' elif source_type == 'gitlab': default_host = 'gitlab.com' url = '{view}/{version}/{container}{page}{suffix}' else: # unsupported source type should not pass here after this # extension's configuration check assert False sourcelink['url'] = url_base + url sourcelink.setdefault('container', '') sourcelink.setdefault('protocol', 'https') sourcelink.setdefault('host', default_host) sourcelink.setdefault('view', default_view) es_node = confluence_source_link() es_node.params.update(sourcelink) header_node.append(es_node) add_header_node = True # add page generation notice if self.config.confluence_page_generation_notice: pgn_node = confluence_page_generation_notice() header_node.append(pgn_node) add_header_node = True # add header next/previous if prev_next_loc in ('top', 'both'): if self._header_footer_inject_navnode(docname, header_node): add_header_node = True # add footer next/previous if prev_next_loc in ('bottom', 'both'): if self._header_footer_inject_navnode(docname, footer_node): add_footer_node = True # inject header/footer nodes into doctree if there is content to add if add_header_node: doctree.insert(0, header_node) if add_footer_node: doctree.append(footer_node) def _header_footer_inject_navnode(self, docname, node): """ inject navigational nodes for a document's header/footer node Requests to build a navigation nodes for a provided document name to be added into a provided header/footer node. If this document has no navigational hints to apply, this method has no effect. Args: docname: the document name node: the node to inject navigational nodes into Returns: whether or not any nodes have been injected """ if docname not in self.nav_prev and docname not in self.nav_next: return False if docname in self.nav_prev: prev_label = '← ' + SL('Previous') reference = nodes.reference(prev_label, prev_label, internal=True, refuri=self.nav_prev[docname]) reference._navnode = True reference._navnode_next = False reference._navnode_previous = True node.append(reference) if docname in self.nav_next: next_label = SL('Next') + ' →' reference = nodes.reference(next_label, next_label, internal=True, refuri=self.nav_next[docname]) reference._navnode = True reference._navnode_next = True reference._navnode_previous = False node.append(reference) return True def _register_doctree_title_targets(self, docname, doctree): """ register title targets for a doctree Compiles a list of title targets which references can link against. This tracked expected targets for sections which are automatically generated in a rendered Confluence instance. Args: docname: the docname of the doctree doctree: the doctree to search for targets """ doc_used_names = {} secnumbers = self.env.toc_secnumbers.get(docname, {}) for node in doctree.traverse(nodes.title): if isinstance(node.parent, nodes.section): section_node = node.parent if 'ids' in section_node: target = ''.join(node.astext().split()) if self.add_secnumbers: anchorname = '#' + section_node['ids'][0] if anchorname not in secnumbers: anchorname = '' secnumber = secnumbers.get(anchorname) if secnumber: target = ('.'.join(map(str, secnumber)) + self.secnumber_suffix + target) section_id = doc_used_names.get(target, 0) doc_used_names[target] = section_id + 1 if section_id > 0: target = '{}.{}'.format(target, section_id) for id_ in section_node['ids']: id_ = '{}#{}'.format(docname, id_) self.state.register_target(id_, target) def _top_ref_check(self, node): """ report if the provided node is consider a #top reference Allows an implementer extending this call to provide a hint if the provided reference node is to be considered a "#top" reference. Args: node: the node to check Returns: whether or not the node should be a #top reference """ return False def _parse_doctree_title(self, docname, doctree): """ parse a doctree for a raw title value Examine a document's doctree value to find a title value from a title section element. If no title is found, a title can be automatically generated (if configuration permits) or a `None` value is returned. """ doctitle = None title_element = self._find_title_element(doctree) if title_element: doctitle = title_element.astext() if not doctitle: if not self.config.confluence_disable_autogen_title: doctitle = "autogen-{}".format(docname) if self.publish: self.warn('document will be published using an ' 'generated title value: {}'.format(docname)) elif self.publish: self.warn('document will not be published since it ' 'has no title: {}'.format(docname)) return doctitle
def wipe_main(args_parser): """ wipe mainline The mainline for the 'wipe' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('--danger', action='store_true') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) work_dir = args.work_dir if args.work_dir else os.getcwd() # protection warning if not args.danger: print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This is a helper utility call to assist a user in cleaning out a space since removing a bulk set of data may not be trivial for a user. Note that this action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. To use this action, the argument '--danger' must be set. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print('') return 1 # check configuration and prepare publisher publisher = None with TemporaryDirectory() as tmp_dir: with docutils_namespace(): app = Sphinx( work_dir, # document sources work_dir, # directory with configuration tmp_dir, # output for generated documents tmp_dir, # output for doctree files 'confluence') # builder to execute aggressive_search = app.config.confluence_adv_aggressive_search server_url = app.config.confluence_server_url space_name = app.config.confluence_space_name # initialize the publisher (if permitted) if app.config.confluence_publish: process_ask_configs(app.config) publisher = ConfluencePublisher() publisher.init(app.config) if not publisher: print('(error) publishing not configured in sphinx configuration') return 1 # reminder warning print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print('') if not ask_question('Are you sure you want to continue?'): return 0 print('') # user has confirmed; start an attempt to wipe publisher.connect() if aggressive_search: legacy_pages = publisher.getDescendantsCompat(None) else: legacy_pages = publisher.getDescendants(None) if not legacy_pages: print('No pages are published on this space. Exiting...') return 0 print(' URL:', server_url) print(' Space:', space_name) logger.note(' Pages: All Pages') print(' Total pages:', len(legacy_pages)) print('') if not ask_question('Are you sure you want to REMOVE these pages?'): return 0 print('') logger.info('Removing pages...', nonl=True) for page_id in legacy_pages: publisher.removePage(page_id) logger.info('.', nonl=True) logger.info(' done\n') return 0
def wipe_main(args_parser): """ wipe mainline The mainline for the 'wipe' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('--danger', action='store_true') args_parser.add_argument('--parent', '-P', action='store_true') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) work_dir = args.work_dir if args.work_dir else os.getcwd() # protection warning if not args.danger: print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This is a helper utility call to assist a user in cleaning out a space since removing a bulk set of data may not be trivial for a user. Note that this action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. To use this action, the argument '--danger' must be set. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') return 1 # check configuration and prepare publisher dryrun = False publisher = None try: with temp_dir() as tmp_dir: with docutils_namespace(): app = Sphinx( work_dir, # document sources work_dir, # directory with configuration tmp_dir, # output for built documents tmp_dir, # output for doctree files 'confluence', # builder to execute status=sys.stdout, # sphinx status output warning=sys.stderr) # sphinx warning output aggressive_search = app.config.confluence_adv_aggressive_search dryrun = app.config.confluence_publish_dryrun server_url = app.config.confluence_server_url space_key = app.config.confluence_space_key parent_ref = app.config.confluence_parent_page # initialize the publisher (if permitted) if app.config.confluence_publish: process_ask_configs(app.config) publisher = ConfluencePublisher() publisher.init(app.config) except Exception: sys.stdout.flush() logger.error(traceback.format_exc()) if os.path.isfile(os.path.join(work_dir, 'conf.py')): logger.error('unable to load configuration') else: logger.error('no documentation/missing configuration') return 1 if not publisher: logger.error('publishing not configured in sphinx configuration') return 1 if args.parent and not parent_ref: logger.error('parent option provided but no parent page is configured') return 1 # reminder warning print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print('') if not ask_question('Are you sure you want to continue?'): return 0 print('') # user has confirmed; start an attempt to wipe publisher.connect() base_page_id = None if args.parent: base_page_id = publisher.get_base_page_id() if aggressive_search: legacy_pages = publisher.get_descendants_compat(base_page_id) else: legacy_pages = publisher.get_descendants(base_page_id) print(' URL:', server_url) print(' Space:', space_key) if base_page_id: logger.note(' Pages: Child pages of ' + parent_ref) else: logger.note(' Pages: All Pages') print(' Total pages:', len(legacy_pages)) if dryrun: print(' Dry run:', 'Enabled (no pages will be removed)') if not legacy_pages: print('') print('No pages detected on this space. Exiting...') return 0 if args.verbose: print('-------------------------') page_names = [] for p in legacy_pages: page_names.append(publisher._name_cache[p]) sorted(page_names) print('\n'.join(page_names)) print('-------------------------') print('') if not ask_question('Are you sure you want to REMOVE these pages?'): return 0 print('') logger.info('Removing pages...', nonl=True) if dryrun: logger.info('') for page_id in legacy_pages: publisher.remove_page(page_id) if not dryrun: logger.info('.', nonl=True) if not dryrun: logger.info(__('done')) return 0