def depart_document(self, node): self.document = '' # prepend header (if any) if self.builder.config.confluence_header_file is not None: headerFile = path.join(self.builder.env.srcdir, self.builder.config.confluence_header_file) try: with io.open(headerFile, encoding='utf-8') as file: self.document += file.read() + self.nl except (IOError, OSError) as err: ConfluenceLogger.warn('error reading file ' '{}: {}'.format(headerFile, err)) self.document += ''.join(self.body) # append footer (if any) if self.builder.config.confluence_footer_file is not None: footerFile = path.join(self.builder.env.srcdir, self.builder.config.confluence_footer_file) try: with io.open(footerFile, encoding='utf-8') as file: self.document += file.read() + self.nl except (IOError, OSError) as err: ConfluenceLogger.warn('error reading file ' '{}: {}'.format(footerFile, err))
def _parse_doctree_title(self, docname, doctree): """ parse a doctree for a raw title value Examine a document's doctree value to find a title value from a title section element. If no title is found, a title can be automatically generated (if configuration permits) or a `None` value is returned. """ doctitle = None title_element = self._find_title_element(doctree) if title_element: doctitle = title_element.astext() if not doctitle: if not self.config.confluence_disable_autogen_title: doctitle = "autogen-{}".format(docname) if self.publish: ConfluenceLogger.warn( "document will be published using an " "generated title value: {}".format(docname)) elif self.publish: ConfluenceLogger.warn( "document will not be published since it " "has no title: {}".format(docname)) return doctitle
def register_title(docname, title, config): """ register the title for the provided document name In Confluence, a page is identified by the name/title of a page (at least, from the user's perspective). When processing a series of document names, the title value used for a document is based off the first heading detected. This register method allows a builder to track a document's title name name, so it may provide a document's contents and target title when passed to the publish operation. If a prefix (or postfix) value is provided, it will be added to the beginning (or at the end) of the provided title value. """ try_max = CONFLUENCE_MAX_TITLE_LEN base_tail = '' postfix = None prefix = None if config and (not config.confluence_ignore_titlefix_on_index or docname != config.root_doc): postfix = config.confluence_publish_postfix prefix = config.confluence_publish_prefix if prefix: title = prefix + title if postfix: base_tail += postfix if len(title) + len(base_tail) > try_max: warning = 'document title has been trimmed due to length: %s' % title if len(base_tail) > 0: warning += '; With postfix: %s' % base_tail logger.warn(warning) title = title[0:try_max - len(base_tail)] base_title = title title += base_tail # check if title is already used; if so, append a new value offset = 2 while title.lower() in ConfluenceState.title2doc: if offset == 2: logger.warn('title conflict detected with ' "'{}' and '{}'".format( ConfluenceState.title2doc[title.lower()], docname)) tail = ' ({}){}'.format(offset, base_tail) if len(base_title) + len(tail) > try_max: base_title = base_title[0:(try_max - len(tail))] title = base_title + tail offset += 1 ConfluenceState.doc2title[docname] = title ConfluenceState.title2doc[title.lower()] = docname logger.verbose('mapping %s to title: %s' % (docname, title)) return title
def remove_page(self, page_id): if self.dryrun: self._dryrun('removing page', page_id) return elif self.onlynew: self._onlynew('page removal restricted', page_id) return try: try: self.rest_client.delete('content', page_id) except ConfluenceBadApiError as ex: if str(ex).find('Transaction rolled back') == -1: raise logger.warn('delete failed; retrying...') time.sleep(3) self.rest_client.delete('content', page_id) except ConfluenceBadApiError as ex: # Check if Confluence reports that this content does not exist. If # so, we want to suppress the API error. This is most likely a # result of a Confluence instance reporting a page descendant # identifier which no longer exists (possibly a caching issue). if str(ex).find('No content found with id') == -1: raise logger.verbose('ignore missing delete for page ' 'identifier: {}'.format(page_id)) except ConfluencePermissionError: raise ConfluencePermissionError( """Publish user does not have permission to delete """ """from the configured space.""")
def build_main(args_parser): """ build mainline The mainline for the 'build' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('-D', action='append', default=[], dest='define') args_parser.add_argument('--output-dir', '-o') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) defines = {} for val in args.define: try: key, val = val.split('=', 1) defines[key] = val except ValueError: logger.error('invalid define provided in command line') return 1 work_dir = args.work_dir if args.work_dir else os.getcwd() if args.output_dir: output_dir = args.output_dir else: output_dir = os.path.join(work_dir, '_build', 'confluence') doctrees_dir = os.path.join(output_dir, '.doctrees') builder = args.action if args.action else DEFAULT_BUILDER verbosity = 0 if args.verbose: try: verbosity = int(args.verbose) except ValueError: pass # run sphinx engine with docutils_namespace(): app = Sphinx( work_dir, # document sources work_dir, # directory with configuration output_dir, # output for generated documents doctrees_dir, # output for doctree files builder, # builder to execute confoverrides=defines, # configuration overload freshenv=True, # fresh environment verbosity=verbosity) # verbosity app.build(force_all=True) return 0
def finish(self): # restore environment's get_doctree if it was temporarily replaced if self._original_get_doctree: self.env.get_doctree = self._original_get_doctree if self.publish: self.legacy_assets = {} self.legacy_pages = None self.parent_id = self.publisher.getBasePageId() for docname in status_iterator(self.publish_docnames, 'publishing documents... ', length=len(self.publish_docnames), verbosity=self.app.verbosity): if self._check_publish_skip(docname): self.verbose(docname + ' skipped due to configuration') continue docfile = path.join(self.outdir, self.file_transform(docname)) try: with io.open(docfile, 'r', encoding='utf-8') as file: output = file.read() self.publish_doc(docname, output) except (IOError, OSError) as err: ConfluenceLogger.warn("error reading file %s: " "%s" % (docfile, err)) def to_asset_name(asset): return asset[0] assets = self.assets.build() for asset in status_iterator(assets, 'publishing assets... ', length=len(assets), verbosity=self.app.verbosity, stringify_func=to_asset_name): key, absfile, type, hash, docname = asset if self._check_publish_skip(docname): self.verbose(key + ' skipped due to configuration') continue try: with open(absfile, 'rb') as file: output = file.read() self.publish_asset(key, docname, output, type, hash) except (IOError, OSError) as err: ConfluenceLogger.warn("error reading asset %s: " "%s" % (key, err)) self.publish_purge() self.publish_finalize() self.info('building intersphinx... ', nonl=True) build_intersphinx(self) self.info('done\n')
def _replace_inheritance_diagram(self, doctree): """ replace inheritance diagrams with images Inheritance diagrams are pre-processed and replaced with respective images in the processed documentation set. Typically, the node support from `sphinx.ext.inheritance_diagram` would be added to the builder; however, this extension renders graphs during the translation phase (which is not ideal for how assets are managed in this extension). Instead, this implementation just traverses for inheritance diagrams, generates renderings and replaces the nodes with image nodes (which in turn will be handled by the existing image-based implementation). Note that the interactive image map is not handled in this implementation since Confluence does not support image maps (without external extensions). Args: doctree: the doctree to replace blocks on """ if inheritance_diagram is None: return # graphviz's render_dot call expects a translator to be passed in; mock # a translator tied to our self-builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(self) for node in doctree.traverse(inheritance_diagram.inheritance_diagram): graph = node['graph'] graph_hash = inheritance_diagram.get_graph_hash(node) name = 'inheritance%s' % graph_hash dotcode = graph.generate_dot(name, {}, env=self.env) try: _, out_filename = render_dot(mock_translator, dotcode, {}, self.graphviz_output_format, 'inheritance') if not out_filename: node.parent.remove(node) continue new_node = nodes.image(candidates={'?'}, uri=out_filename) if 'align' in node: new_node['align'] = node['align'] node.replace_self(new_node) except GraphvizError as exc: ConfluenceLogger.warn('dot code {}: {}'.format(dotcode, exc)) node.parent.remove(node)
def write_doc(self, docname, doctree): if docname in self.omitted_docnames: return if self.prev_next_loc in ('top', 'both'): navnode = self._build_navigation_node(docname) if navnode: navnode.top = True doctree.insert(0, navnode) if self.prev_next_loc in ('bottom', 'both'): navnode = self._build_navigation_node(docname) if navnode: navnode.bottom = True doctree.append(navnode) self.secnumbers = self.env.toc_secnumbers.get(docname, {}) self.fignumbers = self.env.toc_fignumbers.get(docname, {}) # remove title from page contents (if any) if self.config.confluence_remove_title: title_element = self._find_title_element(doctree) if title_element: # If the removed title is referenced to from within the same # document (i.e. a local table of contents entry), flag any # references pointing to it as a "top" (anchor) reference. This # can be used later in a translator to hint at what type of link # to build. if 'refid' in title_element: for node in doctree.traverse(nodes.reference): if 'ids' in node and node['ids']: for id in node['ids']: if id == title_element['refid']: node['top-reference'] = True break title_element.parent.remove(title_element) # This method is taken from TextBuilder.write_doc() # with minor changes to support :confval:`rst_file_transform`. destination = StringOutput(encoding='utf-8') self.writer.write(doctree, destination) outfilename = path.join(self.outdir, self.file_transform(docname)) if self.writer.output: ensuredir(path.dirname(outfilename)) try: with io.open(outfilename, 'w', encoding='utf-8') as file: file.write(self.writer.output) except (IOError, OSError) as err: ConfluenceLogger.warn("error writing file " "%s: %s" % (outfilename, err))
def replace_graphviz_nodes(builder, doctree): """ replace graphviz nodes with images graphviz nodes are pre-processed and replaced with respective images in the processed documentation set. Typically, the node support from `sphinx.ext.graphviz` would be added to the builder; however, this extension renders graphs during the translation phase (which is not ideal for how assets are managed in this extension). Instead, this implementation just traverses for graphviz nodes, generates renderings and replaces the nodes with image nodes (which in turn will be handled by the existing image-based implementation). Args: builder: the builder doctree: the doctree to replace blocks on """ # allow users to disabled implemented extension changes restricted = builder.config.confluence_adv_restricted if 'ext-graphviz' in restricted: return if graphviz is None: return # graphviz's render_dot call expects a translator to be passed in; mock a # translator tied to our builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(builder) for node in doctree.traverse(graphviz): try: _, out_filename = render_dot(mock_translator, node['code'], node['options'], builder.graphviz_output_format, 'graphviz') if not out_filename: node.parent.remove(node) continue new_node = nodes.image(candidates={'?'}, uri=out_filename) if 'align' in node: new_node['align'] = node['align'] node.replace_self(new_node) except GraphvizError as exc: ConfluenceLogger.warn('dot code {}: {}'.format(node['code'], exc)) node.parent.remove(node)
def _wrapper(self, *args, **kwargs): # apply any user-set delay on an api request if self.config.confluence_publish_delay: delay = self.config.confluence_publish_delay logger.verbose('user-set api delay set; ' 'waiting {} seconds...'.format(math.ceil(delay))) time.sleep(delay) # if confluence asked us to wait so many seconds before a next # api request, wait a moment if self.next_delay: delay = self.next_delay logger.verbose('rate-limit header detected; ' 'waiting {} seconds...'.format(math.ceil(delay))) time.sleep(delay) self.next_delay = None # if we have imposed some rate-limiting requests where confluence # did not provide retry information, slowly decrease our tracked # delay if requests are going through self.last_retry = max(self.last_retry / 2, 1) attempt = 1 while True: try: return func(self, *args, **kwargs) except ConfluenceRateLimited as e: # if max attempts have been reached, stop any more attempts if attempt > RATE_LIMITED_MAX_RETRIES: raise e # determine the amount of delay to wait again -- either from the # provided delay (if any) or exponential backoff if self.next_delay: delay = self.next_delay self.next_delay = None else: delay = 2 * self.last_retry # cap delay to a maximum delay = min(delay, RATE_LIMITED_MAX_RETRY_DURATION) # add jitter delay += random.uniform(0.3, 1.3) # wait the calculated delay before retrying again logger.warn('rate-limit response detected; ' 'waiting {} seconds...'.format(math.ceil(delay))) time.sleep(delay) self.last_retry = delay attempt += 1
def _replace_math_blocks(self, doctree): """ replace math blocks with images Math blocks are pre-processed and replaced with respective images in the list of documents to process. This is to help prepare additional images into the asset management for this extension. Math support will work on systems which have latex/dvipng installed. Args: doctree: the doctree to replace blocks on """ if imgmath is None: return # imgmath's render_math call expects a translator to be passed # in; mock a translator tied to our self-builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(self) for node in itertools.chain(doctree.traverse(nodes.math), doctree.traverse(nodes.math_block)): try: if not isinstance(node, nodes.math): if node['nowrap']: latex = node.astext() else: latex = wrap_displaymath(node.astext(), None, False) else: latex = '$' + node.astext() + '$' mf, depth = imgmath.render_math(mock_translator, latex) if not mf: continue new_node = nodes.image(candidates={'?'}, uri=path.join(self.outdir, mf), **node.attributes) new_node['from_math'] = True if not isinstance(node, nodes.math): new_node['align'] = 'center' if depth is not None: new_node['math_depth'] = depth node.replace_self(new_node) except imgmath.MathExtError as exc: ConfluenceLogger.warn('inline latex {}: {}'.format( node.astext(), exc))
def replace_sphinx_diagrams_nodes(builder, doctree): """ replace sphinx-diagrams nodes with images sphinx-diagrams nodes are pre-processed and replaced with respective images in the processed documentation set. Args: builder: the builder doctree: the doctree to replace blocks on """ # allow users to disabled third-party implemented extension changes restricted = builder.config.confluence_adv_restricted if 'ext-sphinx_diagrams' in restricted: return if not sphinx_diagrams: return # sphinx-diagrams's render call expects a translator to be passed in; # mock a translator tied to our builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(builder) for node in doctree.traverse(sphinx_diagrams_diagrams): try: fname, _ = sphinx_diagrams_render(mock_translator, node['code'], node['options'], 'diagrams') if not fname: node.parent.remove(node) continue new_node = nodes.image(candidates={'?'}, uri=fname) if 'align' in node: new_node['align'] = node['align'] new_container = nodes.paragraph() new_container.append(new_node) node.replace_self(new_container) except DiagramsError as exc: ConfluenceLogger.warn('diagrams code %r: ' % node['code'] + str(exc)) node.parent.remove(node)
def replace_sphinxcontrib_mermaid_nodes(builder, doctree): """ replace mermaid nodes with images mermaid nodes are pre-processed and replaced with respective images in the processed documentation set. Args: builder: the builder doctree: the doctree to replace blocks on """ # allow users to disabled third-party implemented extension changes restricted = builder.config.confluence_adv_restricted if 'ext-sphinxcontrib.mermaid' in restricted: return if not sphinxcontrib_mermaid: return # mermaid's mermaid_render call expects a translator to be passed in; mock a # translator tied to our builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(builder) for node in doctree.traverse(mermaid): try: format_ = builder.config.mermaid_output_format if format_ == 'raw': format_ = 'png' fname, _ = mermaid_render(mock_translator, node['code'], node['options'], format_, 'mermaid') if not fname: node.parent.remove(node) continue new_node = nodes.image(candidates={'?'}, uri=fname) if 'align' in node: new_node['align'] = node['align'] node.replace_self(new_node) except MermaidError as exc: ConfluenceLogger.warn('mermaid code %r: ' % node['code'] + str(exc)) node.parent.remove(node)
def warnings(validator): """ inform users of any warnings related to a configuration state This call will check if the provided configuration has any configurations which may be a concern to a user. If a concern is observed in the configuration, a warning message will be provided to the user. Args: validator: the configuration validator """ config = validator.config # check if any user defined mime types are unknown if config.confluence_additional_mime_types is not None: for mime_type in config.confluence_additional_mime_types: if not mimetypes.guess_extension(mime_type): ConfluenceLogger.warn('confluence_additional_mime_types ' 'defines an unknown mime type: ' + mime_type) # warn when ssl validation is disabled if config.confluence_disable_ssl_validation: ConfluenceLogger.warn('confluence_disable_ssl_validation is set; ' 'consider using confluence_ca_cert instead') # confluence_file_suffix "cannot" end with a dot if (config.confluence_file_suffix and config.confluence_file_suffix.endswith('.')): ConfluenceLogger.warn('confluence_file_suffix ends with a period; ' 'a default value will be applied instead')
def deprecated(validator): """ inform users of deprecated configurations This call will check if the provided configuration has any configurations which have been flagged as deprecated. If a deprecated configuration is detected, a warning message will be provided to the user. Args: validator: the configuration validator """ config = validator.config # inform users of a deprecated configuration being used for key, msg in DEPRECATED_CONFIGS.items(): if config[key] is not None: logger.warn('%s deprecated; %s' % (key, msg)) # promote singleconfluence over confluence_max_doc_depth=0 if config.confluence_max_doc_depth == 0: logger.warn('confluence_max_doc_depth with a value of zero ' "is deprecated; use the 'singleconfluence' builder instead") elif config.confluence_max_doc_depth: logger.warn('confluence_max_doc_depth is deprecated and will ' "be removed; consider using the 'singleconfluence' builder instead")
def publish_asset(self, key, docname, output, type, hash): conf = self.config publisher = self.publisher title = ConfluenceState.title(docname) page_id = ConfluenceState.uploadId(docname) if not page_id: # A page identifier may not be tracked in cases where only a subset # of documents are published and the target page an asset will be # published to was not part of the request. In this case, ask the # Confluence instance what the target page's identifier is. page_id, _ = publisher.getPage(title) if page_id: ConfluenceState.registerUploadId(docname, page_id) else: ConfluenceLogger.warn('cannot publish asset since publishing ' 'point cannot be found ({}): {}'.format( key, docname)) return if conf.confluence_asset_override is None: # "automatic" management -- check if already published; if not, push attachment_id = publisher.storeAttachment(page_id, key, output, type, hash) elif conf.confluence_asset_override: # forced publishing of the asset attachment_id = publisher.storeAttachment(page_id, key, output, type, hash, force=True) if attachment_id and conf.confluence_purge: if page_id in self.legacy_assets: legacy_asset_info = self.legacy_assets[page_id] if attachment_id in legacy_asset_info: legacy_asset_info.pop(attachment_id, None)
def _handle_common_request(self, rsp): # if confluence or a proxy reports a retry-after delay (to pace us), # track it to delay the next request made # (https://datatracker.ietf.org/doc/html/rfc2616.html#section-14.37) raw_delay = rsp.headers.get(RSP_HEADER_RETRY_AFTER) if raw_delay: delay = None try: # attempt to parse a seconds value from the header delay = int(raw_delay) except ValueError: # if seconds are not provided, attempt to parse parsed_dtz = parsedate_tz(raw_delay) if parsed_dtz: target_datetime = mktime_tz(parsed_dtz) delay = target_datetime - time.time() if delay > 0: self.next_delay = delay # if this delay is over a minute, provide a notice to a client # that requests are being delayed -- but we'll only notify a # user once if delay >= 60 and not self._reported_large_delay: logger.warn('(warning) site has reported a long ' 'rate-limit delay ({} seconds)'.format( math.ceil(delay))) self._reported_large_delay = True if rsp.status_code == 401: raise ConfluenceAuthenticationFailedUrlError if rsp.status_code == 403: raise ConfluencePermissionError('rest-call') if rsp.status_code == 407: raise ConfluenceProxyPermissionError if rsp.status_code == 429: raise ConfluenceRateLimited
def _inline_all_toctrees(builder, docnameset, docname, tree, colorfunc, traversed, replace): tree = cast(nodes.document, tree.deepcopy()) for toctreenode in list(tree.traverse(addnodes.toctree)): newnodes = [] includefiles = map(str, toctreenode['includefiles']) for includefile in includefiles: if includefile not in traversed: try: traversed.append(includefile) logger.info(colorfunc(includefile) + " ", nonl=True) subtree = _inline_all_toctrees( builder, docnameset, includefile, builder.env.get_doctree(includefile), colorfunc, traversed, replace) docnameset.add(includefile) except Exception: logger.warn( __('toctree contains ref to nonexisting file %r'), includefile, location=docname) else: sof = addnodes.start_of_file(docname=includefile) sof.children = subtree.children for sectionnode in sof.traverse(nodes.section): if 'docname' not in sectionnode: sectionnode['docname'] = includefile newnodes.append(sof) if replace: toctreenode.parent.replace(toctreenode, newnodes) else: for node in newnodes: toctreenode.parent.append(node) return tree
def wipe_main(args_parser): """ wipe mainline The mainline for the 'wipe' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('--danger', action='store_true') args_parser.add_argument('--parent', '-P', action='store_true') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) work_dir = args.work_dir if args.work_dir else os.getcwd() # protection warning if not args.danger: print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This is a helper utility call to assist a user in cleaning out a space since removing a bulk set of data may not be trivial for a user. Note that this action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. To use this action, the argument '--danger' must be set. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') return 1 # check configuration and prepare publisher dryrun = False publisher = None try: with temp_dir() as tmp_dir: with docutils_namespace(): app = Sphinx( work_dir, # document sources work_dir, # directory with configuration tmp_dir, # output for built documents tmp_dir, # output for doctree files 'confluence', # builder to execute status=sys.stdout, # sphinx status output warning=sys.stderr) # sphinx warning output aggressive_search = app.config.confluence_adv_aggressive_search dryrun = app.config.confluence_publish_dryrun server_url = app.config.confluence_server_url space_key = app.config.confluence_space_key parent_ref = app.config.confluence_parent_page # initialize the publisher (if permitted) if app.config.confluence_publish: process_ask_configs(app.config) publisher = ConfluencePublisher() publisher.init(app.config) except Exception: sys.stdout.flush() logger.error(traceback.format_exc()) if os.path.isfile(os.path.join(work_dir, 'conf.py')): logger.error('unable to load configuration') else: logger.error('no documentation/missing configuration') return 1 if not publisher: logger.error('publishing not configured in sphinx configuration') return 1 if args.parent and not parent_ref: logger.error('parent option provided but no parent page is configured') return 1 # reminder warning print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print('') if not ask_question('Are you sure you want to continue?'): return 0 print('') # user has confirmed; start an attempt to wipe publisher.connect() base_page_id = None if args.parent: base_page_id = publisher.get_base_page_id() if aggressive_search: legacy_pages = publisher.get_descendants_compat(base_page_id) else: legacy_pages = publisher.get_descendants(base_page_id) print(' URL:', server_url) print(' Space:', space_key) if base_page_id: logger.note(' Pages: Child pages of ' + parent_ref) else: logger.note(' Pages: All Pages') print(' Total pages:', len(legacy_pages)) if dryrun: print(' Dry run:', 'Enabled (no pages will be removed)') if not legacy_pages: print('') print('No pages detected on this space. Exiting...') return 0 if args.verbose: print('-------------------------') page_names = [] for p in legacy_pages: page_names.append(publisher._name_cache[p]) sorted(page_names) print('\n'.join(page_names)) print('-------------------------') print('') if not ask_question('Are you sure you want to REMOVE these pages?'): return 0 print('') logger.info('Removing pages...', nonl=True) if dryrun: logger.info('') for page_id in legacy_pages: publisher.remove_page(page_id) if not dryrun: logger.info('.', nonl=True) if not dryrun: logger.info(__('done')) return 0
def wipe_main(args_parser): """ wipe mainline The mainline for the 'wipe' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('--danger', action='store_true') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) work_dir = args.work_dir if args.work_dir else os.getcwd() # protection warning if not args.danger: print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This is a helper utility call to assist a user in cleaning out a space since removing a bulk set of data may not be trivial for a user. Note that this action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. To use this action, the argument '--danger' must be set. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print('') return 1 # check configuration and prepare publisher publisher = None with TemporaryDirectory() as tmp_dir: with docutils_namespace(): app = Sphinx( work_dir, # document sources work_dir, # directory with configuration tmp_dir, # output for generated documents tmp_dir, # output for doctree files 'confluence') # builder to execute aggressive_search = app.config.confluence_adv_aggressive_search server_url = app.config.confluence_server_url space_name = app.config.confluence_space_name # initialize the publisher (if permitted) if app.config.confluence_publish: process_ask_configs(app.config) publisher = ConfluencePublisher() publisher.init(app.config) if not publisher: print('(error) publishing not configured in sphinx configuration') return 1 # reminder warning print('') sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print(""" A request has been made to attempt to wipe the pages from a configured Confluence instance. This action is not reversible with this tool and may require assistance from an administrator from a Confluence instance to recover pages. Only use this action if you know what you are doing. """) sys.stdout.flush() logger.warn('!!! DANGER DANGER DANGER !!!') print('') if not ask_question('Are you sure you want to continue?'): return 0 print('') # user has confirmed; start an attempt to wipe publisher.connect() if aggressive_search: legacy_pages = publisher.getDescendantsCompat(None) else: legacy_pages = publisher.getDescendants(None) if not legacy_pages: print('No pages are published on this space. Exiting...') return 0 print(' URL:', server_url) print(' Space:', space_name) logger.note(' Pages: All Pages') print(' Total pages:', len(legacy_pages)) print('') if not ask_question('Are you sure you want to REMOVE these pages?'): return 0 print('') logger.info('Removing pages...', nonl=True) for page_id in legacy_pages: publisher.removePage(page_id) logger.info('.', nonl=True) logger.info(' done\n') return 0
def storePage(self, page_name, data, parent_id=None): uploaded_page_id = None if self.config.confluence_adv_trace_data: ConfluenceLogger.trace('data', data['content']) if self.dryrun: _, page = self.getPage(page_name, 'version,ancestors') if not page: self._dryrun('adding new page ' + page_name) return None else: misc = '' if parent_id and 'ancestors' in page: if not any(a['id'] == parent_id for a in page['ancestors']): if parent_id in self._name_cache: misc += '[new parent page {} ({})]'.format( self._name_cache[parent_id], parent_id) else: misc += '[new parent page]' self._dryrun('updating existing page', page['id'], misc) return page['id'] can_labels = 'labels' not in self.config.confluence_adv_restricted expand = 'version' if can_labels and self.append_labels: expand += ',metadata.labels' _, page = self.getPage(page_name, expand=expand) if self.onlynew and page: self._onlynew('skipping existing page', page['id']) return page['id'] try: # new page if not page: newPage = { 'type': 'page', 'title': page_name, 'body': { 'storage': { 'representation': 'storage', 'value': data['content'], } }, 'space': { 'key': self.space_name }, } if can_labels: self._populate_labels(newPage, data['labels']) if parent_id: newPage['ancestors'] = [{'id': parent_id}] try: rsp = self.rest_client.post('content', newPage) if 'id' not in rsp: api_err = ('Confluence reports a successful page ' + 'creation; however, provided no ' + 'identifier.\n\n') try: api_err += 'DATA: {}'.format(json.dumps( rsp, indent=2)) except TypeError: api_err += 'DATA: <not-or-invalid-json>' raise ConfluenceBadApiError(api_err) uploaded_page_id = rsp['id'] except ConfluenceBadApiError as ex: # Check if Confluence reports that the new page request # fails, indicating it already exists. This is usually # (outside of possible permission use cases) that the page # name's casing does not match. In this case, attempt to # re-check for the page in a case-insensitive fashion. If # found, attempt to perform an update request instead. if str(ex).find('title already exists') == -1: raise ConfluenceLogger.verbose('title already exists warning ' 'for page {}'.format(page_name)) _, page = self.getPageCaseInsensitive(page_name) if not page: raise if self.onlynew: self._onlynew('skipping existing page', page['id']) return page['id'] # update existing page if page: last_version = int(page['version']['number']) updatePage = { 'id': page['id'], 'type': 'page', 'title': page_name, 'body': { 'storage': { 'representation': 'storage', 'value': data['content'], } }, 'space': { 'key': self.space_name }, 'version': { 'number': last_version + 1 }, } if can_labels: labels = list(data['labels']) if self.append_labels: labels.extend([lbl.get('name') for lbl in page.get('metadata', {}).get( 'labels', {}).get('results', {}) ]) self._populate_labels(updatePage, labels) if not self.notify: updatePage['version']['minorEdit'] = True if parent_id: updatePage['ancestors'] = [{'id': parent_id}] try: self.rest_client.put('content', page['id'], updatePage) except ConfluenceBadApiError as ex: if str(ex).find('unreconciled') != -1: raise ConfluenceUnreconciledPageError( page_name, page['id'], self.server_url, ex) # Confluence Cloud may (rarely) fail to complete a # content request with an OptimisticLockException/ # StaleObjectStateException exception. It is suspected # that this is just an instance timing/processing issue. # If this is observed, wait a moment and retry the # content request. If it happens again, the put request # will fail as it normally would. if str(ex).find('OptimisticLockException') == -1: raise ConfluenceLogger.warn( 'remote page updated failed; retrying...') time.sleep(1) self.rest_client.put('content', page['id'], updatePage) uploaded_page_id = page['id'] except ConfluencePermissionError: raise ConfluencePermissionError( """Publish user does not have permission to add page """ """content to the configured space.""" ) if not self.watch: self.rest_client.delete('user/watch/content', uploaded_page_id) return uploaded_page_id
def replace_math_blocks(builder, doctree): """ replace math blocks with images Math blocks are pre-processed and replaced with respective images in the list of documents to process. This is to help prepare additional images into the asset management for this extension. Math support will work on systems which have latex/dvipng installed. Args: builder: the builder doctree: the doctree to replace blocks on """ # allow users to disabled implemented extension changes restricted = builder.config.confluence_adv_restricted if 'ext-imgmath' in restricted: return # phase 1 -- convert math blocks into Confluence LaTeX blocks for node in itertools.chain(doctree.traverse(nodes.math), doctree.traverse(nodes.math_block)): if not isinstance(node, nodes.math): if node['nowrap']: latex = node.astext() else: latex = wrap_displaymath(node.astext(), None, False) new_node_type = confluence_latex_block else: latex = '$' + node.astext() + '$' new_node_type = confluence_latex_inline new_node = new_node_type(latex, latex, **node.attributes) new_node['from_math'] = True if not isinstance(node, nodes.math): new_node['align'] = 'center' node.replace_self(new_node) # disable automatic conversion of latex blocks to images if a latex # macro is configured if builder.config.confluence_latex_macro: return if imgmath is None: return # phase 2 -- convert Confluence LaTeX blocks into image blocks # # imgmath's render_math call expects a translator to be passed # in; mock a translator tied to our builder class MockTranslator: def __init__(self, builder): self.builder = builder mock_translator = MockTranslator(builder) for node in itertools.chain(doctree.traverse(confluence_latex_inline), doctree.traverse(confluence_latex_block)): try: mf, depth = imgmath.render_math(mock_translator, node.astext()) if not mf: continue new_node = nodes.image(candidates={'?'}, uri=path.join(builder.outdir, mf), **node.attributes) if depth is not None: new_node['math_depth'] = depth node.replace_self(new_node) except imgmath.MathExtError as exc: ConfluenceLogger.warn('inline latex {}: {}'.format( node.astext(), exc))
def init(self): validate_configuration(self) apply_defaults(self.config) config = self.config self.add_secnumbers = self.config.confluence_add_secnumbers self.secnumber_suffix = self.config.confluence_secnumber_suffix if self.config.confluence_additional_mime_types: for type_ in self.config.confluence_additional_mime_types: self.supported_image_types.register(type_) if 'graphviz_output_format' in self.config: self.graphviz_output_format = self.config['graphviz_output_format'] else: self.graphviz_output_format = 'png' if self.config.confluence_publish: process_ask_configs(self.config) self.assets = ConfluenceAssetManager(config, self.env, self.outdir) self.writer = ConfluenceWriter(self) self.config.sphinx_verbosity = self.app.verbosity self.publisher.init(self.config) old_url = self.config.confluence_server_url new_url = ConfluenceUtil.normalizeBaseUrl(old_url) if old_url != new_url: ConfluenceLogger.warn('normalizing confluence url from ' '{} to {} '.format(old_url, new_url)) self.config.confluence_server_url = new_url # detect if Confluence Cloud if using the Atlassian domain if new_url: self.cloud = new_url.endswith('.atlassian.net/wiki/') if self.config.confluence_file_suffix is not None: self.file_suffix = self.config.confluence_file_suffix if self.config.confluence_link_suffix is not None: self.link_suffix = self.config.confluence_link_suffix elif self.link_suffix is None: self.link_suffix = self.file_suffix # Function to convert the docname to a reST file name. def file_transform(docname): return docname + self.file_suffix # Function to convert the docname to a relative URI. def link_transform(docname): return docname + self.link_suffix self.prev_next_loc = self.config.confluence_prev_next_buttons_location if self.config.confluence_file_transform is not None: self.file_transform = self.config.confluence_file_transform else: self.file_transform = file_transform if self.config.confluence_link_transform is not None: self.link_transform = self.config.confluence_link_transform else: self.link_transform = link_transform if self.config.confluence_lang_transform is not None: self.lang_transform = self.config.confluence_lang_transform else: self.lang_transform = None if self.config.confluence_publish: self.publish = True self.publisher.connect() else: self.publish = False def prepare_subset(option): value = getattr(config, option) if not value: return None # if provided via command line, treat as a list if option in config['overrides'] and isinstance(value, basestring): value = value.split(',') if isinstance(value, basestring): files = extract_strings_from_file(value) else: files = value return set(files) if files else None self.publish_allowlist = prepare_subset('confluence_publish_allowlist') self.publish_denylist = prepare_subset('confluence_publish_denylist')
def _update_page(self, page, page_name, data, parent_id=None): """ build a page update and publish it to the confluence instance This call is invoked when the updated page data is ready to be published to a Confluence instance (i.e. pre-checks like "dry-run" mode have been completed). Args: page: the page data from confluence to update page_name: the page title to use on the update page data: the new page data to apply parent_id (optional): the id of the ancestor to use """ last_version = int(page['version']['number']) update_page = self._build_page(page_name, data) update_page['id'] = page['id'] update_page['version'] = { 'number': last_version + 1, 'message': self.config.confluence_version_comment, } labels = list(data['labels']) if self.append_labels: labels.extend([ lbl.get('name') for lbl in page.get('metadata', {}).get( 'labels', {}).get('results', {}) ]) self._populate_labels(update_page, labels) if not self.notify: update_page['version']['minorEdit'] = True if parent_id: if page['id'] in self._ancestors_cache: raise ConfluencePublishAncestorError(page_name) update_page['ancestors'] = [{'id': parent_id}] if page['id'] == parent_id: raise ConfluencePublishSelfAncestorError(page_name) page_id_explicit = page['id'] + '?status=current' try: self.rest_client.put('content', page_id_explicit, update_page) except ConfluenceBadApiError as ex: # Handle select API failures by waiting a moment and retrying the # content request. If it happens again, the put request will fail as # it normally would. retry_errors = [ # Confluence Cloud may (rarely) fail to complete a content # request with an OptimisticLockException/ # StaleObjectStateException exception. It is suspected that this # is just an instance timing/processing issue. 'OptimisticLockException', # Confluence Cloud may (rarely) fail to complete a content # request with an UnexpectedRollbackException exception. It is # suspected that this is just a failed update event due to # processing other updates. 'UnexpectedRollbackException', # Confluence may report an unreconciled error -- either from a # conflict with another instance updating the same page or some # select backend issues processing previous updates on a page. 'unreconciled', ] if not any(x in str(ex) for x in retry_errors): raise logger.warn('remote page updated failed; retrying...') time.sleep(3) try: self.rest_client.put('content', page_id_explicit, update_page) except ConfluenceBadApiError as ex: if 'unreconciled' in str(ex): raise ConfluenceUnreconciledPageError( page_name, page['id'], self.server_url, ex) raise
def init(self, suppress_conf_check=False): if not ConfluenceConfig.validate(self, not suppress_conf_check): raise ConfluenceConfigurationError('configuration error') config = self.config if self.config.confluence_publish: if self.config.confluence_ask_user: print('(request to accept username from interactive session)') print(' Instance: ' + self.config.confluence_server_url) default_user = self.config.confluence_server_user u_str = '' if default_user: u_str = ' [{}]'.format(default_user) target_user = input(' User{}: '.format(u_str)) or default_user if not target_user: raise ConfluenceConfigurationError('no user provided') self.config.confluence_server_user = target_user if self.config.confluence_ask_password: print('(request to accept password from interactive session)') if not self.config.confluence_ask_user: print(' Instance: ' + self.config.confluence_server_url) print(' User: '******' Password: '******'') if not self.config.confluence_server_pass: raise ConfluenceConfigurationError('no password provided') self.assets = ConfluenceAssetManager(self.config.master_doc, self.env, self.outdir) self.writer = ConfluenceWriter(self) self.config.sphinx_verbosity = self.app.verbosity self.publisher.init(self.config) old_url = self.config.confluence_server_url new_url = ConfluenceUtil.normalizeBaseUrl(old_url) if old_url != new_url: ConfluenceLogger.warn('normalizing confluence url from ' '{} to {} '.format(old_url, new_url)) self.config.confluence_server_url = new_url # detect if Confluence Cloud if using the Atlassian domain if new_url: self.cloud = new_url.endswith('.atlassian.net/wiki/') if self.config.confluence_file_suffix is not None: self.file_suffix = self.config.confluence_file_suffix if self.config.confluence_link_suffix is not None: self.link_suffix = self.config.confluence_link_suffix elif self.link_suffix is None: self.link_suffix = self.file_suffix # Function to convert the docname to a reST file name. def file_transform(docname): return docname + self.file_suffix # Function to convert the docname to a relative URI. def link_transform(docname): return docname + self.link_suffix self.prev_next_loc = self.config.confluence_prev_next_buttons_location if self.config.confluence_file_transform is not None: self.file_transform = self.config.confluence_file_transform else: self.file_transform = file_transform if self.config.confluence_link_transform is not None: self.link_transform = self.config.confluence_link_transform else: self.link_transform = link_transform if self.config.confluence_lang_transform is not None: self.lang_transform = self.config.confluence_lang_transform else: self.lang_transform = None if self.config.confluence_publish: self.publish = True self.publisher.connect() else: self.publish = False if self.config.confluence_space_name is not None: self.space_name = self.config.confluence_space_name else: self.space_name = None def prepare_subset(option): value = getattr(config, option) if value is None: return None # if provided via command line, treat as a list if option in config['overrides']: value = value.split(',') if isinstance(value, basestring): files = extract_strings_from_file(value) else: files = value return set(files) if files else None self.publish_allowlist = prepare_subset('confluence_publish_allowlist') self.publish_denylist = prepare_subset('confluence_publish_denylist')
def report_main(args_parser): """ report mainline The mainline for the 'report' action. Args: args_parser: the argument parser to use for argument processing Returns: the exit code """ args_parser.add_argument('--full-config', '-C', action='store_true') args_parser.add_argument('--no-sanitize', action='store_true') args_parser.add_argument('--offline', action='store_true') known_args = sys.argv[1:] args, unknown_args = args_parser.parse_known_args(known_args) if unknown_args: logger.warn('unknown arguments: {}'.format(' '.join(unknown_args))) rv = 0 offline = args.offline work_dir = args.work_dir if args.work_dir else os.getcwd() # setup sphinx engine to extract configuration config = {} configuration_load_issue = None confluence_instance_info = None publisher = ConfluencePublisher() try: with temp_dir() as tmp_dir: with docutils_namespace(): print('fetching configuration information...') builder = ConfluenceReportBuilder.name app = Sphinx( work_dir, # document sources work_dir, # directory with configuration tmp_dir, # output for built documents tmp_dir, # output for doctree files builder, # builder to execute status=sys.stdout, # sphinx status output warning=sys.stderr) # sphinx warning output if app.config.confluence_publish: try: process_ask_configs(app.config) except ConfluenceConfigurationError: offline = True # extract configuration information for k, v in app.config.values.items(): raw = getattr(app.config, k) if raw is None: continue if callable(raw): value = '(callable)' else: value = raw prefixes = ( 'confluence_', 'singleconfluence_', ) if not args.full_config and not k.startswith(prefixes): continue # always extract some known builder configurations if args.full_config and k.startswith(IGNORE_BUILDER_CONFS): continue config[k] = value # initialize the publisher (if needed later) publisher.init(app.config) except Exception: sys.stdout.flush() tb_msg = traceback.format_exc() logger.error(tb_msg) if os.path.isfile(os.path.join(work_dir, 'conf.py')): configuration_load_issue = 'unable to load configuration' configuration_load_issue += '\n\n' + tb_msg.strip() else: configuration_load_issue = 'no documentation/missing configuration' rv = 1 # attempt to fetch confluence instance version confluence_publish = config.get('confluence_publish') confluence_server_url = config.get('confluence_server_url') if not offline and confluence_publish and confluence_server_url: base_url = ConfluenceUtil.normalize_base_url(confluence_server_url) info = '' session = None try: print('connecting to confluence instance...') sys.stdout.flush() publisher.connect() info += ' connected: yes\n' session = publisher.rest_client.session except Exception: sys.stdout.flush() logger.error(traceback.format_exc()) info += ' connected: no\n' rv = 1 if session: try: # fetch print('fetching confluence instance information...') manifest_url = base_url + MANIFEST_PATH rsp = session.get(manifest_url) if rsp.status_code == 200: info += ' fetched: yes\n' # extract print('decoding information...') rsp.encoding = 'utf-8' raw_data = rsp.text info += ' decoded: yes\n' # parse print('parsing information...') xml_data = ElementTree.fromstring(raw_data) info += ' parsed: yes\n' root = ElementTree.ElementTree(xml_data) for o in root.findall('typeId'): info += ' type: ' + o.text + '\n' for o in root.findall('version'): info += ' version: ' + o.text + '\n' for o in root.findall('buildNumber'): info += ' build: ' + o.text + '\n' else: logger.error('bad response from server ({})'.format( rsp.status_code)) info += ' fetched: error ({})\n'.format(rsp.status_code) rv = 1 except Exception: sys.stdout.flush() logger.error(traceback.format_exc()) info += 'failure to determine confluence data\n' rv = 1 confluence_instance_info = info def sensitive_config(key): if key in config: if config[key]: config[key] = '(set)' else: config[key] = '(set; empty)' # always sanitize out sensitive information sensitive_config('confluence_client_cert_pass') sensitive_config('confluence_publish_headers') sensitive_config('confluence_publish_token') sensitive_config('confluence_server_pass') # optional sanitization if not args.no_sanitize: sensitive_config('author') sensitive_config('confluence_client_cert') sensitive_config('confluence_global_labels') sensitive_config('confluence_jira_servers') sensitive_config('confluence_mentions') sensitive_config('confluence_parent_page') sensitive_config('confluence_parent_page_id_check') sensitive_config('confluence_proxy') sensitive_config('confluence_publish_root') sensitive_config('confluence_server_auth') sensitive_config('confluence_server_cookies') sensitive_config('confluence_server_user') sensitive_config('project') # remove confluence instance (attempt to keep scheme) if 'confluence_server_url' in config: value = config['confluence_server_url'] parsed = urlparse(value) if parsed.scheme: value = parsed.scheme + '://<removed>' else: value = '(set; no scheme)' if parsed.netloc and parsed.netloc.endswith('atlassian.net'): value += ' (cloud)' config['confluence_server_url'] = value # remove space key, but track casing space_cfgs = [ 'confluence_space_key', 'confluence_space_name', # deprecated ] for space_cfg in space_cfgs: if space_cfg not in config: continue value = config[space_cfg] if value.startswith('~'): value = '(set; user)' elif value.isupper(): value = '(set; upper)' elif value.islower(): value = '(set; lower)' else: value = '(set; mixed)' config[space_cfg] = value print('') print('Confluence builder report has been generated.') print('Please copy the following text for the GitHub issue:') print('') logger.note('------------[ cut here ]------------') print('```') print('(system)') print(' platform:', single_line_version(platform.platform())) print(' python:', single_line_version(sys.version)) print(' sphinx:', single_line_version(sphinx_version)) print(' requests:', single_line_version(requests_version)) print(' builder:', single_line_version(scb_version)) print('') print('(configuration)') if config: for k, v in OrderedDict(sorted(config.items())).items(): print('{}: {}'.format(k, v)) else: print('~default configuration~') if configuration_load_issue: print('') print('(error loading configuration)') print(configuration_load_issue) if confluence_instance_info: print('') print('(confluence instance)') print(confluence_instance_info.rstrip()) print('```') logger.note('------------[ cut here ]------------') return rv