def _handleEntry(self, path, docname, standalone=False):
"""
handle an asset entry
When an asset is detected in a document, the information about the asset
is tracked in this manager. When an asset is detected, there are
considerations to be made. If an asset path has already been registered
(e.g. an asset used twice), only a single asset entry will be created.
If an asset matches the hash of another asset, another entry is *not
created (i.e. a documentation set has duplicate assets; *with the
exception of when ``standalone`` is set to ``True``). In all cases where
an asset is detected, the asset reference is updated to track which
document the asset belongs to.
Args:
path: the absolute path to the asset
docname: the document name this asset was found in
standalone (optional): ignore hash mappings (defaults to False)
"""
if path not in self.path2asset:
hash = ConfluenceUtil.hashAsset(path)
type_ = guess_mimetype(path, default=DEFAULT_CONTENT_TYPE)
else:
hash = self.path2asset[path].hash
type_ = self.path2asset[path].type
asset = self.path2asset.get(path, None)
if not asset:
hash_exists = hash in self.hash2asset
if not hash_exists or standalone:
# no asset entry and no hash entry (or standalone); new asset
key = os.path.basename(path)
# Confluence does not allow attachments with select characters.
# Filter out the asset name to a compatible key value.
for rep in INVALID_CHARS:
key = key.replace(rep, '_')
filename, file_ext = os.path.splitext(key)
idx = 1
while key in self.keys:
idx += 1
key = '{}_{}{}'.format(filename, idx, file_ext)
self.keys.add(key)
asset = ConfluenceAsset(key, path, type_, hash)
self.assets.append(asset)
self.path2asset[path] = asset
if not hash_exists:
self.hash2asset[hash] = asset
else:
# duplicate asset detected; build an asset alias
asset = self.hash2asset[hash]
self.path2asset[path] = asset
else:
assert (self.hash2asset[asset.hash] == asset)
# track (if not already) that this document uses this asset
asset.docnames.add(docname)
def test_normalize_baseurl(self):
data = {
'https://example.atlassian.net/wiki':
'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/':
'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rest/api':
'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rest/api/':
'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rpc/xmlrpc':
'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rpc/xmlrpc/':
'https://example.atlassian.net/wiki/',
'https://intranet-wiki.example.com':
'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/':
'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rest/api':
'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rest/api/':
'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rpc/xmlrpc':
'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rpc/xmlrpc/':
'https://intranet-wiki.example.com/',
'http://example.atlassian.net/wiki':
'http://example.atlassian.net/wiki/',
}
for key in data:
self.assertEqual(UTIL.normalizeBaseUrl(key), data[key])
def processDocument(self, doctree, docname, standalone=False):
"""
process a document for assets
This method will search each the provided document's doctree for
supported assets which could be published. Asset information is tracked
in this manager and other helper methods can be used to pull asset
information when needed.
Args:
doctree: the document's tree
docname: the document's name
standalone (optional): ignore hash mappings (defaults to False)
"""
image_nodes = doctree.traverse(nodes.image)
for node in image_nodes:
uri = node['uri']
if not uri.startswith('data:') and uri.find('://') == -1:
path = self._interpretAssetPath(node)
if not path:
continue
if path not in self.path2asset:
hash = ConfluenceUtil.hashAsset(path)
type = guess_mimetype(path, default=DEFAULT_CONTENT_TYPE)
else:
hash = self.path2asset[path].hash
type = self.path2asset[path].type
self._handleEntry(path, type, hash, docname, standalone)
file_nodes = doctree.traverse(addnodes.download_reference)
for node in file_nodes:
target = node['reftarget']
if target.find('://') == -1:
path = self._interpretAssetPath(node)
if not path:
continue
if path not in self.path2asset:
hash = ConfluenceUtil.hashAsset(path)
type = guess_mimetype(path, default=DEFAULT_CONTENT_TYPE)
else:
hash = self.path2asset[path].hash
type = self.path2asset[path].type
self._handleEntry(path, type, hash, docname, standalone)
def test_normalize_baseurl(self):
data = {
'https://example.atlassian.net/wiki': 'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/': 'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rest/api': 'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rest/api/': 'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rpc/xmlrpc': 'https://example.atlassian.net/wiki/',
'https://example.atlassian.net/wiki/rpc/xmlrpc/': 'https://example.atlassian.net/wiki/',
'https://intranet-wiki.example.com': 'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/': 'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rest/api': 'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rest/api/': 'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rpc/xmlrpc': 'https://intranet-wiki.example.com/',
'https://intranet-wiki.example.com/rpc/xmlrpc/': 'https://intranet-wiki.example.com/',
'http://example.atlassian.net/wiki': 'http://example.atlassian.net/wiki/',
}
for key in data:
self.assertEqual(UTIL.normalizeBaseUrl(key), data[key])
def init(self):
validate_configuration(self)
apply_defaults(self.config)
config = self.config
self.add_secnumbers = self.config.confluence_add_secnumbers
self.secnumber_suffix = self.config.confluence_secnumber_suffix
if self.config.confluence_additional_mime_types:
for type_ in self.config.confluence_additional_mime_types:
self.supported_image_types.register(type_)
if 'graphviz_output_format' in self.config:
self.graphviz_output_format = self.config['graphviz_output_format']
else:
self.graphviz_output_format = 'png'
if self.config.confluence_publish:
process_ask_configs(self.config)
self.assets = ConfluenceAssetManager(config, self.env, self.outdir)
self.writer = ConfluenceWriter(self)
self.config.sphinx_verbosity = self.app.verbosity
self.publisher.init(self.config)
old_url = self.config.confluence_server_url
new_url = ConfluenceUtil.normalizeBaseUrl(old_url)
if old_url != new_url:
ConfluenceLogger.warn('normalizing confluence url from '
'{} to {} '.format(old_url, new_url))
self.config.confluence_server_url = new_url
# detect if Confluence Cloud if using the Atlassian domain
if new_url:
self.cloud = new_url.endswith('.atlassian.net/wiki/')
if self.config.confluence_file_suffix is not None:
self.file_suffix = self.config.confluence_file_suffix
if self.config.confluence_link_suffix is not None:
self.link_suffix = self.config.confluence_link_suffix
elif self.link_suffix is None:
self.link_suffix = self.file_suffix
# Function to convert the docname to a reST file name.
def file_transform(docname):
return docname + self.file_suffix
# Function to convert the docname to a relative URI.
def link_transform(docname):
return docname + self.link_suffix
self.prev_next_loc = self.config.confluence_prev_next_buttons_location
if self.config.confluence_file_transform is not None:
self.file_transform = self.config.confluence_file_transform
else:
self.file_transform = file_transform
if self.config.confluence_link_transform is not None:
self.link_transform = self.config.confluence_link_transform
else:
self.link_transform = link_transform
if self.config.confluence_lang_transform is not None:
self.lang_transform = self.config.confluence_lang_transform
else:
self.lang_transform = None
if self.config.confluence_publish:
self.publish = True
self.publisher.connect()
else:
self.publish = False
def prepare_subset(option):
value = getattr(config, option)
if not value:
return None
# if provided via command line, treat as a list
if option in config['overrides'] and isinstance(value, basestring):
value = value.split(',')
if isinstance(value, basestring):
files = extract_strings_from_file(value)
else:
files = value
return set(files) if files else None
self.publish_allowlist = prepare_subset('confluence_publish_allowlist')
self.publish_denylist = prepare_subset('confluence_publish_denylist')
def report_main(args_parser):
"""
report mainline
The mainline for the 'report' action.
Args:
args_parser: the argument parser to use for argument processing
Returns:
the exit code
"""
args_parser.add_argument('--full-config', '-C', action='store_true')
args_parser.add_argument('--no-sanitize', action='store_true')
args_parser.add_argument('--offline', action='store_true')
known_args = sys.argv[1:]
args, unknown_args = args_parser.parse_known_args(known_args)
if unknown_args:
logger.warn('unknown arguments: {}'.format(' '.join(unknown_args)))
rv = 0
offline = args.offline
work_dir = args.work_dir if args.work_dir else os.getcwd()
# setup sphinx engine to extract configuration
config = {}
configuration_load_issue = None
confluence_instance_info = None
publisher = ConfluencePublisher()
try:
with temp_dir() as tmp_dir:
with docutils_namespace():
print('fetching configuration information...')
builder = ConfluenceReportBuilder.name
app = Sphinx(
work_dir, # document sources
work_dir, # directory with configuration
tmp_dir, # output for built documents
tmp_dir, # output for doctree files
builder, # builder to execute
status=sys.stdout, # sphinx status output
warning=sys.stderr) # sphinx warning output
if app.config.confluence_publish:
try:
process_ask_configs(app.config)
except ConfluenceConfigurationError:
offline = True
# extract configuration information
for k, v in app.config.values.items():
raw = getattr(app.config, k)
if raw is None:
continue
if callable(raw):
value = '(callable)'
else:
value = raw
prefixes = (
'confluence_',
'singleconfluence_',
)
if not args.full_config and not k.startswith(prefixes):
continue
# always extract some known builder configurations
if args.full_config and k.startswith(IGNORE_BUILDER_CONFS):
continue
config[k] = value
# initialize the publisher (if needed later)
publisher.init(app.config)
except Exception:
sys.stdout.flush()
tb_msg = traceback.format_exc()
logger.error(tb_msg)
if os.path.isfile(os.path.join(work_dir, 'conf.py')):
configuration_load_issue = 'unable to load configuration'
configuration_load_issue += '\n\n' + tb_msg.strip()
else:
configuration_load_issue = 'no documentation/missing configuration'
rv = 1
# attempt to fetch confluence instance version
confluence_publish = config.get('confluence_publish')
confluence_server_url = config.get('confluence_server_url')
if not offline and confluence_publish and confluence_server_url:
base_url = ConfluenceUtil.normalize_base_url(confluence_server_url)
info = ''
session = None
try:
print('connecting to confluence instance...')
sys.stdout.flush()
publisher.connect()
info += ' connected: yes\n'
session = publisher.rest_client.session
except Exception:
sys.stdout.flush()
logger.error(traceback.format_exc())
info += ' connected: no\n'
rv = 1
if session:
try:
# fetch
print('fetching confluence instance information...')
manifest_url = base_url + MANIFEST_PATH
rsp = session.get(manifest_url)
if rsp.status_code == 200:
info += ' fetched: yes\n'
# extract
print('decoding information...')
rsp.encoding = 'utf-8'
raw_data = rsp.text
info += ' decoded: yes\n'
# parse
print('parsing information...')
xml_data = ElementTree.fromstring(raw_data)
info += ' parsed: yes\n'
root = ElementTree.ElementTree(xml_data)
for o in root.findall('typeId'):
info += ' type: ' + o.text + '\n'
for o in root.findall('version'):
info += ' version: ' + o.text + '\n'
for o in root.findall('buildNumber'):
info += ' build: ' + o.text + '\n'
else:
logger.error('bad response from server ({})'.format(
rsp.status_code))
info += ' fetched: error ({})\n'.format(rsp.status_code)
rv = 1
except Exception:
sys.stdout.flush()
logger.error(traceback.format_exc())
info += 'failure to determine confluence data\n'
rv = 1
confluence_instance_info = info
def sensitive_config(key):
if key in config:
if config[key]:
config[key] = '(set)'
else:
config[key] = '(set; empty)'
# always sanitize out sensitive information
sensitive_config('confluence_client_cert_pass')
sensitive_config('confluence_publish_headers')
sensitive_config('confluence_publish_token')
sensitive_config('confluence_server_pass')
# optional sanitization
if not args.no_sanitize:
sensitive_config('author')
sensitive_config('confluence_client_cert')
sensitive_config('confluence_global_labels')
sensitive_config('confluence_jira_servers')
sensitive_config('confluence_mentions')
sensitive_config('confluence_parent_page')
sensitive_config('confluence_parent_page_id_check')
sensitive_config('confluence_proxy')
sensitive_config('confluence_publish_root')
sensitive_config('confluence_server_auth')
sensitive_config('confluence_server_cookies')
sensitive_config('confluence_server_user')
sensitive_config('project')
# remove confluence instance (attempt to keep scheme)
if 'confluence_server_url' in config:
value = config['confluence_server_url']
parsed = urlparse(value)
if parsed.scheme:
value = parsed.scheme + '://<removed>'
else:
value = '(set; no scheme)'
if parsed.netloc and parsed.netloc.endswith('atlassian.net'):
value += ' (cloud)'
config['confluence_server_url'] = value
# remove space key, but track casing
space_cfgs = [
'confluence_space_key',
'confluence_space_name', # deprecated
]
for space_cfg in space_cfgs:
if space_cfg not in config:
continue
value = config[space_cfg]
if value.startswith('~'):
value = '(set; user)'
elif value.isupper():
value = '(set; upper)'
elif value.islower():
value = '(set; lower)'
else:
value = '(set; mixed)'
config[space_cfg] = value
print('')
print('Confluence builder report has been generated.')
print('Please copy the following text for the GitHub issue:')
print('')
logger.note('------------[ cut here ]------------')
print('```')
print('(system)')
print(' platform:', single_line_version(platform.platform()))
print(' python:', single_line_version(sys.version))
print(' sphinx:', single_line_version(sphinx_version))
print(' requests:', single_line_version(requests_version))
print(' builder:', single_line_version(scb_version))
print('')
print('(configuration)')
if config:
for k, v in OrderedDict(sorted(config.items())).items():
print('{}: {}'.format(k, v))
else:
print('~default configuration~')
if configuration_load_issue:
print('')
print('(error loading configuration)')
print(configuration_load_issue)
if confluence_instance_info:
print('')
print('(confluence instance)')
print(confluence_instance_info.rstrip())
print('```')
logger.note('------------[ cut here ]------------')
return rv
def init(self, suppress_conf_check=False):
if not ConfluenceConfig.validate(self, not suppress_conf_check):
raise ConfluenceConfigurationError('configuration error')
config = self.config
if self.config.confluence_publish:
if self.config.confluence_ask_user:
print('(request to accept username from interactive session)')
print(' Instance: ' + self.config.confluence_server_url)
default_user = self.config.confluence_server_user
u_str = ''
if default_user:
u_str = ' [{}]'.format(default_user)
target_user = input(' User{}: '.format(u_str)) or default_user
if not target_user:
raise ConfluenceConfigurationError('no user provided')
self.config.confluence_server_user = target_user
if self.config.confluence_ask_password:
print('(request to accept password from interactive session)')
if not self.config.confluence_ask_user:
print(' Instance: ' + self.config.confluence_server_url)
print(' User: '******' Password: '******'')
if not self.config.confluence_server_pass:
raise ConfluenceConfigurationError('no password provided')
self.assets = ConfluenceAssetManager(self.config.master_doc, self.env,
self.outdir)
self.writer = ConfluenceWriter(self)
self.config.sphinx_verbosity = self.app.verbosity
self.publisher.init(self.config)
old_url = self.config.confluence_server_url
new_url = ConfluenceUtil.normalizeBaseUrl(old_url)
if old_url != new_url:
ConfluenceLogger.warn('normalizing confluence url from '
'{} to {} '.format(old_url, new_url))
self.config.confluence_server_url = new_url
# detect if Confluence Cloud if using the Atlassian domain
if new_url:
self.cloud = new_url.endswith('.atlassian.net/wiki/')
if self.config.confluence_file_suffix is not None:
self.file_suffix = self.config.confluence_file_suffix
if self.config.confluence_link_suffix is not None:
self.link_suffix = self.config.confluence_link_suffix
elif self.link_suffix is None:
self.link_suffix = self.file_suffix
# Function to convert the docname to a reST file name.
def file_transform(docname):
return docname + self.file_suffix
# Function to convert the docname to a relative URI.
def link_transform(docname):
return docname + self.link_suffix
self.prev_next_loc = self.config.confluence_prev_next_buttons_location
if self.config.confluence_file_transform is not None:
self.file_transform = self.config.confluence_file_transform
else:
self.file_transform = file_transform
if self.config.confluence_link_transform is not None:
self.link_transform = self.config.confluence_link_transform
else:
self.link_transform = link_transform
if self.config.confluence_lang_transform is not None:
self.lang_transform = self.config.confluence_lang_transform
else:
self.lang_transform = None
if self.config.confluence_publish:
self.publish = True
self.publisher.connect()
else:
self.publish = False
if self.config.confluence_space_name is not None:
self.space_name = self.config.confluence_space_name
else:
self.space_name = None
def prepare_subset(option):
value = getattr(config, option)
if value is None:
return None
# if provided via command line, treat as a list
if option in config['overrides']:
value = value.split(',')
if isinstance(value, basestring):
files = extract_strings_from_file(value)
else:
files = value
return set(files) if files else None
self.publish_allowlist = prepare_subset('confluence_publish_allowlist')
self.publish_denylist = prepare_subset('confluence_publish_denylist')