class Backend(BaseBackend):
_flat_src_file_name = '__all__.md'
targets = ('confluence')
required_preprocessors_after = [
'unescapecode',
{
'confluence_final': {
'cachedir': CACHEDIR_NAME,
'escapedir': ESCAPE_DIR_NAME
}
}
]
defaults = {'mode': 'single',
'toc': False,
'pandoc_path': 'pandoc',
'restore_comments': True,
'resolve_if_changed': False,
'notify_watchers': False,
'test_run': False,
'verify_ssl': True,
'passfile': 'confluence_secrets.yml',
'cloud': False}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._cachedir = (self.project_path / CACHEDIR_NAME).resolve()
self._cachedir.mkdir(exist_ok=True)
self._debug_dir = self._cachedir / DEBUG_DIR_NAME
shutil.rmtree(self._debug_dir, ignore_errors=True)
self._debug_dir.mkdir(exist_ok=True)
self._flat_src_file_path = self._cachedir / self._flat_src_file_name
self._attachments_dir = self._cachedir / ATTACHMENTS_DIR_NAME
config = self.config.get('backend_config', {}).get('confluence', {})
self.options = {**self.defaults, **config}
self.options = Options(self.options, required=['host'])
self.logger = self.logger.getChild('confluence')
self.logger.debug(f'Backend inited: {self.__dict__}')
def _get_options(self, *configs, fallback_title=None) -> Options:
'''
Get a list of dictionaries, all of which will be merged in one and
transfered to an Options object with necessary checks.
Returns the resulting Options object.
'''
options = {}
if fallback_title:
options['title'] = fallback_title
for config in configs:
options.update(config)
options = Options(options,
validators={'host': val_type(str),
'login': val_type(str),
'password': val_type(str),
'id': val_type([str, int]),
'parent_id': val_type([str, int]),
'title': val_type(str),
'space_key': val_type(str),
'pandoc_path': val_type(str),
},
required=[('id',),
('space_key', 'title')])
return options
def _connect(self, host: str, login: str, password: str, verify_ssl: bool) -> Confluence:
"""Connect to Confluence server and test connection"""
self.logger.debug(f'Trying to connect to confluence server at {host}')
host = host.rstrip('/')
self.con = Confluence(host, login, password, verify_ssl=verify_ssl)
try:
res = self.con.get('rest/api/space')
except UnicodeEncodeError:
raise RuntimeError('Sorry, non-ACSII passwords are not supported')
if isinstance(res, str) or 'statusCode' in res:
raise RuntimeError(f'Cannot connect to {host}:\n{res}')
def _get_credentials(self, host: str) -> tuple:
def get_password_for_login(login: str) -> str:
if 'password' in self.options:
return self.options['password']
else:
password = passdict.get(host.rstrip('/'), {}).get(login)
if password:
return password
else:
msg = '\n!!! User input required !!!\n'
msg += f"Please input password for {login}:\n"
return getpass(msg)
self.logger.debug(f'Loading passfile {self.options["passfile"]}')
if os.path.exists(self.options['passfile']):
self.logger.debug(f'Found passfile at {self.options["passfile"]}')
with open(self.options['passfile'], encoding='utf8') as f:
passdict = yaml.load(f, yaml.Loader)
else:
passdict = {}
if 'login' in self.options:
login = self.options['login']
password = get_password_for_login(login)
else: # login not in self.options
host_dict = passdict.get(host, {})
if host_dict:
# getting first login from passdict
login = next(iter(host_dict.keys()))
else:
msg = '\n!!! User input required !!!\n'
msg += f"Please input login for {host}:\n"
login = input(msg)
password = get_password_for_login(login)
return login, password
def _build(self):
'''
Main method. Builds confluence XHTML document from flat md source and
uploads it into the confluence server.
'''
host = self.options['host']
credentials = self._get_credentials(host)
self.logger.debug(f'Got credentials for host {host}: login {credentials[0]}, '
f'password {credentials[1]}')
self._connect(host,
*credentials,
self.options['verify_ssl'])
result = []
if 'id' in self.options or ('title' in self.options and 'space_key' in self.options):
self.logger.debug('Uploading flat project to confluence')
output(f'Building main project', self.quiet)
flatten.Preprocessor(
self.context,
self.logger,
self.quiet,
self.debug,
{'flat_src_file_name': self._flat_src_file_name,
'keep_sources': True}
).apply()
unescapecode.Preprocessor(
self.context,
self.logger,
self.quiet,
self.debug,
{}
).apply()
shutil.move(self.working_dir / self._flat_src_file_name,
self._flat_src_file_path)
with open(self._flat_src_file_path, encoding='utf8') as f:
md_source = f.read()
options = self._get_options(self.options)
self.logger.debug(f'Options: {options}')
uploader = PageUploader(
self._flat_src_file_path,
options,
self.con,
self._cachedir,
self._debug_dir,
self._attachments_dir,
self.logger
)
try:
result.append(uploader.upload(md_source))
except HTTPError as e:
# reraising HTTPError with meaningful message
raise HTTPError(e.response.text, e.response)
self.logger.debug('Searching metadata for confluence properties')
chapters = self.config['chapters']
meta = load_meta(chapters, self.working_dir)
for section in meta.iter_sections():
if not isinstance(section.data.get('confluence'), dict):
self.logger.debug(f'No "confluence" section in {section}), skipping.')
continue
self.logger.debug(f'Found "confluence" section in {section}), preparing to build.')
# getting common options from foliant.yml and merging them with meta fields
common_options = {}
uncommon_options = ['title', 'id', 'space_key', 'parent_id', 'attachments']
common_options = {k: v for k, v in self.options.items()
if k not in uncommon_options}
try:
options = self._get_options(common_options,
section.data['confluence'],
fallback_title=section.title)
except Exception as e:
# output(f'Skipping section {section}, wrong params: {e}', self.quiet)
self.logger.debug(f'Skipping section {section}, wrong params: {e}')
continue
self.logger.debug(f'Building {section.chapter.filename}: {section.title}')
output(f'Building {section.title}', self.quiet)
md_source = section.get_source()
self.logger.debug(f'Options: {options}')
original_file = self.project_path / section.chapter.filename
uploader = PageUploader(
original_file,
options,
self.con,
self._cachedir,
self._debug_dir,
self._attachments_dir,
self.logger
)
try:
result.append(uploader.upload(md_source))
except HTTPError as e:
# reraising HTTPError with meaningful message
raise HTTPError(e.response.text, e.response)
if result:
return '\n' + '\n'.join(result)
else:
return 'nothing to upload'
def make(self, target: str) -> str:
with spinner(f'Making {target}', self.logger, self.quiet, self.debug):
output('', self.quiet) # empty line for better output
try:
if target == 'confluence':
return self._build()
else:
raise ValueError(f'Confluence cannot make {target}')
except Exception as exception:
raise RuntimeError(f'Build failed: {exception}')
class Preprocessor(BasePreprocessorExt):
defaults = {
'pandoc_path': 'pandoc',
'cachedir': '.confluencecache',
'verify_ssl': True,
'passfile': 'confluence_secrets.yml'
}
tags = ('confluence', )
def _get_config(self, tag_options: dict = {}) -> CombinedOptions:
'''
Get merged config from (decreasing priority):
- tag options,
- preprocessir options,
- backend options.
'''
def filter_uncommon(val: dict) -> dict:
uncommon_options = ['title', 'id']
return {k: v for k, v in val.items() if k not in uncommon_options}
backend_config = self.config.get('backend_config',
{}).get('confluence', {})
options = CombinedOptions(
{
'tag': tag_options,
'config': filter_uncommon(self.options),
'backend_config': filter_uncommon(backend_config)
},
priority=['tag', 'config', 'backend_config'],
required=[(
'host',
'id',
), ('host', 'title', 'space_key')])
return options
def _connect(self, host: str, login: str, password: str,
verify_ssl: bool) -> Confluence:
"""Connect to Confluence server and test connection"""
self.logger.debug(f'Trying to connect to confluence server at {host}')
host = host.rstrip('/')
self.con = Confluence(host, login, password, verify_ssl=verify_ssl)
try:
res = self.con.get('rest/api/space')
except UnicodeEncodeError:
raise RuntimeError('Sorry, non-ACSII passwords are not supported')
if isinstance(res, str) or 'statusCode' in res:
raise RuntimeError(f'Cannot connect to {host}:\n{res}')
def _get_credentials(self, host: str, config: CombinedOptions) -> tuple:
def get_password_for_login(login: str) -> str:
if 'password' in config:
return config['password']
else:
password = passdict.get(host.rstrip('/'), {}).get(login)
if password:
return password
else:
msg = '\n!!! User input required !!!\n'
msg += f"Please input password for {login}:\n"
return getpass(msg)
self.logger.debug(f'Loading passfile {config["passfile"]}')
if os.path.exists(config['passfile']):
self.logger.debug(f'Found passfile at {config["passfile"]}')
with open(config['passfile'], encoding='utf8') as f:
passdict = yaml.load(f, yaml.Loader)
else:
passdict = {}
if 'login' in config:
login = config['login']
password = get_password_for_login(login)
else: # login not in self.options
host_dict = passdict.get(host, {})
if host_dict:
# getting first login from passdict
login = next(iter(host_dict.keys()))
else:
msg = '\n!!! User input required !!!\n'
msg += f"Please input login for {host}:\n"
login = input(msg)
password = get_password_for_login(login)
return login, password
def _import_from_confluence(self, match):
tag_options = self.get_options(match.group('options'))
config = self._get_config(tag_options)
cachedir = Path(config['cachedir'])
if not cachedir.exists():
cachedir.mkdir(parents=True)
host = config['host']
credentials = self._get_credentials(host, config)
self.logger.debug(
f'Got credentials for host {host}: login {credentials[0]}, '
f'password {credentials[1]}')
self._connect(host, *credentials, config['verify_ssl'])
page = Page(self.con, config.get('space_key'), config.get('title'),
None, config.get('id'))
body = process(page, self.current_filepath)
debug_filepath = cachedir / DEBUG_FILENAME
with open(debug_filepath, 'w') as f:
f.write(body)
return self._convert_to_markdown(debug_filepath, config['pandoc_path'])
def _convert_to_markdown(self,
source_path: str or PosixPath,
pandoc_path: str = 'pandoc') -> str:
'''Convert HTML to Markdown with Pandoc'''
command = f'{pandoc_path} -f html -t gfm {source_path}'
self.logger.debug('Converting HTML to MD with Pandoc, command:\n' +
command)
p = run(command, shell=True, check=True, stdout=PIPE, stderr=STDOUT)
return p.stdout.decode()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = self.logger.getChild('confluence')
self.logger.debug(f'Preprocessor inited: {self.__dict__}')
def apply(self):
output('', self.quiet) # empty line for better output
self._process_tags_for_all_files(self._import_from_confluence)
self.logger.info(f'Preprocessor applied')
class ConfluenceSearch(MycroftSkill):
def __init__(self):
MycroftSkill.__init__(self)
self.all_confluence_search_results = ''
self.delete_these_results = []
self.parse_these_results = {}
def _establish_confluence_connection(self):
"""
Setup the confluence object to be interacted with later
:return:
"""
# URL needs to be in the settingsmeta.yaml
# confluence username needs to be in the settingsmeta.yaml
# confluence password needs to be in the settingsmeta.yaml
self.username = self.settings.get('user_name')
self.password = self.settings.get('password')
self.confluence_url = self.settings.get('confluence_url')
self.confluence = Confluence(url=self.confluence_url,
username=self.username,
password=self.password)
def _setup_telegram_bot(self):
"""
Sets up the telegram bot key and the channel id to post to
:return: Nothing
"""
# Setup the telegram bot
# botkey needs to be in the settingsmeta.yaml
self.botkey = self.settings.get('telegram_bot_key')
self.updater = Updater(token=self.botkey, use_context=True)
# chat_id needs to be in the settingsmeta.yaml
self.chat_id = self.settings.get('telegram_chat_id')
def create_url_dict(self, confluence_search_results):
"""
This is used to create a dictionary of results so that we can parse it later
:param confluence_search_results: This is a json dict that is obtained from confluence
:return: A simplified dictionary with the key equal to the Page Title and the URL for the user to click on
"""
results_dict = {}
for individual_result in confluence_search_results['results']:
results_dict[individual_result[
'title']] = self.confluence.url + individual_result['_links'][
'webui']
return results_dict
def handle_display_more_context(self):
"""
This function deletes already processed items from the all_confluence_search_results object
Then if there are items left in the list, it calls the process_url_list() to create a loop
:return:
"""
for entry in self.delete_these_results:
del self.all_confluence_search_results[entry]
if len(self.all_confluence_search_results) > 0:
self.process_url_list()
def process_url_list(self):
"""
This function loops over the confluence list in order to send them to Telegram
It also tracks the items that have sent to Telegram.
Finally, it asks the user whether they want more urls to be displayed
:return: Nothing
"""
self.delete_these_results = []
for index, title in enumerate(self.all_confluence_search_results):
text = "%s \n%s" % (title,
self.all_confluence_search_results[title])
self.send_message_to_chat(text)
self.delete_these_results.append(title)
# Only send 2 entries at a time. Once we get to the second index, break the loop
if (index + 1) % 2 == 0:
break
# wait for a few seconds to let the user review the results so they know
# whether they got the page they wanted
time.sleep(5)
if len(self.all_confluence_search_results) > 2:
response = self.get_response(
"Would you like to display more results?")
if response == "no" or response == "nope" or response is None:
exit()
self.handle_display_more_context()
def send_message_to_chat(self, text):
self.updater.bot.send_message(chat_id=self.chat_id, text=text)
@intent_file_handler("search.confluence.intent")
def handle_search_confluence_title(self, message):
"""
This is the main function. It searches confluence for a title containing the user-provided search terms
It optionally filters the results based on the parent page specified by the user
:param message: Mycroft data
:return: Nothing
"""
self._setup_telegram_bot()
self._establish_confluence_connection()
user_specified_title = message.data.get('page')
# The parent page is optional, it will be None if not determined by the intent
parent_page = message.data.get('parentpage')
url = 'rest/api/content/search'
params = {}
params['cql'] = 'type={type} AND title~"{title}"'.format(
type='page', title=user_specified_title)
params['start'] = 0
params['limit'] = 10
# call the atlassian library to get a list of all the possible title matches
response = self.confluence.get(url, params=params)
# Do some extra work if we need to narrow the results by the parent page
if parent_page is not None:
self.parse_these_results['results'] = []
for individual_page_results in response['results']:
page_id = individual_page_results['id']
# get the parent page information
parent_content = ((self.confluence.get_page_by_id(
page_id=page_id, expand='ancestors').get('ancestors')))
# loop over the parent page titles and see if they match the user's utterance
for parent_page_results in parent_content:
if parent_page.lower(
) == parent_page_results['title'].lower():
self.parse_these_results['results'].append(
individual_page_results)
text = "Under the heading: %s \nI found the following results for the search containing: %s " \
% (parent_page.upper(), user_specified_title.upper())
else:
self.parse_these_results = response
text = "I found the following results for the search containing: %s" % user_specified_title.upper(
)
self.all_confluence_search_results = self.create_url_dict(
self.parse_these_results)
self.send_message_to_chat(text)
self.process_url_list()
