def create_dir_landing_page(self, dir_landing_page_file, ancestors):
"""
Create landing page for a directory
:param dir_landing_page_file: the raw markdown file to use for landing page html generation
:param ancestors: the ancestor pages of the new landing page
:return: the created landing page id
"""
landing_page_title = FILE_API.get_title(dir_landing_page_file)
html = FILE_API.get_html(dir_landing_page_file)
return self.create_or_update_page(landing_page_title, html, \
ancestors, dir_landing_page_file)
def create_trash(self):
"""
Create a __ORPHAN__ folder under the root ancestor
"""
file = tempfile.NamedTemporaryFile(mode='w', delete=False)
file.write('''# __ORPHAN__
<p>~!Files under this folder are NOT present in the source repo and and were moved here in lieu of deletion.!~</p>
If these files are no longer needed, it is safe to delete this folder.
''')
file.close()
title = FILE_API.get_title(file.name)
html = FILE_API.get_html(file.name)
root_ancestors = common.get_page_as_ancestor(ANCESTOR)
page_id = self.create_or_update_page(title, html, root_ancestors,
file.name)
return page_id
def upload_folder(directory, ancestors):
"""
Upload everything under a folder, recursively
:param directory: folder to upload
:param ancestors: parent page in ancestors format
"""
LOGGER.info('Folder: %s', directory)
# there must be at least one .md file under this folder or a
# subfolder in order for us to proceed with processing it
if not common.does_path_contain(directory, lambda file : os.path.splitext(file)[1] == '.md'):
LOGGER.info('Skipping folder; no files found')
return
# Make sure there is a landing page for the directory
doc_file = FILE_API.get_landing_page_doc_file(directory)
dir_landing_page_id = PAGE_API.create_dir_landing_page(doc_file, ancestors)
CHILD_PAGES.mark_page_active(dir_landing_page_id)
dir_landing_as_ancestors = common.get_page_as_ancestor(dir_landing_page_id)
# Walk through all other .md files in this directory and upload them all with
# the landing page as its ancestor
for file in os.scandir(directory):
if file.is_file() and os.path.splitext(file)[1] == '.md':
if os.path.normpath(file.path) != os.path.normpath(doc_file):
LOGGER.info('Markdown file: %s', file.name)
title = FILE_API.get_title(file.path)
html = FILE_API.get_html(file.path)
if SIMULATE:
common.log_html(html, title)
else:
page_id = \
PAGE_API.create_or_update_page(title, html, \
dir_landing_as_ancestors, file.path)
CHILD_PAGES.mark_page_active(page_id)
# Walk through all subdirectories and recursively upload them,
# using this directory's landing page as the ancestor for them
for folder in os.scandir(directory):
if folder.is_dir():
upload_folder(folder.path, dir_landing_as_ancestors)
def __update_page_refs_only(self, filepath):
"""
Update refs on a page without changing anything else about it
:param filepath: markdown file full path
"""
title = FILE_API.get_title(filepath)
LOGGER.info('Updating page refs %s...', title)
# update the page
page = PAGE_CACHE.get_page(title)
html = FILE_API.get_html(filepath)
version = page.version
ancestors = common.get_page_as_ancestor(page.ancestor)
LOGGER.info('.. title: %s .. version: %d .. ancestor: %s ..', title,
version, page.ancestor)
PAGE_API.update_page(page.id, title, html, version, ancestors,
filepath)
def __add_images(self, page_id, html, filepath):
"""
Scan for images and upload as attachments or child pages if found
:param page_id: Confluence page id
:param html: html string
:param filepath: markdown file full path
:return: html with modified image reference
"""
source_folder = os.path.dirname(os.path.abspath(filepath))
# <img/> tags turn into attachments
for tag in re.findall('<img(.*?)/>', html):
orig_rel_path = re.search('src="(.*?)"', tag).group(1)
alt_text = re.search('alt="(.*?)"', tag).group(1)
rel_path = urllib.parse.unquote(orig_rel_path)
abs_path = os.path.join(source_folder, rel_path)
basename = os.path.basename(rel_path)
self.__upload_attachment(page_id, abs_path, alt_text)
if re.search('http.*', rel_path) is None:
if CONFLUENCE_API_URL.endswith('/wiki'):
html = html.replace(
'%s' % (orig_rel_path),
'/wiki/download/attachments/%s/%s' %
(page_id, basename))
else:
html = html.replace(
'%s' % (orig_rel_path),
'/download/attachments/%s/%s' % (page_id, basename))
# <a href="<image>">[Name]</a> turns into a sub-page
ancestors = common.get_page_as_ancestor(page_id)
for ref in re.findall(r'<a href=\"([^\"]+)\">([^<]+)</a>', html):
if not ref[0].startswith(
('http', '/')) and ref[0].endswith('.png'):
dirname = os.path.abspath(os.path.dirname(filepath))
rel_image_from_page = os.path.join(dirname, ref[0])
image = os.path.normpath(rel_image_from_page)
alt = ref[1]
if image in self.__IMAGE_LINK_PAGES:
page = self.__IMAGE_LINK_PAGES[image]
else:
file = tempfile.NamedTemporaryFile(mode='w', delete=False)
title = urllib.parse.unquote(os.path.basename(image))
title = "%s - Diagram" % (os.path.splitext(title)[0])
file.write('# %s\n' % title)
temp_dirname = os.path.abspath(os.path.dirname(file.name))
rel_image_from_temp = os.path.relpath(image, temp_dirname)
file.write('\n' % (alt, rel_image_from_temp))
file.close()
title = FILE_API.get_title(file.name)
subhtml = FILE_API.get_html(file.name)
self.create_or_update_page(title, subhtml, ancestors,
file.name)
os.remove(file.name)
page = PAGE_CACHE.get_page(title)
self.__IMAGE_LINK_PAGES[image] = page
CHILD_PAGES.mark_page_active(page.id)
html = html.replace(ref[0], page.link)
return html
def __upload_attachment(self, page_id, file, comment):
"""
Upload an attachment
:param page_id: confluence page id
:param file: attachment file
:param comment: attachment comment
:return: boolean
"""
if re.search('http.*', file):
return False
content_type = mimetypes.guess_type(file)[0]
filename = os.path.basename(file)
if not os.path.isfile(file):
LOGGER.error('File %s cannot be found --> skip ', file)
return False
sha = FILE_API.get_sha_hash(file)
file_to_upload = {
'comment': comment,
'file': (filename, open(file, 'rb'), content_type, {
'Expires': '0'
})
}
attachment = self.__get_attachment(page_id, filename)
if attachment:
if sha == attachment.hash:
LOGGER.info('File %s has not changed --> skip', file)
return True
else:
LOGGER.debug('File %s has changed', file)
url = '%s/rest/api/content/%s/child/attachment/%s/data' % \
(CONFLUENCE_API_URL, page_id, attachment.id)
else:
LOGGER.debug('File %s is new', file)
url = '%s/rest/api/content/%s/child/attachment/' % (
CONFLUENCE_API_URL, page_id)
LOGGER.info('Uploading attachment %s...', filename)
response = common.make_request_upload(url, file_to_upload)
data = response.json()
LOGGER.debug('data: %s', str(data))
# depending on create or update, sometimes you get a collection
# and sometimes you get a single item
if u'results' in data:
data = data[u'results'][0]
attachment_id = data['id']
# Set the SHA hash metadata on the attachment so that it can be later compared
# first, get the current version of the property if it exists
url = '%s/rest/api/content/%s/property/hash' % (CONFLUENCE_API_URL,
attachment_id)
response = common.make_request_get(url, False)
if response.status_code == 200:
data = response.json()
LOGGER.debug('data: %s', str(data))
version = data[u'version'][u'number']
else:
version = 0
# then set the hash propery
page_json = {
"value": {
"sha256": sha
},
"version": {
"number": version + 1,
"minorEdit": True
}
}
LOGGER.debug('data: %s', json.dumps(page_json))
response = common.make_request_put(url, data=json.dumps(page_json))
return True