class ConfluencePublisher():
def __init__(
self,
url,
username,
apiToken,
pageTitlePrefix,
markdownDir,
dbPath,
space,
parentPageId,
forceUpdate=False,
forceDelete=False,
skipUpdate=False,
):
self.api = Confluence(url=url, username=username, password=apiToken)
self.pageTitlePrefix = pageTitlePrefix
self.markdownDir = markdownDir
self.kv = KeyValue(dbPath)
self.space = space
self.parentPageId = parentPageId
self.forceUpdate = forceUpdate
self.forceDelete = forceDelete
self.skipUpdate = skipUpdate
self.confluenceRenderer = ConfluenceRenderer(url)
self.metadataPlugin = MetadataPlugin()
self.renderer = mistune.create_markdown(
renderer=self.confluenceRenderer,
plugins=[
'strikethrough', 'footnotes', 'table', 'url',
self.metadataPlugin.plugin_metadata
])
# Hack to allow metadata plugin to work (See mistune/block_parser.py)
self.renderer.block.rules.remove('thematic_break')
def __getFileContent(self, filepath):
file = open(filepath, mode='r')
content = file.read()
file.close()
return content
def __updatePage(self, space, parentId, filepath, autoindex=False):
if autoindex:
markdown = ''
else:
markdown = self.__getFileContent(filepath)
metadata = self.kv.load(filepath)
currentTitle = metadata['title']
currentHash = metadata['sha256']
hash = self.kv.sha256(markdown)
# --- Render (BEGIN)
self.metadataPlugin.stack['title'] = None
if autoindex:
body = self.confluenceRenderer.generate_autoindex()
else:
body = self.renderer(markdown)
if self.metadataPlugin.stack['title'] is None:
if autoindex:
title = 'Folder ' + os.path.basename(os.path.dirname(filepath))
else:
title = os.path.basename(filepath)
else:
title = self.metadataPlugin.stack['title']
title = self.pageTitlePrefix + title
# >>> Removed: + " [" + self.kv.sha256(filepath)[-6:] + "]"
# --- Render (END)
if currentTitle and currentTitle != title:
print('REN => Title: ' + title)
pageId = self.api.get_page_id(space, currentTitle)
self.api.update_page(pageId, title, body)
if currentHash != hash or self.forceUpdate:
if autoindex:
print('IDX => Title: ' + title)
else:
print('UPD => Title: ' + title)
if self.api.update_or_create(parent_id=parentId,
title=title,
body=body,
representation='storage'):
id = self.api.get_page_id(space, title)
self.kv.save(filepath, {
'id': id,
'title': title,
'sha256': hash
})
return id
else:
return None
else:
print('SKP => Title: ' + title)
return self.api.get_page_id(space, title)
def __deleteAttachment(self, filepath):
metadata = self.kv.load(filepath)
filename = os.path.basename(filepath)
if metadata['id']:
try:
print('DEL Att. => Title: ' + filename)
# https://confluence.atlassian.com/confkb/confluence-rest-api-lacks-delete-method-for-attachments-715361922.html
# self.api.delete_attachment_by_id(metadata['id'], 1)
self.api.remove_content(metadata['id'])
except (HTTPError, ApiError):
pass
def __updateAttachment(self, space, pageId, filepath):
filename = os.path.basename(filepath)
print('UPD Att. => Title: ' + filename)
results = self.api.attach_file(filepath,
name=filename,
page_id=pageId,
space=space)
id = results['id'] if 'id' in results else results['results'][0]['id']
self.kv.save(filepath, {'id': id, 'title': filename, 'sha256': None})
return id
def __publishRecursive(self, space, parentId, path):
# File: _index.md
indexParentId = parentId
indexPath = path + os.sep + '_index.md'
if os.path.isfile(indexPath):
# Use local _index.md file
indexParentId = self.__updatePage(space, parentId, indexPath)
else:
# Autoindex simulate _index.md in Confluence if missing locally
# Except for (root) parentPageId because missing in markdownDir!
if parentId != self.parentPageId:
indexParentId = self.__updatePage(space, parentId, indexPath,
True)
# Directories: */
for f in os.scandir(path):
if f.is_dir():
self.__publishRecursive(space, indexParentId, f.path)
# Files: *.* (Except _index.md)
for f in os.scandir(path):
if f.is_file():
if f.path.endswith(".md"):
if not f.path.endswith(os.sep + '_index.md'):
self.__updatePage(space, indexParentId, f.path)
else:
self.__deleteAttachment(f.path)
self.__updateAttachment(space, indexParentId, f.path)
def delete(self):
for filepath in sorted(self.kv.keys()):
metadata = self.kv.load(filepath)
# Page has Sub-pages (Childs)?
indexWithChilds = False
if filepath.endswith('_index.md'):
childs = 0
if os.path.isdir(os.path.dirname(filepath)):
for f in os.scandir(os.path.dirname(filepath)):
if f.path.endswith(".md") and \
not f.path.endswith('_index.md'):
childs = childs + 1
indexWithChilds = childs > 0
if self.forceDelete \
or (not os.path.isfile(filepath) and not indexWithChilds):
print('DEL => Id: ' + metadata['id'] + ', Title: ' +
metadata['title'])
if filepath.endswith(".md"):
try:
if self.api.get_page_by_id(metadata['id']):
self.api.remove_page(metadata['id'])
except HTTPError as ex:
code = ex.response.status_code
if code != 404:
print("DEL Pag. (Error):" + str(code))
else:
pass
else:
self.__deleteAttachment(filepath)
self.kv.remove(filepath)
def publish(self):
self.__publishRecursive(self.space, self.parentPageId,
self.markdownDir)
]
endpoint = db_instance['Endpoint']['Address'] + ':' + str(
db_instance['Endpoint']['Port'])
dbs[db_vpc].append('<td class="confluenceTd">' + endpoint +
'</td>')
dbs[db_vpc].append('<td class="confluenceTd">' +
db_instance['DBName'] + '</td></tr>')
vpcs = ec2client.describe_vpcs()
for vpc in vpcs['Vpcs']:
if vpc['IsDefault'] == False:
vpc_id = vpc['VpcId']
for tag in vpc['Tags']:
if tag['Key'] == "Name":
vpc_name = tag['Value']
confluenceHTML += '<tr><th colspan="4" class="confluenceTh">'
confluenceHTML += vpc_name + ' | ' + vpc['CidrBlock'] + ' (' + str(int(count(instances, vpc_id)) / 3) + \
' ec2 instances, ' + str(int(count(dbs, vpc_id)) / 2) + ' rds instances)'
confluenceHTML += "</th></tr>"
if vpc_id in instances:
confluenceHTML += ''.join(instances[vpc_id])
if vpc_id in dbs:
confluenceHTML += ''.join(dbs[vpc_id])
confluenceHTML += "</tbody></table></div>"
print(confluenceHTML)
status = confluence.update_or_create(parent_id='REPLACE WITH PARENT PAGE ID',
title='AWS Asset Inventory / Server List',
body=confluenceHTML)
print(status)
class ReportPageBuilder:
def __init__(self,
aepreq_jira_key,
fix_version,
cycle_name,
map_version,
map_revision,
map_region="World",
platform="Linux64"):
self.aepreq_key = aepreq_jira_key
self.fix_version = fix_version
self.cycle_name = cycle_name
self.map_version = map_version
self.map_revision = map_revision
self.map_region = map_region
self.platform = platform
self.zapi = ZephyrAPI()
self.confluence = Confluence(url='https://confluence.in.here.com',
**user_credentials)
def create_page(self):
page_content = self.fill_page_template()
self.confluence.update_or_create(parent_id=self._find_parent_page_id(),
title=self._page_title(),
body=page_content,
representation='storage')
def _page_title(self):
return f"{self.fix_version} - {JiraAPI.get_jira_summary_by_key(self.aepreq_key)}"
def fill_page_template(self) -> str:
page_data = dict(aepreq_key=self.aepreq_key,
aepreq_component=None,
filter_id=self._get_zapi_filter_id(),
fix_version=self.fix_version,
jira_tests=self._get_functional_test_cases(),
map_version=self.map_version,
map_revision=self.map_revision,
map_region=self.map_region,
platform=self.platform,
sdk_version=f"{self.fix_version} - SDK1.X")
page_content = Environment(loader=FileSystemLoader(report_page_templates))\
.get_template('aepreq_report_page.html').render(page_data=page_data)
return page_content
def _get_aepreq_additional_test_info(self) -> tuple:
gerrit = GerritHandler()
linked_aqa_issues = self.zapi.get_test_creation_task_from_aepreq(
self.aepreq_key)
commits = set()
all_labels = set()
log.info(f"linked_aqa_issues -> {linked_aqa_issues}")
for aqa_issue in linked_aqa_issues:
gerrit_commit = gerrit.get_gerrit_commit_url(aqa_issue)
labels = self.zapi.get_issue_labels(aqa_issue)
log.info(f"gerrit_commit -> {gerrit_commit}")
log.info(f"labels -> {labels}")
commits.update([gerrit_commit])
all_labels.update(labels)
return list(commits), list(all_labels)
def _get_functional_test_cases(self):
test_info = self.zapi.get_aepreq_tests_info(self.aepreq_key)
gerrit_commits, labels = [
aqa_issue for aqa_issue in self._get_aepreq_additional_test_info()
]
commit = " ,".join(
gerrit_commits) if len(gerrit_commits) > 1 else gerrit_commits[0]
label = " ,".join(labels) if len(labels) > 1 else labels[0]
for test in test_info:
if "Closed" in test.get("status"):
test["status"] = "done"
test.update({"gerrit_commit": commit, "label": label})
log.info(f"jira tests -> {test_info}")
return test_info
def _get_zapi_filter_id(self):
zapi_filter_query = f'project = "HERESDK" AND fixVersion = "{self.fix_version}" ' \
f'AND cycleName in ("{self.cycle_name}") AND issue in linkedIssues({self.aepreq_key})'
zapi_filter_id = self.zapi.create_execution_filter(
filter_name=f"{self.cycle_name}_{self.aepreq_key}",
query=zapi_filter_query)
log.info(f"zapi_filter_query -> {zapi_filter_query}")
log.info(f"zapi_filter_id -> {zapi_filter_id}")
return zapi_filter_id
def _find_parent_page_id(self) -> int:
hasdk_releases = 561519772
release_page_id = self._find_child_id(hasdk_releases,
f"{self.fix_version} - SDK1.X")
rc_release_page_id = self._find_child_id(
release_page_id, f"RC - {self.fix_version} - SDK1.X")
rc_test_summary_page_id = self._find_child_id(
rc_release_page_id, "Product Requirements Test Summary")
return rc_test_summary_page_id
def _find_child_id(self, parent_id: int, part_of_child_name: str) -> int:
children = self.confluence.get_page_child_by_type(parent_id, limit=100)
log.info(f"children -> {children}")
for child in children:
if part_of_child_name in child.get('title'):
return child.get('id')
return 0
class ConfluencePublisher():
def __init__(
self, url, username, api_token,
page_title_prefix, markdown_dir, db_path, space, parent_pageid,
force_update=False, force_delete=False, skip_update=False,
verbose=False):
self.api = Confluence(url=url, username=username, password=api_token)
self.page_title_prefix = page_title_prefix
self.markdown_dir = markdown_dir
self.kv = KeyValue(db_path)
self.space = space
self.parent_pageid = parent_pageid
self.force_update = force_update
self.force_delete = force_delete
self.skip_update = skip_update
self.confluence_renderer = ConfluenceRenderer(verbose)
self.renderer = mistune.create_markdown(
renderer=self.confluence_renderer,
plugins=[
plugin_front_matter,
DirectiveInclude(),
HugoRefLinkPlugin(self.markdown_dir),
'strikethrough',
'footnotes',
'table',
'url',
Admonition(),
plugin_html_comment,
]
)
def __update_page(self, space, parentid, filepath, autoindex=False):
metadata = self.kv.load(filepath)
current_title = metadata['title']
current_hash = metadata['sha256']
sha_hash = get_file_sha256(filepath)
# --- Render (BEGIN)
body = ''
state = {'front_matter': {}}
if autoindex:
body = generate_autoindex()
state['front_matter']['title'] = \
os.path.basename(os.path.dirname(filepath)).title()
else:
if filepath.endswith("_index.md"):
body = generate_autoindex()
body += self.renderer.read(filepath, state)
title = '{}{}'.format(self.page_title_prefix,
state['front_matter']['title'])
# --- Render (END)
if current_title and current_title != title:
print('REN => Title: ' + title)
confluence_page_id = self.api.get_page_id(space, current_title)
self.api.update_page(confluence_page_id, title, body)
if current_hash != sha_hash or self.force_update:
if autoindex:
print('IDX => Title: ' + title)
else:
print('UPD => Title: ' + title)
if self.api.update_or_create(
parent_id=parentid,
title=title,
body=body,
representation='storage'
):
confluence_page_id = self.api.get_page_id(space, title)
self.kv.save(filepath,
{'id': confluence_page_id, 'title': title, 'sha256': sha_hash})
return confluence_page_id
return None
else:
print('SKP => Title: ' + title)
return self.api.get_page_id(space, title)
def __delete_attachment(self, filepath):
metadata = self.kv.load(filepath)
filename = os.path.basename(filepath)
if metadata['id']:
try:
print('DEL Att. => Title: ' + filename)
# https://confluence.atlassian.com/confkb/confluence-rest-api-lacks-delete-method-for-attachments-715361922.html
# self.api.delete_attachment_by_id(metadata['id'], 1)
self.api.remove_content(metadata['id'])
except (HTTPError, ApiError):
pass
def __update_attachment(self, space, pageid, filepath):
filename = os.path.basename(filepath)
print('UPD Att. => Title: ' + filename)
results = self.api.attach_file(filepath,
name=filename,
page_id=pageid,
space=space)
confluence_page_id = results['id'] if 'id' in results else results['results'][0]['id']
self.kv.save(filepath, {'id': confluence_page_id,
'title': filename, 'sha256': None})
return confluence_page_id
def __publish_recursive(self, space, parentid, path, root=False):
# File: _index.md
index_parentid = parentid
index_path = path + os.sep + '_index.md'
if not root:
if os.path.isfile(index_path):
# Use local _index.md file
index_parentid = self.__update_page(
space, parentid, index_path)
else:
# Autoindex simulate _index.md in Confluence if missing locally
index_parentid = self.__update_page(
space, parentid, index_path, True)
# Directories: */
for f in os.scandir(path):
if f.is_dir():
self.__publish_recursive(space, index_parentid, f.path)
# Files: *.* (Except _index.md)
for f in os.scandir(path):
if f.is_file():
if f.path.endswith(".md"):
if not f.path.endswith(os.sep + '_index.md'):
self.__update_page(space, index_parentid, f.path)
else:
self.__delete_attachment(f.path)
self.__update_attachment(space, index_parentid, f.path)
def delete(self):
for filepath in sorted(self.kv.keys()):
metadata = self.kv.load(filepath)
# Page has Sub-pages (Childs)?
index_with_childs = False
if filepath.endswith('_index.md'):
childs = 0
if os.path.isdir(os.path.dirname(filepath)):
for f in os.scandir(os.path.dirname(filepath)):
if f.path.endswith(".md") and \
not f.path.endswith('_index.md'):
childs = childs + 1
index_with_childs = childs > 0
if self.force_delete \
or (not os.path.isfile(filepath) and not index_with_childs):
print('DEL => Id: '
+ metadata['id'] + ', Title: ' + metadata['title'])
if filepath.endswith(".md"):
try:
if self.api.get_page_by_id(metadata['id']):
self.api.remove_page(metadata['id'])
except HTTPError as ex:
code = ex.response.status_code
if code != 404:
print("DEL Pag. (Error):" + str(code))
else:
pass
else:
self.__delete_attachment(filepath)
self.kv.remove(filepath)
def publish(self):
self.__publish_recursive(
self.space, self.parent_pageid, self.markdown_dir, root=True)
