class ConfluencePublisher(): def __init__( self, url, username, apiToken, pageTitlePrefix, markdownDir, dbPath, space, parentPageId, forceUpdate=False, forceDelete=False, skipUpdate=False, ): self.api = Confluence(url=url, username=username, password=apiToken) self.pageTitlePrefix = pageTitlePrefix self.markdownDir = markdownDir self.kv = KeyValue(dbPath) self.space = space self.parentPageId = parentPageId self.forceUpdate = forceUpdate self.forceDelete = forceDelete self.skipUpdate = skipUpdate self.confluenceRenderer = ConfluenceRenderer(url) self.metadataPlugin = MetadataPlugin() self.renderer = mistune.create_markdown( renderer=self.confluenceRenderer, plugins=[ 'strikethrough', 'footnotes', 'table', 'url', self.metadataPlugin.plugin_metadata ]) # Hack to allow metadata plugin to work (See mistune/block_parser.py) self.renderer.block.rules.remove('thematic_break') def __getFileContent(self, filepath): file = open(filepath, mode='r') content = file.read() file.close() return content def __updatePage(self, space, parentId, filepath, autoindex=False): if autoindex: markdown = '' else: markdown = self.__getFileContent(filepath) metadata = self.kv.load(filepath) currentTitle = metadata['title'] currentHash = metadata['sha256'] hash = self.kv.sha256(markdown) # --- Render (BEGIN) self.metadataPlugin.stack['title'] = None if autoindex: body = self.confluenceRenderer.generate_autoindex() else: body = self.renderer(markdown) if self.metadataPlugin.stack['title'] is None: if autoindex: title = 'Folder ' + os.path.basename(os.path.dirname(filepath)) else: title = os.path.basename(filepath) else: title = self.metadataPlugin.stack['title'] title = self.pageTitlePrefix + title # >>> Removed: + " [" + self.kv.sha256(filepath)[-6:] + "]" # --- Render (END) if currentTitle and currentTitle != title: print('REN => Title: ' + title) pageId = self.api.get_page_id(space, currentTitle) self.api.update_page(pageId, title, body) if currentHash != hash or self.forceUpdate: if autoindex: print('IDX => Title: ' + title) else: print('UPD => Title: ' + title) if self.api.update_or_create(parent_id=parentId, title=title, body=body, representation='storage'): id = self.api.get_page_id(space, title) self.kv.save(filepath, { 'id': id, 'title': title, 'sha256': hash }) return id else: return None else: print('SKP => Title: ' + title) return self.api.get_page_id(space, title) def __deleteAttachment(self, filepath): metadata = self.kv.load(filepath) filename = os.path.basename(filepath) if metadata['id']: try: print('DEL Att. => Title: ' + filename) # https://confluence.atlassian.com/confkb/confluence-rest-api-lacks-delete-method-for-attachments-715361922.html # self.api.delete_attachment_by_id(metadata['id'], 1) self.api.remove_content(metadata['id']) except (HTTPError, ApiError): pass def __updateAttachment(self, space, pageId, filepath): filename = os.path.basename(filepath) print('UPD Att. => Title: ' + filename) results = self.api.attach_file(filepath, name=filename, page_id=pageId, space=space) id = results['id'] if 'id' in results else results['results'][0]['id'] self.kv.save(filepath, {'id': id, 'title': filename, 'sha256': None}) return id def __publishRecursive(self, space, parentId, path): # File: _index.md indexParentId = parentId indexPath = path + os.sep + '_index.md' if os.path.isfile(indexPath): # Use local _index.md file indexParentId = self.__updatePage(space, parentId, indexPath) else: # Autoindex simulate _index.md in Confluence if missing locally # Except for (root) parentPageId because missing in markdownDir! if parentId != self.parentPageId: indexParentId = self.__updatePage(space, parentId, indexPath, True) # Directories: */ for f in os.scandir(path): if f.is_dir(): self.__publishRecursive(space, indexParentId, f.path) # Files: *.* (Except _index.md) for f in os.scandir(path): if f.is_file(): if f.path.endswith(".md"): if not f.path.endswith(os.sep + '_index.md'): self.__updatePage(space, indexParentId, f.path) else: self.__deleteAttachment(f.path) self.__updateAttachment(space, indexParentId, f.path) def delete(self): for filepath in sorted(self.kv.keys()): metadata = self.kv.load(filepath) # Page has Sub-pages (Childs)? indexWithChilds = False if filepath.endswith('_index.md'): childs = 0 if os.path.isdir(os.path.dirname(filepath)): for f in os.scandir(os.path.dirname(filepath)): if f.path.endswith(".md") and \ not f.path.endswith('_index.md'): childs = childs + 1 indexWithChilds = childs > 0 if self.forceDelete \ or (not os.path.isfile(filepath) and not indexWithChilds): print('DEL => Id: ' + metadata['id'] + ', Title: ' + metadata['title']) if filepath.endswith(".md"): try: if self.api.get_page_by_id(metadata['id']): self.api.remove_page(metadata['id']) except HTTPError as ex: code = ex.response.status_code if code != 404: print("DEL Pag. (Error):" + str(code)) else: pass else: self.__deleteAttachment(filepath) self.kv.remove(filepath) def publish(self): self.__publishRecursive(self.space, self.parentPageId, self.markdownDir)
] endpoint = db_instance['Endpoint']['Address'] + ':' + str( db_instance['Endpoint']['Port']) dbs[db_vpc].append('<td class="confluenceTd">' + endpoint + '</td>') dbs[db_vpc].append('<td class="confluenceTd">' + db_instance['DBName'] + '</td></tr>') vpcs = ec2client.describe_vpcs() for vpc in vpcs['Vpcs']: if vpc['IsDefault'] == False: vpc_id = vpc['VpcId'] for tag in vpc['Tags']: if tag['Key'] == "Name": vpc_name = tag['Value'] confluenceHTML += '<tr><th colspan="4" class="confluenceTh">' confluenceHTML += vpc_name + ' | ' + vpc['CidrBlock'] + ' (' + str(int(count(instances, vpc_id)) / 3) + \ ' ec2 instances, ' + str(int(count(dbs, vpc_id)) / 2) + ' rds instances)' confluenceHTML += "</th></tr>" if vpc_id in instances: confluenceHTML += ''.join(instances[vpc_id]) if vpc_id in dbs: confluenceHTML += ''.join(dbs[vpc_id]) confluenceHTML += "</tbody></table></div>" print(confluenceHTML) status = confluence.update_or_create(parent_id='REPLACE WITH PARENT PAGE ID', title='AWS Asset Inventory / Server List', body=confluenceHTML) print(status)
class ReportPageBuilder: def __init__(self, aepreq_jira_key, fix_version, cycle_name, map_version, map_revision, map_region="World", platform="Linux64"): self.aepreq_key = aepreq_jira_key self.fix_version = fix_version self.cycle_name = cycle_name self.map_version = map_version self.map_revision = map_revision self.map_region = map_region self.platform = platform self.zapi = ZephyrAPI() self.confluence = Confluence(url='https://confluence.in.here.com', **user_credentials) def create_page(self): page_content = self.fill_page_template() self.confluence.update_or_create(parent_id=self._find_parent_page_id(), title=self._page_title(), body=page_content, representation='storage') def _page_title(self): return f"{self.fix_version} - {JiraAPI.get_jira_summary_by_key(self.aepreq_key)}" def fill_page_template(self) -> str: page_data = dict(aepreq_key=self.aepreq_key, aepreq_component=None, filter_id=self._get_zapi_filter_id(), fix_version=self.fix_version, jira_tests=self._get_functional_test_cases(), map_version=self.map_version, map_revision=self.map_revision, map_region=self.map_region, platform=self.platform, sdk_version=f"{self.fix_version} - SDK1.X") page_content = Environment(loader=FileSystemLoader(report_page_templates))\ .get_template('aepreq_report_page.html').render(page_data=page_data) return page_content def _get_aepreq_additional_test_info(self) -> tuple: gerrit = GerritHandler() linked_aqa_issues = self.zapi.get_test_creation_task_from_aepreq( self.aepreq_key) commits = set() all_labels = set() log.info(f"linked_aqa_issues -> {linked_aqa_issues}") for aqa_issue in linked_aqa_issues: gerrit_commit = gerrit.get_gerrit_commit_url(aqa_issue) labels = self.zapi.get_issue_labels(aqa_issue) log.info(f"gerrit_commit -> {gerrit_commit}") log.info(f"labels -> {labels}") commits.update([gerrit_commit]) all_labels.update(labels) return list(commits), list(all_labels) def _get_functional_test_cases(self): test_info = self.zapi.get_aepreq_tests_info(self.aepreq_key) gerrit_commits, labels = [ aqa_issue for aqa_issue in self._get_aepreq_additional_test_info() ] commit = " ,".join( gerrit_commits) if len(gerrit_commits) > 1 else gerrit_commits[0] label = " ,".join(labels) if len(labels) > 1 else labels[0] for test in test_info: if "Closed" in test.get("status"): test["status"] = "done" test.update({"gerrit_commit": commit, "label": label}) log.info(f"jira tests -> {test_info}") return test_info def _get_zapi_filter_id(self): zapi_filter_query = f'project = "HERESDK" AND fixVersion = "{self.fix_version}" ' \ f'AND cycleName in ("{self.cycle_name}") AND issue in linkedIssues({self.aepreq_key})' zapi_filter_id = self.zapi.create_execution_filter( filter_name=f"{self.cycle_name}_{self.aepreq_key}", query=zapi_filter_query) log.info(f"zapi_filter_query -> {zapi_filter_query}") log.info(f"zapi_filter_id -> {zapi_filter_id}") return zapi_filter_id def _find_parent_page_id(self) -> int: hasdk_releases = 561519772 release_page_id = self._find_child_id(hasdk_releases, f"{self.fix_version} - SDK1.X") rc_release_page_id = self._find_child_id( release_page_id, f"RC - {self.fix_version} - SDK1.X") rc_test_summary_page_id = self._find_child_id( rc_release_page_id, "Product Requirements Test Summary") return rc_test_summary_page_id def _find_child_id(self, parent_id: int, part_of_child_name: str) -> int: children = self.confluence.get_page_child_by_type(parent_id, limit=100) log.info(f"children -> {children}") for child in children: if part_of_child_name in child.get('title'): return child.get('id') return 0
class ConfluencePublisher(): def __init__( self, url, username, api_token, page_title_prefix, markdown_dir, db_path, space, parent_pageid, force_update=False, force_delete=False, skip_update=False, verbose=False): self.api = Confluence(url=url, username=username, password=api_token) self.page_title_prefix = page_title_prefix self.markdown_dir = markdown_dir self.kv = KeyValue(db_path) self.space = space self.parent_pageid = parent_pageid self.force_update = force_update self.force_delete = force_delete self.skip_update = skip_update self.confluence_renderer = ConfluenceRenderer(verbose) self.renderer = mistune.create_markdown( renderer=self.confluence_renderer, plugins=[ plugin_front_matter, DirectiveInclude(), HugoRefLinkPlugin(self.markdown_dir), 'strikethrough', 'footnotes', 'table', 'url', Admonition(), plugin_html_comment, ] ) def __update_page(self, space, parentid, filepath, autoindex=False): metadata = self.kv.load(filepath) current_title = metadata['title'] current_hash = metadata['sha256'] sha_hash = get_file_sha256(filepath) # --- Render (BEGIN) body = '' state = {'front_matter': {}} if autoindex: body = generate_autoindex() state['front_matter']['title'] = \ os.path.basename(os.path.dirname(filepath)).title() else: if filepath.endswith("_index.md"): body = generate_autoindex() body += self.renderer.read(filepath, state) title = '{}{}'.format(self.page_title_prefix, state['front_matter']['title']) # --- Render (END) if current_title and current_title != title: print('REN => Title: ' + title) confluence_page_id = self.api.get_page_id(space, current_title) self.api.update_page(confluence_page_id, title, body) if current_hash != sha_hash or self.force_update: if autoindex: print('IDX => Title: ' + title) else: print('UPD => Title: ' + title) if self.api.update_or_create( parent_id=parentid, title=title, body=body, representation='storage' ): confluence_page_id = self.api.get_page_id(space, title) self.kv.save(filepath, {'id': confluence_page_id, 'title': title, 'sha256': sha_hash}) return confluence_page_id return None else: print('SKP => Title: ' + title) return self.api.get_page_id(space, title) def __delete_attachment(self, filepath): metadata = self.kv.load(filepath) filename = os.path.basename(filepath) if metadata['id']: try: print('DEL Att. => Title: ' + filename) # https://confluence.atlassian.com/confkb/confluence-rest-api-lacks-delete-method-for-attachments-715361922.html # self.api.delete_attachment_by_id(metadata['id'], 1) self.api.remove_content(metadata['id']) except (HTTPError, ApiError): pass def __update_attachment(self, space, pageid, filepath): filename = os.path.basename(filepath) print('UPD Att. => Title: ' + filename) results = self.api.attach_file(filepath, name=filename, page_id=pageid, space=space) confluence_page_id = results['id'] if 'id' in results else results['results'][0]['id'] self.kv.save(filepath, {'id': confluence_page_id, 'title': filename, 'sha256': None}) return confluence_page_id def __publish_recursive(self, space, parentid, path, root=False): # File: _index.md index_parentid = parentid index_path = path + os.sep + '_index.md' if not root: if os.path.isfile(index_path): # Use local _index.md file index_parentid = self.__update_page( space, parentid, index_path) else: # Autoindex simulate _index.md in Confluence if missing locally index_parentid = self.__update_page( space, parentid, index_path, True) # Directories: */ for f in os.scandir(path): if f.is_dir(): self.__publish_recursive(space, index_parentid, f.path) # Files: *.* (Except _index.md) for f in os.scandir(path): if f.is_file(): if f.path.endswith(".md"): if not f.path.endswith(os.sep + '_index.md'): self.__update_page(space, index_parentid, f.path) else: self.__delete_attachment(f.path) self.__update_attachment(space, index_parentid, f.path) def delete(self): for filepath in sorted(self.kv.keys()): metadata = self.kv.load(filepath) # Page has Sub-pages (Childs)? index_with_childs = False if filepath.endswith('_index.md'): childs = 0 if os.path.isdir(os.path.dirname(filepath)): for f in os.scandir(os.path.dirname(filepath)): if f.path.endswith(".md") and \ not f.path.endswith('_index.md'): childs = childs + 1 index_with_childs = childs > 0 if self.force_delete \ or (not os.path.isfile(filepath) and not index_with_childs): print('DEL => Id: ' + metadata['id'] + ', Title: ' + metadata['title']) if filepath.endswith(".md"): try: if self.api.get_page_by_id(metadata['id']): self.api.remove_page(metadata['id']) except HTTPError as ex: code = ex.response.status_code if code != 404: print("DEL Pag. (Error):" + str(code)) else: pass else: self.__delete_attachment(filepath) self.kv.remove(filepath) def publish(self): self.__publish_recursive( self.space, self.parent_pageid, self.markdown_dir, root=True)