def __init__(
self,
configuration: GitHubConfiguration,
):
"""
Initialize self.
Args:
configuration: a GitHub configuration
"""
super().__init__(configuration=configuration, )
self._github = Github(
base_url=cast(str, self.configuration.url),
login_or_token=cast(str, self.configuration.token),
verify=cast(bool, self.configuration.verify),
)
self._attachment_uploader = GitHubAttachmentUploader(
configuration=self.configuration, )
self._message_formatter = ReportMessageMarkdownFormatter()
def __init__(
self,
configuration: GitLabConfiguration,
):
"""
Initialize self.
Args:
configuration: a GitLab configuration
"""
super().__init__(configuration=configuration, )
self._default_author_name = 'Anonymous'
self._session = requests.Session()
self._session.verify = configuration.verify
self._gitlab = Gitlab(
url=configuration.url,
private_token=configuration.token,
session=self._session,
)
self._message_formatter = ReportMessageMarkdownFormatter()
class GitHubTrackerClient(TrackerClient[GitHubConfiguration]):
"""A GitHub tracker client."""
_attachment_name_regex_template = Template(
r'(?:!?\[)([^\[\]]*)(?:\])(?:\(${url}\))')
_attachment_substitute_regex_template = Template(
r'!?\[[^\[\]]*\]\(${url}\)')
_github: Github
_attachment_uploader: GitHubAttachmentUploader
_message_formatter: ReportMessageMarkdownFormatter
_default_timezone: timezone = timezone.utc
def __init__(
self,
configuration: GitHubConfiguration,
):
"""
Initialize self.
Args:
configuration: a GitHub configuration
"""
super().__init__(configuration=configuration, )
self._github = Github(
base_url=cast(str, self.configuration.url),
login_or_token=cast(str, self.configuration.token),
verify=cast(bool, self.configuration.verify),
)
self._attachment_uploader = GitHubAttachmentUploader(
configuration=self.configuration, )
self._message_formatter = ReportMessageMarkdownFormatter()
@property
def tracker_type(self) -> str:
"""
Get the type of the tracker client.
Returns:
the type of the tracker client
"""
return 'GitHub'
def _build_tracker_issue(
self,
issue_id: str,
issue_url: str,
closed: bool,
) -> TrackerIssue:
return TrackerIssue(
tracker_url=cast(str, self.configuration.url),
project=cast(str, self.configuration.project),
issue_id=issue_id,
issue_url=issue_url,
closed=closed,
)
def get_tracker_issue(
self,
issue_id: str,
) -> Optional[TrackerIssue]:
"""
Get a tracker issue.
Args:
issue_id: an issue id
Returns:
The issue if it exists, else None
"""
self._ensure_auth()
try:
github_issue = self._get_github_issue(issue_id=issue_id, )
except GitHubTrackerClientError:
return None
if github_issue is None:
return None
return self._build_tracker_issue(
issue_id=issue_id,
issue_url=github_issue.html_url,
closed=github_issue.closed_at is not None,
)
def get_tracker_issue_comments(
self,
issue_id: str,
exclude_comments: Optional[List[str]] = None,
) -> List[TrackerIssueComment]:
"""
Get a list of comments on an issue.
Args:
issue_id: an issue id
exclude_comments: an optional list of comment to exclude
Returns:
The list of comments
"""
self._ensure_auth()
try:
github_issue = self._get_github_issue(issue_id=issue_id, )
except GitHubTrackerClientError:
return []
if not github_issue:
return []
return self._extract_comments(
github_issue=github_issue,
exclude_comments=exclude_comments,
)
def send_report(
self,
report: Report,
) -> TrackerIssue:
"""
Send a report to the tracker.
Args:
report: a report
Returns:
information about the sent report
"""
self._ensure_auth()
repository = self._get_repository()
title = self._message_formatter.format_report_title(report=report, )
body = self._message_formatter.format_report_description(
report=report, )
issue = self._create_issue(
github_repository=repository,
title=title,
body=body,
)
self._upload_issue_attachments(
issue=issue,
attachments=report.attachments,
)
return self._build_tracker_issue(
issue_id=str(issue.id),
issue_url=issue.html_url,
closed=False,
)
def send_logs(
self,
tracker_issue: TrackerIssue,
logs: Iterable[Log],
) -> SendLogsResult:
"""
Send logs to the tracker.
Args:
tracker_issue: information about the tracker issue
logs: a list of comments
Raises:
GitHubTrackerClientError: if an error occurs
Returns:
information about the sent comments
"""
self._ensure_auth()
tracker_comments = SendLogsResult(
tracker_issue=tracker_issue,
added_comments=[],
)
github_issue = self._get_github_issue(
issue_id=tracker_issue.issue_id, )
if not github_issue:
raise GitHubTrackerClientError(
f'GitHub issue {tracker_issue.issue_id} not found')
for log in logs:
github_comment = self._add_comment(
github_issue=github_issue,
log=log,
)
self._upload_comment_attachments(
issue=github_issue,
comment=github_comment,
attachments=log.attachments,
)
tracker_comments.added_comments.append(
TrackerIssueComment(
author=github_comment.user.name
or github_comment.user.login,
created_at=self._ensure_timezone(
dt=github_comment.created_at,
tz=self._default_timezone,
),
comment_id=str(github_comment.id),
body=github_comment.body,
attachments={},
), )
return tracker_comments
def _ensure_timezone(
self,
dt: datetime,
tz: timezone,
) -> datetime:
if not dt.tzinfo:
dt = dt.replace(tzinfo=tz)
return dt
def test(self, ) -> None:
"""
Test the client.
Raises:
GitHubTrackerClientError: if the test failed
"""
try:
login = self._github.get_user().login
except GithubException as e:
raise GitHubTrackerClientError(
'Unable to log in with GitHub API client') from e
if not login:
raise GitHubTrackerClientError(
'Unable to log in with GitHub API client')
def _extract_comments(
self,
github_issue: Issue,
exclude_comments: Optional[List[str]] = None,
) -> List[TrackerIssueComment]:
return [
self._extract_comment(github_comment=github_comment, )
for github_comment in github_issue.get_comments()
if exclude_comments is None
or str(github_comment.id) not in exclude_comments
]
def _extract_comment(
self,
github_comment: IssueComment,
) -> TrackerIssueComment:
github_body = github_comment.body
inline_images = _RE_IMAGE.findall(github_body)
comment_attachments: Dict[str, TrackerAttachment] = {}
for _, inline_url in inline_images:
attachment = self._download_attachment(url=inline_url, )
if attachment:
comment_attachments[inline_url] = attachment
return TrackerIssueComment(
created_at=self._ensure_timezone(
dt=github_comment.created_at,
tz=self._default_timezone,
),
author=github_comment.user.name or github_comment.user.login,
comment_id=str(github_comment.id),
body=github_body,
attachments=comment_attachments,
)
def _download_attachment(
self,
url: str,
) -> Optional[TrackerAttachment]:
try:
response = requests.get(url)
except requests.RequestException:
return None
content_disposition = response.headers.get('Content-Disposition')
filename = None
if content_disposition:
match = _RE_CONTENT_DISPOSITION_FILENAME.search(
content_disposition)
if match:
filename = match.group(1)
if not content_disposition or not filename:
filename = os.path.basename(url)
return TrackerAttachment(
filename=filename,
mime_type=response.headers.get('Content-Type', 'text/plain'),
content=response.content,
)
def _get_repository(self, ) -> Repository:
project = cast(str, self.configuration.project)
if project[0] == '/':
project = project[1:]
try:
return self._github.get_repo(project, )
except (GithubException, requests.RequestException) as e:
raise GitHubTrackerClientError(
f'Unable to get GitHub repository {self.configuration.project}',
) from e
def _create_issue(
self,
github_repository: Repository,
title: str,
body: str,
) -> Issue:
try:
return github_repository.create_issue(
title=title,
body=body,
)
except GithubException as e:
raise GitHubTrackerClientError(
f'Unable to create issue for project {self.configuration.project} to GitHub',
) from e
def _add_comment(
self,
github_issue: Issue,
log: Log,
) -> IssueComment:
comment_body = self._message_formatter.format_log(log=log, )
try:
return github_issue.create_comment(body=comment_body, )
except GithubException as e:
raise GitHubTrackerClientError(
f'Unable to add GitHub comment for issue {github_issue} in project {self.configuration.project}',
) from e
def _get_github_issue(
self,
issue_id: str,
) -> Optional[Issue]:
github_repository = self._get_repository()
issue_id_int = int(issue_id)
try:
for issue in github_repository.get_issues(state='all'):
if issue.id == issue_id_int:
return issue
except GithubException as e:
raise GitHubTrackerClientError(
f'GitHub issue {issue_id} not found in project {self.configuration.project}',
) from e
return None
def _upload_issue_attachments(
self,
issue: Issue,
attachments: List[Attachment],
) -> None:
body = self._upload_attachments(
issue=issue,
body=issue.body,
attachments=attachments,
)
issue.edit(body=body, )
def _upload_comment_attachments(
self,
issue: Issue,
comment: IssueComment,
attachments: List[Attachment],
) -> None:
body = self._upload_attachments(
issue=issue,
body=comment.body,
attachments=attachments,
)
comment.edit(body=body, )
def _upload_attachments(
self,
issue: Issue,
body: str,
attachments: List[Attachment],
) -> str:
for attachment in attachments:
attachment_name = self._extract_attachment_name(
body=body,
attachment=attachment,
)
if self.configuration.github_cdn_on:
url = self._attachment_uploader.upload_attachment(
issue=issue,
attachment=attachment,
)
if url:
body = body.replace(
attachment.url,
url,
)
else:
substitution = f'(Attachment "{attachment_name}" not available due to upload error)'
body = self._substitute_attachment(
body=body,
attachment=attachment,
substitution=substitution,
)
else:
substitution = f'(Attachment "{attachment_name}" not available due to export script’s configuration)'
body = self._substitute_attachment(
body=body,
attachment=attachment,
substitution=substitution,
)
return body
def _extract_attachment_name(
self,
body: str,
attachment: Attachment,
) -> str:
attachment_name_regex = self._attachment_name_regex_template.substitute(
url=re.escape(attachment.url), )
matches = re.findall(
attachment_name_regex,
body,
)
if matches:
return cast(str, matches[0])
return attachment.original_name
def _substitute_attachment(
self,
body: str,
attachment: Attachment,
substitution: str,
) -> str:
attachment_substitute_regex = self._attachment_substitute_regex_template.substitute(
url=re.escape(attachment.url), )
return re.sub(
attachment_substitute_regex,
substitution,
body,
)
def _ensure_auth(self, ) -> None:
try:
user_id = self._github.get_user().id
except GithubException as e:
raise GitHubTrackerClientError(
'Unable to authenticate to GitHub') from e
if not isinstance(user_id, int):
raise GitHubTrackerClientError('Unable to authenticate to GitHub')
class GitLabTrackerClient(TrackerClient[GitLabConfiguration]):
"""A GitLab tracker client."""
_session: requests.Session
_gitlab: Gitlab
_message_formatter: ReportMessageMarkdownFormatter
_default_author_name: str
def __init__(
self,
configuration: GitLabConfiguration,
):
"""
Initialize self.
Args:
configuration: a GitLab configuration
"""
super().__init__(configuration=configuration, )
self._default_author_name = 'Anonymous'
self._session = requests.Session()
self._session.verify = configuration.verify
self._gitlab = Gitlab(
url=configuration.url,
private_token=configuration.token,
session=self._session,
)
self._message_formatter = ReportMessageMarkdownFormatter()
@property
def tracker_type(self) -> str:
"""
Get the type of the tracker client.
Returns:
the type of the tracker client
"""
return 'GitLab'
def _build_tracker_issue(
self,
issue_id: str,
issue_url: str,
closed: bool,
) -> TrackerIssue:
return TrackerIssue(
tracker_url=cast(str, self.configuration.url),
project=cast(str, self.configuration.project),
issue_id=issue_id,
issue_url=issue_url,
closed=closed,
)
def get_tracker_issue(
self,
issue_id: str,
) -> Optional[TrackerIssue]:
"""
Get a tracker issue.
Args:
issue_id: an issue id
Returns:
The issue if it exists, else None
"""
try:
gitlab_issue = self._get_gitlab_issue(issue_id=issue_id, )
except GitLabTrackerClientError:
return None
if gitlab_issue is None:
return None
return self._build_tracker_issue(
issue_id=issue_id,
issue_url=gitlab_issue.web_url,
closed=gitlab_issue.state == 'closed',
)
def get_tracker_issue_comments(
self,
issue_id: str,
exclude_comments: Optional[List[str]] = None,
) -> TrackerIssueComments:
"""
Get a list of comments on an issue.
Args:
issue_id: an issue id
exclude_comments: an optional list of comment to exclude
Returns:
The list of comments
"""
try:
gitlab_issue = self._get_gitlab_issue(issue_id=issue_id, )
except GitLabTrackerClientError:
return []
if not gitlab_issue:
return []
return self._extract_comments(
gitlab_issue=gitlab_issue,
exclude_comments=exclude_comments,
)
def send_report(
self,
report: Report,
) -> TrackerIssue:
"""
Send a report to the tracker.
Args:
report: a report
Returns:
information about the sent report
"""
self._ensure_auth()
gitlab_project = self._get_gitlab_project()
title = self._message_formatter.format_report_title(report=report, )
description = self._message_formatter.format_report_description(
report=report, ) + self._get_attachments_list_description(
attachments=report.attachments, )
external_description = ''
description_attachment = None
if len(description) > _TEXT_MAX_SIZE:
external_description = description
description_attachment = self._build_external_description_attachment(
name=
f'report-{report.local_id.replace("#", "")}-description.md', )
report_copy = deepcopy(report)
report_copy.description_html = (
'<p>This report description is too large to fit into a GitLab issue. '
+
f'See attachment <a href="{description_attachment.url}">{description_attachment.original_name}</a> '
+ 'for more details.</p>')
description = self._message_formatter.format_report_description(
report=report_copy, ) + self._get_attachments_list_description(
attachments=[
description_attachment,
*report.attachments,
], )
description, external_description = self._replace_attachments_references(
uploads=self._upload_attachments(
gitlab_project=gitlab_project,
attachments=report.attachments,
),
referencing_texts=[
description,
external_description,
],
)
if description_attachment:
description_attachment.data_loader = lambda: bytes(
external_description, 'utf-8')
description = self._replace_attachments_references(
uploads=self._upload_attachments(
gitlab_project=gitlab_project,
attachments=[
description_attachment,
],
),
referencing_texts=[
description,
],
)[0]
gitlab_issue = self._create_issue(
gitlab_project=gitlab_project,
title=title,
description=description,
confidential=self.configuration.confidential or False,
)
return self._build_tracker_issue(
issue_id=str(gitlab_issue.id),
issue_url=gitlab_issue.web_url,
closed=False,
)
def _build_external_description_attachment(
self,
name: str,
) -> Attachment:
return Attachment(
attachment_id=0,
name=name,
original_name=name,
mime_type='text/markdown',
size=0,
url=f'http://tracker/external/{name}',
data_loader=lambda: bytes('', 'utf-8'),
)
def _extract_comments(
self,
gitlab_issue: ProjectIssue,
exclude_comments: Optional[List[str]] = None,
) -> List[TrackerIssueComment]:
return [
self._extract_comment(gitlab_note=gitlab_note, )
for gitlab_note in reversed(gitlab_issue.notes.list())
if exclude_comments is None
or str(gitlab_note.id) not in exclude_comments
]
def _extract_comment(
self,
gitlab_note: ProjectIssueNote,
) -> TrackerIssueComment:
gitlab_body = gitlab_note.body
inline_images = _RE_IMAGE.findall(gitlab_body)
comment_attachments: Dict[str, TrackerAttachment] = {}
for _, inline_path in inline_images:
attachment = self._download_attachment(path=inline_path, )
if attachment:
comment_attachments[inline_path] = attachment
return TrackerIssueComment(
created_at=self._parse_date(date=gitlab_note.created_at, ),
author=gitlab_note.author.get('name', self._default_author_name),
comment_id=str(gitlab_note.id),
body=gitlab_body,
attachments=comment_attachments,
)
def _download_attachment(
self,
path: str,
) -> Optional[TrackerAttachment]:
url = f'{self.configuration.url}/{self.configuration.project}/{path}'
try:
response = requests.get(url)
except requests.RequestException:
return None
if not response.ok:
return None
content_disposition = response.headers.get('Content-Disposition')
filename = None
if content_disposition:
match = _RE_CONTENT_DISPOSITION_FILENAME.search(
content_disposition)
if match:
filename = match.group(1)
if not content_disposition or not filename:
filename = os.path.basename(path)
return TrackerAttachment(
filename=filename,
mime_type=response.headers.get('Content-Type', 'text/plain'),
content=response.content,
)
def send_logs(
self,
tracker_issue: TrackerIssue,
logs: Iterable[Log],
) -> SendLogsResult:
"""
Send logs to the tracker.
Args:
tracker_issue: information about the tracker issue
logs: a list of comments
Raises:
GitLabTrackerClientError: if an error occurs
Returns:
information about the sent comments
"""
self._ensure_auth()
tracker_comments = SendLogsResult(
tracker_issue=tracker_issue,
added_comments=[],
)
gitlab_project = self._get_gitlab_project()
gitlab_issue = self._get_gitlab_issue(
issue_id=tracker_issue.issue_id, )
if not gitlab_issue:
raise GitLabTrackerClientError(
f'GitLab issue {tracker_issue.issue_id} not found in project {self.configuration.project}',
)
for log in logs:
gitlab_comment = self._add_comment(
gitlab_project=gitlab_project,
gitlab_issue=gitlab_issue,
log=log,
)
tracker_comments.added_comments.append(
TrackerIssueComment(
created_at=self._parse_date(
date=gitlab_comment.created_at, ),
author=gitlab_comment.author.get(
'name', self._default_author_name),
comment_id=str(gitlab_comment.id),
body=gitlab_comment.body,
attachments={},
), )
return tracker_comments
def _parse_date(
self,
date: str,
) -> datetime:
return datetime.strptime(
date,
'%Y-%m-%dT%H:%M:%S.%f%z',
)
def test(self, ) -> None:
"""
Test the client.
Raises:
GitLabTrackerClientError: if the test failed
"""
try:
self._gitlab.auth()
except GitlabError as e:
raise GitLabTrackerClientError(
'Unable to log in with GitLab API client') from e
def _get_gitlab_project(self, ) -> Project:
try:
return self._gitlab.projects.get(self.configuration.project, )
except GitlabError as e:
raise GitLabTrackerClientError(
f'Unable to get GitLab project {self.configuration.project}',
) from e
def _get_gitlab_issue(
self,
issue_id: str,
) -> Optional[ProjectIssue]:
gitlab_project = self._get_gitlab_project()
try:
gitlab_issues = gitlab_project.issues.list(all=False,
as_list=False)
except GitlabError as e:
raise GitLabTrackerClientError(
f'Unable to get GitLab issues for project {self.configuration.project}',
) from e
issue_id_int = int(issue_id)
for gitlab_issue in gitlab_issues:
if gitlab_issue.id == issue_id_int:
return gitlab_issue
return None
def _replace_attachments_references(
self,
uploads: List[Tuple[Attachment, str]],
referencing_texts: List[str],
) -> List[str]:
for attachment, upload_url in uploads:
referencing_texts = [
text.replace(
attachment.url,
f'{self.configuration.url}/{self.configuration.project}{upload_url}',
) for text in referencing_texts
]
return referencing_texts
def _upload_attachments(
self,
gitlab_project: Project,
attachments: List[Attachment],
) -> List[Tuple[Attachment, str]]:
try:
return [(
attachment,
gitlab_project.upload(attachment.original_name,
attachment.data)['url'],
) for attachment in attachments]
except GitlabError as e:
raise GitLabTrackerClientError(
f'Unable to upload attachments for project {self.configuration.project} to GitLab',
) from e
def _get_attachments_list_description(
self,
attachments: List[Attachment],
) -> str:
attachments_lines = []
if attachments:
attachments_lines = [
'',
'**Attachments**:',
]
for attachment in attachments:
attachments_lines.append(
f'- [{attachment.original_name}]({attachment.url})', )
attachments_lines.append('')
return '\n'.join(attachments_lines)
def _create_issue(
self,
gitlab_project: Project,
title: str,
description: str,
confidential: bool,
) -> ProjectIssue:
issue_data = {
'title': title,
'description': description,
'confidential': confidential,
}
try:
return gitlab_project.issues.create(issue_data)
except GitlabError as e:
raise GitLabTrackerClientError(
f'Unable to create issue for project {self.configuration.project} to GitLab',
) from e
def _add_comment(
self,
gitlab_project: Project,
gitlab_issue: ProjectIssue,
log: Log,
) -> ProjectIssueNote:
comment_body = self._message_formatter.format_log(
log=log, ) + self._get_attachments_list_description(
attachments=log.attachments, )
external_body = ''
body_attachment = None
if len(comment_body) > _TEXT_MAX_SIZE:
external_body = comment_body
body_attachment = self._build_external_description_attachment(
name=f'comment-{log.log_id}-description.md', )
log_copy = deepcopy(log)
log_copy.message_html = (
'<p>This comment is too large to fit into a GitLab comment. ' +
f'See attachment <a href="{body_attachment.url}">{body_attachment.original_name}</a> '
+ 'for more details.</p>')
comment_body = self._message_formatter.format_log(
log=log_copy, ) + self._get_attachments_list_description(
attachments=[
body_attachment,
*log.attachments,
], )
comment_body, external_body = self._replace_attachments_references(
uploads=self._upload_attachments(
gitlab_project=gitlab_project,
attachments=log.attachments,
),
referencing_texts=[
comment_body,
external_body,
],
)
if body_attachment:
body_attachment.data_loader = lambda: bytes(external_body, 'utf-8')
comment_body = self._replace_attachments_references(
uploads=self._upload_attachments(
gitlab_project=gitlab_project,
attachments=[
body_attachment,
],
),
referencing_texts=[
comment_body,
],
)[0]
try:
return gitlab_issue.notes.create({
'body': comment_body,
})
except GitlabError as e:
raise GitLabTrackerClientError(
f'Unable to add GitLab comment for issue {gitlab_issue} in project {self.configuration.project}',
) from e
def _ensure_auth(self, ) -> None:
try:
self._gitlab.auth()
except GitlabError as e:
raise GitLabTrackerClientError(
'Unable to authenticate to GitLab') from e
def _add_comment(
self,
issue: JIRAIssue,
log: Log,
) -> JIRAComment:
comment_body = self._message_formatter.format_log(
log=log, ) + self._get_attachments_list_description(
title='*Attachments*:',
item_template=self.
_attachments_list_description_item_jira_template,
attachments=log.attachments,
)
markdown_description = ''
body_attachment = None
if len(comment_body) > _TEXT_MAX_SIZE:
body_attachment = self._build_external_description_attachment(
name=f'comment-{log.log_id}-description.md', )
markdown_description = ReportMessageMarkdownFormatter().format_log(
log=log, ) + self._get_attachments_list_description(
title='**Attachments**:',
item_template=self.
_attachments_list_description_item_markdown_template,
attachments=log.attachments,
)
log_copy = deepcopy(log)
log_copy.message_html = (
'<p>This comment is too large to fit into a JIRA comment. ' +
f'See attachment <a href="{body_attachment.url}">{body_attachment.original_name}</a> '
+ 'for more details.</p>')
comment_body = self._message_formatter.format_log(
log=log_copy, ) + self._get_attachments_list_description(
title='*Attachments*:',
item_template=self.
_attachments_list_description_item_jira_template,
attachments=[
body_attachment,
*log.attachments,
],
)
comment_body, markdown_description = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=issue,
attachments=log.attachments,
),
referencing_texts=[
comment_body,
markdown_description,
],
)
if body_attachment:
body_attachment.data_loader = lambda: bytes(
markdown_description, 'utf-8')
comment_body = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=issue,
attachments=[
body_attachment,
],
),
referencing_texts=[
comment_body,
],
)[0]
try:
return self._get_client().add_comment(
issue=str(issue),
body=comment_body,
)
except JIRAError as e:
raise JiraTrackerClientError(
f'Unable to add JIRA comment for issue {issue} in project {self.configuration.project}',
) from e
def send_report(
self,
report: Report,
) -> TrackerIssue:
"""
Send a report to the tracker.
Args:
report: a report
Returns:
information about the sent report
"""
self._ensure_auth()
title = self._message_formatter.format_report_title(report=report, )
description = self._message_formatter.format_report_description(
report=report, ) + self._get_attachments_list_description(
title='*Attachments*:',
item_template=self.
_attachments_list_description_item_jira_template,
attachments=report.attachments,
)
markdown_description = ''
description_attachment = None
if len(description) > _TEXT_MAX_SIZE:
description_attachment = self._build_external_description_attachment(
name=
f'report-{report.local_id.replace("#", "")}-description.md', )
markdown_description = ReportMessageMarkdownFormatter(
).format_report_description(
report=report, ) + self._get_attachments_list_description(
title='**Attachments**:',
item_template=self.
_attachments_list_description_item_markdown_template,
attachments=report.attachments,
)
report_copy = deepcopy(report)
report_copy.description_html = (
'<p>This report description is too large to fit into a JIRA issue. '
+
f'See attachment <a href="{description_attachment.url}">{description_attachment.original_name}</a> '
+ 'for more details.</p>')
description = self._message_formatter.format_report_description(
report=report_copy, ) + self._get_attachments_list_description(
title='*Attachments*:',
item_template=self.
_attachments_list_description_item_jira_template,
attachments=[
description_attachment,
*report.attachments,
],
)
jira_issue = self._create_issue(
title=title,
description=
'This issue is being synchronized. Please check back in a moment.',
)
description, markdown_description = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=jira_issue,
attachments=report.attachments,
),
referencing_texts=[
description,
markdown_description,
],
)
if description_attachment:
description_attachment.data_loader = lambda: bytes(
markdown_description, 'utf-8')
description = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=jira_issue,
attachments=[
description_attachment,
],
),
referencing_texts=[
description,
],
)[0]
jira_issue.update(description=description, )
return self._build_tracker_issue(
issue_id=jira_issue.key,
issue_url=jira_issue.permalink(),
closed=False,
)
class GitHubTrackerClient(TrackerClient[GitHubConfiguration]):
"""A GitHub tracker client."""
_attachment_name_regex_template = Template(r'(?:!?\[)([^\[\]]*)(?:\])(?:\(${url}\))')
_attachment_substitute_regex_template = Template(r'!?\[[^\[\]]*\]\(${url}\)')
_github: Github
_attachment_uploader: GitHubAttachmentUploader
_message_formatter: ReportMessageMarkdownFormatter
_default_timezone: timezone = timezone.utc
def __init__(
self,
configuration: GitHubConfiguration,
):
"""
Initialize self.
Args:
configuration: a GitHub configuration
"""
super().__init__(
configuration=configuration,
)
self._github = Github(
base_url=cast(str, self.configuration.url),
login_or_token=cast(str, self.configuration.token),
verify=cast(bool, self.configuration.verify),
)
self._attachment_uploader = GitHubAttachmentUploader(
configuration=self.configuration,
)
self._message_formatter = ReportMessageMarkdownFormatter()
@property
def tracker_type(self) -> str:
"""
Get the type of the tracker client.
Returns:
the type of the tracker client
"""
return 'GitHub'
def _build_tracker_issue(
self,
issue_id: str,
issue_url: str,
closed: bool,
) -> TrackerIssue:
return TrackerIssue(
tracker_url=cast(str, self.configuration.url),
project=cast(str, self.configuration.project),
issue_id=issue_id,
issue_url=issue_url,
closed=closed,
)
def get_tracker_issue(
self,
issue_id: str,
) -> Optional[TrackerIssue]:
"""
Get a tracker issue.
Args:
issue_id: an issue id
Returns:
The issue if it exists, else None
"""
self._ensure_auth()
try:
github_issue = self._get_github_issue(
issue_id=issue_id,
)
except GitHubTrackerClientError:
return None
if github_issue is None:
return None
return self._build_tracker_issue(
issue_id=issue_id,
issue_url=github_issue.html_url,
closed=github_issue.closed_at is not None,
)
def get_tracker_issue_comments(
self,
issue_id: str,
exclude_comments: Optional[List[str]] = None,
) -> List[TrackerIssueComment]:
"""
Get a list of comments on an issue.
Args:
issue_id: an issue id
exclude_comments: an optional list of comment to exclude
Returns:
The list of comments
"""
self._ensure_auth()
try:
github_issue = self._get_github_issue(
issue_id=issue_id,
)
except GitHubTrackerClientError:
return []
if not github_issue:
return []
return self._extract_comments(
github_issue=github_issue,
exclude_comments=exclude_comments,
)
def send_report(
self,
report: Report,
) -> TrackerIssue:
"""
Send a report to the tracker.
Args:
report: a report
Returns:
information about the sent report
"""
self._ensure_auth()
repository = self._get_repository()
title = self._message_formatter.format_report_title(
report=report,
)
description = self._message_formatter.format_report_description(
report=report,
) + self._get_attachments_list_description(
attachments=report.attachments,
)
external_description = ''
description_attachment = None
if len(description) > _TEXT_MAX_SIZE:
external_description = description
description_attachment = self._build_external_description_attachment(
name=f'report-{report.local_id.replace("#", "")}-description.md',
)
report_copy = deepcopy(report)
report_copy.description_html = (
'<p>This report description is too large to fit into a GitHub issue. '
+ f'See attachment <a href="{description_attachment.url}">{description_attachment.original_name}</a> '
+ 'for more details.</p>'
)
description = self._message_formatter.format_report_description(
report=report_copy,
) + self._get_attachments_list_description(
attachments=[
description_attachment,
*report.attachments,
],
)
issue = self._create_issue(
github_repository=repository,
title=title,
body='This issue is being synchronized. Please check back in a moment.',
)
description, external_description = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=issue,
attachments=report.attachments,
),
referencing_texts=[
description,
external_description,
],
)
if description_attachment:
description_attachment.data_loader = lambda: bytes(external_description, 'utf-8')
description = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=issue,
attachments=[
description_attachment,
],
),
referencing_texts=[
description,
],
)[0]
issue.edit(
body=description,
)
return self._build_tracker_issue(
issue_id=str(issue.id),
issue_url=issue.html_url,
closed=False,
)
def _build_external_description_attachment(
self,
name: str,
) -> Attachment:
return Attachment(
attachment_id=0,
name=name,
original_name=name,
mime_type='text/markdown',
size=0,
url=f'http://tracker/external/{name}',
data_loader=lambda: bytes('', 'utf-8'),
)
def send_logs(
self,
tracker_issue: TrackerIssue,
logs: Iterable[Log],
) -> SendLogsResult:
"""
Send logs to the tracker.
Args:
tracker_issue: information about the tracker issue
logs: a list of comments
Raises:
GitHubTrackerClientError: if an error occurs
Returns:
information about the sent comments
"""
self._ensure_auth()
tracker_comments = SendLogsResult(
tracker_issue=tracker_issue,
added_comments=[],
)
github_issue = self._get_github_issue(
issue_id=tracker_issue.issue_id,
)
if not github_issue:
raise GitHubTrackerClientError(f'GitHub issue {tracker_issue.issue_id} not found')
for log in logs:
github_comment = self._add_comment(
github_issue=github_issue,
log=log,
)
tracker_comments.added_comments.append(
TrackerIssueComment(
author=github_comment.user.name or github_comment.user.login,
created_at=self._ensure_timezone(
dt=github_comment.created_at,
tz=self._default_timezone,
),
comment_id=str(github_comment.id),
body=github_comment.body,
attachments={},
),
)
return tracker_comments
def _ensure_timezone(
self,
dt: datetime,
tz: timezone,
) -> datetime:
if not dt.tzinfo:
dt = dt.replace(tzinfo=tz)
return dt
def test(
self,
) -> None:
"""
Test the client.
Raises:
GitHubTrackerClientError: if the test failed
"""
try:
login = self._github.get_user().login
except GithubException as e:
raise GitHubTrackerClientError('Unable to log in with GitHub API client') from e
if not login:
raise GitHubTrackerClientError('Unable to log in with GitHub API client')
def _extract_comments(
self,
github_issue: Issue,
exclude_comments: Optional[List[str]] = None,
) -> List[TrackerIssueComment]:
return [
self._extract_comment(
github_comment=github_comment,
)
for github_comment in github_issue.get_comments()
if exclude_comments is None or str(github_comment.id) not in exclude_comments
]
def _extract_comment(
self,
github_comment: IssueComment,
) -> TrackerIssueComment:
github_body = github_comment.body
inline_images = _RE_IMAGE.findall(github_body)
comment_attachments: Dict[str, TrackerAttachment] = {}
for _, inline_url in inline_images:
attachment = self._download_attachment(
url=inline_url,
)
if attachment:
comment_attachments[inline_url] = attachment
return TrackerIssueComment(
created_at=self._ensure_timezone(
dt=github_comment.created_at,
tz=self._default_timezone,
),
author=github_comment.user.name or github_comment.user.login,
comment_id=str(github_comment.id),
body=github_body,
attachments=comment_attachments,
)
def _download_attachment(
self,
url: str,
) -> Optional[TrackerAttachment]:
try:
response = requests.get(url)
except requests.RequestException:
return None
content_disposition = response.headers.get('Content-Disposition')
filename = None
if content_disposition:
match = _RE_CONTENT_DISPOSITION_FILENAME.search(content_disposition)
if match:
filename = match.group(1)
if not content_disposition or not filename:
filename = os.path.basename(url)
return TrackerAttachment(
filename=filename,
mime_type=response.headers.get('Content-Type', 'text/plain'),
content=response.content,
)
def _get_repository(
self,
) -> Repository:
project = cast(str, self.configuration.project)
if project[0] == '/':
project = project[1:]
try:
return self._github.get_repo(
project,
)
except (GithubException, requests.RequestException) as e:
raise GitHubTrackerClientError(
f'Unable to get GitHub repository {self.configuration.project}',
) from e
def _create_issue(
self,
github_repository: Repository,
title: str,
body: str,
) -> Issue:
try:
return github_repository.create_issue(
title=title,
body=body,
)
except GithubException as e:
raise GitHubTrackerClientError(
f'Unable to create issue for project {self.configuration.project} to GitHub',
) from e
def _add_comment(
self,
github_issue: Issue,
log: Log,
) -> IssueComment:
comment_body = self._message_formatter.format_log(
log=log,
) + self._get_attachments_list_description(
attachments=log.attachments,
)
external_body = ''
body_attachment = None
if len(comment_body) > _TEXT_MAX_SIZE:
external_body = comment_body
body_attachment = self._build_external_description_attachment(
name=f'comment-{log.log_id}-description.md',
)
log_copy = deepcopy(log)
log_copy.message_html = (
'<p>This comment is too large to fit into a GitHub comment. '
+ f'See attachment <a href="{body_attachment.url}">{body_attachment.original_name}</a> '
+ 'for more details.</p>'
)
comment_body = self._message_formatter.format_log(
log=log_copy,
) + self._get_attachments_list_description(
attachments=[
body_attachment,
*log.attachments,
],
)
try:
github_comment = github_issue.create_comment(
body='This comment is being synchronized. Please check back in a moment.',
)
except GithubException as e:
raise GitHubTrackerClientError(
f'Unable to add GitHub comment for issue {github_issue} in project {self.configuration.project}',
) from e
comment_body, external_body = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=github_issue,
attachments=log.attachments,
),
referencing_texts=[
comment_body,
external_body,
],
)
if body_attachment:
body_attachment.data_loader = lambda: bytes(external_body, 'utf-8')
comment_body = self._replace_attachments_references(
uploads=self._upload_attachments(
issue=github_issue,
attachments=[
body_attachment,
],
),
referencing_texts=[
comment_body,
],
)[0]
github_comment.edit(
body=comment_body,
)
return github_comment
def _get_github_issue(
self,
issue_id: str,
) -> Optional[Issue]:
github_repository = self._get_repository()
issue_id_int = int(issue_id)
try:
for issue in github_repository.get_issues(state='all'):
if issue.id == issue_id_int:
return issue
except GithubException as e:
raise GitHubTrackerClientError(
f'GitHub issue {issue_id} not found in project {self.configuration.project}',
) from e
return None
def _replace_attachments_references(
self,
uploads: List[AttachmentUploadResult],
referencing_texts: List[str],
) -> List[str]:
for attachment, upload_url, error_message in uploads:
if upload_url:
referencing_texts = [
text.replace(
attachment.url,
upload_url,
)
for text in referencing_texts
]
elif error_message:
referencing_texts = [
self._substitute_attachment_url(
body=text,
url=attachment.url,
substitution=error_message,
)
for text in referencing_texts
]
return referencing_texts
def _upload_attachments(
self,
issue: Issue,
attachments: List[Attachment],
) -> List[AttachmentUploadResult]:
uploads = []
for attachment in attachments:
url = None
error_message = None
if self.configuration.github_cdn_on:
url = self._attachment_uploader.upload_attachment(
issue=issue,
attachment=attachment,
)
if not url:
error_message = f'(Attachment "{attachment.original_name}" not available due to upload error)'
else:
error_message = (
f'(Attachment "{attachment.original_name}" not available '
+ 'due to export script’s configuration)'
)
uploads.append(
(
attachment,
url,
error_message,
),
)
return uploads
def _get_attachments_list_description(
self,
attachments: List[Attachment],
) -> str:
attachments_lines = []
if attachments:
attachments_lines = [
'',
'**Attachments**:',
]
for attachment in attachments:
attachments_lines.append(
f'- [{attachment.original_name}]({attachment.url})',
)
attachments_lines.append('')
return '\n'.join(attachments_lines)
def _extract_attachment_name(
self,
referencing_texts: List[str],
attachment: Attachment,
) -> str:
attachment_name_regex = self._attachment_name_regex_template.substitute(
url=re.escape(attachment.url),
)
for text in referencing_texts:
matches = re.findall(
attachment_name_regex,
text,
)
if matches:
return cast(str, matches[0])
return attachment.original_name
def _substitute_attachment_url(
self,
body: str,
url: str,
substitution: str,
) -> str:
attachment_substitute_regex = self._attachment_substitute_regex_template.substitute(
url=re.escape(url),
)
return re.sub(
attachment_substitute_regex,
substitution,
body,
)
def _ensure_auth(
self,
) -> None:
try:
user_id = self._github.get_user().id
except GithubException as e:
raise GitHubTrackerClientError('Unable to authenticate to GitHub') from e
if not isinstance(user_id, int):
raise GitHubTrackerClientError('Unable to authenticate to GitHub')