def build_package_from_pr_number(gh_token, sdk_id, pr_number, output_folder, *, with_comment=False):
"""Will clone the given PR branch and vuild the package with the given name."""
con = Github(gh_token)
repo = con.get_repo(sdk_id)
sdk_pr = repo.get_pull(pr_number)
package_names = get_package_names(sdk_pr)
absolute_output_folder = Path(output_folder).resolve()
with tempfile.TemporaryDirectory() as temp_dir, manage_git_folder(
gh_token, Path(temp_dir) / Path("sdk"), sdk_id, pr_number=pr_number
) as sdk_repo_root:
for _, package_name in package_names:
_LOGGER.debug("Build {}".format(package_name))
execute_simple_command(
["python", "./build_package.py", "--dest", str(absolute_output_folder), package_name], cwd=sdk_repo_root
)
_LOGGER.debug("Build finished: {}".format(package_name))
if with_comment:
files = [f.name for f in absolute_output_folder.iterdir()]
comment_message = None
dashboard = DashboardCommentableObject(sdk_pr, "(message created by the CI based on PR content)")
try:
installation_message = build_installation_message(sdk_pr)
download_message = build_download_message(sdk_pr, files)
comment_message = installation_message + "\n\n" + download_message
dashboard.create_comment(comment_message)
except Exception:
_LOGGER.critical("Unable to do PR comment:\n%s", comment_message)
def build_package_from_pr_number(gh_token, sdk_id, pr_number, output_folder, *, with_comment=False):
"""Will clone the given PR branch and vuild the package with the given name."""
con = Github(gh_token)
repo = con.get_repo(sdk_id)
sdk_pr = repo.get_pull(pr_number)
# "get_files" of Github only download the first 300 files. Might not be enough.
package_names = {f.filename.split('/')[0] for f in sdk_pr.get_files() if f.filename.startswith("azure")}
absolute_output_folder = Path(output_folder).resolve()
with tempfile.TemporaryDirectory() as temp_dir, \
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), sdk_id, pr_number=pr_number) as sdk_folder:
for package_name in package_names:
_LOGGER.debug("Build {}".format(package_name))
execute_simple_command(
["python", "./build_package.py", "--dest", str(absolute_output_folder), package_name],
cwd=sdk_folder
)
_LOGGER.debug("Build finished: {}".format(package_name))
if with_comment:
files = [f.name for f in absolute_output_folder.iterdir()]
comment_message = None
dashboard = DashboardCommentableObject(sdk_pr, "(message created by the CI based on PR content)")
try:
installation_message = build_installation_message(sdk_pr)
download_message = build_download_message(sdk_pr, files)
comment_message = installation_message + "\n\n" + download_message
dashboard.create_comment(comment_message)
except Exception:
_LOGGER.critical("Unable to do PR comment:\n%s", comment_message)
def test_dashboard(self, format_exc):
format_exc.return_value = "something to do with an exception"
# Prepare
repo = self.g.get_repo("lmazuel/TestingRepo")
issue = repo.get_issue(15)
initial_size = len(list(issue.get_comments()))
header = "# MYHEADER"
dashboard = DashboardCommentableObject(issue, header)
with exception_to_github(dashboard, "Python bot") as error:
"Test".fakemethod(12) # pylint: disable=no-member
after_size = len(list(issue.get_comments()))
assert after_size == initial_size + 1
assert error.comment is not None
assert "Encountered an unknown error" in error.comment.body
dashboard.create_comment("New text comment")
after_size_2 = len(list(issue.get_comments()))
assert after_size == after_size_2
# Clean my mess
error.comment.delete()
def build_package_from_pr_number(gh_token,
sdk_id,
pr_number,
output_folder,
*,
with_comment=False):
"""Will clone the given PR branch and vuild the package with the given name."""
con = Github(gh_token)
repo = con.get_repo(sdk_id)
sdk_pr = repo.get_pull(pr_number)
files = [
one_file.filename for one_file in sdk_pr.get_files()
if one_file.status not in ['removed']
]
# "get_files" of Github only download the first 300 files. Might not be enough.
package_names = {('.', f.split('/')[0])
for f in files if f.startswith("azure")}
# Handle the SDK folder as well
matches = {_SDK_FOLDER_RE.search(f) for f in files}
package_names.update(
{match.groups()
for match in matches if match is not None})
absolute_output_folder = Path(output_folder).resolve()
with tempfile.TemporaryDirectory() as temp_dir, \
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), sdk_id, pr_number=pr_number) as sdk_repo_root:
for _, package_name in package_names:
_LOGGER.debug("Build {}".format(package_name))
execute_simple_command([
"python", "./build_package.py", "--dest",
str(absolute_output_folder), package_name
],
cwd=sdk_repo_root)
_LOGGER.debug("Build finished: {}".format(package_name))
if with_comment:
files = [f.name for f in absolute_output_folder.iterdir()]
comment_message = None
dashboard = DashboardCommentableObject(
sdk_pr, "(message created by the CI based on PR content)")
try:
installation_message = build_installation_message(sdk_pr)
download_message = build_download_message(sdk_pr, files)
comment_message = installation_message + "\n\n" + download_message
dashboard.create_comment(comment_message)
except Exception:
_LOGGER.critical("Unable to do PR comment:\n%s", comment_message)
def rest_pr_management(rest_pr,
sdk_repo,
sdk_tag,
sdk_default_base=_DEFAULT_SDK_BRANCH):
"""What to do when something happen to a PR in the Rest repo.
:param restpr: a PyGithub pull object
:type restpr: github.PullRequest.PullRequest
:param sdk_repo: a PyGithub repository
:type sdk_repo: github.Repository.Repository
:param str sdk_tag: repotag to use to filter SwaggerToSDK conf
:param str sdk_default_base: Default SDK branch.
"""
# Extract some metadata as variables
rest_repo = rest_pr.base.repo
# "repo" can be None if fork has been deleted.
is_from_a_fork = rest_pr.head.repo is None or rest_pr.head.repo.full_name != rest_repo.full_name
# THE comment were we put everything
dashboard = DashboardCommentableObject(
rest_pr, "# Automation for {}".format(sdk_tag))
#
# Work on context, ext if context is not good
#
context_tags = list(get_context_tag_from_git_object(rest_pr))
if not context_tags:
dashboard.create_comment(
"Unable to detect any generation context from this PR.")
return
if len(context_tags) > _CONTEXT_TAG_LIMITS:
dashboard.create_comment(
"This PR contains more than {} context, SDK generation is not enabled. Contexts found:\n{}"
.format(_CONTEXT_TAG_LIMITS,
"\n".join(["- {}".format(ctxt) for ctxt in context_tags])))
return
#
# Decide if this PR will use a context branch
#
# A RestPR will have a context branch if:
# - This is from a fork. Local branch are considered of the context system.
# - There is one context only. Too much complicated to handle two context branches.
# - Base is master. If fork to a feature branch, keep that flow.
is_pushed_to_context_branch = is_from_a_fork and len(
context_tags) == 1 and rest_pr.base.ref == _DEFAULT_REST_BRANCH
#
# Compute the "head" of future SDK PR.
#
if is_from_a_fork:
sdk_pr_head = _SDK_PR_TEMPLATE.format(rest_pr.number)
else:
sdk_pr_head = _SDK_PR_TEMPLATE.format(rest_pr.head.ref)
#
# Compute the "base" of future SDK PR.
# "sdk_checkout_bases" is an ordered list of branchs to checkout, since this can be several
# branches that derives from "master".
# This are branches that SwaggerToSDK should have "push" permission. Do NOT add "master" or a protected
# branch to that list.
#
sdk_checkout_bases = []
if rest_pr.base.ref == _DEFAULT_REST_BRANCH:
sdk_pr_base = sdk_default_base
else:
sdk_pr_base = _SDK_PR_TEMPLATE.format(rest_pr.base.ref)
sdk_checkout_bases.append(sdk_pr_base)
# In special case where I use context branch
if is_pushed_to_context_branch:
sdk_pr_base = _SDK_PR_TEMPLATE.format(context_tags[0])
sdk_checkout_bases.insert(0, sdk_pr_base)
#
# Try to generate on "head", whatever the state of the PR.
#
generate_sdk_from_git_object(
rest_pr,
sdk_pr_head,
None, # We don't need repo id if it's a PR, infer from PR itself.
sdk_repo.full_name,
sdk_checkout_bases,
fallback_base_branch_name=sdk_default_base,
sdk_tag=sdk_tag)
#
# Try to create/get a SDK PR.
#
# There is a lot of reasons why a SDK PR could not exist even on a "close" event, so don't assume this exists.
#
try:
sdk_pr = get_or_create_pull(
sdk_repo,
title='[AutoPR {}] {}'.format("/".join(context_tags),
rest_pr.title),
body="Created to sync {}".format(rest_pr.html_url),
head=sdk_repo.owner.login + ":" + sdk_pr_head,
base=sdk_pr_base,
)
except Exception as err:
_LOGGER.warning("Unable to create SDK PR: %s", err)
dashboard.create_comment("Nothing to generate for {}".format(sdk_tag))
return
# Replace whatever message it was if we were able to do a PR
dashboard.create_comment("A PR has been created for you:\n{}".format(
sdk_pr.html_url))
#
# Manage labels/state on this SDK PR.
#
sdk_pr_as_issue = sdk_repo.get_issue(sdk_pr.number)
sdk_pr_merged = False
if rest_pr.closed_at: # If there is a date, this is closed
head_ref = sdk_repo.get_git_ref("heads/{}".format(sdk_pr_head))
if rest_pr.merged:
manage_labels(sdk_pr_as_issue,
to_add=[SwaggerToSdkLabels.merged],
to_remove=[SwaggerToSdkLabels.in_progress])
if sdk_pr.base.ref.startswith(_SDK_PR_PREFIX):
try:
# Merge "single context PRs" automatically
sdk_pr.merge(merge_method="squash")
sdk_pr_merged = True
# Delete branch from merged PR
head_ref.delete()
except Exception as err:
_LOGGER.warning("Was unable to merge: %s", err)
else:
manage_labels(sdk_pr_as_issue,
to_add=[SwaggerToSdkLabels.refused],
to_remove=[SwaggerToSdkLabels.in_progress])
sdk_pr.edit(state="closed")
# Delete branch from closed PR
head_ref.delete()
else:
# Try to remove "refused", if it was re-opened
manage_labels(sdk_pr_as_issue,
to_add=[SwaggerToSdkLabels.in_progress],
to_remove=[SwaggerToSdkLabels.refused])
#
# Extra work: if this was a context branch
#
if is_pushed_to_context_branch:
try:
context_pr = get_or_create_pull(
sdk_repo,
title='[AutoPR] {}'.format("/".join(context_tags)),
body="Created to accumulate context: {}".format(
context_tags[0]),
head=sdk_repo.owner.login + ":" + sdk_pr_base,
base=sdk_default_base,
)
except Exception as err:
_LOGGER.warning("Unable to create context PR: %s", err)
return
# We got the context PR!
context_pr_as_issue = sdk_repo.get_issue(context_pr.number)
manage_labels(context_pr_as_issue, [SwaggerToSdkLabels.service_pr])
# Put a link into the SDK single PR
if sdk_pr_merged:
sdk_pr.create_issue_comment(
"This PR has been merged into {}".format(context_pr.html_url))
# Update dashboar to talk about this PR
if sdk_pr.merged:
msg = "The initial [PR]({}) has been merged into your service PR:\n{}".format(
sdk_pr.html_url, context_pr.html_url)
else:
msg = "A [PR]({}) has been created for you based on this PR content.\n\n".format(
sdk_pr.html_url)
msg += "Once this PR will be merged, content will be added to your service PR:\n{}".format(
context_pr.html_url)
dashboard.create_comment(msg)
def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK_BRANCH):
"""What to do when something happen to a PR in the Rest repo.
:param restpr: a PyGithub pull object
:type restpr: github.PullRequest.PullRequest
:param sdk_repo: a PyGithub repository
:type sdk_repo: github.Repository.Repository
:param str sdk_tag: repotag to use to filter SwaggerToSDK conf
:param str sdk_default_base: Default SDK branch.
"""
# Extract some metadata as variables
rest_repo = rest_pr.base.repo
# "repo" can be None if fork has been deleted.
is_from_a_fork = rest_pr.head.repo is None or rest_pr.head.repo.full_name != rest_repo.full_name
# THE comment were we put everything
dashboard = DashboardCommentableObject(rest_pr, "# Automation for {}".format(sdk_tag))
#
# Work on context, ext if context is not good
#
context_tags = list(get_context_tag_from_git_object(rest_pr))
if not context_tags:
dashboard.create_comment("Unable to detect any generation context from this PR.")
return
if len(context_tags) > _CONTEXT_TAG_LIMITS:
dashboard.create_comment(
"This PR contains more than {} context, SDK generation is not enabled. Contexts found:\n{}".format(
_CONTEXT_TAG_LIMITS,
"\n".join(["- {}".format(ctxt) for ctxt in context_tags])
))
return
#
# Decide if this PR will use a context branch
#
# A RestPR will have a context branch if:
# - This is from a fork. Local branch are considered of the context system.
# - There is one context only. Too much complicated to handle two context branches.
# - Base is master. If fork to a feature branch, keep that flow.
is_pushed_to_context_branch = is_from_a_fork and len(context_tags) == 1 and rest_pr.base.ref == _DEFAULT_REST_BRANCH
#
# Compute the "head" of future SDK PR.
#
if is_from_a_fork:
sdk_pr_head = _SDK_PR_TEMPLATE.format(rest_pr.number)
else:
sdk_pr_head = _SDK_PR_TEMPLATE.format(rest_pr.head.ref)
#
# Compute the "base" of future SDK PR.
# "sdk_checkout_bases" is an ordered list of branchs to checkout, since this can be several
# branches that derives from "master".
# This are branches that SwaggerToSDK should have "push" permission. Do NOT add "master" or a protected
# branch to that list.
#
sdk_checkout_bases = []
if rest_pr.base.ref == _DEFAULT_REST_BRANCH:
sdk_pr_base = sdk_default_base
else:
sdk_pr_base = _SDK_PR_TEMPLATE.format(rest_pr.base.ref)
sdk_checkout_bases.append(sdk_pr_base)
# In special case where I use context branch
if is_pushed_to_context_branch:
sdk_pr_base = _SDK_PR_TEMPLATE.format(context_tags[0])
sdk_checkout_bases.insert(0, sdk_pr_base)
#
# Try to generate on "head", whatever the state of the PR.
#
generate_sdk_from_git_object(
rest_pr,
sdk_pr_head,
None, # We don't need repo id if it's a PR, infer from PR itself.
sdk_repo.full_name,
sdk_checkout_bases,
fallback_base_branch_name=sdk_default_base,
sdk_tag=sdk_tag
)
#
# Try to create/get a SDK PR.
#
# There is a lot of reasons why a SDK PR could not exist even on a "close" event, so don't assume this exists.
#
try:
sdk_pr = get_or_create_pull(
sdk_repo,
title='[AutoPR {}] {}'.format("/".join(context_tags), rest_pr.title),
body="Created to sync {}".format(rest_pr.html_url),
head=sdk_repo.owner.login+":"+sdk_pr_head,
base=sdk_pr_base,
)
except Exception as err:
_LOGGER.warning("Unable to create SDK PR: %s", err)
dashboard.create_comment("Nothing to generate for {}".format(sdk_tag))
return
# Replace whatever message it was if we were able to do a PR
dashboard.create_comment("A PR has been created for you:\n{}".format(sdk_pr.html_url))
#
# Manage labels/state on this SDK PR.
#
sdk_pr_as_issue = sdk_repo.get_issue(sdk_pr.number)
sdk_pr_merged = False
if rest_pr.closed_at: # If there is a date, this is closed
head_ref = sdk_repo.get_git_ref("heads/{}".format(sdk_pr_head))
if rest_pr.merged:
manage_labels(sdk_pr_as_issue,
to_add=[SwaggerToSdkLabels.merged],
to_remove=[SwaggerToSdkLabels.in_progress])
if sdk_pr.base.ref.startswith(_SDK_PR_PREFIX):
try:
# Merge "single context PRs" automatically
sdk_pr.merge(merge_method="squash")
sdk_pr_merged = True
# Delete branch from merged PR
head_ref.delete()
except Exception as err:
_LOGGER.warning("Was unable to merge: %s", err)
else:
manage_labels(sdk_pr_as_issue,
to_add=[SwaggerToSdkLabels.refused],
to_remove=[SwaggerToSdkLabels.in_progress])
sdk_pr.edit(state="closed")
# Delete branch from closed PR
head_ref.delete()
else:
# Try to remove "refused", if it was re-opened
manage_labels(sdk_pr_as_issue,
to_add=[SwaggerToSdkLabels.in_progress],
to_remove=[SwaggerToSdkLabels.refused])
#
# Extra work: if this was a context branch
#
if is_pushed_to_context_branch:
try:
context_pr = get_or_create_pull(
sdk_repo,
title='[AutoPR] {}'.format("/".join(context_tags)),
body="Created to accumulate context: {}".format(context_tags[0]),
head=sdk_repo.owner.login+":"+sdk_pr_base,
base=sdk_default_base,
)
except Exception as err:
_LOGGER.warning("Unable to create context PR: %s", err)
return
# We got the context PR!
context_pr_as_issue = sdk_repo.get_issue(context_pr.number)
manage_labels(context_pr_as_issue, [SwaggerToSdkLabels.service_pr])
# Put a link into the SDK single PR
if sdk_pr_merged:
sdk_pr.create_issue_comment("This PR has been merged into {}".format(context_pr.html_url))
# Update dashboar to talk about this PR
if sdk_pr.merged:
msg = "The initial [PR]({}) has been merged into your service PR:\n{}".format(
sdk_pr.html_url,
context_pr.html_url
)
else:
msg = "A [PR]({}) has been created for you based on this PR content.\n\n".format(
sdk_pr.html_url
)
msg += "Once this PR will be merged, content will be added to your service PR:\n{}".format(
context_pr.html_url
)
dashboard.create_comment(msg)