class GitHubPR:
def __init__(self, token: str, repo: str, user: str) -> None:
self.repo = Github(token,
user_agent=USER_AGENT).get_repo(f'{user}/{repo}')
def create_pull_request(self,
title: str,
branch: str,
body: str,
labels: Optional[List[str]] = None) -> None:
try:
pull = self.repo.create_pull(title, body, 'master', branch)
logging.info('Pull request for %s opened at %s', branch,
pull.html_url)
if labels:
# Labels have to be set separately
pull.set_labels(*labels)
except GithubException as exc:
logging.error('Pull request for %s failed: %s',
branch,
self.get_error_message(exc.data),
exc_info=exc)
@staticmethod
def get_error_message(exc_data: Dict[str, Any]) -> str:
return ' - '.join([
exc_data.get('message', 'Unknown error'),
*(err['message']
for err in exc_data.get('errors', []) if 'message' in err)
])
def create_git_pr(s3_client, hyper_params,
deployment_type): # pragma: no cover
"""Create a git PR automatically if recall_at_30 is higher than previous iteration."""
upstream_repo = Github(GITHUB_TOKEN).get_repo(
f'{UPSTREAM_REPO_NAME}/{PROJECT_NAME}')
deployed_data, yaml_data, latest_commit_hash = read_deployed_data(
upstream_repo, s3_client, deployment_type)
recall_at_30 = hyper_params['recall_at_30']
deployed_recall_at_30 = deployed_data['hyperparams'].get(
'recall_at_30', 0.55)
logger.info(
'create_git_pr:: Deployed => Model %s, Recall %f Current => Model %s, Recall %f',
deployed_data['version'], deployed_recall_at_30, MODEL_VERSION,
recall_at_30)
if recall_at_30 >= deployed_recall_at_30:
promotion_creteria = 'current_recall_at_30 >= deployed_recall_at_30'
params = hyper_params.copy()
params.update({'promotion_criteria': str(promotion_creteria)})
branch_name, commit_message = create_branch_and_update_yaml(
deployment_type, deployed_data, yaml_data, params,
latest_commit_hash)
hyper_params_formated = build_hyper_params_message(hyper_params)
prev_hyper_params_formated = build_hyper_params_message(
deployed_data['hyperparams'])
body = f'''Current deployed model details:
- Model version :: `{deployed_data['version']}`
{prev_hyper_params_formated}
New model details:
- Model version :: `{MODEL_VERSION}`
{hyper_params_formated}
Criteria for promotion is `{promotion_creteria}`
'''
pr = upstream_repo.create_pull(title=commit_message,
body=format_body(body),
head=f'{FORK_REPO_NAME}:{branch_name}',
base='refs/heads/master')
logger.info('Raised SAAS %s for review', pr)
else:
logger.warn(
'Ignoring latest model %s as its recall %f is less than '
'existing model %s recall %f', MODEL_VERSION, recall_at_30,
deployed_data['version'], deployed_recall_at_30)
def process_event(event_name, event_data, repo, branch, base, remote_exists):
# Fetch required environment variables
github_token = os.environ['GITHUB_TOKEN']
github_repository = os.environ['GITHUB_REPOSITORY']
# Fetch optional environment variables with default values
commit_message = os.getenv(
'COMMIT_MESSAGE',
"Auto-committed changes by create-pull-request action")
title = os.getenv(
'PULL_REQUEST_TITLE',
"Auto-generated by create-pull-request action")
body = os.getenv(
'PULL_REQUEST_BODY', "Auto-generated pull request by "
"[create-pull-request](https://github.com/peter-evans/create-pull-request) GitHub Action")
# Fetch optional environment variables with no default values
pull_request_labels = os.environ.get('PULL_REQUEST_LABELS')
pull_request_assignees = os.environ.get('PULL_REQUEST_ASSIGNEES')
pull_request_milestone = os.environ.get('PULL_REQUEST_MILESTONE')
pull_request_reviewers = os.environ.get('PULL_REQUEST_REVIEWERS')
pull_request_team_reviewers = os.environ.get('PULL_REQUEST_TEAM_REVIEWERS')
# Update URL for the 'origin' remote
set_git_remote_url(repo.git, github_token, github_repository)
# Push the local changes to the remote branch
print("Pushing changes.")
push_result = push_changes(repo.git, branch, commit_message)
print(push_result)
# If the remote existed then we are using fixed branch strategy.
# A PR should already exist and we can finish here.
if remote_exists:
print("Updated pull request branch %s." % branch)
sys.exit()
# Create the pull request
print("Creating a request to pull %s into %s." % (branch, base))
github_repo = Github(github_token).get_repo(github_repository)
pull_request = github_repo.create_pull(
title=title,
body=body,
base=base,
head=branch)
print("Created pull request %d." % pull_request.number)
os.system(
'echo ::set-env name=PULL_REQUEST_NUMBER::%d' %
pull_request.number)
# Set labels, assignees and milestone
if pull_request_labels is not None:
print("Applying labels")
pull_request.as_issue().edit(labels=cs_string_to_list(pull_request_labels))
if pull_request_assignees is not None:
print("Applying assignees")
pull_request.as_issue().edit(assignees=cs_string_to_list(pull_request_assignees))
if pull_request_milestone is not None:
print("Applying milestone")
milestone = github_repo.get_milestone(int(pull_request_milestone))
pull_request.as_issue().edit(milestone=milestone)
# Set pull request reviewers and team reviewers
if pull_request_reviewers is not None:
print("Requesting reviewers")
pull_request.create_review_request(
reviewers=cs_string_to_list(pull_request_reviewers))
if pull_request_team_reviewers is not None:
print("Requesting team reviewers")
pull_request.create_review_request(
team_reviewers=cs_string_to_list(pull_request_team_reviewers))
def autobuild(reason, **kwargs):
# TODO: fail gracefully if these aren't set
gh_user = os.environ['GITHUB_USER']
gh_token = os.environ['GITHUB_TOKEN']
gh_push_url = os.environ['PUSH_URL']
print "Let's autobuild this sucker!"
print "GITHUB_USER: %s" % gh_user
verbose = kwargs.get('verbose', False)
rebuild = kwargs.get('rebuild', False)
clean = kwargs.get('clean', False)
pdf_generator = 'phantomjs'
dont_remove = [
'.git', '.gitignore', '.travis.yml', 'CNAME', 'README.md',
'requirements.txt'
]
output_dir = 'output/codeclubworld_autobuild'
gh_repo = 'CodeClubWorld-Projects'
r = Github(gh_user, gh_token).get_repo('CodeClub/%s' % gh_repo)
# clone the curricula repos (the lazy way)
subprocess.call('make clone'.split())
# clone the output repo
subprocess.call(('git clone https://%s:%[email protected]/CodeClub/%s.git %s' %
(gh_user, gh_token, gh_repo, output_dir)).split())
if clean:
# delete everything in the output dir
rm_files(output_dir, dont_remove)
# init gitpython!
repo = Repo(output_dir)
# run the build
print "** running the build"
build.build(pdf_generator,
['lessons/scratch', 'lessons/webdev', 'lessons/python'],
"world", output_dir, verbose, repo, rebuild)
repo.git.remote('set-url', '--push', 'origin', gh_push_url)
# stage everything...
print "** stage everything"
repo.git.add('--all')
# NB. it seems weird, but this reason can disagree
# with the PR (since we force push)
reason_text = get_reason_text(reason)
try:
# ... commit it...
print "** commiting the rebuild"
repo.git.commit('-m', 'Rebuild', '-m', reason_text)
# ...and push!
# TODO: Don't force push here!
print "** pushing the changes"
repo.git.push('-f', 'origin', 'gh-pages')
except GitCommandError as e:
print "** ERROR GitCommandError: "
print e
sys.exit()
# submit pull request
try:
print "** creating the PR"
msg = "Hello!\n\n"
msg += "I've been hard at work, rebuilding the Code Club World's projects website from the latest markdown.\n\n"
msg += "%s and I found some updates, so I thought I'd best send a pull request. You can view my updated version here:\nhttp://%s.github.io/%s/\n\n" % (
reason_text, gh_user, gh_repo)
msg += "Have a nice day!"
r.create_pull(title='Rebuild',
body=msg,
head='%s:gh-pages' % gh_user,
base='gh-pages')
except GithubException as e:
print "** ERROR GithubException: "
print e
# TODO: handle this.
# Usually it just means the PR already exists, which is
# nothing too much to worry about.
pass
def autobuild(reason, **kwargs):
# TODO: fail gracefully if these aren't set
gh_user = os.environ['GITHUB_USER']
gh_token = os.environ['GITHUB_TOKEN']
gh_push_url = os.environ['PUSH_URL']
print "Let's autobuild this sucker!"
print "GITHUB_USER: %s" % gh_user
verbose = kwargs.get('verbose', False)
rebuild = kwargs.get('rebuild', False)
clean = kwargs.get('clean', False)
pdf_generator = 'phantomjs'
dont_remove = ['.git', '.gitignore', '.travis.yml', 'CNAME', 'README.md', 'requirements.txt']
output_dir = 'output/codeclubworld_autobuild'
gh_repo = 'CodeClubWorld-Projects'
r = Github(gh_user, gh_token).get_repo('CodeClub/%s' % gh_repo)
# clone the curricula repos (the lazy way)
subprocess.call('make clone'.split())
# clone the output repo
subprocess.call(('git clone https://%s:%[email protected]/CodeClub/%s.git %s' % (gh_user, gh_token, gh_repo, output_dir)).split())
if clean:
# delete everything in the output dir
rm_files(output_dir, dont_remove)
# init gitpython!
repo = Repo(output_dir)
# run the build
print "** running the build"
build.build(pdf_generator, ['lessons/scratch', 'lessons/webdev', 'lessons/python'], "world", output_dir, verbose, repo, rebuild)
repo.git.remote('set-url', '--push', 'origin', gh_push_url)
# stage everything...
print "** stage everything"
repo.git.add('--all')
# NB. it seems weird, but this reason can disagree
# with the PR (since we force push)
reason_text = get_reason_text(reason)
try:
# ... commit it...
print "** commiting the rebuild"
repo.git.commit('-m', 'Rebuild', '-m', reason_text)
# ...and push!
# TODO: Don't force push here!
print "** pushing the changes"
repo.git.push('-f', 'origin', 'gh-pages')
except GitCommandError as e:
print "** ERROR GitCommandError: "
print e
sys.exit()
# submit pull request
try:
print "** creating the PR"
msg = "Hello!\n\n"
msg += "I've been hard at work, rebuilding the Code Club World's projects website from the latest markdown.\n\n"
msg += "%s and I found some updates, so I thought I'd best send a pull request. You can view my updated version here:\nhttp://%s.github.io/%s/\n\n" % (reason_text, gh_user, gh_repo)
msg += "Have a nice day!"
r.create_pull(title='Rebuild', body=msg, head='%s:gh-pages' % gh_user, base='gh-pages')
except GithubException as e:
print "** ERROR GithubException: "
print e
# TODO: handle this.
# Usually it just means the PR already exists, which is
# nothing too much to worry about.
pass
def autobuild(region, reason, **kwargs):
# TODO: fail gracefully if these aren't set
gh_user = os.environ['GITHUB_USER']
gh_token = os.environ['GITHUB_TOKEN']
rebuild = kwargs.get('rebuild', False)
clean = kwargs.get('clean', False)
dont_remove = ['.git', '.gitignore', '.travis.yml', 'CNAME', 'README.md', 'requirements.txt']
output_dir = 'output/codeclub%s' % region
# case sensitivity issues
pp_region = {
'uk': 'UK',
'world': 'World'
}[region]
gh_repo = 'CodeClub%s-Projects' % pp_region
r = Github(gh_user, gh_token).get_repo('CodeClub/%s' % gh_repo)
pdf_generator = 'phantomjs'
# clone the curricula repos (the lazy way)
subprocess.call('make clone'.split())
# clone the output repo
subprocess.call(('git clone https://github.com/CodeClub/%s.git %s' % (gh_repo, output_dir)).split())
if clean:
# delete everything in the output dir
rm_files(output_dir, dont_remove)
# init gitpython!
repo = Repo(output_dir)
# run the build
build.build(pdf_generator, ['lessons/scratch', 'lessons/webdev', 'lessons/python'], region, output_dir, repo, rebuild)
# add username and token to remote url
# (so we can write)
origin_url = repo.remotes.origin.url
origin_url = 'https://%s:%[email protected]/%s/%s' % (gh_user, gh_token, gh_user, origin_url[28:])
repo.git.remote('set-url', '--push', 'origin', origin_url)
# stage everything...
repo.git.add('--all')
# NB. it seems weird, but this reason can disagree
# with the PR (since we force push)
reason_text = get_reason_text(reason)
try:
# ... commit it...
repo.git.commit('-m', 'Rebuild', '-m', reason_text)
# ...and push!
# TODO: Don't force push here!
repo.git.push('-f', 'origin', 'gh-pages')
except GitCommandError:
sys.exit()
# submit pull request
try:
msg = "Hello!\n\n"
msg += "I've been hard at work, rebuilding the Code Club %s projects website from the latest markdown.\n\n" % pp_region
msg += "%s and I found some updates, so I thought I'd best send a pull request. You can view my updated version here:\nhttp://%s.github.io/%s/\n\n" % (reason_text, gh_user, gh_repo)
msg += "Have a nice day!"
r.create_pull(title='Rebuild', body=msg, head='%s:gh-pages' % gh_user, base='gh-pages')
except GithubException:
# TODO: handle this.
# Usually it just means the PR already exists, which is
# nothing too much to worry about.
pass
def create_or_update_pull_request(
github_token,
github_repository,
branch,
base,
title,
body,
labels,
assignees,
milestone,
reviewers,
team_reviewers,
project_name,
project_column_name,
):
# Create the pull request
github_repo = Github(github_token).get_repo(github_repository)
try:
pull_request = github_repo.create_pull(title=title,
body=body,
base=base,
head=branch)
print(
f"Created pull request #{pull_request.number} ({branch} => {base})"
)
except GithubException as e:
if e.status == 422:
# A pull request exists for this branch and base
head_branch = "{}:{}".format(
github_repository.split("/")[0], branch)
# Get the pull request
pull_request = github_repo.get_pulls(state="open",
base=base,
head=head_branch)[0]
print(
f"Updated pull request #{pull_request.number} ({branch} => {base})"
)
else:
print(str(e))
raise
# Set the output variables
os.system(
f"echo ::set-env name=PULL_REQUEST_NUMBER::{pull_request.number}")
os.system(f"echo ::set-output name=pr_number::{pull_request.number}")
# Set labels, assignees and milestone
if labels is not None:
print(f"Applying labels '{labels}'")
pull_request.as_issue().edit(labels=cs_string_to_list(labels))
if assignees is not None:
print(f"Applying assignees '{assignees}'")
pull_request.as_issue().edit(assignees=cs_string_to_list(assignees))
if milestone is not None:
print(f"Applying milestone '{milestone}'")
milestone = github_repo.get_milestone(int(milestone))
pull_request.as_issue().edit(milestone=milestone)
# Set pull request reviewers
if reviewers is not None:
print(f"Requesting reviewers '{reviewers}'")
try:
pull_request.create_review_request(
reviewers=cs_string_to_list(reviewers))
except GithubException as e:
# Likely caused by "Review cannot be requested from pull request author."
if e.status == 422:
print("Request reviewers failed - {}".format(
e.data["message"]))
# Set pull request team reviewers
if team_reviewers is not None:
print(f"Requesting team reviewers '{team_reviewers}'")
pull_request.create_review_request(
team_reviewers=cs_string_to_list(team_reviewers))
# Create a project card for the pull request
if project_name is not None and project_column_name is not None:
try:
create_project_card(github_repo, project_name, project_column_name,
pull_request)
except GithubException as e:
# Likely caused by "Project already has the associated issue."
if e.status == 422:
print("Create project card failed - {}".format(
e.data["errors"][0]["message"]))
def process_event(github_token, github_repository, repo, branch, base):
# Fetch optional environment variables with default values
commit_message = os.getenv(
'COMMIT_MESSAGE',
"Auto-committed changes by create-pull-request action")
title = os.getenv('PULL_REQUEST_TITLE',
"Auto-generated by create-pull-request action")
body = os.getenv(
'PULL_REQUEST_BODY', "Auto-generated pull request by "
"[create-pull-request](https://github.com/peter-evans/create-pull-request) GitHub Action"
)
# Fetch optional environment variables with no default values
pull_request_labels = os.environ.get('PULL_REQUEST_LABELS')
pull_request_assignees = os.environ.get('PULL_REQUEST_ASSIGNEES')
pull_request_milestone = os.environ.get('PULL_REQUEST_MILESTONE')
pull_request_reviewers = os.environ.get('PULL_REQUEST_REVIEWERS')
pull_request_team_reviewers = os.environ.get('PULL_REQUEST_TEAM_REVIEWERS')
# Push the local changes to the remote branch
print("Pushing changes.")
push_result = push_changes(repo.git, branch, commit_message)
print(push_result)
# Create the pull request
github_repo = Github(github_token).get_repo(github_repository)
try:
pull_request = github_repo.create_pull(title=title,
body=body,
base=base,
head=branch)
print("Created pull request #%d (%s => %s)" %
(pull_request.number, branch, base))
except GithubException as e:
if e.status == 422:
pull_request = github_repo.get_pulls(state='open',
base=base,
head=branch)[1]
print("Updated pull request #%d (%s => %s)" %
(pull_request.number, branch, base))
else:
print(str(e))
sys.exit(1)
# Set the output variable
os.system('echo ::set-env name=PULL_REQUEST_NUMBER::%d' %
pull_request.number)
# Set labels, assignees and milestone
if pull_request_labels is not None:
print("Applying labels")
pull_request.as_issue().edit(
labels=cs_string_to_list(pull_request_labels))
if pull_request_assignees is not None:
print("Applying assignees")
pull_request.as_issue().edit(
assignees=cs_string_to_list(pull_request_assignees))
if pull_request_milestone is not None:
print("Applying milestone")
milestone = github_repo.get_milestone(int(pull_request_milestone))
pull_request.as_issue().edit(milestone=milestone)
# Set pull request reviewers
if pull_request_reviewers is not None:
print("Requesting reviewers")
try:
pull_request.create_review_request(
reviewers=cs_string_to_list(pull_request_reviewers))
except GithubException as e:
# Likely caused by "Review cannot be requested from pull request author."
if e.status == 422:
print("Requesting reviewers failed - %s" % e.data["message"])
# Set pull request team reviewers
if pull_request_team_reviewers is not None:
print("Requesting team reviewers")
pull_request.create_review_request(
team_reviewers=cs_string_to_list(pull_request_team_reviewers))
def make_pull():
repo = Github(os.environ.get("gitpat")).get_repo("QEDK/goodbot") # gitpat is parsebot's personal access token
return repo, repo.create_pull(title="Update project list", body="parsebot 🤖 hard at work 🛠️", head="parsebot", base="master", maintainer_can_modify=True)
def autobuild(region, reason, **kwargs):
# TODO: fail gracefully if these aren't set
gh_user = os.environ['GITHUB_USER']
gh_token = os.environ['GITHUB_TOKEN']
verbose = kwargs.get('verbose', False)
rebuild = kwargs.get('rebuild', False)
clean = kwargs.get('clean', False)
dont_remove = [
'.git', '.gitignore', '.travis.yml', 'CNAME', 'README.md',
'requirements.txt'
]
output_dir = 'output/codeclub%s' % region
# case sensitivity issues
pp_region = {'uk': 'UK', 'world': 'World'}[region]
gh_repo = 'CodeClub%s-Projects' % pp_region
r = Github(gh_user, gh_token).get_repo('CodeClub/%s' % gh_repo)
pdf_generator = 'phantomjs'
# clone the curricula repos (the lazy way)
subprocess.call('make clone'.split())
# clone the output repo
subprocess.call(('git clone https://github.com/CodeClub/%s.git %s' %
(gh_repo, output_dir)).split())
if clean:
# delete everything in the output dir
rm_files(output_dir, dont_remove)
# init gitpython!
repo = Repo(output_dir)
# run the build
build.build(pdf_generator,
['lessons/scratch', 'lessons/webdev', 'lessons/python'],
region, output_dir, verbose, repo, rebuild)
# add username and token to remote url
# (so we can write)
origin_url = repo.remotes.origin.url
origin_url = 'https://%s:%[email protected]/%s/%s' % (gh_user, gh_token,
gh_user, origin_url[28:])
repo.git.remote('set-url', '--push', 'origin', origin_url)
# stage everything...
repo.git.add('--all')
# NB. it seems weird, but this reason can disagree
# with the PR (since we force push)
reason_text = get_reason_text(reason)
try:
# ... commit it...
repo.git.commit('-m', 'Rebuild', '-m', reason_text)
# ...and push!
# TODO: Don't force push here!
repo.git.push('-f', 'origin', 'gh-pages')
except GitCommandError:
sys.exit()
# submit pull request
try:
msg = "Hello!\n\n"
msg += "I've been hard at work, rebuilding the Code Club %s projects website from the latest markdown.\n\n" % pp_region
msg += "%s and I found some updates, so I thought I'd best send a pull request. You can view my updated version here:\nhttp://%s.github.io/%s/\n\n" % (
reason_text, gh_user, gh_repo)
msg += "Have a nice day!"
r.create_pull(title='Rebuild',
body=msg,
head='%s:gh-pages' % gh_user,
base='gh-pages')
except GithubException:
# TODO: handle this.
# Usually it just means the PR already exists, which is
# nothing too much to worry about.
pass
def make_release(repo, blurb_file, prs_to_ignore):
"""
Generate a new changelog, run the script, and then make a PR on GitHub
"""
gh_token = os.getenv(config.GH_TOKEN_VAR)
default_branch, new_version, new_markdown, changelog = new_changelog(
repo, blurb_file, prs_to_ignore)
if changelog:
# Freshly clone repo
tmp = os.path.join(tempfile.gettempdir(), "release_maker")
shutil.rmtree(tmp, ignore_errors=True)
print(f"Cloning https://github.com/{repo}.git to {tmp} ...")
subprocess.run(
[
"git",
"clone",
"--depth",
"1",
f"https://{gh_token}@github.com/{repo}.git",
tmp,
],
check=True,
capture_output=True,
)
os.chdir(tmp)
# Get the configuration
cfg = load_config()
# Attach project header
changelog = f"# {cfg['project_title']} Change History\n\n{changelog}"
print("Writing updated changelog file ...")
with open(CHANGEFILE, "w") as f:
f.write(changelog)
pre_release_script = cfg.get("pre_release_script")
if pre_release_script:
print(f"Executing pre-release script {pre_release_script} ...")
mode = os.stat(pre_release_script).st_mode
os.chmod(pre_release_script, mode | stat.S_IXUSR)
subprocess.run(
[pre_release_script, new_version],
check=True,
capture_output=True,
)
# Create and push new release branch
release_branch_name = (
f"release-{new_version}-{config.NEW_RELEASE_EMOJI}")
print(f"Submitting release branch {release_branch_name} ...")
subprocess.run(
["git", "checkout", "-b", release_branch_name],
check=True,
capture_output=True,
)
subprocess.run(["git", "add", "-A"], check=True, capture_output=True)
commit_emoji = (config.NEW_RELEASE_EMOJI_SHORTCODE if hasattr(
config, "NEW_RELEASE_EMOJI_SHORTCODE") else
config.NEW_RELEASE_EMOJI)
subprocess.run(
[
"git",
"commit",
"-m",
f"{commit_emoji} Release {new_version}\n\n{new_markdown}",
],
check=True,
capture_output=True,
)
subprocess.run(
["git", "push", "--force", "origin", release_branch_name],
check=True,
capture_output=True,
)
# Create GitHub Pull Request
print("Submitting PR for release ...")
gh_repo = Github(gh_token, base_url=GH_API).get_repo(repo)
pr_title = f"{config.NEW_RELEASE_EMOJI} Release {new_version}"
pr_url = None
for p in gh_repo.get_pulls(state="open", base=default_branch):
if p.title == pr_title:
pr_url = p.html_url
break
if pr_url:
print(f"Updated release PR: {pr_url}")
else:
pr = gh_repo.create_pull(
title=pr_title,
body=new_markdown,
head=release_branch_name,
base=default_branch,
)
pr.add_to_labels("release")
print(f"Created release PR: {pr.html_url}")
else:
print("Doing nothing.")