def make_changelog_pr(auth, branch, repo, title, commit_message, body, dry_run=False):
repo = repo or util.get_repo()
# Make a new branch with a uuid suffix
pr_branch = f"changelog-{uuid.uuid1().hex}"
if not dry_run:
util.run("git --no-pager diff")
util.run("git stash")
util.run(f"git fetch origin {branch}")
util.run(f"git checkout -b {pr_branch} origin/{branch}")
util.run("git stash apply")
# Add a commit with the message
util.run(commit_message)
# Create the pull
owner, repo_name = repo.split("/")
gh = GhApi(owner=owner, repo=repo_name, token=auth)
base = branch
head = pr_branch
maintainer_can_modify = True
if dry_run:
util.log("Skipping pull request due to dry run")
return
util.run(f"git push origin {pr_branch}")
# title, head, base, body, maintainer_can_modify, draft, issue
pull = gh.pulls.create(title, head, base, body, maintainer_can_modify, False, None)
util.actions_output("pr_url", pull.html_url)
def run_script(target, script):
"""Run a script on the target pull request URL"""
# e.g. https://github.com/foo/bar/pull/81
owner, repo = target.replace("https://github.com/", "").split('/')[:2]
number = target.split("/")[-1]
auth = os.environ['GITHUB_ACCESS_TOKEN']
gh = GhApi(owner=owner, repo=repo, token=auth)
# here we get the target owner and branch so we can check it out below
pull = gh.pulls.get(number)
user_name = pull.head.repo.owner.login
branch = pull.head.ref
if Path("./test").exists():
shutil.rmtree("./test")
run(f"git clone https://{maintainer}:{auth}@github.com/{user_name}/{repo} -b {branch} test"
)
os.chdir("test")
run("pip install -e '.[test]'")
for cmd in script:
try:
run(cmd)
except Exception:
continue
# Use email address for the GitHub Actions bot
# https://github.community/t/github-actions-bot-email-address/17204/6
run('git config user.email "41898282+github-actions[bot]@users.noreply.github.com"'
)
run('git config user.name "GitHub Action"')
run(f"git commit -a -m 'Run maintainer script' -m 'by {maintainer}' -m '{json.dumps(script)}'"
)
run(f"git push origin {branch}")
def format_pr_entry(target, number, auth=None):
"""Format a PR entry in the style used by our changelogs.
Parameters
----------
target : str
The GitHub owner/repo
number : int
The PR number to resolve
auth : str, optional
The GitHub authorization token
Returns
-------
str
A formatted PR entry
"""
owner, repo = target.split("/")
gh = GhApi(owner=owner, repo=repo, token=auth)
pull = gh.pulls.get(number)
title = pull.title
url = pull.html_url
user_name = pull.user.login
user_url = pull.user.html_url
return f"- {title} [#{number}]({url}) ([@{user_name}]({user_url}))"
def cli_remove_runners(args):
# Authenticate
github = GhApi(token=args.api)
if args.all is True:
for runner in Runner.discover(github):
runner.kill()
else:
Runner.from_name(github, args.runner).kill()
def delete_release(auth, release_url):
"""Delete a draft GitHub release by url to the release page"""
match = re.match(util.RELEASE_HTML_PATTERN, release_url)
match = match or re.match(util.RELEASE_API_PATTERN, release_url)
if not match:
raise ValueError(f"Release url is not valid: {release_url}")
gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth)
release = util.release_for_url(gh, release_url)
for asset in release.assets:
gh.repos.delete_release_asset(asset.id)
gh.repos.delete_release(release.id)
def get(self, repo: str, filename: str) -> str | None:
if self.online:
try:
self.contents[filename][repo] = (GhApi(
*repo.split("/")).get_content(filename).decode())
except HTTP404NotFoundError:
self.contents[filename][repo] = None
return self.contents[filename][repo]
elif repo in self.contents[filename]:
return self.contents[filename][repo]
else:
raise RuntimeError(
f"Trying to access {repo}:{filename} and not in cache, rebuild cache"
)
def cli_add_runners(args):
# Authenticate
github = GhApi(token=args.api)
# Get Repo info
repo = GithubRepo(github, args.repo)
logging.info("Will add runner for %s", repo)
runner = []
for n in range(args.n):
r = Runner(repo)
logging.debug("Adding Runner %d of %d: %s", n, args.n, r)
r.install()
r.register()
r.start()
def __init__(self, api: GhApi, url) -> None:
# Maintain link back to ghapi
if isinstance(api, str):
self.api = GhApi(token=api)
else:
self.api = api
# Parse info from URL
m = re.match(r".*?github.com/([^/]*)/([^/]*)", url)
if m is None:
raise RuntimeError(f"Invalid GitHub Repo: {url}")
else:
self.owner = m.group(1)
self.name = m.group(2)
super().__init__()
def __init__(self, *, token: Optional[str] = None,
users_page_size: int = DEFAULT_USER_PAGE_SIZE,
repositories_page_size: int = DEFAULT_REPOSITORY_PAGE_SIZE):
"""
Initializes a GitHub Scraper with a determined page size for users and repositories.
:param token: Github OAuth token to get a better rate limit
:param users_page_size: The amount of users each github users api call will fetch. max: 100
:param repositories_page_size: The amount of repositories each github repositories api call will fetch. max: 100
"""
self.api: GhApi = GhApi(token=token)
self.repositories_processed: int = 0
self.users_processed: int = 0
self.repositories_added: int = 0
self.users_added: int = 0
# Set bound for the page sizes to the minimum and the maximum values
self.users_page_size: int = max(min(self.MAX_PAGE_SIZE, users_page_size), self.MIN_PAGE_SIZE)
self.repositories_page_size: int = max(min(self.MAX_PAGE_SIZE, repositories_page_size), self.MIN_PAGE_SIZE)
logger.debug(f'scrapper instance created with token: {token}.')
def publish_release(auth, dist_dir, npm_token, npm_cmd, twine_cmd, dry_run,
release_url):
"""Publish release asset(s) and finalize GitHub release"""
util.log(f"Publishing {release_url} in with dry run: {dry_run}")
match = parse_release_url(release_url)
if npm_token:
npm.handle_auth_token(npm_token)
found = False
for path in glob(f"{dist_dir}/*.*"):
name = Path(path).name
suffix = Path(path).suffix
if suffix in [".gz", ".whl"]:
util.run(f"{twine_cmd} {name}", cwd=dist_dir)
found = True
elif suffix == ".tgz":
util.run(f"{npm_cmd} {name}", cwd=dist_dir)
found = True
else:
util.log(f"Nothing to upload for {name}")
if not found: # pragma: no cover
raise ValueError("No assets published, refusing to finalize release")
# Take the release out of draft
gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth)
release = util.release_for_url(gh, release_url)
release = gh.repos.update_release(
release.id,
release.tag_name,
release.target_commitish,
release.name,
release.body,
dry_run,
release.prerelease,
)
# Set the GitHub action output
util.actions_output("release_url", release.html_url)
def publish_release(auth, release_url):
"""Publish GitHub release"""
util.log(f"Publishing {release_url}")
match = parse_release_url(release_url)
# Take the release out of draft
gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth)
release = util.release_for_url(gh, release_url)
release = gh.repos.update_release(
release.id,
release.tag_name,
release.target_commitish,
release.name,
release.body,
False,
release.prerelease,
)
# Set the GitHub action output
util.actions_output("release_url", release.html_url)
def format_pr_entry(target, number):
"""Format a PR entry in the style used by our changelogs.
Parameters
----------
target : str
The GitHub owner/repo
number : int
The PR number to resolve
Returns
-------
str
A formatted PR entry
"""
owner, repo = target.split("/")
auth = os.environ['GITHUB_ACCESS_TOKEN']
gh = GhApi(owner=owner, repo=repo, token=auth)
pull = gh.pulls.get(number)
title = pull.title
url = pull.html_url
user_name = pull.user.login
user_url = pull.user.html_url
return f"- {title} [#{number}]({url}) ([@{user_name}]({user_url}))"
timeout=15)
# Set up the Github REST API client.
# Note that ghapi contains a bug in the `paged` method as of December 2020, therefore
# it's safer to install my fork (see README.md for instructions).
load_dotenv()
if os.getenv("GH_TOKENS"):
GH_TOKENS = os.getenv("GH_TOKENS").split(",")
print(f"Found {len(GH_TOKENS)} token(s) for Github API")
else:
raise RuntimeError(
"Couldn't find a token for Github API! Specify via env variable GH_TOKENS"
)
api = GhApi(
token=random.choice(GH_TOKENS),
limit_cb=lambda rem, quota: print(f"Quota remaining: {rem} of {quota}"),
)
def switch_api_token():
"""Update API to use a new, random token from the env variable GH_TOKENS."""
api.headers["Authorization"] = f"token {random.choice(GH_TOKENS)}"
print("Switched API token")
def rate_limit_info() -> Dict:
"""Return information about reamining API calls (on REST API and GraphQL API)."""
limits = api.rate_limit.get()
d = {
"core_remaining":
limits.resources.core.remaining,
def forwardport_changelog(
auth, ref, branch, repo, username, changelog_path, dry_run, git_url, release_url
):
"""Forwardport Changelog Entries to the Default Branch"""
# Set up the git repo with the branch
match = parse_release_url(release_url)
gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth)
release = util.release_for_url(gh, release_url)
tag = release.tag_name
repo = f'{match["owner"]}/{match["repo"]}'
# We want to target the main branch
orig_dir = os.getcwd()
branch = prep_git(None, None, repo, auth, username, git_url, install=False)
os.chdir(util.CHECKOUT_NAME)
# Bail if the tag has been merged to the branch
tags = util.run(f"git --no-pager tag --merged {branch}")
if tag in tags.splitlines():
util.log(f"Skipping since tag is already merged into {branch}")
return
# Get the entry for the tag
util.run(f"git checkout {tag}")
entry = changelog.extract_current(changelog_path)
# Get the previous header for the branch
full_log = Path(changelog_path).read_text(encoding="utf-8")
start = full_log.index(changelog.END_MARKER)
prev_header = ""
for line in full_log[start:].splitlines():
if line.strip().startswith("#"):
prev_header = line
break
if not prev_header:
raise ValueError("No anchor for previous entry")
# Check out the branch again
util.run(f"git checkout -B {branch} origin/{branch}")
default_entry = changelog.extract_current(changelog_path)
# Look for the previous header
default_log = Path(changelog_path).read_text(encoding="utf-8")
if not prev_header in default_log:
util.log(
f'Could not find previous header "{prev_header}" in {changelog_path} on branch {branch}'
)
return
# If the previous header is the current entry in the default branch, we need to move the change markers
if prev_header in default_entry:
default_log = changelog.insert_entry(default_log, entry)
# Otherwise insert the new entry ahead of the previous header
else:
insertion_point = default_log.index(prev_header)
default_log = changelog.format(
default_log[:insertion_point] + entry + default_log[insertion_point:]
)
Path(changelog_path).write_text(default_log, encoding="utf-8")
# Create a forward port PR
title = f"{changelog.PR_PREFIX} Forward Ported from {tag}"
commit_message = f'git commit -a -m "{title}"'
body = title
pr = make_changelog_pr(
auth, ref, branch, repo, title, commit_message, body, dry_run=dry_run
)
# Clean up after ourselves
os.chdir(orig_dir)
shutil.rmtree(util.CHECKOUT_NAME)
def extract_release(auth, dist_dir, dry_run, release_url):
"""Download and verify assets from a draft GitHub release"""
match = parse_release_url(release_url)
owner, repo = match["owner"], match["repo"]
gh = GhApi(owner=owner, repo=repo, token=auth)
release = util.release_for_url(gh, release_url)
assets = release.assets
# Clean the dist folder
dist = Path(dist_dir)
if dist.exists():
shutil.rmtree(dist, ignore_errors=True)
os.makedirs(dist)
# Fetch, validate, and publish assets
for asset in assets:
util.log(f"Fetching {asset.name}...")
url = asset.url
headers = dict(Authorization=f"token {auth}", Accept="application/octet-stream")
path = dist / asset.name
with requests.get(url, headers=headers, stream=True) as r:
r.raise_for_status()
with open(path, "wb") as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
suffix = Path(asset.name).suffix
if suffix in [".gz", ".whl"]:
python.check_dist(path)
elif suffix == ".tgz":
npm.check_dist(path)
else:
util.log(f"Nothing to check for {asset.name}")
# Skip sha validation for dry runs since the remote tag will not exist
if dry_run:
return
branch = release.target_commitish
tag_name = release.tag_name
sha = None
for tag in gh.list_tags():
if tag.ref == f"refs/tags/{tag_name}":
sha = tag.object.sha
if sha is None:
raise ValueError("Could not find tag")
# Run a git checkout
# Fetch the branch
# Get the commmit message for the branch
commit_message = ""
with TemporaryDirectory() as td:
url = gh.repos.get().html_url
util.run(f"git clone {url} local", cwd=td)
checkout = osp.join(td, "local")
if not osp.exists(url):
util.run(f"git fetch origin {branch}", cwd=checkout)
commit_message = util.run(f"git log --format=%B -n 1 {sha}", cwd=checkout)
for asset in assets:
# Check the sha against the published sha
valid = False
path = dist / asset.name
sha = util.compute_sha256(path)
for line in commit_message.splitlines():
if asset.name in line:
if sha in line:
valid = True
else:
util.log("Mismatched sha!")
if not valid: # pragma: no cover
raise ValueError(f"Invalid file {asset.name}")
def draft_release(
ref,
branch,
repo,
auth,
changelog_path,
version_cmd,
dist_dir,
dry_run,
post_version_spec,
assets,
):
"""Publish Draft GitHub release and handle post version bump"""
branch = branch or util.get_branch()
repo = repo or util.get_repo()
assets = assets or glob(f"{dist_dir}/*")
version = util.get_version()
body = changelog.extract_current(changelog_path)
prerelease = util.is_prerelease(version)
# Bump to post version if given
if post_version_spec:
post_version = bump_version(post_version_spec, version_cmd)
util.log(f"Bumped version to {post_version}")
util.run(f'git commit -a -m "Bump to {post_version}"')
if dry_run:
return
owner, repo_name = repo.split("/")
gh = GhApi(owner=owner, repo=repo_name, token=auth)
# Remove draft releases over a day old
if bool(os.environ.get("GITHUB_ACTIONS")):
for release in gh.repos.list_releases():
if str(release.draft).lower() == "false":
continue
created = release.created_at
d_created = datetime.strptime(created, r"%Y-%m-%dT%H:%M:%SZ")
delta = datetime.utcnow() - d_created
if delta.days > 0:
gh.repos.delete_release(release.id)
remote_url = util.run("git config --get remote.origin.url")
if not os.path.exists(remote_url):
util.run(f"git push origin HEAD:{branch} --follow-tags --tags")
util.log(f"Creating release for {version}")
util.log(f"With assets: {assets}")
release = gh.create_release(
f"v{version}",
branch,
f"Release v{version}",
body,
True,
prerelease,
files=assets,
)
# Set the GitHub action output
util.actions_output("release_url", release.html_url)
def generate_bullets(search_start: datetime, detailed: bool = False):
akst = tz.tzoffset('AKST', timedelta(hours=-9))
aknow = datetime.now(akst)
search_start = search_start.astimezone(akst)
meta = {
'title':
'Tools Team bullets',
'description':
f"Tools team bullets for {search_start.isoformat(timespec='seconds')}"
f" through {aknow.isoformat(timespec='seconds')}",
}
log.info(f'Generating {meta["description"]}')
gh = GhApi()
release_details = {}
dev_prs = {}
open_prs = {}
opened_issues = {}
for repo in tqdm(gh.repos.list_for_org('ASFHyP3')):
# FIXME: Returns issues and PRs... simpler to filter this one list or make the three calls?
for issue in gh.issues.list_for_repo(
repo.owner.login,
repo.name,
state='open',
sort='created',
direction='desc',
since=search_start.isoformat(timespec='seconds')):
if issue.get('pull_request') is None:
opened_issues[issue.id] = util.get_details(issue)
try:
last_release = parse_date(
gh.repos.get_latest_release(repo.owner.login,
repo.name).created_at)
for release in gh.repos.list_releases(repo.owner.login, repo.name):
created_at = parse_date(release.created_at)
if created_at >= search_start:
release_details[
release.target_commitish] = util.get_details(release)
else:
break
except HTTP404NotFoundError:
last_release = search_start
# FIXME: might be able to use issues.list_for_repo with since=... to simplify logic
for pull in gh.pulls.list(repo.owner.login,
repo.name,
state='closed',
base='develop',
sort='updated',
direction='desc'):
merged_at = pull.get('merged_at')
if merged_at and parse_date(merged_at) > max(
search_start, last_release):
dev_prs[pull.merge_commit_sha] = util.get_details(pull)
for pull in gh.pulls.list(repo.owner.login,
repo.name,
state='open',
sort='created',
direction='desc'):
open_prs[pull.head.sha] = util.get_details(pull)
template = 'report_detailed.md.j2' if detailed else 'report.md.j2'
report_name = 'report_detailed.md' if detailed else 'report.md'
report = util.render_template(
template,
releases=release_details,
meta=meta,
dev_prs=dev_prs,
open_prs=open_prs,
opened_issues=opened_issues,
)
with open(report_name, 'w') as f:
f.write(report)
from ghapi.core import GhApi
import os
api = GhApi(owner="streamlit", repo="streamlit", token=os.environ["GITHUB_API_KEY"])
params = {
'state': 'open',
'labels': ['media'],
'per_page': 100,
}
issues = api.issues.list(state="open")
# issues.list_labels_for_repo
# issues.list_events
# issues.get(owner, repo, issue_number): Get an issue
# issues.list_comments_for_repo(owner, repo, sort, direction, since, per_page, page): List issue comments for a repository
# issues.list_for_repo(owner, repo, milestone, state, assignee, creator, mentioned, labels, sort, direction, since, per_page, page): List repository issues
print(api.limit_rem)
issues = api.issues.list_for_repo(
state=params['state'],
labels=params['labels'],
per_page=params['per_page'],
)
def publish_release(auth, dist_dir, npm_token, npm_cmd, twine_cmd, dry_run,
release_url):
"""Publish release asset(s) and finalize GitHub release"""
util.log(f"Publishing {release_url} in with dry run: {dry_run}")
if dry_run:
# Start local pypi server with no auth, allowing overwrites,
# in a temporary directory
temp_dir = TemporaryDirectory()
cmd = f"pypi-server -p 8081 -P . -a . -o -v {temp_dir.name}"
proc = Popen(shlex.split(cmd), stderr=PIPE)
# Wait for the server to start
while True:
line = proc.stderr.readline().decode("utf-8").strip()
util.log(line)
if "Listening on" in line:
break
atexit.register(proc.kill)
atexit.register(temp_dir.cleanup)
twine_cmd = "twine upload --repository-url=http://localhost:8081"
os.environ["TWINE_USERNAME"] = "******"
os.environ["TWINE_PASSWORD"] = "******"
npm_cmd = "npm publish --dry-run"
match = parse_release_url(release_url)
if npm_token:
npm.handle_auth_token(npm_token)
found = False
for path in glob(f"{dist_dir}/*.*"):
name = Path(path).name
suffix = Path(path).suffix
if suffix in [".gz", ".whl"]:
util.run(f"{twine_cmd} {name}", cwd=dist_dir)
found = True
elif suffix == ".tgz":
util.run(f"{npm_cmd} {name}", cwd=dist_dir)
found = True
else:
util.log(f"Nothing to upload for {name}")
if not found: # pragma: no cover
raise ValueError("No assets published, refusing to finalize release")
# Take the release out of draft
gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth)
release = util.release_for_url(gh, release_url)
release = gh.repos.update_release(
release.id,
release.tag_name,
release.target_commitish,
release.name,
release.body,
dry_run,
release.prerelease,
)
# Set the GitHub action output
util.actions_output("release_url", release.html_url)
