def make_changelog_pr(auth, branch, repo, title, commit_message, body, dry_run=False): repo = repo or util.get_repo() # Make a new branch with a uuid suffix pr_branch = f"changelog-{uuid.uuid1().hex}" if not dry_run: util.run("git --no-pager diff") util.run("git stash") util.run(f"git fetch origin {branch}") util.run(f"git checkout -b {pr_branch} origin/{branch}") util.run("git stash apply") # Add a commit with the message util.run(commit_message) # Create the pull owner, repo_name = repo.split("/") gh = GhApi(owner=owner, repo=repo_name, token=auth) base = branch head = pr_branch maintainer_can_modify = True if dry_run: util.log("Skipping pull request due to dry run") return util.run(f"git push origin {pr_branch}") # title, head, base, body, maintainer_can_modify, draft, issue pull = gh.pulls.create(title, head, base, body, maintainer_can_modify, False, None) util.actions_output("pr_url", pull.html_url)
def run_script(target, script): """Run a script on the target pull request URL""" # e.g. https://github.com/foo/bar/pull/81 owner, repo = target.replace("https://github.com/", "").split('/')[:2] number = target.split("/")[-1] auth = os.environ['GITHUB_ACCESS_TOKEN'] gh = GhApi(owner=owner, repo=repo, token=auth) # here we get the target owner and branch so we can check it out below pull = gh.pulls.get(number) user_name = pull.head.repo.owner.login branch = pull.head.ref if Path("./test").exists(): shutil.rmtree("./test") run(f"git clone https://{maintainer}:{auth}@github.com/{user_name}/{repo} -b {branch} test" ) os.chdir("test") run("pip install -e '.[test]'") for cmd in script: try: run(cmd) except Exception: continue # Use email address for the GitHub Actions bot # https://github.community/t/github-actions-bot-email-address/17204/6 run('git config user.email "41898282+github-actions[bot]@users.noreply.github.com"' ) run('git config user.name "GitHub Action"') run(f"git commit -a -m 'Run maintainer script' -m 'by {maintainer}' -m '{json.dumps(script)}'" ) run(f"git push origin {branch}")
def format_pr_entry(target, number, auth=None): """Format a PR entry in the style used by our changelogs. Parameters ---------- target : str The GitHub owner/repo number : int The PR number to resolve auth : str, optional The GitHub authorization token Returns ------- str A formatted PR entry """ owner, repo = target.split("/") gh = GhApi(owner=owner, repo=repo, token=auth) pull = gh.pulls.get(number) title = pull.title url = pull.html_url user_name = pull.user.login user_url = pull.user.html_url return f"- {title} [#{number}]({url}) ([@{user_name}]({user_url}))"
def cli_remove_runners(args): # Authenticate github = GhApi(token=args.api) if args.all is True: for runner in Runner.discover(github): runner.kill() else: Runner.from_name(github, args.runner).kill()
def delete_release(auth, release_url): """Delete a draft GitHub release by url to the release page""" match = re.match(util.RELEASE_HTML_PATTERN, release_url) match = match or re.match(util.RELEASE_API_PATTERN, release_url) if not match: raise ValueError(f"Release url is not valid: {release_url}") gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth) release = util.release_for_url(gh, release_url) for asset in release.assets: gh.repos.delete_release_asset(asset.id) gh.repos.delete_release(release.id)
def get(self, repo: str, filename: str) -> str | None: if self.online: try: self.contents[filename][repo] = (GhApi( *repo.split("/")).get_content(filename).decode()) except HTTP404NotFoundError: self.contents[filename][repo] = None return self.contents[filename][repo] elif repo in self.contents[filename]: return self.contents[filename][repo] else: raise RuntimeError( f"Trying to access {repo}:{filename} and not in cache, rebuild cache" )
def cli_add_runners(args): # Authenticate github = GhApi(token=args.api) # Get Repo info repo = GithubRepo(github, args.repo) logging.info("Will add runner for %s", repo) runner = [] for n in range(args.n): r = Runner(repo) logging.debug("Adding Runner %d of %d: %s", n, args.n, r) r.install() r.register() r.start()
def __init__(self, api: GhApi, url) -> None: # Maintain link back to ghapi if isinstance(api, str): self.api = GhApi(token=api) else: self.api = api # Parse info from URL m = re.match(r".*?github.com/([^/]*)/([^/]*)", url) if m is None: raise RuntimeError(f"Invalid GitHub Repo: {url}") else: self.owner = m.group(1) self.name = m.group(2) super().__init__()
def __init__(self, *, token: Optional[str] = None, users_page_size: int = DEFAULT_USER_PAGE_SIZE, repositories_page_size: int = DEFAULT_REPOSITORY_PAGE_SIZE): """ Initializes a GitHub Scraper with a determined page size for users and repositories. :param token: Github OAuth token to get a better rate limit :param users_page_size: The amount of users each github users api call will fetch. max: 100 :param repositories_page_size: The amount of repositories each github repositories api call will fetch. max: 100 """ self.api: GhApi = GhApi(token=token) self.repositories_processed: int = 0 self.users_processed: int = 0 self.repositories_added: int = 0 self.users_added: int = 0 # Set bound for the page sizes to the minimum and the maximum values self.users_page_size: int = max(min(self.MAX_PAGE_SIZE, users_page_size), self.MIN_PAGE_SIZE) self.repositories_page_size: int = max(min(self.MAX_PAGE_SIZE, repositories_page_size), self.MIN_PAGE_SIZE) logger.debug(f'scrapper instance created with token: {token}.')
def publish_release(auth, dist_dir, npm_token, npm_cmd, twine_cmd, dry_run, release_url): """Publish release asset(s) and finalize GitHub release""" util.log(f"Publishing {release_url} in with dry run: {dry_run}") match = parse_release_url(release_url) if npm_token: npm.handle_auth_token(npm_token) found = False for path in glob(f"{dist_dir}/*.*"): name = Path(path).name suffix = Path(path).suffix if suffix in [".gz", ".whl"]: util.run(f"{twine_cmd} {name}", cwd=dist_dir) found = True elif suffix == ".tgz": util.run(f"{npm_cmd} {name}", cwd=dist_dir) found = True else: util.log(f"Nothing to upload for {name}") if not found: # pragma: no cover raise ValueError("No assets published, refusing to finalize release") # Take the release out of draft gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth) release = util.release_for_url(gh, release_url) release = gh.repos.update_release( release.id, release.tag_name, release.target_commitish, release.name, release.body, dry_run, release.prerelease, ) # Set the GitHub action output util.actions_output("release_url", release.html_url)
def publish_release(auth, release_url): """Publish GitHub release""" util.log(f"Publishing {release_url}") match = parse_release_url(release_url) # Take the release out of draft gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth) release = util.release_for_url(gh, release_url) release = gh.repos.update_release( release.id, release.tag_name, release.target_commitish, release.name, release.body, False, release.prerelease, ) # Set the GitHub action output util.actions_output("release_url", release.html_url)
def format_pr_entry(target, number): """Format a PR entry in the style used by our changelogs. Parameters ---------- target : str The GitHub owner/repo number : int The PR number to resolve Returns ------- str A formatted PR entry """ owner, repo = target.split("/") auth = os.environ['GITHUB_ACCESS_TOKEN'] gh = GhApi(owner=owner, repo=repo, token=auth) pull = gh.pulls.get(number) title = pull.title url = pull.html_url user_name = pull.user.login user_url = pull.user.html_url return f"- {title} [#{number}]({url}) ([@{user_name}]({user_url}))"
timeout=15) # Set up the Github REST API client. # Note that ghapi contains a bug in the `paged` method as of December 2020, therefore # it's safer to install my fork (see README.md for instructions). load_dotenv() if os.getenv("GH_TOKENS"): GH_TOKENS = os.getenv("GH_TOKENS").split(",") print(f"Found {len(GH_TOKENS)} token(s) for Github API") else: raise RuntimeError( "Couldn't find a token for Github API! Specify via env variable GH_TOKENS" ) api = GhApi( token=random.choice(GH_TOKENS), limit_cb=lambda rem, quota: print(f"Quota remaining: {rem} of {quota}"), ) def switch_api_token(): """Update API to use a new, random token from the env variable GH_TOKENS.""" api.headers["Authorization"] = f"token {random.choice(GH_TOKENS)}" print("Switched API token") def rate_limit_info() -> Dict: """Return information about reamining API calls (on REST API and GraphQL API).""" limits = api.rate_limit.get() d = { "core_remaining": limits.resources.core.remaining,
def forwardport_changelog( auth, ref, branch, repo, username, changelog_path, dry_run, git_url, release_url ): """Forwardport Changelog Entries to the Default Branch""" # Set up the git repo with the branch match = parse_release_url(release_url) gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth) release = util.release_for_url(gh, release_url) tag = release.tag_name repo = f'{match["owner"]}/{match["repo"]}' # We want to target the main branch orig_dir = os.getcwd() branch = prep_git(None, None, repo, auth, username, git_url, install=False) os.chdir(util.CHECKOUT_NAME) # Bail if the tag has been merged to the branch tags = util.run(f"git --no-pager tag --merged {branch}") if tag in tags.splitlines(): util.log(f"Skipping since tag is already merged into {branch}") return # Get the entry for the tag util.run(f"git checkout {tag}") entry = changelog.extract_current(changelog_path) # Get the previous header for the branch full_log = Path(changelog_path).read_text(encoding="utf-8") start = full_log.index(changelog.END_MARKER) prev_header = "" for line in full_log[start:].splitlines(): if line.strip().startswith("#"): prev_header = line break if not prev_header: raise ValueError("No anchor for previous entry") # Check out the branch again util.run(f"git checkout -B {branch} origin/{branch}") default_entry = changelog.extract_current(changelog_path) # Look for the previous header default_log = Path(changelog_path).read_text(encoding="utf-8") if not prev_header in default_log: util.log( f'Could not find previous header "{prev_header}" in {changelog_path} on branch {branch}' ) return # If the previous header is the current entry in the default branch, we need to move the change markers if prev_header in default_entry: default_log = changelog.insert_entry(default_log, entry) # Otherwise insert the new entry ahead of the previous header else: insertion_point = default_log.index(prev_header) default_log = changelog.format( default_log[:insertion_point] + entry + default_log[insertion_point:] ) Path(changelog_path).write_text(default_log, encoding="utf-8") # Create a forward port PR title = f"{changelog.PR_PREFIX} Forward Ported from {tag}" commit_message = f'git commit -a -m "{title}"' body = title pr = make_changelog_pr( auth, ref, branch, repo, title, commit_message, body, dry_run=dry_run ) # Clean up after ourselves os.chdir(orig_dir) shutil.rmtree(util.CHECKOUT_NAME)
def extract_release(auth, dist_dir, dry_run, release_url): """Download and verify assets from a draft GitHub release""" match = parse_release_url(release_url) owner, repo = match["owner"], match["repo"] gh = GhApi(owner=owner, repo=repo, token=auth) release = util.release_for_url(gh, release_url) assets = release.assets # Clean the dist folder dist = Path(dist_dir) if dist.exists(): shutil.rmtree(dist, ignore_errors=True) os.makedirs(dist) # Fetch, validate, and publish assets for asset in assets: util.log(f"Fetching {asset.name}...") url = asset.url headers = dict(Authorization=f"token {auth}", Accept="application/octet-stream") path = dist / asset.name with requests.get(url, headers=headers, stream=True) as r: r.raise_for_status() with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) suffix = Path(asset.name).suffix if suffix in [".gz", ".whl"]: python.check_dist(path) elif suffix == ".tgz": npm.check_dist(path) else: util.log(f"Nothing to check for {asset.name}") # Skip sha validation for dry runs since the remote tag will not exist if dry_run: return branch = release.target_commitish tag_name = release.tag_name sha = None for tag in gh.list_tags(): if tag.ref == f"refs/tags/{tag_name}": sha = tag.object.sha if sha is None: raise ValueError("Could not find tag") # Run a git checkout # Fetch the branch # Get the commmit message for the branch commit_message = "" with TemporaryDirectory() as td: url = gh.repos.get().html_url util.run(f"git clone {url} local", cwd=td) checkout = osp.join(td, "local") if not osp.exists(url): util.run(f"git fetch origin {branch}", cwd=checkout) commit_message = util.run(f"git log --format=%B -n 1 {sha}", cwd=checkout) for asset in assets: # Check the sha against the published sha valid = False path = dist / asset.name sha = util.compute_sha256(path) for line in commit_message.splitlines(): if asset.name in line: if sha in line: valid = True else: util.log("Mismatched sha!") if not valid: # pragma: no cover raise ValueError(f"Invalid file {asset.name}")
def draft_release( ref, branch, repo, auth, changelog_path, version_cmd, dist_dir, dry_run, post_version_spec, assets, ): """Publish Draft GitHub release and handle post version bump""" branch = branch or util.get_branch() repo = repo or util.get_repo() assets = assets or glob(f"{dist_dir}/*") version = util.get_version() body = changelog.extract_current(changelog_path) prerelease = util.is_prerelease(version) # Bump to post version if given if post_version_spec: post_version = bump_version(post_version_spec, version_cmd) util.log(f"Bumped version to {post_version}") util.run(f'git commit -a -m "Bump to {post_version}"') if dry_run: return owner, repo_name = repo.split("/") gh = GhApi(owner=owner, repo=repo_name, token=auth) # Remove draft releases over a day old if bool(os.environ.get("GITHUB_ACTIONS")): for release in gh.repos.list_releases(): if str(release.draft).lower() == "false": continue created = release.created_at d_created = datetime.strptime(created, r"%Y-%m-%dT%H:%M:%SZ") delta = datetime.utcnow() - d_created if delta.days > 0: gh.repos.delete_release(release.id) remote_url = util.run("git config --get remote.origin.url") if not os.path.exists(remote_url): util.run(f"git push origin HEAD:{branch} --follow-tags --tags") util.log(f"Creating release for {version}") util.log(f"With assets: {assets}") release = gh.create_release( f"v{version}", branch, f"Release v{version}", body, True, prerelease, files=assets, ) # Set the GitHub action output util.actions_output("release_url", release.html_url)
def generate_bullets(search_start: datetime, detailed: bool = False): akst = tz.tzoffset('AKST', timedelta(hours=-9)) aknow = datetime.now(akst) search_start = search_start.astimezone(akst) meta = { 'title': 'Tools Team bullets', 'description': f"Tools team bullets for {search_start.isoformat(timespec='seconds')}" f" through {aknow.isoformat(timespec='seconds')}", } log.info(f'Generating {meta["description"]}') gh = GhApi() release_details = {} dev_prs = {} open_prs = {} opened_issues = {} for repo in tqdm(gh.repos.list_for_org('ASFHyP3')): # FIXME: Returns issues and PRs... simpler to filter this one list or make the three calls? for issue in gh.issues.list_for_repo( repo.owner.login, repo.name, state='open', sort='created', direction='desc', since=search_start.isoformat(timespec='seconds')): if issue.get('pull_request') is None: opened_issues[issue.id] = util.get_details(issue) try: last_release = parse_date( gh.repos.get_latest_release(repo.owner.login, repo.name).created_at) for release in gh.repos.list_releases(repo.owner.login, repo.name): created_at = parse_date(release.created_at) if created_at >= search_start: release_details[ release.target_commitish] = util.get_details(release) else: break except HTTP404NotFoundError: last_release = search_start # FIXME: might be able to use issues.list_for_repo with since=... to simplify logic for pull in gh.pulls.list(repo.owner.login, repo.name, state='closed', base='develop', sort='updated', direction='desc'): merged_at = pull.get('merged_at') if merged_at and parse_date(merged_at) > max( search_start, last_release): dev_prs[pull.merge_commit_sha] = util.get_details(pull) for pull in gh.pulls.list(repo.owner.login, repo.name, state='open', sort='created', direction='desc'): open_prs[pull.head.sha] = util.get_details(pull) template = 'report_detailed.md.j2' if detailed else 'report.md.j2' report_name = 'report_detailed.md' if detailed else 'report.md' report = util.render_template( template, releases=release_details, meta=meta, dev_prs=dev_prs, open_prs=open_prs, opened_issues=opened_issues, ) with open(report_name, 'w') as f: f.write(report)
from ghapi.core import GhApi import os api = GhApi(owner="streamlit", repo="streamlit", token=os.environ["GITHUB_API_KEY"]) params = { 'state': 'open', 'labels': ['media'], 'per_page': 100, } issues = api.issues.list(state="open") # issues.list_labels_for_repo # issues.list_events # issues.get(owner, repo, issue_number): Get an issue # issues.list_comments_for_repo(owner, repo, sort, direction, since, per_page, page): List issue comments for a repository # issues.list_for_repo(owner, repo, milestone, state, assignee, creator, mentioned, labels, sort, direction, since, per_page, page): List repository issues print(api.limit_rem) issues = api.issues.list_for_repo( state=params['state'], labels=params['labels'], per_page=params['per_page'], )
def publish_release(auth, dist_dir, npm_token, npm_cmd, twine_cmd, dry_run, release_url): """Publish release asset(s) and finalize GitHub release""" util.log(f"Publishing {release_url} in with dry run: {dry_run}") if dry_run: # Start local pypi server with no auth, allowing overwrites, # in a temporary directory temp_dir = TemporaryDirectory() cmd = f"pypi-server -p 8081 -P . -a . -o -v {temp_dir.name}" proc = Popen(shlex.split(cmd), stderr=PIPE) # Wait for the server to start while True: line = proc.stderr.readline().decode("utf-8").strip() util.log(line) if "Listening on" in line: break atexit.register(proc.kill) atexit.register(temp_dir.cleanup) twine_cmd = "twine upload --repository-url=http://localhost:8081" os.environ["TWINE_USERNAME"] = "******" os.environ["TWINE_PASSWORD"] = "******" npm_cmd = "npm publish --dry-run" match = parse_release_url(release_url) if npm_token: npm.handle_auth_token(npm_token) found = False for path in glob(f"{dist_dir}/*.*"): name = Path(path).name suffix = Path(path).suffix if suffix in [".gz", ".whl"]: util.run(f"{twine_cmd} {name}", cwd=dist_dir) found = True elif suffix == ".tgz": util.run(f"{npm_cmd} {name}", cwd=dist_dir) found = True else: util.log(f"Nothing to upload for {name}") if not found: # pragma: no cover raise ValueError("No assets published, refusing to finalize release") # Take the release out of draft gh = GhApi(owner=match["owner"], repo=match["repo"], token=auth) release = util.release_for_url(gh, release_url) release = gh.repos.update_release( release.id, release.tag_name, release.target_commitish, release.name, release.body, dry_run, release.prerelease, ) # Set the GitHub action output util.actions_output("release_url", release.html_url)