def _create_pr(self, reposlug: str, repo: Repository, branch: str, onto: str) -> None: repo.create_pull(title="Harden ignore files", body="Autofix created by devsecops", base=onto, head=branch) print("[I] Created PR {}".format(reposlug))
def create_github_release( repository: Repository, version: str, artifacts: Set[Path], ) -> None: """ Create a tag and release on GitHub. """ changelog_url = 'https://dcos-e2e.readthedocs.io/en/latest/changelog.html' release_name = 'Release ' + version release_message = 'See ' + changelog_url github_release = repository.create_git_tag_and_release( tag=version, tag_message='Release ' + version, release_name=release_name, release_message=release_message, type='commit', object=repository.get_commits()[0].sha, draft=True, ) for artifact_path in artifacts: github_release.upload_asset( path=str(artifact_path), label=artifact_path.name, ) github_release.update_release( name=release_name, message=release_message, draft=False, )
def open_issue(pr, pr_checks, hub_repo: Repository, module, jina_core_version): """opens an issue for the PR with the failed checks (if not already open)""" issue_name = f'{FIX_MODULE_TEMPLATE}{module}' existing_issue_for_pr = [ i for i in list(hub_repo.get_issues(state='open')) if i.title == issue_name ] if len(existing_issue_for_pr) > 0: print(f'Found existing issue: {existing_issue_for_pr}') return existing_issue_for_pr[0] else: # open the issue body = f""" **[This is an automated issue opened as part of the hub modules update GH action. DO NOT EDIT THIS DESCRIPTION]** Could not build module {module} for Jina core version {jina_core_version} because some of the checks failed: ``` {[(c['name'], c['status'], c['conclusion']) for c in pr_checks]} ``` See {pr.html_url} for more info. {TAG_IN_ISSUES} """ issue = hub_repo.create_issue( title=issue_name, body=body, ) print(f'opened issue at {issue.html_url}') return issue
def _create_label_if_needed(self, reposlug: str, repo: Repository, name: str, desc: str, color: str) -> Label: try: label = repo.get_label("devsecops") except UnknownObjectException: if self._verbose: print("[I] Creating label {}:{}".format(reposlug, name)) label = repo.create_label(name, color, desc) return label
def _create_fix(self, reposlug: str, repo: Repository, branch: str, result: ScanResult) -> None: new_content = "{}\n{}\n".format(result.content, "\n".join(result.missings)) repo.update_file(result.filename, message="Autofix by devsecops", content=new_content, sha=result.filesha, branch=branch) print("[I] Updated {}: {}".format(reposlug, result.filename))
def _scan_repository(self, results: ScanResults, user: NamedUser, org: Organization, repo: Repository) -> None: try: branch = repo.get_branch(branch="master") tree = repo.get_git_tree(branch.commit.sha, recursive=True).tree except GithubException as e: print("[W] {}/{} - {}".format(org.login, repo.name, str(e))) # Skip if no master branch return filelist = [x.path for x in tree] for element in tree: self._scan_file(results, user, org, repo, branch.commit.sha, element.path, filelist)
def extract_branches(repo: Repository, from_pulls: bool = False) -> Set[str]: """ Returns a set of branch name strings by wrapping PyGithub's get_branches(). See https://pygithub.readthedocs.io/en/latest/github_objects/Repository.html#github.Repository.Repository.get_branches """ if from_pulls: branches: Set[str] = set(b.head.ref for b in repo.get_pulls()) else: branches: Set[str] = set(b.name for b in repo.get_branches()) return branches
def process_milestone( repo: Repository, milestone: Milestone, in_progress_label: Optional[Label], blocked_label: Optional[Label], ) -> str: now = datetime.now() html_url = milestone._rawData["html_url"] total_issues = milestone.open_issues + milestone.closed_issues percentage_complete = as_percentage(milestone.closed_issues, total_issues) detail_lines = [] status_line = f":heavy_check_mark: {percentage_complete}% completed" if in_progress_label: in_progress_issues = list( repo.get_issues(milestone=milestone, labels=[in_progress_label], state="open")) if in_progress_issues: percentage_in_progress = as_percentage(len(in_progress_issues), total_issues) status_line += ( f" - :hourglass_flowing_sand: {percentage_in_progress}% in progress" ) if blocked_label: blocked_issues = list( repo.get_issues(milestone=milestone, labels=[blocked_label], state="open")) if blocked_issues: percentage_blocked = as_percentage(len(blocked_issues), total_issues) status_line += f" - :octagonal_sign: {percentage_blocked}% blocked" detail_lines.append(status_line) if milestone.due_on: duration = milestone.due_on - milestone.created_at remaining = milestone.due_on - now time_used = 100 - as_percentage(remaining.days, duration.days) detail_lines.append( f":date: {milestone.due_on.date().isoformat()} - :alarm_clock: {time_used}% time used" ) rendered_line = ( f"<{html_url}|{milestone.title}> - {milestone.closed_issues}/{total_issues}" ) for line in detail_lines: rendered_line += f"\n\t{line}" return rendered_line
def create_github_release( repository: Repository, version: str, ) -> None: """ Create a tag and release on GitHub. """ changelog_url = 'https://dcos-e2e.readthedocs.io/en/latest/changelog.html' release_name = 'Release ' + version release_message = 'See ' + changelog_url github_release = repository.create_git_tag_and_release( tag=version, tag_message='Release ' + version, release_name=release_name, release_message=release_message, type='commit', object=repository.get_commits()[0].sha, draft=False, ) # The artifacts we build must be built from the tag we just created. # This tag is created remotely on GitHub using the GitHub HTTP API. # # We fetch all tags from GitHub and set our local HEAD to the latest master # from GitHub. # # One symptom of this is that ``minidcos --version`` from the PyInstaller # binary shows the correct version. local_repository = Repo('.') client = HttpGitClient(repository.owner.html_url) remote_refs = client.fetch(repository.name + '.git', local_repository) # Update the local tags and references with the remote ones. for key, value in remote_refs.items(): local_repository.refs[key] = value # Advance local HEAD to remote master HEAD. local_repository[b'HEAD'] = remote_refs[b'refs/heads/master'] # We need to make the artifacts just after creating a tag so that the # --version output is exactly the one of the tag. # No tag exists when the GitHub release is a draft. # This means that temporarily we have a release without binaries. linux_artifacts = make_linux_binaries(repo_root=Path('.')) for installer_path in linux_artifacts: github_release.upload_asset( path=str(installer_path), label=installer_path.name, )
def create_github_release( repository: Repository, version: str, ) -> None: """ Create a tag and release on GitHub. """ repository.create_git_tag_and_release( tag=version, tag_message='Release ' + version, release_name='Release ' + version, release_message='See CHANGELOG.rst', type='commit', object=repository.get_commits()[0].sha, )
def install(self, name_or_path: str, version: str) -> None: try: with State() as s: name_or_path = self._validate_and_expand_path(name_or_path) self.log.step(f"Installing from {name_or_path}@{version}") repo = Repository.from_url(name_or_path) if repo.name in s.state: raise ValueError( f"{repo.name} is already installed. Use `binman update` to switch versions, or `binman uninstall` to remove" ) release = repo.get_release(tag=version) self.log.step(release.tag_name, padding=1) arts = [] for artifact in release.get_platform_assets(): if "md5" in artifact.download_url or "sha256" in artifact.download_url: continue tgt_dir = self._install_location / artifact.name self.log.progress( f"{artifact.name}/{artifact.platform}-{artifact.architecture} ~> {tgt_dir}", padding=2 ) artifact.download(self._install_location) arts.append(tgt_dir) s.update_application(name=repo.name, url=name_or_path, artifacts=arts, version=release.tag_name) self.log.info("Installation complete") except Exception as e: self.log.error(f"{type(e).__name__}: {e}.") raise
def _is_branch_exists(self, repo: Repository, branch: str) -> Optional[str]: try: b = repo.get_branch(branch) return b.commit.sha except GithubException: return None
def create_git_repo_from_template(user, new_owner, new_repo, template_owner, template_repo, private=True): """Creates a new github repository from a template Args: user (AuthenticatedUser): user object new_owner (string): owner of the new repository new_repo (string): name of the new repository template_owner (string): owner of the template repository template_repo (string): name of the template repository private (boolean): if the repo is private (default True) Returns: The new repository object """ post_parameters = { "owner": new_owner, "name": new_repo, "private": private } headers, data = user._requester.requestJsonAndCheck( "POST", "/repos/" + template_owner + "/" + template_repo + "/generate", headers = {"Accept":"application/vnd.github.baptiste-preview+json"}, #required because templating api in preview input=post_parameters ) return Repository.Repository( user._requester, headers, data, completed=True )
def get_repo_attributes_dict(input_repo: Repository, last_commit_within_years: int = 2): result_dict = {} try: result_dict = { 'repo_path': input_repo.full_name, 'created_at': input_repo.created_at, 'last_commit': get_last_commit_date(input_repo), 'last_update': input_repo.updated_at, 'star_count': input_repo.stargazers_count, 'fork_count': input_repo.forks_count, 'contributors_count': input_repo.get_contributors().totalCount } today = datetime.datetime.today() check_start_date = datetime.datetime( today.year - last_commit_within_years, today.month, today.day) if result_dict['last_commit'] >= check_start_date: repo_status = 'active' else: repo_status = 'inactive' result_dict['repo_status'] = repo_status except Exception as e: print(e) return result_dict
def _clone_branch(self, reposlug: str, repo: Repository, branch: str, fromsha: str) -> Optional[str]: try: ref = repo.create_git_ref(ref='refs/heads/' + branch, sha=fromsha) return ref.ref except GithubException: return None
def get_result_from_network(commit: pygit2.Commit, gh_repo: github.Repository, db: ResultsDatabase) -> commit_result: # Fetch a commit_result from the network for 'commit'. # Saves the new commit_result in db before returning. # Prints signs of life to stderr. # Get the commit_result for commit from gh_repo. sha = str(commit.oid) gh_prs = list(gh_repo.get_commit(sha).get_pulls()) if len(gh_prs) == 0: gh_pr = None result = commit_result(sha, gh_repo.owner.login, gh_repo.name, NO_PULL_REQUEST, '', '') elif len(gh_prs) == 1: gh_pr = gh_prs[0] result = commit_result(sha, gh_repo.owner.login, gh_repo.name, gh_pr.number, gh_pr.title, gh_pr.html_url) else: sys.exit(f'{sha} has {len(gh_prs)} prs (expected 1): {gh_prs}') # Cache the result in the database. db.add_result(result) # Print sign of life. if gh_pr: print(f'\t{sha[:10]} {commit_shortlog(commit)}\n' f'\t PR #{gh_pr.number:5}: {gh_pr.title}', file=sys.stderr) else: print(f'\t{sha[:10]} {commit_shortlog(commit)}\n' f'\t Not merged via pull request', file=sys.stderr) return result
def create_release(repo: github.Repository, tag: str, artifacts_dir: str, draft: bool = False, prerelease: bool = False): commit = get_commit_from_tag(repo, tag) if not commit: logging.warning(f'Unable to find tag "{tag}". Skipping release.') return None, [] release = repo.create_git_release(tag, tag, commit.commit.message, draft=draft, prerelease=prerelease) artifacts = [] for filename in os.listdir(artifacts_dir): artifact = os.path.join(artifacts_dir, filename) if os.path.isfile(artifact): artifact = release.upload_asset(artifact, label=artifact_label(filename)) artifacts.append(artifact) logging.info(f'Uploading artifact "{artifact.name}" ...') else: logging.warning(f'Cannot upload directory "{artifact}". Skipping.') logging.info(f'Created release for tag "{tag}".') return release, artifacts
def assign_issues(repo: github.Repository): open_unassigned_issues = [ i for i in repo.get_issues() if i.state == 'open' and len(i.assignees) == 0 ] for issue in open_unassigned_issues: try_assign(issue)
def get_issues(r: github.Repository, data_dir: str): issues = dict() all_issues = r.get_issues(state="all") total = all_issues.totalCount cur = 0 for i in all_issues: cur += 1 if cur == 1 or cur % 20 == 0: print("Processing issue {} of {}".format(cur, total), file=sys.stderr) wait(verbose=True) else: wait() login = i.user.login if login not in issues.keys(): issues[login] = { "num_issues": 0, "open": 0, "closed": 0, "num_comments": 0, "num_labels": 0, "wordcount": 0, "title_wordcount": 0, "body_vocab": set(), "body_vocab_filescope": 0, "title_vocab": set(), "title_vocab_filescope": 0, "total_vocab_filescope": 0, } data = issues[login] data["num_issues"] += 1 if i.state == "open": data["open"] += 1 elif i.state == "closed": data["closed"] += 1 data["num_comments"] += i.get_comments().totalCount data["num_labels"] += len(i.labels) data["wordcount"] += len(i.body.split()) data["title_wordcount"] += len(i.title.split()) data["body_vocab"].update(set(i.body.split())) data["body_vocab_filescope"] += len(set(i.body.split())) data["title_vocab"].update(set(i.title.split())) data["title_vocab_filescope"] += len(set(i.title.split())) data["total_vocab_filescope"] += len( set(i.body.split()).union(set(i.title.split()))) # Convert vocab sets to numbers for login in issues.keys(): data = issues[login] data["total_vocab"] = len(data["body_vocab"].union( data["title_vocab"])) data["body_vocab"] = len(data["body_vocab"]) data["title_vocab"] = len(data["title_vocab"]) filename = os.path.join(data_dir, "issues.json") with open(filename + ".part", mode="w") as f: json.dump(issues, f) os.rename(filename + ".part", filename)
def get_last_commit_date(input_repo: Repository): """ get latest commit from repo :param input_repo: :return: """ page = input_repo.get_commits().get_page(0)[0] return page.commit.author.date
def get_time_of_last_run(repo: Repository) -> datetime: try: workflow = repo.get_workflow(WORKFLOW_FILE_NAME) last_run = next(iter(workflow.get_runs(branch='master', status='success'))) except (UnknownObjectException, StopIteration): return datetime(2021, 1, 1) else: return last_run.created_at - timedelta(minutes=5)
def get_compare_status(repo: Repository, fork: Repository, head: str) -> str: """ Compares a branch in a fork to the origin's default branch. :return: Status of the compared branch """ label = construct_fork_label(fork, head) compare = repo.compare(repo.default_branch, label) return compare.status
def _create_label(self, repo: Repository, label: Label) -> bool: try: github_label = repo.get_label(label.name) if self._verbose: print("[I] Edit existing label: {}".format(label.name)) github_label.edit(name=label.name, color=label.color, description=label.description) except UnknownObjectException: try: if self._verbose: print("[I] Creating label {}".format(label.name)) label = repo.create_label(label.name, label.color, label.description) except Exception as e: print("[E] Error creating label {}".format(label.name), str(e)) return False return True
def _repo_releases(self, repo: Repository, release_ids: Optional[list[str]]) -> list[Release]: if not release_ids: releases = [r for r in repo.get_releases() if r.published_at][:self.CONFIG.EXPLORE_RELEASES_DEPTH] else: releases = list(map(repo.get_release, release_ids)) return list( map(self._create_release_object, sorted(releases, key=lambda r: r.published_at, reverse=True)))
def update_changelog(version: str, github_repository: Repository) -> None: """ Add a version title to the changelog. """ changelog_path = Path('CHANGELOG.rst') branch = 'master' changelog_content_file = github_repository.get_contents( path=str(changelog_path), ref=branch, ) changelog_bytes = changelog_content_file.decoded_content changelog_contents = changelog_bytes.decode('utf-8') new_changelog_contents = changelog_contents.replace( 'Next\n----', f'Next\n----\n\n{version}\n------------', ) github_repository.update_file( path=str(changelog_path), message=f'Update for release {version}', content=new_changelog_contents, sha=changelog_content_file.sha, )
def get_contributors(repo: Repository, path: str, branch_name: str): """Gets a sorted list of contributors to a file in a repository.""" commit_list = repo.get_commits(sha=branch_name, path=path) # Remove duplicate users and map users to their commit counts. unique_users = defaultdict(int) for commit in commit_list: unique_users[commit.author] += 1 # Sort users by commit count, name them. return sorted(list(unique_users.keys()), key=lambda user: unique_users[user], reverse=True)
def repo_isempty(repo: github.Repository) -> bool: """ is a GitHub repo empty? Parameters ---------- repo : github.Repository handle to GitHub repo Results ------- empty : bool GitHub repo empty """ try: repo.get_contents('/') empty = False except github.GithubException as e: logging.error(f'{repo.name} is empty. \n') empty = True logging.info(str(e)) return empty
def update_labels(repository: Repository, new: Dict[str, Dict[str, str]]): logging.info(f'Working on {repository.name}') # Edit or delete existing labels for old in repository.get_labels(): # Edit labels if fmt(old.name) in new: # Edit only if there is a difference if (old.name, old.color, old.description) != tuple(new[fmt(old.name)].values()): logging.info(f'Editing {old.name}') old.edit(*new[fmt(old.name)].values()) else: logging.info(f'Deleting {old.name}') old.delete() # Create new labels existing_labels = {fmt(x.name) for x in repository.get_labels()} for new_label in new.values(): if fmt(new_label['name']) not in existing_labels: logging.info(f'Creating {new_label["name"]}') repository.create_label(*[x for x in new_label.values() if x is not None])
def set(repo: Repository, config): print(" Processing repo settings...") newsettings = {} if 'features' in config: for feat in config['features']: newsettings[f"has_{feat}"] = config['features'][feat] if 'allow' in config: for allow in config['allow']: newsettings[f"allow_{allow.replace('-', '_')}"] = config[ 'allow'][allow] if 'delete-branch-on-merge' in config: newsettings['delete_branch_on_merge'] = config[ 'delete-branch-on-merge'] if not RepoSetter.has_changes(newsettings, repo): print(" Repo settings unchanged.") return print(" Applying new repo settings...") repo.edit(**newsettings)
def get_compare_status(repo: Repository, fork: Repository, head: str) -> str: """ Compares a branch in a fork to the origin's default branch. :return: Status of the compared branch """ label = construct_fork_label(fork, head) try: compare = repo.compare(repo.default_branch, label) return compare.status except GithubException: # Because each head that is not "diverged" is removed, # the exact return make no difference here, like # if GithubException.status == "404": return GithubException.data
def dump_api(repo: Repository): result = [] # We use the first (latest) issue as indicator of how many data we have # to fetch issues = repo.get_issues(state="all") latest_issue = issues[0] count = latest_issue.number logger.info("Pulling {} items", count) for i in range(1, count + 1): try: issue = repo.get_issue(i) logger.info("{}: {}", issue.number, issue.title) result.append(issue.raw_data) except Exception as err: logger.info("Unable to get data, waiting a minute: {}", err) time.sleep(60) with open(Data.API_DATA_JSON, "w") as data_file: json.dump(result, data_file) logger.info("Done exporting {} items", i) Data.api_to_tarball()