def _create_pr(self, reposlug: str, repo: Repository, branch: str,
onto: str) -> None:
repo.create_pull(title="Harden ignore files",
body="Autofix created by devsecops",
base=onto,
head=branch)
print("[I] Created PR {}".format(reposlug))
def create_github_release(
repository: Repository,
version: str,
artifacts: Set[Path],
) -> None:
"""
Create a tag and release on GitHub.
"""
changelog_url = 'https://dcos-e2e.readthedocs.io/en/latest/changelog.html'
release_name = 'Release ' + version
release_message = 'See ' + changelog_url
github_release = repository.create_git_tag_and_release(
tag=version,
tag_message='Release ' + version,
release_name=release_name,
release_message=release_message,
type='commit',
object=repository.get_commits()[0].sha,
draft=True,
)
for artifact_path in artifacts:
github_release.upload_asset(
path=str(artifact_path),
label=artifact_path.name,
)
github_release.update_release(
name=release_name,
message=release_message,
draft=False,
)
def open_issue(pr, pr_checks, hub_repo: Repository, module, jina_core_version):
"""opens an issue for the PR with the failed checks (if not already open)"""
issue_name = f'{FIX_MODULE_TEMPLATE}{module}'
existing_issue_for_pr = [
i for i in list(hub_repo.get_issues(state='open'))
if i.title == issue_name
]
if len(existing_issue_for_pr) > 0:
print(f'Found existing issue: {existing_issue_for_pr}')
return existing_issue_for_pr[0]
else:
# open the issue
body = f"""
**[This is an automated issue opened as part of the hub modules update GH action. DO NOT EDIT THIS DESCRIPTION]**
Could not build module {module} for Jina core version {jina_core_version} because some of the checks failed:
```
{[(c['name'], c['status'], c['conclusion']) for c in pr_checks]}
```
See {pr.html_url} for more info. {TAG_IN_ISSUES}
"""
issue = hub_repo.create_issue(
title=issue_name,
body=body,
)
print(f'opened issue at {issue.html_url}')
return issue
def _create_label_if_needed(self, reposlug: str, repo: Repository,
name: str, desc: str, color: str) -> Label:
try:
label = repo.get_label("devsecops")
except UnknownObjectException:
if self._verbose:
print("[I] Creating label {}:{}".format(reposlug, name))
label = repo.create_label(name, color, desc)
return label
def _create_fix(self, reposlug: str, repo: Repository, branch: str,
result: ScanResult) -> None:
new_content = "{}\n{}\n".format(result.content,
"\n".join(result.missings))
repo.update_file(result.filename,
message="Autofix by devsecops",
content=new_content,
sha=result.filesha,
branch=branch)
print("[I] Updated {}: {}".format(reposlug, result.filename))
def _scan_repository(self, results: ScanResults, user: NamedUser, org: Organization, repo: Repository) -> None:
try:
branch = repo.get_branch(branch="master")
tree = repo.get_git_tree(branch.commit.sha, recursive=True).tree
except GithubException as e:
print("[W] {}/{} - {}".format(org.login, repo.name, str(e)))
# Skip if no master branch
return
filelist = [x.path for x in tree]
for element in tree:
self._scan_file(results, user, org, repo, branch.commit.sha, element.path, filelist)
def extract_branches(repo: Repository, from_pulls: bool = False) -> Set[str]:
"""
Returns a set of branch name strings by wrapping PyGithub's get_branches().
See https://pygithub.readthedocs.io/en/latest/github_objects/Repository.html#github.Repository.Repository.get_branches
"""
if from_pulls:
branches: Set[str] = set(b.head.ref for b in repo.get_pulls())
else:
branches: Set[str] = set(b.name for b in repo.get_branches())
return branches
def process_milestone(
repo: Repository,
milestone: Milestone,
in_progress_label: Optional[Label],
blocked_label: Optional[Label],
) -> str:
now = datetime.now()
html_url = milestone._rawData["html_url"]
total_issues = milestone.open_issues + milestone.closed_issues
percentage_complete = as_percentage(milestone.closed_issues, total_issues)
detail_lines = []
status_line = f":heavy_check_mark: {percentage_complete}% completed"
if in_progress_label:
in_progress_issues = list(
repo.get_issues(milestone=milestone,
labels=[in_progress_label],
state="open"))
if in_progress_issues:
percentage_in_progress = as_percentage(len(in_progress_issues),
total_issues)
status_line += (
f" - :hourglass_flowing_sand: {percentage_in_progress}% in progress"
)
if blocked_label:
blocked_issues = list(
repo.get_issues(milestone=milestone,
labels=[blocked_label],
state="open"))
if blocked_issues:
percentage_blocked = as_percentage(len(blocked_issues),
total_issues)
status_line += f" - :octagonal_sign: {percentage_blocked}% blocked"
detail_lines.append(status_line)
if milestone.due_on:
duration = milestone.due_on - milestone.created_at
remaining = milestone.due_on - now
time_used = 100 - as_percentage(remaining.days, duration.days)
detail_lines.append(
f":date: {milestone.due_on.date().isoformat()} - :alarm_clock: {time_used}% time used"
)
rendered_line = (
f"<{html_url}|{milestone.title}> - {milestone.closed_issues}/{total_issues}"
)
for line in detail_lines:
rendered_line += f"\n\t{line}"
return rendered_line
def create_github_release(
repository: Repository,
version: str,
) -> None:
"""
Create a tag and release on GitHub.
"""
changelog_url = 'https://dcos-e2e.readthedocs.io/en/latest/changelog.html'
release_name = 'Release ' + version
release_message = 'See ' + changelog_url
github_release = repository.create_git_tag_and_release(
tag=version,
tag_message='Release ' + version,
release_name=release_name,
release_message=release_message,
type='commit',
object=repository.get_commits()[0].sha,
draft=False,
)
# The artifacts we build must be built from the tag we just created.
# This tag is created remotely on GitHub using the GitHub HTTP API.
#
# We fetch all tags from GitHub and set our local HEAD to the latest master
# from GitHub.
#
# One symptom of this is that ``minidcos --version`` from the PyInstaller
# binary shows the correct version.
local_repository = Repo('.')
client = HttpGitClient(repository.owner.html_url)
remote_refs = client.fetch(repository.name + '.git', local_repository)
# Update the local tags and references with the remote ones.
for key, value in remote_refs.items():
local_repository.refs[key] = value
# Advance local HEAD to remote master HEAD.
local_repository[b'HEAD'] = remote_refs[b'refs/heads/master']
# We need to make the artifacts just after creating a tag so that the
# --version output is exactly the one of the tag.
# No tag exists when the GitHub release is a draft.
# This means that temporarily we have a release without binaries.
linux_artifacts = make_linux_binaries(repo_root=Path('.'))
for installer_path in linux_artifacts:
github_release.upload_asset(
path=str(installer_path),
label=installer_path.name,
)
def create_github_release(
repository: Repository,
version: str,
) -> None:
"""
Create a tag and release on GitHub.
"""
repository.create_git_tag_and_release(
tag=version,
tag_message='Release ' + version,
release_name='Release ' + version,
release_message='See CHANGELOG.rst',
type='commit',
object=repository.get_commits()[0].sha,
)
def install(self, name_or_path: str, version: str) -> None:
try:
with State() as s:
name_or_path = self._validate_and_expand_path(name_or_path)
self.log.step(f"Installing from {name_or_path}@{version}")
repo = Repository.from_url(name_or_path)
if repo.name in s.state:
raise ValueError(
f"{repo.name} is already installed. Use `binman update` to switch versions, or `binman uninstall` to remove"
)
release = repo.get_release(tag=version)
self.log.step(release.tag_name, padding=1)
arts = []
for artifact in release.get_platform_assets():
if "md5" in artifact.download_url or "sha256" in artifact.download_url:
continue
tgt_dir = self._install_location / artifact.name
self.log.progress(
f"{artifact.name}/{artifact.platform}-{artifact.architecture} ~> {tgt_dir}", padding=2
)
artifact.download(self._install_location)
arts.append(tgt_dir)
s.update_application(name=repo.name, url=name_or_path, artifacts=arts, version=release.tag_name)
self.log.info("Installation complete")
except Exception as e:
self.log.error(f"{type(e).__name__}: {e}.")
raise
def _is_branch_exists(self, repo: Repository,
branch: str) -> Optional[str]:
try:
b = repo.get_branch(branch)
return b.commit.sha
except GithubException:
return None
def create_git_repo_from_template(user, new_owner, new_repo, template_owner, template_repo, private=True):
"""Creates a new github repository from a template
Args:
user (AuthenticatedUser): user object
new_owner (string): owner of the new repository
new_repo (string): name of the new repository
template_owner (string): owner of the template repository
template_repo (string): name of the template repository
private (boolean): if the repo is private (default True)
Returns:
The new repository object
"""
post_parameters = {
"owner": new_owner,
"name": new_repo,
"private": private
}
headers, data = user._requester.requestJsonAndCheck(
"POST",
"/repos/" + template_owner + "/" + template_repo + "/generate",
headers = {"Accept":"application/vnd.github.baptiste-preview+json"}, #required because templating api in preview
input=post_parameters
)
return Repository.Repository(
user._requester, headers, data, completed=True
)
def get_repo_attributes_dict(input_repo: Repository,
last_commit_within_years: int = 2):
result_dict = {}
try:
result_dict = {
'repo_path': input_repo.full_name,
'created_at': input_repo.created_at,
'last_commit': get_last_commit_date(input_repo),
'last_update': input_repo.updated_at,
'star_count': input_repo.stargazers_count,
'fork_count': input_repo.forks_count,
'contributors_count': input_repo.get_contributors().totalCount
}
today = datetime.datetime.today()
check_start_date = datetime.datetime(
today.year - last_commit_within_years, today.month, today.day)
if result_dict['last_commit'] >= check_start_date:
repo_status = 'active'
else:
repo_status = 'inactive'
result_dict['repo_status'] = repo_status
except Exception as e:
print(e)
return result_dict
def _clone_branch(self, reposlug: str, repo: Repository, branch: str,
fromsha: str) -> Optional[str]:
try:
ref = repo.create_git_ref(ref='refs/heads/' + branch, sha=fromsha)
return ref.ref
except GithubException:
return None
def get_result_from_network(commit: pygit2.Commit,
gh_repo: github.Repository,
db: ResultsDatabase) -> commit_result:
# Fetch a commit_result from the network for 'commit'.
# Saves the new commit_result in db before returning.
# Prints signs of life to stderr.
# Get the commit_result for commit from gh_repo.
sha = str(commit.oid)
gh_prs = list(gh_repo.get_commit(sha).get_pulls())
if len(gh_prs) == 0:
gh_pr = None
result = commit_result(sha, gh_repo.owner.login, gh_repo.name,
NO_PULL_REQUEST, '', '')
elif len(gh_prs) == 1:
gh_pr = gh_prs[0]
result = commit_result(sha, gh_repo.owner.login, gh_repo.name,
gh_pr.number, gh_pr.title, gh_pr.html_url)
else:
sys.exit(f'{sha} has {len(gh_prs)} prs (expected 1): {gh_prs}')
# Cache the result in the database.
db.add_result(result)
# Print sign of life.
if gh_pr:
print(f'\t{sha[:10]} {commit_shortlog(commit)}\n'
f'\t PR #{gh_pr.number:5}: {gh_pr.title}',
file=sys.stderr)
else:
print(f'\t{sha[:10]} {commit_shortlog(commit)}\n'
f'\t Not merged via pull request',
file=sys.stderr)
return result
def create_release(repo: github.Repository,
tag: str,
artifacts_dir: str,
draft: bool = False,
prerelease: bool = False):
commit = get_commit_from_tag(repo, tag)
if not commit:
logging.warning(f'Unable to find tag "{tag}". Skipping release.')
return None, []
release = repo.create_git_release(tag,
tag,
commit.commit.message,
draft=draft,
prerelease=prerelease)
artifacts = []
for filename in os.listdir(artifacts_dir):
artifact = os.path.join(artifacts_dir, filename)
if os.path.isfile(artifact):
artifact = release.upload_asset(artifact,
label=artifact_label(filename))
artifacts.append(artifact)
logging.info(f'Uploading artifact "{artifact.name}" ...')
else:
logging.warning(f'Cannot upload directory "{artifact}". Skipping.')
logging.info(f'Created release for tag "{tag}".')
return release, artifacts
def assign_issues(repo: github.Repository):
open_unassigned_issues = [
i for i in repo.get_issues()
if i.state == 'open' and len(i.assignees) == 0
]
for issue in open_unassigned_issues:
try_assign(issue)
def get_issues(r: github.Repository, data_dir: str):
issues = dict()
all_issues = r.get_issues(state="all")
total = all_issues.totalCount
cur = 0
for i in all_issues:
cur += 1
if cur == 1 or cur % 20 == 0:
print("Processing issue {} of {}".format(cur, total),
file=sys.stderr)
wait(verbose=True)
else:
wait()
login = i.user.login
if login not in issues.keys():
issues[login] = {
"num_issues": 0,
"open": 0,
"closed": 0,
"num_comments": 0,
"num_labels": 0,
"wordcount": 0,
"title_wordcount": 0,
"body_vocab": set(),
"body_vocab_filescope": 0,
"title_vocab": set(),
"title_vocab_filescope": 0,
"total_vocab_filescope": 0,
}
data = issues[login]
data["num_issues"] += 1
if i.state == "open":
data["open"] += 1
elif i.state == "closed":
data["closed"] += 1
data["num_comments"] += i.get_comments().totalCount
data["num_labels"] += len(i.labels)
data["wordcount"] += len(i.body.split())
data["title_wordcount"] += len(i.title.split())
data["body_vocab"].update(set(i.body.split()))
data["body_vocab_filescope"] += len(set(i.body.split()))
data["title_vocab"].update(set(i.title.split()))
data["title_vocab_filescope"] += len(set(i.title.split()))
data["total_vocab_filescope"] += len(
set(i.body.split()).union(set(i.title.split())))
# Convert vocab sets to numbers
for login in issues.keys():
data = issues[login]
data["total_vocab"] = len(data["body_vocab"].union(
data["title_vocab"]))
data["body_vocab"] = len(data["body_vocab"])
data["title_vocab"] = len(data["title_vocab"])
filename = os.path.join(data_dir, "issues.json")
with open(filename + ".part", mode="w") as f:
json.dump(issues, f)
os.rename(filename + ".part", filename)
def get_last_commit_date(input_repo: Repository):
"""
get latest commit from repo
:param input_repo:
:return:
"""
page = input_repo.get_commits().get_page(0)[0]
return page.commit.author.date
def get_time_of_last_run(repo: Repository) -> datetime:
try:
workflow = repo.get_workflow(WORKFLOW_FILE_NAME)
last_run = next(iter(workflow.get_runs(branch='master', status='success')))
except (UnknownObjectException, StopIteration):
return datetime(2021, 1, 1)
else:
return last_run.created_at - timedelta(minutes=5)
def get_compare_status(repo: Repository, fork: Repository, head: str) -> str:
"""
Compares a branch in a fork to the origin's default branch.
:return: Status of the compared branch
"""
label = construct_fork_label(fork, head)
compare = repo.compare(repo.default_branch, label)
return compare.status
def _create_label(self, repo: Repository, label: Label) -> bool:
try:
github_label = repo.get_label(label.name)
if self._verbose:
print("[I] Edit existing label: {}".format(label.name))
github_label.edit(name=label.name,
color=label.color,
description=label.description)
except UnknownObjectException:
try:
if self._verbose:
print("[I] Creating label {}".format(label.name))
label = repo.create_label(label.name, label.color,
label.description)
except Exception as e:
print("[E] Error creating label {}".format(label.name), str(e))
return False
return True
def _repo_releases(self, repo: Repository,
release_ids: Optional[list[str]]) -> list[Release]:
if not release_ids:
releases = [r for r in repo.get_releases()
if r.published_at][:self.CONFIG.EXPLORE_RELEASES_DEPTH]
else:
releases = list(map(repo.get_release, release_ids))
return list(
map(self._create_release_object,
sorted(releases, key=lambda r: r.published_at, reverse=True)))
def update_changelog(version: str, github_repository: Repository) -> None:
"""
Add a version title to the changelog.
"""
changelog_path = Path('CHANGELOG.rst')
branch = 'master'
changelog_content_file = github_repository.get_contents(
path=str(changelog_path),
ref=branch,
)
changelog_bytes = changelog_content_file.decoded_content
changelog_contents = changelog_bytes.decode('utf-8')
new_changelog_contents = changelog_contents.replace(
'Next\n----',
f'Next\n----\n\n{version}\n------------',
)
github_repository.update_file(
path=str(changelog_path),
message=f'Update for release {version}',
content=new_changelog_contents,
sha=changelog_content_file.sha,
)
def get_contributors(repo: Repository, path: str, branch_name: str):
"""Gets a sorted list of contributors to a file in a repository."""
commit_list = repo.get_commits(sha=branch_name, path=path)
# Remove duplicate users and map users to their commit counts.
unique_users = defaultdict(int)
for commit in commit_list:
unique_users[commit.author] += 1
# Sort users by commit count, name them.
return sorted(list(unique_users.keys()),
key=lambda user: unique_users[user],
reverse=True)
def repo_isempty(repo: github.Repository) -> bool:
"""
is a GitHub repo empty?
Parameters
----------
repo : github.Repository
handle to GitHub repo
Results
-------
empty : bool
GitHub repo empty
"""
try:
repo.get_contents('/')
empty = False
except github.GithubException as e:
logging.error(f'{repo.name} is empty. \n')
empty = True
logging.info(str(e))
return empty
def update_labels(repository: Repository, new: Dict[str, Dict[str, str]]):
logging.info(f'Working on {repository.name}')
# Edit or delete existing labels
for old in repository.get_labels():
# Edit labels
if fmt(old.name) in new:
# Edit only if there is a difference
if (old.name, old.color, old.description) != tuple(new[fmt(old.name)].values()):
logging.info(f'Editing {old.name}')
old.edit(*new[fmt(old.name)].values())
else:
logging.info(f'Deleting {old.name}')
old.delete()
# Create new labels
existing_labels = {fmt(x.name) for x in repository.get_labels()}
for new_label in new.values():
if fmt(new_label['name']) not in existing_labels:
logging.info(f'Creating {new_label["name"]}')
repository.create_label(*[x for x in new_label.values() if x is not None])
def set(repo: Repository, config):
print(" Processing repo settings...")
newsettings = {}
if 'features' in config:
for feat in config['features']:
newsettings[f"has_{feat}"] = config['features'][feat]
if 'allow' in config:
for allow in config['allow']:
newsettings[f"allow_{allow.replace('-', '_')}"] = config[
'allow'][allow]
if 'delete-branch-on-merge' in config:
newsettings['delete_branch_on_merge'] = config[
'delete-branch-on-merge']
if not RepoSetter.has_changes(newsettings, repo):
print(" Repo settings unchanged.")
return
print(" Applying new repo settings...")
repo.edit(**newsettings)
def get_compare_status(repo: Repository, fork: Repository, head: str) -> str:
"""
Compares a branch in a fork to the origin's default branch.
:return: Status of the compared branch
"""
label = construct_fork_label(fork, head)
try:
compare = repo.compare(repo.default_branch, label)
return compare.status
except GithubException:
# Because each head that is not "diverged" is removed,
# the exact return make no difference here, like
# if GithubException.status == "404":
return GithubException.data
def dump_api(repo: Repository):
result = []
# We use the first (latest) issue as indicator of how many data we have
# to fetch
issues = repo.get_issues(state="all")
latest_issue = issues[0]
count = latest_issue.number
logger.info("Pulling {} items", count)
for i in range(1, count + 1):
try:
issue = repo.get_issue(i)
logger.info("{}: {}", issue.number, issue.title)
result.append(issue.raw_data)
except Exception as err:
logger.info("Unable to get data, waiting a minute: {}", err)
time.sleep(60)
with open(Data.API_DATA_JSON, "w") as data_file:
json.dump(result, data_file)
logger.info("Done exporting {} items", i)
Data.api_to_tarball()
