def check_prs_for_current_release_are_released(
repo: github.Repository.Repository) -> None:
"""Checks that all pull requests for current release have a
'PR: released' label.
Args:
repo: github.Repository.Repository. The PyGithub object for the repo.
Raises:
Exception. Some pull requests for current release do not have a
"PR: released" label.
"""
current_release_label = repo.get_label(
constants.release_constants.LABEL_FOR_CURRENT_RELEASE_PRS)
current_release_prs = repo.get_issues(state='all',
labels=[current_release_label])
for pr in current_release_prs:
label_names = [label.name for label in pr.labels]
if constants.release_constants.LABEL_FOR_RELEASED_PRS not in (
label_names):
open_new_tab_in_browser_if_possible(
'https://github.com/oppia/oppia/pulls?utf8=%E2%9C%93&q=is%3Apr'
'+label%3A%22PR%3A+for+current+release%22+')
raise Exception(
'There are PRs for current release which do not have '
'a \'PR: released\' label. Please ensure that they are '
'released before release summary generation.')
def update_packages(repo: github.Repository.Repository, branch_name: str,
config: dict) -> None:
try:
packages_content = repo.get_contents("packages.yml")
packages = ruamel.yaml.load(
packages_content.decoded_content,
Loader=ruamel.yaml.RoundTripLoader,
preserve_quotes=True,
)
for package in packages["packages"]:
if "package" in package:
name = package["package"]
if name in config["packages"]:
package["version"] = config["packages"][name]
if "git" in package:
name = package["git"]
if name in config["packages"]:
package["revision"] = config["packages"][name]
repo.update_file(
path=packages_content.path,
message="Updating package dependendcies",
content=ruamel.yaml.dump(packages,
Dumper=ruamel.yaml.RoundTripDumper),
sha=packages_content.sha,
branch=branch_name,
)
except github.GithubException:
print("'packages.yml' not found in repo.")
def create_labels(current: List[github.Label.Label], expected: List[Label],
repo: github.Repository.Repository, run: bool) -> None:
def __creation_plan(current: List[github.Label.Label],
expected: List[Label]) -> List[Label]:
current_names: List[str] = [l.name for l in current]
expected_names: List[str] = [l.name for l in expected]
names: List[str] = list(set(expected_names) - set(current_names))
return [l for l in expected if l.name in names]
for l in __creation_plan(current, expected):
print(' Create `' + l.name + '`')
if run:
repo.create_label(l.name, l.color, l.description)
def search_repo(self, repo: github.Repository.Repository,
repo_out: github.Repository.Repository):
for issue_state in self._issue_state:
issues = repo.get_issues(state=issue_state, sort='updated')
total = 0
for _ in issues:
total = total + 1
for i, issue in enumerate(issues):
upstream_tag = False
confirm_tag = False
tag = ''
for label in issue.labels:
if label.name == 'upstream':
upstream_tag = True
elif label.name == 'confirm':
confirm_tag = True
else:
tag = label.name.title()
if not (not confirm_tag and upstream_tag):
continue
# GitHub's REST API v3 considers every pull request an issue, but not every issue is a pull request.
# You can identify pull requests by the pull_request key.
if issue.pull_request:
continue
print("--------")
print('[ISSUE][%s][%d/%d]\t%s %s' %
(issue_state, i + 1, total, repo.name, issue.title))
title = '%s: %s' % (tag, issue.title)
print(title)
print('Moved from: %s#%d' % (repo.full_name, issue.number))
print('Original author: @%s' % (issue.user.login))
issue_body = issue.body.strip()
if not issue_body:
issue_body = '**No description provided.** :sleeping:'
body = \
'''
Moved from: %s#%d
Original author: @%s
### Issue description
%s
'''% (repo.full_name, issue.number, issue.user.login, issue_body)
print(body)
# close issue and create a new one
issue.edit(state='closed')
repo_out.create_issue(title=title, body=body)
def search_repo(self, repo: github.Repository.Repository):
if not self._pr and not self._issue:
self._pr = True
if self._pr:
for pr_state in self._pr_state:
prs = repo.get_pulls(state=pr_state, sort='updated')
# totalCount does not work
# https://github.com/PyGithub/PyGithub/issues/870
total = 0
for _ in prs:
total = total + 1
for i, pr in enumerate(prs):
self.printcl('[PR][%s][%d/%d]\t\t%s' % (self._state_to_utf[pr_state], i, total, repo.name))
# It's not me
if self._search_username not in pr.user.login:
continue
# Check for max age
if (datetime.now().date() - pr.updated_at.date()).days > self._age:
break
self.print_pr(pr)
if self._issue:
for issue_state in self._issue_state:
issues = repo.get_issues(state=issue_state, sort='updated')
total = 0
for _ in issues:
total = total + 1
for i, issue in enumerate(issues):
# GitHub's REST API v3 considers every pull request an issue, but not every issue is a pull request.
# You can identify pull requests by the pull_request key.
if issue.pull_request:
continue
self.printcl('[ISSUE][%s][%d/%d]\t%s' % (self._state_to_utf[issue_state], i + 1, total, repo.name))
# It's not me
check_for_name_in = [issue.user.login]
if issue.closed_by:
check_for_name_in.append(issue.closed_by.login)
if issue.assignee:
check_for_name_in.append(issue.assignee.login)
if self._search_username not in check_for_name_in:
continue
# Check for max age
if (datetime.now().date() - issue.updated_at.date()).days > self._age:
break
self.print_issue(issue)
def phab_login_to_github_login(phab_token: str,
repo: github.Repository.Repository,
phab_login: str) -> str:
"""
Tries to translate a Phabricator login to a github login by
finding a commit made in Phabricator's Differential.
The commit's SHA1 is then looked up in the github repo and
the committer's login associated with that commit is returned.
:param str phab_token: The Conduit API token to use for communication with Pabricator
:param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
:param str phab_login: The Phabricator login to be translated.
"""
args = {
"constraints[authors][0]": phab_login,
# PHID for "LLVM Github Monorepo" repository
"constraints[repositories][0]": "PHID-REPO-f4scjekhnkmh7qilxlcy",
"limit": 1
}
# API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
r = phab_api_call(phab_token,
"https://reviews.llvm.org/api/diffusion.commit.search",
args)
data = r['result']['data']
if len(data) == 0:
# Can't find any commits associated with this user
return None
commit_sha = data[0]['fields']['identifier']
return repo.get_commit(commit_sha).committer.login
def add_base_files(moss: mosspy.Moss, base_files: Iterable,
repo: github.Repository.Repository):
print(f'Adding base files from repo: {repo.full_name}')
for path in base_files:
print(f'\t{path}')
content_file = repo.get_contents(path=path)
head, tail = os.path.split(path)
file_path = f'{repo.name}_{tail}'
with open(file_path, 'wb') as f:
f.write(content_file.decoded_content)
moss.addBaseFile(file_path=file_path, display_name=file_path)
def get_prs(
repo: github.Repository.Repository,
t_a: datetime.datetime,
t_b: datetime.datetime,
):
merged = [] # PRs merged during the month
for pr in repo.get_pulls("closed", sort="updated",
direction="desc"): # most recently merged first
if pr.merged:
if t_a <= pr.merged_at < t_b: # could check `.month` instead...
merged.append(pr)
if pr.merged_at < t_a:
break
wip = [] # WIP PRs (not merged, still open at this time)
for pr in repo.get_pulls("open"):
if pr.created_at < t_b: # and pr.updated_at >= t_a:
wip.append(pr)
return _Prs(merged, wip)
def search_repo(self, repo: github.Repository.Repository,
repo_out: github.Repository.Repository):
repo_path = "/tmp/%s" % (repo.name)
call([
"git", "clone",
"https://github.com/%s" % (repo.full_name), repo_path
])
call(("git --git-dir %s/.git remote add output https://github.com/%s" %
(repo_path, repo_out.full_name)).split(' '))
call(["ls", "-l", repo_path])
for pr_state in self._pr_state:
prs = repo.get_pulls(state=pr_state, sort='updated')
total = 0
for _ in prs:
total = total + 1
for i, pr in enumerate(prs):
print('[PR][%s][%d/%d]\t%s %s' %
(pr_state, i + 1, total, repo.name, pr.title))
#print('Moved from: %s#%d' % (repo.full_name, pr.number))
#print('Original author: @%s' % (pr.user.login))
#print('Body: %s' % (pr.body))
call(("git --git-dir %s/.git fetch origin pull/%d/head:%d" %
(repo_path, pr.number, pr.number)).split(' '))
call(("git --git-dir %s/.git push output %d" %
(repo_path, pr.number)).split(' '))
head = "%s:%d" % (repo_out.owner.login, pr.number)
body = \
'''
%s
Moved from: %s#%d
Original author: @%s
''' % (pr.body, repo.full_name, pr.number, pr.user.login)
repo_out.create_pull(title=pr.title,
head=head,
base="master",
body=body)
def get_repo_clones_traffic(self,
repo: github.Repository.Repository,
save_dir: Union[str, pathlib.Path] = 'clone_data',
load=False):
"""gets clones traffic for `repo` and saves as csv in `save_dir`
Parameters
----------
repo : Union[str,github.Repository.Repository]
repository from `org`
save_dir : Union[str, pathlib.Path], optional
directory where output CSV should be saved, by default 'clone_data'
load : bool, optional
load data into a Pandas DataFrame, by default False
Returns
-------
pd.DataFrame
contains unique and total clones for `repo` with dates
"""
if type(repo) == str:
repo = self.org.get_repo(repo)
clones = repo.get_clones_traffic()
clones_dict = {
view.timestamp: {
"total_clones": view.count,
"unique_clones": view.uniques,
}
for view in clones['clones']
}
try:
old_clones_data = pd.read_csv(
f'{save_dir}/{repo.name}_clones_traffic.csv',
index_col="_date",
parse_dates=["_date"]).to_dict(orient="index")
updated_clones_dict = {**old_clones_data, **clones_dict}
clones_frame = pd.DataFrame.from_dict(
data=updated_clones_dict,
orient="index",
columns=["total_clones", "unique_clones"])
except:
clones_frame = pd.DataFrame.from_dict(
data=clones_dict,
orient="index",
columns=["total_clones", "unique_clones"])
clones_frame.index.name = "_date"
if not Path(save_dir).exists():
Path(save_dir).mkdir()
clones_frame.to_csv(f'{save_dir}/{repo.name}_clones_traffic.csv')
if load:
return clones_frame
def update_requirements(repo: github.Repository.Repository, branch_name: str,
config: str) -> None:
try:
requirements_content = repo.get_contents(
"integration_tests/requirements.txt")
new_content = ""
for requirement in config["requirements"]:
new_content += f"{requirement['name']}=={requirement['version']}\n"
repo.update_file(
path=requirements_content.path,
message="Updating dbt version in requirements.txt",
content=new_content,
sha=requirements_content.sha,
branch=branch_name,
)
except github.GithubException:
repo.create_file(
path="integration_tests/requirements.txt",
message="Updating dbt version in requirements.txt",
content=new_content,
branch=branch_name,
)
def repo_isempty(repo: github.Repository.Repository) -> bool:
"""
is a GitHub repo empty?
Parameters
----------
repo : github.Repository
handle to GitHub repo
Results
-------
empty : bool
GitHub repo empty
"""
try:
repo.get_contents("/")
empty = False
except github.GithubException as e:
logging.error(f"{repo.name} is empty. \n")
empty = True
logging.info(str(e))
return empty
def get_prs(
repo: github.Repository.Repository,
t_a: datetime.datetime,
t_b: datetime.datetime,
):
merged = [] # PRs merged during the month
pl = repo.get_pulls("closed", sort="updated",
direction="desc") # most recently *updated* first
for pr in _maybe_prog(pl, desc=repo.name):
if pr.merged:
if t_a <= pr.merged_at < t_b:
merged.append(pr)
# if pr.merged_at < t_a:
# break
# ^ This causes some to be missed since recently updated not equiv. to recently merged.
merged.sort(key=lambda pr: pr.merged_at) # earliest merged first
wip = [] # WIP PRs (not merged, still open at this time)
for pr in repo.get_pulls("open"):
if pr.created_at < t_b: # and pr.updated_at >= t_a:
wip.append(pr)
return _Prs(merged, wip)
def update_project(repo: github.Repository.Repository, branch_name: str,
config: str) -> None:
project_content = repo.get_contents("dbt_project.yml")
project = ruamel.yaml.load(
project_content.decoded_content,
Loader=ruamel.yaml.RoundTripLoader,
preserve_quotes=True,
)
project["require-dbt-version"] = config["require-dbt-version"]
current_version = project["version"]
bump_type = config["version-bump-type"]
current_version_split = current_version.split(".")
if bump_type == "patch":
current_version_split[2] = str(int(current_version_split[2]) + 1)
elif bump_type == "minor":
current_version_split[1] = str(int(current_version_split[1]) + 1)
current_version_split[2] = "0"
elif bump_type == "major":
current_version_split[0] = str(int(current_version_split[0]) + 1)
current_version_split[1] = "0"
current_version_split[2] = "0"
new_version = ".".join(current_version_split)
project["version"] = new_version
repo.update_file(
path=project_content.path,
message="Updating require-dbt-version",
content=ruamel.yaml.dump(project, Dumper=ruamel.yaml.RoundTripDumper),
sha=project_content.sha,
branch=branch_name,
)
def open_pull_request(repo: github.Repository.Repository, branch_name: str,
default_branch: str) -> None:
body = """
#### This pull request was created automatically 🎉
Before merging this PR:
- [ ] Verify that all the tests pass.
- [ ] Tag a release
"""
pull = repo.create_pull(
title="[MagicBot] Bumping package version",
body=body,
head=branch_name,
base=default_branch,
)
print(pull.html_url)
def create_scrum_issue(
repo: github.Repository.Repository,
date: datetime.date,
previous_issue: github.Issue.Issue = None,
) -> typing.Optional[github.Issue.Issue]:
"""
Create a scrum issue for the given date.
If not None, previous_issue is used to set an issue body
that refers to the previous issue.
"""
kwargs = {'title': f"{date}: e-scrum for {date:%A, %B %-d, %Y}"}
if previous_issue:
kwargs['body'] = 'Preceeding e-scrum in {}.'.format(previous_issue.html_url)
print('Creating {title!r}'.format(**kwargs), file=sys.stderr)
try:
return repo.create_issue(**kwargs)
except Exception:
print('Creating issue failed:\n{}'.fomrat(traceback.format_exc()), file=sys.stderr)
def check_isolated_branch(repo: github.Repository.Repository,
branch_name: str) -> bool:
"""Check if a branch is old and not associated with open PRs."""
# find all commits in the branch
commits = repo.get_commits(sha=branch_name)
# find the latest commit
latest_commit = max(commits,
key=lambda commit: commit.commit.committer.date)
# find out if the latest commit is older than our time range
latest_commit_timestamp = latest_commit.commit.committer.date
if datetime.today() - latest_commit_timestamp > timedelta(
days=DAYS_DEFINED_AS_STALE):
pull_requests = latest_commit.get_pulls()
# not associated with any open PR
return all([pr.state == 'closed' for pr in pull_requests])
# ignore if branch is recent
return False
def has_python_code(repo: github.Repository.Repository):
return LANGUAGE in repo.get_languages()
def check_if_pull_request_exists(self, repo: github.Repository.Repository,
head: str) -> bool:
pulls = repo.get_pulls(head=head)
return pulls.totalCount != 0
def has_python_code(repo: github.Repository.Repository):
return LANGUAGE in repo.get_languages()
def fork_prober(
repo: github.Repository.Repository,
sess: github.Github,
ahead: list[tuple[str, int]],
branch: str = None,
verbose: bool = False,
) -> list[tuple[str, int]]:
"""
check a GitHub repo for forks
Parameters
----------
repo :
handle to GitHub repo
sess :
handle to GitHub session
ahead : list of tuple of str, int
forked with repos with number of commits they're ahead of your repo
branch : str, optional
Git branch to examine
verbose : bool, optional
verbosity
Results
-------
ahead : list of tuple of str, int
forked with repos with number of commits they're ahead of your repo
"""
check_api_limit(sess)
b = repo.default_branch if not branch else branch
try:
master = repo.get_branch(b)
except github.GithubException as e:
logging.error(f"{repo.full_name} {e}")
return ahead
forks = repo.get_forks()
for fork in forks:
sleep(0.1) # don't hammer the API, avoiding 502 errors
check_api_limit(sess)
try:
fmaster = fork.get_branch(b)
except github.GithubException as e:
if (
e.data["message"] == "Not Found"
): # repo/branch that they deleted FIXME: should we check their default branch?
continue
logging.error(f"{repo.full_name} {fork.full_name} {e}")
continue
try:
comp = repo.compare(master.commit.sha, fmaster.commit.sha)
except github.GithubException as excp:
# if excp.data["message"].startswith("No common ancestor"):
# continue
logging.error(f"{repo.full_name} {fork.full_name} {excp}")
continue
if comp.ahead_by:
ahead.append((fork.full_name, comp.ahead_by))
print(f"{fork.full_name} ahead by {comp.ahead_by}", end="")
if verbose and comp.behind_by:
print(f"behind by {comp.behind_by}", end="")
print()
return ahead