def process_milestone(
repo: Repository,
milestone: Milestone,
in_progress_label: Optional[Label],
blocked_label: Optional[Label],
) -> str:
now = datetime.now()
html_url = milestone._rawData["html_url"]
total_issues = milestone.open_issues + milestone.closed_issues
percentage_complete = as_percentage(milestone.closed_issues, total_issues)
detail_lines = []
status_line = f":heavy_check_mark: {percentage_complete}% completed"
if in_progress_label:
in_progress_issues = list(
repo.get_issues(milestone=milestone,
labels=[in_progress_label],
state="open"))
if in_progress_issues:
percentage_in_progress = as_percentage(len(in_progress_issues),
total_issues)
status_line += (
f" - :hourglass_flowing_sand: {percentage_in_progress}% in progress"
)
if blocked_label:
blocked_issues = list(
repo.get_issues(milestone=milestone,
labels=[blocked_label],
state="open"))
if blocked_issues:
percentage_blocked = as_percentage(len(blocked_issues),
total_issues)
status_line += f" - :octagonal_sign: {percentage_blocked}% blocked"
detail_lines.append(status_line)
if milestone.due_on:
duration = milestone.due_on - milestone.created_at
remaining = milestone.due_on - now
time_used = 100 - as_percentage(remaining.days, duration.days)
detail_lines.append(
f":date: {milestone.due_on.date().isoformat()} - :alarm_clock: {time_used}% time used"
)
rendered_line = (
f"<{html_url}|{milestone.title}> - {milestone.closed_issues}/{total_issues}"
)
for line in detail_lines:
rendered_line += f"\n\t{line}"
return rendered_line
def assign_issues(repo: github.Repository):
open_unassigned_issues = [
i for i in repo.get_issues()
if i.state == 'open' and len(i.assignees) == 0
]
for issue in open_unassigned_issues:
try_assign(issue)
def open_issue(pr, pr_checks, hub_repo: Repository, module, jina_core_version):
"""opens an issue for the PR with the failed checks (if not already open)"""
issue_name = f'{FIX_MODULE_TEMPLATE}{module}'
existing_issue_for_pr = [
i for i in list(hub_repo.get_issues(state='open'))
if i.title == issue_name
]
if len(existing_issue_for_pr) > 0:
print(f'Found existing issue: {existing_issue_for_pr}')
return existing_issue_for_pr[0]
else:
# open the issue
body = f"""
**[This is an automated issue opened as part of the hub modules update GH action. DO NOT EDIT THIS DESCRIPTION]**
Could not build module {module} for Jina core version {jina_core_version} because some of the checks failed:
```
{[(c['name'], c['status'], c['conclusion']) for c in pr_checks]}
```
See {pr.html_url} for more info. {TAG_IN_ISSUES}
"""
issue = hub_repo.create_issue(
title=issue_name,
body=body,
)
print(f'opened issue at {issue.html_url}')
return issue
def get_issues(r: github.Repository, data_dir: str):
issues = dict()
all_issues = r.get_issues(state="all")
total = all_issues.totalCount
cur = 0
for i in all_issues:
cur += 1
if cur == 1 or cur % 20 == 0:
print("Processing issue {} of {}".format(cur, total),
file=sys.stderr)
wait(verbose=True)
else:
wait()
login = i.user.login
if login not in issues.keys():
issues[login] = {
"num_issues": 0,
"open": 0,
"closed": 0,
"num_comments": 0,
"num_labels": 0,
"wordcount": 0,
"title_wordcount": 0,
"body_vocab": set(),
"body_vocab_filescope": 0,
"title_vocab": set(),
"title_vocab_filescope": 0,
"total_vocab_filescope": 0,
}
data = issues[login]
data["num_issues"] += 1
if i.state == "open":
data["open"] += 1
elif i.state == "closed":
data["closed"] += 1
data["num_comments"] += i.get_comments().totalCount
data["num_labels"] += len(i.labels)
data["wordcount"] += len(i.body.split())
data["title_wordcount"] += len(i.title.split())
data["body_vocab"].update(set(i.body.split()))
data["body_vocab_filescope"] += len(set(i.body.split()))
data["title_vocab"].update(set(i.title.split()))
data["title_vocab_filescope"] += len(set(i.title.split()))
data["total_vocab_filescope"] += len(
set(i.body.split()).union(set(i.title.split())))
# Convert vocab sets to numbers
for login in issues.keys():
data = issues[login]
data["total_vocab"] = len(data["body_vocab"].union(
data["title_vocab"]))
data["body_vocab"] = len(data["body_vocab"])
data["title_vocab"] = len(data["title_vocab"])
filename = os.path.join(data_dir, "issues.json")
with open(filename + ".part", mode="w") as f:
json.dump(issues, f)
os.rename(filename + ".part", filename)
def update_api(repo: Repository):
(update_file,
date) = Export.get_update_file_date(Export.API_UPDATE_FILE)
json_list = []
for issue in repo.get_issues(
since=date,
sort="updated",
state="all",
):
logger.info("{}: {}", issue.number, issue.title)
json_list.append(issue.raw_data)
data = Data()
logger.info("Updating data")
data.update_api_data(json_list)
logger.info("Saving data")
data.dump_api()
Data.api_to_tarball()
Export.write_update_file_date(update_file)
def dump_api(repo: Repository):
result = []
# We use the first (latest) issue as indicator of how many data we have
# to fetch
issues = repo.get_issues(state="all")
latest_issue = issues[0]
count = latest_issue.number
logger.info("Pulling {} items", count)
for i in range(1, count + 1):
try:
issue = repo.get_issue(i)
logger.info("{}: {}", issue.number, issue.title)
result.append(issue.raw_data)
except Exception as err:
logger.info("Unable to get data, waiting a minute: {}", err)
time.sleep(60)
with open(Data.API_DATA_JSON, "w") as data_file:
json.dump(result, data_file)
logger.info("Done exporting {} items", i)
Data.api_to_tarball()
def clone(self, base_repository: Repository, new_repository: Repository):
# copy issues
for issue in base_repository.get_issues(direction="asc"):
new_repository.create_issue(issue.title, issue.body)
def get_issues_for_modules(hub_repo: Repository):
issues: List[Issue] = list(hub_repo.get_issues(state='open'))
issues = [i for i in issues if FIX_MODULE_TEMPLATE in i.title]
names = [i.title.split(FIX_MODULE_TEMPLATE)[-1] for i in issues]
print(f'found {len(names)} modules to be fixed: {names}')
return names
def _find_issue(self, repo: Repository, label: Label, title: str) -> Issue:
issues = repo.get_issues(labels=[label])
for issue in issues:
if issue.title == title and issue.state == "open":
return issue
return None
def assign_issues(repo: github.Repository):
open_issues = [i for i in repo.get_issues() if i.state == 'open']
for issue in open_issues:
try_assign(issue)