def repo_is_on_pypi(repo):
"""returns True when the provided repository is in pypi"""
is_on = False
the_page = pypi.get("/pypi/" + repo["name"] + "/json")
if the_page and the_page.status_code == 200:
is_on = True
return is_on
def run_library_checks():
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = list_repos()
output_handler("Found {} repos to check.".format(len(repos)))
global bundle_submodules
bundle_submodules = get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = github.get("/user").json()
output_handler("Running GitHub checks as " + github_user["login"])
travis_user = travis.get("/user").json()
output_handler("Running Travis checks as " + travis_user["login"])
need_work = 0
lib_insights = {
"merged_prs": 0,
"closed_prs": 0,
"new_prs": 0,
"active_prs": 0,
"open_prs": [],
"pr_authors": set(),
"pr_merged_authors": set(),
"pr_reviewers": set(),
"closed_issues": 0,
"new_issues": 0,
"active_issues": 0,
"open_issues": [],
"issue_authors": set(),
"issue_closers": set(),
}
core_insights = copy.deepcopy(lib_insights)
for k in core_insights:
if isinstance(core_insights[k], set):
core_insights[k] = set()
if isinstance(core_insights[k], list):
core_insights[k] = []
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
for repo in repos:
errors = validate_repo(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
insights = lib_insights
if repo["name"] == "circuitpython" and repo["owner"][
"login"] == "adafruit":
insights = core_insights
gather_insights(repo, insights, since)
output_handler()
output_handler("State of CircuitPython + Libraries")
output_handler("Overall")
print_pr_overview(lib_insights, core_insights)
print_issue_overview(lib_insights, core_insights)
output_handler()
output_handler("Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
for pr in core_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler()
print_circuitpython_download_stats()
output_handler()
output_handler("Libraries")
print_pr_overview(lib_insights)
output_handler("* {} open pull requests".format(
len(lib_insights["open_prs"])))
for pr in lib_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
for issue in lib_insights["open_issues"]:
output_handler(" * {}".format(issue))
lib_repos = []
for repo in repos:
if repo["owner"]["login"] == "adafruit" and repo["name"].startswith(
"Adafruit_CircuitPython"):
lib_repos.append(repo)
# print("- [ ] [{0}](https://github.com/{1})".format(repo["name"], repo["full_name"]))
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [ERROR_NOT_IN_BUNDLE]
output_handler()
for error in repos_by_error:
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = list_repos()
output_handler("Found {} repos to check.".format(len(repos)))
bundle_submodules = get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = github.get("/user").json()
output_handler("Running GitHub checks as " + github_user["login"])
travis_user = travis.get("/user").json()
output_handler("Running Travis checks as " + travis_user["login"])
need_work = 0
lib_insights = {
"merged_prs": 0,
"closed_prs": 0,
"new_prs": 0,
"active_prs": 0,
"open_prs": [],
"pr_authors": set(),
"pr_merged_authors": set(),
"pr_reviewers": set(),
"closed_issues": 0,
"new_issues": 0,
"active_issues": 0,
"open_issues": [],
"issue_authors": set(),
"issue_closers": set(),
}
core_insights = copy.deepcopy(lib_insights)
for k in core_insights:
if isinstance(core_insights[k], set):
core_insights[k] = set()
if isinstance(core_insights[k], list):
core_insights[k] = []
core_insights["milestones"] = dict()
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
validator = circuitpython_library_validators.library_validator(
validators, bundle_submodules, latest_pylint, **kw_args)
for repo in repos:
if len(validators) != 0:
errors = validator.run_repo_validation(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if not isinstance(error, tuple):
# check for an error occurring in the valiator module
if error == circuitpython_library_validators.ERROR_OUTPUT_HANDLER:
#print(errors, "repo output handler error:", validator.output_file_data)
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
else:
if error[0] not in repos_by_error:
repos_by_error[error[0]] = []
repos_by_error[error[0]].append("{0} ({1} days)".format(
repo["html_url"], error[1]))
insights = lib_insights
if repo["name"] == "circuitpython" and repo["owner"][
"login"] == "adafruit":
insights = core_insights
errors = validator.gather_insights(repo, insights, since)
if errors:
print("insights error")
for error in errors:
if error == circuitpython_library_validators.ERROR_OUTPUT_HANDLER:
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
output_handler()
output_handler("State of CircuitPython + Libraries")
output_handler("Overall")
print_pr_overview(lib_insights, core_insights)
print_issue_overview(lib_insights, core_insights)
output_handler()
output_handler("Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
for pr in core_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(
core_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler("* {} active milestones".format(
len(core_insights["milestones"])))
ms_count = 0
for milestone in sorted(core_insights["milestones"].keys()):
ms_count += core_insights["milestones"][milestone]
output_handler(" * {0}: {1} open issues".format(
milestone, core_insights["milestones"][milestone]))
output_handler(" * {} issues not assigned a milestone".format(
len(core_insights["open_issues"]) - ms_count))
output_handler()
print_circuitpython_download_stats()
output_handler()
output_handler("Libraries")
print_pr_overview(lib_insights)
output_handler("* {} open pull requests".format(
len(lib_insights["open_prs"])))
for pr in lib_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
output_handler(" * https://circuitpython.org/libraries/contributing")
if len(validators) != 0:
lib_repos = []
for repo in repos:
if repo["owner"]["login"] == "adafruit" and repo[
"name"].startswith("Adafruit_CircuitPython"):
lib_repos.append(repo)
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [
circuitpython_library_validators.ERROR_NOT_IN_BUNDLE
]
output_handler()
for error in sorted(repos_by_error):
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
updated_libs = {}
open_issues_by_repo = {}
open_prs_by_repo = {}
contributors = []
reviewers = []
merged_pr_count_total = 0
repos_by_error = {}
default_validators = [
vals[1] for vals in inspect.getmembers(cpy_vals.library_validator)
if vals[0].startswith("validate")
]
bundle_submodules = common_funcs.get_bundle_submodules()
latest_pylint = ""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
validator = cpy_vals.library_validator(default_validators,
bundle_submodules, latest_pylint)
for repo in repos:
if (repo["name"] in cpy_vals.BUNDLE_IGNORE_LIST
or repo["name"] == "circuitpython"):
continue
repo_name = repo["name"]
# get a list of new & updated libraries for the last week
check_releases = common_funcs.is_new_or_updated(repo)
if check_releases == "new":
def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = common_funcs.list_repos(include_repos=tuple(blinka_repos) +
("CircuitPython_Community_Bundle",
"cookiecutter-adafruit-circuitpython"))
output_handler("Found {} repos to check.".format(len(repos)))
bundle_submodules = common_funcs.get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = common_funcs.whois_github_user()
output_handler("Running GitHub checks as " + github_user)
need_work = 0
lib_insights = common_funcs.InsightData()
blinka_insights = common_funcs.InsightData()
core_insights = common_funcs.InsightData()
core_insights["milestones"] = dict()
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
new_libs = {}
updated_libs = {}
validator = cirpy_lib_vals.library_validator(validators, bundle_submodules,
latest_pylint, **kw_args)
for repo in repos:
if len(validators) != 0:
errors = validator.run_repo_validation(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if not isinstance(error, tuple):
# check for an error occurring in the valiator module
if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER:
#print(errors, "repo output handler error:", validator.output_file_data)
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
else:
if error[0] not in repos_by_error:
repos_by_error[error[0]] = []
repos_by_error[error[0]].append("{0} ({1} days)".format(
repo["html_url"], error[1]))
insights = lib_insights
if repo["owner"]["login"] == "adafruit":
if repo["name"] in blinka_repos:
insights = blinka_insights
elif repo["name"] == "circuitpython":
insights = core_insights
closed_metric = bool(insights == lib_insights)
errors = validator.gather_insights(repo,
insights,
since,
show_closed_metric=closed_metric)
if errors:
print("insights error")
for error in errors:
if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER:
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
# get a list of new & updated libraries for the last week
if repo["name"] != "Adafruit_CircuitPython_Bundle":
check_releases = common_funcs.is_new_or_updated(repo)
if check_releases == "new":
new_libs[repo["name"]] = repo["html_url"]
elif check_releases == "updated":
updated_libs[repo["name"]] = repo["html_url"]
output_handler()
output_handler("State of CircuitPython + Libraries + Blinka")
output_handler("### Overall")
print_pr_overview(lib_insights, core_insights, blinka_insights)
print_issue_overview(lib_insights, core_insights, blinka_insights)
output_handler()
output_handler("### Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
sorted_prs = sorted(core_insights["open_prs"],
key=lambda days: int(pr_sort_re.search(days).group(1)),
reverse=True)
for pr in sorted_prs:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(
core_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler("* {} active milestones".format(
len(core_insights["milestones"])))
ms_count = 0
for milestone in sorted(core_insights["milestones"].keys()):
ms_count += core_insights["milestones"][milestone]
output_handler(" * {0}: {1} open issues".format(
milestone, core_insights["milestones"][milestone]))
output_handler(" * {} issues not assigned a milestone".format(
len(core_insights["open_issues"]) - ms_count))
output_handler()
## temporarily disabling core download stats:
# - GitHub API has been broken, due to the number of release artifacts
# - Release asset delivery is being moved to AWS CloudFront/S3
#print_circuitpython_download_stats()
output_handler(
"* Core download stats available at https://circuitpython.org/stats")
output_handler()
output_handler("### Libraries")
print_pr_overview(lib_insights)
output_handler(" * Merged pull requests:")
sorted_prs = sorted(
lib_insights["merged_prs"],
key=lambda days: int(close_pr_sort_re.search(days).group(1)),
reverse=True)
for pr in sorted_prs:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* https://circuitpython.org/contributing")
output_handler(" * {} open issues".format(len(
lib_insights["open_issues"])))
output_handler(" * {} good first issues".format(
lib_insights["good_first_issues"]))
open_pr_days = [
int(pr_sort_re.search(pr).group(1)) for pr in lib_insights["open_prs"]
if pr_sort_re.search(pr) is not None
]
if len(lib_insights["open_prs"]) != 0:
output_handler(
" * {0} open pull requests (Oldest: {1}, Newest: {2})".format(
len(lib_insights["open_prs"]),
max(open_pr_days),
max((min(open_pr_days), 1)) # ensure the minumum is '1'
))
output_handler("Library updates in the last seven days:")
if len(new_libs) != 0:
output_handler("**New Libraries**")
for new in new_libs:
output_handler(" * [{}]({})".format(new, new_libs[new]))
if len(updated_libs) != 0:
output_handler("**Updated Libraries**")
for updated in updated_libs:
output_handler(" * [{}]({})".format(updated,
updated_libs[updated]))
if len(validators) != 0:
lib_repos = []
for repo in repos:
if (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
lib_repos.append(repo)
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [cirpy_lib_vals.ERROR_NOT_IN_BUNDLE]
output_handler()
for error in sorted(repos_by_error):
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
output_handler()
output_handler("### Blinka")
print_pr_overview(blinka_insights)
output_handler("* {} open pull requests".format(
len(blinka_insights["open_prs"])))
sorted_prs = sorted(blinka_insights["open_prs"],
key=lambda days: int(pr_sort_re.search(days).group(1)),
reverse=True)
for pr in sorted_prs:
output_handler(" * {}".format(pr))
print_issue_overview(blinka_insights)
output_handler("* {} open issues".format(
len(blinka_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/Adafruit_Blinka/issues")
blinka_dl, _ = dl_stats.pypistats_get('adafruit-blinka')
output_handler("* PyPI Downloads in the last week: {}".format(blinka_dl))
output_handler("Number of supported boards: {}".format(
blinka_funcs.board_count()))
