def get_pypi_stats():
successful_stats = {}
failed_stats = []
repos = common_funcs.list_repos()
dl_stats = piwheels_stats()
for repo in repos:
if (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
if common_funcs.repo_is_on_pypi(repo):
pkg_name = repo["name"].replace("_", "-").lower()
if pkg_name in dl_stats:
successful_stats[repo["name"]] = (
dl_stats[pkg_name]["month"],
dl_stats[pkg_name]["total"])
else:
failed_stats.append(repo["name"])
for lib in PYPI_FORCE_NON_CIRCUITPYTHON:
pkg_name = lib.lower()
if pkg_name in dl_stats:
successful_stats[lib] = (dl_stats[pkg_name]["month"],
dl_stats[pkg_name]["total"])
else:
failed_stats.append(repo["name"])
return successful_stats, failed_stats
def get_repo_list():
""" Uses adabot.circuitpython_libraries module to get a list of
CircuitPython repositories. Filters the list down to adafruit
owned/sponsored CircuitPython libraries.
"""
repo_list = []
get_repos = common_funcs.list_repos()
for repo in get_repos:
if not (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
continue
repo_list.append(dict(name=repo["name"], url=repo["clone_url"]))
return repo_list
def get_pypi_stats():
successful_stats = {}
failed_stats = []
repos = common_funcs.list_repos()
for repo in repos:
if (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
if common_funcs.repo_is_on_pypi(repo):
pypi_dl_last_week, pypi_dl_total = pypistats_get(
repo["name"].replace("_", "-").lower())
if pypi_dl_last_week is None:
failed_stats.append(repo["name"])
continue
successful_stats[repo["name"]] = (pypi_dl_last_week,
pypi_dl_total)
for lib in PYPI_FORCE_NON_CIRCUITPYTHON:
pypi_dl_last_week, pypi_dl_total = pypistats_get(lib.lower())
if pypi_dl_last_week is None:
failed_stats.append(lib)
continue
successful_stats[lib] = (pypi_dl_last_week, pypi_dl_total)
return successful_stats, failed_stats
def test_list_repos():
repos = common_funcs.list_repos()
assert isinstance(repos, list)
startup_message = [
"Run Date: {}".format(run_time.strftime("%d %B %Y, %I:%M%p"))
]
output_filename = os.path.join(cp_org_dir, "_data/libraries.json")
local_file_output = False
if cmd_line_args.output_file:
output_filename = os.path.abspath(cmd_line_args.output_file)
local_file_output = True
startup_message.append(
" - Output will be saved to: {}".format(output_filename))
print("\n".join(startup_message))
repos = common_funcs.list_repos()
new_libs = {}
updated_libs = {}
open_issues_by_repo = {}
open_prs_by_repo = {}
contributors = []
reviewers = []
merged_pr_count_total = 0
repos_by_error = {}
default_validators = [
vals[1] for vals in inspect.getmembers(cpy_vals.library_validator)
if vals[0].startswith("validate")
]
bundle_submodules = common_funcs.get_bundle_submodules()
startup_message = [
"Run Date: {}".format(run_time.strftime("%d %B %Y, %I:%M%p"))
]
output_filename = ""
local_file_output = False
if cmd_line_args.output_file:
output_filename = os.path.abspath(cmd_line_args.output_file)
local_file_output = True
startup_message.append(
" - Output will be saved to: {}".format(output_filename))
print("\n".join(startup_message))
repos = common_funcs.list_repos(include_repos=(
"CircuitPython_Community_Bundle",
'cookiecutter-adafruit-circuitpython',
))
new_libs = {}
updated_libs = {}
open_issues_by_repo = {}
open_prs_by_repo = {}
contributors = []
reviewers = []
merged_pr_count_total = 0
repos_by_error = {}
default_validators = [
vals[1] for vals in inspect.getmembers(cpy_vals.library_validator)
if vals[0].startswith("validate")
]
def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = common_funcs.list_repos(include_repos=tuple(blinka_repos) +
("CircuitPython_Community_Bundle",
"cookiecutter-adafruit-circuitpython"))
output_handler("Found {} repos to check.".format(len(repos)))
bundle_submodules = common_funcs.get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = common_funcs.whois_github_user()
output_handler("Running GitHub checks as " + github_user)
need_work = 0
lib_insights = common_funcs.InsightData()
blinka_insights = common_funcs.InsightData()
core_insights = common_funcs.InsightData()
core_insights["milestones"] = dict()
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
new_libs = {}
updated_libs = {}
validator = cirpy_lib_vals.library_validator(validators, bundle_submodules,
latest_pylint, **kw_args)
for repo in repos:
if len(validators) != 0:
errors = validator.run_repo_validation(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if not isinstance(error, tuple):
# check for an error occurring in the valiator module
if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER:
#print(errors, "repo output handler error:", validator.output_file_data)
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
else:
if error[0] not in repos_by_error:
repos_by_error[error[0]] = []
repos_by_error[error[0]].append("{0} ({1} days)".format(
repo["html_url"], error[1]))
insights = lib_insights
if repo["owner"]["login"] == "adafruit":
if repo["name"] in blinka_repos:
insights = blinka_insights
elif repo["name"] == "circuitpython":
insights = core_insights
closed_metric = bool(insights == lib_insights)
errors = validator.gather_insights(repo,
insights,
since,
show_closed_metric=closed_metric)
if errors:
print("insights error")
for error in errors:
if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER:
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
# get a list of new & updated libraries for the last week
if repo["name"] != "Adafruit_CircuitPython_Bundle":
check_releases = common_funcs.is_new_or_updated(repo)
if check_releases == "new":
new_libs[repo["name"]] = repo["html_url"]
elif check_releases == "updated":
updated_libs[repo["name"]] = repo["html_url"]
output_handler()
output_handler("State of CircuitPython + Libraries + Blinka")
output_handler("### Overall")
print_pr_overview(lib_insights, core_insights, blinka_insights)
print_issue_overview(lib_insights, core_insights, blinka_insights)
output_handler()
output_handler("### Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
sorted_prs = sorted(core_insights["open_prs"],
key=lambda days: int(pr_sort_re.search(days).group(1)),
reverse=True)
for pr in sorted_prs:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(
core_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler("* {} active milestones".format(
len(core_insights["milestones"])))
ms_count = 0
for milestone in sorted(core_insights["milestones"].keys()):
ms_count += core_insights["milestones"][milestone]
output_handler(" * {0}: {1} open issues".format(
milestone, core_insights["milestones"][milestone]))
output_handler(" * {} issues not assigned a milestone".format(
len(core_insights["open_issues"]) - ms_count))
output_handler()
## temporarily disabling core download stats:
# - GitHub API has been broken, due to the number of release artifacts
# - Release asset delivery is being moved to AWS CloudFront/S3
#print_circuitpython_download_stats()
output_handler(
"* Core download stats available at https://circuitpython.org/stats")
output_handler()
output_handler("### Libraries")
print_pr_overview(lib_insights)
output_handler(" * Merged pull requests:")
sorted_prs = sorted(
lib_insights["merged_prs"],
key=lambda days: int(close_pr_sort_re.search(days).group(1)),
reverse=True)
for pr in sorted_prs:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* https://circuitpython.org/contributing")
output_handler(" * {} open issues".format(len(
lib_insights["open_issues"])))
output_handler(" * {} good first issues".format(
lib_insights["good_first_issues"]))
open_pr_days = [
int(pr_sort_re.search(pr).group(1)) for pr in lib_insights["open_prs"]
if pr_sort_re.search(pr) is not None
]
if len(lib_insights["open_prs"]) != 0:
output_handler(
" * {0} open pull requests (Oldest: {1}, Newest: {2})".format(
len(lib_insights["open_prs"]),
max(open_pr_days),
max((min(open_pr_days), 1)) # ensure the minumum is '1'
))
output_handler("Library updates in the last seven days:")
if len(new_libs) != 0:
output_handler("**New Libraries**")
for new in new_libs:
output_handler(" * [{}]({})".format(new, new_libs[new]))
if len(updated_libs) != 0:
output_handler("**Updated Libraries**")
for updated in updated_libs:
output_handler(" * [{}]({})".format(updated,
updated_libs[updated]))
if len(validators) != 0:
lib_repos = []
for repo in repos:
if (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
lib_repos.append(repo)
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [cirpy_lib_vals.ERROR_NOT_IN_BUNDLE]
output_handler()
for error in sorted(repos_by_error):
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
output_handler()
output_handler("### Blinka")
print_pr_overview(blinka_insights)
output_handler("* {} open pull requests".format(
len(blinka_insights["open_prs"])))
sorted_prs = sorted(blinka_insights["open_prs"],
key=lambda days: int(pr_sort_re.search(days).group(1)),
reverse=True)
for pr in sorted_prs:
output_handler(" * {}".format(pr))
print_issue_overview(blinka_insights)
output_handler("* {} open issues".format(
len(blinka_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/Adafruit_Blinka/issues")
blinka_dl, _ = dl_stats.pypistats_get('adafruit-blinka')
output_handler("* PyPI Downloads in the last week: {}".format(blinka_dl))
output_handler("Number of supported boards: {}".format(
blinka_funcs.board_count()))
def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = common_funcs.list_repos()
output_handler("Found {} repos to check.".format(len(repos)))
bundle_submodules = common_funcs.get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = github.get("/user").json()
output_handler("Running GitHub checks as " + github_user["login"])
travis_user = travis.get("/user").json()
output_handler("Running Travis checks as " + travis_user["login"])
need_work = 0
lib_insights = {
"merged_prs": 0,
"closed_prs": 0,
"new_prs": 0,
"active_prs": 0,
"open_prs": [],
"pr_authors": set(),
"pr_merged_authors": set(),
"pr_reviewers": set(),
"closed_issues": 0,
"new_issues": 0,
"active_issues": 0,
"open_issues": [],
"issue_authors": set(),
"issue_closers": set(),
}
core_insights = copy.deepcopy(lib_insights)
for k in core_insights:
if isinstance(core_insights[k], set):
core_insights[k] = set()
if isinstance(core_insights[k], list):
core_insights[k] = []
core_insights["milestones"] = dict()
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
new_libs = {}
updated_libs = {}
validator = cirpy_lib_vals.library_validator(validators, bundle_submodules,
latest_pylint, **kw_args)
for repo in repos:
if len(validators) != 0:
errors = validator.run_repo_validation(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if not isinstance(error, tuple):
# check for an error occurring in the valiator module
if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER:
#print(errors, "repo output handler error:", validator.output_file_data)
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
else:
if error[0] not in repos_by_error:
repos_by_error[error[0]] = []
repos_by_error[error[0]].append("{0} ({1} days)".format(
repo["html_url"], error[1]))
insights = lib_insights
if (repo["name"] == "circuitpython"
and repo["owner"]["login"] == "adafruit"):
insights = core_insights
errors = validator.gather_insights(repo, insights, since)
if errors:
print("insights error")
for error in errors:
if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER:
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
# get a list of new & updated libraries for the last week
if repo["name"] != "Adafruit_CircuitPython_Bundle":
check_releases = common_funcs.is_new_or_updated(repo)
if check_releases == "new":
new_libs[repo["name"]] = repo["html_url"]
elif check_releases == "updated":
updated_libs[repo["name"]] = repo["html_url"]
output_handler()
output_handler("State of CircuitPython + Libraries")
output_handler("Overall")
print_pr_overview(lib_insights, core_insights)
print_issue_overview(lib_insights, core_insights)
output_handler()
output_handler("Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
for pr in core_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(
core_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler("* {} active milestones".format(
len(core_insights["milestones"])))
ms_count = 0
for milestone in sorted(core_insights["milestones"].keys()):
ms_count += core_insights["milestones"][milestone]
output_handler(" * {0}: {1} open issues".format(
milestone, core_insights["milestones"][milestone]))
output_handler(" * {} issues not assigned a milestone".format(
len(core_insights["open_issues"]) - ms_count))
output_handler()
print_circuitpython_download_stats()
output_handler()
output_handler("Libraries")
print_pr_overview(lib_insights)
output_handler("* {} open pull requests".format(
len(lib_insights["open_prs"])))
for pr in lib_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
output_handler(" * https://circuitpython.org/libraries/contributing")
output_handler("Library updates in the last seven days:")
if len(new_libs) != 0:
output_handler("**New Libraries**")
for new in new_libs:
output_handler(" * [{}]({})".format(new, new_libs[new]))
if len(updated_libs) != 0:
output_handler("**Updated Libraries**")
for updated in updated_libs:
output_handler(" * [{}]({})".format(updated,
updated_libs[updated]))
if len(validators) != 0:
lib_repos = []
for repo in repos:
if (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
lib_repos.append(repo)
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [cirpy_lib_vals.ERROR_NOT_IN_BUNDLE]
output_handler()
for error in sorted(repos_by_error):
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
startup_message = [
"Run Date: {}".format(run_time.strftime("%d %B %Y, %I:%M%p"))
]
output_filename = ""
local_file_output = False
if cmd_line_args.output_file:
output_filename = os.path.abspath(cmd_line_args.output_file)
local_file_output = True
startup_message.append(
" - Output will be saved to: {}".format(output_filename))
print("\n".join(startup_message))
repos = common_funcs.list_repos(
include_repos=("CircuitPython_Community_Bundle", ))
new_libs = {}
updated_libs = {}
open_issues_by_repo = {}
open_prs_by_repo = {}
contributors = []
reviewers = []
merged_pr_count_total = 0
repos_by_error = {}
default_validators = [
vals[1] for vals in inspect.getmembers(cpy_vals.library_validator)
if vals[0].startswith("validate")
]
bundle_submodules = common_funcs.get_bundle_submodules()