def run_library_checks():
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = list_repos()
output_handler("Found {} repos to check.".format(len(repos)))
global bundle_submodules
bundle_submodules = get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = github.get("/user").json()
output_handler("Running GitHub checks as " + github_user["login"])
travis_user = travis.get("/user").json()
output_handler("Running Travis checks as " + travis_user["login"])
need_work = 0
lib_insights = {
"merged_prs": 0,
"closed_prs": 0,
"new_prs": 0,
"active_prs": 0,
"open_prs": [],
"pr_authors": set(),
"pr_merged_authors": set(),
"pr_reviewers": set(),
"closed_issues": 0,
"new_issues": 0,
"active_issues": 0,
"open_issues": [],
"issue_authors": set(),
"issue_closers": set(),
}
core_insights = copy.deepcopy(lib_insights)
for k in core_insights:
if isinstance(core_insights[k], set):
core_insights[k] = set()
if isinstance(core_insights[k], list):
core_insights[k] = []
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
for repo in repos:
errors = validate_repo(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
insights = lib_insights
if repo["name"] == "circuitpython" and repo["owner"][
"login"] == "adafruit":
insights = core_insights
gather_insights(repo, insights, since)
output_handler()
output_handler("State of CircuitPython + Libraries")
output_handler("Overall")
print_pr_overview(lib_insights, core_insights)
print_issue_overview(lib_insights, core_insights)
output_handler()
output_handler("Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
for pr in core_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler()
print_circuitpython_download_stats()
output_handler()
output_handler("Libraries")
print_pr_overview(lib_insights)
output_handler("* {} open pull requests".format(
len(lib_insights["open_prs"])))
for pr in lib_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
for issue in lib_insights["open_issues"]:
output_handler(" * {}".format(issue))
lib_repos = []
for repo in repos:
if repo["owner"]["login"] == "adafruit" and repo["name"].startswith(
"Adafruit_CircuitPython"):
lib_repos.append(repo)
# print("- [ ] [{0}](https://github.com/{1})".format(repo["name"], repo["full_name"]))
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [ERROR_NOT_IN_BUNDLE]
output_handler()
for error in repos_by_error:
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
"""runs the various library checking functions"""
pylint_info = pypi.get("/pypi/pylint/json")
if pylint_info and pylint_info.ok:
latest_pylint = pylint_info.json()["info"]["version"]
output_handler("Latest pylint is: {}".format(latest_pylint))
repos = list_repos()
output_handler("Found {} repos to check.".format(len(repos)))
bundle_submodules = get_bundle_submodules()
output_handler("Found {} submodules in the bundle.".format(
len(bundle_submodules)))
github_user = github.get("/user").json()
output_handler("Running GitHub checks as " + github_user["login"])
travis_user = travis.get("/user").json()
output_handler("Running Travis checks as " + travis_user["login"])
need_work = 0
lib_insights = {
"merged_prs": 0,
"closed_prs": 0,
"new_prs": 0,
"active_prs": 0,
"open_prs": [],
"pr_authors": set(),
"pr_merged_authors": set(),
"pr_reviewers": set(),
"closed_issues": 0,
"new_issues": 0,
"active_issues": 0,
"open_issues": [],
"issue_authors": set(),
"issue_closers": set(),
}
core_insights = copy.deepcopy(lib_insights)
for k in core_insights:
if isinstance(core_insights[k], set):
core_insights[k] = set()
if isinstance(core_insights[k], list):
core_insights[k] = []
core_insights["milestones"] = dict()
repo_needs_work = []
since = datetime.datetime.now() - datetime.timedelta(days=7)
repos_by_error = {}
validator = circuitpython_library_validators.library_validator(
validators, bundle_submodules, latest_pylint, **kw_args)
for repo in repos:
if len(validators) != 0:
errors = validator.run_repo_validation(repo)
if errors:
need_work += 1
repo_needs_work.append(repo)
# print(repo["full_name"])
# print("\n".join(errors))
# print()
for error in errors:
if not isinstance(error, tuple):
# check for an error occurring in the valiator module
if error == circuitpython_library_validators.ERROR_OUTPUT_HANDLER:
#print(errors, "repo output handler error:", validator.output_file_data)
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
if error not in repos_by_error:
repos_by_error[error] = []
repos_by_error[error].append(repo["html_url"])
else:
if error[0] not in repos_by_error:
repos_by_error[error[0]] = []
repos_by_error[error[0]].append("{0} ({1} days)".format(
repo["html_url"], error[1]))
insights = lib_insights
if repo["name"] == "circuitpython" and repo["owner"][
"login"] == "adafruit":
insights = core_insights
errors = validator.gather_insights(repo, insights, since)
if errors:
print("insights error")
for error in errors:
if error == circuitpython_library_validators.ERROR_OUTPUT_HANDLER:
output_handler(", ".join(validator.output_file_data))
validator.output_file_data.clear()
output_handler()
output_handler("State of CircuitPython + Libraries")
output_handler("Overall")
print_pr_overview(lib_insights, core_insights)
print_issue_overview(lib_insights, core_insights)
output_handler()
output_handler("Core")
print_pr_overview(core_insights)
output_handler("* {} open pull requests".format(
len(core_insights["open_prs"])))
for pr in core_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(core_insights)
output_handler("* {} open issues".format(len(
core_insights["open_issues"])))
output_handler(" * https://github.com/adafruit/circuitpython/issues")
output_handler("* {} active milestones".format(
len(core_insights["milestones"])))
ms_count = 0
for milestone in sorted(core_insights["milestones"].keys()):
ms_count += core_insights["milestones"][milestone]
output_handler(" * {0}: {1} open issues".format(
milestone, core_insights["milestones"][milestone]))
output_handler(" * {} issues not assigned a milestone".format(
len(core_insights["open_issues"]) - ms_count))
output_handler()
print_circuitpython_download_stats()
output_handler()
output_handler("Libraries")
print_pr_overview(lib_insights)
output_handler("* {} open pull requests".format(
len(lib_insights["open_prs"])))
for pr in lib_insights["open_prs"]:
output_handler(" * {}".format(pr))
print_issue_overview(lib_insights)
output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
output_handler(" * https://circuitpython.org/libraries/contributing")
if len(validators) != 0:
lib_repos = []
for repo in repos:
if repo["owner"]["login"] == "adafruit" and repo[
"name"].startswith("Adafruit_CircuitPython"):
lib_repos.append(repo)
output_handler("{} out of {} repos need work.".format(
need_work, len(lib_repos)))
list_repos_for_errors = [
circuitpython_library_validators.ERROR_NOT_IN_BUNDLE
]
output_handler()
for error in sorted(repos_by_error):
if not repos_by_error[error]:
continue
output_handler()
error_count = len(repos_by_error[error])
output_handler("{} - {}".format(error, error_count))
if error_count <= error_depth or error in list_repos_for_errors:
output_handler("\n".join(
[" * " + x for x in repos_by_error[error]]))
def validate_travis(repo):
"""Validate and configure a repository has the expected state in Travis
CI. This will both check Travis state and attempt to enable Travis CI
and setup the expected state in Travis if not enabled. Expects a
dictionary with a GitHub API repository state (like from the list_repos
function). Returns a list of string error messages for the repository.
"""
if not (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
return []
repo_url = "/repo/" + repo["owner"]["login"] + "%2F" + repo["name"]
result = travis.get(repo_url)
if not result.ok:
#print(result, result.request.url, result.request.headers)
#print(result.text)
return [ERROR_TRAVIS_DOESNT_KNOW_REPO]
result = result.json()
if not result["active"]:
activate = travis.post(repo_url + "/activate")
if not activate.ok:
#output_handler(activate.request.url)
#output_handler("{} {}".format(activate, activate.text))
return [ERROR_ENABLE_TRAVIS]
env_variables = travis.get(repo_url + "/env_vars")
if not env_variables.ok:
#print(env_variables, env_variables.text)
#print(env_variables.request.headers)
return [ERROR_TRAVIS_ENV]
env_variables = env_variables.json()
found_token = False
for var in env_variables["env_vars"]:
found_token = found_token or var["name"] == "GITHUB_TOKEN"
ok = True
if not found_token:
if not github_token:
return [ERROR_TRAVIS_GITHUB_TOKEN]
else:
global full_auth
if not full_auth:
#github_user = github_token
github_user = github.get("/user").json()
password = input("Password for " + github_user["login"] + ": ")
full_auth = (github_user["login"], password.strip())
if not full_auth:
return [ERROR_TRAVIS_GITHUB_TOKEN]
new_access_token = {
"scopes": ["public_repo"],
"note": "TravisCI release token for " + repo["full_name"],
"note_url": "https://travis-ci.org/" + repo["full_name"]
}
token = github.post("/authorizations",
json=new_access_token,
auth=full_auth)
if not token.ok:
print(token.text)
return [ERROR_TRAVIS_TOKEN_CREATE]
token = token.json()
grant_id = token["id"]
token = token["token"]
new_var = {
"env_var.name": "GITHUB_TOKEN",
"env_var.value": token,
"env_var.public": False
}
new_var_result = travis.post(repo_url + "/env_vars", json=new_var)
if not new_var_result.ok:
#print(new_var_result.headers, new_var_result.text)
github.delete("/authorizations/{}".format(grant_id),
auth=full_auth)
return [ERROR_TRAVIS_GITHUB_TOKEN]
return []
# of Github API repository object state).
validators = [
validate_repo_state, validate_travis, validate_contents,
validate_readthedocs, validate_core_driver_page
]
# Submodules inside the bundle (result of get_bundle_submodules)
bundle_submodules = []
if __name__ == "__main__":
repos = list_repos()
print("Found {} repos to check.".format(len(repos)))
bundle_submodules = get_bundle_submodules()
print("Found {} submodules in the bundle.".format(len(bundle_submodules)))
github_user = github.get("/user").json()
print("Running GitHub checks as " + github_user["login"])
travis_user = travis.get("/user").json()
print("Running Travis checks as " + travis_user["login"])
need_work = 0
insights = {
"merged_prs": 0,
"closed_prs": 0,
"new_prs": 0,
"active_prs": 0,
"open_prs": [],
"pr_authors": set(),
"pr_merged_authors": set(),
"pr_reviewers": set(),
"closed_issues": 0,
"new_issues": 0,
"active_issues": 0,
"open_issues": [],