def run_library_checks(): """runs the various library checking functions""" pylint_info = pypi.get("/pypi/pylint/json") if pylint_info and pylint_info.ok: latest_pylint = pylint_info.json()["info"]["version"] output_handler("Latest pylint is: {}".format(latest_pylint)) repos = list_repos() output_handler("Found {} repos to check.".format(len(repos))) global bundle_submodules bundle_submodules = get_bundle_submodules() output_handler("Found {} submodules in the bundle.".format( len(bundle_submodules))) github_user = github.get("/user").json() output_handler("Running GitHub checks as " + github_user["login"]) travis_user = travis.get("/user").json() output_handler("Running Travis checks as " + travis_user["login"]) need_work = 0 lib_insights = { "merged_prs": 0, "closed_prs": 0, "new_prs": 0, "active_prs": 0, "open_prs": [], "pr_authors": set(), "pr_merged_authors": set(), "pr_reviewers": set(), "closed_issues": 0, "new_issues": 0, "active_issues": 0, "open_issues": [], "issue_authors": set(), "issue_closers": set(), } core_insights = copy.deepcopy(lib_insights) for k in core_insights: if isinstance(core_insights[k], set): core_insights[k] = set() if isinstance(core_insights[k], list): core_insights[k] = [] repo_needs_work = [] since = datetime.datetime.now() - datetime.timedelta(days=7) repos_by_error = {} for repo in repos: errors = validate_repo(repo) if errors: need_work += 1 repo_needs_work.append(repo) # print(repo["full_name"]) # print("\n".join(errors)) # print() for error in errors: if error not in repos_by_error: repos_by_error[error] = [] repos_by_error[error].append(repo["html_url"]) insights = lib_insights if repo["name"] == "circuitpython" and repo["owner"][ "login"] == "adafruit": insights = core_insights gather_insights(repo, insights, since) output_handler() output_handler("State of CircuitPython + Libraries") output_handler("Overall") print_pr_overview(lib_insights, core_insights) print_issue_overview(lib_insights, core_insights) output_handler() output_handler("Core") print_pr_overview(core_insights) output_handler("* {} open pull requests".format( len(core_insights["open_prs"]))) for pr in core_insights["open_prs"]: output_handler(" * {}".format(pr)) print_issue_overview(core_insights) output_handler("* {} open issues".format(len(insights["open_issues"]))) output_handler(" * https://github.com/adafruit/circuitpython/issues") output_handler() print_circuitpython_download_stats() output_handler() output_handler("Libraries") print_pr_overview(lib_insights) output_handler("* {} open pull requests".format( len(lib_insights["open_prs"]))) for pr in lib_insights["open_prs"]: output_handler(" * {}".format(pr)) print_issue_overview(lib_insights) output_handler("* {} open issues".format(len(lib_insights["open_issues"]))) for issue in lib_insights["open_issues"]: output_handler(" * {}".format(issue)) lib_repos = [] for repo in repos: if repo["owner"]["login"] == "adafruit" and repo["name"].startswith( "Adafruit_CircuitPython"): lib_repos.append(repo) # print("- [ ] [{0}](https://github.com/{1})".format(repo["name"], repo["full_name"])) output_handler("{} out of {} repos need work.".format( need_work, len(lib_repos))) list_repos_for_errors = [ERROR_NOT_IN_BUNDLE] output_handler() for error in repos_by_error: if not repos_by_error[error]: continue output_handler() error_count = len(repos_by_error[error]) output_handler("{} - {}".format(error, error_count)) if error_count <= error_depth or error in list_repos_for_errors: output_handler("\n".join( [" * " + x for x in repos_by_error[error]]))
def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args): """runs the various library checking functions""" pylint_info = pypi.get("/pypi/pylint/json") if pylint_info and pylint_info.ok: latest_pylint = pylint_info.json()["info"]["version"] output_handler("Latest pylint is: {}".format(latest_pylint)) repos = list_repos() output_handler("Found {} repos to check.".format(len(repos))) bundle_submodules = get_bundle_submodules() output_handler("Found {} submodules in the bundle.".format( len(bundle_submodules))) github_user = github.get("/user").json() output_handler("Running GitHub checks as " + github_user["login"]) travis_user = travis.get("/user").json() output_handler("Running Travis checks as " + travis_user["login"]) need_work = 0 lib_insights = { "merged_prs": 0, "closed_prs": 0, "new_prs": 0, "active_prs": 0, "open_prs": [], "pr_authors": set(), "pr_merged_authors": set(), "pr_reviewers": set(), "closed_issues": 0, "new_issues": 0, "active_issues": 0, "open_issues": [], "issue_authors": set(), "issue_closers": set(), } core_insights = copy.deepcopy(lib_insights) for k in core_insights: if isinstance(core_insights[k], set): core_insights[k] = set() if isinstance(core_insights[k], list): core_insights[k] = [] core_insights["milestones"] = dict() repo_needs_work = [] since = datetime.datetime.now() - datetime.timedelta(days=7) repos_by_error = {} validator = circuitpython_library_validators.library_validator( validators, bundle_submodules, latest_pylint, **kw_args) for repo in repos: if len(validators) != 0: errors = validator.run_repo_validation(repo) if errors: need_work += 1 repo_needs_work.append(repo) # print(repo["full_name"]) # print("\n".join(errors)) # print() for error in errors: if not isinstance(error, tuple): # check for an error occurring in the valiator module if error == circuitpython_library_validators.ERROR_OUTPUT_HANDLER: #print(errors, "repo output handler error:", validator.output_file_data) output_handler(", ".join(validator.output_file_data)) validator.output_file_data.clear() if error not in repos_by_error: repos_by_error[error] = [] repos_by_error[error].append(repo["html_url"]) else: if error[0] not in repos_by_error: repos_by_error[error[0]] = [] repos_by_error[error[0]].append("{0} ({1} days)".format( repo["html_url"], error[1])) insights = lib_insights if repo["name"] == "circuitpython" and repo["owner"][ "login"] == "adafruit": insights = core_insights errors = validator.gather_insights(repo, insights, since) if errors: print("insights error") for error in errors: if error == circuitpython_library_validators.ERROR_OUTPUT_HANDLER: output_handler(", ".join(validator.output_file_data)) validator.output_file_data.clear() output_handler() output_handler("State of CircuitPython + Libraries") output_handler("Overall") print_pr_overview(lib_insights, core_insights) print_issue_overview(lib_insights, core_insights) output_handler() output_handler("Core") print_pr_overview(core_insights) output_handler("* {} open pull requests".format( len(core_insights["open_prs"]))) for pr in core_insights["open_prs"]: output_handler(" * {}".format(pr)) print_issue_overview(core_insights) output_handler("* {} open issues".format(len( core_insights["open_issues"]))) output_handler(" * https://github.com/adafruit/circuitpython/issues") output_handler("* {} active milestones".format( len(core_insights["milestones"]))) ms_count = 0 for milestone in sorted(core_insights["milestones"].keys()): ms_count += core_insights["milestones"][milestone] output_handler(" * {0}: {1} open issues".format( milestone, core_insights["milestones"][milestone])) output_handler(" * {} issues not assigned a milestone".format( len(core_insights["open_issues"]) - ms_count)) output_handler() print_circuitpython_download_stats() output_handler() output_handler("Libraries") print_pr_overview(lib_insights) output_handler("* {} open pull requests".format( len(lib_insights["open_prs"]))) for pr in lib_insights["open_prs"]: output_handler(" * {}".format(pr)) print_issue_overview(lib_insights) output_handler("* {} open issues".format(len(lib_insights["open_issues"]))) output_handler(" * https://circuitpython.org/libraries/contributing") if len(validators) != 0: lib_repos = [] for repo in repos: if repo["owner"]["login"] == "adafruit" and repo[ "name"].startswith("Adafruit_CircuitPython"): lib_repos.append(repo) output_handler("{} out of {} repos need work.".format( need_work, len(lib_repos))) list_repos_for_errors = [ circuitpython_library_validators.ERROR_NOT_IN_BUNDLE ] output_handler() for error in sorted(repos_by_error): if not repos_by_error[error]: continue output_handler() error_count = len(repos_by_error[error]) output_handler("{} - {}".format(error, error_count)) if error_count <= error_depth or error in list_repos_for_errors: output_handler("\n".join( [" * " + x for x in repos_by_error[error]]))
def validate_travis(repo): """Validate and configure a repository has the expected state in Travis CI. This will both check Travis state and attempt to enable Travis CI and setup the expected state in Travis if not enabled. Expects a dictionary with a GitHub API repository state (like from the list_repos function). Returns a list of string error messages for the repository. """ if not (repo["owner"]["login"] == "adafruit" and repo["name"].startswith("Adafruit_CircuitPython")): return [] repo_url = "/repo/" + repo["owner"]["login"] + "%2F" + repo["name"] result = travis.get(repo_url) if not result.ok: #print(result, result.request.url, result.request.headers) #print(result.text) return [ERROR_TRAVIS_DOESNT_KNOW_REPO] result = result.json() if not result["active"]: activate = travis.post(repo_url + "/activate") if not activate.ok: #output_handler(activate.request.url) #output_handler("{} {}".format(activate, activate.text)) return [ERROR_ENABLE_TRAVIS] env_variables = travis.get(repo_url + "/env_vars") if not env_variables.ok: #print(env_variables, env_variables.text) #print(env_variables.request.headers) return [ERROR_TRAVIS_ENV] env_variables = env_variables.json() found_token = False for var in env_variables["env_vars"]: found_token = found_token or var["name"] == "GITHUB_TOKEN" ok = True if not found_token: if not github_token: return [ERROR_TRAVIS_GITHUB_TOKEN] else: global full_auth if not full_auth: #github_user = github_token github_user = github.get("/user").json() password = input("Password for " + github_user["login"] + ": ") full_auth = (github_user["login"], password.strip()) if not full_auth: return [ERROR_TRAVIS_GITHUB_TOKEN] new_access_token = { "scopes": ["public_repo"], "note": "TravisCI release token for " + repo["full_name"], "note_url": "https://travis-ci.org/" + repo["full_name"] } token = github.post("/authorizations", json=new_access_token, auth=full_auth) if not token.ok: print(token.text) return [ERROR_TRAVIS_TOKEN_CREATE] token = token.json() grant_id = token["id"] token = token["token"] new_var = { "env_var.name": "GITHUB_TOKEN", "env_var.value": token, "env_var.public": False } new_var_result = travis.post(repo_url + "/env_vars", json=new_var) if not new_var_result.ok: #print(new_var_result.headers, new_var_result.text) github.delete("/authorizations/{}".format(grant_id), auth=full_auth) return [ERROR_TRAVIS_GITHUB_TOKEN] return []
# of Github API repository object state). validators = [ validate_repo_state, validate_travis, validate_contents, validate_readthedocs, validate_core_driver_page ] # Submodules inside the bundle (result of get_bundle_submodules) bundle_submodules = [] if __name__ == "__main__": repos = list_repos() print("Found {} repos to check.".format(len(repos))) bundle_submodules = get_bundle_submodules() print("Found {} submodules in the bundle.".format(len(bundle_submodules))) github_user = github.get("/user").json() print("Running GitHub checks as " + github_user["login"]) travis_user = travis.get("/user").json() print("Running Travis checks as " + travis_user["login"]) need_work = 0 insights = { "merged_prs": 0, "closed_prs": 0, "new_prs": 0, "active_prs": 0, "open_prs": [], "pr_authors": set(), "pr_merged_authors": set(), "pr_reviewers": set(), "closed_issues": 0, "new_issues": 0, "active_issues": 0, "open_issues": [],