def update_json_file(json_string):
""" Uses GitHub API to do the following:
- Creates branch on fork 'adafruit-adabot/circuipython-org'
- Updates '_data/libraries.json'
- Creates pull request from fork to upstream
Note: adapted from Scott Shawcroft's code found here
https://github.com/adafruit/circuitpython/blob/master/tools/build_board_info.py
"""
master_url = "/repos/adafruit/circuitpython-org/"
fork_url = "/repos/adafruit-adabot/circuitpython-org/"
commit_date = datetime.date.today()
branch_name = "libraries_update_" + commit_date.strftime("%d-%b-%y")
response = github.get(master_url + "git/refs/heads/master")
if not response.ok:
raise RuntimeError("Failed to retrieve master sha:\n{}".format(
response.text))
commit_sha = response.json()["object"]["sha"]
response = github.get(master_url + "contents/_data/libraries.json?ref=" +
commit_sha)
if not response.ok:
raise RuntimeError("Failed to retrieve libraries.json sha:\n{}".format(
response.text))
blob_sha = response.json()["sha"]
branch_info = {"ref": "refs/heads/" + branch_name, "sha": commit_sha}
response = github.post(fork_url + "git/refs", json=branch_info)
if not response.ok and response.json(
)["message"] != "Reference already exists":
raise RuntimeError("Failed to create branch:\n{}".format(
response.text))
commit_msg = "Automated Libraries update for {}".format(
commit_date.strftime("%d-%b-%y"))
content = json_string.encode("utf-8") + b"\n"
update_json = {
"message": commit_msg,
"content": base64.b64encode(content).decode("utf-8"),
"sha": blob_sha,
"branch": branch_name
}
response = github.put(fork_url + "contents/_data/libraries.json",
json=update_json)
if not response.ok:
raise RuntimeError("Failed to update libraries.json:\n{}".format(
response.text))
pr_info = {
"title": commit_msg,
"head": "adafruit-adabot:" + branch_name,
"base": "master",
"body": commit_msg,
"maintainer_can_modify": True
}
response = github.post(master_url + "pulls", json=pr_info)
if not response.ok:
raise RuntimeError("Failed to create pull request:\n{}".format(
response.text))
def create_pr(changes, updated, git_info):
commit_sha, original_blob_sha = git_info
branch_name = "new_release_" + changes["new_release"]
updated = json.dumps(updated, sort_keys=True,
indent=4).encode("utf-8") + b"\n"
print(updated.decode("utf-8"))
pr_title = "Automated website update for release {}".format(
changes["new_release"])
boards = ""
if changes["new_boards"]:
boards = "New boards:\n* " + "\n* ".join(changes["new_boards"])
languages = ""
if changes["new_languages"]:
languages = "New languages:\n* " + "\n* ".join(
changes["new_languages"])
message = "Automated website update for release {} by AdaBot.\n\n{}\n\n{}\n".format(
changes["new_release"], boards, languages)
create_branch = {"ref": "refs/heads/" + branch_name, "sha": commit_sha}
response = github.post("/repos/adafruit-adabot/circuitpython-org/git/refs",
json=create_branch)
if not response.ok and response.json(
)["message"] != "Reference already exists":
print("unable to create branch")
print(response.text)
return
update_file = {
"message": message,
"content": base64.b64encode(updated).decode("utf-8"),
"sha": original_blob_sha,
"branch": branch_name
}
response = github.put(
"/repos/adafruit-adabot/circuitpython-org/contents/_data/files.json",
json=update_file)
if not response.ok:
print("unable to post new file")
print(response.text)
return
pr_info = {
"title": pr_title,
"head": "adafruit-adabot:" + branch_name,
"base": "master",
"body": message,
"maintainer_can_modify": True
}
response = github.post("/repos/adafruit/circuitpython-org/pulls",
json=pr_info)
if not response.ok:
print("unable to create pr")
print(response.text)
return
print(changes)
print(pr_info)
def validate_labels(self, repo):
"""ensures the repo has the standard labels available"""
response = github.get("/repos/" + repo["full_name"] + "/labels")
if not response.ok:
# replace 'output_handler' with ERROR_OUTPUT_HANDLER
self.output_file_data.append("Labels request failed: {}".format(repo["full_name"]))
return [ERROR_OUTPUT_HANDLER]
errors = []
repo_labels = [label["name"] for label in response.json()]
has_all_labels = True
for label, info in STD_REPO_LABELS.items():
if not label in repo_labels:
response = github.post(
"/repos/" + repo["full_name"] + "/labels",
json={"name": label, "color": info["color"]}
)
if not response.ok:
has_all_labels = False
self.output_file_data.append(
"Request to add '{}' label failed: {}".format(label,
repo["full_name"])
)
if ERROR_OUTPUT_HANDLER not in errors:
errors.append(ERROR_OUTPUT_HANDLER)
if not has_all_labels:
errors.append(ERROR_MISSING_STANDARD_LABELS)
return errors
def ensure_hacktober_label_exists(repo):
""" Checks if the 'Hacktoberfest' label exists on the repo.
If not, creates the label.
"""
response = github.get("/repos/" + repo["full_name"] + "/labels")
if not response.ok:
print("Failed to retrieve labels for '{}'".format(repo["name"]))
return False
repo_labels = [label["name"] for label in response.json()]
hacktober_exists = {"Hacktoberfest", "hacktoberfest"} & set(repo_labels)
if not hacktober_exists:
params = {
"name": "Hacktoberfest",
"color": "f2b36f",
"description": "DigitalOcean's Hacktoberfest"
}
result = github.post("/repos/" + repo["full_name"] + "/labels", json=params)
if not result.status_code == 201:
print("Failed to create new Hacktoberfest label for: {}".format(repo["name"]))
return False
return True
def validate_travis(repo):
"""Validate and configure a repository has the expected state in Travis
CI. This will both check Travis state and attempt to enable Travis CI
and setup the expected state in Travis if not enabled. Expects a
dictionary with a GitHub API repository state (like from the list_repos
function). Returns a list of string error messages for the repository.
"""
if not (repo["owner"]["login"] == "adafruit"
and repo["name"].startswith("Adafruit_CircuitPython")):
return []
repo_url = "/repo/" + repo["owner"]["login"] + "%2F" + repo["name"]
result = travis.get(repo_url)
if not result.ok:
#print(result, result.request.url, result.request.headers)
#print(result.text)
return [ERROR_TRAVIS_DOESNT_KNOW_REPO]
result = result.json()
if not result["active"]:
activate = travis.post(repo_url + "/activate")
if not activate.ok:
#output_handler(activate.request.url)
#output_handler("{} {}".format(activate, activate.text))
return [ERROR_ENABLE_TRAVIS]
env_variables = travis.get(repo_url + "/env_vars")
if not env_variables.ok:
#print(env_variables, env_variables.text)
#print(env_variables.request.headers)
return [ERROR_TRAVIS_ENV]
env_variables = env_variables.json()
found_token = False
for var in env_variables["env_vars"]:
found_token = found_token or var["name"] == "GITHUB_TOKEN"
ok = True
if not found_token:
if not github_token:
return [ERROR_TRAVIS_GITHUB_TOKEN]
else:
global full_auth
if not full_auth:
#github_user = github_token
github_user = github.get("/user").json()
password = input("Password for " + github_user["login"] + ": ")
full_auth = (github_user["login"], password.strip())
if not full_auth:
return [ERROR_TRAVIS_GITHUB_TOKEN]
new_access_token = {
"scopes": ["public_repo"],
"note": "TravisCI release token for " + repo["full_name"],
"note_url": "https://travis-ci.org/" + repo["full_name"]
}
token = github.post("/authorizations",
json=new_access_token,
auth=full_auth)
if not token.ok:
print(token.text)
return [ERROR_TRAVIS_TOKEN_CREATE]
token = token.json()
grant_id = token["id"]
token = token["token"]
new_var = {
"env_var.name": "GITHUB_TOKEN",
"env_var.value": token,
"env_var.public": False
}
new_var_result = travis.post(repo_url + "/env_vars", json=new_var)
if not new_var_result.ok:
#print(new_var_result.headers, new_var_result.text)
github.delete("/authorizations/{}".format(grant_id),
auth=full_auth)
return [ERROR_TRAVIS_GITHUB_TOKEN]
return []
def create_pr(changes, updated, git_info, user):
commit_sha, original_blob_sha = git_info
branch_name = "new_release_" + changes["new_release"]
# Convert the dictionary to a list of boards. Liquid templates only handle arrays.
updated_list = []
all_ids = sorted(updated.keys())
for id in all_ids:
info = updated[id]
info["id"] = id
updated_list.append(info)
updated = json.dumps(updated_list, sort_keys=True,
indent=4).encode("utf-8") + b"\n"
#print(updated.decode("utf-8"))
pr_title = "Automated website update for release {}".format(
changes["new_release"])
boards = ""
if changes["new_boards"]:
boards = "New boards:\n* " + "\n* ".join(changes["new_boards"])
languages = ""
if changes["new_languages"]:
languages = "New languages:\n* " + "\n* ".join(
changes["new_languages"])
message = "Automated website update for release {} by Blinka.\n\n{}\n\n{}\n".format(
changes["new_release"], boards, languages)
create_branch = {"ref": "refs/heads/" + branch_name, "sha": commit_sha}
response = github.post("/repos/{}/circuitpython-org/git/refs".format(user),
json=create_branch)
if not response.ok and response.json(
)["message"] != "Reference already exists":
print("unable to create branch")
print(response.text)
return
update_file = {
"message": message,
"content": base64.b64encode(updated).decode("utf-8"),
"sha": original_blob_sha,
"branch": branch_name
}
response = github.put(
"/repos/{}/circuitpython-org/contents/_data/files.json".format(user),
json=update_file)
if not response.ok:
print("unable to post new file")
print(response.text)
return
pr_info = {
"title": pr_title,
"head": user + ":" + branch_name,
"base": "master",
"body": message,
"maintainer_can_modify": True
}
response = github.post("/repos/adafruit/circuitpython-org/pulls",
json=pr_info)
if not response.ok:
print("unable to create pr")
print(response.text)
return
print(changes)
print(pr_info)
def new_release(bundle, bundle_path):
working_directory = os.path.abspath(os.getcwd())
os.chdir(bundle_path)
print(bundle)
current_release = github.get(
"/repos/adafruit/{}/releases/latest".format(bundle))
last_tag = current_release.json()["tag_name"]
contributors = get_contributors("adafruit/" + bundle, last_tag + "..")
added_submodules = []
updated_submodules = []
repo_links = {}
output = StringIO()
git.diff("--submodule=short", last_tag + "..", _out=output)
output = output.getvalue().strip()
if not output:
print("Everything is already released.")
return
current_submodule = None
current_index = None
for line in output.split("\n"):
if line.startswith("diff"):
current_submodule = line.split()[-1][len("b/"):]
continue
elif "index" in line:
current_index = line
continue
elif not line.startswith("+Subproject"):
continue
# We have a candidate submodule change.
directory = current_submodule
commit_range = current_index.split()[1]
library_name = directory.split("/")[-1]
if commit_range.startswith("0000000"):
added_submodules.append(library_name)
commit_range = commit_range.split(".")[-1]
elif commit_range.endswith("0000000"):
# For now, skip documenting deleted modules.
continue
else:
updated_submodules.append(library_name)
repo_url = repo_remote_url(directory)
new_commit = commit_range.split(".")[-1]
release_tag = commit_to_tag(directory, new_commit)
with Submodule(directory):
submodule_contributors = get_contributors(repo_name(repo_url),
commit_range)
add_contributors(contributors, submodule_contributors)
repo_links[library_name] = repo_url[:-4] + "/releases/" + release_tag
release_description = []
if added_submodules:
additions = []
for library in added_submodules:
additions.append("[{}]({})".format(library, repo_links[library]))
release_description.append("New libraries: " + ", ".join(additions))
if updated_submodules:
updates = []
for library in updated_submodules:
updates.append("[{}]({})".format(library, repo_links[library]))
release_description.append("Updated libraries: " + ", ".join(updates))
release_description.append("")
contributors = sorted(contributors, key=contributors.__getitem__, reverse=True)
contributors = ["@" + x for x in contributors]
release_description.append("As always, thank you to all of our contributors: " + ", ".join(contributors))
release_description.append("\n--------------------------\n")
release_description.append("The libraries in each release are compiled for all recent major versions of CircuitPython. Please download the one that matches the major version of your CircuitPython. For example, if you are running 4.0.0 you should download the `4.x` bundle.\n")
release_description.append("To install, simply download the matching zip file, unzip it, and selectively copy the libraries you would like to install into the lib folder on your CIRCUITPY drive. This is especially important for non-express boards such as the [Trinket M0](https://www.adafruit.com/product/3500), [Gemma M0](https://www.adafruit.com/product/3501) and [Feather M0 Basic](https://www.adafruit.com/product/2772).")
release = {
"tag_name": "{0:%Y%m%d}".format(date.today()),
"target_commitish": repo_sha(),
"name": "{0:%B} {0:%d}, {0:%Y} auto-release".format(date.today()),
"body": "\n".join(release_description),
"draft": False,
"prerelease": False}
print("Releasing {}".format(release["tag_name"]))
print(release["body"])
response = github.post("/repos/adafruit/" + bundle + "/releases", json=release)
if not response.ok:
print("Failed to create release")
print(release)
print(response.request.url)
print(response.text)
os.chdir(working_directory)
import os.path
import sys
import uritemplate
sys.path.append("adabot")
import adabot.github_requests as github
exit_status = 0
for dirpath, dirnames, filenames in os.walk("../bin"):
if not filenames:
continue
for filename in filenames:
full_filename = os.path.join(dirpath, filename)
label = filename.replace("adafruit-circuitpython-", "")
url_vars = {}
url_vars["name"] = filename
url_vars["label"] = label
url = uritemplate.expand(os.environ["UPLOAD_URL"], url_vars)
headers = {"content-type": "application/octet-stream"}
print(url)
with open(full_filename, "rb") as f:
response = github.post(url, data=f, headers=headers)
if not response.ok:
print("Upload of {} failed with {}.".format(
filename, response.status_code))
print(response.text)
sys.exit(response.status_code)
sys.exit(exit_status)
