def UpdateGitHubRelease(spec, tag, token, tmp_dir):
logging.debug("GH release spec: %s", spec)
repo = spec["repo"]
rel, ok = github_api.CallReleasesAPI(repo, token, releases_url=("/tags/%s" % tag))
if not ok:
logging.error("Failed to get release info for %s/%s: %s; (re)creating", repo, tag, rel)
# Assume release doesn't exist and create it.
return CreateGitHubRelease(spec, tag, token, tmp_dir)
logging.info(" Updating release %s / %s (%d)", repo, tag, rel["id"])
for asset_name, asset_file in spec["assets"]:
ct = "application/zip" if asset_name.endswith(".zip") else "application/octet-stream"
for a in rel.get("assets", []):
if a["name"] == asset_name:
logging.info(" Deleting asset %s (%d)", asset_name, a["id"])
github_api.CallReleasesAPI(
repo, token,
method="DELETE",
releases_url=("/assets/%d" % a["id"]),
decode_json=False)
logging.info(" Uploading %s to %s", asset_file, asset_name)
with open(asset_file, "rb") as f:
r, ok = github_api.CallReleasesAPI(
repo, token, method="POST", subdomain="uploads", data=f,
releases_url=("/%d/assets" % rel["id"]),
headers = {"Content-Type": ct},
params = {"name": asset_name})
if not ok:
logging.error("Failed to upload %s: %s", asset_name, r)
raise RuntimeError
logging.info(" Updated release %s / %s (%d)", repo, tag, rel["id"])
def del_release(repo_name, tag):
r, ok = github_api.CallReleasesAPI(repo_name,
releases_url="/tags/%s" % tag)
if ok:
print("%s: Deleting existing %s release %d" %
(repo_name, tag, r["id"]))
r, ok = github_api.CallReleasesAPI(repo_name,
releases_url="/%d" % r["id"],
method="DELETE",
decode_json=False)
def UpdateGitHubRelease(spec, tag, token, tmp_dir):
logging.debug("GH release spec: %s", spec)
repo = spec["repo"]
rel, ok = github_api.CallReleasesAPI(repo,
token,
releases_url=("/tags/%s" % tag))
if not ok:
logging.error("Failed to get release info for %s/%s: %s", repo, tag,
rel)
raise RuntimeError
logging.info(" Updating release %s / %s (%d)", repo, tag, rel["id"])
for asset_name, asset_file in spec["assets"]:
ct = "application/zip" if asset_name.endswith(
".zip") else "application/octet-stream"
for a in rel.get("assets", []):
if a["name"] == asset_name:
logging.info(" Deleting asset %s (%d)", asset_name, a["id"])
github_api.CallReleasesAPI(repo,
token,
method="DELETE",
releases_url=("/assets/%d" %
a["id"]),
decode_json=False)
logging.info(" Uploading %s to %s", asset_file, asset_name)
with open(asset_file, "rb") as f:
r, ok = github_api.CallReleasesAPI(repo,
token,
method="POST",
subdomain="uploads",
data=f,
releases_url=("/%d/assets" %
rel["id"]),
headers={"Content-Type": ct},
params={"name": asset_name})
if not ok:
logging.error("Failed to upload %s: %s", asset_name, r)
if r and r.get("errors", [{}])[0].get("code",
"") == "already_exists":
# This is a bug in GitHub where sometimes "phantom asset" will block an upload.
# The asset is not listed (or it would've been deleted), but an uplaod will fail.
# There is no way around it exept re-creating a release.
# Here we'll just delete it and next run will re-create properly. Ugh.
logging.error("*BUG* Phantom asset, nuking release")
DeleteRelease(repo, token, rel["id"])
raise RuntimeError
logging.info(" Updated release %s / %s (%d)", repo, tag, rel["id"])
def handle_repo(repo_name, tag):
res, ok = github_api.CallReleasesAPI(repo_name, TOKEN, f"/tags/{tag}")
if not ok:
print(f"{repo_name}: No {tag} release")
return
repo_out_dir = os.path.join(args.out_dir, repo_name, tag)
meta_file = os.path.join(repo_out_dir, ".meta.json")
meta = {"assets": {}}
if os.path.exists(meta_file):
try:
with open(meta_file) as f:
meta = json.load(f)
except Exception as e:
print("Invalid meta {meta_file}: {e}")
token = None
if args.token_filepath:
with open(args.token_filepath) as f:
token = f.read().strip()
for asset in res["assets"]:
asset_name, asset_url, asset_mtime = asset["name"], asset[
"url"], asset["updated_at"]
asset_meta = meta["assets"].get(asset_name)
if asset_meta and asset_meta["updated_at"] == asset_mtime:
print(f"{repo_name}: {asset_name} is up to date ({asset_mtime})")
continue
r = requests.get(asset_url,
auth=(token, ""),
headers={"Accept": "application/octet-stream"})
if r.status_code == 200 and len(r.content) == asset["size"]:
asset_hash = hashlib.sha256(r.content).hexdigest()
asset_file = os.path.join(repo_out_dir,
f"{asset_name}.{asset_hash}")
print(
f"{repo_name}: {asset_name} {len(r.content)} bytes, hash {asset_hash}"
)
os.makedirs(os.path.dirname(asset_file), mode=0o755, exist_ok=True)
with open(asset_file, "wb") as f:
f.write(r.content)
meta["assets"][asset_name] = asset
else:
print(f"Failed to download asset {asset_url}: {r.text}")
meta["last_update"] = time.time()
with open(meta_file, "w") as f:
json.dump(meta, f, indent=" ", sort_keys=True)
def handle_repo(repo_name, from_tag, to_tag):
if not args.no_cleanup_drafts:
rr, ok = github_api.CallReleasesAPI(repo_name,
TOKEN,
method="GET",
releases_url="")
if not ok:
raise Exception("Failed to list releases: %s" % r)
for r in rr:
if not r["draft"]:
continue
print("%s: Cleaning up draft release %d (%s)" %
(repo_name, r["id"], r["name"]))
github_api.CallReleasesAPI(repo_name,
TOKEN,
releases_url="/%d" % r["id"],
method="DELETE",
decode_json=False)
# Delete existing release, if any.
del_release(repo_name, to_tag)
# Tag the repo (deletes existing tag, if any).
make_repo_tag(repo_name, to_tag)
res, ok = github_api.CallReleasesAPI(repo_name, TOKEN,
"/tags/%s" % from_tag)
if not ok:
print("%s: No %s release, not creating tagged" % (repo_name, from_tag))
# No release - no problem.
return
# Create a release draft {{{
print("%s: Creating a new draft of %s" % (repo_name, to_tag))
r, ok = github_api.CallReleasesAPI(repo_name,
TOKEN,
method="POST",
releases_url="",
json_data={
"tag_name": to_tag,
"name": to_tag,
"draft": True,
})
if not ok:
raise Exception("Failed to create a draft: %s" % r)
new_rel_id = r["id"]
# }}}
with open(args.token_filepath) as f:
token = f.read().strip()
for asset in res["assets"]:
asset_url = asset["url"]
print("%s: Downloading %s (%s)" %
(repo_name, asset["name"], asset_url))
r = requests.get(asset_url,
auth=(token, ""),
headers={"Accept": "application/octet-stream"})
if r.status_code == 200:
print("%s: Uploading a new asset %s" % (repo_name, asset["name"]))
r, ok = github_api.CallReleasesAPI(
repo_name,
TOKEN,
method="POST",
subdomain="uploads",
data=r.content,
releases_url="/%d/assets" % new_rel_id,
headers={"Content-Type": asset["content_type"]},
params={"name": asset["name"]})
if not ok:
raise Exception("Failed to upload %s: %s" % (asset["name"], r))
else:
raise Exception("Failed to download asset: %s" % r.text)
# Undraft the release {{{
print("%s: Undraft the %s release" % (repo_name, to_tag))
r, ok = github_api.CallReleasesAPI(repo_name,
TOKEN,
method="PATCH",
releases_url="/%d" % new_rel_id,
json_data={
"draft": False,
})
if not ok:
raise Exception("Failed to undraft the release: %s" % r)
def CreateGitHubRelease(spec, tag, token, tmp_dir, re_create=False):
logging.debug("GH release spec: %s", spec)
repo = spec["repo"]
logging.info(" Publishing release %s / %s", repo, tag)
# If tag already exists, make sure it points to master.
tag_ref, ok1 = github_api.CallRefsAPI(repo, token, ("/tags/%s" % tag))
master_ref, ok2 = github_api.CallRefsAPI(repo, token, "/heads/master")
if ok1 and ok2 and tag_ref["object"]["sha"] != master_ref["object"]["sha"]:
logging.info(" Updating tag %s (%s -> %s)", tag,
tag_ref["object"]["sha"], master_ref["object"]["sha"])
r, ok = github_api.CallRefsAPI(repo,
token,
method="PATCH",
uri=("/tags/%s" % tag),
json_data={
"sha": master_ref["object"]["sha"],
"force": False,
})
# If target release already exists, avoid re-creating it - simply delete previous assets.
rel, ok = github_api.CallReleasesAPI(repo,
token,
releases_url=("/tags/%s" % tag))
if ok:
if re_create:
logging.info(" Release already exists (id %d), deleting",
rel["id"])
DeleteRelease(repo, token, rel["id"])
rel = None
else:
logging.info(" Release already exists (id %d), deleting assets",
rel["id"])
for a in rel.get("assets", []):
logging.info(" Deleting asset %s (%d)", a["name"],
a["id"])
github_api.CallReleasesAPI(repo,
token,
method="DELETE",
releases_url=("/assets/%d" %
a["id"]),
decode_json=False)
else:
rel = None
if rel is None:
logging.info(" Creating release")
rel, ok = github_api.CallReleasesAPI(repo,
token,
"",
method="POST",
json_data={
"name": tag,
"draft": False,
"tag_name": tag,
"target_commitish": "master",
})
if not ok:
logging.error("Failed to create a release draft: %s", r)
raise RuntimeError
rel_id = rel["id"]
logging.debug("Release id: %d", rel_id)
logging.info(" Uploading assets")
for asset_name, asset_file in spec["assets"]:
ct = "application/zip" if asset_name.endswith(
".zip") else "application/octet-stream"
logging.info(" Uploading %s to %s", asset_file, asset_name)
with open(asset_file, "rb") as f:
r, ok = github_api.CallReleasesAPI(repo,
token,
method="POST",
subdomain="uploads",
data=f,
releases_url=("/%d/assets" %
rel_id),
headers={"Content-Type": ct},
params={"name": asset_name})
if not ok:
logging.error("Failed to upload %s: %s", asset_name, r)
raise RuntimeError
logging.info(" Published release %s / %s (%d)", repo, tag, rel["id"])
def DeleteRelease(repo, token, rel_id):
github_api.CallReleasesAPI(repo,
token,
method="DELETE",
releases_url=("/%d" % rel_id),
decode_json=False)
def ProcessLoc(e, loc, args, created_repos):
parts = loc.split("/")
pre, name, i, repo_loc, repo_subdir = "", "", 0, loc, ""
for p in parts:
pre = name.split(":")[-1]
name = p
i += 1
if p.endswith(".git"):
repo_loc = "/".join(parts[:i])
repo_subdir = "/".join(parts[i:])
name = p[:-4]
break
repo_dir = os.path.join(args.tmp_dir, pre, name)
if os.path.exists(repo_loc):
rl = repo_loc + ("/" if not repo_loc.endswith("/") else "")
os.makedirs(repo_dir, exist_ok=True)
cmd = ["rsync", "-a", "--delete", rl, repo_dir + "/"]
name = os.path.basename(repo_loc)
else:
if not os.path.exists(repo_dir):
logging.info("Cloning into %s", repo_dir)
cmd = ["git", "clone", repo_loc, repo_dir]
else:
logging.info("Pulling %s", repo_dir)
cmd = ["git", "-C", repo_dir, "pull"]
if repo_subdir:
name = os.path.basename(repo_subdir)
logging.info("== %s: %s / %s", name, repo_loc, repo_subdir)
else:
name = os.path.basename(repo_loc)
logging.info("== %s: %s", name, repo_loc)
RunCmd(cmd)
if repo_subdir:
tgt_dir = os.path.join(repo_dir, repo_subdir)
else:
tgt_dir = repo_dir
tgt_name = os.path.split(tgt_dir)[-1]
assets = []
common = e.get("common", {})
# Build all the variants, collect assets
for v in e["variants"]:
logging.info(" %s %s", tgt_name, v["name"])
mos_cmd = [args.mos, "build", "-C", tgt_dir, "--local", "--clean"]
if args.repo_dir:
mos_cmd.append("--repo=%s" % args.repo_dir)
if args.deps_dir:
mos_cmd.append("--deps-dir=%s" % args.deps_dir)
if args.binary_libs_dir:
mos_cmd.append("--binary-libs-dir=%s" % args.binary_libs_dir)
if args.lib:
for lib in args.lib:
mos_cmd.append("--lib=%s" % lib)
if args.libs_dir:
mos_cmd.append("--libs-dir=%s" % args.libs_dir)
if args.no_libs_update:
mos_cmd.append("--no-libs-update")
mos_cmd.append("--platform=%s" % v["platform"])
for bvk, bvv in sorted(
list(common.get("build_vars", {}).items()) +
list(v.get("build_vars", {}).items())):
mos_cmd.append("--build-var=%s=%s" % (bvk, bvv))
cflags = (common.get("cflags", "") + " " + v.get("cflags", "")).strip()
if cflags:
mos_cmd.append("--cflags-extra=%s" % cflags)
cxxflags = (common.get("cxxflags", "") + " " +
v.get("cxxflags", "")).strip()
if cflags:
mos_cmd.append("--cxxflags-extra=%s" % cflags)
mos_args = (common.get("mos_args", []) + v.get("mos_args", []))
if mos_args:
mos_cmd.extend(mos_args)
RunCmd(mos_cmd)
bl = os.path.join(args.tmp_dir,
"%s-%s-build.log" % (tgt_name, v["name"]))
logging.info(" Saving build log to %s", bl)
shutil.copy(os.path.join(tgt_dir, "build", "build.log"), bl)
# Ok, what did we just build?
with open(os.path.join(tgt_dir, "mos.yml")) as f:
m = yaml.safe_load(f)
if m.get("type", "") == "lib":
assets.append(
MakeAsset("lib%s-%s.a" % (tgt_name, v["name"]),
os.path.join(tgt_dir, "build", "lib.a"),
args.tmp_dir))
else:
assets.append(
MakeAsset("%s-%s.zip" % (tgt_name, v["name"]),
os.path.join(tgt_dir, "build", "fw.zip"),
args.tmp_dir))
for fn in glob.glob(
os.path.join(tgt_dir, "build", "objs", "*.elf")):
an = os.path.basename(fn).replace(
tgt_name, "%s-%s" % (tgt_name, v["name"]))
assets.append(MakeAsset(an, fn, args.tmp_dir))
outs = e.get("out", [])
if not outs and loc.startswith("https://github.com/"):
outs = [{"github": {"repo": "%s/%s" % (pre, tgt_name)}}]
for out in outs:
gh_out = copy.deepcopy(out.get("github", {}))
# Push to GitHub
if gh_out:
gh_out["assets"] = assets
gh_out["repo"] = gh_out["repo"] % {
"name": name,
"repo_subdir": repo_subdir,
}
if not args.gh_token_file:
logging.info("Token file not set, GH uploads disabled")
return
if not os.path.isfile(args.gh_token_file):
logging.error("Token file %s does not exist",
args.gh_token_file)
exit(1)
logging.debug("Using token file at %s", args.gh_token_file)
with open(args.gh_token_file, "r") as f:
token = f.read().strip()
i = 1
while True:
try:
should_update = gh_out.get("update", False)
# If we already (re)created this repo during this run, do not do it again.
if created_repos.get(gh_out["repo"]):
should_update = True
if not should_update:
# Looks like after some number of asset deletions / uploads GH release
# gets into a bad state where new asset uploads are just failing.
# So we try once, try twice and on the third time we re-create the release
# even if it exists.
re_create = (i > 2)
CreateGitHubRelease(gh_out,
args.gh_release_tag,
token,
args.tmp_dir,
re_create=re_create)
created_repos[gh_out["repo"]] = True
else:
UpdateGitHubRelease(gh_out, args.gh_release_tag, token,
args.tmp_dir)
break
except (Exception, KeyboardInterrupt) as e:
logging.exception("Exception (attempt %d): %s", i, e)
if not isinstance(e, KeyboardInterrupt) and i < 5:
time.sleep(1)
i += 1
else:
if not isinstance(
e, KeyboardInterrupt) and gh_out.get("update"):
logging.error(
"*BUG* Phantom asset (probably), nuking release"
)
rel, ok = github_api.CallReleasesAPI(
gh_out["repo"],
token,
releases_url=("/tags/%s" %
args.gh_release_tag))
if ok:
DeleteRelease(gh_out["repo"], token, rel["id"])
raise
