def collect_milestone(milestone): label_groups = get_label_groups() mrs = [] with Spinner( text=f"Loading merge requests associated with %{milestone.iid}"): for mr in milestone.merge_requests(): if mr.state == "merged": mrs.append(mr) # need to get issues from merged MRs with Spinner(text=f"Collecting issues from {len(mrs)} merged MRs"): issue_ids = [] issues = [] for mr in mrs: for issue in mr.closes_issues(): if issue.id not in issue_ids: issue_ids.append(issue.id) issues.append(issue) issues_grouped = group_items(label_groups, issues) mrs_grouped = group_items(label_groups, mrs) return mrs_grouped, issues_grouped
def relnotes(start, end, gitlab): start = tuple(start) end = tuple(end) print(start, end, file=sys.stderr) project = gitlab.projects.get("acts/acts-core") all_milestones = get_milestones(project) milestones = [] for ms in all_milestones.values(): try: ver = split_version(ms.title) milestones.append(ms) except: pass sorted_milestones = list(sorted(all_milestones.keys())) start_ms = all_milestones[start] end_ms = all_milestones[end] ms_range = (sorted_milestones.index(start), sorted_milestones.index(end)) md = "" for mst in sorted_milestones[ms_range[0] + 1:ms_range[1] + 1]: ms = all_milestones[mst] print(ms.title, file=sys.stderr) mrs_grouped, issues_grouped = collect_milestone(ms) with Spinner(text="Assembling release notes", stream=sys.stderr): md += f"## {format_version(mst)}\n\n" md += make_release_notes(ms, mrs_grouped, issues_grouped, badges=False, links=False) print(md)
def main(): p = argparse.ArgumentParser() p = def_arguments(p, acc=True, gl=True) p.add_argument("--doc-source", required=True) p.add_argument("--dry-run", action="store_true") p.add_argument("--ref", default=os.getenv("CI_COMMIT_REF_NAME", None), required=True) p.add_argument( "--doc-root", default=os.getenv("DOC_WEBSITE_ROOT", "/eos/user/a/atsjenkins/www/ACTS/"), ) p.add_argument( "--doc-public-url", default=os.getenv("DOC_WEBSITE_URL", "https://acts.web.cern.ch/ACTS/"), ) args = p.parse_args() src_fs = OSFS(os.path.abspath(args.doc_source)) www_fs = get_lxplus_fs(args).opendir(os.path.join(args.doc_root)) if not www_fs.exists(args.ref): www_fs.makedirs(os.path.join(args.ref, "doc")) refdir = www_fs.opendir(os.path.join(args.ref, "doc")) # refdir = OSFS("/tmp/doctest") print( os.path.abspath(args.doc_source), "->", os.path.join(args.doc_root, args.ref, "doc"), ) with Spinner(f"Publishing doc for {args.ref}"): if not args.dry_run: fs.copy.copy_dir(src_fs, ".", refdir, ".") doc_url = os.path.join(args.doc_public_url, args.ref, "doc") print("Doc is available at", doc_url) # write tag info json file if not args.dry_run: with www_fs.open("latest_release.json", "w") as f: json.dump( { "subject": "release", "status": args.ref, "color": "yellow" }, f) gl = gitlab(args) project = gl.projects.get("acts/acts-core") version = parse_version(args.ref) with Spinner(text="Loading milestone"): milestones = project.milestones.list(all=True) milestone = None for ms in milestones: if ms.title == version: milestone = ms break relnotes = make_release_notes(milestone, *collect_milestone(milestone)) message = MIMEMultipart("alternative") message["Subject"] = f"New Acts release: {args.ref}" message["From"] = sender_email message["To"] = receiver_email text = """ Dear Acts enthusiasts, a new tag '{ref}' of the Acts project has been created. You can get the source code from git using: git clone https://gitlab.cern.ch/acts/acts-core.git cd acts-core/ git checkout {ref} or download a tarball with the source from https://gitlab.cern.ch/acts/acts-core/-/archive/{ref}/acts-core-{ref}.tar.gz The documentation is deployed at https://acts.web.cern.ch/ACTS/{ref}/doc/index.html Cheers, your friendly Acts robot """ text = textwrap.dedent(text).format(ref=args.ref, relnotes=relnotes) md = """ Dear Acts enthusiasts, a new tag of the Acts project has been created. --- # {ref} [![](https://badgen.net/badge/release/{ref}/yellow)](https://gitlab.cern.ch/acts/acts-core/tags/{ref}) {relnotes} --- You can get the source code from git using: ```bash git clone https://gitlab.cern.ch/acts/acts-core.git cd acts-core/ git checkout {ref} ``` or download a tarball with the source from https://gitlab.cern.ch/acts/acts-core/-/archive/{ref}/acts-core-{ref}.tar.gz The documentation is deployed at https://acts.web.cern.ch/ACTS/{ref}/doc/index.html Cheers,<br/> your friendly Acts robot """ md = textwrap.dedent(md).format(ref=args.ref, relnotes=relnotes) html = """\ <html> <body> {text} </body> </html> """.format(text=markdown(textwrap.dedent(md))) # print(html) part1 = MIMEText(text, "plain") part2 = MIMEText(html, "html") message.attach(part1) message.attach(part2) with Spinner("Sending email"): if not args.dry_run: with smtp(args) as server: server.sendmail(sender_email, receiver_email, message.as_string())
def zenodo(version, gitlab, zenodo_token, deposition): version = split_version(version) # print(version, gitlab, zenodo_token) zenodo = Zenodo(zenodo_token) with Spinner(text="Creating new version of existing deposition"): create_res = zenodo.post( f"deposit/depositions/{deposition}/actions/newversion") # print(create_res) create_res = create_res.json() draft_id = create_res["links"]["latest_draft"].split("/")[-1] # pprint(create_res) print("Created new version with id", draft_id) with Spinner(text="Delete all files for draft"): draft = zenodo.get(f"deposit/depositions/{draft_id}") # pprint(draft) for file in draft["files"]: file_id = file["id"] r = zenodo.delete( f"deposit/depositions/{draft_id}/files/{file_id}") assert r.status_code == 204 with Spinner(text="Assembling authors"): creator_file = os.path.join(os.path.dirname(__file__), "../AUTHORS.md") with open(creator_file) as fh: md = fh.read().strip().split("\n") md = [ l.strip() for l in md if not l.strip().startswith("#") and not l.strip() == "" ] creators = [] for line in md: assert line.startswith("- ") line = line[2:] split = line.split(",", 1) creator = {"name": split[0].strip()} if len(split) == 2: creator["affiliation"] = split[1].strip() creators.append(creator) with Spinner(text="Collection milestones for description"): project = gitlab.projects.get("acts/acts-core") milestones = project.milestones.list() milestone = find_milestone(version, milestones) mrs_grouped, issues_grouped = collect_milestone(milestone) assert milestone.state == "closed" tag = project.tags.get(format_version(version)) # print(tag) tag_date = dateutil.parser.parse( tag.commit["created_at"]).date().strftime("%Y-%m-%d") description = f'Milestone: <a href="{milestone.web_url}">%{milestone.title}</a> <br/> Merge requested accepted for this version: \n <ul>\n' for mr in sum(mrs_grouped.values(), []): description += f'<li><a href="{mr.web_url}">!{mr.iid} - {mr.title}</a></li>\n' description += "</ul>" with Spinner(text="Updating deposition metadata"): data = { "metadata": { "title": f"Acts Project: {format_version(version)}", "upload_type": "software", "description": description, "creators": creators, "version": format_version(version), "publication_date": tag_date, "license": "MPL-2.0", } } zenodo.put(f"deposit/depositions/{draft_id}", data).json() with tempfile.TemporaryFile() as fh: with Spinner(text="Downloading release archive from Gitlab"): r = requests.get( f"https://gitlab.cern.ch/acts/acts-core/-/archive/{format_version(version)}/acts-core-{format_version(version)}.zip", stream=True) r.raw.decode_content = True fh.write(r.raw.read()) fh.seek(0) with Spinner(text="Uploading release archive to zenodo"): name = f"acts-core-{format_version(version)}.zip" zenodo.upload(draft_id, name, fh) print(f"Done: https://zenodo.org/deposit/{draft_id}")
def message(version, gitlab): dfmt = "%Y-%m-%d" current_version = split_version(version) next_version = current_version[:] next_version[1] += 1 print(current_version, next_version, file=sys.stderr) project = gitlab.projects.get("acts/acts-core") milestones = project.milestones.list() current_milestone = find_milestone(current_version, milestones) assert current_milestone is not None next_milestone = find_milestone(next_version, milestones) if next_milestone is None: print("Milestone for", format_version(next_version), "does not exist") if click.confirm("Want me to create it?"): title = click.prompt("What title?", format_version(next_version)) next_milestone = project.milestones.create(title=title) else: sys.exit(1) if current_milestone.due_date != date.today().strftime(dfmt): if sys.stdout.isatty(): if click.confirm( f"Do you want me to set due date of %{current_milestone.title} to {date.today()}? (is {current_milestone.due_date})" ): current_milestone.due_date = date.today().strftime(dfmt) current_milestone.save() if next_milestone.due_date is None: dt = date.today() delta = relativedelta(weekday=FR(1)) next_due = dt + delta else: next_due = datetime.strptime(next_milestone.due_date, dfmt) if sys.stdout.isatty(): next_due = datetime.strptime( click.prompt( f"Due date for milestone %{next_milestone.title}", next_due.strftime(dfmt), ), dfmt, ) start_date = datetime.strptime(next_milestone.start_date, dfmt) or date.today() start_date_str = start_date.strftime(dfmt) next_due_str = next_due.strftime(dfmt) if (next_milestone.start_date != start_date_str or next_milestone.due_date != next_due_str): if click.confirm(f"Update milestone %{next_milestone.title}?"): with Spinner(text=f"Updating milestone %{next_milestone.title}"): next_milestone.start_date = start_date_str next_milestone.due_date = next_due_str next_milestone.save() release_branch = "release/v{:d}.{:>02d}.X".format(*current_version) tpl = jinja2.Template(""" I've just tagged [`{{cv}}`](https://gitlab.cern.ch/acts/acts-core/-/tags/{{cv}}) from milestone [`%{{cm.title}}`](https://gitlab.cern.ch/acts/acts-core/-/milestones/{{cm.iid}}). Bugfixes should be targeted at [`{{ release_branch }}`](https://gitlab.cern.ch/acts/acts-core/tree/{{ release_branch }}). We will tag the next release `{{nv}}` on {{humanize.naturaldate(next_due)}} from [`%{{nm.title}}`](https://gitlab.cern.ch/acts/acts-core/-/milestones/{{nm.iid}}). This release can be cancelled if a sufficient number of merges does not happen before that date. """.strip()) tpl.globals["humanize"] = humanize text = tpl.render( next_due=datetime.strptime(next_milestone.due_date, dfmt).date(), release_branch=release_branch, cm=current_milestone, cv=format_version(current_version), nm=next_milestone, nv=format_version(next_version), ) print(text)
def patch(version, dry_run, gitlab): project = gitlab.projects.get("acts/acts-core") version = split_version(version) milestone = find_milestone(version, project.milestones.list(state="active")) assert (milestone is not None ), f"Didn't find milestone for {version}. Is it closed already?" branches = get_branches() release_branch = "release/v{:d}.{:>02d}.X".format(*version) version_file = Path() / "version_number" tag_name = format_version(version) if release_branch not in branches: print("Release branch", release_branch, "does not exist. I'm bailing") print("Will make new patch version tag %s from milestone %s on branch %s" % (format_version(version), milestone.title, release_branch)) if click.confirm("Do you want to run local preparation?"): with Spinner( text=f"Checkout and update release branch {release_branch}"): if not dry_run: git.checkout(release_branch) assert current_branch() == release_branch git.pull() with Spinner(text=f"Bumping version to {format_version(version)}"): if not dry_run: assert current_branch() == release_branch with version_file.open("w") as fh: fh.write(".".join(map(str, version))) with Spinner( text= f"Committing bumped version on release branch {release_branch}" ): if not dry_run: git.add(str(version_file)) git.commit(message="Bump version to %s" % ".".join(map(str, version))) with Spinner( text=f"Creating local tag {tag_name} on {release_branch}"): if not dry_run: git.tag(tag_name) print(f"You might want to run 'git push REMOTE {tag_name}'") if click.confirm(f"Do you want me to try to push {release_branch}?"): with Spinner(text=f"Pushing {release_branch}"): if not dry_run: git.push() if click.confirm(f"Do you want me to close %{milestone.title}?"): with Spinner(text=f"Closing milestone %{milestone.title}"): if not dry_run: milestone.state_event = "close" milestone.save()
def main(): p = argparse.ArgumentParser() p = def_arguments(p, acc=True, gl=True) p.add_argument("--coverage-source", required=True) p.add_argument("--ref", default=os.getenv("CI_COMMIT_TAG", os.getenv("CI_COMMIT_SHA", None))) p.add_argument( "--coverage-commit-limit", default=int(os.getenv("COVERAGE_COMMIT_LIMIT", 10)), type=int, ) p.add_argument( "--coverage-root", default=os.getenv("COVERAGE_WEBSITE_ROOT", "/eos/user/a/atsjenkins/www/ACTS/coverage"), ) p.add_argument( "--website-public-url", default=os.getenv("COVERAGE_WEBSITE_URL", "https://acts.web.cern.ch/ACTS/coverage/"), ) p.add_argument("--project-id", default=3031, type=int) p.add_argument("--dry-run", "-s", action="store_true") args = p.parse_args() try: www_fs = get_lxplus_fs(args).opendir(args.coverage_root) # www_fs = OSFS("www") listdir = www_fs.listdir(".") except: print("Unable to establish SSH connection to lxplus") print("This might indicate a problem with the credentials") print("or a temporary connection / configuration problem") raise sys.exit(1) gl = gitlab(args) project = gl.projects.get(args.project_id) if len(args.ref) == 40: # is commit hash deploy_name = args.ref[:8] else: # probably tag deploy_name = args.ref coverage_dest = os.path.join(args.coverage_root, deploy_name) print("Going to deploy coverage for", deploy_name, "to", coverage_dest) print( "Will be publicly available under", urljoin(args.website_public_url, deploy_name), ) src_fs = OSFS(args.coverage_source) with Spinner(f"Publishing ref {deploy_name}"): if not args.dry_run: fs.copy.copy_dir(src_fs, ".", www_fs, deploy_name) # cleanup # get all deployed commits with Spinner(text="Getting deployed commits"): deployed_commits = set() for item in www_fs.listdir("."): if not www_fs.isdir(item): continue if item.startswith("v"): # skip versions continue deployed_commits.add(item) with Spinner(text="Getting info for deployed commits"): with ThreadPoolExecutor(max_workers=20) as tp: # deployed_commit_info = p.map(project.commits.get, deployed_commits) futures = [ tp.submit(project.commits.get, c) for c in deployed_commits ] wait(futures) deployed_commits_with_time = [] for commit, future in zip(deployed_commits, futures): try: info = future.result() date = parse(info.committed_date) deployed_commits_with_time.append((commit, date)) except gitlab.exceptions.GitlabGetError as e: print("Commit", commit, "not found, will remove") deployed_commits_with_time = list( reversed(sorted(deployed_commits_with_time, key=lambda i: i[1]))) # take the n newest commits commits_to_keep = set( h for h, _ in deployed_commits_with_time[:args.coverage_commit_limit]) print("Currently deployed commits:") for idx, (h, t) in enumerate(deployed_commits_with_time): if idx < args.coverage_commit_limit: print(" o", h, "-", t) else: print(" x", h, "-", t) print("Keeping commits:", ", ".join(commits_to_keep)) commits_to_delete = deployed_commits - commits_to_keep if len(commits_to_delete) > 0: with Spinner("Removing: %s" % ", ".join(commits_to_delete)): if not args.dry_run: for commit in commits_to_delete: www_fs.removetree(commit) # install / update indexfile latest_commit = deployed_commits_with_time[0][0] latest_coverage_url = urljoin(args.website_public_url, latest_commit) index_content = """ <!DOCTYPE html> <html> <head> <meta http-equiv="refresh" content="0; url={0}" /> </head> <body> Redirecting to <a href"{0}">{0}</a> </body> </html> """.format(latest_coverage_url) with Spinner("Writing index file redirecting to %s" % latest_coverage_url): if not args.dry_run: with www_fs.open("index.html", "w") as f: f.write(index_content)
def tag(obj, tag_name, remote, yes): current_branch = get_current_branch() remote_url = git.remote("get-url", remote).strip() gh, repo = obj tag = split_version(tag_name) tag_name = format_version(tag) major, minor, fix = tag with Spinner(f"Checking for milestone for tag {tag_name}"): tag_milestone = None for ms in repo.get_milestones(state="all"): if ms.title == tag_name: tag_milestone = ms break assert tag_milestone is not None, "Did not find milestone for tag" release_branch_name = f"release/v{major}.{minor:>02}.X" with Spinner("Refreshing branches"): git.fetch(all=True, prune=True) if fix == 0: # new minor release with Spinner(f"Checking out and updating {default_branch_name}"): git.checkout(default_branch_name) git.pull() assert not check_branch_exists( release_branch_name ), "For new minor: release branch CANNOT exist yet" with Spinner(f"Creating {release_branch_name}"): git.checkout("-b", release_branch_name) else: assert check_branch_exists( release_branch_name), "For new fix: release brunch MUST exist" with Spinner(f"Checking out {release_branch_name}"): git.checkout(release_branch_name) # we are not on release branch version_file = Path("version_number") assert version_file.exists(), "Version number file not found" current_version_string = version_file.read_text() print(f"Current version: [bold]{current_version_string}[/bold]") if fix == 0: assert current_version_string == "9.9.9", "Unexpected current version string found" else: assert current_version_string != f"{major}.{minor}.{fix-1}", "Unexpected current version string found" version_string = f"{major}.{minor}.{fix}" with Spinner( f"Bumping version number in '{version_file}' to '{version_string}'" ): with version_file.open("w") as fh: fh.write(version_string) with Spinner("Comitting"): git.add(version_file) git.commit(m=f"Bump version number to {version_string}") with Spinner(f"Creating tag {tag_name}"): git.tag(tag_name) print( f"I will now: push tag [bold green]{tag_name}[/bold green] and branch [bold green]{release_branch_name}[/bold green] to [bold]{remote_url}[/bold]" ) if not confirm("Continue?", yes=yes): raise SystemExit("Aborting") with Spinner(f"Pushing branch {release_branch_name}"): git.push("-u", remote, release_branch_name) with Spinner(f"Pushing tag {tag_name}"): git.push(remote, tag_name)
def notes(obj, tag_name, draft, yes): gh, repo = obj label_file = repo.get_contents(".labels.yml", ref=default_branch_name).decoded_content labels = yaml.safe_load(io.BytesIO(label_file))["labels"] with Spinner(f"Finding tag {tag_name}"): tag = None for t in repo.get_tags(): if t.name == tag_name: tag = t break assert tag is not None, "Did not find tag" with Spinner(f"Loading milestone for tag {tag_name}"): tag_milestone = None for ms in repo.get_milestones(state="all"): if ms.title == tag_name: tag_milestone = ms break assert tag_milestone is not None, "Did not find milestone for tag" with Spinner(f"Getting PRs for milestone {tag_milestone.title}"): prs = list( gh.search_issues("", milestone=tag_milestone.title, repo=repo.full_name, type="pr", **{"is": "merged"})) assert not any([pr.state == "open" for pr in prs ]), "PRs assigned to milestone that are still open!" click.echo("Have " + click.style(str(len(prs)), bold=True) + " PRs, all closed.") body = "" groups = {l: [] for l in sorted(labels)} groups["Uncategorized"] = [] for pr in prs: pr_labels = [l.name for l in pr.labels] assigned = False for label in labels: if label in pr_labels: groups[label].append(pr) assigned = True break if not assigned: groups["Uncategorized"].append(pr) for group, prs in groups.items(): if len(prs) == 0: continue name = group if name.lower() == "bug": name = "Bug Fixes" body += f"#### {name}:\n\n" for pr in prs: body += f"- {pr.title} [#{pr.number}]({pr.html_url})\n" body += "\n" body = body.strip() width, _ = click.get_terminal_size() print() click.secho( "\n".join([l.ljust(width) for l in [""] + body.split("\n") + [""]]), fg="black", bg="white", ) print() release = None with Spinner("Getting release"): try: release = repo.get_release(tag.name) except github.UnknownObjectException: pass if release is not None: # existing release, update click.echo("Existing release {} is at {}".format( click.style(release.title, bold=True), click.style(release.html_url, bold=True), )) if confirm(f"Update release {release.title}?", yes=yes): with Spinner(f"Updating release {release.title}"): release.update_release(name=release.title, message=body) click.echo("Updated release is at {}".format( click.style(release.html_url, bold=True))) else: # new release if confirm(f"Create release for tag {tag.name} (draft: {draft})?", yes=yes): with Spinner(f"Creating release {tag.name}"): release = repo.create_git_release(tag=tag.name, name=tag.name, message=body, draft=draft) click.echo("Created release is at {}".format( click.style(release.html_url, bold=True))) else: print("Not creating a release") if tag_milestone.state == "open": if confirm(f"Do you want me to close milestone {tag_milestone.title}?", yes=yes): with Spinner(f"Closing milestone {tag_milestone.title}"): tag_milestone.edit(title=tag_milestone.title, state="closed") else: print("Not closing milestone")
def main(): p = argparse.ArgumentParser() p = def_arguments(p, gl=True) p.add_argument("--dry-run", "-s", action="store_true") p.add_argument("--verbose", "-v", action="store_true") print("Label groups:", ", ".join(get_label_groups())) args = p.parse_args() gl = gitlab(args) project = gl.projects.get("acts/acts-core") with Spinner(text="Loading tags"): tags = project.tags.list(all=True) with Spinner(text="Loading milestones"): milestones = project.milestones.list(all=True) ms_map = {} for ms in milestones: ms_map[ms.title] = ms for tag in tags: version = parse_version(tag.name) if not version in ms_map: print(f"No milestone found for tag f{tag.name} => skipping") milestone = ms_map[version] print(tag.name, milestone.title) mrs_grouped, issues_grouped = collect_milestone(milestone) if args.verbose: print("Issues:", ", ".join([str(i.iid) for i in issues])) for g, issues in issues_grouped.items(): print(g, ", ".join([str(i.iid) for i in issues])) print("MRs:", ", ".join([str(mr.iid) for mr in mrs])) for g, mrs in mrs_grouped.items(): print(g, ", ".join([str(mr.iid) for mr in mrs])) with Spinner(text="Assembling release notes"): md = make_release_notes(milestone, mrs_grouped, issues_grouped) # print(md) if not args.dry_run: with Spinner(text=f"Saving release notes on {tag.name}"): tag.set_release_description(md) # there should be a release now with Spinner(f"Getting release for tag {tag.name}"): rel = project.releases.get(tag.name) if not args.dry_run: with Spinner( f"Setting milestones on release {tag.name} to [{milestone.title}]" ): gl.http_put(f"/projects/{project.id}/releases/{tag.name}", post_data={"milestones": [milestone.title]}) # rel.milestones = [milestone.title] # rel.save() if args.verbose: print("---")