def new_rc(
create_branch: Optional[str],
checkout: Optional[str],
affect_remote: bool,
level: str,
) -> None:
"""Start a brand new release
\b
Arguments:
level Which part of the version to change:
* weekly - The Z in X.Y.Z
* feature - The Y in X.Y.Z
* major - The X in X.Y.Z
* rc - increases the N in -rcN, should only be used if
you need to create a second or greater release candidate
"""
tag = get_latest_tag(fetch=True)
new_version = None
if level == "rc":
if tag.prerelease is None or not tag.prerelease.startswith("rc"):
raise errors.MzConfigurationError(
"Attempted to bump an rc version without starting an RC"
)
next_rc = int(tag.prerelease[2:]) + 1
new_version = tag.replace(prerelease=f"rc{next_rc}")
elif level == "weekly":
new_version = tag.bump_patch().replace(prerelease="rc1")
elif level == "feature":
new_version = tag.bump_minor().replace(prerelease="rc1")
elif level == "major":
new_version = tag.bump_major().replace(prerelease="rc1")
assert new_version is not None
release(new_version, checkout, create_branch, True, affect_remote)
def assert_docker_compose_version() -> None:
"""Check the version of docker-compose installed.
Raises `MzRuntimeError` if the version is not recent enough.
"""
cmd = ["docker-compose", "version", "--short"]
output = spawn.capture(cmd, unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
def change_line(fname: str, line_start: str, replacement: str) -> None:
with open(fname, "r") as fh:
content = fh.read().splitlines()
changes = 0
for i, line in enumerate(content):
if line.startswith(line_start):
content[i] = replacement
changes += 1
with open(fname, "w") as fh:
fh.write("\n".join(content))
fh.write("\n")
if changes != 1:
raise errors.MzConfigurationError(f"Found {changes} {line_start}s in {fname}")
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root,
release_mode=(args.mz_build_mode == "release"))
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "lint":
return lint(repo)
elif args.command == "list-compositions":
return list_compositions(repo)
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
return list_workflows(composition)
elif args.command == "list-ports":
return list_ports(composition, args.first_command_arg)
elif args.command == "web":
return web(composition, args.first_command_arg)
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# The `run` command requires special handling.
if args.command == "run":
try:
workflow = composition.get_workflow(dict(os.environ),
args.first_command_arg)
except KeyError:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
composition.run([
"up",
"-d",
"--scale",
f"{args.first_command_arg}=0",
args.first_command_arg,
])
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode
def main(version: str, checkout: Optional[str], create_branch: str,
tag: bool) -> None:
"""Update documents for a release and create tags
If both `-b` and `-c` are specified, the checkout happens before the branch creation,
meaning that the new branch is created on the target of `-c`.
For example make release::
mkrelease -b prepare-v0.1.2 -c v0.1.1-rc1 v0.1.2-dev
Has the same git semantics as::
git checkout -b prepare-v0.1.2 v0.1.1-rc1
\b
Arguments:
version: The version to release. The `v` prefix is optional
"""
if git.is_dirty():
raise errors.MzConfigurationError(
"working directory is not clean, stash or commit your changes")
sys.exit(1)
version = version.lstrip("v")
the_tag = f"v{version}"
confirm_version_is_next(version)
if git.is_dirty():
raise errors.MzConfigurationError(
"working directory is not clean, stash or commit your changes")
sys.exit(1)
if checkout is not None:
git.checkout(checkout)
if create_branch is not None:
git.create_branch(create_branch)
change_line(BIN_CARGO_TOML, "version", f'version = "{version}"')
change_line(
LICENSE,
"Licensed Work:",
f"Licensed Work: Materialize Version {version}",
)
# Don't update the change date unless some code has changed
if "-rc" in version or "-dev" in version:
future = four_years_hence()
change_line(LICENSE, "Change Date",
f"Change Date: {future}")
say("Updating Cargo.lock")
spawn.runv(["cargo", "check", "-p", "materialized"])
if tag:
git.commit_all_changed(f"release: {the_tag}")
git.tag_annotated(the_tag)
else:
git.commit_all_changed(f"Prepare next phase of development: {the_tag}")
matching = git.first_remote_matching("MaterializeInc/materialize")
if tag:
if matching is not None:
if ui.confirm(
f"\nWould you like me to run: git push {matching} {the_tag}"
):
spawn.runv(["git", "push", matching, the_tag])
else:
say("")
say(f"Next step is to push {the_tag} to the MaterializeInc/materialize repo"
)
else:
branch = git.rev_parse("HEAD", abbrev=True)
say("")
say(f"Create a PR with your branch: '{branch}'")
def list_prs(recent_ref: Optional[str], ancestor_ref: Optional[str]) -> None:
"""
List PRs between a range of refs
If no refs are specified, then this will find the refs between the most
recent tag and the previous semver tag (i.e. excluding RCs)
"""
git.fetch()
if recent_ref is None or ancestor_ref is None:
tags = git.get_version_tags(fetch=False)
if recent_ref is None:
recent = tags[0]
recent_ref = str(tags[0])
else:
recent = Version.parse(recent_ref)
if ancestor_ref is None:
for ref in tags[1:]:
ancestor = ref
if (
ancestor.major < recent.major
or ancestor.minor < recent.minor
or ancestor.patch < recent.patch
):
ancestor_ref = str(ref)
break
say(
f"Using recent_ref={recent_ref} ancestor_ref={ancestor_ref}",
)
commit_range = f"v{ancestor_ref}..v{recent_ref}"
commits = spawn.capture(
[
"git",
"log",
"--pretty=format:%d %s",
"--abbrev-commit",
"--date=iso",
commit_range,
"--",
],
unicode=True,
)
pattern = re.compile(r"^\s*\(refs/pullreqs/(\d+)|\(#(\d+)")
prs = []
found_ref = False
for commit in commits.splitlines():
if "build(deps)" in commit:
continue
match = pattern.search(commit)
if match is not None:
pr = match.group(1)
if pr:
found_ref = True
else:
pr = match.group(2)
prs.append(pr)
if not found_ref:
say(
"WARNING: you probably don't have pullreqs configured for your repo",
)
say(
"Add the following line to the MaterializeInc/materialize remote section in your .git/config",
)
say(" fetch = +refs/pull/*/head:refs/pullreqs/*")
username = input("Enter your github username: "******"~/.config/materialize/dev-tools-access-token")
try:
with open(creds_path) as fh:
token = fh.read().strip()
except FileNotFoundError:
raise errors.MzConfigurationError(
f"""No developer tool api token at {creds_path!r}
please create an access token at https://github.com/settings/tokens"""
)
def get(pr: str) -> Any:
return requests.get(
f"https://{username}:{token}@api.github.com/repos/MaterializeInc/materialize/pulls/{pr}",
headers={
"Accept": "application/vnd.github.v3+json",
},
).json()
collected = []
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
futures = {pool.submit(get, pr): pr for pr in prs}
for future in concurrent.futures.as_completed(futures):
pr = futures[future]
contents = future.result()
try:
url = contents["html_url"]
title = contents["title"]
collected.append((url, title))
except KeyError:
raise errors.MzRuntimeError(contents)
for url, title in sorted(collected):
print(url, title)
def release(
version: Version,
checkout: Optional[str],
create_branch: Optional[str],
tag: bool,
affect_remote: bool,
) -> None:
"""Update documents for a release and create tags
If both `-b` and `-c` are specified, the checkout happens before the branch creation,
meaning that the new branch is created on the target of `-c`.
For example make release::
mkrelease -b prepare-v0.1.2 -c v0.1.1-rc1 v0.1.2-dev
Has the same git semantics as::
git checkout -b prepare-v0.1.2 v0.1.1-rc1
\b
Arguments:
version: The version to release. The `v` prefix is optional
"""
if git.is_dirty():
raise errors.MzConfigurationError(
"working directory is not clean, stash or commit your changes"
)
the_tag = f"v{version}"
confirm_version_is_next(version, affect_remote)
if checkout is not None:
git.checkout(checkout)
if create_branch is not None:
git.create_branch(create_branch)
confirm_on_latest_rc(affect_remote)
change_line(BIN_CARGO_TOML, "version", f'version = "{version}"')
change_line(
LICENSE,
"Licensed Work:",
f"Licensed Work: Materialize Version {version}",
)
# Don't update the change date unless some code has changed
if version.prerelease:
future = four_years_hence()
change_line(LICENSE, "Change Date", f"Change Date: {future}")
say("Updating Cargo.lock")
spawn.runv(["cargo", "check", "-p", "materialized"])
spawn.runv(["cargo", "check", "-p", "materialized"])
spawn.runv(["cargo", "check", "-p", "materialized", "--locked"])
if tag:
git.commit_all_changed(f"release: {the_tag}")
git.tag_annotated(the_tag)
else:
git.commit_all_changed(f"Prepare next phase of development: {the_tag}")
latest_tag = get_latest_tag(fetch=False)
# we have made an actual release
if latest_tag.prerelease is None and click.confirm(
f"Update doc/user/config.toml marking v{latest_tag} as released"
):
update_versions_list(latest_tag)
git.commit_all_changed(f"Update released versions to include v{latest_tag}")
matching = git.first_remote_matching("MaterializeInc/materialize")
if tag:
if matching is not None:
spawn.runv(["git", "show", "HEAD"])
if affect_remote and ui.confirm(
f"\nWould you like to push the above changes as: git push {matching} {the_tag}"
):
spawn.runv(["git", "push", matching, the_tag])
else:
say("")
say(
f"Next step is to push {the_tag} to the MaterializeInc/materialize repo"
)
else:
branch = git.rev_parse("HEAD", abbrev=True)
say("")
say(f"Create a PR with your branch: '{branch}'")
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "list-compositions":
for name in repo.compositions:
print(name)
return 0
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
for name in composition.workflows:
print(name)
return 0
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# Check if this is a run command that names a workflow. If so, run the
# workflow instead of Docker Compose.
if args.command == "run":
workflow = composition.workflows.get(args.first_command_arg, None)
if workflow is not None:
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Check if we are being asked to list ports
elif args.command == "list-ports":
for port in composition.find_host_ports(args.first_command_arg):
print(port)
return 0
# Check if we are being asked to open a web connection to this service
elif args.command == "web":
ports = composition.find_host_ports(args.first_command_arg)
if len(ports) == 1:
webbrowser.open(f"http://localhost:{ports[0]}")
elif not ports:
raise errors.MzRuntimeError(
f"No running services matched {args.first_command_arg}")
else:
raise errors.MzRuntimeError(
f"Too many ports matched {args.first_command_arg}, found: {ports}"
)
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode