def web(composition: mzcompose.Composition, service: Optional[str]) -> int:
if service is None:
raise errors.MzRuntimeError(f"web command requires a service argument")
ports = composition.find_host_ports(service)
if len(ports) == 1:
webbrowser.open(f"http://localhost:{ports[0]}")
elif not ports:
raise errors.MzRuntimeError(f"No running services matched {service!r}")
else:
raise errors.MzRuntimeError(
f"Too many ports matched {service!r}, found: {ports}")
return 0
def list_ports(composition: mzcompose.Composition,
service: Optional[str]) -> int:
if service is None:
raise errors.MzRuntimeError(
f"list-ports command requires a service argument")
for port in composition.find_host_ports(service):
print(port)
return 0
def update_versions_list(released_version: Version) -> None:
"""Update the doc config with the passed-in version"""
today = date.today().strftime("%d %B %Y")
toml_line = f' {{ name = "v{released_version}", date = "{today}" }},\n'
with open(USER_DOC_CONFIG) as fh:
docs = fh.readlines()
wrote_line = False
with open(USER_DOC_CONFIG, "w") as fh:
for line in docs:
fh.write(line)
if line == "versions = [\n":
fh.write(toml_line)
wrote_line = True
if not wrote_line:
raise errors.MzRuntimeError("Couldn't determine where to insert new version")
def rev_parse(rev: str, *, abbrev: bool = False) -> str:
"""Compute the hash for a revision.
Args:
rev: A Git revision in any format known to the Git CLI.
abbrev: Return a branch or tag name instead of a git sha
Returns:
ref: A 40 character hex-encoded SHA-1 hash representing the ID of the
named revision in Git's object database.
With "abbrev=True" this will return an abbreviated ref, or throw an
error if there is no abbrev.
"""
a = ["--abbrev-ref"] if abbrev else []
out = spawn.capture(["git", "rev-parse", *a, "--verify", rev], unicode=True).strip()
if not out:
raise errors.MzRuntimeError(f"No parsed rev for {rev}")
return out
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root,
release_mode=(args.mz_build_mode == "release"))
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "lint":
return lint(repo)
elif args.command == "list-compositions":
return list_compositions(repo)
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
return list_workflows(composition)
elif args.command == "list-ports":
return list_ports(composition, args.first_command_arg)
elif args.command == "web":
return web(composition, args.first_command_arg)
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# The `run` command requires special handling.
if args.command == "run":
try:
workflow = composition.get_workflow(dict(os.environ),
args.first_command_arg)
except KeyError:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
composition.run([
"up",
"-d",
"--scale",
f"{args.first_command_arg}=0",
args.first_command_arg,
])
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode
def list_prs(recent_ref: Optional[str], ancestor_ref: Optional[str]) -> None:
"""
List PRs between a range of refs
If no refs are specified, then this will find the refs between the most
recent tag and the previous semver tag (i.e. excluding RCs)
"""
git.fetch()
if recent_ref is None or ancestor_ref is None:
tags = git.get_version_tags(fetch=False)
if recent_ref is None:
recent = tags[0]
recent_ref = str(tags[0])
else:
recent = Version.parse(recent_ref)
if ancestor_ref is None:
for ref in tags[1:]:
ancestor = ref
if (
ancestor.major < recent.major
or ancestor.minor < recent.minor
or ancestor.patch < recent.patch
):
ancestor_ref = str(ref)
break
say(
f"Using recent_ref={recent_ref} ancestor_ref={ancestor_ref}",
)
commit_range = f"v{ancestor_ref}..v{recent_ref}"
commits = spawn.capture(
[
"git",
"log",
"--pretty=format:%d %s",
"--abbrev-commit",
"--date=iso",
commit_range,
"--",
],
unicode=True,
)
pattern = re.compile(r"^\s*\(refs/pullreqs/(\d+)|\(#(\d+)")
prs = []
found_ref = False
for commit in commits.splitlines():
if "build(deps)" in commit:
continue
match = pattern.search(commit)
if match is not None:
pr = match.group(1)
if pr:
found_ref = True
else:
pr = match.group(2)
prs.append(pr)
if not found_ref:
say(
"WARNING: you probably don't have pullreqs configured for your repo",
)
say(
"Add the following line to the MaterializeInc/materialize remote section in your .git/config",
)
say(" fetch = +refs/pull/*/head:refs/pullreqs/*")
username = input("Enter your github username: "******"~/.config/materialize/dev-tools-access-token")
try:
with open(creds_path) as fh:
token = fh.read().strip()
except FileNotFoundError:
raise errors.MzConfigurationError(
f"""No developer tool api token at {creds_path!r}
please create an access token at https://github.com/settings/tokens"""
)
def get(pr: str) -> Any:
return requests.get(
f"https://{username}:{token}@api.github.com/repos/MaterializeInc/materialize/pulls/{pr}",
headers={
"Accept": "application/vnd.github.v3+json",
},
).json()
collected = []
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
futures = {pool.submit(get, pr): pr for pr in prs}
for future in concurrent.futures.as_completed(futures):
pr = futures[future]
contents = future.result()
try:
url = contents["html_url"]
title = contents["title"]
collected.append((url, title))
except KeyError:
raise errors.MzRuntimeError(contents)
for url, title in sorted(collected):
print(url, title)
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "list-compositions":
for name in repo.compositions:
print(name)
return 0
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
for name in composition.workflows:
print(name)
return 0
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# Check if this is a run command that names a workflow. If so, run the
# workflow instead of Docker Compose.
if args.command == "run":
workflow = composition.workflows.get(args.first_command_arg, None)
if workflow is not None:
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Check if we are being asked to list ports
elif args.command == "list-ports":
for port in composition.find_host_ports(args.first_command_arg):
print(port)
return 0
# Check if we are being asked to open a web connection to this service
elif args.command == "web":
ports = composition.find_host_ports(args.first_command_arg)
if len(ports) == 1:
webbrowser.open(f"http://localhost:{ports[0]}")
elif not ports:
raise errors.MzRuntimeError(
f"No running services matched {args.first_command_arg}")
else:
raise errors.MzRuntimeError(
f"Too many ports matched {args.first_command_arg}, found: {ports}"
)
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode
