def run(cls, args: argparse.Namespace) -> None:
repo = mzbuild.Repository.from_arguments(ROOT, args)
for name, path in sorted(repo.compositions.items(), key=lambda item: item[1]):
print(os.path.relpath(path, repo.root))
composition = mzcompose.Composition(repo, name, munge_services=False)
if composition.description:
# Emit the first paragraph of the description.
for line in composition.description.split("\n"):
if line.strip() == "":
break
print(f" {line}")
def load_composition(args: argparse.Namespace) -> mzcompose.Composition:
"""Loads the composition specified by the command-line arguments."""
repo = mzbuild.Repository.from_arguments(ROOT, args)
try:
return mzcompose.Composition(repo,
name=args.find or Path.cwd().name,
preserve_ports=args.preserve_ports)
except mzcompose.UnknownCompositionError as e:
if args.find:
hint = "available compositions:\n"
for name in repo.compositions:
hint += f" {name}\n"
e.set_hint(hint)
raise e
else:
hint = "enter one of the following directories and run ./mzcompose:\n"
for path in repo.compositions.values():
hint += f" {path.relative_to(Path.cwd())}\n"
raise UIError(
"directory does not contain an mzcompose.yml or mzcompose.py",
hint,
)
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root,
release_mode=(args.mz_build_mode == "release"))
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "lint":
return lint(repo)
elif args.command == "list-compositions":
return list_compositions(repo)
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
return list_workflows(composition)
elif args.command == "list-ports":
return list_ports(composition, args.first_command_arg)
elif args.command == "web":
return web(composition, args.first_command_arg)
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# The `run` command requires special handling.
if args.command == "run":
try:
workflow = composition.get_workflow(dict(os.environ),
args.first_command_arg)
except KeyError:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
composition.run([
"up",
"-d",
"--scale",
f"{args.first_command_arg}=0",
args.first_command_arg,
])
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode
def trim_pipeline(pipeline: Any) -> None:
"""Trim pipeline steps whose inputs have not changed in this branch.
Steps are assigned inputs in two ways:
1. An explicit glob in the `inputs` key.
2. An implicit dependency on any number of mzbuild images via the
mzcompose plugin. Any steps which use the mzcompose plugin will
have inputs autodiscovered based on the images used in that
mzcompose configuration.
A step is trimmed if a) none of its inputs have changed, and b) there are
no other untrimmed steps that depend on it.
"""
repo = mzbuild.Repository(Path("."))
steps = OrderedDict()
for config in pipeline["steps"]:
if "wait" in config:
continue
step = PipelineStep(config["id"])
if "inputs" in config:
for inp in config["inputs"]:
step.extra_inputs.add(inp)
if "depends_on" in config:
d = config["depends_on"]
if isinstance(d, str):
step.step_dependencies.add(d)
elif isinstance(d, list):
step.step_dependencies.update(d)
else:
raise ValueError(
f"unexpected non-str non-list for depends_on: {d}")
if "plugins" in config:
for plugin in config["plugins"]:
for plugin_name, plugin_config in plugin.items():
if plugin_name == "./ci/plugins/mzcompose":
name = plugin_config["composition"]
composition = mzcompose.Composition(repo, name)
for dep in composition.dependencies:
step.image_dependencies.add(dep)
step.extra_inputs.add(str(repo.compositions[name]))
steps[step.id] = step
# Find all the steps whose inputs have changed with respect to main.
# We delegate this hard work to Git.
changed = set()
for step in steps.values():
inputs = step.inputs()
if not inputs:
# No inputs means there is no way this step can be considered
# changed, but `git diff` with no pathspecs means "diff everything",
# not "diff nothing", so explicitly skip.
continue
if have_paths_changed(inputs):
changed.add(step.id)
# Then collect all changed steps, and all the steps that those changed steps
# depend on.
needed = set()
def visit(step: PipelineStep) -> None:
if step.id not in needed:
needed.add(step.id)
for d in step.step_dependencies:
visit(steps[d])
for step_id in changed:
visit(steps[step_id])
# Print decisions, for debugging.
for step in steps.values():
print(f'{"✓" if step.id in needed else "✗"} {step.id}')
if step.step_dependencies:
print(" wait:", " ".join(step.step_dependencies))
if step.extra_inputs:
print(" globs:", " ".join(step.extra_inputs))
if step.image_dependencies:
print(" images:",
" ".join(image.name for image in step.image_dependencies))
# Restrict the pipeline to the needed steps.
pipeline["steps"] = [
step for step in pipeline["steps"]
if "wait" in step or step["id"] in needed
]
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "list-compositions":
for name in repo.compositions:
print(name)
return 0
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
for name in composition.workflows:
print(name)
return 0
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# Check if this is a run command that names a workflow. If so, run the
# workflow instead of Docker Compose.
if args.command == "run":
workflow = composition.workflows.get(args.first_command_arg, None)
if workflow is not None:
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Check if we are being asked to list ports
elif args.command == "list-ports":
for port in composition.find_host_ports(args.first_command_arg):
print(port)
return 0
# Check if we are being asked to open a web connection to this service
elif args.command == "web":
ports = composition.find_host_ports(args.first_command_arg)
if len(ports) == 1:
webbrowser.open(f"http://localhost:{ports[0]}")
elif not ports:
raise errors.MzRuntimeError(
f"No running services matched {args.first_command_arg}")
else:
raise errors.MzRuntimeError(
f"Too many ports matched {args.first_command_arg}, found: {ports}"
)
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode