def main() -> None:
repos = [
mzbuild.Repository(Path("."), Arch.X86_64),
mzbuild.Repository(Path("."), Arch.AARCH64),
]
buildkite_tag = os.environ["BUILDKITE_TAG"]
print(f"--- Tagging Docker images")
deps = [
repo.resolve_dependencies(image for image in repo if image.publish)
for repo in repos
]
if buildkite_tag:
# On tag builds, always tag the images as such.
mzbuild.publish_multiarch_images(buildkite_tag, deps)
# Also tag the images as `latest` if this is the latest version.
version = semver.VersionInfo.parse(buildkite_tag.lstrip("v"))
latest_version = next(t for t in git.get_version_tags() if t.prerelease is None)
if version == latest_version:
mzbuild.publish_multiarch_images("latest", deps)
else:
mzbuild.publish_multiarch_images("unstable", deps)
mzbuild.publish_multiarch_images(f'unstable-{git.rev_parse("HEAD")}', deps)
def main() -> None:
repos = [
mzbuild.Repository(Path("."), Arch.X86_64),
mzbuild.Repository(Path("."), Arch.AARCH64),
]
print(f"--- Tagging development Docker images")
deps = [
repo.resolve_dependencies(image for image in repo if image.publish)
for repo in repos
]
mzbuild.publish_multiarch_images(f'devel-{git.rev_parse("HEAD")}', deps)
def generate_data(n_records: int, distribution: str) -> None:
repo = mzbuild.Repository(Path(MZ_ROOT))
deps = repo.resolve_dependencies([repo.images["kafka-avro-generator"]])
image = deps["kafka-avro-generator"]
deps.acquire()
stdout = open("gen_data.out", "a")
stderr = open("gen_data.err", "a")
spawn.runv(
args=[
"docker",
"run",
"--network",
"host",
image.spec(),
"-n",
str(n_records),
"-b",
"confluent:9093",
"-r",
"http://confluent:8081",
"-t",
"bench_data",
"-d",
distribution,
],
stdout=stdout,
stderr=stderr,
print_to=sys.stderr,
)
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if not args.file:
config_files = ["mzcompose.yml"]
else:
config_files = args.file
ui.Verbosity.init_from_env(args.mz_quiet)
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
assert_docker_compose_version()
announce("Collecting mzbuild dependencies")
composes = []
for config_file in config_files:
composes.append(build_compose_file(repo, args.command, config_file))
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
dc_args = [
"docker-compose",
*[f"-f/dev/fd/{comp.fileno()}" for comp in composes],
"--project-directory",
args.project_directory or str(Path(config_file).parent),
*unknown_args,
*([args.command] if args.command is not None else []),
*args.extra,
]
os.execvp("docker-compose", dc_args)
def main() -> None:
repo = mzbuild.Repository(Path("."))
workspace = cargo.Workspace(repo.root)
print(f"--- Publishing Debian package")
if os.environ["BUILDKITE_TAG"]:
version = workspace.crates["materialized"].version
if version.prerelease is None:
publish_deb("materialized", str(version))
else:
print(f"Detected prerelease version ({version}); skipping...")
else:
publish_deb("materialized-unstable", deb.unstable_version(workspace))
print(f"--- Tagging Docker images")
if os.environ["BUILDKITE_TAG"]:
tag_docker(repo, os.environ["BUILDKITE_TAG"])
# TODO(benesch): figure out how to push a latest tag. We want to be
# careful to not overwrite a tag for a newer release if we are building
# a historical release (e.g., don't overwrite v1.1.0 with v1.0.1).
else:
tag_docker(repo, f'unstable-{git.rev_parse("HEAD")}')
tag_docker(repo, "unstable")
print("--- Uploading binary tarball")
mz_path = Path("materialized")
ci_util.acquire_materialized(repo, mz_path)
deploy_util.deploy_tarball("x86_64-unknown-linux-gnu", mz_path)
def main() -> None:
repo = mzbuild.Repository(Path("."))
workspace = cargo.Workspace(repo.root)
# Acquire all the mzbuild images in the repository, while pushing any
# images that we build to Docker Hub, where they will be accessible to
# other build agents.
print("--- Acquiring mzbuild images")
deps = repo.resolve_dependencies(image for image in repo if image.publish)
deps.acquire()
for d in deps:
if not d.pushed():
d.push()
print("--- Staging Debian package")
if os.environ["BUILDKITE_BRANCH"] == "main":
stage_deb(repo, "materialized-unstable",
deb.unstable_version(workspace))
elif os.environ["BUILDKITE_TAG"]:
version = workspace.crates["materialized"].version
assert (
f"v{version}" == os.environ["BUILDKITE_TAG"]
), f'materialized version {version} does not match tag {os.environ["BUILDKITE_TAG"]}'
stage_deb(repo, "materialized", str(version))
elif os.environ["BUILDKITE_BRANCH"] == "master":
raise errors.MzError(
f"Tried to build branch master {git.rev_parse('HEAD')}")
else:
print("Not on main branch or tag; skipping")
def main() -> None:
repo = mzbuild.Repository(Path("."))
workspace = cargo.Workspace(repo.root)
buildkite_tag = os.environ["BUILDKITE_TAG"]
print(f"--- Publishing Debian package")
if buildkite_tag:
version = workspace.crates["materialized"].version
if version.prerelease is None:
publish_deb("materialized", str(version))
else:
print(f"Detected prerelease version ({version}); skipping...")
else:
publish_deb("materialized-unstable", deb.unstable_version(workspace))
print(f"--- Tagging Docker images")
if buildkite_tag:
tag_docker(repo, buildkite_tag)
tag_docker_latest_maybe(repo, buildkite_tag)
else:
tag_docker(repo, f'unstable-{git.rev_parse("HEAD")}')
tag_docker(repo, "unstable")
print("--- Uploading binary tarball")
mz_path = Path("materialized")
ci_util.acquire_materialized(repo, mz_path)
deploy_util.deploy_tarball("x86_64-unknown-linux-gnu", mz_path)
def launch_mz() -> None:
# We can't run bin/mzimage directly,
# because we need to pass in various flags;
# notably `--cidfile` to get the container ID.
# Otherwise, this all does the same thing as `bin/mzimage materialized`
repo = mzbuild.Repository(Path(MZ_ROOT))
deps = repo.resolve_dependencies([repo.images["materialized"]])
image = deps["materialized"]
deps.acquire()
stdout = open("build_mz.out", "a")
stderr = open("build_mz.err", "a")
spawn.runv(
args=[
"docker",
"run",
"--network",
"host",
"--cidfile",
"docker.cid",
image.spec(),
],
stdout=stdout,
stderr=stderr,
print_to=sys.stderr,
)
def main() -> None:
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
deps = repo.resolve_dependencies([repo.images[name] for name in IMAGES])
deps.acquire()
for mzimage in IMAGES:
spawn.runv(["docker", "tag", deps[mzimage].spec(), f"{REGISTRY}/{mzimage}"])
spawn.runv(["docker", "push", f"{REGISTRY}/{mzimage}"])
def acquire_images(self) -> None:
repo = mzbuild.Repository(ROOT)
for image in self.images:
deps = repo.resolve_dependencies([repo.images[image]])
deps.acquire()
for dep in deps:
subprocess.check_call(
[
"kind",
"load",
"docker-image",
dep.spec(),
]
)
def main() -> int:
args = parse_args()
ui.Verbosity.init_from_env(explicit=None)
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(
root,
release_mode=(args.build_mode == "release"),
coverage=args.coverage,
)
if args.command == "list":
for image in repo:
print(image.name)
else:
if args.image not in repo.images:
print(f"fatal: unknown image: {args.image}", file=sys.stderr)
return 1
deps = repo.resolve_dependencies([repo.images[args.image]])
rimage = deps[args.image]
if args.command == "build":
deps.acquire(force_build=True)
elif args.command == "run":
deps.acquire()
rimage.run(args.image_args)
elif args.command == "acquire":
deps.acquire()
elif args.command == "fingerprint":
print(rimage.fingerprint())
elif args.command == "spec":
print(rimage.spec())
elif args.command == "describe":
inputs = sorted(rimage.inputs(args.transitive))
dependencies = sorted(rimage.list_dependencies(args.transitive))
print(f"Image: {rimage.name}")
print(f"Fingerprint: {rimage.fingerprint()}")
print("Input files:")
for inp in inputs:
print(f" {inp}")
print("Dependencies:")
for d in dependencies:
print(f" {d}")
if not dependencies:
print(" (none)")
else:
raise RuntimeError("unreachable")
return 0
def main() -> None:
repo = mzbuild.Repository(Path("."))
workspace = cargo.Workspace(repo.root)
# Build and push any images that are not already available on Docker Hub,
# so they are accessible to other build agents.
print("--- Acquiring mzbuild images")
deps = repo.resolve_dependencies(image for image in repo if image.publish)
deps.ensure()
annotate_buildkite_with_tags(repo.rd.arch, deps)
print("--- Staging Debian package")
if os.environ["BUILDKITE_BRANCH"] == "main":
stage_deb(repo, "materialized-unstable", deb.unstable_version(workspace))
elif os.environ["BUILDKITE_TAG"]:
version = workspace.crates["materialized"].version
assert (
f"v{version}" == os.environ["BUILDKITE_TAG"]
), f'materialized version {version} does not match tag {os.environ["BUILDKITE_TAG"]}'
stage_deb(repo, "materialized", str(version))
else:
print("Not on main branch or tag; skipping")
def main() -> None:
repo = mzbuild.Repository(Path("."))
workspace = cargo.Workspace(repo.root)
buildkite_tag = os.environ["BUILDKITE_TAG"]
print(f"--- Publishing Debian package")
if buildkite_tag:
version = workspace.crates["materialized"].version
if version.prerelease is None:
publish_deb("materialized", str(version))
else:
print(f"Detected prerelease version ({version}); skipping...")
else:
publish_deb("materialized-unstable", deb.unstable_version(workspace))
print(f"--- Tagging Docker images")
deps = repo.resolve_dependencies(image for image in repo if image.publish)
deps.acquire()
if buildkite_tag:
# On tag builds, always tag the images as such.
deps.push_tagged(buildkite_tag)
# Also tag the images as `latest` if this is the latest version.
version = semver.VersionInfo.parse(buildkite_tag.lstrip("v"))
latest_version = next(t for t in git.get_version_tags()
if t.prerelease is None)
if version == latest_version:
deps.push_tagged("latest")
else:
deps.push_tagged("unstable")
print("--- Uploading binary tarball")
mz_path = Path("materialized")
ci_util.acquire_materialized(repo, mz_path)
deploy_util.deploy_tarball("x86_64-unknown-linux-gnu", mz_path)
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "list-compositions":
for name in repo.compositions:
print(name)
return 0
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
for name in composition.workflows:
print(name)
return 0
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# Check if this is a run command that names a workflow. If so, run the
# workflow instead of Docker Compose.
if args.command == "run":
workflow = composition.workflows.get(args.first_command_arg, None)
if workflow is not None:
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Check if we are being asked to list ports
elif args.command == "list-ports":
for port in composition.find_host_ports(args.first_command_arg):
print(port)
return 0
# Check if we are being asked to open a web connection to this service
elif args.command == "web":
ports = composition.find_host_ports(args.first_command_arg)
if len(ports) == 1:
webbrowser.open(f"http://localhost:{ports[0]}")
elif not ports:
raise errors.MzRuntimeError(
f"No running services matched {args.first_command_arg}")
else:
raise errors.MzRuntimeError(
f"Too many ports matched {args.first_command_arg}, found: {ports}"
)
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode
def main() -> int:
parser = argparse.ArgumentParser(
prog="mzimage",
formatter_class=argparse.RawDescriptionHelpFormatter,
description="Swiss army knife for mzbuild images.",
epilog=
"For additional help on a subcommand, run:\n\n %(prog)s <command> -h",
)
subparsers = parser.add_subparsers(dest="command", metavar="<command>")
def add_subcommand(name: str, **kwargs: Any) -> argparse.ArgumentParser:
subparser = subparsers.add_parser(name, **kwargs)
return subparser
def add_image_subcommand(name: str,
**kwargs: Any) -> argparse.ArgumentParser:
subparser = add_subcommand(name, **kwargs)
subparser.add_argument(
"image", help="the name of an mzbuild image in this repository")
return subparser
add_subcommand(
"list",
description="List all images in this repository.",
help="list all images",
)
add_image_subcommand("build",
description="Unconditionally build an image.",
help="build an image")
add_image_subcommand("acquire",
description="Download or build an image.",
help="acquire an image")
run_parser = add_image_subcommand("run",
description="Acquire and run an image.",
help="run an image")
run_parser.add_argument("image_args", nargs=argparse.REMAINDER)
add_image_subcommand(
"fingerprint",
description="Compute the fingerprint for an image.",
help="fingerprint an image",
)
add_image_subcommand(
"spec",
description="Compute the Docker Hub specification for an image.",
help="compute image spec",
)
describe_parser = add_image_subcommand(
"describe",
description="Print information about an image.",
help="show image details",
)
describe_parser.add_argument(
"--transitive",
action="store_true",
help="compute transitive inputs and dependencies",
)
describe_parser.add_argument(
"--resolve-inputs",
action="store_true",
help="resolve input patterns (by running `git ls-files`)",
)
args = parser.parse_args()
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
if args.command is None:
# TODO(benesch): we can set `required=True` in the call to
# `add_subparsers` when we upgrade to Python v3.7+.
parser.print_help(file=sys.stderr)
return 1
elif args.command == "list":
for image in repo:
print(image.name)
else:
if args.image not in repo.images:
print(f"fatal: unknown image: {args.image}", file=sys.stderr)
return 1
deps = repo.resolve_dependencies([repo.images[args.image]])
rimage = deps[args.image]
if args.command == "build":
deps.acquire(force_build=True)
elif args.command == "run":
deps.acquire()
rimage.run(args.image_args)
elif args.command == "acquire":
deps.acquire()
elif args.command == "fingerprint":
print(rimage.fingerprint())
elif args.command == "spec":
print(rimage.spec())
elif args.command == "describe":
inputs = sorted(rimage.inputs(args.transitive))
dependencies = sorted(rimage.list_dependencies(args.transitive))
print(f"Image: {rimage.name}")
print(f"Fingerprint: {rimage.fingerprint()}")
print("Input patterns:")
for inp in inputs:
print(f" {inp}")
if args.resolve_inputs:
print("Resolved inputs:")
for path in sorted(mzbuild.ls_files(root, *inputs)):
print(f" {path.decode()}")
print("Dependencies:")
for d in dependencies:
print(f" {d}")
if not dependencies:
print(" (none)")
else:
raise RuntimeError("unreachable")
return 0
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if args.file:
raise errors.MzConfigurationError("-f/--file option not supported")
elif args.project_directory:
raise errors.MzConfigurationError(
"--project-directory option not supported")
ui.Verbosity.init_from_env(args.mz_quiet)
# Load repository.
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root,
release_mode=(args.mz_build_mode == "release"))
# Handle special mzcompose commands that apply to the repo.
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
elif args.command == "lint":
return lint(repo)
elif args.command == "list-compositions":
return list_compositions(repo)
# Load composition.
try:
composition = mzcompose.Composition(repo, args.mz_find
or Path.cwd().name)
except errors.UnknownComposition:
if args.mz_find:
print(f"unknown composition {args.mz_find!r}", file=sys.stderr)
print("hint: available compositions:", file=sys.stderr)
for name in repo.compositions:
print(f" {name}", file=sys.stderr)
else:
print("error: directory does not contain mzcompose.yml",
file=sys.stderr)
print(
"hint: enter one of the following directories and run ./mzcompose:",
file=sys.stderr,
)
for path in repo.compositions.values():
print(f" {path.relative_to(Path.cwd())}", file=sys.stderr)
return 1
# Handle special mzcompose commands that apply to the composition.
if args.command == "list-workflows":
return list_workflows(composition)
elif args.command == "list-ports":
return list_ports(composition, args.first_command_arg)
elif args.command == "web":
return web(composition, args.first_command_arg)
# From here on out we're definitely invoking Docker Compose, so make sure
# it's new enough.
output = spawn.capture(["docker-compose", "version", "--short"],
unicode=True).strip()
version = tuple(int(i) for i in output.split("."))
if version < MIN_COMPOSE_VERSION:
msg = f"Unsupported docker-compose version: {version}, min required: {MIN_COMPOSE_VERSION}"
raise errors.MzConfigurationError(msg)
announce("Collecting mzbuild dependencies")
deps = repo.resolve_dependencies(composition.images)
for d in deps:
say(d.spec())
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# The `run` command requires special handling.
if args.command == "run":
try:
workflow = composition.get_workflow(dict(os.environ),
args.first_command_arg)
except KeyError:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
composition.run([
"up",
"-d",
"--scale",
f"{args.first_command_arg}=0",
args.first_command_arg,
])
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.remainder:
raise errors.MzRuntimeError(
f"cannot specify extra arguments ({' '.join(args.remainder)}) "
"when specifying a workflow (rather than a container)")
workflow.run()
return 0
# Hand over control to Docker Compose.
announce("Delegating to Docker Compose")
proc = composition.run(
[
*unknown_args,
*([args.command] if args.command is not None else []),
*([args.first_command_arg]
if args.first_command_arg is not None else []),
*args.remainder,
],
check=False,
)
return proc.returncode
def trim_pipeline(pipeline: Any) -> None:
"""Trim pipeline steps whose inputs have not changed in this branch.
Steps are assigned inputs in two ways:
1. An explicit glob in the `inputs` key.
2. An implicit dependency on any number of mzbuild images via the
mzcompose plugin. Any steps which use the mzcompose plugin will
have inputs autodiscovered based on the images used in that
mzcompose configuration.
A step is trimmed if a) none of its inputs have changed, and b) there are
no other untrimmed steps that depend on it.
"""
repo = mzbuild.Repository(Path("."))
steps = OrderedDict()
for config in pipeline["steps"]:
if "wait" in config:
continue
step = PipelineStep(config["id"])
if "inputs" in config:
for inp in config["inputs"]:
step.extra_inputs.add(inp)
if "depends_on" in config:
d = config["depends_on"]
if isinstance(d, str):
step.step_dependencies.add(d)
elif isinstance(d, list):
step.step_dependencies.update(d)
else:
raise ValueError(
f"unexpected non-str non-list for depends_on: {d}")
if "plugins" in config:
for plugin in config["plugins"]:
for plugin_name, plugin_config in plugin.items():
if plugin_name == "./ci/plugins/mzcompose":
name = plugin_config["composition"]
composition = mzcompose.Composition(repo, name)
for dep in composition.dependencies:
step.image_dependencies.add(dep)
step.extra_inputs.add(str(repo.compositions[name]))
steps[step.id] = step
# Find all the steps whose inputs have changed with respect to main.
# We delegate this hard work to Git.
changed = set()
for step in steps.values():
inputs = step.inputs()
if not inputs:
# No inputs means there is no way this step can be considered
# changed, but `git diff` with no pathspecs means "diff everything",
# not "diff nothing", so explicitly skip.
continue
if have_paths_changed(inputs):
changed.add(step.id)
# Then collect all changed steps, and all the steps that those changed steps
# depend on.
needed = set()
def visit(step: PipelineStep) -> None:
if step.id not in needed:
needed.add(step.id)
for d in step.step_dependencies:
visit(steps[d])
for step_id in changed:
visit(steps[step_id])
# Print decisions, for debugging.
for step in steps.values():
print(f'{"✓" if step.id in needed else "✗"} {step.id}')
if step.step_dependencies:
print(" wait:", " ".join(step.step_dependencies))
if step.extra_inputs:
print(" globs:", " ".join(step.extra_inputs))
if step.image_dependencies:
print(" images:",
" ".join(image.name for image in step.image_dependencies))
# Restrict the pipeline to the needed steps.
pipeline["steps"] = [
step for step in pipeline["steps"]
if "wait" in step or step["id"] in needed
]
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"--confluent-host",
default="confluent",
help="The hostname of a machine running the Confluent Platform",
)
parser.add_argument(
"-n",
"--trials",
default=1,
type=int,
help="Number of measurements to take",
)
parser.add_argument(
"-r",
"--records",
default=1000000,
type=int,
help="Number of Avro records to generate",
)
args = parser.parse_args()
os.chdir(ROOT)
repo = mzbuild.Repository(ROOT)
wait_for_confluent(args.confluent_host)
images = ["kgen", "materialized"]
deps = repo.resolve_dependencies([repo.images[name] for name in images])
deps.acquire()
docker_client = docker.from_env()
mz_container = docker_client.containers.run(
deps["materialized"].spec(),
detach=True,
network_mode="host",
)
docker_client.containers.run(
deps["kgen"].spec(),
[
f"--num-records={args.records}",
f"--bootstrap-server={args.confluent_host}:9092",
f"--schema-registry-url=http://{args.confluent_host}:8081",
"--topic=bench_data",
"--keys=avro",
"--values=avro",
f"--avro-schema={VALUE_SCHEMA}",
f"--avro-distribution={VALUE_DISTRIBUTION}",
f"--avro-key-schema={KEY_SCHEMA}",
f"--avro-key-distribution={KEY_DISTRIBUTION}",
],
network_mode="host",
)
conn = pg8000.connect(host="localhost", port=6875, user="******")
conn.autocommit = True
cur = conn.cursor()
cur.execute(
f"""CREATE CONNECTION IF NOT EXISTS csr_conn
FOR CONFLUENT SCHEMA REGISTRY
URL 'http://{args.confluent_host}:8081'"""
)
cur.execute(
f"""CREATE SOURCE src
FROM KAFKA BROKER '{args.confluent_host}:9092' TOPIC 'bench_data'
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn"""
)
results_file = open("results.csv", "w")
print("Rss,Vms,User Cpu,System Cpu,Wall Time", file=results_file, flush=True)
prev = PrevStats(time.time(), 0.0, 0.0)
for _ in range(args.trials):
cur.execute("DROP VIEW IF EXISTS cnt")
cur.execute("CREATE MATERIALIZED VIEW cnt AS SELECT count(*) FROM src")
while True:
try:
cur.execute("SELECT * FROM cnt")
n = cur.fetchone()[0]
if n >= args.records:
break
except ProgrammingError:
pass
time.sleep(1)
prev = print_stats(mz_container, prev, results_file)
def image(self, service: str) -> str:
repo = mzbuild.Repository(ROOT)
deps = repo.resolve_dependencies([repo.images[service]])
rimage = deps[service]
return rimage.spec()
def main(argv: List[str]) -> int:
# Lightly parse the arguments so we know what to do.
args, unknown_args = ArgumentParser().parse_known_args(argv)
if not args.file:
config_file = "mzcompose.yml"
elif len(args.file) > 1:
print(
"mzcompose: multiple -f/--file options are not yet supported",
file=sys.stderr,
)
return 1
else:
config_file = args.file[0]
def say(s: str) -> None:
if not args.mz_quiet:
print(s)
root = Path(os.environ["MZ_ROOT"])
repo = mzbuild.Repository(root)
if args.command == "gen-shortcuts":
return gen_shortcuts(repo)
# Determine what images this particular compose file depends upon.
say("==> Collecting mzbuild dependencies")
images = []
with open(config_file) as f:
compose = yaml.safe_load(f)
# strip mzconduct top-level key, if it exists
compose.pop("mzconduct", None)
for config in compose["services"].values():
if "mzbuild" in config:
image_name = config["mzbuild"]
if image_name not in repo.images:
print(
f"mzcompose: unknown image {image_name}",
file=sys.stderr,
)
return 1
image = repo.images[image_name]
override_tag = os.environ.get(
f"MZBUILD_{image.env_var_name()}_TAG", None)
if override_tag is not None:
config["image"] = image.docker_name(override_tag)
print(
f"mzcompose: warning: overriding {image_name} image to tag {override_tag}",
file=sys.stderr,
)
del config["mzbuild"]
else:
images.append(image)
if "propagate-uid-gid" in config:
config["user"] = f"{os.getuid()}:{os.getgid()}"
del config["propagate-uid-gid"]
deps = repo.resolve_dependencies(images)
for d in deps:
say(d.spec())
for config in compose["services"].values():
if "mzbuild" in config:
config["image"] = deps[config["mzbuild"]].spec()
del config["mzbuild"]
# Check if the command is going to create or start containers, and if so
# build the dependencies. This can be slow, so we don't want to do it if we
# can help it (e.g., for `down` or `ps`).
if args.command in ["create", "run", "start", "up"]:
deps.acquire()
# Construct a configuration that will point Docker Compose at the correct
# images.
tempfile = TemporaryFile()
os.set_inheritable(tempfile.fileno(), True)
yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore
tempfile.flush()
tempfile.seek(0)
# Hand over control to Docker Compose.
say("==> Delegating to Docker Compose")
dc_args = [
"docker-compose",
"-f",
f"/dev/fd/{tempfile.fileno()}",
"--project-directory",
args.project_directory or str(Path(config_file).parent),
*unknown_args,
*([args.command] if args.command is not None else []),
*args.extra,
]
os.execvp("docker-compose", dc_args)
def trim_pipeline(pipeline: Any) -> None:
"""Trim pipeline steps whose inputs have not changed in this branch.
Steps are assigned inputs in two ways:
1. An explicit glob in the `inputs` key.
2. An implicit dependency on any number of mzbuild images via the
mzcompose plugin. Any steps which use the mzcompose plugin will
have inputs autodiscovered based on the images used in that
mzcompose configuration.
A step is trimmed if a) none of its inputs have changed, and b) there are
no other untrimmed steps that depend on it.
"""
repo = mzbuild.Repository(Path("."))
images = repo.resolve_dependencies(image for image in repo)
steps = OrderedDict()
for config in pipeline["steps"]:
step = PipelineStep(config["id"])
if "inputs" in config:
for inp in config["inputs"]:
step.extra_inputs.add(inp)
if "depends_on" in config:
d = config["depends_on"]
if isinstance(d, str):
step.step_dependencies.add(d)
elif isinstance(d, list):
step.step_dependencies.update(d)
else:
raise ValueError(
f"unexpected non-str non-list for depends_on: {d}")
if "plugins" in config:
for plugin in config["plugins"]:
for name, plugin_config in plugin.items():
if name == "./ci/plugins/mzcompose":
step.image_dependencies.update(
find_compose_images(images,
plugin_config["config"]))
step.extra_inputs.add(plugin_config["config"])
elif name == "./ci/plugins/mzconduct":
comp = Composition.find(plugin_config["test"]).path
config = comp / "mzcompose.yml"
step.image_dependencies.update(
find_compose_images(images, config))
step.extra_inputs.add(str(config))
steps[step.id] = step
# Make sure we have an up to date view of master.
spawn.runv(["git", "fetch", "origin", "master"])
# Print out a summary of all changes.
os.environ["GIT_PAGER"] = ""
spawn.runv(["git", "diff", "--stat", "origin/master..."])
# Find all the steps whose inputs have changed with respect to master.
# We delegate this hard work to Git.
changed = set()
for step in steps.values():
inputs = step.inputs()
if not inputs:
# No inputs means there is no way this step can be considered
# changed, but `git diff` with no pathspecs means "diff everything",
# not "diff nothing", so explicitly skip.
continue
diff = subprocess.run([
"git", "diff", "--no-patch", "--quiet", "origin/master...", "--",
*inputs
])
if diff.returncode != 0:
changed.add(step.id)
# Then collect all changed steps, and all the steps that those changed steps
# depend on.
needed = set()
def visit(step: PipelineStep) -> None:
if step.id not in needed:
needed.add(step.id)
for d in step.step_dependencies:
visit(steps[d])
for step_id in changed:
visit(steps[step_id])
# Print decisions, for debugging.
for step in steps.values():
print(f'{"✓" if step.id in needed else "✗"} {step.id}')
if step.step_dependencies:
print(" wait:", " ".join(step.step_dependencies))
if step.extra_inputs:
print(" globs:", " ".join(step.extra_inputs))
if step.image_dependencies:
print(" images:",
" ".join(image.name for image in step.image_dependencies))
# Restrict the pipeline to the needed steps.
pipeline["steps"] = [
step for step in pipeline["steps"] if step["id"] in needed
]
