def __init__(
self,
root: Path,
arch: Arch = Arch.host(),
release_mode: bool = True,
coverage: bool = False,
):
self.rd = RepositoryDetails(root, arch, release_mode, coverage)
self.images: Dict[str, Image] = {}
self.compositions: Dict[str, Path] = {}
for (path, dirs, files) in os.walk(self.root, topdown=True):
# Filter out some particularly massive ignored directories to keep
# things snappy. Not required for correctness.
dirs[:] = set(dirs) - {".git", "target", "mzdata", "node_modules"}
if "mzbuild.yml" in files:
image = Image(self.rd, Path(path))
if not image.name:
raise ValueError(f"config at {path} missing name")
if image.name in self.images:
raise ValueError(f"image {image.name} exists twice")
self.images[image.name] = image
if "mzcompose.yml" in files or "mzcompose.py" in files:
name = Path(path).name
if name in self.compositions:
raise ValueError(f"composition {name} exists twice")
self.compositions[name] = Path(path)
# Validate dependencies.
for image in self.images.values():
for d in image.depends_on:
if d not in self.images:
raise ValueError(
f"image {image.name} depends on non-existent image {d}"
)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run testdrive."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
parser.add_argument(
"--workers",
type=int,
metavar="N",
help="set the number of materialized dataflow workers",
)
parser.add_argument(
"--persistent-user-tables",
action="store_true",
help="enable the --persistent-user-tables materialized option",
)
parser.add_argument(
"files",
nargs="*",
default=["*.td", "esoteric/*.td"],
help="run against the specified files",
)
args = parser.parse_args()
if not args.redpanda and Arch.host() == Arch.AARCH64:
ui.warn(
"Running the Confluent Platform in Docker on ARM-based machines is "
"nearly unusably slow. Consider using Redpanda instead (--redpanda) "
"or running tests without mzcompose.")
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
materialized = Materialized(
workers=args.workers,
options=["--persistent-user-tables"]
if args.persistent_user_tables else [],
)
testdrive = Testdrive(
forward_buildkite_shard=True,
entrypoint_extra=[f"--aws-region={args.aws_region}"]
if args.aws_region else ["--aws-endpoint=http://localstack:4566"],
)
with c.override(materialized, testdrive):
c.start_and_wait_for_tcp(services=dependencies)
c.wait_for_materialized("materialized")
c.run("testdrive-svc", *args.files)
c.kill("materialized")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run testdrive."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
parser.add_argument(
"--kafka-default-partitions",
type=int,
metavar="N",
help="set the default number of kafka partitions per topic",
)
parser.add_argument(
"files",
nargs="*",
default=["*.td"],
help="run against the specified files",
)
args = parser.parse_args()
if not args.redpanda and Arch.host() == Arch.AARCH64:
ui.warn(
"Running the Confluent Platform in Docker on ARM-based machines is "
"nearly unusably slow. Consider using Redpanda instead (--redpanda) "
"or running tests without mzcompose."
)
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
if args.aws_region is None:
dependencies += ["localstack"]
testdrive = Testdrive(
forward_buildkite_shard=True,
kafka_default_partitions=args.kafka_default_partitions,
aws_region=args.aws_region,
validate_postgres_stash=True,
)
with c.override(testdrive):
c.start_and_wait_for_tcp(services=dependencies)
c.wait_for_materialized("materialized")
try:
junit_report = ci_util.junit_report_filename(c.name)
c.run("testdrive", f"--junit-report={junit_report}", *args.files)
finally:
ci_util.upload_junit_report(
"testdrive", Path(__file__).parent / junit_report
)
def install_arguments(parser: argparse.ArgumentParser) -> None:
"""Install options to configure a repository into an argparse parser.
This function installs the following options:
* The mutually-exclusive `--dev`/`--release` options to control the
`release_mode` repository attribute.
* The `--coverage` booelan option to control the `coverage` repository
attribute.
Use `Repository.from_arguments` to construct a repository from the
parsed command-line arguments.
"""
build_mode = parser.add_mutually_exclusive_group()
build_mode.add_argument(
"--dev",
dest="release",
action="store_false",
help="build Rust binaries with the dev profile",
)
build_mode.add_argument(
"--release",
action="store_true",
help="build Rust binaries with the release profile (default)",
)
parser.add_argument(
"--coverage",
help="whether to enable code coverage compilation flags",
action="store_true",
)
parser.add_argument(
"--arch",
default=Arch.host(),
help="the CPU architecture to build for",
type=Arch,
choices=Arch,
)