def __init__( self, root: Path, arch: Arch = Arch.host(), release_mode: bool = True, coverage: bool = False, ): self.rd = RepositoryDetails(root, arch, release_mode, coverage) self.images: Dict[str, Image] = {} self.compositions: Dict[str, Path] = {} for (path, dirs, files) in os.walk(self.root, topdown=True): # Filter out some particularly massive ignored directories to keep # things snappy. Not required for correctness. dirs[:] = set(dirs) - {".git", "target", "mzdata", "node_modules"} if "mzbuild.yml" in files: image = Image(self.rd, Path(path)) if not image.name: raise ValueError(f"config at {path} missing name") if image.name in self.images: raise ValueError(f"image {image.name} exists twice") self.images[image.name] = image if "mzcompose.yml" in files or "mzcompose.py" in files: name = Path(path).name if name in self.compositions: raise ValueError(f"composition {name} exists twice") self.compositions[name] = Path(path) # Validate dependencies. for image in self.images.values(): for d in image.depends_on: if d not in self.images: raise ValueError( f"image {image.name} depends on non-existent image {d}" )
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: """Run testdrive.""" parser.add_argument( "--redpanda", action="store_true", help="run against Redpanda instead of the Confluent Platform", ) parser.add_argument( "--aws-region", help="run against the specified AWS region instead of localstack", ) parser.add_argument( "--workers", type=int, metavar="N", help="set the number of materialized dataflow workers", ) parser.add_argument( "--persistent-user-tables", action="store_true", help="enable the --persistent-user-tables materialized option", ) parser.add_argument( "files", nargs="*", default=["*.td", "esoteric/*.td"], help="run against the specified files", ) args = parser.parse_args() if not args.redpanda and Arch.host() == Arch.AARCH64: ui.warn( "Running the Confluent Platform in Docker on ARM-based machines is " "nearly unusably slow. Consider using Redpanda instead (--redpanda) " "or running tests without mzcompose.") dependencies = ["materialized"] if args.redpanda: dependencies += ["redpanda"] else: dependencies += ["zookeeper", "kafka", "schema-registry"] materialized = Materialized( workers=args.workers, options=["--persistent-user-tables"] if args.persistent_user_tables else [], ) testdrive = Testdrive( forward_buildkite_shard=True, entrypoint_extra=[f"--aws-region={args.aws_region}"] if args.aws_region else ["--aws-endpoint=http://localstack:4566"], ) with c.override(materialized, testdrive): c.start_and_wait_for_tcp(services=dependencies) c.wait_for_materialized("materialized") c.run("testdrive-svc", *args.files) c.kill("materialized")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: """Run testdrive.""" parser.add_argument( "--redpanda", action="store_true", help="run against Redpanda instead of the Confluent Platform", ) parser.add_argument( "--aws-region", help="run against the specified AWS region instead of localstack", ) parser.add_argument( "--kafka-default-partitions", type=int, metavar="N", help="set the default number of kafka partitions per topic", ) parser.add_argument( "files", nargs="*", default=["*.td"], help="run against the specified files", ) args = parser.parse_args() if not args.redpanda and Arch.host() == Arch.AARCH64: ui.warn( "Running the Confluent Platform in Docker on ARM-based machines is " "nearly unusably slow. Consider using Redpanda instead (--redpanda) " "or running tests without mzcompose." ) dependencies = ["materialized"] if args.redpanda: dependencies += ["redpanda"] else: dependencies += ["zookeeper", "kafka", "schema-registry"] if args.aws_region is None: dependencies += ["localstack"] testdrive = Testdrive( forward_buildkite_shard=True, kafka_default_partitions=args.kafka_default_partitions, aws_region=args.aws_region, validate_postgres_stash=True, ) with c.override(testdrive): c.start_and_wait_for_tcp(services=dependencies) c.wait_for_materialized("materialized") try: junit_report = ci_util.junit_report_filename(c.name) c.run("testdrive", f"--junit-report={junit_report}", *args.files) finally: ci_util.upload_junit_report( "testdrive", Path(__file__).parent / junit_report )
def install_arguments(parser: argparse.ArgumentParser) -> None: """Install options to configure a repository into an argparse parser. This function installs the following options: * The mutually-exclusive `--dev`/`--release` options to control the `release_mode` repository attribute. * The `--coverage` booelan option to control the `coverage` repository attribute. Use `Repository.from_arguments` to construct a repository from the parsed command-line arguments. """ build_mode = parser.add_mutually_exclusive_group() build_mode.add_argument( "--dev", dest="release", action="store_false", help="build Rust binaries with the dev profile", ) build_mode.add_argument( "--release", action="store_true", help="build Rust binaries with the release profile (default)", ) parser.add_argument( "--coverage", help="whether to enable code coverage compilation flags", action="store_true", ) parser.add_argument( "--arch", default=Arch.host(), help="the CPU architecture to build for", type=Arch, choices=Arch, )