def test_cluster_compute_storage(c: Composition, *glob: str) -> None:
c.up("dataflowd_storage")
c.up("dataflowd_compute_1")
c.up("dataflowd_compute_2")
c.up("materialized_compute_storage")
c.wait_for_materialized(service="materialized_compute_storage")
c.run("testdrive-svc-compute-storage", *glob)
def workflow_user_tables(
c: Composition, args_or_parser: Union[WorkflowArgumentParser,
Namespace]) -> None:
start_deps(c, args_or_parser)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive",
f"--seed={seed}",
f"user-tables/table-persistence-before-{td_test}.td",
)
c.kill("materialized")
c.up("materialized")
c.kill("materialized")
c.up("materialized")
c.run(
"testdrive",
f"--seed={seed}",
f"user-tables/table-persistence-after-{td_test}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def test_testdrive(c: Composition, mz: Materialized, aws: str, tests: str) -> None:
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", mz.name]
)
c.wait_for_materialized(mz.name)
c.run("testdrive-svc", aws, tests)
c.kill(mz.name)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
parser.add_argument("--message-count", type=int, default=1000)
parser.add_argument("--partitions", type=int, default=1)
parser.add_argument("--check-sink", action="store_true")
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
args = parser.parse_args()
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
c.start_and_wait_for_tcp(services=dependencies)
c.run(
"billing-demo",
"--materialized-host=materialized",
"--kafka-host=kafka",
"--schema-registry-url=http://schema-registry:8081",
"--create-topic",
"--replication-factor=1",
f"--message-count={args.message_count}",
f"--partitions={args.partitions}",
*(["--check-sink"] if args.check_sink else []),
)
def workflow_test_upsert(c: Composition) -> None:
"""Test creating upsert sources and continuing to ingest them after a restart."""
with c.override(
Testdrive(default_timeout="30s",
no_reset=True,
consistent_seed=True), ):
c.down(destroy_volumes=True)
dependencies = [
"materialized",
"zookeeper",
"kafka",
"schema-registry",
]
c.start_and_wait_for_tcp(services=dependencies, )
c.run("testdrive", "upsert/01-create-sources.td")
# Sleep to make sure the errors have made it to persist.
# This isn't necessary for correctness,
# as we should be able to crash at any point and re-start.
# But if we don't sleep here, then we might be ingesting the errored
# records in the new process, and so we won't actually be testing
# the ability to retract error values that make it to persist.
print("Sleeping for ten seconds")
time.sleep(10)
c.exec("materialized", "bash", "-c", "kill -9 `pidof storaged`")
c.run("testdrive", "upsert/02-after-storaged-restart.td")
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(["zookeeper", "kafka", "schema-registry"])
c.run("ci-cargo-test", "run-tests")
token = os.environ["BUILDKITE_TEST_ANALYTICS_API_KEY_CARGO_TEST"]
if len(token) < 1:
print("Analytics API key empty, skipping junit reporting")
return
with open(f"{ROOT.as_posix()}/results.json") as f:
junit_xml = spawn.capture(args=["cargo2junit"], stdin=f.read())
requests.post(
"https://analytics-api.buildkite.com/v1/uploads",
headers={"Authorization": f"Token {token}"},
json={
"format": "junit",
"run_env": {
"key": os.environ["BUILDKITE_BUILD_ID"],
"CI": "buildkite",
"number": os.environ["BUILDKITE_BUILD_NUMBER"],
"job_id": os.environ["BUILDKITE_JOB_ID"],
"branch": os.environ["BUILDKITE_BRANCH"],
"commit_sha": os.environ["BUILDKITE_COMMIT"],
"message": os.environ["BUILDKITE_MESSAGE"],
"url": os.environ["BUILDKITE_BUILD_URL"],
},
"data": junit_xml,
},
)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized", "toxiproxy"
])
c.wait_for_materialized()
seed = random.getrandbits(16)
for i, failure_mode in enumerate([
"toxiproxy-close-connection.td",
"toxiproxy-timeout.td",
]):
c.run(
"testdrive-svc",
"--no-reset",
"--max-errors=1",
f"--seed={seed}{i}",
f"--temp-dir=/share/tmp/kafka-resumption-{seed}{i}",
"setup.td",
failure_mode,
"during.td",
"sleep.td",
"toxiproxy-restore-connection.td",
"verify-success.td",
"cleanup.td",
)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Runs the dbt adapter test suite against Materialize in various configurations."""
parser.add_argument(
"filter", nargs="?", default="", help="limit to test cases matching filter"
)
args = parser.parse_args()
for test_case in test_cases:
if args.filter in test_case.name:
print(f"> Running test case {test_case.name}")
materialized = Materialized(
options=test_case.materialized_options,
image=test_case.materialized_image,
depends_on=["test-certs"],
volumes=["secrets:/secrets"],
)
with c.override(materialized):
c.up("materialized")
c.wait_for_tcp(host="materialized", port=6875)
c.run(
"dbt-test",
"pytest",
"dbt-materialize/test",
env_extra=test_case.dbt_env,
)
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(services=["localstack"])
for version in CONFLUENT_PLATFORM_VERSIONS:
print(f"==> Testing Confluent Platform {version}")
confluent_platform_services = [
Zookeeper(tag=version),
Kafka(tag=version),
SchemaRegistry(tag=version),
]
with c.override(*confluent_platform_services):
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized"
])
c.wait_for_materialized()
c.run("testdrive", "kafka-matrix.td", "testdrive/kafka-*.td")
c.kill(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
)
c.rm(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
"testdrive",
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
def test_proxy(c: Composition, aws: str) -> None:
for test in tests:
mz: Materialized = test["mz"]
c.up(mz.name)
c.wait_for_materialized(mz.name)
c.run("testdrive-svc", aws, test["td"])
c.kill(mz.name)
def workflow_default(c: Composition) -> None:
c.up("test-certs")
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry"])
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive", "*.td")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
parser.add_argument(
"--seed",
help="an alternate seed to use to avoid clashing with existing topics",
type=int,
default=1,
)
args = parser.parse_args()
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "materialized"])
c.run(
"testdrive",
f"--seed={args.seed}",
"--kafka-option=group.id=group1",
"before-restart.td",
)
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive",
f"--seed={args.seed}",
"--no-reset",
"--kafka-option=group.id=group2",
"after-restart.td",
)
def restart_mz_during_replication(c: Composition) -> None:
c.run("testdrive", "wait-for-snapshot.td", "delete-rows-t1.td",
"alter-table.td")
restart_mz(c)
c.run("testdrive", "delete-rows-t2.td")
def workflow_kafka_sources(c: Composition) -> None:
seed = round(time.time())
c.start_and_wait_for_tcp(services=prerequisites)
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", f"--seed={seed}",
f"kafka-sources/*{td_test}*-before.td")
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
# And restart again, for extra stress
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", f"--seed={seed}",
f"kafka-sources/*{td_test}*-after.td")
# Do one more restart, just in case and just confirm that Mz is able to come up
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_disable_user_indexes(c: Composition) -> None:
seed = round(time.time())
c.start_and_wait_for_tcp(services=prerequisites)
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", "disable-user-indexes/before.td")
c.kill("materialized")
with c.override(
Materialized(options=f"{mz_options} --disable-user-indexes", )):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}",
"disable-user-indexes/after.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def run_one_failpoint(c: Composition, failpoint: str, action: str) -> None:
print(
f">>> Running failpoint test for failpoint {failpoint} with action {action}"
)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive-svc",
f"--seed={seed}",
f"--var=failpoint={failpoint}",
f"--var=action={action}",
"failpoints/before.td",
)
time.sleep(2)
# kill Mz if the failpoint has not killed it
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", "failpoints/after.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_kafka_sources(
c: Composition, args_or_parser: Union[WorkflowArgumentParser,
Namespace]) -> None:
start_deps(c, args_or_parser)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", f"--seed={seed}",
f"kafka-sources/*{td_test}*-before.td")
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
# And restart again, for extra stress
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", f"--seed={seed}", f"kafka-sources/*{td_test}*-after.td")
# Do one more restart, just in case and just confirm that Mz is able to come up
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run the proxy tests."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
args = parser.parse_args()
dependencies = ["squid"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
if not args.aws_region:
dependencies += ["localstack"]
c.start_and_wait_for_tcp(dependencies)
aws_arg = (f"--aws-region={args.aws_region}"
if args.aws_region else "--aws-endpoint=http://localstack:4566")
for test_case in test_cases:
print(f"Running test case {test_case.name!r}")
with c.override(Materialized(environment_extra=test_case.env)):
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", aws_arg, *test_case.files)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run testdrive."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
parser.add_argument(
"--workers",
type=int,
metavar="N",
help="set the number of materialized dataflow workers",
)
parser.add_argument(
"--persistent-user-tables",
action="store_true",
help="enable the --persistent-user-tables materialized option",
)
parser.add_argument(
"files",
nargs="*",
default=["*.td", "esoteric/*.td"],
help="run against the specified files",
)
args = parser.parse_args()
if not args.redpanda and Arch.host() == Arch.AARCH64:
ui.warn(
"Running the Confluent Platform in Docker on ARM-based machines is "
"nearly unusably slow. Consider using Redpanda instead (--redpanda) "
"or running tests without mzcompose.")
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
materialized = Materialized(
workers=args.workers,
options=["--persistent-user-tables"]
if args.persistent_user_tables else [],
)
testdrive = Testdrive(
forward_buildkite_shard=True,
entrypoint_extra=[f"--aws-region={args.aws_region}"]
if args.aws_region else ["--aws-endpoint=http://localstack:4566"],
)
with c.override(materialized, testdrive):
c.start_and_wait_for_tcp(services=dependencies)
c.wait_for_materialized("materialized")
c.run("testdrive-svc", *args.files)
c.kill("materialized")
def run_sqllogictest(c: Composition, command: str) -> None:
c.up("postgres")
c.wait_for_postgres(dbname="postgres")
try:
junit_report = ci_util.junit_report_filename(c.name)
c.run("sqllogictest-svc", command, f"--junit-report={junit_report}")
finally:
ci_util.upload_junit_report(c.name, ROOT / junit_report)
def disconnect_pg_during_snapshot(c: Composition) -> None:
c.run(
"testdrive-svc",
"toxiproxy-close-connection.td",
"toxiproxy-restore-connection.td",
"delete-rows-t1.td",
"delete-rows-t2.td",
)
def workflow_testdrive(c: Composition) -> None:
c.start_and_wait_for_tcp(services=[
"zookeeper",
"kafka",
"schema-registry",
"materialized",
])
c.run("testdrive-svc", tests)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run testdrive."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
parser.add_argument(
"--kafka-default-partitions",
type=int,
metavar="N",
help="set the default number of kafka partitions per topic",
)
parser.add_argument(
"files",
nargs="*",
default=["*.td"],
help="run against the specified files",
)
args = parser.parse_args()
if not args.redpanda and Arch.host() == Arch.AARCH64:
ui.warn(
"Running the Confluent Platform in Docker on ARM-based machines is "
"nearly unusably slow. Consider using Redpanda instead (--redpanda) "
"or running tests without mzcompose."
)
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
if args.aws_region is None:
dependencies += ["localstack"]
testdrive = Testdrive(
forward_buildkite_shard=True,
kafka_default_partitions=args.kafka_default_partitions,
aws_region=args.aws_region,
validate_postgres_stash=True,
)
with c.override(testdrive):
c.start_and_wait_for_tcp(services=dependencies)
c.wait_for_materialized("materialized")
try:
junit_report = ci_util.junit_report_filename(c.name)
c.run("testdrive", f"--junit-report={junit_report}", *args.files)
finally:
ci_util.upload_junit_report(
"testdrive", Path(__file__).parent / junit_report
)
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(
services=["localstack", "materialized", "toxiproxy"])
c.wait_for_materialized()
# For different values of bytes_allowed, the following happens:
# 0 - Connection is dropped immediately
# 1K - SQS queue and bucket listing are both prevented
# 2K - SQS and key fetching are both prevented
# 10K - only key fetching is prevented
for toxiproxy_bytes_allowed in [
0,
256,
512,
768,
1024,
1536,
2 * 1024,
3 * 1024,
5 * 1024,
10 * 1024,
20 * 1024,
]:
# For small values of toxiproxy_bytes_allowed, we need to allow for CREATE SOURCE to go undisturbed first, otherwise it fails immediately
toxiproxy_setup = (
["configure-materialize.td", "toxiproxy-close-connection.td"]
if toxiproxy_bytes_allowed < 1024 else
["toxiproxy-close-connection.td", "configure-materialize.td"])
c.run(
"testdrive",
"--no-reset",
"--max-errors=1",
f"--seed={toxiproxy_bytes_allowed}",
"--aws-endpoint=http://toxiproxy:4566",
f"--var=toxiproxy-bytes-allowed={toxiproxy_bytes_allowed}",
"configure-toxiproxy.td",
"s3-create.td",
"s3-insert-long.td",
"s3-insert-long-gzip.td",
#
# Confirm that short network interruptions are tolerated
#
*toxiproxy_setup,
"short-sleep.td",
"toxiproxy-restore-connection.td",
"materialize-verify-success.td",
#
# Confirm that long network interruptions cause source error
# Disabled due to https://github.com/MaterializeInc/materialize/issues/7009
# "s3-insert-long.td s3-insert-long-gzip.td toxiproxy-close-connection.td materialize-verify-failure.td",
#
# Cleanup
#
"materialize-drop-source.td",
"toxiproxy-remove.td",
)
def test_upgrade_from_version(
c: Composition, from_version: str, priors: List[str], filter: str, style: str = ""
) -> None:
print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.")
version_glob = "{" + ",".join(["any_version", *priors, from_version]) + "}"
print(">>> Version glob pattern: " + version_glob)
c.rm("materialized", "testdrive-svc", stop=True)
c.rm_volumes("mzdata", "tmp")
if from_version != "current_source":
mz_from = Materialized(
image=f"materialize/materialized:{from_version}",
options=" ".join(
opt
for start_version, opt in mz_options.items()
if from_version[1:] >= start_version
),
environment=[
"SSL_KEY_PASSWORD=mzmzmz",
],
volumes_extra=["secrets:/share/secrets"],
)
with c.override(mz_from):
c.up("materialized")
else:
c.up("materialized")
c.wait_for_materialized("materialized")
temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}"
seed = f"--seed={random.getrandbits(32)}"
c.run(
"testdrive-svc",
"--no-reset",
f"--var=upgrade-from-version={from_version}",
temp_dir,
seed,
f"create-{style}in-{version_glob}-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run(
"testdrive-svc",
"--no-reset",
f"--var=upgrade-from-version={from_version}",
temp_dir,
seed,
"--validate-catalog=/share/mzdata/catalog",
f"check-{style}from-{version_glob}-{filter}.td",
)
def workflow_versioned_mz(c: Composition) -> None:
for mz in versioned_mz:
c.up(mz.name)
c.wait_for_materialized(mz.name)
c.run("testdrive-svc", "test*.td")
c.kill(mz.name)
def begin(c: Composition) -> None:
"""Configure Toxiproxy and Mz and populate initial data"""
c.run(
"testdrive-svc",
"configure-toxiproxy.td",
"populate-tables.td",
"configure-materalize.td",
)
def workflow_postgres(c: Composition) -> None:
c.start_and_wait_for_tcp(services=prerequisites)
c.start_and_wait_for_tcp(services=["postgres"])
c.wait_for_postgres(service="postgres")
c.wait_for_materialized("materialized")
c.run("testdrive", "postgres/debezium-postgres.td.initialize")
c.run("testdrive", "postgres/*.td")
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "materialized"])
with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp:
with contextlib.redirect_stdout(tmp):
[cls.generate() for cls in Generator.__subclasses__()]
sys.stdout.flush()
c.run("testdrive-svc", os.path.basename(tmp.name))
def run_test(c: Composition, materialized: str, env: Dict[str, str]) -> None:
c.up(materialized)
c.wait_for_tcp(host=materialized, port=6875)
c.run(
"dbt-test",
"pytest",
"dbt-materialize/test",
env=env,
)
