def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(services=["localstack"])
for version in CONFLUENT_PLATFORM_VERSIONS:
print(f"==> Testing Confluent Platform {version}")
confluent_platform_services = [
Zookeeper(tag=version),
Kafka(tag=version),
SchemaRegistry(tag=version),
]
with c.override(*confluent_platform_services):
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized"
])
c.wait_for_materialized()
c.run("testdrive", "kafka-matrix.td", "testdrive/kafka-*.td")
c.kill(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
)
c.rm(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
"testdrive",
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
def test_testdrive(c: Composition, mz: Materialized, aws: str, tests: str) -> None:
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", mz.name]
)
c.wait_for_materialized(mz.name)
c.run("testdrive-svc", aws, tests)
c.kill(mz.name)
def test_cluster(c: Composition, *glob: str) -> None:
c.up("materialized")
c.wait_for_materialized()
# Create a remote cluster and verify that tests pass.
c.up("computed_1")
c.up("computed_2")
c.sql("DROP CLUSTER IF EXISTS cluster1 CASCADE;")
c.sql(
"CREATE CLUSTER cluster1 REPLICAS (replica1 (REMOTE ['computed_1:2100', 'computed_2:2100']));"
)
c.run("testdrive", *glob)
# Add a replica to that remote cluster and verify that tests still pass.
c.up("computed_3")
c.up("computed_4")
c.sql(
"CREATE CLUSTER REPLICA cluster1.replica2 REMOTE ['computed_3:2100', 'computed_4:2100']"
)
c.run("testdrive", *glob)
# Kill one of the nodes in the first replica of the compute cluster and
# verify that tests still pass.
c.kill("computed_1")
c.run("testdrive", *glob)
# Leave only replica 2 up and verify that tests still pass.
c.sql("DROP CLUSTER REPLICA cluster1.replica1")
c.run("testdrive", *glob)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized", "toxiproxy"
])
c.wait_for_materialized()
seed = random.getrandbits(16)
for i, failure_mode in enumerate([
"toxiproxy-close-connection.td",
"toxiproxy-limit-connection.td",
"toxiproxy-timeout.td",
# TODO: Enable https://github.com/MaterializeInc/materialize/issues/11085
# "toxiproxy-timeout-hold.td",
]):
c.start_and_wait_for_tcp(["toxiproxy"])
c.run(
"testdrive-svc",
"--no-reset",
"--max-errors=1",
f"--seed={seed}{i}",
f"--temp-dir=/share/tmp/kafka-resumption-{seed}{i}",
"setup.td",
failure_mode,
"during.td",
"sleep.td",
"toxiproxy-restore-connection.td",
"verify-success.td",
"cleanup.td",
)
c.kill("toxiproxy")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
parser.add_argument(
"--seed",
help="an alternate seed to use to avoid clashing with existing topics",
type=int,
default=1,
)
args = parser.parse_args()
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "materialized"])
c.run(
"testdrive",
f"--seed={args.seed}",
"--kafka-option=group.id=group1",
"before-restart.td",
)
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive",
f"--seed={args.seed}",
"--no-reset",
"--kafka-option=group.id=group2",
"after-restart.td",
)
def test_proxy(c: Composition, aws: str) -> None:
for test in tests:
mz: Materialized = test["mz"]
c.up(mz.name)
c.wait_for_materialized(mz.name)
c.run("testdrive-svc", aws, test["td"])
c.kill(mz.name)
def workflow_disable_user_indexes(c: Composition) -> None:
seed = round(time.time())
c.start_and_wait_for_tcp(services=prerequisites)
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", "disable-user-indexes/before.td")
c.kill("materialized")
with c.override(
Materialized(options=f"{mz_options} --disable-user-indexes", )):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}",
"disable-user-indexes/after.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def run_one_failpoint(c: Composition, failpoint: str, action: str) -> None:
print(
f">>> Running failpoint test for failpoint {failpoint} with action {action}"
)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive-svc",
f"--seed={seed}",
f"--var=failpoint={failpoint}",
f"--var=action={action}",
"failpoints/before.td",
)
time.sleep(2)
# kill Mz if the failpoint has not killed it
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", "failpoints/after.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run testdrive."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
parser.add_argument(
"--workers",
type=int,
metavar="N",
help="set the number of materialized dataflow workers",
)
parser.add_argument(
"--persistent-user-tables",
action="store_true",
help="enable the --persistent-user-tables materialized option",
)
parser.add_argument(
"files",
nargs="*",
default=["*.td", "esoteric/*.td"],
help="run against the specified files",
)
args = parser.parse_args()
if not args.redpanda and Arch.host() == Arch.AARCH64:
ui.warn(
"Running the Confluent Platform in Docker on ARM-based machines is "
"nearly unusably slow. Consider using Redpanda instead (--redpanda) "
"or running tests without mzcompose.")
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
materialized = Materialized(
workers=args.workers,
options=["--persistent-user-tables"]
if args.persistent_user_tables else [],
)
testdrive = Testdrive(
forward_buildkite_shard=True,
entrypoint_extra=[f"--aws-region={args.aws_region}"]
if args.aws_region else ["--aws-endpoint=http://localstack:4566"],
)
with c.override(materialized, testdrive):
c.start_and_wait_for_tcp(services=dependencies)
c.wait_for_materialized("materialized")
c.run("testdrive-svc", *args.files)
c.kill("materialized")
def workflow_mz_with_options(c: Composition) -> None:
c.up("mz_2_workers")
c.wait_for_materialized("mz_2_workers")
c.kill("mz_2_workers")
c.up("mz_4_workers")
c.wait_for_materialized("mz_4_workers")
c.kill("mz_4_workers")
def test_upgrade_from_version(
c: Composition, from_version: str, priors: List[str], filter: str, style: str = ""
) -> None:
print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.")
version_glob = "{" + ",".join(["any_version", *priors, from_version]) + "}"
print(">>> Version glob pattern: " + version_glob)
c.rm("materialized", "testdrive-svc", stop=True)
c.rm_volumes("mzdata", "tmp")
if from_version != "current_source":
mz_from = Materialized(
image=f"materialize/materialized:{from_version}",
options=" ".join(
opt
for start_version, opt in mz_options.items()
if from_version[1:] >= start_version
),
environment=[
"SSL_KEY_PASSWORD=mzmzmz",
],
volumes_extra=["secrets:/share/secrets"],
)
with c.override(mz_from):
c.up("materialized")
else:
c.up("materialized")
c.wait_for_materialized("materialized")
temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}"
seed = f"--seed={random.getrandbits(32)}"
c.run(
"testdrive-svc",
"--no-reset",
f"--var=upgrade-from-version={from_version}",
temp_dir,
seed,
f"create-{style}in-{version_glob}-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run(
"testdrive-svc",
"--no-reset",
f"--var=upgrade-from-version={from_version}",
temp_dir,
seed,
"--validate-catalog=/share/mzdata/catalog",
f"check-{style}from-{version_glob}-{filter}.td",
)
def workflow_smoke_test(c: Composition) -> None:
c.workflow(
"default",
"--num-seconds=15",
"--records-per-second=1000",
)
c.kill("materialized")
c.rm("materialized", "testdrive", "kafka", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_versioned_mz(c: Composition) -> None:
for mz in versioned_mz:
c.up(mz.name)
c.wait_for_materialized(mz.name)
c.run("testdrive-svc", "test*.td")
c.kill(mz.name)
def run_test(c: Composition, disruption: Disruption, id: int) -> None:
print(f"+++ Running disruption scenario {disruption.name}")
c.up("testdrive", persistent=True)
nodes = [
Computed(
name="computed_1_1",
peers=["computed_1_1", "computed_1_2"],
),
Computed(
name="computed_1_2",
peers=["computed_1_1", "computed_1_2"],
),
Computed(
name="computed_2_1",
peers=["computed_2_1", "computed_2_2"],
),
Computed(
name="computed_2_2",
peers=["computed_2_1", "computed_2_2"],
),
]
with c.override(*nodes):
c.up("materialized", *[n.name for n in nodes])
c.wait_for_materialized()
c.sql(
"""
CREATE CLUSTER cluster1 REPLICAS (
replica1 (REMOTE ['computed_1_1:2100', 'computed_1_2:2100']),
replica2 (REMOTE ['computed_2_1:2100', 'computed_2_2:2100'])
)
"""
)
with c.override(
Testdrive(
validate_data_dir=False,
no_reset=True,
materialize_params={"cluster": "cluster1"},
seed=id,
)
):
populate(c)
# Disrupt replica1 by some means
disruption.disruption(c)
validate(c)
cleanup_list = ["materialized", "testdrive", *[n.name for n in nodes]]
c.kill(*cleanup_list)
c.rm(*cleanup_list, destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_github_8021(c: Composition) -> None:
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", "github-8021.td")
# Ensure MZ can boot
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
def run_one_scenario(
c: Composition, scenario: Type[Scenario], args: argparse.Namespace
) -> Comparator:
name = scenario.__name__
print(f"--- Now benchmarking {name} ...")
comparator = make_comparator(name)
common_seed = round(time.time())
mzs = {
"this": Materialized(
image=f"materialize/materialized:{args.this_tag}"
if args.this_tag
else None,
options=args.this_options,
),
"other": Materialized(
image=f"materialize/materialized:{args.other_tag}"
if args.other_tag
else None,
options=args.other_options,
),
}
for mz_id, instance in enumerate(["this", "other"]):
with c.override(mzs[instance]):
print(f"The version of the '{instance.upper()}' Mz instance is:")
c.run("materialized", "--version")
c.start_and_wait_for_tcp(services=["materialized"])
c.wait_for_materialized()
executor = Docker(
composition=c,
seed=common_seed,
)
benchmark = Benchmark(
mz_id=mz_id,
scenario=scenario,
scale=args.scale,
executor=executor,
filter=make_filter(args),
termination_conditions=make_termination_conditions(args),
aggregation=make_aggregation(),
)
outcome, iterations = benchmark.run()
comparator.append(outcome)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.rm_volumes("mzdata")
return comparator
def workflow_failpoints(c: Composition) -> None:
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", f"failpoints/{td_test}.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_smoke_test(c: Composition) -> None:
for arg in ["--upsert", "--enable-persistence"]:
c.workflow(
"default",
"--num-seconds=15",
"--records-per-second=1000",
arg,
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc", "kafka", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_compaction(c: Composition) -> None:
with c.override(mz_fast_metrics):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", "compaction/compaction.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_compaction(c: Composition) -> None:
with c.override(Materialized(options=f"--metrics-scraping-interval=1s", )):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive", "compaction/compaction.td")
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def test_upgrade_from_version(
c: Composition, from_version: str, priors: List[str], filter: str
) -> None:
print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.")
version_glob = "|".join(["any_version", *priors, from_version])
print(">>> Version glob pattern: " + version_glob)
if from_version != "current_source":
mz_from = Materialized(
image=f"materialize/materialized:{from_version}",
options=" ".join(
opt
for start_version, opt in mz_options.items()
if from_version[1:] >= start_version
),
)
with c.override(mz_from):
c.up("materialized")
else:
c.up("materialized")
c.wait_for_materialized("materialized")
temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}"
with patch.dict(os.environ, {"UPGRADE_FROM_VERSION": from_version}):
c.run(
"testdrive-svc",
"--seed=1",
"--no-reset",
temp_dir,
f"create-in-@({version_glob})-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.up("materialized")
c.wait_for_materialized("materialized")
with patch.dict(os.environ, {"UPGRADE_FROM_VERSION": from_version}):
c.run(
"testdrive-svc",
"--seed=1",
"--no-reset",
temp_dir,
f"--validate-catalog=/share/mzdata/catalog check-from-@({version_glob})-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.rm_volumes("mzdata", "tmp")
def workflow_kafka_sources(
c: Composition, args_or_parser: Union[WorkflowArgumentParser,
Namespace]) -> None:
start_deps(c, args_or_parser)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", f"--seed={seed}",
f"kafka-sources/*{td_test}*-before.td")
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
# And restart again, for extra stress
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", f"--seed={seed}", f"kafka-sources/*{td_test}*-after.td")
# Do one more restart, just in case and just confirm that Mz is able to come up
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_kafka_sources(c: Composition) -> None:
seed = round(time.time())
c.start_and_wait_for_tcp(services=prerequisites)
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", f"--seed={seed}",
f"kafka-sources/*{td_test}*-before.td")
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
# And restart again, for extra stress
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", f"--seed={seed}",
f"kafka-sources/*{td_test}*-after.td")
# Do one more restart, just in case and just confirm that Mz is able to come up
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_test_builtin_migration(c: Composition) -> None:
"""Exercise the builtin object migration code by upgrading between two versions
that will have a migration triggered between them. Create a materialized view
over the affected builtin object to confirm that the migration was successful
"""
c.down(destroy_volumes=True)
with c.override(
# Random commit before pg_roles was updated.
Materialized(
image=
"materialize/materialized:devel-9efd269199b1510b3e8f90196cb4fa3072a548a1",
),
Testdrive(default_timeout="15s",
no_reset=True,
consistent_seed=True),
):
c.up("testdrive", persistent=True)
c.up("materialized")
c.wait_for_materialized()
c.testdrive(input=dedent("""
> CREATE VIEW v1 AS SELECT COUNT(*) FROM pg_roles;
> SELECT * FROM v1;
2
! SELECT DISTINCT rolconnlimit FROM pg_roles;
contains:column "rolconnlimit" does not exist
"""))
c.kill("materialized")
with c.override(
# This will stop working if we introduce a breaking change.
Materialized(),
Testdrive(default_timeout="15s",
no_reset=True,
consistent_seed=True),
):
c.up("testdrive", persistent=True)
c.up("materialized")
c.wait_for_materialized()
c.testdrive(input=dedent("""
> SELECT * FROM v1;
2
# This column is new after the migration
> SELECT DISTINCT rolconnlimit FROM pg_roles;
-1
"""))
def workflow_disable_user_indexes(c: Composition) -> None:
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry"])
# Create catalog with vanilla MZ
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", "user-indexes-enabled.td")
c.kill("materialized")
# Test semantics of disabling user indexes
c.up("mz_disable_user_indexes")
c.wait_for_materialized("mz_disable_user_indexes")
c.run("testdrive_no_reset", "user-indexes-disabled.td")
c.kill("mz_disable_user_indexes")
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(services=[
"zookeeper",
"kafka1",
"kafka2",
"kafka3",
"schema-registry",
"materialized",
])
c.run("testdrive", "--kafka-addr=kafka2", "01-init.td")
time.sleep(10)
c.kill("kafka1")
time.sleep(10)
c.run("testdrive", "--kafka-addr=kafka2,kafka3", "--no-reset",
"02-after-leave.td")
c.up("kafka1")
time.sleep(10)
c.run("testdrive", "--kafka-addr=kafka1", "--no-reset", "03-after-join.td")
def workflow_test_cluster(c: Composition,
parser: WorkflowArgumentParser) -> None:
"""Run testdrive in a variety of compute cluster configurations."""
parser.add_argument(
"glob",
nargs="*",
default=["smoke/*.td"],
help="run against the specified files",
)
args = parser.parse_args()
c.down(destroy_volumes=True)
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "localstack"])
c.up("materialized")
c.wait_for_materialized()
# Create a remote cluster and verify that tests pass.
c.up("computed_1")
c.up("computed_2")
c.sql("DROP CLUSTER IF EXISTS cluster1 CASCADE;")
c.sql(
"CREATE CLUSTER cluster1 REPLICAS (replica1 (REMOTE ['computed_1:2100', 'computed_2:2100']));"
)
c.run("testdrive", *args.glob)
# Add a replica to that remote cluster and verify that tests still pass.
c.up("computed_3")
c.up("computed_4")
c.sql(
"CREATE CLUSTER REPLICA cluster1.replica2 REMOTE ['computed_3:2100', 'computed_4:2100']"
)
c.run("testdrive", *args.glob)
# Kill one of the nodes in the first replica of the compute cluster and
# verify that tests still pass.
c.kill("computed_1")
c.run("testdrive", *args.glob)
# Leave only replica 2 up and verify that tests still pass.
c.sql("DROP CLUSTER REPLICA cluster1.replica1")
c.run("testdrive", *args.glob)
def workflow_test_github_13603(c: Composition) -> None:
"""Test that multi woker replicas terminate eagerly upon rehydration"""
c.down(destroy_volumes=True)
c.up("materialized")
c.wait_for_materialized()
c.up("computed_1")
c.up("computed_2")
c.sql(
"CREATE CLUSTER cluster1 REPLICAS (replica1 (REMOTE ['computed_1:2100', 'computed_2:2100']));"
)
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
# Ensure the computeds have crashed
c1 = c.invoke("logs", "computed_1", capture=True)
assert "panicked" in c1.stdout
c2 = c.invoke("logs", "computed_2", capture=True)
assert "panicked" in c2.stdout
def workflow_audit_log(c: Composition) -> None:
c.up("materialized")
c.wait_for_materialized(service="materialized")
# Create some audit log entries.
c.sql("CREATE TABLE t (i INT)")
c.sql("CREATE DEFAULT INDEX ON t")
log = c.sql_query("SELECT * FROM mz_audit_events ORDER BY id")
# Restart mz.
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
# Verify the audit log entries are still present and have not changed.
restart_log = c.sql_query("SELECT * FROM mz_audit_events ORDER BY id")
if log != restart_log:
print("initial audit log:", log)
print("audit log after restart:", restart_log)
raise Exception("audit logs not equal after restart")
def workflow_user_tables(
c: Composition, args_or_parser: Union[WorkflowArgumentParser,
Namespace]) -> None:
start_deps(c, args_or_parser)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive",
f"--seed={seed}",
f"user-tables/table-persistence-before-{td_test}.td",
)
c.kill("materialized")
c.up("materialized")
c.kill("materialized")
c.up("materialized")
c.run(
"testdrive",
f"--seed={seed}",
f"user-tables/table-persistence-after-{td_test}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
