def run_one_failpoint(c: Composition, failpoint: str, action: str) -> None:
print(
f">>> Running failpoint test for failpoint {failpoint} with action {action}"
)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive-svc",
f"--seed={seed}",
f"--var=failpoint={failpoint}",
f"--var=action={action}",
"failpoints/before.td",
)
time.sleep(2)
# kill Mz if the failpoint has not killed it
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", "failpoints/after.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_user_tables(
c: Composition, args_or_parser: Union[WorkflowArgumentParser,
Namespace]) -> None:
start_deps(c, args_or_parser)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive",
f"--seed={seed}",
f"user-tables/table-persistence-before-{td_test}.td",
)
c.kill("materialized")
c.up("materialized")
c.kill("materialized")
c.up("materialized")
c.run(
"testdrive",
f"--seed={seed}",
f"user-tables/table-persistence-after-{td_test}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_kafka_sources(c: Composition) -> None:
seed = round(time.time())
c.start_and_wait_for_tcp(services=prerequisites)
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", f"--seed={seed}",
f"kafka-sources/*{td_test}*-before.td")
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
# And restart again, for extra stress
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", f"--seed={seed}",
f"kafka-sources/*{td_test}*-after.td")
# Do one more restart, just in case and just confirm that Mz is able to come up
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_kafka_sources(
c: Composition, args_or_parser: Union[WorkflowArgumentParser,
Namespace]) -> None:
start_deps(c, args_or_parser)
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", f"--seed={seed}",
f"kafka-sources/*{td_test}*-before.td")
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
# And restart again, for extra stress
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive", f"--seed={seed}", f"kafka-sources/*{td_test}*-after.td")
# Do one more restart, just in case and just confirm that Mz is able to come up
c.kill("materialized")
c.up("materialized")
c.wait_for_materialized("materialized")
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(services=["localstack"])
for version in CONFLUENT_PLATFORM_VERSIONS:
print(f"==> Testing Confluent Platform {version}")
confluent_platform_services = [
Zookeeper(tag=version),
Kafka(tag=version),
SchemaRegistry(tag=version),
]
with c.override(*confluent_platform_services):
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized"
])
c.wait_for_materialized()
c.run("testdrive", "kafka-matrix.td", "testdrive/kafka-*.td")
c.kill(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
)
c.rm(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
"testdrive",
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
def workflow_disable_user_indexes(c: Composition) -> None:
seed = round(time.time())
c.start_and_wait_for_tcp(services=prerequisites)
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", "disable-user-indexes/before.td")
c.kill("materialized")
with c.override(
Materialized(options=f"{mz_options} --disable-user-indexes", )):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}",
"disable-user-indexes/after.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def test_upgrade_from_version(
c: Composition, from_version: str, priors: List[str], filter: str, style: str = ""
) -> None:
print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.")
version_glob = "{" + ",".join(["any_version", *priors, from_version]) + "}"
print(">>> Version glob pattern: " + version_glob)
c.rm("materialized", "testdrive-svc", stop=True)
c.rm_volumes("mzdata", "tmp")
if from_version != "current_source":
mz_from = Materialized(
image=f"materialize/materialized:{from_version}",
options=" ".join(
opt
for start_version, opt in mz_options.items()
if from_version[1:] >= start_version
),
environment=[
"SSL_KEY_PASSWORD=mzmzmz",
],
volumes_extra=["secrets:/share/secrets"],
)
with c.override(mz_from):
c.up("materialized")
else:
c.up("materialized")
c.wait_for_materialized("materialized")
temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}"
seed = f"--seed={random.getrandbits(32)}"
c.run(
"testdrive-svc",
"--no-reset",
f"--var=upgrade-from-version={from_version}",
temp_dir,
seed,
f"create-{style}in-{version_glob}-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.up("materialized")
c.wait_for_materialized("materialized")
c.run(
"testdrive-svc",
"--no-reset",
f"--var=upgrade-from-version={from_version}",
temp_dir,
seed,
"--validate-catalog=/share/mzdata/catalog",
f"check-{style}from-{version_glob}-{filter}.td",
)
def workflow_smoke_test(c: Composition) -> None:
c.workflow(
"default",
"--num-seconds=15",
"--records-per-second=1000",
)
c.kill("materialized")
c.rm("materialized", "testdrive", "kafka", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def run_test(c: Composition, disruption: Disruption, id: int) -> None:
print(f"+++ Running disruption scenario {disruption.name}")
c.up("testdrive", persistent=True)
nodes = [
Computed(
name="computed_1_1",
peers=["computed_1_1", "computed_1_2"],
),
Computed(
name="computed_1_2",
peers=["computed_1_1", "computed_1_2"],
),
Computed(
name="computed_2_1",
peers=["computed_2_1", "computed_2_2"],
),
Computed(
name="computed_2_2",
peers=["computed_2_1", "computed_2_2"],
),
]
with c.override(*nodes):
c.up("materialized", *[n.name for n in nodes])
c.wait_for_materialized()
c.sql(
"""
CREATE CLUSTER cluster1 REPLICAS (
replica1 (REMOTE ['computed_1_1:2100', 'computed_1_2:2100']),
replica2 (REMOTE ['computed_2_1:2100', 'computed_2_2:2100'])
)
"""
)
with c.override(
Testdrive(
validate_data_dir=False,
no_reset=True,
materialize_params={"cluster": "cluster1"},
seed=id,
)
):
populate(c)
# Disrupt replica1 by some means
disruption.disruption(c)
validate(c)
cleanup_list = ["materialized", "testdrive", *[n.name for n in nodes]]
c.kill(*cleanup_list)
c.rm(*cleanup_list, destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def workflow_smoke_test(c: Composition) -> None:
for arg in ["--upsert", "--enable-persistence"]:
c.workflow(
"default",
"--num-seconds=15",
"--records-per-second=1000",
arg,
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc", "kafka", destroy_volumes=True)
c.rm_volumes("mzdata")
def run_one_scenario(
c: Composition, scenario: Type[Scenario], args: argparse.Namespace
) -> Comparator:
name = scenario.__name__
print(f"--- Now benchmarking {name} ...")
comparator = make_comparator(name)
common_seed = round(time.time())
mzs = {
"this": Materialized(
image=f"materialize/materialized:{args.this_tag}"
if args.this_tag
else None,
options=args.this_options,
),
"other": Materialized(
image=f"materialize/materialized:{args.other_tag}"
if args.other_tag
else None,
options=args.other_options,
),
}
for mz_id, instance in enumerate(["this", "other"]):
with c.override(mzs[instance]):
print(f"The version of the '{instance.upper()}' Mz instance is:")
c.run("materialized", "--version")
c.start_and_wait_for_tcp(services=["materialized"])
c.wait_for_materialized()
executor = Docker(
composition=c,
seed=common_seed,
)
benchmark = Benchmark(
mz_id=mz_id,
scenario=scenario,
scale=args.scale,
executor=executor,
filter=make_filter(args),
termination_conditions=make_termination_conditions(args),
aggregation=make_aggregation(),
)
outcome, iterations = benchmark.run()
comparator.append(outcome)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.rm_volumes("mzdata")
return comparator
def workflow_failpoints(c: Composition) -> None:
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", f"--seed={seed}", f"failpoints/{td_test}.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_compaction(c: Composition) -> None:
with c.override(mz_fast_metrics):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive-svc", "compaction/compaction.td")
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_compaction(c: Composition) -> None:
with c.override(Materialized(options=f"--metrics-scraping-interval=1s", )):
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive", "compaction/compaction.td")
c.kill("materialized")
c.rm("materialized", "testdrive", destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata")
def test_upgrade_from_version(
c: Composition, from_version: str, priors: List[str], filter: str
) -> None:
print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.")
version_glob = "|".join(["any_version", *priors, from_version])
print(">>> Version glob pattern: " + version_glob)
if from_version != "current_source":
mz_from = Materialized(
image=f"materialize/materialized:{from_version}",
options=" ".join(
opt
for start_version, opt in mz_options.items()
if from_version[1:] >= start_version
),
)
with c.override(mz_from):
c.up("materialized")
else:
c.up("materialized")
c.wait_for_materialized("materialized")
temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}"
with patch.dict(os.environ, {"UPGRADE_FROM_VERSION": from_version}):
c.run(
"testdrive-svc",
"--seed=1",
"--no-reset",
temp_dir,
f"create-in-@({version_glob})-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.up("materialized")
c.wait_for_materialized("materialized")
with patch.dict(os.environ, {"UPGRADE_FROM_VERSION": from_version}):
c.run(
"testdrive-svc",
"--seed=1",
"--no-reset",
temp_dir,
f"--validate-catalog=/share/mzdata/catalog check-from-@({version_glob})-{filter}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc")
c.rm_volumes("mzdata", "tmp")
def workflow_timelines(c: Composition) -> None:
for _ in range(3):
c.start_and_wait_for_tcp(services=[
"zookeeper",
"kafka",
"schema-registry",
"materialized",
])
c.wait_for_materialized()
c.run("testdrive", "timelines.td")
c.rm(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
destroy_volumes=True,
)
def workflow_stash(c: Composition) -> None:
c.rm(
"testdrive",
"materialized",
stop=True,
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
materialized = Materialized(options=[
"--adapter-stash-url", "postgres://*****:*****@postgres"
], )
postgres = Postgres(image="postgres:14.4")
with c.override(materialized, postgres):
c.up("postgres")
c.wait_for_postgres()
c.start_and_wait_for_tcp(services=["materialized"])
c.wait_for_materialized("materialized")
c.sql("CREATE TABLE a (i INT)")
c.stop("postgres")
c.up("postgres")
c.wait_for_postgres()
c.sql("CREATE TABLE b (i INT)")
c.rm("postgres", stop=True, destroy_volumes=True)
c.up("postgres")
c.wait_for_postgres()
# Postgres cleared its database, so this should fail.
try:
c.sql("CREATE TABLE c (i INT)")
raise Exception("expected unreachable")
except Exception as e:
# Depending on timing, either of these errors can occur. The stash error comes
# from the stash complaining. The network error comes from pg8000 complaining
# because materialize panic'd.
if "stash error: postgres: db error" not in str(
e) and "network error" not in str(e):
raise e
def workflow_user_tables(c: Composition) -> None:
seed = round(time.time())
c.up("materialized")
c.wait_for_materialized()
c.run(
"testdrive-svc",
f"--seed {seed} user-tables/table-persistence-before-{td_test}.td",
)
c.kill("materialized")
c.up("materialized")
c.run(
"testdrive-svc",
f"--seed {seed} user-tables/table-persistence-after-{td_test}.td",
)
c.kill("materialized")
c.rm("materialized", "testdrive-svc", destroy_volumes=True)
c.rm_volumes("mzdata")
def workflow_default(c: Composition) -> None:
"Test that materialize can use a multitude of auth schemes to connect to AWS"
LOCAL_DIR.mkdir()
session = boto3.Session()
sts: STSClient = session.client("sts")
iam: IAMClient = session.client("iam")
identity = sts.get_caller_identity()
current_user = identity["Arn"]
aws_region = session.region_name
created_roles: List[CreatedRole] = []
try:
allowed = create_role(iam, "Allow", current_user, created_roles)
denied = create_role(iam, "Deny", current_user, created_roles)
requires_eid = create_role(
iam, "Allow", current_user, created_roles, external_id=EXTERNAL_ID
)
profile_contents = gen_profile_text(
session, allowed.arn, requires_eid.arn, denied.arn
)
wait_for_role(sts, allowed.arn)
td_args = [
f"--aws-region={aws_region}",
f"--var=allowed-role-arn={allowed.arn}",
f"--var=denied-role-arn={denied.arn}",
f"--var=role-requires-eid={requires_eid.arn}",
]
# == Run core tests ==
c.up("materialized")
write_aws_config(LOCAL_DIR, profile_contents)
c.wait_for_materialized("materialized")
c.run(
"testdrive",
*td_args,
"test.td",
)
c.run(
"testdrive",
*td_args,
# no reset because the next test wants to validate behavior with
# the previous catalog
"--no-reset",
"test-externalid-missing.td",
)
# == Tests that restarting materialized without a profile doesn't bork mz ==
print("+++ Test Restarts with and without profile files")
# Historically, a missing aws config file would cause all SQL
# commands to hang entirely after a restart, this no longer happens
# but this step restarts to catch it if it comes back.
c.stop("materialized")
rm_aws_config(LOCAL_DIR)
c.up("materialized")
c.run(
"testdrive",
"--no-reset",
"test-restart-no-creds.td",
)
# now test that with added credentials things can be done
write_aws_config(LOCAL_DIR, profile_contents)
c.run("testdrive", *td_args, "test-restart-with-creds.td")
# == Test that requires --aws-external-id has been supplied ==
print("+++ Test AWS External IDs")
c.stop("materialized")
c.rm("materialized")
with c.override(MZ_EID):
c.up("materialized")
c.wait_for_materialized("materialized")
write_aws_config(LOCAL_DIR, profile_contents)
c.run("testdrive", *td_args, "test-externalid-present.td")
finally:
errored = False
for role in created_roles:
try:
iam.delete_role_policy(RoleName=role.name, PolicyName=role.policy_name)
except Exception as e:
errored = True
print(
f"> Unable to delete role policy {role.name}/{role.policy_name}: {e}"
)
try:
iam.delete_role(RoleName=role.name)
print(f"> Deleted IAM role {role.name}")
except Exception as e:
errored = True
print(f"> Unable to delete role {role.name}: {e}")
rm_aws_config(LOCAL_DIR)
LOCAL_DIR.rmdir()
if errored:
raise UIError("Unable to completely clean up AWS resources")
def stop_services(c: Composition, cluster_services: List[Service]) -> None:
service_names = [s.name for s in cluster_services] + ["testdrive"]
c.kill(*service_names)
c.rm(*service_names)
c.rm_volumes("mzdata", "pgdata")
def teardown(c: Composition) -> None:
c.rm(*[s.name for s in SERVICES], stop=True, destroy_volumes=True)
c.rm_volumes("mzdata", "pgdata", "tmp", force=True)
