def test_cluster_compute_storage(c: Composition, *glob: str) -> None:
c.up("dataflowd_storage")
c.up("dataflowd_compute_1")
c.up("dataflowd_compute_2")
c.up("materialized_compute_storage")
c.wait_for_materialized(service="materialized_compute_storage")
c.run("testdrive-svc-compute-storage", *glob)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Runs the dbt adapter test suite against Materialize in various configurations."""
parser.add_argument("filter",
nargs="?",
default="",
help="limit to test cases matching filter")
args = parser.parse_args()
for test_case in test_cases:
if args.filter in test_case.name:
print(f"> Running test case {test_case.name}")
materialized = Materialized(
options=test_case.materialized_options,
image=test_case.materialized_image,
depends_on=["test-certs"],
volumes_extra=["secrets:/secrets"],
)
with c.test_case(test_case.name):
with c.override(materialized):
c.down()
c.up("materialized")
c.wait_for_tcp(host="materialized", port=6875)
c.run(
"dbt-test",
"pytest",
"dbt-materialize/test",
env_extra=test_case.dbt_env,
)
def run(self, c: Composition) -> None:
if not self.new_sink:
return
c.testdrive(
dedent(f"""
> CREATE CONNECTION IF NOT EXISTS {self.sink.name}_kafka_conn FOR KAFKA BROKER '${{testdrive.kafka-addr}}';
> CREATE CONNECTION IF NOT EXISTS {self.sink.name}_csr_conn FOR CONFLUENT SCHEMA REGISTRY URL '${{testdrive.schema-registry-url}}';
> CREATE SINK {self.sink.name} FROM {self.source_view.name}
INTO KAFKA CONNECTION {self.sink.name}_kafka_conn
TOPIC 'sink-{self.sink.name}'
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION {self.sink.name}_csr_conn;
# Ingest the sink again in order to be able to validate its contents
> CREATE SOURCE {self.sink.name}_source
FROM KAFKA CONNECTION {self.sink.name}_kafka_conn
TOPIC 'sink-{self.sink.name}'
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION {self.sink.name}_csr_conn
ENVELOPE NONE
# The sink-dervied source has upsert semantics, so produce a "normal" ViewExists output
# from the 'before' and the 'after'
> CREATE MATERIALIZED VIEW {self.dest_view.name} AS
SELECT SUM(min)::int AS min, SUM(max)::int AS max, SUM(c1)::int AS c1, SUM(c2)::int AS c2 FROM (
SELECT (after).min, (after).max, (after).c1, (after).c2 FROM {self.sink.name}_source
UNION ALL
SELECT - (before).min, - (before).max, -(before).c1, -(before).c2 FROM {self.sink.name}_source
);
"""))
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
parser.add_argument("--message-count", type=int, default=1000)
parser.add_argument("--partitions", type=int, default=1)
parser.add_argument("--check-sink", action="store_true")
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
args = parser.parse_args()
dependencies = ["materialized"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
c.start_and_wait_for_tcp(services=dependencies)
c.run(
"billing-demo",
"--materialized-host=materialized",
"--kafka-host=kafka",
"--schema-registry-url=http://schema-registry:8081",
"--create-topic",
"--replication-factor=1",
f"--message-count={args.message_count}",
f"--partitions={args.partitions}",
*(["--check-sink"] if args.check_sink else []),
)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Run the proxy tests."""
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--aws-region",
help="run against the specified AWS region instead of localstack",
)
args = parser.parse_args()
dependencies = ["squid"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
if not args.aws_region:
dependencies += ["localstack"]
c.start_and_wait_for_tcp(dependencies)
aws_arg = (f"--aws-region={args.aws_region}"
if args.aws_region else "--aws-endpoint=http://localstack:4566")
for test_case in test_cases:
with c.test_case(test_case.name):
with c.override(Materialized(environment_extra=test_case.env)):
c.up("materialized")
c.wait_for_materialized("materialized")
c.run("testdrive-svc", aws_arg, *test_case.files)
def workflow_default(c: Composition) -> None:
c.up("test-certs")
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry"])
c.up("materialized")
c.wait_for_materialized()
c.run("testdrive", "*.td")
def populate(c: Composition) -> None:
# Create some database objects
c.testdrive(
dedent("""
> CREATE TABLE t1 (f1 INTEGER);
> INSERT INTO t1 SELECT * FROM generate_series(1, 10);
> CREATE MATERIALIZED VIEW v1 AS SELECT COUNT(*) AS c1 FROM t1;
> CREATE TABLE ten (f1 INTEGER);
> INSERT INTO ten SELECT * FROM generate_series(1, 10);
> CREATE MATERIALIZED VIEW expensive AS SELECT (a1.f1 * 1) +
(a2.f1 * 10) +
(a3.f1 * 100) +
(a4.f1 * 1000) +
(a5.f1 * 10000) +
(a6.f1 * 100000) +
(a7.f1 * 1000000)
FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5, ten AS a6, ten AS a7;
$ kafka-create-topic topic=source1
$ kafka-ingest format=bytes topic=source1 repeat=1000000
A${kafka-ingest.iteration}
> CREATE SOURCE source1
FROM KAFKA BROKER '${testdrive.kafka-addr}' TOPIC 'testdrive-source1-${testdrive.seed}'
FORMAT BYTES
> CREATE MATERIALIZED VIEW v2 AS SELECT COUNT(*) FROM source1
"""), )
def handle_composition(self, args: argparse.Namespace,
composition: mzcompose.Composition) -> None:
if args.workflow not in composition.workflows:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
if args.workflow:
composition.invoke(
"up",
"-d",
"--scale",
f"{args.workflow}=0",
args.workflow,
)
super().handle_composition(args, composition)
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.unknown_args:
bad_arg = args.unknown_args[0]
elif args.unknown_subargs[0].startswith("-"):
bad_arg = args.unknown_subargs[0]
else:
bad_arg = None
if bad_arg:
raise UIError(
f"unknown option {bad_arg!r}",
hint=f"if {bad_arg!r} is a valid Docker Compose option, "
f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
"is a custom mzcompose workflow, not a Docker Compose service",
)
composition.workflow(args.workflow, *args.unknown_subargs[1:])
def restart_mz_during_replication(c: Composition) -> None:
c.run("testdrive", "wait-for-snapshot.td", "delete-rows-t1.td",
"alter-table.td")
restart_mz(c)
c.run("testdrive", "delete-rows-t2.td")
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
parser.add_argument(
"--scenario",
metavar="SCENARIO",
type=str,
help="Scenario to run",
required=True,
)
parser.add_argument("--seed",
metavar="N",
type=int,
help="Random seed",
default=1)
args = parser.parse_args()
scenario_class = globals()[args.scenario]
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry"])
c.up("testdrive", persistent=True)
random.seed(args.seed)
print("Generating test...")
test = Test(scenario=scenario_class(), max_actions=500)
print("Running test...")
test.run(c)
def workflow_start_two_mzs(c: Composition,
parser: WorkflowArgumentParser) -> None:
"""Starts two Mz instances from different git tags for the purpose of manually running
RQG comparison tests.
"""
parser.add_argument("--this-tag",
help="Run Materialize with this git tag on port 6875")
parser.add_argument("--other-tag",
help="Run Materialize with this git tag on port 16875")
args = parser.parse_args()
with c.override(
Materialized(
name="mz_this",
image=f"materialize/materialized:{args.this_tag}"
if args.this_tag else None,
volumes=
[], # Keep the mzdata, pgdata, etc. private to the container
allow_host_ports=True,
ports=["6875:6875"],
),
Materialized(
name="mz_other",
image=f"materialize/materialized:{args.other_tag}"
if args.other_tag else None,
volumes=[],
allow_host_ports=True,
ports=["16875:6875"],
),
):
for mz in ["mz_this", "mz_other"]:
c.up(mz)
c.wait_for_materialized(service=mz)
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(["zookeeper", "kafka", "schema-registry"])
c.run("ci-cargo-test", "run-tests")
token = os.environ["BUILDKITE_TEST_ANALYTICS_API_KEY_CARGO_TEST"]
if len(token) < 1:
print("Analytics API key empty, skipping junit reporting")
return
with open(f"{ROOT.as_posix()}/results.json") as f:
junit_xml = spawn.capture(args=["cargo2junit"], stdin=f.read())
requests.post(
"https://analytics-api.buildkite.com/v1/uploads",
headers={"Authorization": f"Token {token}"},
json={
"format": "junit",
"run_env": {
"key": os.environ["BUILDKITE_BUILD_ID"],
"CI": "buildkite",
"number": os.environ["BUILDKITE_BUILD_NUMBER"],
"job_id": os.environ["BUILDKITE_JOB_ID"],
"branch": os.environ["BUILDKITE_BRANCH"],
"commit_sha": os.environ["BUILDKITE_COMMIT"],
"message": os.environ["BUILDKITE_MESSAGE"],
"url": os.environ["BUILDKITE_BUILD_URL"],
},
"data": junit_xml,
},
)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
c.silent = True
parser.add_argument("--scenario",
metavar="SCENARIO",
type=str,
help="Scenario to run.")
parser.add_argument("--check",
metavar="CHECK",
type=str,
help="Check to run.")
args = parser.parse_args()
c.up("testdrive", persistent=True)
# c.start_and_wait_for_tcp(
# services=["zookeeper", "kafka", "schema-registry", "postgres"]
# )
scenarios = ([globals()[args.scenario]]
if args.scenario else Scenario.__subclasses__())
checks = [globals()[args.check]] if args.check else Check.__subclasses__()
for scenario_class in scenarios:
print(f"Testing upgrade scenario {scenario_class}")
scenario = scenario_class(checks=checks)
scenario.run(c)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized", "toxiproxy"
])
c.wait_for_materialized()
seed = random.getrandbits(16)
for i, failure_mode in enumerate([
"toxiproxy-close-connection.td",
"toxiproxy-limit-connection.td",
"toxiproxy-timeout.td",
# TODO: Enable https://github.com/MaterializeInc/materialize/issues/11085
# "toxiproxy-timeout-hold.td",
]):
c.start_and_wait_for_tcp(["toxiproxy"])
c.run(
"testdrive-svc",
"--no-reset",
"--max-errors=1",
f"--seed={seed}{i}",
f"--temp-dir=/share/tmp/kafka-resumption-{seed}{i}",
"setup.td",
failure_mode,
"during.td",
"sleep.td",
"toxiproxy-restore-connection.td",
"verify-success.td",
"cleanup.td",
)
c.kill("toxiproxy")
def workflow_upgrade(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Test upgrades from various versions."""
parser.add_argument(
"--min-version",
metavar="VERSION",
type=Version.parse,
default=Version.parse("0.8.0"),
help="the minimum version to test from",
)
parser.add_argument(
"--most-recent",
metavar="N",
type=int,
help="limit testing to the N most recent versions",
)
parser.add_argument(
"filter", nargs="?", default="*", help="limit to only the files matching filter"
)
args = parser.parse_args()
tested_versions = [v for v in all_versions if v >= args.min_version]
if args.most_recent is not None:
tested_versions = tested_versions[: args.most_recent]
tested_versions.reverse()
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "postgres"]
)
for version in tested_versions:
priors = [f"v{v}" for v in all_versions if v < version]
test_upgrade_from_version(c, f"v{version}", priors, filter=args.filter)
test_upgrade_from_version(c, "current_source", priors=["*"], filter=args.filter)
def workflow_load_test(c: Composition) -> None:
"""Run CH-benCHmark with a selected amount of load against Materialize."""
c.workflow(
"default",
"--peek-conns=1",
"--mz-views=q01,q02,q05,q06,q08,q09,q12,q14,q17,q19",
"--transactional-threads=2",
)
def workflow_testdrive(c: Composition) -> None:
c.start_and_wait_for_tcp(services=[
"zookeeper",
"kafka",
"schema-registry",
"materialized",
])
c.run("testdrive-svc", tests)
def create_invalid_replica(c: Composition) -> None:
c.testdrive(
dedent(
"""
> CREATE CLUSTER REPLICA cluster1.replica3 REMOTE ['no_such_host:2100']
"""
)
)
def disconnect_pg_during_snapshot(c: Composition) -> None:
c.run(
"testdrive-svc",
"toxiproxy-close-connection.td",
"toxiproxy-restore-connection.td",
"delete-rows-t1.td",
"delete-rows-t2.td",
)
def run(self, c: Composition) -> None:
if self.new_source:
c.testdrive(f"""
> CREATE MATERIALIZED SOURCE {self.source.name}
FROM KAFKA BROKER '${{testdrive.kafka-addr}}'
TOPIC 'testdrive-{self.topic.name}-${{testdrive.seed}}'
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY '${{testdrive.schema-registry-url}}'
ENVELOPE UPSERT
""")
def drop_create_replica(c: Composition) -> None:
c.testdrive(
dedent(
"""
> DROP CLUSTER REPLICA cluster1.replica1
> CREATE CLUSTER REPLICA cluster1.replica3 REMOTE ['computed_1_1:2100', 'computed_1_2:2100']
"""
)
)
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "materialized"])
with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp:
with contextlib.redirect_stdout(tmp):
[cls.generate() for cls in Generator.__subclasses__()]
sys.stdout.flush()
c.run("testdrive-svc", os.path.basename(tmp.name))
def workflow_test_drop_default_cluster(c: Composition) -> None:
"""Test that the default cluster can be dropped"""
c.down(destroy_volumes=True)
c.up("materialized")
c.wait_for_materialized()
c.sql("DROP CLUSTER default CASCADE")
c.sql("CREATE CLUSTER default REPLICAS (default (SIZE '1'))")
def begin(c: Composition) -> None:
"""Configure Toxiproxy and Mz and populate initial data"""
c.run(
"testdrive-svc",
"configure-toxiproxy.td",
"populate-tables.td",
"configure-materalize.td",
)
def workflow_default(c: Composition) -> None:
"""Test cluster isolation by introducing faults of various kinds in cluster1
and then making sure that cluster2 continues to operate properly
"""
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry"])
for id, disruption in enumerate(disruptions):
run_test(c, disruption, id)
def handle_composition(self, args: argparse.Namespace,
composition: mzcompose.Composition) -> None:
if args.workflow not in composition.workflows:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
if args.workflow:
composition.invoke(
"up",
"-d",
"--scale",
f"{args.workflow}=0",
args.workflow,
)
super().handle_composition(args, composition)
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.unknown_args:
bad_arg = args.unknown_args[0]
elif args.unknown_subargs[0].startswith("-"):
bad_arg = args.unknown_subargs[0]
else:
bad_arg = None
if bad_arg:
raise UIError(
f"unknown option {bad_arg!r}",
hint=f"if {bad_arg!r} is a valid Docker Compose option, "
f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
"is a custom mzcompose workflow, not a Docker Compose service",
)
# Run the workflow inside of a test case so that we get some basic
# test analytics, even if the workflow doesn't define more granular
# test cases.
with composition.test_case(f"workflow-{args.workflow}"):
composition.workflow(args.workflow, *args.unknown_subargs[1:])
# Upload test report to Buildkite Test Analytics.
junit_suite = junit_xml.TestSuite(composition.name)
for (name, result) in composition.test_results.items():
test_case = junit_xml.TestCase(name, composition.name,
result.duration)
if result.error:
test_case.add_error_info(message=result.error)
junit_suite.test_cases.append(test_case)
junit_report = ci_util.junit_report_filename("mzcompose")
with junit_report.open("w") as f:
junit_xml.to_xml_report_file(f, [junit_suite])
ci_util.upload_junit_report("mzcompose", junit_report)
if any(result.error
for result in composition.test_results.values()):
raise UIError("at least one test case failed")
def workflow_cluster_testdrive(c: Composition) -> None:
c.start_and_wait_for_tcp(services=["zookeeper", "kafka", "schema-registry"])
# Skip tests that use features that are not supported yet
test_cluster(
c,
"grep",
"-LE",
"mz_catalog|mz_kafka_|mz_records_|mz_metrics",
"testdrive/*.td",
)
def run(self, c: Composition) -> None:
watermarks = self.view.get_watermarks()
view_min = watermarks.min
view_max = watermarks.max
if view_min <= view_max:
c.testdrive(f"""
> SELECT * FROM {self.view.name} /* {view_min} {view_max} {(view_max-view_min)+1} {(view_max-view_min)+1} */ ;
{view_min} {view_max} {(view_max-view_min)+1} {(view_max-view_min)+1}
""")
def disconnect_pg_during_replication(c: Composition) -> None:
c.run(
"testdrive",
"wait-for-snapshot.td",
"delete-rows-t1.td",
"delete-rows-t2.td",
"alter-table.td",
"toxiproxy-close-connection.td",
"toxiproxy-restore-connection.td",
)
def run(self, c: Composition) -> None:
if self.new_topic:
c.testdrive(f"""
$ kafka-create-topic topic={self.topic.name} partitions={self.topic.partitions}
{SCHEMA}
$ kafka-ingest format=avro key-format=avro topic={self.topic.name} schema=${{schema}} key-schema=${{keyschema}} repeat=1
{{"key": 0}} {{"f1": 0}}
""")
