from materialize.mzcompose import (
Debezium,
Kafka,
Materialized,
Postgres,
SchemaRegistry,
SqlServer,
Testdrive,
Workflow,
Zookeeper,
)
prerequisites = [
Zookeeper(),
Kafka(auto_create_topics=True),
SchemaRegistry(),
Debezium(),
Materialized(),
]
postgres = Postgres()
sa_password = "******" + "".join(
random.choices(string.ascii_uppercase + string.digits, k=10)
)
sql_server = SqlServer(sa_password=sa_password)
services = prerequisites + [
postgres,
name=f"materialized_v{tag}",
image=f"materialize/materialized:v{tag}",
hostname="materialized",
options="".join(option
for starting_version, option in mz_options.items()
if tag >= starting_version),
),
) for tag in all_tested_tags)
mz_versioned["current_source"] = Materialized(
name="materialized_current_source",
hostname="materialized",
options="".join(mz_options.values()),
)
prerequisites = [Zookeeper(), Kafka(), SchemaRegistry(), Postgres()]
TESTDRIVE_VALIDATE_CATALOG_SVC_NAME = "testdrive-svc"
TESTDRIVE_NOVALIDATE_CATALOG_SVC_NAME = "testdrive-svc-novalidate"
services = [
*prerequisites,
*(mz_versioned.values()),
Testdrive(name=TESTDRIVE_VALIDATE_CATALOG_SVC_NAME),
# N.B.: we need to use `validate_catalog=False` because testdrive uses HEAD version to read
# from disk to compare to the in-memory representation. Doing this invokes SQL planning.
# However, the SQL planner cannot be guaranteed to handle all old syntaxes directly. For
# example, HEAD will panic when attempting to plan the <= v0.9.12 representation of protobufs
# because it expects them to be either be:
# 1) purified (input from the console)
# 2) upgraded (reading from old catalog)
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
from materialize.mzcompose import (
Kafka,
Materialized,
SchemaRegistry,
Testdrive,
Workflow,
Zookeeper,
)
confluents = [Zookeeper(), Kafka(), SchemaRegistry()]
versioned_mz = [
Materialized(
name=f"materialized_{version}",
image=f"materialize/materialized:{version}",
hostname="materialized",
) for version in ["v0.7.0", "v0.8.0"]
]
multiple_mz = [
Materialized(name=f"materialized{i}",
data_directory=f"/share/materialized{i}",
port=6875 + i) for i in [1, 2]
]
print(f"{cls.COUNT}")
class RowsJoinOuter(Generator):
COUNT = 10_000_000
@classmethod
def body(cls):
print(
f"> CREATE MATERIALIZED VIEW v1 AS SELECT generate_series AS f1, generate_series AS f2 FROM (SELECT * FROM generate_series(1, {cls.COUNT}));"
)
print(
f"> SELECT COUNT(*) FROM v1 AS a1 LEFT JOIN v1 AS a2 USING (f1);")
print(f"{cls.COUNT}")
servers = [Zookeeper(), Kafka(), SchemaRegistry(), Materialized(memory="8G")]
services = [*servers, Testdrive()]
def workflow_limits(w: Workflow):
w.start_and_wait_for_tcp(services=servers)
with tempfile.NamedTemporaryFile(mode="w", dir=w.composition.path) as tmp:
with contextlib.redirect_stdout(tmp):
[cls.generate() for cls in Generator.__subclasses__()]
sys.stdout.flush()
w.run_service(service="testdrive-svc",
command=os.path.basename(tmp.name))
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
from materialize.mzcompose import (
Kafka,
Localstack,
Materialized,
SchemaRegistry,
Squid,
Testdrive,
Workflow,
Zookeeper,
)
localstack = Localstack()
prerequisites = [Zookeeper(), Kafka(), SchemaRegistry(), Squid()]
# Run certain testdrive tests for each combination of env variables under test
tests = [
{
"name": "with_proxy",
"env": ["ALL_PROXY=http://squid:3128"],
"td": "testdrive/avro-registry.td testdrive/esoteric/s3.td",
},
{
"name": "proxy_failure",
"env": ["ALL_PROXY=http://localhost:1234"],
"td": "proxy-failure.td",
},
{
"name":