def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(services=["localstack"])
for version in CONFLUENT_PLATFORM_VERSIONS:
print(f"==> Testing Confluent Platform {version}")
confluent_platform_services = [
Zookeeper(tag=version),
Kafka(tag=version),
SchemaRegistry(tag=version),
]
with c.override(*confluent_platform_services):
c.start_and_wait_for_tcp(services=[
"zookeeper", "kafka", "schema-registry", "materialized"
])
c.wait_for_materialized()
c.run("testdrive", "kafka-matrix.td", "testdrive/kafka-*.td")
c.kill(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
)
c.rm(
"zookeeper",
"kafka",
"schema-registry",
"materialized",
"testdrive",
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
class RowsJoinOuter(Generator):
COUNT = 1_000_000
@classmethod
def body(cls) -> None:
print(
f"> CREATE MATERIALIZED VIEW v1 AS SELECT generate_series AS f1, generate_series AS f2 FROM (SELECT * FROM generate_series(1, {cls.COUNT}));"
)
print(
f"> SELECT COUNT(*) FROM v1 AS a1 LEFT JOIN v1 AS a2 USING (f1);")
print(f"{cls.COUNT}")
SERVICES = [
Zookeeper(),
Kafka(),
SchemaRegistry(),
Materialized(
memory="8G",
options="--persistent-user-tables --persistent-kafka-sources"),
Testdrive(),
]
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "materialized"])
with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp:
with contextlib.redirect_stdout(tmp):
from materialize.mzcompose.services import (
Kafka,
Materialized,
SchemaRegistry,
TestCerts,
Testdrive,
Zookeeper,
)
SERVICES = [
TestCerts(),
Zookeeper(
environment=[
"ZOOKEEPER_CLIENT_PORT=2181",
# Despite the environment variable name, these are JVM options that are
# passed through to Zookeeper.
"KAFKA_OPTS=-Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dzookeeper.sessionRequireClientSASLAuth=true -Djava.security.auth.login.config=/etc/zookeeper/sasl.jaas.config",
],
volumes=["./sasl.jaas.config:/etc/zookeeper/sasl.jaas.config"],
),
Kafka(
environment=[
# "KAFKA_INTER_BROKER_LISTENER_NAME=SSL",
"KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181",
"KAFKA_SASL_ENABLED_MECHANISMS=PLAIN",
"KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN",
"KAFKA_SSL_KEYSTORE_FILENAME=kafka.keystore.jks",
"KAFKA_SSL_KEYSTORE_CREDENTIALS=cert_creds",
"KAFKA_SSL_KEY_CREDENTIALS=cert_creds",
"KAFKA_SSL_TRUSTSTORE_FILENAME=kafka.truststore.jks",
"KAFKA_SSL_TRUSTSTORE_CREDENTIALS=cert_creds",
