def workflow_default(c: Composition) -> None: c.start_and_wait_for_tcp(services=["localstack"]) for version in CONFLUENT_PLATFORM_VERSIONS: print(f"==> Testing Confluent Platform {version}") confluent_platform_services = [ Zookeeper(tag=version), Kafka(tag=version), SchemaRegistry(tag=version), ] with c.override(*confluent_platform_services): c.start_and_wait_for_tcp(services=[ "zookeeper", "kafka", "schema-registry", "materialized" ]) c.wait_for_materialized() c.run("testdrive", "kafka-matrix.td", "testdrive/kafka-*.td") c.kill( "zookeeper", "kafka", "schema-registry", "materialized", ) c.rm( "zookeeper", "kafka", "schema-registry", "materialized", "testdrive", destroy_volumes=True, ) c.rm_volumes("mzdata", "pgdata", force=True)
class RowsJoinOuter(Generator): COUNT = 1_000_000 @classmethod def body(cls) -> None: print( f"> CREATE MATERIALIZED VIEW v1 AS SELECT generate_series AS f1, generate_series AS f2 FROM (SELECT * FROM generate_series(1, {cls.COUNT}));" ) print( f"> SELECT COUNT(*) FROM v1 AS a1 LEFT JOIN v1 AS a2 USING (f1);") print(f"{cls.COUNT}") SERVICES = [ Zookeeper(), Kafka(), SchemaRegistry(), Materialized( memory="8G", options="--persistent-user-tables --persistent-kafka-sources"), Testdrive(), ] def workflow_default(c: Composition) -> None: c.start_and_wait_for_tcp( services=["zookeeper", "kafka", "schema-registry", "materialized"]) with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp: with contextlib.redirect_stdout(tmp):
from materialize.mzcompose.services import ( Kafka, Materialized, SchemaRegistry, TestCerts, Testdrive, Zookeeper, ) SERVICES = [ TestCerts(), Zookeeper( environment=[ "ZOOKEEPER_CLIENT_PORT=2181", # Despite the environment variable name, these are JVM options that are # passed through to Zookeeper. "KAFKA_OPTS=-Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dzookeeper.sessionRequireClientSASLAuth=true -Djava.security.auth.login.config=/etc/zookeeper/sasl.jaas.config", ], volumes=["./sasl.jaas.config:/etc/zookeeper/sasl.jaas.config"], ), Kafka( environment=[ # "KAFKA_INTER_BROKER_LISTENER_NAME=SSL", "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181", "KAFKA_SASL_ENABLED_MECHANISMS=PLAIN", "KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN", "KAFKA_SSL_KEYSTORE_FILENAME=kafka.keystore.jks", "KAFKA_SSL_KEYSTORE_CREDENTIALS=cert_creds", "KAFKA_SSL_KEY_CREDENTIALS=cert_creds", "KAFKA_SSL_TRUSTSTORE_FILENAME=kafka.truststore.jks", "KAFKA_SSL_TRUSTSTORE_CREDENTIALS=cert_creds",