def workflow_pg_snapshot_resumption(c: Composition) -> None: """Test creating sources in a remote storaged process.""" c.down(destroy_volumes=True) with c.override( # Start postgres for the pg source Postgres(), Testdrive(no_reset=True), Storaged(environment=["FAILPOINTS=pg_snapshot_failure=return"]), ): dependencies = [ "materialized", "postgres", "storaged", ] c.start_and_wait_for_tcp(services=dependencies, ) c.run("testdrive", "pg-snapshot-resumption/01-configure-postgres.td") c.run("testdrive", "pg-snapshot-resumption/02-create-sources.td") # storaged should crash c.run("testdrive", "pg-snapshot-resumption/03-while-storaged-down.td") print("Sleeping to ensure that storaged crashes") time.sleep(10) with c.override( # turn off the failpoint Storaged()): c.start_and_wait_for_tcp(services=["storaged"], ) c.run("testdrive", "pg-snapshot-resumption/04-verify-data.td")
def workflow_stash(c: Composition) -> None: c.rm( "testdrive", "materialized", stop=True, destroy_volumes=True, ) c.rm_volumes("mzdata", "pgdata", force=True) materialized = Materialized(options=[ "--adapter-stash-url", "postgres://*****:*****@postgres" ], ) postgres = Postgres(image="postgres:14.4") with c.override(materialized, postgres): c.up("postgres") c.wait_for_postgres() c.start_and_wait_for_tcp(services=["materialized"]) c.wait_for_materialized("materialized") c.sql("CREATE TABLE a (i INT)") c.stop("postgres") c.up("postgres") c.wait_for_postgres() c.sql("CREATE TABLE b (i INT)") c.rm("postgres", stop=True, destroy_volumes=True) c.up("postgres") c.wait_for_postgres() # Postgres cleared its database, so this should fail. try: c.sql("CREATE TABLE c (i INT)") raise Exception("expected unreachable") except Exception as e: # Depending on timing, either of these errors can occur. The stash error comes # from the stash complaining. The network error comes from pg8000 complaining # because materialize panic'd. if "stash error: postgres: db error" not in str( e) and "network error" not in str(e): raise e
def workflow_test_remote_storaged(c: Composition) -> None: """Test creating sources in a remote storaged process.""" c.down(destroy_volumes=True) with c.override( Testdrive(default_timeout="15s", no_reset=True, consistent_seed=True), # Use a separate PostgreSQL service for persist rather than the one in # the `Materialized` service, so that crashing `environmentd` does not # also take down PostgreSQL. Postgres(), Materialized( options= "--persist-consensus-url=postgres://postgres:postgres@postgres" ), ): dependencies = [ "materialized", "postgres", "storaged", "redpanda", ] c.start_and_wait_for_tcp(services=dependencies, ) c.run("testdrive", "storaged/01-create-sources.td") c.kill("materialized") c.up("materialized") c.run("testdrive", "storaged/02-after-environmentd-restart.td") c.kill("storaged") c.run("testdrive", "storaged/03-while-storaged-down.td") c.up("storaged") c.run("testdrive", "storaged/04-after-storaged-restart.td")
# Copyright Materialize, Inc. and contributors. All rights reserved. # # Use of this software is governed by the Business Source License # included in the LICENSE file at the root of this repository. # # As of the Change Date specified in that file, in accordance with # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. from materialize.mzcompose import Composition from materialize.mzcompose.services import Materialized, Postgres, Testdrive, Toxiproxy SERVICES = [ Materialized(), Postgres(), Toxiproxy(), Testdrive(no_reset=True, default_timeout="60s"), ] def workflow_pg_cdc_resumption(c: Composition) -> None: """Test Postgres direct replication's failure handling by disrupting replication at various stages using Toxiproxy or service restarts """ initialize(c) for scenario in [ disconnect_pg_during_snapshot, disconnect_pg_during_replication, restart_pg_during_snapshot,
# Copyright Materialize, Inc. and contributors. All rights reserved. # # Use of this software is governed by the Business Source License # included in the LICENSE file at the root of this repository. # # As of the Change Date specified in that file, in accordance with # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. from materialize import ROOT, ci_util from materialize.mzcompose import Composition from materialize.mzcompose.services import Postgres, SqlLogicTest SERVICES = [Postgres(), SqlLogicTest()] def workflow_default(c: Composition) -> None: "Run fast SQL logic tests" run_sqllogictest(c, "ci/test/slt-fast.sh") def workflow_sqllogictest(c: Composition) -> None: "Run slow SQL logic tests" run_sqllogictest(c, "ci/slt/slt.sh") def run_sqllogictest(c: Composition, command: str) -> None: c.up("postgres") c.wait_for_postgres(dbname="postgres") try: junit_report = ci_util.junit_report_filename(c.name)
from materialize.checks.update import * # noqa: F401 F403 from materialize.checks.upsert import * # noqa: F401 F403 from materialize.checks.users import * # noqa: F401 F403 from materialize.checks.window_functions import * # noqa: F401 F403 from materialize.mzcompose import Composition, WorkflowArgumentParser from materialize.mzcompose.services import ( Computed, Debezium, Materialized, Postgres, Redpanda, ) from materialize.mzcompose.services import Testdrive as TestdriveService SERVICES = [ Postgres(name="postgres-backend"), Postgres(name="postgres-source"), Redpanda(auto_create_topics=True), Debezium(), Computed( name="computed_1" ), # Started by some Scenarios, defined here only for the teardown Materialized( options=" ".join( [ "--persist-consensus-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=consensus", "--storage-stash-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=storage", "--adapter-stash-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=adapter", ] ) ),
# Use of this software is governed by the Business Source License # included in the LICENSE file at the root of this repository. # # As of the Change Date specified in that file, in accordance with # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. from materialize import ROOT, ci_util, spawn from materialize.mzcompose import Composition, Service from materialize.mzcompose.services import Kafka, Postgres, SchemaRegistry, Zookeeper SERVICES = [ Zookeeper(), Kafka(), SchemaRegistry(), Postgres(image="postgres:14.2"), Service( name="ci-cargo-test", config={ "mzbuild": "ci-cargo-test", "environment": [ "ZOOKEEPER_ADDR=zookeeper:2181", "KAFKA_ADDRS=kafka:9092", "SCHEMA_REGISTRY_URL=http://schema-registry:8081", "POSTGRES_URL=postgres://postgres:postgres@postgres", "MZ_SOFT_ASSERTIONS=1", "MZ_PERSIST_EXTERNAL_STORAGE_TEST_S3_BUCKET=mtlz-test-persist-1d-lifecycle-delete", "MZ_PERSIST_EXTERNAL_STORAGE_TEST_POSTGRES_URL=postgres://postgres:postgres@postgres", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID",