def workflow_pg_snapshot_resumption(c: Composition) -> None:
"""Test creating sources in a remote storaged process."""
c.down(destroy_volumes=True)
with c.override(
# Start postgres for the pg source
Postgres(),
Testdrive(no_reset=True),
Storaged(environment=["FAILPOINTS=pg_snapshot_failure=return"]),
):
dependencies = [
"materialized",
"postgres",
"storaged",
]
c.start_and_wait_for_tcp(services=dependencies, )
c.run("testdrive", "pg-snapshot-resumption/01-configure-postgres.td")
c.run("testdrive", "pg-snapshot-resumption/02-create-sources.td")
# storaged should crash
c.run("testdrive", "pg-snapshot-resumption/03-while-storaged-down.td")
print("Sleeping to ensure that storaged crashes")
time.sleep(10)
with c.override(
# turn off the failpoint
Storaged()):
c.start_and_wait_for_tcp(services=["storaged"], )
c.run("testdrive", "pg-snapshot-resumption/04-verify-data.td")
def workflow_stash(c: Composition) -> None:
c.rm(
"testdrive",
"materialized",
stop=True,
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
materialized = Materialized(options=[
"--adapter-stash-url", "postgres://*****:*****@postgres"
], )
postgres = Postgres(image="postgres:14.4")
with c.override(materialized, postgres):
c.up("postgres")
c.wait_for_postgres()
c.start_and_wait_for_tcp(services=["materialized"])
c.wait_for_materialized("materialized")
c.sql("CREATE TABLE a (i INT)")
c.stop("postgres")
c.up("postgres")
c.wait_for_postgres()
c.sql("CREATE TABLE b (i INT)")
c.rm("postgres", stop=True, destroy_volumes=True)
c.up("postgres")
c.wait_for_postgres()
# Postgres cleared its database, so this should fail.
try:
c.sql("CREATE TABLE c (i INT)")
raise Exception("expected unreachable")
except Exception as e:
# Depending on timing, either of these errors can occur. The stash error comes
# from the stash complaining. The network error comes from pg8000 complaining
# because materialize panic'd.
if "stash error: postgres: db error" not in str(
e) and "network error" not in str(e):
raise e
def workflow_test_remote_storaged(c: Composition) -> None:
"""Test creating sources in a remote storaged process."""
c.down(destroy_volumes=True)
with c.override(
Testdrive(default_timeout="15s",
no_reset=True,
consistent_seed=True),
# Use a separate PostgreSQL service for persist rather than the one in
# the `Materialized` service, so that crashing `environmentd` does not
# also take down PostgreSQL.
Postgres(),
Materialized(
options=
"--persist-consensus-url=postgres://postgres:postgres@postgres"
),
):
dependencies = [
"materialized",
"postgres",
"storaged",
"redpanda",
]
c.start_and_wait_for_tcp(services=dependencies, )
c.run("testdrive", "storaged/01-create-sources.td")
c.kill("materialized")
c.up("materialized")
c.run("testdrive", "storaged/02-after-environmentd-restart.td")
c.kill("storaged")
c.run("testdrive", "storaged/03-while-storaged-down.td")
c.up("storaged")
c.run("testdrive", "storaged/04-after-storaged-restart.td")
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
from materialize.mzcompose import Composition
from materialize.mzcompose.services import Materialized, Postgres, Testdrive, Toxiproxy
SERVICES = [
Materialized(),
Postgres(),
Toxiproxy(),
Testdrive(no_reset=True, default_timeout="60s"),
]
def workflow_pg_cdc_resumption(c: Composition) -> None:
"""Test Postgres direct replication's failure handling by
disrupting replication at various stages using Toxiproxy or service restarts
"""
initialize(c)
for scenario in [
disconnect_pg_during_snapshot,
disconnect_pg_during_replication,
restart_pg_during_snapshot,
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
from materialize import ROOT, ci_util
from materialize.mzcompose import Composition
from materialize.mzcompose.services import Postgres, SqlLogicTest
SERVICES = [Postgres(), SqlLogicTest()]
def workflow_default(c: Composition) -> None:
"Run fast SQL logic tests"
run_sqllogictest(c, "ci/test/slt-fast.sh")
def workflow_sqllogictest(c: Composition) -> None:
"Run slow SQL logic tests"
run_sqllogictest(c, "ci/slt/slt.sh")
def run_sqllogictest(c: Composition, command: str) -> None:
c.up("postgres")
c.wait_for_postgres(dbname="postgres")
try:
junit_report = ci_util.junit_report_filename(c.name)
from materialize.checks.update import * # noqa: F401 F403
from materialize.checks.upsert import * # noqa: F401 F403
from materialize.checks.users import * # noqa: F401 F403
from materialize.checks.window_functions import * # noqa: F401 F403
from materialize.mzcompose import Composition, WorkflowArgumentParser
from materialize.mzcompose.services import (
Computed,
Debezium,
Materialized,
Postgres,
Redpanda,
)
from materialize.mzcompose.services import Testdrive as TestdriveService
SERVICES = [
Postgres(name="postgres-backend"),
Postgres(name="postgres-source"),
Redpanda(auto_create_topics=True),
Debezium(),
Computed(
name="computed_1"
), # Started by some Scenarios, defined here only for the teardown
Materialized(
options=" ".join(
[
"--persist-consensus-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=consensus",
"--storage-stash-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=storage",
"--adapter-stash-url=postgresql://postgres:postgres@postgres-backend:5432?options=--search_path=adapter",
]
)
),
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
from materialize import ROOT, ci_util, spawn
from materialize.mzcompose import Composition, Service
from materialize.mzcompose.services import Kafka, Postgres, SchemaRegistry, Zookeeper
SERVICES = [
Zookeeper(),
Kafka(),
SchemaRegistry(),
Postgres(image="postgres:14.2"),
Service(
name="ci-cargo-test",
config={
"mzbuild":
"ci-cargo-test",
"environment": [
"ZOOKEEPER_ADDR=zookeeper:2181",
"KAFKA_ADDRS=kafka:9092",
"SCHEMA_REGISTRY_URL=http://schema-registry:8081",
"POSTGRES_URL=postgres://postgres:postgres@postgres",
"MZ_SOFT_ASSERTIONS=1",
"MZ_PERSIST_EXTERNAL_STORAGE_TEST_S3_BUCKET=mtlz-test-persist-1d-lifecycle-delete",
"MZ_PERSIST_EXTERNAL_STORAGE_TEST_POSTGRES_URL=postgres://postgres:postgres@postgres",
"AWS_DEFAULT_REGION",
"AWS_ACCESS_KEY_ID",
