예제 #1
0
def test_cluster_compute_storage(c: Composition, *glob: str) -> None:
    c.up("dataflowd_storage")
    c.up("dataflowd_compute_1")
    c.up("dataflowd_compute_2")
    c.up("materialized_compute_storage")
    c.wait_for_materialized(service="materialized_compute_storage")
    c.run("testdrive-svc-compute-storage", *glob)
예제 #2
0
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    """Runs the dbt adapter test suite against Materialize in various configurations."""
    parser.add_argument("filter",
                        nargs="?",
                        default="",
                        help="limit to test cases matching filter")
    args = parser.parse_args()

    for test_case in test_cases:
        if args.filter in test_case.name:
            print(f"> Running test case {test_case.name}")
            materialized = Materialized(
                options=test_case.materialized_options,
                image=test_case.materialized_image,
                depends_on=["test-certs"],
                volumes_extra=["secrets:/secrets"],
            )

            with c.test_case(test_case.name):
                with c.override(materialized):
                    c.down()
                    c.up("materialized")
                    c.wait_for_tcp(host="materialized", port=6875)
                    c.run(
                        "dbt-test",
                        "pytest",
                        "dbt-materialize/test",
                        env_extra=test_case.dbt_env,
                    )
예제 #3
0
    def run(self, c: Composition) -> None:
        if not self.new_sink:
            return

        c.testdrive(
            dedent(f"""
                > CREATE CONNECTION IF NOT EXISTS {self.sink.name}_kafka_conn FOR KAFKA BROKER '${{testdrive.kafka-addr}}';
                > CREATE CONNECTION IF NOT EXISTS {self.sink.name}_csr_conn FOR CONFLUENT SCHEMA REGISTRY URL '${{testdrive.schema-registry-url}}';

                > CREATE SINK {self.sink.name} FROM {self.source_view.name}
                  INTO KAFKA CONNECTION {self.sink.name}_kafka_conn
                  TOPIC 'sink-{self.sink.name}'
                  FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION {self.sink.name}_csr_conn;

                # Ingest the sink again in order to be able to validate its contents

                > CREATE SOURCE {self.sink.name}_source
                  FROM KAFKA CONNECTION {self.sink.name}_kafka_conn
                  TOPIC 'sink-{self.sink.name}'
                  FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION {self.sink.name}_csr_conn
                  ENVELOPE NONE

                # The sink-dervied source has upsert semantics, so produce a "normal" ViewExists output
                # from the 'before' and the 'after'

                > CREATE MATERIALIZED VIEW {self.dest_view.name} AS
                  SELECT SUM(min)::int AS min, SUM(max)::int AS max, SUM(c1)::int AS c1, SUM(c2)::int AS c2 FROM (
                    SELECT (after).min, (after).max, (after).c1, (after).c2 FROM {self.sink.name}_source
                    UNION ALL
                    SELECT - (before).min, - (before).max, -(before).c1, -(before).c2 FROM {self.sink.name}_source
                  );
            """))
예제 #4
0
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    parser.add_argument("--message-count", type=int, default=1000)
    parser.add_argument("--partitions", type=int, default=1)
    parser.add_argument("--check-sink", action="store_true")
    parser.add_argument(
        "--redpanda",
        action="store_true",
        help="run against Redpanda instead of the Confluent Platform",
    )
    args = parser.parse_args()

    dependencies = ["materialized"]
    if args.redpanda:
        dependencies += ["redpanda"]
    else:
        dependencies += ["zookeeper", "kafka", "schema-registry"]

    c.start_and_wait_for_tcp(services=dependencies)
    c.run(
        "billing-demo",
        "--materialized-host=materialized",
        "--kafka-host=kafka",
        "--schema-registry-url=http://schema-registry:8081",
        "--create-topic",
        "--replication-factor=1",
        f"--message-count={args.message_count}",
        f"--partitions={args.partitions}",
        *(["--check-sink"] if args.check_sink else []),
    )
예제 #5
0
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    """Run the proxy tests."""
    parser.add_argument(
        "--redpanda",
        action="store_true",
        help="run against Redpanda instead of the Confluent Platform",
    )
    parser.add_argument(
        "--aws-region",
        help="run against the specified AWS region instead of localstack",
    )
    args = parser.parse_args()

    dependencies = ["squid"]
    if args.redpanda:
        dependencies += ["redpanda"]
    else:
        dependencies += ["zookeeper", "kafka", "schema-registry"]
    if not args.aws_region:
        dependencies += ["localstack"]
    c.start_and_wait_for_tcp(dependencies)

    aws_arg = (f"--aws-region={args.aws_region}"
               if args.aws_region else "--aws-endpoint=http://localstack:4566")

    for test_case in test_cases:
        with c.test_case(test_case.name):
            with c.override(Materialized(environment_extra=test_case.env)):
                c.up("materialized")
                c.wait_for_materialized("materialized")
                c.run("testdrive-svc", aws_arg, *test_case.files)
예제 #6
0
def workflow_default(c: Composition) -> None:
    c.up("test-certs")
    c.start_and_wait_for_tcp(
        services=["zookeeper", "kafka", "schema-registry"])
    c.up("materialized")
    c.wait_for_materialized()
    c.run("testdrive", "*.td")
예제 #7
0
def populate(c: Composition) -> None:
    # Create some database objects
    c.testdrive(
        dedent("""
            > CREATE TABLE t1 (f1 INTEGER);
            > INSERT INTO t1 SELECT * FROM generate_series(1, 10);
            > CREATE MATERIALIZED VIEW v1 AS SELECT COUNT(*) AS c1 FROM t1;
            > CREATE TABLE ten (f1 INTEGER);
            > INSERT INTO ten SELECT * FROM generate_series(1, 10);
            > CREATE MATERIALIZED VIEW expensive AS SELECT (a1.f1 * 1) +
              (a2.f1 * 10) +
              (a3.f1 * 100) +
              (a4.f1 * 1000) +
              (a5.f1 * 10000) +
              (a6.f1 * 100000) +
              (a7.f1 * 1000000)
              FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5, ten AS a6, ten AS a7;
            $ kafka-create-topic topic=source1
            $ kafka-ingest format=bytes topic=source1 repeat=1000000
            A${kafka-ingest.iteration}
            > CREATE SOURCE source1
              FROM KAFKA BROKER '${testdrive.kafka-addr}' TOPIC 'testdrive-source1-${testdrive.seed}'
              FORMAT BYTES
            > CREATE MATERIALIZED VIEW v2 AS SELECT COUNT(*) FROM source1
            """), )
예제 #8
0
 def handle_composition(self, args: argparse.Namespace,
                        composition: mzcompose.Composition) -> None:
     if args.workflow not in composition.workflows:
         # Restart any dependencies whose definitions have changed. This is
         # Docker Compose's default behavior for `up`, but not for `run`,
         # which is a constant irritation that we paper over here. The trick,
         # taken from Buildkite's Docker Compose plugin, is to run an `up`
         # command that requests zero instances of the requested service.
         if args.workflow:
             composition.invoke(
                 "up",
                 "-d",
                 "--scale",
                 f"{args.workflow}=0",
                 args.workflow,
             )
         super().handle_composition(args, composition)
     else:
         # The user has specified a workflow rather than a service. Run the
         # workflow instead of Docker Compose.
         if args.unknown_args:
             bad_arg = args.unknown_args[0]
         elif args.unknown_subargs[0].startswith("-"):
             bad_arg = args.unknown_subargs[0]
         else:
             bad_arg = None
         if bad_arg:
             raise UIError(
                 f"unknown option {bad_arg!r}",
                 hint=f"if {bad_arg!r} is a valid Docker Compose option, "
                 f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
                 "is a custom mzcompose workflow, not a Docker Compose service",
             )
         composition.workflow(args.workflow, *args.unknown_subargs[1:])
예제 #9
0
def restart_mz_during_replication(c: Composition) -> None:
    c.run("testdrive", "wait-for-snapshot.td", "delete-rows-t1.td",
          "alter-table.td")

    restart_mz(c)

    c.run("testdrive", "delete-rows-t2.td")
예제 #10
0
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    parser.add_argument(
        "--scenario",
        metavar="SCENARIO",
        type=str,
        help="Scenario to run",
        required=True,
    )

    parser.add_argument("--seed",
                        metavar="N",
                        type=int,
                        help="Random seed",
                        default=1)

    args = parser.parse_args()
    scenario_class = globals()[args.scenario]

    c.start_and_wait_for_tcp(
        services=["zookeeper", "kafka", "schema-registry"])
    c.up("testdrive", persistent=True)

    random.seed(args.seed)

    print("Generating test...")
    test = Test(scenario=scenario_class(), max_actions=500)
    print("Running test...")
    test.run(c)
예제 #11
0
def workflow_start_two_mzs(c: Composition,
                           parser: WorkflowArgumentParser) -> None:
    """Starts two Mz instances from different git tags for the purpose of manually running
    RQG comparison tests.
    """
    parser.add_argument("--this-tag",
                        help="Run Materialize with this git tag on port 6875")

    parser.add_argument("--other-tag",
                        help="Run Materialize with this git tag on port 16875")
    args = parser.parse_args()

    with c.override(
            Materialized(
                name="mz_this",
                image=f"materialize/materialized:{args.this_tag}"
                if args.this_tag else None,
                volumes=
                [],  # Keep the mzdata, pgdata, etc. private to the container
                allow_host_ports=True,
                ports=["6875:6875"],
            ),
            Materialized(
                name="mz_other",
                image=f"materialize/materialized:{args.other_tag}"
                if args.other_tag else None,
                volumes=[],
                allow_host_ports=True,
                ports=["16875:6875"],
            ),
    ):
        for mz in ["mz_this", "mz_other"]:
            c.up(mz)
            c.wait_for_materialized(service=mz)
예제 #12
0
def workflow_default(c: Composition) -> None:
    c.start_and_wait_for_tcp(["zookeeper", "kafka", "schema-registry"])
    c.run("ci-cargo-test", "run-tests")
    token = os.environ["BUILDKITE_TEST_ANALYTICS_API_KEY_CARGO_TEST"]
    if len(token) < 1:
        print("Analytics API key empty, skipping junit reporting")
        return
    with open(f"{ROOT.as_posix()}/results.json") as f:
        junit_xml = spawn.capture(args=["cargo2junit"], stdin=f.read())
        requests.post(
            "https://analytics-api.buildkite.com/v1/uploads",
            headers={"Authorization": f"Token {token}"},
            json={
                "format": "junit",
                "run_env": {
                    "key": os.environ["BUILDKITE_BUILD_ID"],
                    "CI": "buildkite",
                    "number": os.environ["BUILDKITE_BUILD_NUMBER"],
                    "job_id": os.environ["BUILDKITE_JOB_ID"],
                    "branch": os.environ["BUILDKITE_BRANCH"],
                    "commit_sha": os.environ["BUILDKITE_COMMIT"],
                    "message": os.environ["BUILDKITE_MESSAGE"],
                    "url": os.environ["BUILDKITE_BUILD_URL"],
                },
                "data": junit_xml,
            },
        )
예제 #13
0
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    c.silent = True

    parser.add_argument("--scenario",
                        metavar="SCENARIO",
                        type=str,
                        help="Scenario to run.")

    parser.add_argument("--check",
                        metavar="CHECK",
                        type=str,
                        help="Check to run.")

    args = parser.parse_args()

    c.up("testdrive", persistent=True)

    #    c.start_and_wait_for_tcp(
    #        services=["zookeeper", "kafka", "schema-registry", "postgres"]
    #    )

    scenarios = ([globals()[args.scenario]]
                 if args.scenario else Scenario.__subclasses__())

    checks = [globals()[args.check]] if args.check else Check.__subclasses__()

    for scenario_class in scenarios:
        print(f"Testing upgrade scenario {scenario_class}")
        scenario = scenario_class(checks=checks)
        scenario.run(c)
예제 #14
0
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
    c.start_and_wait_for_tcp(services=[
        "zookeeper", "kafka", "schema-registry", "materialized", "toxiproxy"
    ])
    c.wait_for_materialized()

    seed = random.getrandbits(16)
    for i, failure_mode in enumerate([
            "toxiproxy-close-connection.td",
            "toxiproxy-limit-connection.td",
            "toxiproxy-timeout.td",
            # TODO: Enable https://github.com/MaterializeInc/materialize/issues/11085
            # "toxiproxy-timeout-hold.td",
    ]):
        c.start_and_wait_for_tcp(["toxiproxy"])
        c.run(
            "testdrive-svc",
            "--no-reset",
            "--max-errors=1",
            f"--seed={seed}{i}",
            f"--temp-dir=/share/tmp/kafka-resumption-{seed}{i}",
            "setup.td",
            failure_mode,
            "during.td",
            "sleep.td",
            "toxiproxy-restore-connection.td",
            "verify-success.td",
            "cleanup.td",
        )
        c.kill("toxiproxy")
예제 #15
0
def workflow_upgrade(c: Composition, parser: WorkflowArgumentParser) -> None:
    """Test upgrades from various versions."""

    parser.add_argument(
        "--min-version",
        metavar="VERSION",
        type=Version.parse,
        default=Version.parse("0.8.0"),
        help="the minimum version to test from",
    )
    parser.add_argument(
        "--most-recent",
        metavar="N",
        type=int,
        help="limit testing to the N most recent versions",
    )
    parser.add_argument(
        "filter", nargs="?", default="*", help="limit to only the files matching filter"
    )
    args = parser.parse_args()

    tested_versions = [v for v in all_versions if v >= args.min_version]
    if args.most_recent is not None:
        tested_versions = tested_versions[: args.most_recent]
    tested_versions.reverse()

    c.start_and_wait_for_tcp(
        services=["zookeeper", "kafka", "schema-registry", "postgres"]
    )

    for version in tested_versions:
        priors = [f"v{v}" for v in all_versions if v < version]
        test_upgrade_from_version(c, f"v{version}", priors, filter=args.filter)

    test_upgrade_from_version(c, "current_source", priors=["*"], filter=args.filter)
예제 #16
0
def workflow_load_test(c: Composition) -> None:
    """Run CH-benCHmark with a selected amount of load against Materialize."""
    c.workflow(
        "default",
        "--peek-conns=1",
        "--mz-views=q01,q02,q05,q06,q08,q09,q12,q14,q17,q19",
        "--transactional-threads=2",
    )
예제 #17
0
def workflow_testdrive(c: Composition) -> None:
    c.start_and_wait_for_tcp(services=[
        "zookeeper",
        "kafka",
        "schema-registry",
        "materialized",
    ])
    c.run("testdrive-svc", tests)
예제 #18
0
def create_invalid_replica(c: Composition) -> None:
    c.testdrive(
        dedent(
            """
            > CREATE CLUSTER REPLICA cluster1.replica3 REMOTE ['no_such_host:2100']
            """
        )
    )
예제 #19
0
def disconnect_pg_during_snapshot(c: Composition) -> None:
    c.run(
        "testdrive-svc",
        "toxiproxy-close-connection.td",
        "toxiproxy-restore-connection.td",
        "delete-rows-t1.td",
        "delete-rows-t2.td",
    )
예제 #20
0
    def run(self, c: Composition) -> None:
        if self.new_source:
            c.testdrive(f"""
> CREATE MATERIALIZED SOURCE {self.source.name}
  FROM KAFKA BROKER '${{testdrive.kafka-addr}}'
  TOPIC 'testdrive-{self.topic.name}-${{testdrive.seed}}'
  FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY '${{testdrive.schema-registry-url}}'
  ENVELOPE UPSERT
""")
예제 #21
0
def drop_create_replica(c: Composition) -> None:
    c.testdrive(
        dedent(
            """
            > DROP CLUSTER REPLICA cluster1.replica1
            > CREATE CLUSTER REPLICA cluster1.replica3 REMOTE ['computed_1_1:2100', 'computed_1_2:2100']
            """
        )
    )
예제 #22
0
def workflow_default(c: Composition) -> None:
    c.start_and_wait_for_tcp(
        services=["zookeeper", "kafka", "schema-registry", "materialized"])

    with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp:
        with contextlib.redirect_stdout(tmp):
            [cls.generate() for cls in Generator.__subclasses__()]
            sys.stdout.flush()
            c.run("testdrive-svc", os.path.basename(tmp.name))
예제 #23
0
def workflow_test_drop_default_cluster(c: Composition) -> None:
    """Test that the default cluster can be dropped"""

    c.down(destroy_volumes=True)
    c.up("materialized")
    c.wait_for_materialized()

    c.sql("DROP CLUSTER default CASCADE")
    c.sql("CREATE CLUSTER default REPLICAS (default (SIZE '1'))")
예제 #24
0
def begin(c: Composition) -> None:
    """Configure Toxiproxy and Mz and populate initial data"""

    c.run(
        "testdrive-svc",
        "configure-toxiproxy.td",
        "populate-tables.td",
        "configure-materalize.td",
    )
예제 #25
0
def workflow_default(c: Composition) -> None:
    """Test cluster isolation by introducing faults of various kinds in cluster1
    and then making sure that cluster2 continues to operate properly
    """

    c.start_and_wait_for_tcp(
        services=["zookeeper", "kafka", "schema-registry"])
    for id, disruption in enumerate(disruptions):
        run_test(c, disruption, id)
예제 #26
0
    def handle_composition(self, args: argparse.Namespace,
                           composition: mzcompose.Composition) -> None:
        if args.workflow not in composition.workflows:
            # Restart any dependencies whose definitions have changed. This is
            # Docker Compose's default behavior for `up`, but not for `run`,
            # which is a constant irritation that we paper over here. The trick,
            # taken from Buildkite's Docker Compose plugin, is to run an `up`
            # command that requests zero instances of the requested service.
            if args.workflow:
                composition.invoke(
                    "up",
                    "-d",
                    "--scale",
                    f"{args.workflow}=0",
                    args.workflow,
                )
            super().handle_composition(args, composition)
        else:
            # The user has specified a workflow rather than a service. Run the
            # workflow instead of Docker Compose.
            if args.unknown_args:
                bad_arg = args.unknown_args[0]
            elif args.unknown_subargs[0].startswith("-"):
                bad_arg = args.unknown_subargs[0]
            else:
                bad_arg = None
            if bad_arg:
                raise UIError(
                    f"unknown option {bad_arg!r}",
                    hint=f"if {bad_arg!r} is a valid Docker Compose option, "
                    f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
                    "is a custom mzcompose workflow, not a Docker Compose service",
                )

            # Run the workflow inside of a test case so that we get some basic
            # test analytics, even if the workflow doesn't define more granular
            # test cases.
            with composition.test_case(f"workflow-{args.workflow}"):
                composition.workflow(args.workflow, *args.unknown_subargs[1:])

            # Upload test report to Buildkite Test Analytics.
            junit_suite = junit_xml.TestSuite(composition.name)
            for (name, result) in composition.test_results.items():
                test_case = junit_xml.TestCase(name, composition.name,
                                               result.duration)
                if result.error:
                    test_case.add_error_info(message=result.error)
                junit_suite.test_cases.append(test_case)
            junit_report = ci_util.junit_report_filename("mzcompose")
            with junit_report.open("w") as f:
                junit_xml.to_xml_report_file(f, [junit_suite])
            ci_util.upload_junit_report("mzcompose", junit_report)

            if any(result.error
                   for result in composition.test_results.values()):
                raise UIError("at least one test case failed")
예제 #27
0
def workflow_cluster_testdrive(c: Composition) -> None:
    c.start_and_wait_for_tcp(services=["zookeeper", "kafka", "schema-registry"])
    # Skip tests that use features that are not supported yet
    test_cluster(
        c,
        "grep",
        "-LE",
        "mz_catalog|mz_kafka_|mz_records_|mz_metrics",
        "testdrive/*.td",
    )
예제 #28
0
    def run(self, c: Composition) -> None:
        watermarks = self.view.get_watermarks()
        view_min = watermarks.min
        view_max = watermarks.max

        if view_min <= view_max:
            c.testdrive(f"""
> SELECT * FROM {self.view.name} /* {view_min} {view_max} {(view_max-view_min)+1} {(view_max-view_min)+1} */ ;
{view_min} {view_max} {(view_max-view_min)+1} {(view_max-view_min)+1}
""")
예제 #29
0
def disconnect_pg_during_replication(c: Composition) -> None:
    c.run(
        "testdrive",
        "wait-for-snapshot.td",
        "delete-rows-t1.td",
        "delete-rows-t2.td",
        "alter-table.td",
        "toxiproxy-close-connection.td",
        "toxiproxy-restore-connection.td",
    )
예제 #30
0
    def run(self, c: Composition) -> None:
        if self.new_topic:
            c.testdrive(f"""
$ kafka-create-topic topic={self.topic.name} partitions={self.topic.partitions}

{SCHEMA}

$ kafka-ingest format=avro key-format=avro topic={self.topic.name} schema=${{schema}} key-schema=${{keyschema}} repeat=1
{{"key": 0}} {{"f1": 0}}
""")