def workflow_disable_user_indexes(c: Composition) -> None: seed = round(time.time()) c.start_and_wait_for_tcp(services=prerequisites) c.up("materialized") c.wait_for_materialized() c.run("testdrive-svc", f"--seed={seed}", "disable-user-indexes/before.td") c.kill("materialized") with c.override( Materialized(options=f"{mz_options} --disable-user-indexes", )): c.up("materialized") c.wait_for_materialized() c.run("testdrive-svc", f"--seed={seed}", "disable-user-indexes/after.td") c.kill("materialized") c.rm("materialized", "testdrive-svc", destroy_volumes=True) c.rm_volumes("mzdata")
def workflow_kafka_sources(c: Composition) -> None: seed = round(time.time()) c.start_and_wait_for_tcp(services=prerequisites) c.up("materialized") c.wait_for_materialized("materialized") c.run("testdrive-svc", f"--seed={seed}", f"kafka-sources/*{td_test}*-before.td") c.kill("materialized") c.up("materialized") c.wait_for_materialized("materialized") # And restart again, for extra stress c.kill("materialized") c.up("materialized") c.wait_for_materialized("materialized") c.run("testdrive-svc", f"--seed={seed}", f"kafka-sources/*{td_test}*-after.td") # Do one more restart, just in case and just confirm that Mz is able to come up c.kill("materialized") c.up("materialized") c.wait_for_materialized("materialized") c.kill("materialized") c.rm("materialized", "testdrive-svc", destroy_volumes=True) c.rm_volumes("mzdata")
def workflow_kafka_sources( c: Composition, args_or_parser: Union[WorkflowArgumentParser, Namespace]) -> None: start_deps(c, args_or_parser) seed = round(time.time()) c.up("materialized") c.wait_for_materialized("materialized") c.run("testdrive", f"--seed={seed}", f"kafka-sources/*{td_test}*-before.td") c.kill("materialized") c.up("materialized") c.wait_for_materialized("materialized") # And restart again, for extra stress c.kill("materialized") c.up("materialized") c.wait_for_materialized("materialized") c.run("testdrive", f"--seed={seed}", f"kafka-sources/*{td_test}*-after.td") # Do one more restart, just in case and just confirm that Mz is able to come up c.kill("materialized") c.up("materialized") c.wait_for_materialized("materialized") c.kill("materialized") c.rm("materialized", "testdrive", destroy_volumes=True) c.rm_volumes("mzdata", "pgdata")
def run_one_failpoint(c: Composition, failpoint: str, action: str) -> None: print( f">>> Running failpoint test for failpoint {failpoint} with action {action}" ) seed = round(time.time()) c.up("materialized") c.wait_for_materialized() c.run( "testdrive-svc", f"--seed={seed}", f"--var=failpoint={failpoint}", f"--var=action={action}", "failpoints/before.td", ) time.sleep(2) # kill Mz if the failpoint has not killed it c.kill("materialized") c.up("materialized") c.wait_for_materialized() c.run("testdrive-svc", f"--seed={seed}", "failpoints/after.td") c.kill("materialized") c.rm("materialized", "testdrive-svc", destroy_volumes=True) c.rm_volumes("mzdata")
def workflow_default(c: Composition) -> None: c.start_and_wait_for_tcp(services=["localstack"]) for version in CONFLUENT_PLATFORM_VERSIONS: print(f"==> Testing Confluent Platform {version}") confluent_platform_services = [ Zookeeper(tag=version), Kafka(tag=version), SchemaRegistry(tag=version), ] with c.override(*confluent_platform_services): c.start_and_wait_for_tcp(services=[ "zookeeper", "kafka", "schema-registry", "materialized" ]) c.wait_for_materialized() c.run("testdrive", "kafka-matrix.td", "testdrive/kafka-*.td") c.kill( "zookeeper", "kafka", "schema-registry", "materialized", ) c.rm( "zookeeper", "kafka", "schema-registry", "materialized", "testdrive", destroy_volumes=True, ) c.rm_volumes("mzdata", "pgdata", force=True)
def workflow_user_tables( c: Composition, args_or_parser: Union[WorkflowArgumentParser, Namespace]) -> None: start_deps(c, args_or_parser) seed = round(time.time()) c.up("materialized") c.wait_for_materialized() c.run( "testdrive", f"--seed={seed}", f"user-tables/table-persistence-before-{td_test}.td", ) c.kill("materialized") c.up("materialized") c.kill("materialized") c.up("materialized") c.run( "testdrive", f"--seed={seed}", f"user-tables/table-persistence-after-{td_test}.td", ) c.kill("materialized") c.rm("materialized", "testdrive", destroy_volumes=True) c.rm_volumes("mzdata", "pgdata")
def workflow_smoke_test(c: Composition) -> None: c.workflow( "default", "--num-seconds=15", "--records-per-second=1000", ) c.kill("materialized") c.rm("materialized", "testdrive", "kafka", destroy_volumes=True) c.rm_volumes("mzdata", "pgdata")
def test_upgrade_from_version( c: Composition, from_version: str, priors: List[str], filter: str, style: str = "" ) -> None: print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.") version_glob = "{" + ",".join(["any_version", *priors, from_version]) + "}" print(">>> Version glob pattern: " + version_glob) c.rm("materialized", "testdrive-svc", stop=True) c.rm_volumes("mzdata", "tmp") if from_version != "current_source": mz_from = Materialized( image=f"materialize/materialized:{from_version}", options=" ".join( opt for start_version, opt in mz_options.items() if from_version[1:] >= start_version ), environment=[ "SSL_KEY_PASSWORD=mzmzmz", ], volumes_extra=["secrets:/share/secrets"], ) with c.override(mz_from): c.up("materialized") else: c.up("materialized") c.wait_for_materialized("materialized") temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}" seed = f"--seed={random.getrandbits(32)}" c.run( "testdrive-svc", "--no-reset", f"--var=upgrade-from-version={from_version}", temp_dir, seed, f"create-{style}in-{version_glob}-{filter}.td", ) c.kill("materialized") c.rm("materialized", "testdrive-svc") c.up("materialized") c.wait_for_materialized("materialized") c.run( "testdrive-svc", "--no-reset", f"--var=upgrade-from-version={from_version}", temp_dir, seed, "--validate-catalog=/share/mzdata/catalog", f"check-{style}from-{version_glob}-{filter}.td", )
def run_test(c: Composition, disruption: Disruption, id: int) -> None: print(f"+++ Running disruption scenario {disruption.name}") c.up("testdrive", persistent=True) nodes = [ Computed( name="computed_1_1", peers=["computed_1_1", "computed_1_2"], ), Computed( name="computed_1_2", peers=["computed_1_1", "computed_1_2"], ), Computed( name="computed_2_1", peers=["computed_2_1", "computed_2_2"], ), Computed( name="computed_2_2", peers=["computed_2_1", "computed_2_2"], ), ] with c.override(*nodes): c.up("materialized", *[n.name for n in nodes]) c.wait_for_materialized() c.sql( """ CREATE CLUSTER cluster1 REPLICAS ( replica1 (REMOTE ['computed_1_1:2100', 'computed_1_2:2100']), replica2 (REMOTE ['computed_2_1:2100', 'computed_2_2:2100']) ) """ ) with c.override( Testdrive( validate_data_dir=False, no_reset=True, materialize_params={"cluster": "cluster1"}, seed=id, ) ): populate(c) # Disrupt replica1 by some means disruption.disruption(c) validate(c) cleanup_list = ["materialized", "testdrive", *[n.name for n in nodes]] c.kill(*cleanup_list) c.rm(*cleanup_list, destroy_volumes=True) c.rm_volumes("mzdata", "pgdata")
def workflow_failpoints(c: Composition) -> None: seed = round(time.time()) c.up("materialized") c.wait_for_materialized() c.run("testdrive-svc", f"--seed={seed}", f"failpoints/{td_test}.td") c.kill("materialized") c.rm("materialized", "testdrive-svc", destroy_volumes=True) c.rm_volumes("mzdata")
def run_one_scenario( c: Composition, scenario: Type[Scenario], args: argparse.Namespace ) -> Comparator: name = scenario.__name__ print(f"--- Now benchmarking {name} ...") comparator = make_comparator(name) common_seed = round(time.time()) mzs = { "this": Materialized( image=f"materialize/materialized:{args.this_tag}" if args.this_tag else None, options=args.this_options, ), "other": Materialized( image=f"materialize/materialized:{args.other_tag}" if args.other_tag else None, options=args.other_options, ), } for mz_id, instance in enumerate(["this", "other"]): with c.override(mzs[instance]): print(f"The version of the '{instance.upper()}' Mz instance is:") c.run("materialized", "--version") c.start_and_wait_for_tcp(services=["materialized"]) c.wait_for_materialized() executor = Docker( composition=c, seed=common_seed, ) benchmark = Benchmark( mz_id=mz_id, scenario=scenario, scale=args.scale, executor=executor, filter=make_filter(args), termination_conditions=make_termination_conditions(args), aggregation=make_aggregation(), ) outcome, iterations = benchmark.run() comparator.append(outcome) c.kill("materialized") c.rm("materialized", "testdrive-svc") c.rm_volumes("mzdata") return comparator
def workflow_smoke_test(c: Composition) -> None: for arg in ["--upsert", "--enable-persistence"]: c.workflow( "default", "--num-seconds=15", "--records-per-second=1000", arg, ) c.kill("materialized") c.rm("materialized", "testdrive-svc", "kafka", destroy_volumes=True) c.rm_volumes("mzdata")
def workflow_compaction(c: Composition) -> None: with c.override(Materialized(options=f"--metrics-scraping-interval=1s", )): c.up("materialized") c.wait_for_materialized() c.run("testdrive", "compaction/compaction.td") c.kill("materialized") c.rm("materialized", "testdrive", destroy_volumes=True) c.rm_volumes("mzdata", "pgdata")
def workflow_compaction(c: Composition) -> None: with c.override(mz_fast_metrics): c.up("materialized") c.wait_for_materialized() c.run("testdrive-svc", "compaction/compaction.td") c.kill("materialized") c.rm("materialized", "testdrive-svc", destroy_volumes=True) c.rm_volumes("mzdata")
def test_upgrade_from_version( c: Composition, from_version: str, priors: List[str], filter: str ) -> None: print(f"===>>> Testing upgrade from Materialize {from_version} to current_source.") version_glob = "|".join(["any_version", *priors, from_version]) print(">>> Version glob pattern: " + version_glob) if from_version != "current_source": mz_from = Materialized( image=f"materialize/materialized:{from_version}", options=" ".join( opt for start_version, opt in mz_options.items() if from_version[1:] >= start_version ), ) with c.override(mz_from): c.up("materialized") else: c.up("materialized") c.wait_for_materialized("materialized") temp_dir = f"--temp-dir=/share/tmp/upgrade-from-{from_version}" with patch.dict(os.environ, {"UPGRADE_FROM_VERSION": from_version}): c.run( "testdrive-svc", "--seed=1", "--no-reset", temp_dir, f"create-in-@({version_glob})-{filter}.td", ) c.kill("materialized") c.rm("materialized", "testdrive-svc") c.up("materialized") c.wait_for_materialized("materialized") with patch.dict(os.environ, {"UPGRADE_FROM_VERSION": from_version}): c.run( "testdrive-svc", "--seed=1", "--no-reset", temp_dir, f"--validate-catalog=/share/mzdata/catalog check-from-@({version_glob})-{filter}.td", ) c.kill("materialized") c.rm("materialized", "testdrive-svc") c.rm_volumes("mzdata", "tmp")
def workflow_stash(c: Composition) -> None: c.rm( "testdrive", "materialized", stop=True, destroy_volumes=True, ) c.rm_volumes("mzdata", "pgdata", force=True) materialized = Materialized(options=[ "--adapter-stash-url", "postgres://*****:*****@postgres" ], ) postgres = Postgres(image="postgres:14.4") with c.override(materialized, postgres): c.up("postgres") c.wait_for_postgres() c.start_and_wait_for_tcp(services=["materialized"]) c.wait_for_materialized("materialized") c.sql("CREATE TABLE a (i INT)") c.stop("postgres") c.up("postgres") c.wait_for_postgres() c.sql("CREATE TABLE b (i INT)") c.rm("postgres", stop=True, destroy_volumes=True) c.up("postgres") c.wait_for_postgres() # Postgres cleared its database, so this should fail. try: c.sql("CREATE TABLE c (i INT)") raise Exception("expected unreachable") except Exception as e: # Depending on timing, either of these errors can occur. The stash error comes # from the stash complaining. The network error comes from pg8000 complaining # because materialize panic'd. if "stash error: postgres: db error" not in str( e) and "network error" not in str(e): raise e
def workflow_user_tables(c: Composition) -> None: seed = round(time.time()) c.up("materialized") c.wait_for_materialized() c.run( "testdrive-svc", f"--seed {seed} user-tables/table-persistence-before-{td_test}.td", ) c.kill("materialized") c.up("materialized") c.run( "testdrive-svc", f"--seed {seed} user-tables/table-persistence-after-{td_test}.td", ) c.kill("materialized") c.rm("materialized", "testdrive-svc", destroy_volumes=True) c.rm_volumes("mzdata")
def stop_services(c: Composition, cluster_services: List[Service]) -> None: service_names = [s.name for s in cluster_services] + ["testdrive"] c.kill(*service_names) c.rm(*service_names) c.rm_volumes("mzdata", "pgdata")
def teardown(c: Composition) -> None: c.rm(*[s.name for s in SERVICES], stop=True, destroy_volumes=True) c.rm_volumes("mzdata", "pgdata", "tmp", force=True)