def workflow_test_upsert(c: Composition) -> None: """Test creating upsert sources and continuing to ingest them after a restart.""" with c.override( Testdrive(default_timeout="30s", no_reset=True, consistent_seed=True), ): c.down(destroy_volumes=True) dependencies = [ "materialized", "zookeeper", "kafka", "schema-registry", ] c.start_and_wait_for_tcp(services=dependencies, ) c.run("testdrive", "upsert/01-create-sources.td") # Sleep to make sure the errors have made it to persist. # This isn't necessary for correctness, # as we should be able to crash at any point and re-start. # But if we don't sleep here, then we might be ingesting the errored # records in the new process, and so we won't actually be testing # the ability to retract error values that make it to persist. print("Sleeping for ten seconds") time.sleep(10) c.exec("materialized", "bash", "-c", "kill -9 `pidof storaged`") c.run("testdrive", "upsert/02-after-storaged-restart.td")
def run_test(c: Composition, args: argparse.Namespace) -> None: c.up("testdrive", persistent=True) scenarios = ([globals()[args.scenario]] if args.scenario else Generator.__subclasses__()) for scenario in scenarios: with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp: with contextlib.redirect_stdout(tmp): scenario.generate() sys.stdout.flush() c.exec("testdrive", os.path.basename(tmp.name))
def execute(self, c: Composition) -> None: # Depending on the workload, storaged may not be running, hence the || true c.exec("materialized", "bash", "-c", "kill -9 `pidof storaged` || true")
def workflow_default(c: Composition) -> None: c.start_and_wait_for_tcp(services=["materialized"]) c.wait_for_materialized("materialized") # ensure that the directory has restricted permissions c.exec( "materialized", "bash", "-c", "[[ `stat -c \"%a\" /mzdata/secrets` == '700' ]] && exit 0 || exit 1", ) c.sql("CREATE SECRET secret AS 's3cret'") # Check that the contents of the secret have made it to the storage c.exec( "materialized", "bash", "-c", "[[ `cat /mzdata/secrets/*` == 's3cret' ]] && exit 0 || exit 1", ) # Check that the file permissions are restrictive c.exec( "materialized", "bash", "-c", "[[ `stat -c \"%a\" /mzdata/secrets/*` == '600' ]] && exit 0 || exit 1", ) # Check that alter secret gets reflected on disk c.sql("ALTER SECRET secret AS 'tops3cret'") c.exec( "materialized", "bash", "-c", "[[ `cat /mzdata/secrets/*` == 'tops3cret' ]] && exit 0 || exit 1", ) # check that replacing the file did not change permissions c.exec( "materialized", "bash", "-c", "[[ `stat -c \"%a\" /mzdata/secrets/*` == '600' ]] && exit 0 || exit 1", ) # Rename should not change the contents on disk c.sql("ALTER SECRET secret RENAME TO renamed_secret") # Check that the contents of the secret have made it to the storage c.exec( "materialized", "bash", "-c", "[[ `cat /mzdata/secrets/*` == 'tops3cret' ]] && exit 0 || exit 1", ) c.sql("DROP SECRET renamed_secret") # Check that the file has been deleted from the storage c.exec( "materialized", "bash", "-c", "[[ -z `ls -A /mzdata/secrets` ]] && exit 0 || exit 1", )