def workflow_test_upsert(c: Composition) -> None:
"""Test creating upsert sources and continuing to ingest them after a restart."""
with c.override(
Testdrive(default_timeout="30s",
no_reset=True,
consistent_seed=True), ):
c.down(destroy_volumes=True)
dependencies = [
"materialized",
"zookeeper",
"kafka",
"schema-registry",
]
c.start_and_wait_for_tcp(services=dependencies, )
c.run("testdrive", "upsert/01-create-sources.td")
# Sleep to make sure the errors have made it to persist.
# This isn't necessary for correctness,
# as we should be able to crash at any point and re-start.
# But if we don't sleep here, then we might be ingesting the errored
# records in the new process, and so we won't actually be testing
# the ability to retract error values that make it to persist.
print("Sleeping for ten seconds")
time.sleep(10)
c.exec("materialized", "bash", "-c", "kill -9 `pidof storaged`")
c.run("testdrive", "upsert/02-after-storaged-restart.td")
def run_test(c: Composition, args: argparse.Namespace) -> None:
c.up("testdrive", persistent=True)
scenarios = ([globals()[args.scenario]]
if args.scenario else Generator.__subclasses__())
for scenario in scenarios:
with tempfile.NamedTemporaryFile(mode="w", dir=c.path) as tmp:
with contextlib.redirect_stdout(tmp):
scenario.generate()
sys.stdout.flush()
c.exec("testdrive", os.path.basename(tmp.name))
def execute(self, c: Composition) -> None:
# Depending on the workload, storaged may not be running, hence the || true
c.exec("materialized", "bash", "-c",
"kill -9 `pidof storaged` || true")
def workflow_default(c: Composition) -> None:
c.start_and_wait_for_tcp(services=["materialized"])
c.wait_for_materialized("materialized")
# ensure that the directory has restricted permissions
c.exec(
"materialized",
"bash",
"-c",
"[[ `stat -c \"%a\" /mzdata/secrets` == '700' ]] && exit 0 || exit 1",
)
c.sql("CREATE SECRET secret AS 's3cret'")
# Check that the contents of the secret have made it to the storage
c.exec(
"materialized",
"bash",
"-c",
"[[ `cat /mzdata/secrets/*` == 's3cret' ]] && exit 0 || exit 1",
)
# Check that the file permissions are restrictive
c.exec(
"materialized",
"bash",
"-c",
"[[ `stat -c \"%a\" /mzdata/secrets/*` == '600' ]] && exit 0 || exit 1",
)
# Check that alter secret gets reflected on disk
c.sql("ALTER SECRET secret AS 'tops3cret'")
c.exec(
"materialized",
"bash",
"-c",
"[[ `cat /mzdata/secrets/*` == 'tops3cret' ]] && exit 0 || exit 1",
)
# check that replacing the file did not change permissions
c.exec(
"materialized",
"bash",
"-c",
"[[ `stat -c \"%a\" /mzdata/secrets/*` == '600' ]] && exit 0 || exit 1",
)
# Rename should not change the contents on disk
c.sql("ALTER SECRET secret RENAME TO renamed_secret")
# Check that the contents of the secret have made it to the storage
c.exec(
"materialized",
"bash",
"-c",
"[[ `cat /mzdata/secrets/*` == 'tops3cret' ]] && exit 0 || exit 1",
)
c.sql("DROP SECRET renamed_secret")
# Check that the file has been deleted from the storage
c.exec(
"materialized",
"bash",
"-c",
"[[ -z `ls -A /mzdata/secrets` ]] && exit 0 || exit 1",
)
