def workflow_stash(c: Composition) -> None:
c.rm(
"testdrive",
"materialized",
stop=True,
destroy_volumes=True,
)
c.rm_volumes("mzdata", "pgdata", force=True)
materialized = Materialized(options=[
"--adapter-stash-url", "postgres://*****:*****@postgres"
], )
postgres = Postgres(image="postgres:14.4")
with c.override(materialized, postgres):
c.up("postgres")
c.wait_for_postgres()
c.start_and_wait_for_tcp(services=["materialized"])
c.wait_for_materialized("materialized")
c.sql("CREATE TABLE a (i INT)")
c.stop("postgres")
c.up("postgres")
c.wait_for_postgres()
c.sql("CREATE TABLE b (i INT)")
c.rm("postgres", stop=True, destroy_volumes=True)
c.up("postgres")
c.wait_for_postgres()
# Postgres cleared its database, so this should fail.
try:
c.sql("CREATE TABLE c (i INT)")
raise Exception("expected unreachable")
except Exception as e:
# Depending on timing, either of these errors can occur. The stash error comes
# from the stash complaining. The network error comes from pg8000 complaining
# because materialize panic'd.
if "stash error: postgres: db error" not in str(
e) and "network error" not in str(e):
raise e
def workflow_default(c: Composition) -> None:
"Test that materialize can use a multitude of auth schemes to connect to AWS"
LOCAL_DIR.mkdir()
session = boto3.Session()
sts: STSClient = session.client("sts")
iam: IAMClient = session.client("iam")
identity = sts.get_caller_identity()
current_user = identity["Arn"]
aws_region = session.region_name
created_roles: List[CreatedRole] = []
try:
allowed = create_role(iam, "Allow", current_user, created_roles)
denied = create_role(iam, "Deny", current_user, created_roles)
requires_eid = create_role(
iam, "Allow", current_user, created_roles, external_id=EXTERNAL_ID
)
profile_contents = gen_profile_text(
session, allowed.arn, requires_eid.arn, denied.arn
)
wait_for_role(sts, allowed.arn)
td_args = [
f"--aws-region={aws_region}",
f"--var=allowed-role-arn={allowed.arn}",
f"--var=denied-role-arn={denied.arn}",
f"--var=role-requires-eid={requires_eid.arn}",
]
# == Run core tests ==
c.up("materialized")
write_aws_config(LOCAL_DIR, profile_contents)
c.wait_for_materialized("materialized")
c.run(
"testdrive",
*td_args,
"test.td",
)
c.run(
"testdrive",
*td_args,
# no reset because the next test wants to validate behavior with
# the previous catalog
"--no-reset",
"test-externalid-missing.td",
)
# == Tests that restarting materialized without a profile doesn't bork mz ==
print("+++ Test Restarts with and without profile files")
# Historically, a missing aws config file would cause all SQL
# commands to hang entirely after a restart, this no longer happens
# but this step restarts to catch it if it comes back.
c.stop("materialized")
rm_aws_config(LOCAL_DIR)
c.up("materialized")
c.run(
"testdrive",
"--no-reset",
"test-restart-no-creds.td",
)
# now test that with added credentials things can be done
write_aws_config(LOCAL_DIR, profile_contents)
c.run("testdrive", *td_args, "test-restart-with-creds.td")
# == Test that requires --aws-external-id has been supplied ==
print("+++ Test AWS External IDs")
c.stop("materialized")
c.rm("materialized")
with c.override(MZ_EID):
c.up("materialized")
c.wait_for_materialized("materialized")
write_aws_config(LOCAL_DIR, profile_contents)
c.run("testdrive", *td_args, "test-externalid-present.td")
finally:
errored = False
for role in created_roles:
try:
iam.delete_role_policy(RoleName=role.name, PolicyName=role.policy_name)
except Exception as e:
errored = True
print(
f"> Unable to delete role policy {role.name}/{role.policy_name}: {e}"
)
try:
iam.delete_role(RoleName=role.name)
print(f"> Deleted IAM role {role.name}")
except Exception as e:
errored = True
print(f"> Unable to delete role {role.name}: {e}")
rm_aws_config(LOCAL_DIR)
LOCAL_DIR.rmdir()
if errored:
raise UIError("Unable to completely clean up AWS resources")
