def workflow_stash(c: Composition) -> None: c.rm( "testdrive", "materialized", stop=True, destroy_volumes=True, ) c.rm_volumes("mzdata", "pgdata", force=True) materialized = Materialized(options=[ "--adapter-stash-url", "postgres://*****:*****@postgres" ], ) postgres = Postgres(image="postgres:14.4") with c.override(materialized, postgres): c.up("postgres") c.wait_for_postgres() c.start_and_wait_for_tcp(services=["materialized"]) c.wait_for_materialized("materialized") c.sql("CREATE TABLE a (i INT)") c.stop("postgres") c.up("postgres") c.wait_for_postgres() c.sql("CREATE TABLE b (i INT)") c.rm("postgres", stop=True, destroy_volumes=True) c.up("postgres") c.wait_for_postgres() # Postgres cleared its database, so this should fail. try: c.sql("CREATE TABLE c (i INT)") raise Exception("expected unreachable") except Exception as e: # Depending on timing, either of these errors can occur. The stash error comes # from the stash complaining. The network error comes from pg8000 complaining # because materialize panic'd. if "stash error: postgres: db error" not in str( e) and "network error" not in str(e): raise e
def workflow_default(c: Composition) -> None: "Test that materialize can use a multitude of auth schemes to connect to AWS" LOCAL_DIR.mkdir() session = boto3.Session() sts: STSClient = session.client("sts") iam: IAMClient = session.client("iam") identity = sts.get_caller_identity() current_user = identity["Arn"] aws_region = session.region_name created_roles: List[CreatedRole] = [] try: allowed = create_role(iam, "Allow", current_user, created_roles) denied = create_role(iam, "Deny", current_user, created_roles) requires_eid = create_role( iam, "Allow", current_user, created_roles, external_id=EXTERNAL_ID ) profile_contents = gen_profile_text( session, allowed.arn, requires_eid.arn, denied.arn ) wait_for_role(sts, allowed.arn) td_args = [ f"--aws-region={aws_region}", f"--var=allowed-role-arn={allowed.arn}", f"--var=denied-role-arn={denied.arn}", f"--var=role-requires-eid={requires_eid.arn}", ] # == Run core tests == c.up("materialized") write_aws_config(LOCAL_DIR, profile_contents) c.wait_for_materialized("materialized") c.run( "testdrive", *td_args, "test.td", ) c.run( "testdrive", *td_args, # no reset because the next test wants to validate behavior with # the previous catalog "--no-reset", "test-externalid-missing.td", ) # == Tests that restarting materialized without a profile doesn't bork mz == print("+++ Test Restarts with and without profile files") # Historically, a missing aws config file would cause all SQL # commands to hang entirely after a restart, this no longer happens # but this step restarts to catch it if it comes back. c.stop("materialized") rm_aws_config(LOCAL_DIR) c.up("materialized") c.run( "testdrive", "--no-reset", "test-restart-no-creds.td", ) # now test that with added credentials things can be done write_aws_config(LOCAL_DIR, profile_contents) c.run("testdrive", *td_args, "test-restart-with-creds.td") # == Test that requires --aws-external-id has been supplied == print("+++ Test AWS External IDs") c.stop("materialized") c.rm("materialized") with c.override(MZ_EID): c.up("materialized") c.wait_for_materialized("materialized") write_aws_config(LOCAL_DIR, profile_contents) c.run("testdrive", *td_args, "test-externalid-present.td") finally: errored = False for role in created_roles: try: iam.delete_role_policy(RoleName=role.name, PolicyName=role.policy_name) except Exception as e: errored = True print( f"> Unable to delete role policy {role.name}/{role.policy_name}: {e}" ) try: iam.delete_role(RoleName=role.name) print(f"> Deleted IAM role {role.name}") except Exception as e: errored = True print(f"> Unable to delete role {role.name}: {e}") rm_aws_config(LOCAL_DIR) LOCAL_DIR.rmdir() if errored: raise UIError("Unable to completely clean up AWS resources")