Exemplos de S3 em Python

Exemplo n.º 1

def test_s3_empty_serialize():
    s3 = storage.S3(bucket="bucket")
    serialized = S3Schema().dump(s3)

    assert serialized
    assert serialized["__version__"] == prefect.__version__
    assert serialized["bucket"]
    assert not serialized["key"]

Exemplo n.º 2

def test_s3_full_serialize():
    s3 = storage.S3(bucket="bucket", key="key", secrets=["hidden", "auth"],)
    serialized = S3Schema().dump(s3)

    assert serialized
    assert serialized["__version__"] == prefect.__version__
    assert serialized["bucket"] == "bucket"
    assert serialized["key"] == "key"
    assert serialized["secrets"] == ["hidden", "auth"]

Exemplo n.º 3

def test_s3_full_serialize():
    s3 = storage.S3(
        aws_access_key_id="id",
        aws_secret_access_key="secret",
        aws_session_token="session",
        bucket="bucket",
        key="key",
    )
    serialized = S3Schema().dump(s3)

    assert serialized
    assert serialized["__version__"] == prefect.__version__
    assert serialized["bucket"] == "bucket"
    assert serialized["key"] == "key"

Exemplo n.º 4

def test_s3_serialize_with_flows():
    s3 = storage.S3(bucket="bucket", key="key", secrets=["hidden", "auth"],)
    f = prefect.Flow("test")
    s3.flows["test"] = "key"
    serialized = S3Schema().dump(s3)

    assert serialized
    assert serialized["__version__"] == prefect.__version__
    assert serialized["bucket"] == "bucket"
    assert serialized["key"] == "key"
    assert serialized["flows"] == {"test": "key"}

    deserialized = S3Schema().load(serialized)
    assert f.name in deserialized
    assert deserialized.secrets == ["hidden", "auth"]

Exemplo n.º 5

def test_s3_aws_creds_not_serialized():
    s3 = storage.S3(
        aws_access_key_id="id",
        aws_secret_access_key="secret",
        aws_session_token="session",
        bucket="bucket",
        key="key",
    )
    serialized = S3Schema().dump(s3)

    assert serialized
    assert serialized["__version__"] == prefect.__version__
    assert serialized["bucket"] == "bucket"
    assert serialized["key"] == "key"
    assert serialized.get("aws_access_key_id") is None
    assert serialized.get("aws_secret_access_key") is None
    assert serialized.get("aws_session_token") is None

Exemplo n.º 6

def test_s3_serialize_with_flows():
    s3 = storage.S3(
        aws_access_key_id="id",
        aws_secret_access_key="secret",
        aws_session_token="session",
        bucket="bucket",
        key="key",
    )
    f = prefect.Flow("test")
    s3.flows["test"] = "key"
    serialized = S3Schema().dump(s3)

    assert serialized
    assert serialized["__version__"] == prefect.__version__
    assert serialized["bucket"] == "bucket"
    assert serialized["key"] == "key"
    assert serialized["flows"] == {"test": "key"}

    deserialized = S3Schema().load(serialized)
    assert f.name in deserialized