def test_boto3_client_is_created_with_session(self, mocked_boto_client):
""" Tests the fix for #3925 """
task = S3Download("test")
task.run("key")
assert (
"session.Session()" in mocked_boto_client.return_value._extract_mock_name()
)
def test_raises_on_invalid_compression_method(self, monkeypatch):
task = S3Download("test")
client = MagicMock()
boto3 = MagicMock(client=MagicMock(return_value=client))
monkeypatch.setattr("prefect.utilities.aws.boto3", boto3)
with pytest.raises(ValueError, match="gz_fake"):
task.run("key", compression="gz_fake")
def test_gzip_compression(self, mocked_boto_client):
task = S3Download("bucket")
byte_string = b"col1,col2,col3\nfake,data,1\nfalse,data,2\n"
gzip_data = gzip.compress(byte_string)
def modify_stream(Bucket=None, Key=None, Fileobj=None):
Fileobj.write(gzip_data)
mocked_boto_client.download_fileobj.side_effect = modify_stream
returned_data = task.run("key", compression="gzip")
assert returned_data == str(byte_string, "utf-8")
def test_creds_default_to_environment(self, monkeypatch):
task = S3Download(bucket="bob")
client = MagicMock()
boto3 = MagicMock(client=client)
monkeypatch.setattr("prefect.tasks.aws.s3.boto3", boto3)
task.run(key="")
kwargs = client.call_args[1]
assert kwargs == {
"aws_access_key_id": None,
"aws_secret_access_key": None
}
def test_as_bytes(self, mocked_boto_client, as_bytes):
task = S3Download("bucket")
data = b"some bytes"
def modify_stream(Bucket=None, Key=None, Fileobj=None):
Fileobj.write(data)
mocked_boto_client.download_fileobj.side_effect = modify_stream
res = task.run("key", as_bytes=as_bytes)
if as_bytes:
assert res == data
else:
assert res == data.decode("utf-8")
def test_gzip_compression(self, monkeypatch):
task = S3Download("bucket")
byte_string = b"col1,col2,col3\nfake,data,1\nfalse,data,2\n"
gzip_data = gzip.compress(byte_string)
def modify_stream(Bucket=None, Key=None, Fileobj=None):
Fileobj.write(gzip_data)
client = MagicMock()
boto3 = MagicMock(client=MagicMock(return_value=client))
monkeypatch.setattr("prefect.utilities.aws.boto3", boto3)
client.download_fileobj.side_effect = modify_stream
returned_data = task.run("key", compression="gzip")
assert returned_data == str(byte_string, "utf-8")
def test_creds_are_pulled_from_secret(self, monkeypatch):
task = S3Download(bucket="bob")
client = MagicMock()
boto3 = MagicMock(client=client)
monkeypatch.setattr("prefect.tasks.aws.s3.boto3", boto3)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(AWS_CREDENTIALS={
"ACCESS_KEY": "42",
"SECRET_ACCESS_KEY": "99"
})):
task.run(key="")
kwargs = client.call_args[1]
assert kwargs == {
"aws_access_key_id": "42",
"aws_secret_access_key": "99"
}
def test_raises_if_bucket_not_eventually_provided(self):
task = S3Download()
with pytest.raises(ValueError, match="bucket"):
task.run(key="")
def test_initialization_passes_to_task_constructor(self):
task = S3Download(name="test", tags=["AWS"])
assert task.name == "test"
assert task.tags == {"AWS"}
def test_initialization(self):
task = S3Download()
def test_raises_on_invalid_compression_method(self, mocked_boto_client):
task = S3Download("test")
with pytest.raises(ValueError, match="gz_fake"):
task.run("key", compression="gz_fake")
def test_raises_if_bucket_not_eventually_provided(self):
task = S3Download()
with pytest.raises(ValueError) as exc:
task.run(key="")
assert "bucket" in str(exc.value)
def test_initialization(self):
task = S3Download()
assert task.aws_credentials_secret == "AWS_CREDENTIALS"
def test_initialization(self):
task = S3Download()
assert task.aws_credentials_secret is None
Docker Storage - ECR
one image per flow
ECS RUN
Infra can be abstracted to another file
Execution
Run on ECS Agent
One K8s cluster, two agents
1 - ECS Agent to submit Fargate Tasks, "serverless" labels
2 - **Kubernetes Agent to submit K8s Jobs, "eks" labels
"""
my_s3_key = PrefectSecret(name="Secret Key")
buckets = Parameter(name="Bucket List", default=["users", "transactions"])
upload_data = S3Download(key=my_s3_key)
event_trigger = LambdaCreate()
@task
def transform_data(data):
pass
@task
def upload_to_redshift(data):
pass
with Flow("S3 to Redshift") as flow:
upload_data()
