def test_boto3_client_is_created_with_session(self, mocked_boto_client): """ Tests the fix for #3925 """ task = S3Download("test") task.run("key") assert ( "session.Session()" in mocked_boto_client.return_value._extract_mock_name() )
def test_raises_on_invalid_compression_method(self, monkeypatch): task = S3Download("test") client = MagicMock() boto3 = MagicMock(client=MagicMock(return_value=client)) monkeypatch.setattr("prefect.utilities.aws.boto3", boto3) with pytest.raises(ValueError, match="gz_fake"): task.run("key", compression="gz_fake")
def test_gzip_compression(self, mocked_boto_client): task = S3Download("bucket") byte_string = b"col1,col2,col3\nfake,data,1\nfalse,data,2\n" gzip_data = gzip.compress(byte_string) def modify_stream(Bucket=None, Key=None, Fileobj=None): Fileobj.write(gzip_data) mocked_boto_client.download_fileobj.side_effect = modify_stream returned_data = task.run("key", compression="gzip") assert returned_data == str(byte_string, "utf-8")
def test_creds_default_to_environment(self, monkeypatch): task = S3Download(bucket="bob") client = MagicMock() boto3 = MagicMock(client=client) monkeypatch.setattr("prefect.tasks.aws.s3.boto3", boto3) task.run(key="") kwargs = client.call_args[1] assert kwargs == { "aws_access_key_id": None, "aws_secret_access_key": None }
def test_as_bytes(self, mocked_boto_client, as_bytes): task = S3Download("bucket") data = b"some bytes" def modify_stream(Bucket=None, Key=None, Fileobj=None): Fileobj.write(data) mocked_boto_client.download_fileobj.side_effect = modify_stream res = task.run("key", as_bytes=as_bytes) if as_bytes: assert res == data else: assert res == data.decode("utf-8")
def test_gzip_compression(self, monkeypatch): task = S3Download("bucket") byte_string = b"col1,col2,col3\nfake,data,1\nfalse,data,2\n" gzip_data = gzip.compress(byte_string) def modify_stream(Bucket=None, Key=None, Fileobj=None): Fileobj.write(gzip_data) client = MagicMock() boto3 = MagicMock(client=MagicMock(return_value=client)) monkeypatch.setattr("prefect.utilities.aws.boto3", boto3) client.download_fileobj.side_effect = modify_stream returned_data = task.run("key", compression="gzip") assert returned_data == str(byte_string, "utf-8")
def test_creds_are_pulled_from_secret(self, monkeypatch): task = S3Download(bucket="bob") client = MagicMock() boto3 = MagicMock(client=client) monkeypatch.setattr("prefect.tasks.aws.s3.boto3", boto3) with set_temporary_config({"cloud.use_local_secrets": True}): with prefect.context(secrets=dict(AWS_CREDENTIALS={ "ACCESS_KEY": "42", "SECRET_ACCESS_KEY": "99" })): task.run(key="") kwargs = client.call_args[1] assert kwargs == { "aws_access_key_id": "42", "aws_secret_access_key": "99" }
def test_raises_if_bucket_not_eventually_provided(self): task = S3Download() with pytest.raises(ValueError, match="bucket"): task.run(key="")
def test_initialization_passes_to_task_constructor(self): task = S3Download(name="test", tags=["AWS"]) assert task.name == "test" assert task.tags == {"AWS"}
def test_initialization(self): task = S3Download()
def test_raises_on_invalid_compression_method(self, mocked_boto_client): task = S3Download("test") with pytest.raises(ValueError, match="gz_fake"): task.run("key", compression="gz_fake")
def test_raises_if_bucket_not_eventually_provided(self): task = S3Download() with pytest.raises(ValueError) as exc: task.run(key="") assert "bucket" in str(exc.value)
def test_initialization(self): task = S3Download() assert task.aws_credentials_secret == "AWS_CREDENTIALS"
def test_initialization(self): task = S3Download() assert task.aws_credentials_secret is None
Docker Storage - ECR one image per flow ECS RUN Infra can be abstracted to another file Execution Run on ECS Agent One K8s cluster, two agents 1 - ECS Agent to submit Fargate Tasks, "serverless" labels 2 - **Kubernetes Agent to submit K8s Jobs, "eks" labels """ my_s3_key = PrefectSecret(name="Secret Key") buckets = Parameter(name="Bucket List", default=["users", "transactions"]) upload_data = S3Download(key=my_s3_key) event_trigger = LambdaCreate() @task def transform_data(data): pass @task def upload_to_redshift(data): pass with Flow("S3 to Redshift") as flow: upload_data()