def test_successful_file_handle_to_s3(): foo_bytes = "foo".encode() with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle: # Uses mock S3 s3 = boto3.client("s3") s3.create_bucket(Bucket="some-bucket") result = execute_pipeline( create_file_handle_pipeline(temp_file_handle, s3), run_config={ "solids": { "file_handle_to_s3": { "config": { "Bucket": "some-bucket", "Key": "some-key" } } } }, ) assert result.success assert s3.get_object(Bucket="some-bucket", Key="some-key")["Body"].read() == foo_bytes materializations = result.result_for_solid( "file_handle_to_s3").materializations_during_compute assert len(materializations) == 1 assert len(materializations[0].metadata_entries) == 1 assert (materializations[0].metadata_entries[0].entry_data.path == "s3://some-bucket/some-key") assert materializations[0].metadata_entries[0].label == "some-key"
def test_successful_file_handle_to_s3(): foo_bytes = 'foo'.encode() with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle: # Uses mock S3 s3 = boto3.client('s3') s3.create_bucket(Bucket='some-bucket') result = execute_pipeline( create_file_handle_pipeline(temp_file_handle, s3), environment_dict={ 'solids': { 'file_handle_to_s3': { 'config': { 'Bucket': 'some-bucket', 'Key': 'some-key' } } } }, ) assert result.success assert s3.get_object(Bucket='some-bucket', Key='some-key')['Body'].read() == foo_bytes materializations = result.result_for_solid( 'file_handle_to_s3').materializations_during_compute assert len(materializations) == 1 assert len(materializations[0].metadata_entries) == 1 assert (materializations[0].metadata_entries[0].entry_data.path == 's3://some-bucket/some-key') assert materializations[0].metadata_entries[0].label == 'some-key'
def test_successful_file_handle_to_s3(): foo_bytes = 'foo'.encode() with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle: s3_fake_resource = create_s3_fake_resource() result = execute_pipeline( create_file_handle_pipeline(temp_file_handle, s3_fake_resource), environment_dict={ 'solids': { 'file_handle_to_s3': { 'config': { 'Bucket': 'some-bucket', 'Key': 'some-key' } } } }, ) assert result.success assert s3_fake_resource.session.mock_extras.upload_fileobj.call_count == 1 assert (s3_fake_resource.session.get_object( 'some-bucket', 'some-key')['Body'].read() == foo_bytes) materializations = result.result_for_solid( 'file_handle_to_s3').materializations_during_compute assert len(materializations) == 1 assert len(materializations[0].metadata_entries) == 1 assert (materializations[0].metadata_entries[0].entry_data.path == 's3://some-bucket/some-key') assert materializations[0].metadata_entries[0].label == 'some-key'
def test_successful_file_handle_to_s3(bucket): foo_bytes = "foo".encode() with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle: result = execute_pipeline( create_file_handle_pipeline(temp_file_handle), run_config={ "solids": { "file_handle_to_s3": { "config": { "Bucket": bucket.name, "Key": "some-key" } } } }, ) assert result.success assert bucket.Object(key="some-key").get()["Body"].read() == foo_bytes materializations = result.result_for_solid( "file_handle_to_s3").materializations_during_compute assert len(materializations) == 1 assert len(materializations[0].metadata_entries) == 1 assert materializations[0].metadata_entries[ 0].entry_data.path == "s3://{bucket}/some-key".format( bucket=bucket.name) assert materializations[0].metadata_entries[0].label == "some-key"
def test_successful_file_handle_to_s3_with_configs(): foo_bytes = 'foo'.encode() with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle: s3_fake_resource = create_s3_fake_resource() result = execute_pipeline( create_file_handle_pipeline(temp_file_handle, s3_fake_resource), environment_dict={ 'solids': { 'file_handle_to_s3': { 'config': { 'Bucket': 'some-bucket', 'Key': 'some-key', 'CacheControl': 'some-value', } } } }, ) assert result.success s3_fake_resource.session.mock_extras.put_object.assert_called_once_with( CacheControl='some-value' )
def test_basic_file_manager_copy_handle_to_local_temp(): foo_data = 'foo'.encode() with get_temp_dir() as temp_dir: with get_temp_file_handle_with_data(foo_data) as foo_handle: with local_file_manager(temp_dir) as manager: local_temp = manager.copy_handle_to_local_temp(foo_handle) assert local_temp != foo_handle.path with open(local_temp, 'rb') as ff: assert ff.read() == foo_data