def test_successful_file_handle_to_s3():
foo_bytes = "foo".encode()
with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle:
# Uses mock S3
s3 = boto3.client("s3")
s3.create_bucket(Bucket="some-bucket")
result = execute_pipeline(
create_file_handle_pipeline(temp_file_handle, s3),
run_config={
"solids": {
"file_handle_to_s3": {
"config": {
"Bucket": "some-bucket",
"Key": "some-key"
}
}
}
},
)
assert result.success
assert s3.get_object(Bucket="some-bucket",
Key="some-key")["Body"].read() == foo_bytes
materializations = result.result_for_solid(
"file_handle_to_s3").materializations_during_compute
assert len(materializations) == 1
assert len(materializations[0].metadata_entries) == 1
assert (materializations[0].metadata_entries[0].entry_data.path ==
"s3://some-bucket/some-key")
assert materializations[0].metadata_entries[0].label == "some-key"
def test_successful_file_handle_to_s3():
foo_bytes = 'foo'.encode()
with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle:
# Uses mock S3
s3 = boto3.client('s3')
s3.create_bucket(Bucket='some-bucket')
result = execute_pipeline(
create_file_handle_pipeline(temp_file_handle, s3),
environment_dict={
'solids': {
'file_handle_to_s3': {
'config': {
'Bucket': 'some-bucket',
'Key': 'some-key'
}
}
}
},
)
assert result.success
assert s3.get_object(Bucket='some-bucket',
Key='some-key')['Body'].read() == foo_bytes
materializations = result.result_for_solid(
'file_handle_to_s3').materializations_during_compute
assert len(materializations) == 1
assert len(materializations[0].metadata_entries) == 1
assert (materializations[0].metadata_entries[0].entry_data.path ==
's3://some-bucket/some-key')
assert materializations[0].metadata_entries[0].label == 'some-key'
def test_successful_file_handle_to_s3():
foo_bytes = 'foo'.encode()
with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle:
s3_fake_resource = create_s3_fake_resource()
result = execute_pipeline(
create_file_handle_pipeline(temp_file_handle, s3_fake_resource),
environment_dict={
'solids': {
'file_handle_to_s3': {
'config': {
'Bucket': 'some-bucket',
'Key': 'some-key'
}
}
}
},
)
assert result.success
assert s3_fake_resource.session.mock_extras.upload_fileobj.call_count == 1
assert (s3_fake_resource.session.get_object(
'some-bucket', 'some-key')['Body'].read() == foo_bytes)
materializations = result.result_for_solid(
'file_handle_to_s3').materializations_during_compute
assert len(materializations) == 1
assert len(materializations[0].metadata_entries) == 1
assert (materializations[0].metadata_entries[0].entry_data.path ==
's3://some-bucket/some-key')
assert materializations[0].metadata_entries[0].label == 'some-key'
def test_successful_file_handle_to_s3(bucket):
foo_bytes = "foo".encode()
with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle:
result = execute_pipeline(
create_file_handle_pipeline(temp_file_handle),
run_config={
"solids": {
"file_handle_to_s3": {
"config": {
"Bucket": bucket.name,
"Key": "some-key"
}
}
}
},
)
assert result.success
assert bucket.Object(key="some-key").get()["Body"].read() == foo_bytes
materializations = result.result_for_solid(
"file_handle_to_s3").materializations_during_compute
assert len(materializations) == 1
assert len(materializations[0].metadata_entries) == 1
assert materializations[0].metadata_entries[
0].entry_data.path == "s3://{bucket}/some-key".format(
bucket=bucket.name)
assert materializations[0].metadata_entries[0].label == "some-key"
def test_successful_file_handle_to_s3_with_configs():
foo_bytes = 'foo'.encode()
with get_temp_file_handle_with_data(foo_bytes) as temp_file_handle:
s3_fake_resource = create_s3_fake_resource()
result = execute_pipeline(
create_file_handle_pipeline(temp_file_handle, s3_fake_resource),
environment_dict={
'solids': {
'file_handle_to_s3': {
'config': {
'Bucket': 'some-bucket',
'Key': 'some-key',
'CacheControl': 'some-value',
}
}
}
},
)
assert result.success
s3_fake_resource.session.mock_extras.put_object.assert_called_once_with(
CacheControl='some-value'
)
def test_basic_file_manager_copy_handle_to_local_temp():
foo_data = 'foo'.encode()
with get_temp_dir() as temp_dir:
with get_temp_file_handle_with_data(foo_data) as foo_handle:
with local_file_manager(temp_dir) as manager:
local_temp = manager.copy_handle_to_local_temp(foo_handle)
assert local_temp != foo_handle.path
with open(local_temp, 'rb') as ff:
assert ff.read() == foo_data
