def test_upload_numpy_to_s3_shards():
mock_s3 = Mock()
mock_object = Mock()
mock_s3.Object = Mock(return_value=mock_object)
mock_put = mock_s3.Object.return_value.put
array = np.array([[j for j in range(10)] for i in range(10)])
labels = np.array([i for i in range(10)])
num_shards = 3
num_objects = num_shards + 1 # Account for the manifest file.
def make_all_put_calls(**kwargs):
return [call(Body=ANY, **kwargs) for i in range(num_objects)]
upload_numpy_to_s3_shards(num_shards, mock_s3, BUCKET_NAME, "key-prefix",
array, labels)
mock_s3.Object.assert_has_calls(
[call(BUCKET_NAME, "key-prefix/matrix_0.pbr")])
mock_s3.Object.assert_has_calls(
[call(BUCKET_NAME, "key-prefix/matrix_1.pbr")])
mock_s3.Object.assert_has_calls(
[call(BUCKET_NAME, "key-prefix/matrix_2.pbr")])
mock_put.assert_has_calls(make_all_put_calls())
mock_put.reset()
upload_numpy_to_s3_shards(3,
mock_s3,
BUCKET_NAME,
"key-prefix",
array,
labels,
encrypt=True)
mock_put.assert_has_calls(
make_all_put_calls(ServerSideEncryption="AES256"))
def test_upload_numpy_to_s3_shards():
mock_s3 = Mock()
mock_object = Mock()
mock_s3.Object = Mock(return_value=mock_object)
array = np.array([[j for j in range(10)] for i in range(10)])
labels = np.array([i for i in range(10)])
upload_numpy_to_s3_shards(3, mock_s3, BUCKET_NAME, "key-prefix", array, labels)
mock_s3.Object.assert_has_calls([call(BUCKET_NAME, 'key-prefix/matrix_0.pbr')])
mock_s3.Object.assert_has_calls([call(BUCKET_NAME, 'key-prefix/matrix_1.pbr')])
mock_s3.Object.assert_has_calls([call(BUCKET_NAME, 'key-prefix/matrix_2.pbr')])
def test_upload_numpy_to_s3_shards():
mock_s3 = Mock()
mock_object = Mock()
mock_s3.Object = Mock(return_value=mock_object)
array = np.array([[j for j in range(10)] for i in range(10)])
labels = np.array([i for i in range(10)])
upload_numpy_to_s3_shards(3, mock_s3, BUCKET_NAME, "key-prefix", array, labels)
mock_s3.Object.assert_has_calls([call(BUCKET_NAME, 'key-prefix/matrix_0.pbr')])
mock_s3.Object.assert_has_calls([call(BUCKET_NAME, 'key-prefix/matrix_1.pbr')])
mock_s3.Object.assert_has_calls([call(BUCKET_NAME, 'key-prefix/matrix_2.pbr')])
