def test_init_grpc(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, projection = gcs.bucket.Bucket.init(request, "")
assert bucket.metadata.name == "bucket"
assert projection == 1
assert list(bucket.metadata.acl) == utils.acl.compute_predefined_bucket_acl(
"bucket", 3, ""
)
assert list(
bucket.metadata.default_object_acl
) == utils.acl.compute_predefined_default_object_acl("bucket", 5, "")
# WITH ACL
request = storage_pb2.InsertBucketRequest(
bucket={
"name": "bucket",
"acl": utils.acl.compute_predefined_bucket_acl("bucket", 1, ""),
}
)
bucket, projection = gcs.bucket.Bucket.init(request, "")
assert bucket.metadata.name == "bucket"
assert projection == 2
assert list(bucket.metadata.acl) == utils.acl.compute_predefined_bucket_acl(
"bucket", 1, ""
)
assert list(
bucket.metadata.default_object_acl
) == utils.acl.compute_predefined_default_object_acl("bucket", 5, "")
# WITH PREDEFINED ACL
request = storage_pb2.InsertBucketRequest(
bucket={"name": "bucket"}, predefined_acl=5
)
bucket, projection = gcs.bucket.Bucket.init(request, "")
assert bucket.metadata.name == "bucket"
assert projection == 1
assert list(bucket.metadata.acl) == utils.acl.compute_predefined_bucket_acl(
"bucket", 5, ""
)
assert list(
bucket.metadata.default_object_acl
) == utils.acl.compute_predefined_default_object_acl("bucket", 5, "")
# WITH ACL AND PREDEFINED ACL
request = storage_pb2.InsertBucketRequest(
bucket={
"name": "bucket",
"acl": utils.acl.compute_predefined_bucket_acl("bucket", 1, ""),
},
predefined_acl=2,
)
bucket, projection = gcs.bucket.Bucket.init(request, "")
assert bucket.metadata.name == "bucket"
assert projection == 2
assert list(bucket.metadata.acl) == utils.acl.compute_predefined_bucket_acl(
"bucket", 1, ""
)
assert list(
bucket.metadata.default_object_acl
) == utils.acl.compute_predefined_default_object_acl("bucket", 5, "")
def test_default_object_acl(self):
# Both REST and GRPC share almost the same implementation so we only test GRPC here.
entity = "*****@*****.**"
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, projection = gcs.bucket.Bucket.init(request, "")
request = storage_pb2.InsertDefaultObjectAccessControlRequest(
bucket="bucket",
object_access_control={
"entity": entity,
"role": "READER"
})
bucket.insert_default_object_acl(request, "")
acl = bucket.get_default_object_acl(entity, "")
self.assertEqual(acl.entity, entity)
self.assertEqual(acl.email, "*****@*****.**")
self.assertEqual(acl.role, "READER")
request = storage_pb2.PatchDefaultObjectAccessControlRequest(
bucket="bucket",
entity=entity,
object_access_control={"role": "OWNER"})
bucket.patch_default_object_acl(request, entity, "")
acl = bucket.get_default_object_acl(entity, "")
self.assertEqual(acl.entity, entity)
self.assertEqual(acl.email, "*****@*****.**")
self.assertEqual(acl.role, "OWNER")
bucket.delete_default_object_acl(entity, "")
with self.assertRaises(Exception):
bucket.get_default_object_acl(entity, None)
def test_extract_predefined_default_object_acl(self):
request = storage_pb2.InsertBucketRequest()
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, ""
)
self.assertEqual(
predefined_default_object_acl,
CommonEnums.PredefinedObjectAcl.PREDEFINED_OBJECT_ACL_UNSPECIFIED,
)
request.predefined_default_object_acl = (
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_AUTHENTICATED_READ
)
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, ""
)
self.assertEqual(
predefined_default_object_acl,
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_AUTHENTICATED_READ,
)
request = utils.common.FakeRequest(args={})
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, None
)
self.assertEqual(predefined_default_object_acl, "")
request.args["predefinedDefaultObjectAcl"] = "authenticatedRead"
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, None
)
self.assertEqual(predefined_default_object_acl, "authenticatedRead")
def test_default_object_acl(self):
# Both REST and GRPC share almost the same implementation so we only test GRPC here.
entity = "*****@*****.**"
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, projection = gcs.bucket.Bucket.init(request, "")
request = storage_pb2.InsertDefaultObjectAccessControlRequest(
bucket="bucket",
object_access_control={
"entity": entity,
"role": "READER"
})
bucket.insert_default_object_acl(request, "")
acl = bucket.get_default_object_acl(entity, "")
assert acl.entity == entity
assert acl.email == "*****@*****.**"
assert acl.role == "READER"
request = storage_pb2.PatchDefaultObjectAccessControlRequest(
bucket="bucket",
entity=entity,
object_access_control={"role": "OWNER"})
bucket.patch_default_object_acl(request, entity, "")
acl = bucket.get_default_object_acl(entity, "")
assert acl.entity == entity
assert acl.email == "*****@*****.**"
assert acl.role == "OWNER"
bucket.delete_default_object_acl(entity, "")
for acl in bucket.metadata.default_object_acl:
assert acl.entity != entity
def test_extract_predefined_acl(self):
request = storage_pb2.InsertBucketRequest()
predefined_acl = utils.acl.extract_predefined_acl(request, False, "")
self.assertEqual(
predefined_acl,
CommonEnums.PredefinedBucketAcl.PREDEFINED_BUCKET_ACL_UNSPECIFIED,
)
request.predefined_acl = (
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_AUTHENTICATED_READ)
predefined_acl = utils.acl.extract_predefined_acl(request, False, "")
self.assertEqual(
predefined_acl,
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_AUTHENTICATED_READ,
)
request = storage_pb2.CopyObjectRequest(
destination_predefined_acl=CommonEnums.PredefinedBucketAcl.
BUCKET_ACL_PRIVATE)
predefined_acl = utils.acl.extract_predefined_acl(request, True, "")
self.assertEqual(predefined_acl,
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_PRIVATE)
request = utils.common.FakeRequest(args={})
predefined_acl = utils.acl.extract_predefined_acl(request, False, None)
self.assertEqual(predefined_acl, "")
request.args["predefinedAcl"] = "authenticatedRead"
predefined_acl = utils.acl.extract_predefined_acl(request, False, None)
self.assertEqual(predefined_acl, "authenticatedRead")
request.args["destinationPredefinedAcl"] = "bucketOwnerFullControl"
predefined_acl = utils.acl.extract_predefined_acl(request, True, None)
self.assertEqual(predefined_acl, "bucketOwnerFullControl")
def testInitMedia(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, projection = gcs.bucket.Bucket.init(request, "")
request = utils.common.FakeRequest(
args={"name": "object"}, data=b"12345678", headers={}
)
blob, _ = gcs.object.Object.init_media(request, bucket.metadata)
assert blob.metadata.name == "object"
assert blob.media == b"12345678"
def testInitMultiPart(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
request = utils.common.FakeRequest(
args={},
headers={"content-type": "multipart/related; boundary=foo_bar_baz"},
data=b'--foo_bar_baz\r\nContent-Type: application/json; charset=UTF-8\r\n{"name": "object", "metadata": {"key": "value"}}\r\n--foo_bar_baz\r\nContent-Type: image/jpeg\r\n123456789\r\n--foo_bar_baz--\r\n',
)
blob, _ = gcs.object.Object.init_multipart(request, bucket.metadata)
assert blob.metadata.name == "object"
assert blob.media == b"123456789"
assert blob.metadata.metadata["key"] == "value"
def test_patch(self):
# Updating requires a full metadata so we don't test it here.
request = storage_pb2.InsertBucketRequest(
bucket={
"name": "bucket",
"labels": {"init": "true", "patch": "false"},
"website": {"not_found_page": "notfound.html"},
}
)
bucket, projection = gcs.bucket.Bucket.init(request, "")
assert bucket.metadata.labels.get("init") == "true"
assert bucket.metadata.labels.get("patch") == "false"
assert bucket.metadata.labels.get("method") is None
assert bucket.metadata.website.main_page_suffix == ""
assert bucket.metadata.website.not_found_page == "notfound.html"
request = storage_pb2.PatchBucketRequest(
bucket="bucket",
metadata={
"labels": {"patch": "true", "method": "grpc"},
"website": {"main_page_suffix": "bucket", "not_found_page": "404"},
},
update_mask={"paths": ["labels", "website.main_page_suffix"]},
)
bucket.patch(request, "")
# GRPC can not update a part of map field.
assert bucket.metadata.labels.get("init") is None
assert bucket.metadata.labels.get("patch") == "true"
assert bucket.metadata.labels.get("method") == "grpc"
assert bucket.metadata.website.main_page_suffix == "bucket"
# `update_mask` does not update `website.not_found_page`
assert bucket.metadata.website.not_found_page == "notfound.html"
request = utils.common.FakeRequest(
args={},
data=json.dumps(
{
"name": "new_bucket",
"labels": {"method": "rest"},
"website": {"notFoundPage": "404.html"},
}
),
)
bucket.patch(request, None)
# REST should only update modifiable field.
assert bucket.metadata.name == "bucket"
# REST can update a part of map field.
assert bucket.metadata.labels.get("init") is None
assert bucket.metadata.labels.get("patch") == "true"
assert bucket.metadata.labels.get("method") == "rest"
assert bucket.metadata.website.main_page_suffix == "bucket"
assert bucket.metadata.website.not_found_page == "404.html"
def insert_test_bucket(self, context):
bucket_name = os.environ.get(
"GOOGLE_CLOUD_CPP_STORAGE_TEST_BUCKET_NAME", "bucket")
if self.buckets.get(bucket_name) is None:
if context is not None:
request = storage_pb2.InsertBucketRequest(
bucket={"name": bucket_name})
else:
request = utils.common.FakeRequest(args={},
data=simdjson.dumps(
{"name": bucket_name}))
bucket_test, _ = gcs.bucket.Bucket.init(request, context)
self.insert_bucket(request, bucket_test, context)
bucket_test.metadata.metageneration = 4
bucket_test.metadata.versioning.enabled = True
def test_init_resumable_grpc(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
bucket = bucket.metadata
insert_object_spec = storage_pb2.InsertObjectSpec(
resource={
"name": "object",
"bucket": "bucket"
},
predefined_acl=CommonEnums.PredefinedObjectAcl.
OBJECT_ACL_PROJECT_PRIVATE,
if_generation_not_match={"value": 1},
if_metageneration_match={"value": 2},
if_metageneration_not_match={"value": 3},
projection=CommonEnums.Projection.FULL,
)
request = storage_pb2.InsertObjectRequest(
insert_object_spec=insert_object_spec, write_offset=0)
upload = gcs.holder.DataHolder.init_resumable_grpc(request, bucket, "")
# Verify the annotations inserted by the emulator.
annotations = upload.metadata.metadata
self.assertGreaterEqual(
set([
"x_emulator_upload", "x_emulator_no_crc32c",
"x_emulator_no_md5"
]),
set(annotations.keys()),
)
# Clear any annotations created by the emulator
upload.metadata.metadata.clear()
self.assertEqual(upload.metadata,
resources_pb2.Object(name="object", bucket="bucket"))
predefined_acl = utils.acl.extract_predefined_acl(
upload.request, False, "")
self.assertEqual(
predefined_acl,
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE)
match, not_match = utils.generation.extract_precondition(
upload.request, False, False, "")
self.assertIsNone(match)
self.assertEqual(not_match, 1)
match, not_match = utils.generation.extract_precondition(
upload.request, True, False, "")
self.assertEqual(match, 2)
self.assertEqual(not_match, 3)
projection = utils.common.extract_projection(upload.request, False, "")
self.assertEqual(projection, CommonEnums.Projection.FULL)
def test_extract_predefined_default_object_acl(self):
request = storage_pb2.InsertBucketRequest()
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, "")
assert predefined_default_object_acl == 0
request.predefined_default_object_acl = 1
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, "")
assert predefined_default_object_acl == 1
request = utils.common.FakeRequest(args={})
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, None)
assert predefined_default_object_acl == ""
request.args["predefinedDefaultObjectAcl"] = "authenticatedRead"
predefined_default_object_acl = utils.acl.extract_predefined_default_object_acl(
request, None)
assert predefined_default_object_acl == "authenticatedRead"
def test_grpc_to_rest(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, projection = gcs.bucket.Bucket.init(request, "")
self.assertEqual(bucket.metadata.name, "bucket")
# `REST` GET
rest_metadata = bucket.rest()
self.assertEqual(rest_metadata["name"], "bucket")
self.assertIsNone(bucket.metadata.labels.get("method"))
# `REST` PATCH
request = utils.common.FakeRequest(args={},
data=json.dumps(
{"labels": {
"method": "rest"
}}))
bucket.patch(request, None)
self.assertEqual(bucket.metadata.labels["method"], "rest")
def test_init_resumable_rest(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
bucket = bucket.metadata
data = json.dumps({
# Empty payload checksums
"crc32c": "AAAAAA==",
"md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==",
"name": "test-object-name",
})
environ = create_environ(
base_url="http://localhost:8080",
content_length=len(data),
data=data,
content_type="application/json",
method="POST",
)
upload = gcs.holder.DataHolder.init_resumable_rest(
Request(environ), bucket)
self.assertEqual(upload.metadata.name, "test-object-name")
self.assertEqual(upload.metadata.crc32c.value, 0)
self.assertEqual(upload.metadata.md5_hash, "1B2M2Y8AsgTpgAmY7PhCfg==")
def test_extract_predefined_acl(self):
request = storage_pb2.InsertBucketRequest()
predefined_acl = utils.acl.extract_predefined_acl(request, False, "")
assert predefined_acl == 0
request.predefined_acl = 1
predefined_acl = utils.acl.extract_predefined_acl(request, False, "")
assert predefined_acl == 1
request = storage_pb2.CopyObjectRequest(destination_predefined_acl=2)
predefined_acl = utils.acl.extract_predefined_acl(request, True, "")
assert predefined_acl == 2
request = utils.common.FakeRequest(args={})
predefined_acl = utils.acl.extract_predefined_acl(request, False, None)
assert predefined_acl == ""
request.args["predefinedAcl"] = "authenticatedRead"
predefined_acl = utils.acl.extract_predefined_acl(request, False, None)
assert predefined_acl == "authenticatedRead"
request.args["destinationPredefinedAcl"] = "bucketOwnerFullControl"
predefined_acl = utils.acl.extract_predefined_acl(request, True, None)
assert predefined_acl == "bucketOwnerFullControl"
def test_init_resumable_grpc(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
bucket = bucket.metadata
insert_object_spec = storage_pb2.InsertObjectSpec(
resource={
"name": "object",
"bucket": "bucket"
},
predefined_acl=CommonEnums.PredefinedObjectAcl.
OBJECT_ACL_PROJECT_PRIVATE,
if_generation_not_match={"value": 1},
if_metageneration_match={"value": 2},
if_metageneration_not_match={"value": 3},
projection=CommonEnums.Projection.FULL,
)
request = storage_pb2.InsertObjectRequest(
insert_object_spec=insert_object_spec, write_offset=0)
upload = gcs.holder.DataHolder.init_resumable_grpc(request, bucket, "")
self.assertEqual(upload.metadata,
resources_pb2.Object(name="object", bucket="bucket"))
predefined_acl = utils.acl.extract_predefined_acl(
upload.request, False, "")
self.assertEqual(
predefined_acl,
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE)
match, not_match = utils.generation.extract_precondition(
upload.request, False, False, "")
self.assertIsNone(match)
self.assertEqual(not_match, 1)
match, not_match = utils.generation.extract_precondition(
upload.request, True, False, "")
self.assertEqual(match, 2)
self.assertEqual(not_match, 3)
projection = utils.common.extract_projection(upload.request, False, "")
self.assertEqual(projection, CommonEnums.Projection.FULL)
def setUp(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
self.bucket = bucket
def test_init_grpc(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, projection = gcs.bucket.Bucket.init(request, "")
self.assertEqual(bucket.metadata.name, "bucket")
self.assertEqual(projection, CommonEnums.Projection.NO_ACL)
self.assertListEqual(
list(bucket.metadata.acl),
utils.acl.compute_predefined_bucket_acl(
"bucket",
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_PROJECT_PRIVATE,
""),
)
self.assertListEqual(
list(bucket.metadata.default_object_acl),
utils.acl.compute_predefined_default_object_acl(
"bucket",
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE,
""),
)
# WITH ACL
request = storage_pb2.InsertBucketRequest(
bucket={
"name":
"bucket",
"acl":
utils.acl.compute_predefined_bucket_acl(
"bucket",
CommonEnums.PredefinedBucketAcl.
BUCKET_ACL_AUTHENTICATED_READ,
"",
),
})
bucket, projection = gcs.bucket.Bucket.init(request, "")
self.assertEqual(bucket.metadata.name, "bucket")
self.assertEqual(projection, CommonEnums.Projection.FULL)
self.assertEqual(
list(bucket.metadata.acl),
utils.acl.compute_predefined_bucket_acl(
"bucket",
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_AUTHENTICATED_READ,
"",
),
)
self.assertListEqual(
list(bucket.metadata.default_object_acl),
utils.acl.compute_predefined_default_object_acl(
"bucket",
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE,
""),
)
# WITH PREDEFINED ACL
request = storage_pb2.InsertBucketRequest(
bucket={"name": "bucket"},
predefined_acl=CommonEnums.PredefinedBucketAcl.
BUCKET_ACL_PUBLIC_READ_WRITE,
)
bucket, projection = gcs.bucket.Bucket.init(request, "")
self.assertEqual(bucket.metadata.name, "bucket")
self.assertEqual(projection, CommonEnums.Projection.NO_ACL)
self.assertEqual(
list(bucket.metadata.acl),
utils.acl.compute_predefined_bucket_acl(
"bucket",
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_PUBLIC_READ_WRITE,
"",
),
)
self.assertListEqual(
list(bucket.metadata.default_object_acl),
utils.acl.compute_predefined_default_object_acl(
"bucket",
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE,
""),
)
# WITH ACL AND PREDEFINED ACL
request = storage_pb2.InsertBucketRequest(
bucket={
"name":
"bucket",
"acl":
utils.acl.compute_predefined_bucket_acl(
"bucket",
CommonEnums.PredefinedBucketAcl.
BUCKET_ACL_AUTHENTICATED_READ,
"",
),
},
predefined_acl=CommonEnums.PredefinedBucketAcl.BUCKET_ACL_PRIVATE,
)
bucket, projection = gcs.bucket.Bucket.init(request, "")
self.assertEqual(bucket.metadata.name, "bucket")
self.assertEqual(projection, CommonEnums.Projection.FULL)
self.assertEqual(
list(bucket.metadata.acl),
utils.acl.compute_predefined_bucket_acl(
"bucket",
CommonEnums.PredefinedBucketAcl.BUCKET_ACL_AUTHENTICATED_READ,
"",
),
)
self.assertListEqual(
list(bucket.metadata.default_object_acl),
utils.acl.compute_predefined_default_object_acl(
"bucket",
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE,
""),
)
def test_patch(self):
# Updating requires a full metadata so we don't test it here.
request = storage_pb2.InsertBucketRequest(
bucket={
"name": "bucket",
"labels": {
"init": "true",
"patch": "false"
},
"website": {
"not_found_page": "notfound.html"
},
})
bucket, projection = gcs.bucket.Bucket.init(request, "")
self.assertEqual(bucket.metadata.labels.get("init"), "true")
self.assertEqual(bucket.metadata.labels.get("patch"), "false")
self.assertIsNone(bucket.metadata.labels.get("method"))
self.assertEqual(bucket.metadata.website.main_page_suffix, "")
self.assertEqual(bucket.metadata.website.not_found_page,
"notfound.html")
request = storage_pb2.PatchBucketRequest(
bucket="bucket",
metadata={
"labels": {
"patch": "true",
"method": "grpc"
},
"website": {
"main_page_suffix": "bucket",
"not_found_page": "404"
},
},
update_mask={"paths": ["labels", "website.main_page_suffix"]},
)
bucket.patch(request, "")
# GRPC can not update a part of map field.
self.assertIsNone(bucket.metadata.labels.get("init"))
self.assertEqual(bucket.metadata.labels.get("patch"), "true")
self.assertEqual(bucket.metadata.labels.get("method"), "grpc")
self.assertEqual(bucket.metadata.website.main_page_suffix, "bucket")
# `update_mask` does not update `website.not_found_page`
self.assertEqual(bucket.metadata.website.not_found_page,
"notfound.html")
request = utils.common.FakeRequest(
args={},
data=json.dumps({
"name": "new_bucket",
"labels": {
"method": "rest"
},
"website": {
"notFoundPage": "404.html"
},
}),
)
bucket.patch(request, None)
# REST should only update modifiable field.
self.assertEqual(bucket.metadata.name, "bucket")
# REST can update a part of map field.
self.assertIsNone(bucket.metadata.labels.get("init"))
self.assertEqual(bucket.metadata.labels.get("patch"), "true")
self.assertEqual(bucket.metadata.labels.get("method"), "rest")
self.assertEqual(bucket.metadata.website.main_page_suffix, "bucket")
self.assertEqual(bucket.metadata.website.not_found_page, "404.html")
# We want to make sure REST `UPDATE` does not throw any exception.
request = utils.common.FakeRequest(
args={}, data=json.dumps({"labels": {
"method": "rest_update"
}}))
bucket.update(request, None)
self.assertEqual(bucket.metadata.labels["method"], "rest_update")