def init_resumable_rest(cls, request, bucket):
name = request.args.get("name", "")
rest_only = {}
if len(request.data) > 0 and request.data.decode("utf-8") != "{}":
if name != "":
utils.error.invalid("name argument in non-empty payload", None)
data = json.loads(request.data)
rest_only = cls.__extract_rest_only(data)
metadata = json_format.ParseDict(data, resources_pb2.Object())
else:
metadata = resources_pb2.Object()
metadata.name = name
if metadata.content_type == "":
metadata.content_type = request.headers.get(
"x-upload-content-type", "application/octet-stream")
upload_id = hashlib.sha256(
("%s/o/%s" %
(bucket.name, metadata.name)).encode("utf-8")).hexdigest()
location = (
request.host_url +
"upload/storage/v1/b/%s/o?uploadType=resumable&upload_id=%s" %
(bucket.name, upload_id))
headers = {
key.lower(): value
for key, value in request.headers.items()
if key.lower().startswith("x-")
}
request = utils.common.FakeRequest(args=request.args.to_dict(),
headers=headers,
data=b"")
return cls.init_upload(request, metadata, bucket, location, upload_id,
rest_only)
def objects_rewrite(src_bucket_name, src_object_name, dst_bucket_name,
dst_object_name):
db.insert_test_bucket(None)
token, rewrite = flask.request.args.get("rewriteToken"), None
src_object = None
if token is None:
rewrite = gcs_type.holder.DataHolder.init_rewrite_rest(
flask.request,
src_bucket_name,
src_object_name,
dst_bucket_name,
dst_object_name,
)
db.insert_rewrite(rewrite)
else:
rewrite = db.get_rewrite(token, None)
src_object = db.get_object(rewrite.request, src_bucket_name,
src_object_name, True, None)
utils.csek.validation(rewrite.request,
src_object.metadata.customer_encryption.key_sha256,
True, None)
total_bytes_rewritten = len(rewrite.media)
total_bytes_rewritten += min(rewrite.max_bytes_rewritten_per_call,
len(src_object.media) - len(rewrite.media))
rewrite.media += src_object.media[len(rewrite.media):total_bytes_rewritten]
done, dst_object = total_bytes_rewritten == len(src_object.media), None
response = {
"kind": "storage#rewriteResponse",
"totalBytesRewritten": len(rewrite.media),
"objectSize": len(src_object.media),
"done": done,
}
if done:
dst_bucket = db.get_bucket_without_generation(dst_bucket_name,
None).metadata
dst_metadata = resources_pb2.Object()
dst_metadata.CopyFrom(src_object.metadata)
dst_rest_only = dict(src_object.rest_only)
dst_metadata.bucket = dst_bucket_name
dst_metadata.name = dst_object_name
dst_media = rewrite.media
dst_object, _ = gcs_type.object.Object.init(
flask.request,
dst_metadata,
dst_media,
dst_bucket,
True,
None,
dst_rest_only,
)
db.insert_object(flask.request, dst_bucket_name, dst_object, None)
dst_object.patch(rewrite.request, None)
dst_object.metadata.metageneration = 1
dst_object.metadata.updated.FromDatetime(
dst_object.metadata.time_created.ToDatetime())
resources = dst_object.rest_metadata()
response["resource"] = resources
else:
response["rewriteToken"] = rewrite.token
return response
def init_dict(cls, request, metadata, media, bucket, is_destination):
rest_only = {}
if "customTime" in metadata:
rest_only["customTime"] = metadata.pop("customTime")
metadata = json_format.ParseDict(metadata, resources_pb2.Object())
return cls.init(request, metadata, media, bucket, is_destination, None,
rest_only)
def objects_copy(src_bucket_name, src_object_name, dst_bucket_name,
dst_object_name):
db.insert_test_bucket(None)
dst_bucket = db.get_bucket_without_generation(dst_bucket_name,
None).metadata
src_object = db.get_object(flask.request, src_bucket_name, src_object_name,
True, None)
utils.csek.validation(flask.request,
src_object.metadata.customer_encryption.key_sha256,
False, None)
dst_metadata = resources_pb2.Object()
dst_metadata.CopyFrom(src_object.metadata)
del dst_metadata.acl[:]
dst_metadata.bucket = dst_bucket_name
dst_metadata.name = dst_object_name
dst_media = b""
dst_media += src_object.media
dst_rest_only = dict(src_object.rest_only)
dst_object, _ = gcs_type.object.Object.init(flask.request, dst_metadata,
dst_media, dst_bucket, True,
None, dst_rest_only)
db.insert_object(flask.request, dst_bucket_name, dst_object, None)
dst_object.patch(flask.request, None)
dst_object.metadata.metageneration = 1
dst_object.metadata.updated.FromDatetime(
dst_object.metadata.time_created.ToDatetime())
return dst_object.rest_metadata()
def init_resumable_rest(cls, request, bucket):
query_name = request.args.get("name", None)
rest_only = {}
metadata = resources_pb2.Object()
if len(request.data) > 0:
data = json.loads(request.data)
data_name = data.get("name", None)
if (
query_name is not None
and data_name is not None
and query_name != data_name
):
utils.error.invalid(
"Value '%s' in content does not agree with value '%s'."
% (data_name, query_name),
context=None,
)
rest_only = cls.__extract_rest_only(data)
metadata = json_format.ParseDict(
cls.__preprocess_rest_metadata(data), metadata
)
# Add some annotations to make it easier to write tests
metadata.metadata["x_emulator_upload"] = "resumable"
if data.get("crc32c", None) is not None:
metadata.metadata["x_emulator_crc32c"] = data.get("crc32c")
if data.get("md5Hash", None) is not None:
metadata.metadata["x_emulator_md5"] = data.get("md5Hash")
if metadata.metadata.get("x_emulator_crc32c", None) is None:
metadata.metadata["x_emulator_no_crc32c"] = "true"
if metadata.metadata.get("x_emulator_md5", None) is None:
metadata.metadata["x_emulator_no_md5"] = "true"
if query_name:
metadata.name = query_name
if metadata.name == "":
utils.error.invalid("No object name", context=None)
if metadata.content_type == "":
metadata.content_type = request.headers.get(
"x-upload-content-type", "application/octet-stream"
)
upload_id = cls.__create_upload_id(bucket.name, metadata.name)
location = (
request.host_url
+ "upload/storage/v1/b/%s/o?uploadType=resumable&upload_id=%s"
% (bucket.name, upload_id)
)
headers = {
key.lower(): value
for key, value in request.headers.items()
if key.lower().startswith("x-")
}
request = utils.common.FakeRequest(
args=request.args.to_dict(), headers=headers, data=b""
)
return cls.init_upload(
request, metadata, bucket, location, upload_id, rest_only
)
def update(self, request, context):
metadata = None
if context is not None:
metadata = request.metadata
else:
metadata = json_format.ParseDict(
self.__preprocess_rest(json.loads(request.data)),
resources_pb2.Object())
self.__update_metadata(metadata, None)
self.__insert_predefined_acl(
metadata,
self.bucket,
utils.acl.extract_predefined_acl(request, False, context),
context,
)
def test_init_resumable_grpc(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
bucket = bucket.metadata
insert_object_spec = storage_pb2.InsertObjectSpec(
resource={
"name": "object",
"bucket": "bucket"
},
predefined_acl=CommonEnums.PredefinedObjectAcl.
OBJECT_ACL_PROJECT_PRIVATE,
if_generation_not_match={"value": 1},
if_metageneration_match={"value": 2},
if_metageneration_not_match={"value": 3},
projection=CommonEnums.Projection.FULL,
)
request = storage_pb2.InsertObjectRequest(
insert_object_spec=insert_object_spec, write_offset=0)
upload = gcs.holder.DataHolder.init_resumable_grpc(request, bucket, "")
# Verify the annotations inserted by the emulator.
annotations = upload.metadata.metadata
self.assertGreaterEqual(
set([
"x_emulator_upload", "x_emulator_no_crc32c",
"x_emulator_no_md5"
]),
set(annotations.keys()),
)
# Clear any annotations created by the emulator
upload.metadata.metadata.clear()
self.assertEqual(upload.metadata,
resources_pb2.Object(name="object", bucket="bucket"))
predefined_acl = utils.acl.extract_predefined_acl(
upload.request, False, "")
self.assertEqual(
predefined_acl,
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE)
match, not_match = utils.generation.extract_precondition(
upload.request, False, False, "")
self.assertIsNone(match)
self.assertEqual(not_match, 1)
match, not_match = utils.generation.extract_precondition(
upload.request, True, False, "")
self.assertEqual(match, 2)
self.assertEqual(not_match, 3)
projection = utils.common.extract_projection(upload.request, False, "")
self.assertEqual(projection, CommonEnums.Projection.FULL)
def InsertObject(self, request_iterator, context):
db.insert_test_bucket(context)
upload, is_resumable = self.handle_insert_object_streaming_rpc(
request_iterator, context)
if upload is None:
return utils.error.missing(
"missing initial upload_id or insert_object_spec", context)
if not upload.complete:
if not is_resumable:
utils.error.missing("finish_write in request", context)
else:
return resources_pb2.Object()
blob, _ = gcs_type.object.Object.init(upload.request, upload.metadata,
upload.media, upload.bucket,
False, context)
db.insert_object(upload.request, upload.bucket.name, blob, context)
return blob.metadata
def patch(self, request, context):
update_mask = field_mask_pb2.FieldMask()
metadata = None
if context is not None:
metadata = request.metadata
update_mask = request.update_mask
else:
data = json.loads(request.data)
if "customTime" in data:
if self.rest_only is None:
self.rest_only = {}
self.rest_only["customTime"] = data.pop("customTime")
if "metadata" in data:
if data["metadata"] is None:
self.metadata.metadata.clear()
else:
for key, value in data["metadata"].items():
if value is None:
self.metadata.metadata.pop(key, None)
else:
self.metadata.metadata[key] = value
data.pop("metadata", None)
metadata = json_format.ParseDict(data, resources_pb2.Object())
paths = set()
for key in utils.common.nested_key(data):
key = utils.common.to_snake_case(key)
head = key
for i, c in enumerate(key):
if c == "." or c == "[":
head = key[0:i]
break
if head in Object.modifiable_fields:
if "[" in key:
paths.add(head)
else:
paths.add(key)
update_mask = field_mask_pb2.FieldMask(paths=list(paths))
self.__update_metadata(metadata, update_mask)
self.__insert_predefined_acl(
metadata,
self.bucket,
utils.acl.extract_predefined_acl(request, False, context),
context,
)
def test_init_resumable_grpc(self):
request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"})
bucket, _ = gcs.bucket.Bucket.init(request, "")
bucket = bucket.metadata
insert_object_spec = storage_pb2.InsertObjectSpec(
resource={
"name": "object",
"bucket": "bucket"
},
predefined_acl=CommonEnums.PredefinedObjectAcl.
OBJECT_ACL_PROJECT_PRIVATE,
if_generation_not_match={"value": 1},
if_metageneration_match={"value": 2},
if_metageneration_not_match={"value": 3},
projection=CommonEnums.Projection.FULL,
)
request = storage_pb2.InsertObjectRequest(
insert_object_spec=insert_object_spec, write_offset=0)
upload = gcs.holder.DataHolder.init_resumable_grpc(request, bucket, "")
self.assertEqual(upload.metadata,
resources_pb2.Object(name="object", bucket="bucket"))
predefined_acl = utils.acl.extract_predefined_acl(
upload.request, False, "")
self.assertEqual(
predefined_acl,
CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE)
match, not_match = utils.generation.extract_precondition(
upload.request, False, False, "")
self.assertIsNone(match)
self.assertEqual(not_match, 1)
match, not_match = utils.generation.extract_precondition(
upload.request, True, False, "")
self.assertEqual(match, 2)
self.assertEqual(not_match, 3)
projection = utils.common.extract_projection(upload.request, False, "")
self.assertEqual(projection, CommonEnums.Projection.FULL)
def init_dict(cls, request, metadata, media, bucket, is_destination):
metadata = json_format.ParseDict(metadata, resources_pb2.Object())
return cls.init(request, metadata, media, bucket, is_destination, None)
def test_grpc_to_rest(self):
# Make sure that object created by `gRPC` works with `REST`'s request.
spec = storage_pb2.InsertObjectSpec(resource=resources_pb2.Object(
name="test-object-name",
bucket="bucket",
metadata={"label0": "value0"},
cache_control="no-cache",
content_disposition="test-value",
content_encoding="test-value",
content_language="test-value",
content_type="octet-stream",
storage_class="regional",
customer_encryption=resources_pb2.Object.CustomerEncryption(
encryption_algorithm="AES", key_sha256="123456"),
# TODO(#6982) - add these fields when moving to storage/v2
# custom_time=utils.common.rest_rfc3339_to_proto("2021-08-01T12:00:00Z"),
event_based_hold={"value": True},
kms_key_name="test-value",
retention_expiration_time=utils.common.rest_rfc3339_to_proto(
"2022-01-01T00:00:00Z"),
temporary_hold=True,
time_deleted=utils.common.rest_rfc3339_to_proto(
"2021-06-01T00:00:00Z"),
time_storage_class_updated=utils.common.rest_rfc3339_to_proto(
"2021-07-01T00:00:00Z"),
))
request = storage_pb2.StartResumableWriteRequest(
insert_object_spec=spec)
upload = gcs.holder.DataHolder.init_resumable_grpc(
request, self.bucket.metadata, "")
blob, _ = gcs.object.Object.init(upload.request, upload.metadata,
b"123456789", upload.bucket, False,
"")
self.assertDictEqual(blob.rest_only, {})
self.assertEqual(blob.metadata.bucket, "bucket")
self.assertEqual(blob.metadata.name, "test-object-name")
self.assertEqual(blob.media, b"123456789")
# `REST` GET
rest_metadata = blob.rest_metadata()
self.assertEqual(rest_metadata["bucket"], "bucket")
self.assertEqual(rest_metadata["name"], "test-object-name")
self.assertIsNone(blob.metadata.metadata.get("method"))
# Verify the ObjectAccessControl entries have the desired fields
acl = rest_metadata.pop("acl", None)
self.assertIsNotNone(acl)
for entry in acl:
self.assertEqual(entry.pop("kind", None),
"storage#objectAccessControl")
self.assertEqual(entry.pop("bucket", None), "bucket")
self.assertEqual(entry.pop("object", None), "test-object-name")
self.assertIsNotNone(entry.pop("entity", None))
self.assertIsNotNone(entry.pop("role", None))
# Verify the remaining keys are a subset of the expected keys
self.assertLessEqual(
set(entry.keys()),
set([
"id",
"selfLink",
"generation",
"email",
"entityId",
"domain",
"projectTeam",
"etag",
]),
)
# Some fields we only care that they exist.
for key in self.__REST_FIELDS_KEY_ONLY:
self.assertIsNotNone(rest_metadata.pop(key, None),
msg="key=%s" % key)
# Some fields we need to manually extract to check their values
generation = rest_metadata.pop("generation", None)
self.assertIsNotNone(generation)
self.assertEqual("bucket/o/test-object-name#" + generation,
rest_metadata.pop("id"))
self.maxDiff = None
self.assertDictEqual(
rest_metadata,
{
"kind": "storage#object",
"bucket": "bucket",
"name": "test-object-name",
"cacheControl": "no-cache",
"contentDisposition": "test-value",
"contentEncoding": "test-value",
"contentLanguage": "test-value",
"contentType": "octet-stream",
"eventBasedHold": True,
"crc32c": "4waSgw==",
"customerEncryption": {
"encryptionAlgorithm": "AES",
"keySha256": "123456",
},
"kmsKeyName": "test-value",
"md5Hash": "JfnnlDI7RTiF9RgfG2JNCw==",
"metadata": {
"label0": "value0",
# The emulator adds useful annotations
"x_emulator_upload": "resumable",
"x_emulator_no_crc32c": "true",
"x_emulator_no_md5": "true",
"x_testbench_upload": "resumable",
"x_testbench_no_crc32c": "true",
"x_testbench_no_md5": "true",
},
"metageneration": "1",
"retentionExpirationTime": "2022-01-01T00:00:00Z",
"size": "9",
"storageClass": "regional",
"temporaryHold": True,
},
)
# `REST` PATCH
request = utils.common.FakeRequest(
args={}, data=json.dumps({"metadata": {
"method": "rest"
}}))
blob.patch(request, None)
self.assertEqual(blob.metadata.metadata["method"], "rest")
def init_dict(cls, request, metadata, media, bucket, is_destination):
rest_only = cls.__extract_rest_only(metadata)
metadata = json_format.ParseDict(metadata, resources_pb2.Object())
return cls.init(request, metadata, media, bucket, is_destination, None,
rest_only)
