def init_resumable_rest(cls, request, bucket): name = request.args.get("name", "") rest_only = {} if len(request.data) > 0 and request.data.decode("utf-8") != "{}": if name != "": utils.error.invalid("name argument in non-empty payload", None) data = json.loads(request.data) rest_only = cls.__extract_rest_only(data) metadata = json_format.ParseDict(data, resources_pb2.Object()) else: metadata = resources_pb2.Object() metadata.name = name if metadata.content_type == "": metadata.content_type = request.headers.get( "x-upload-content-type", "application/octet-stream") upload_id = hashlib.sha256( ("%s/o/%s" % (bucket.name, metadata.name)).encode("utf-8")).hexdigest() location = ( request.host_url + "upload/storage/v1/b/%s/o?uploadType=resumable&upload_id=%s" % (bucket.name, upload_id)) headers = { key.lower(): value for key, value in request.headers.items() if key.lower().startswith("x-") } request = utils.common.FakeRequest(args=request.args.to_dict(), headers=headers, data=b"") return cls.init_upload(request, metadata, bucket, location, upload_id, rest_only)
def objects_rewrite(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name): db.insert_test_bucket(None) token, rewrite = flask.request.args.get("rewriteToken"), None src_object = None if token is None: rewrite = gcs_type.holder.DataHolder.init_rewrite_rest( flask.request, src_bucket_name, src_object_name, dst_bucket_name, dst_object_name, ) db.insert_rewrite(rewrite) else: rewrite = db.get_rewrite(token, None) src_object = db.get_object(rewrite.request, src_bucket_name, src_object_name, True, None) utils.csek.validation(rewrite.request, src_object.metadata.customer_encryption.key_sha256, True, None) total_bytes_rewritten = len(rewrite.media) total_bytes_rewritten += min(rewrite.max_bytes_rewritten_per_call, len(src_object.media) - len(rewrite.media)) rewrite.media += src_object.media[len(rewrite.media):total_bytes_rewritten] done, dst_object = total_bytes_rewritten == len(src_object.media), None response = { "kind": "storage#rewriteResponse", "totalBytesRewritten": len(rewrite.media), "objectSize": len(src_object.media), "done": done, } if done: dst_bucket = db.get_bucket_without_generation(dst_bucket_name, None).metadata dst_metadata = resources_pb2.Object() dst_metadata.CopyFrom(src_object.metadata) dst_rest_only = dict(src_object.rest_only) dst_metadata.bucket = dst_bucket_name dst_metadata.name = dst_object_name dst_media = rewrite.media dst_object, _ = gcs_type.object.Object.init( flask.request, dst_metadata, dst_media, dst_bucket, True, None, dst_rest_only, ) db.insert_object(flask.request, dst_bucket_name, dst_object, None) dst_object.patch(rewrite.request, None) dst_object.metadata.metageneration = 1 dst_object.metadata.updated.FromDatetime( dst_object.metadata.time_created.ToDatetime()) resources = dst_object.rest_metadata() response["resource"] = resources else: response["rewriteToken"] = rewrite.token return response
def init_dict(cls, request, metadata, media, bucket, is_destination): rest_only = {} if "customTime" in metadata: rest_only["customTime"] = metadata.pop("customTime") metadata = json_format.ParseDict(metadata, resources_pb2.Object()) return cls.init(request, metadata, media, bucket, is_destination, None, rest_only)
def objects_copy(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name): db.insert_test_bucket(None) dst_bucket = db.get_bucket_without_generation(dst_bucket_name, None).metadata src_object = db.get_object(flask.request, src_bucket_name, src_object_name, True, None) utils.csek.validation(flask.request, src_object.metadata.customer_encryption.key_sha256, False, None) dst_metadata = resources_pb2.Object() dst_metadata.CopyFrom(src_object.metadata) del dst_metadata.acl[:] dst_metadata.bucket = dst_bucket_name dst_metadata.name = dst_object_name dst_media = b"" dst_media += src_object.media dst_rest_only = dict(src_object.rest_only) dst_object, _ = gcs_type.object.Object.init(flask.request, dst_metadata, dst_media, dst_bucket, True, None, dst_rest_only) db.insert_object(flask.request, dst_bucket_name, dst_object, None) dst_object.patch(flask.request, None) dst_object.metadata.metageneration = 1 dst_object.metadata.updated.FromDatetime( dst_object.metadata.time_created.ToDatetime()) return dst_object.rest_metadata()
def init_resumable_rest(cls, request, bucket): query_name = request.args.get("name", None) rest_only = {} metadata = resources_pb2.Object() if len(request.data) > 0: data = json.loads(request.data) data_name = data.get("name", None) if ( query_name is not None and data_name is not None and query_name != data_name ): utils.error.invalid( "Value '%s' in content does not agree with value '%s'." % (data_name, query_name), context=None, ) rest_only = cls.__extract_rest_only(data) metadata = json_format.ParseDict( cls.__preprocess_rest_metadata(data), metadata ) # Add some annotations to make it easier to write tests metadata.metadata["x_emulator_upload"] = "resumable" if data.get("crc32c", None) is not None: metadata.metadata["x_emulator_crc32c"] = data.get("crc32c") if data.get("md5Hash", None) is not None: metadata.metadata["x_emulator_md5"] = data.get("md5Hash") if metadata.metadata.get("x_emulator_crc32c", None) is None: metadata.metadata["x_emulator_no_crc32c"] = "true" if metadata.metadata.get("x_emulator_md5", None) is None: metadata.metadata["x_emulator_no_md5"] = "true" if query_name: metadata.name = query_name if metadata.name == "": utils.error.invalid("No object name", context=None) if metadata.content_type == "": metadata.content_type = request.headers.get( "x-upload-content-type", "application/octet-stream" ) upload_id = cls.__create_upload_id(bucket.name, metadata.name) location = ( request.host_url + "upload/storage/v1/b/%s/o?uploadType=resumable&upload_id=%s" % (bucket.name, upload_id) ) headers = { key.lower(): value for key, value in request.headers.items() if key.lower().startswith("x-") } request = utils.common.FakeRequest( args=request.args.to_dict(), headers=headers, data=b"" ) return cls.init_upload( request, metadata, bucket, location, upload_id, rest_only )
def update(self, request, context): metadata = None if context is not None: metadata = request.metadata else: metadata = json_format.ParseDict( self.__preprocess_rest(json.loads(request.data)), resources_pb2.Object()) self.__update_metadata(metadata, None) self.__insert_predefined_acl( metadata, self.bucket, utils.acl.extract_predefined_acl(request, False, context), context, )
def test_init_resumable_grpc(self): request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"}) bucket, _ = gcs.bucket.Bucket.init(request, "") bucket = bucket.metadata insert_object_spec = storage_pb2.InsertObjectSpec( resource={ "name": "object", "bucket": "bucket" }, predefined_acl=CommonEnums.PredefinedObjectAcl. OBJECT_ACL_PROJECT_PRIVATE, if_generation_not_match={"value": 1}, if_metageneration_match={"value": 2}, if_metageneration_not_match={"value": 3}, projection=CommonEnums.Projection.FULL, ) request = storage_pb2.InsertObjectRequest( insert_object_spec=insert_object_spec, write_offset=0) upload = gcs.holder.DataHolder.init_resumable_grpc(request, bucket, "") # Verify the annotations inserted by the emulator. annotations = upload.metadata.metadata self.assertGreaterEqual( set([ "x_emulator_upload", "x_emulator_no_crc32c", "x_emulator_no_md5" ]), set(annotations.keys()), ) # Clear any annotations created by the emulator upload.metadata.metadata.clear() self.assertEqual(upload.metadata, resources_pb2.Object(name="object", bucket="bucket")) predefined_acl = utils.acl.extract_predefined_acl( upload.request, False, "") self.assertEqual( predefined_acl, CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE) match, not_match = utils.generation.extract_precondition( upload.request, False, False, "") self.assertIsNone(match) self.assertEqual(not_match, 1) match, not_match = utils.generation.extract_precondition( upload.request, True, False, "") self.assertEqual(match, 2) self.assertEqual(not_match, 3) projection = utils.common.extract_projection(upload.request, False, "") self.assertEqual(projection, CommonEnums.Projection.FULL)
def InsertObject(self, request_iterator, context): db.insert_test_bucket(context) upload, is_resumable = self.handle_insert_object_streaming_rpc( request_iterator, context) if upload is None: return utils.error.missing( "missing initial upload_id or insert_object_spec", context) if not upload.complete: if not is_resumable: utils.error.missing("finish_write in request", context) else: return resources_pb2.Object() blob, _ = gcs_type.object.Object.init(upload.request, upload.metadata, upload.media, upload.bucket, False, context) db.insert_object(upload.request, upload.bucket.name, blob, context) return blob.metadata
def patch(self, request, context): update_mask = field_mask_pb2.FieldMask() metadata = None if context is not None: metadata = request.metadata update_mask = request.update_mask else: data = json.loads(request.data) if "customTime" in data: if self.rest_only is None: self.rest_only = {} self.rest_only["customTime"] = data.pop("customTime") if "metadata" in data: if data["metadata"] is None: self.metadata.metadata.clear() else: for key, value in data["metadata"].items(): if value is None: self.metadata.metadata.pop(key, None) else: self.metadata.metadata[key] = value data.pop("metadata", None) metadata = json_format.ParseDict(data, resources_pb2.Object()) paths = set() for key in utils.common.nested_key(data): key = utils.common.to_snake_case(key) head = key for i, c in enumerate(key): if c == "." or c == "[": head = key[0:i] break if head in Object.modifiable_fields: if "[" in key: paths.add(head) else: paths.add(key) update_mask = field_mask_pb2.FieldMask(paths=list(paths)) self.__update_metadata(metadata, update_mask) self.__insert_predefined_acl( metadata, self.bucket, utils.acl.extract_predefined_acl(request, False, context), context, )
def test_init_resumable_grpc(self): request = storage_pb2.InsertBucketRequest(bucket={"name": "bucket"}) bucket, _ = gcs.bucket.Bucket.init(request, "") bucket = bucket.metadata insert_object_spec = storage_pb2.InsertObjectSpec( resource={ "name": "object", "bucket": "bucket" }, predefined_acl=CommonEnums.PredefinedObjectAcl. OBJECT_ACL_PROJECT_PRIVATE, if_generation_not_match={"value": 1}, if_metageneration_match={"value": 2}, if_metageneration_not_match={"value": 3}, projection=CommonEnums.Projection.FULL, ) request = storage_pb2.InsertObjectRequest( insert_object_spec=insert_object_spec, write_offset=0) upload = gcs.holder.DataHolder.init_resumable_grpc(request, bucket, "") self.assertEqual(upload.metadata, resources_pb2.Object(name="object", bucket="bucket")) predefined_acl = utils.acl.extract_predefined_acl( upload.request, False, "") self.assertEqual( predefined_acl, CommonEnums.PredefinedObjectAcl.OBJECT_ACL_PROJECT_PRIVATE) match, not_match = utils.generation.extract_precondition( upload.request, False, False, "") self.assertIsNone(match) self.assertEqual(not_match, 1) match, not_match = utils.generation.extract_precondition( upload.request, True, False, "") self.assertEqual(match, 2) self.assertEqual(not_match, 3) projection = utils.common.extract_projection(upload.request, False, "") self.assertEqual(projection, CommonEnums.Projection.FULL)
def init_dict(cls, request, metadata, media, bucket, is_destination): metadata = json_format.ParseDict(metadata, resources_pb2.Object()) return cls.init(request, metadata, media, bucket, is_destination, None)
def test_grpc_to_rest(self): # Make sure that object created by `gRPC` works with `REST`'s request. spec = storage_pb2.InsertObjectSpec(resource=resources_pb2.Object( name="test-object-name", bucket="bucket", metadata={"label0": "value0"}, cache_control="no-cache", content_disposition="test-value", content_encoding="test-value", content_language="test-value", content_type="octet-stream", storage_class="regional", customer_encryption=resources_pb2.Object.CustomerEncryption( encryption_algorithm="AES", key_sha256="123456"), # TODO(#6982) - add these fields when moving to storage/v2 # custom_time=utils.common.rest_rfc3339_to_proto("2021-08-01T12:00:00Z"), event_based_hold={"value": True}, kms_key_name="test-value", retention_expiration_time=utils.common.rest_rfc3339_to_proto( "2022-01-01T00:00:00Z"), temporary_hold=True, time_deleted=utils.common.rest_rfc3339_to_proto( "2021-06-01T00:00:00Z"), time_storage_class_updated=utils.common.rest_rfc3339_to_proto( "2021-07-01T00:00:00Z"), )) request = storage_pb2.StartResumableWriteRequest( insert_object_spec=spec) upload = gcs.holder.DataHolder.init_resumable_grpc( request, self.bucket.metadata, "") blob, _ = gcs.object.Object.init(upload.request, upload.metadata, b"123456789", upload.bucket, False, "") self.assertDictEqual(blob.rest_only, {}) self.assertEqual(blob.metadata.bucket, "bucket") self.assertEqual(blob.metadata.name, "test-object-name") self.assertEqual(blob.media, b"123456789") # `REST` GET rest_metadata = blob.rest_metadata() self.assertEqual(rest_metadata["bucket"], "bucket") self.assertEqual(rest_metadata["name"], "test-object-name") self.assertIsNone(blob.metadata.metadata.get("method")) # Verify the ObjectAccessControl entries have the desired fields acl = rest_metadata.pop("acl", None) self.assertIsNotNone(acl) for entry in acl: self.assertEqual(entry.pop("kind", None), "storage#objectAccessControl") self.assertEqual(entry.pop("bucket", None), "bucket") self.assertEqual(entry.pop("object", None), "test-object-name") self.assertIsNotNone(entry.pop("entity", None)) self.assertIsNotNone(entry.pop("role", None)) # Verify the remaining keys are a subset of the expected keys self.assertLessEqual( set(entry.keys()), set([ "id", "selfLink", "generation", "email", "entityId", "domain", "projectTeam", "etag", ]), ) # Some fields we only care that they exist. for key in self.__REST_FIELDS_KEY_ONLY: self.assertIsNotNone(rest_metadata.pop(key, None), msg="key=%s" % key) # Some fields we need to manually extract to check their values generation = rest_metadata.pop("generation", None) self.assertIsNotNone(generation) self.assertEqual("bucket/o/test-object-name#" + generation, rest_metadata.pop("id")) self.maxDiff = None self.assertDictEqual( rest_metadata, { "kind": "storage#object", "bucket": "bucket", "name": "test-object-name", "cacheControl": "no-cache", "contentDisposition": "test-value", "contentEncoding": "test-value", "contentLanguage": "test-value", "contentType": "octet-stream", "eventBasedHold": True, "crc32c": "4waSgw==", "customerEncryption": { "encryptionAlgorithm": "AES", "keySha256": "123456", }, "kmsKeyName": "test-value", "md5Hash": "JfnnlDI7RTiF9RgfG2JNCw==", "metadata": { "label0": "value0", # The emulator adds useful annotations "x_emulator_upload": "resumable", "x_emulator_no_crc32c": "true", "x_emulator_no_md5": "true", "x_testbench_upload": "resumable", "x_testbench_no_crc32c": "true", "x_testbench_no_md5": "true", }, "metageneration": "1", "retentionExpirationTime": "2022-01-01T00:00:00Z", "size": "9", "storageClass": "regional", "temporaryHold": True, }, ) # `REST` PATCH request = utils.common.FakeRequest( args={}, data=json.dumps({"metadata": { "method": "rest" }})) blob.patch(request, None) self.assertEqual(blob.metadata.metadata["method"], "rest")
def init_dict(cls, request, metadata, media, bucket, is_destination): rest_only = cls.__extract_rest_only(metadata) metadata = json_format.ParseDict(metadata, resources_pb2.Object()) return cls.init(request, metadata, media, bucket, is_destination, None, rest_only)