def objects_insert(bucket_name):
"""Implement the 'Objects: insert' API. Insert a new GCS Object."""
gcs_url = flask.url_for(
"objects_insert", bucket_name=bucket_name, _external=True
).replace("/upload/", "/")
insert_magic_bucket(gcs_url)
upload_type = flask.request.args.get("uploadType")
if upload_type is None:
raise error_response.ErrorResponse(
"uploadType not set in Objects: insert", status_code=400
)
if upload_type not in {"multipart", "media", "resumable"}:
raise error_response.ErrorResponse(
"testbench does not support %s uploadType" % upload_type, status_code=400
)
if upload_type == "resumable":
bucket = testbench_utils.lookup_bucket(bucket_name)
upload_url = flask.url_for(
"objects_insert", bucket_name=bucket_name, _external=True
)
return bucket.create_resumable_upload(upload_url, flask.request)
object_path = None
blob = None
current_version = None
if upload_type == "media":
object_name = flask.request.args.get("name", None)
if object_name is None:
raise error_response.ErrorResponse(
"name not set in Objects: insert", status_code=412
)
object_path, blob = testbench_utils.get_object(
bucket_name, object_name, gcs_object.GcsObject(bucket_name, object_name)
)
blob.check_preconditions(flask.request)
current_version = blob.insert(gcs_url, flask.request)
else:
resource, media_headers, media_body = testbench_utils.parse_multi_part(
flask.request
)
object_name = flask.request.args.get("name", resource.get("name", None))
if object_name is None:
raise error_response.ErrorResponse(
"name not set in Objects: insert", status_code=412
)
object_path, blob = testbench_utils.get_object(
bucket_name, object_name, gcs_object.GcsObject(bucket_name, object_name)
)
blob.check_preconditions(flask.request)
current_version = blob.insert_multipart(
gcs_url, flask.request, resource, media_headers, media_body
)
testbench_utils.insert_object(object_path, blob)
return testbench_utils.filtered_response(flask.request, current_version.metadata)
def objects_copy(source_bucket, source_object, destination_bucket,
destination_object):
"""Implement the 'Objects: copy' API, copy an object."""
object_path, blob = testbench_utils.lookup_object(source_bucket,
source_object)
blob.check_preconditions(
flask.request,
if_generation_match='ifSourceGenerationMatch',
if_generation_not_match='ifSourceGenerationNotMatch',
if_metageneration_match='ifSourceMetagenerationMatch',
if_metageneration_not_match='ifSourceMetagenerationNotMatch')
source_revision = blob.get_revision(flask.request, 'sourceGeneration')
if source_revision is None:
raise error_response.ErrorResponse('Revision not found %s' %
object_path,
status_code=404)
destination_path, destination = testbench_utils.get_object(
destination_bucket, destination_object,
gcs_object.GcsObject(destination_bucket, destination_object))
base_url = flask.url_for('gcs_index', _external=True)
current_version = destination.copy_from(base_url, flask.request,
source_revision)
testbench_utils.insert_object(destination_path, destination)
return testbench_utils.filtered_response(flask.request,
current_version.metadata)
def xmlapi_put_object(bucket_name, object_name):
"""Inserts a new GCS Object.
Implement the PUT request in the XML API.
"""
gcs_url = flask.url_for(
"xmlapi_put_object",
bucket_name=bucket_name,
object_name=object_name,
_external=True,
).replace("/xmlapi/", "/")
insert_magic_bucket(gcs_url)
object_path, blob = testbench_utils.get_object(
bucket_name, object_name, gcs_object.GcsObject(bucket_name,
object_name))
generation_match = flask.request.headers.get("x-goog-if-generation-match")
metageneration_match = flask.request.headers.get(
"x-goog-if-metageneration-match")
blob.check_preconditions_by_value(generation_match, None,
metageneration_match, None)
revision = blob.insert_xml(gcs_url, flask.request)
testbench_utils.insert_object(object_path, blob)
response = flask.make_response("")
response.headers["x-goog-hash"] = revision.x_goog_hash_header()
return response
def objects_insert(bucket_name):
"""Implement the 'Objects: insert' API. Insert a new GCS Object."""
gcs_url = flask.url_for('objects_insert',
bucket_name=bucket_name,
_external=True).replace('/upload/', '/')
insert_magic_bucket(gcs_url)
object_name = flask.request.args.get('name', None)
if object_name is None:
raise error_response.ErrorResponse('name not set in Objects: insert',
status_code=412)
upload_type = flask.request.args.get('uploadType')
if upload_type is None:
raise error_response.ErrorResponse(
'uploadType not set in Objects: insert', status_code=412)
if upload_type not in {'multipart', 'media'}:
raise error_response.ErrorResponse(
'testbench does not support %s uploadType' % upload_type,
status_code=400)
object_path, blob = testbench_utils.get_object(
bucket_name, object_name, gcs_object.GcsObject(bucket_name,
object_name))
blob.check_preconditions(flask.request)
if upload_type == 'media':
current_version = blob.insert(gcs_url, flask.request)
else:
current_version = blob.insert_multipart(gcs_url, flask.request)
testbench_utils.insert_object(object_path, blob)
return testbench_utils.filtered_response(flask.request,
current_version.metadata)
def xml_put_object(gcs_url, bucket_name, object_name):
"""Implement PUT for the XML API."""
insert_magic_bucket(gcs_url)
object_path, blob = testbench_utils.get_object(
bucket_name, object_name, gcs_object.GcsObject(bucket_name,
object_name))
generation_match = flask.request.headers.get("x-goog-if-generation-match")
metageneration_match = flask.request.headers.get(
"x-goog-if-metageneration-match")
blob.check_preconditions_by_value(generation_match, None,
metageneration_match, None)
revision = blob.insert_xml(gcs_url, flask.request)
testbench_utils.insert_object(object_path, blob)
response = flask.make_response("")
response.headers["x-goog-hash"] = revision.x_goog_hash_header()
return response
def objects_compose(bucket_name, object_name):
"""Implement the 'Objects: compose' API: concatenate Objects."""
payload = json.loads(flask.request.data)
source_objects = payload["sourceObjects"]
if source_objects is None:
raise error_response.ErrorResponse(
"You must provide at least one source component.", status_code=400
)
if len(source_objects) > 32:
raise error_response.ErrorResponse(
"The number of source components provided"
" (%d) exceeds the maximum (32)" % len(source_objects),
status_code=400,
)
composed_media = b""
for source_object in source_objects:
source_object_name = source_object.get("name")
if source_object_name is None:
raise error_response.ErrorResponse("Required.", status_code=400)
source_object_path, source_blob = testbench_utils.lookup_object(
bucket_name, source_object_name
)
source_revision = source_blob.get_latest()
generation = source_object.get("generation")
if generation is not None:
source_revision = source_blob.get_revision_by_generation(generation)
if source_revision is None:
raise error_response.ErrorResponse(
"No such object: %s" % source_object_path, status_code=404
)
object_preconditions = source_object.get("objectPreconditions")
if object_preconditions is not None:
if_generation_match = object_preconditions.get("ifGenerationMatch")
source_blob.check_preconditions_by_value(
if_generation_match, None, None, None
)
composed_media += source_revision.media
composed_object_path, composed_object = testbench_utils.get_object(
bucket_name, object_name, gcs_object.GcsObject(bucket_name, object_name)
)
composed_object.check_preconditions(flask.request)
base_url = flask.url_for("gcs_index", _external=True)
current_version = composed_object.compose_from(
base_url, flask.request, composed_media
)
testbench_utils.insert_object(composed_object_path, composed_object)
return testbench_utils.filtered_response(flask.request, current_version.metadata)
def receive_upload_chunk(self, gcs_url, request):
"""Receive a new upload chunk.
:param gcs_url: str the base URL for the service.
:param request: flask.Request the original http request.
:return: the HTTP response.
"""
upload_id = request.args.get('upload_id')
if upload_id is None:
raise error_response.ErrorResponse(
'Missing upload_id in resumable_upload_chunk', status_code=400)
upload = self.resumable_uploads.get(upload_id)
if upload is None:
raise error_response.ErrorResponse(
'Cannot find resumable upload %s' % upload_id, status_code=404)
# Be gracious in what you accept, if the Content-Range header is not
# set we assume it is a good header and it is the end of the file.
next_byte = upload['next_byte']
begin = next_byte
end = next_byte + len(request.data)
total = end
final_chunk = False
content_range = request.headers.get('content-range')
if content_range is not None:
if content_range.startswith('bytes */*'):
# This is just a query to resume an upload, if it is done, return
# an empty
response = flask.make_response('')
if next_byte > 1:
response.headers['Range'] = 'bytes=0-%d' % (next_byte - 1)
response.status_code = 308
return response
match = re.match('bytes \*/(\\*|[0-9]+)', content_range)
if match:
if match.group(1) == '*':
total = 0
else:
total = int(match.group(1))
final_chunk = True
else:
match = re.match('bytes ([0-9]+)-([0-9]+)/(\\*|[0-9]+)', content_range)
if not match:
raise error_response.ErrorResponse(
'Invalid Content-Range in upload %s' % content_range,
status_code=400)
begin = int(match.group(1))
end = int(match.group(2))
if match.group(3) == '*':
total = 0
else:
total = int(match.group(3))
final_chunk = True
if begin != next_byte:
raise error_response.ErrorResponse(
'Mismatched data range, expected data at %d, got %d' % (
next_byte, begin), status_code=400)
if len(request.data) != end - begin + 1:
raise error_response.ErrorResponse(
'Mismatched data range (%d) vs. content-length (%d)' % (
end - begin + 1, len(request.data)), status_code=400)
upload['media'] = upload.get('media', '') + request.data
next_byte = len(upload.get('media', ''))
upload['next_byte'] = next_byte
response_payload = ''
if final_chunk and next_byte >= total:
upload['done'] = True
object_name = upload.get('object_name')
object_path, blob = testbench_utils.get_object(
self.name, object_name,
gcs_object.GcsObject(self.name, object_name))
# Release a few resources to control memory usage.
original_metadata = upload.pop('metadata', None)
media = upload.pop('media', None)
blob.check_preconditions_by_value(
upload.get('ifGenerationMatch'),
upload.get('ifGenerationNotMatch'),
upload.get('ifMetagenerationMatch'),
upload.get('ifMetagenerationNotMatch')
)
if upload.pop('instructions', None) == 'inject-upload-data-error':
media = testbench_utils.corrupt_media(media)
revision = blob.insert_resumable(
gcs_url, request, media,
original_metadata)
response_payload = testbench_utils.filter_fields_from_response(
upload.get('fields'), revision.metadata)
testbench_utils.insert_object(object_path, blob)
response = flask.make_response(response_payload)
if next_byte == 0:
response.headers['Range'] = 'bytes=0-0'
else:
response.headers['Range'] = 'bytes=0-%d' % (next_byte - 1)
if upload.get('done', False):
response.status_code = 200
else:
response.status_code = 308
return response
def rewrite_step(self, gcs_url, request, destination_bucket,
destination_object):
"""Execute an iteration of `Objects: rewrite.
Objects: rewrite may need to be called multiple times before it
succeeds. Only objects in the same location, with the same encryption,
are guaranteed to complete in a single request.
The implementation simulates some, but not all, the behaviors of the
server, in particular, only rewrites within the same bucket and smaller
than 1MiB complete immediately.
:param gcs_url:str the root URL for the fake GCS service.
:param request:flask.Request the contents of the HTTP request.
:param destination_bucket:str where will the object be placed after the
rewrite operation completes.
:param destination_object:str the name of the object when the rewrite
operation completes.
:return: a dictionary prepared for JSON encoding of a
`Objects: rewrite` response.
:rtype:dict
"""
body = json.loads(request.data)
rewrite_token = request.args.get("rewriteToken")
if rewrite_token is not None and rewrite_token != "":
# Note that we remove the rewrite operation, not just look it up.
# That way if the operation completes in this call, and/or fails,
# it is already removed. We need to insert it with a new token
# anyway, so this makes sense.
rewrite = self.rewrite_operations.pop(rewrite_token, None)
if rewrite is None:
raise error_response.ErrorResponse(
"Invalid or expired token in rewrite", status_code=410)
else:
rewrite_token, rewrite = self.make_rewrite_operation(
request, destination_bucket, destination_bucket)
# Compare the difference to the original arguments, on the first call
# this is a waste, but the code is easier to follow.
current_arguments = self.capture_rewrite_operation_arguments(
request, destination_bucket, destination_object)
diff = set(current_arguments) ^ set(rewrite.get("original_arguments"))
if len(diff) != 0:
raise error_response.ErrorResponse(
"Mismatched arguments to rewrite", status_code=412)
# This will raise if the version is deleted while the operation is in
# progress.
source = self.get_revision_by_generation(
rewrite.get("actual_generation"))
source.validate_encryption_for_read(
request, prefix="x-goog-copy-source-encryption")
bytes_rewritten = rewrite.get("bytes_rewritten")
bytes_rewritten += 1024 * 1024
result = {
"kind": "storage#rewriteResponse",
"objectSize": len(source.media)
}
if bytes_rewritten >= len(source.media):
bytes_rewritten = len(source.media)
rewrite["bytes_rewritten"] = bytes_rewritten
# Success, the operation completed. Return the new object:
object_path, destination = testbench_utils.get_object(
destination_bucket,
destination_object,
GcsObject(destination_bucket, destination_object),
)
revision = destination.rewrite_finish(gcs_url, request, body,
source)
testbench_utils.insert_object(object_path, destination)
result["done"] = True
result["resource"] = revision.metadata
rewrite_token = ""
else:
rewrite["bytes_rewritten"] = bytes_rewritten
rewrite_token = GcsObject.make_rewrite_token(
rewrite, destination_bucket, destination_object,
source.generation)
self.rewrite_operations[rewrite_token] = rewrite
result["done"] = False
result.update({
"totalBytesRewritten": bytes_rewritten,
"rewriteToken": rewrite_token
})
return result
def receive_upload_chunk(self, gcs_url, request):
"""Receive a new upload chunk.
:param gcs_url: str the base URL for the service.
:param request: flask.Request the original http request.
:return: the HTTP response.
"""
upload_id = request.args.get("upload_id")
if upload_id is None:
raise error_response.ErrorResponse(
"Missing upload_id in resumable_upload_chunk", status_code=400)
upload = self.resumable_uploads.get(upload_id)
if upload is None:
raise error_response.ErrorResponse(
"Cannot find resumable upload %s" % upload_id, status_code=404)
# Be gracious in what you accept, if the Content-Range header is not
# set we assume it is a good header and it is the end of the file.
next_byte = upload["next_byte"]
begin = next_byte
end = next_byte + len(request.data)
total = end
final_chunk = False
content_range = request.headers.get("content-range")
if content_range is not None:
if content_range.startswith("bytes */*"):
# This is just a query to resume an upload, if it is done, return
# the completed upload payload and an empty range header.
response = flask.make_response(upload.get("payload", ""))
if next_byte > 1 and not upload["done"]:
response.headers["Range"] = "bytes=0-%d" % (next_byte - 1)
response.status_code = 200 if upload["done"] else 308
return response
match = re.match("bytes \*/(\\*|[0-9]+)", content_range)
if match:
if match.group(1) == "*":
total = 0
else:
total = int(match.group(1))
final_chunk = True
else:
match = re.match("bytes ([0-9]+)-([0-9]+)/(\\*|[0-9]+)",
content_range)
if not match:
raise error_response.ErrorResponse(
"Invalid Content-Range in upload %s" % content_range,
status_code=400,
)
begin = int(match.group(1))
end = int(match.group(2))
if match.group(3) == "*":
total = 0
else:
total = int(match.group(3))
final_chunk = True
if begin != next_byte:
raise error_response.ErrorResponse(
"Mismatched data range, expected data at %d, got %d" %
(next_byte, begin),
status_code=400,
)
if len(request.data) != end - begin + 1:
raise error_response.ErrorResponse(
"Mismatched data range (%d) vs. content-length (%d)" %
(end - begin + 1, len(request.data)),
status_code=400,
)
upload["media"] = upload.get("media", b"") + request.data
next_byte = len(upload.get("media", ""))
upload["next_byte"] = next_byte
response_payload = ""
if final_chunk and next_byte >= total:
upload["done"] = True
object_name = upload.get("object_name")
object_path, blob = testbench_utils.get_object(
self.name, object_name,
gcs_object.GcsObject(self.name, object_name))
# Release a few resources to control memory usage.
original_metadata = upload.pop("metadata", None)
media = upload.pop("media", None)
blob.check_preconditions_by_value(
upload.get("ifGenerationMatch"),
upload.get("ifGenerationNotMatch"),
upload.get("ifMetagenerationMatch"),
upload.get("ifMetagenerationNotMatch"),
)
if upload.pop("instructions", None) == "inject-upload-data-error":
media = testbench_utils.corrupt_media(media)
revision = blob.insert_resumable(gcs_url, request, media,
original_metadata)
response_payload = testbench_utils.filter_fields_from_response(
upload.get("fields"), revision.metadata)
upload["payload"] = response_payload
testbench_utils.insert_object(object_path, blob)
response = flask.make_response(response_payload)
if next_byte == 0:
response.headers["Range"] = "bytes=0-0"
else:
response.headers["Range"] = "bytes=0-%d" % (next_byte - 1)
if upload.get("done", False):
response.status_code = 200
else:
response.status_code = 308
return response
def receive_upload_chunk(self, gcs_url, request):
"""Receive a new upload chunk.
:param gcs_url: str the base URL for the service.
:param request: flask.Request the original http request.
:return: the HTTP response.
"""
upload_id = request.args.get('upload_id')
if upload_id is None:
raise error_response.ErrorResponse(
'Missing upload_id in resumable_upload_chunk', status_code=400)
upload = self.resumable_uploads.get(upload_id)
if upload is None:
raise error_response.ErrorResponse(
'Cannot find resumable upload %s' % upload_id, status_code=404)
# Be gracious in what you accept, if the Content-Range header is not
# set we assume it is a good header and it is the end of the file.
received_bytes = upload['received_bytes']
begin = received_bytes
end = received_bytes + len(request.data)
total = end
content_range = request.headers.get('content-range')
if content_range is not None:
if content_range.starts_with('*/'):
# This is just a query to resume an upload, if it is done, return
# an empty
response = flask.make_response('')
response.headers['Range'] = 'bytes=0-%d' % upload.get('received_bytes')
if upload.get('done', False):
# We could also return 200, but this is a better test for the
# library.
response.status_code = 201
else:
response.status_code = 308
return response
match = re.match('([0-9]+)-([0-9]+)/(\*|[0-9]+)', content_range)
if not match:
raise error_response.ErrorResponse(
'Invalid Content-Range in upload %s' % content_range,
status_code=400)
begin = int(match.group(1))
end = int(match.group(2))
total = int(match.group(3))
if begin != received_bytes:
raise error_response.ErrorResponse(
'Mismatched data range, expected data at %d, got %d' % (
received_bytes, begin), status_code=400)
if end - begin != len(request.data):
raise error_response.ErrorResponse(
'Mismatched data range (%d) vs. content-length (%d)' % (
end - begin, len(request.data)), status_code=400)
received_bytes = received_bytes + len(request.data)
upload['media'] = upload.get('media', '') + request.data
upload['received_bytes'] = received_bytes
response_payload = ''
if received_bytes == total:
upload['done'] = True
object_name = upload.get('object_name')
object_path, blob = testbench_utils.get_object(
self.name, object_name,
gcs_object.GcsObject(self.name, object_name))
# Release a few resources to control memory usage.
original_metadata = upload.pop('metadata', None)
media = upload.pop('media', None)
original_request = upload.pop('request', None)
blob.check_preconditions(original_request)
revision = blob.insert_resumable(
gcs_url, original_request, media,
original_metadata)
print("\n\n\n%d %s\n\n" % (len(media), object_name))
response_payload = testbench_utils.filtered_response(
request, revision.metadata)
testbench_utils.insert_object(object_path, blob)
response = flask.make_response(response_payload)
response.headers['Range'] = 'bytes=0-%d' % received_bytes
if upload.get('done', False):
response.status_code = 200
else:
response.status_code = 308
return response