def insert_multipart(self, gcs_url, request):
"""Insert a new revision based on the give flask request.
:param gcs_url:str the root URL for the fake GCS service.
:param request:flask.Request the contents of the HTTP request.
:return: the newly created object version.
:rtype: GcsObjectVersion
"""
content_type = request.headers.get('content-type')
if content_type is None or not content_type.startswith(
'multipart/related'):
raise error_response.ErrorResponse(
'Missing or invalid content-type header in multipart upload')
_, _, boundary = content_type.partition('boundary=')
if boundary is None:
raise error_response.ErrorResponse(
'Missing boundary (%s) in content-type header in multipart upload'
% boundary)
marker = '--' + boundary + '\r\n'
body = testbench_utils.extract_media(request)
parts = body.split(marker)
# parts[0] is the empty string, `multipart` should start with the boundary
# parts[1] is the JSON resource object part, with some headers
resource_headers, resource_body = self._parse_part(parts[1])
# parts[2] is the media, with some headers
media_headers, media_body = self._parse_part(parts[2])
end = media_body.find('\r\n--' + boundary + '--\r\n')
if end == -1:
raise error_response.ErrorResponse(
'Missing end marker (--%s--) in media body' % boundary)
media_body = media_body[:end]
resource = json.loads(resource_body)
# There are two ways to specify the content-type, the 'content-type'
# header and the resource['contentType'] field. They must be consistent,
# and the service generates an error when they are not.
if (resource.get('contentType') is not None and
media_headers.get('content-type') is not None and
resource.get('contentType') != media_headers.get('content-type')):
raise error_response.ErrorResponse(
('Content-Type specified in the upload (%s) does not match' +
'contentType specified in the metadata (%s).') % (
media_headers.get('content-type'),
resource.get('contentType')),
status_code=400)
# Set the contentType in the resource from the header. Note that if both
# are set they have the same value.
resource.setdefault('contentType', media_headers.get('content-type'))
self.generation += 1
revision = GcsObjectVersion(
gcs_url, self.bucket_name, self.name, self.generation, request,
media_body)
meta = revision.metadata.setdefault('metadata', {})
meta['x_testbench_upload'] = 'multipart'
meta['x_testbench_md5'] = resource.get('md5Hash', '')
meta['x_testbench_crc32c'] = resource.get('crc32c', '')
# Apply any overrides from the resource object part.
revision.update_from_metadata(resource)
self._insert_revision(revision)
return revision
def insert_xml(self, gcs_url, request):
"""Implement the insert operation using the XML API.
:param gcs_url:str the root URL for the fake GCS service.
:param request:flask.Request the contents of the HTTP request.
:return: the newly created object version.
:rtype: GcsObjectVersion
"""
media = testbench_utils.extract_media(request)
self.generation_generator += 1
goog_hash = request.headers.get('x-goog-hash')
md5hash = None
crc32c = None
if goog_hash is not None:
for hash in goog_hash.split(','):
if hash.startswith('md5='):
md5hash = hash[4:]
if hash.startswith('crc32c='):
crc32c = hash[7:]
revision = GcsObjectVersion(gcs_url, self.bucket_name, self.name,
self.generation_generator, request, media)
meta = revision.metadata.setdefault('metadata', {})
meta['x_testbench_upload'] = 'xml'
if md5hash is not None:
meta['x_testbench_md5'] = md5hash
revision.update_from_metadata({
'md5Hash': md5hash,
})
if crc32c is not None:
meta['x_testbench_crc32c'] = crc32c
revision.update_from_metadata({
'crc32c': crc32c,
})
self._insert_revision(revision)
return revision
def insert_multipart(self, gcs_url, request):
"""Insert a new revision based on the give flask request.
:param gcs_url:str the root URL for the fake GCS service.
:param request:flask.Request the contents of the HTTP request.
:return: the newly created object version.
:rtype: GcsObjectVersion
"""
content_type = request.headers.get('content-type')
if content_type is None or not content_type.startswith(
'multipart/related'):
raise error_response.ErrorResponse(
'Missing or invalid content-type header in multipart upload')
_, _, boundary = content_type.partition('boundary=')
if boundary is None:
raise error_response.ErrorResponse(
'Missing boundary (%s) in content-type header in multipart upload'
% boundary)
marker = '--' + boundary + '\r\n'
body = testbench_utils.extract_media(request)
parts = body.split(marker)
# parts[0] is the empty string, `multipart` should start with the boundary
# parts[1] is the JSON resource object part, with some headers
resource_headers, resource_body = self._parse_part(parts[1])
# parts[2] is the media, with some headers
media_headers, media_body = self._parse_part(parts[2])
end = media_body.find('\r\n--' + boundary + '--\r\n')
if end == -1:
raise error_response.ErrorResponse(
'Missing end marker (--%s--) in media body' % boundary)
media_body = media_body[:end]
self.generation += 1
revision = GcsObjectVersion(
gcs_url, self.bucket_name, self.name, self.generation, request,
media_body)
resource = json.loads(resource_body)
meta = revision.metadata.setdefault('metadata', {})
meta['x_testbench_upload'] = 'multipart'
meta['x_testbench_md5'] = resource.get('md5Hash', '')
meta['x_testbench_crc32c'] = resource.get('crc32c', '')
# Apply any overrides from the resource object part.
revision.update_from_metadata(resource)
# The content-type needs to be patched up, yuck.
if media_headers.get('content-type') is not None:
revision.update_from_metadata({
'contentType':
media_headers.get('content-type')
})
self._insert_revision(revision)
return revision
def insert(self, gcs_url, request):
"""Insert a new revision based on the give flask request.
:param gcs_url:str the root URL for the fake GCS service.
:param request:flask.Request the contents of the HTTP request.
:return: the newly created object version.
:rtype: GcsObjectVersion
"""
media = testbench_utils.extract_media(request)
self.generation_generator += 1
revision = GcsObjectVersion(gcs_url, self.bucket_name, self.name,
self.generation_generator, request, media)
meta = revision.metadata.setdefault('metadata', {})
meta['x_testbench_upload'] = 'simple'
self._insert_revision(revision)
return revision
def receive_upload_chunk(self, gcs_url, request):
"""Receive a new upload chunk.
:param gcs_url: str the base URL for the service.
:param request: flask.Request the original http request.
:return: the HTTP response.
"""
upload_id = request.args.get("upload_id")
if upload_id is None:
raise error_response.ErrorResponse(
"Missing upload_id in resumable_upload_chunk", status_code=400
)
upload = self.resumable_uploads.get(upload_id)
if upload is None:
raise error_response.ErrorResponse(
"Cannot find resumable upload %s" % upload_id, status_code=404
)
# Be gracious in what you accept, if the Content-Range header is not
# set we assume it is a good header and it is the end of the file.
next_byte = upload["next_byte"]
upload["transfer"].add(request.environ.get("HTTP_TRANSFER_ENCODING", ""))
end = next_byte + len(request.data)
total = end
final_chunk = False
payload = testbench_utils.extract_media(request)
content_range = request.headers.get("content-range")
if content_range is not None:
if content_range.startswith("bytes */*"):
# This is just a query to resume an upload, if it is done, return
# the completed upload payload and an empty range header.
response = flask.make_response(upload.get("payload", ""))
if next_byte > 1 and not upload["done"]:
response.headers["Range"] = "bytes=0-%d" % (next_byte - 1)
response.status_code = 200 if upload["done"] else 308
return response
match = re.match("bytes \*/(\\*|[0-9]+)", content_range)
if match:
if match.group(1) == "*":
total = 0
else:
total = int(match.group(1))
final_chunk = True
else:
match = re.match("bytes ([0-9]+)-([0-9]+)/(\\*|[0-9]+)", content_range)
if not match:
raise error_response.ErrorResponse(
"Invalid Content-Range in upload %s" % content_range,
status_code=400,
)
begin = int(match.group(1))
end = int(match.group(2))
if match.group(3) == "*":
total = 0
else:
total = int(match.group(3))
final_chunk = True
if begin != next_byte:
raise error_response.ErrorResponse(
"Mismatched data range, expected data at %d, got %d"
% (next_byte, begin),
status_code=400,
)
if len(payload) != end - begin + 1:
raise error_response.ErrorResponse(
"Mismatched data range (%d) vs. received data (%d)"
% (end - begin + 1, len(payload)),
status_code=400,
)
upload["media"] = upload.get("media", b"") + payload
next_byte = len(upload.get("media", ""))
upload["next_byte"] = next_byte
response_payload = ""
if final_chunk and next_byte >= total:
expected_bytes = upload["expected_bytes"]
if expected_bytes is not None and expected_bytes != total:
raise error_response.ErrorResponse(
"X-Upload-Content-Length"
"validation failed. Expected=%d, got %d." % (expected_bytes, total)
)
upload["done"] = True
object_name = upload.get("object_name")
object_path, blob = testbench_utils.get_object(
self.name, object_name, gcs_object.GcsObject(self.name, object_name)
)
# Release a few resources to control memory usage.
original_metadata = upload.pop("metadata", None)
media = upload.pop("media", None)
blob.check_preconditions_by_value(
upload.get("ifGenerationMatch"),
upload.get("ifGenerationNotMatch"),
upload.get("ifMetagenerationMatch"),
upload.get("ifMetagenerationNotMatch"),
)
if upload.pop("instructions", None) == "inject-upload-data-error":
media = testbench_utils.corrupt_media(media)
revision = blob.insert_resumable(gcs_url, request, media, original_metadata)
revision.metadata.setdefault("metadata", {})
revision.metadata["metadata"]["x_testbench_transfer_encoding"] = ":".join(
upload["transfer"]
)
response_payload = testbench_utils.filter_fields_from_response(
upload.get("fields"), revision.metadata
)
upload["payload"] = response_payload
testbench_utils.insert_object(object_path, blob)
response = flask.make_response(response_payload)
if next_byte == 0:
response.headers["Range"] = "bytes=0-0"
else:
response.headers["Range"] = "bytes=0-%d" % (next_byte - 1)
if upload.get("done", False):
response.status_code = 200
else:
response.status_code = 308
return response
