def insert_multipart(self, gcs_url, request): """Insert a new revision based on the give flask request. :param gcs_url:str the root URL for the fake GCS service. :param request:flask.Request the contents of the HTTP request. :return: the newly created object version. :rtype: GcsObjectVersion """ content_type = request.headers.get('content-type') if content_type is None or not content_type.startswith( 'multipart/related'): raise error_response.ErrorResponse( 'Missing or invalid content-type header in multipart upload') _, _, boundary = content_type.partition('boundary=') if boundary is None: raise error_response.ErrorResponse( 'Missing boundary (%s) in content-type header in multipart upload' % boundary) marker = '--' + boundary + '\r\n' body = testbench_utils.extract_media(request) parts = body.split(marker) # parts[0] is the empty string, `multipart` should start with the boundary # parts[1] is the JSON resource object part, with some headers resource_headers, resource_body = self._parse_part(parts[1]) # parts[2] is the media, with some headers media_headers, media_body = self._parse_part(parts[2]) end = media_body.find('\r\n--' + boundary + '--\r\n') if end == -1: raise error_response.ErrorResponse( 'Missing end marker (--%s--) in media body' % boundary) media_body = media_body[:end] resource = json.loads(resource_body) # There are two ways to specify the content-type, the 'content-type' # header and the resource['contentType'] field. They must be consistent, # and the service generates an error when they are not. if (resource.get('contentType') is not None and media_headers.get('content-type') is not None and resource.get('contentType') != media_headers.get('content-type')): raise error_response.ErrorResponse( ('Content-Type specified in the upload (%s) does not match' + 'contentType specified in the metadata (%s).') % ( media_headers.get('content-type'), resource.get('contentType')), status_code=400) # Set the contentType in the resource from the header. Note that if both # are set they have the same value. resource.setdefault('contentType', media_headers.get('content-type')) self.generation += 1 revision = GcsObjectVersion( gcs_url, self.bucket_name, self.name, self.generation, request, media_body) meta = revision.metadata.setdefault('metadata', {}) meta['x_testbench_upload'] = 'multipart' meta['x_testbench_md5'] = resource.get('md5Hash', '') meta['x_testbench_crc32c'] = resource.get('crc32c', '') # Apply any overrides from the resource object part. revision.update_from_metadata(resource) self._insert_revision(revision) return revision
def insert_xml(self, gcs_url, request): """Implement the insert operation using the XML API. :param gcs_url:str the root URL for the fake GCS service. :param request:flask.Request the contents of the HTTP request. :return: the newly created object version. :rtype: GcsObjectVersion """ media = testbench_utils.extract_media(request) self.generation_generator += 1 goog_hash = request.headers.get('x-goog-hash') md5hash = None crc32c = None if goog_hash is not None: for hash in goog_hash.split(','): if hash.startswith('md5='): md5hash = hash[4:] if hash.startswith('crc32c='): crc32c = hash[7:] revision = GcsObjectVersion(gcs_url, self.bucket_name, self.name, self.generation_generator, request, media) meta = revision.metadata.setdefault('metadata', {}) meta['x_testbench_upload'] = 'xml' if md5hash is not None: meta['x_testbench_md5'] = md5hash revision.update_from_metadata({ 'md5Hash': md5hash, }) if crc32c is not None: meta['x_testbench_crc32c'] = crc32c revision.update_from_metadata({ 'crc32c': crc32c, }) self._insert_revision(revision) return revision
def insert_multipart(self, gcs_url, request): """Insert a new revision based on the give flask request. :param gcs_url:str the root URL for the fake GCS service. :param request:flask.Request the contents of the HTTP request. :return: the newly created object version. :rtype: GcsObjectVersion """ content_type = request.headers.get('content-type') if content_type is None or not content_type.startswith( 'multipart/related'): raise error_response.ErrorResponse( 'Missing or invalid content-type header in multipart upload') _, _, boundary = content_type.partition('boundary=') if boundary is None: raise error_response.ErrorResponse( 'Missing boundary (%s) in content-type header in multipart upload' % boundary) marker = '--' + boundary + '\r\n' body = testbench_utils.extract_media(request) parts = body.split(marker) # parts[0] is the empty string, `multipart` should start with the boundary # parts[1] is the JSON resource object part, with some headers resource_headers, resource_body = self._parse_part(parts[1]) # parts[2] is the media, with some headers media_headers, media_body = self._parse_part(parts[2]) end = media_body.find('\r\n--' + boundary + '--\r\n') if end == -1: raise error_response.ErrorResponse( 'Missing end marker (--%s--) in media body' % boundary) media_body = media_body[:end] self.generation += 1 revision = GcsObjectVersion( gcs_url, self.bucket_name, self.name, self.generation, request, media_body) resource = json.loads(resource_body) meta = revision.metadata.setdefault('metadata', {}) meta['x_testbench_upload'] = 'multipart' meta['x_testbench_md5'] = resource.get('md5Hash', '') meta['x_testbench_crc32c'] = resource.get('crc32c', '') # Apply any overrides from the resource object part. revision.update_from_metadata(resource) # The content-type needs to be patched up, yuck. if media_headers.get('content-type') is not None: revision.update_from_metadata({ 'contentType': media_headers.get('content-type') }) self._insert_revision(revision) return revision
def insert(self, gcs_url, request): """Insert a new revision based on the give flask request. :param gcs_url:str the root URL for the fake GCS service. :param request:flask.Request the contents of the HTTP request. :return: the newly created object version. :rtype: GcsObjectVersion """ media = testbench_utils.extract_media(request) self.generation_generator += 1 revision = GcsObjectVersion(gcs_url, self.bucket_name, self.name, self.generation_generator, request, media) meta = revision.metadata.setdefault('metadata', {}) meta['x_testbench_upload'] = 'simple' self._insert_revision(revision) return revision
def receive_upload_chunk(self, gcs_url, request): """Receive a new upload chunk. :param gcs_url: str the base URL for the service. :param request: flask.Request the original http request. :return: the HTTP response. """ upload_id = request.args.get("upload_id") if upload_id is None: raise error_response.ErrorResponse( "Missing upload_id in resumable_upload_chunk", status_code=400 ) upload = self.resumable_uploads.get(upload_id) if upload is None: raise error_response.ErrorResponse( "Cannot find resumable upload %s" % upload_id, status_code=404 ) # Be gracious in what you accept, if the Content-Range header is not # set we assume it is a good header and it is the end of the file. next_byte = upload["next_byte"] upload["transfer"].add(request.environ.get("HTTP_TRANSFER_ENCODING", "")) end = next_byte + len(request.data) total = end final_chunk = False payload = testbench_utils.extract_media(request) content_range = request.headers.get("content-range") if content_range is not None: if content_range.startswith("bytes */*"): # This is just a query to resume an upload, if it is done, return # the completed upload payload and an empty range header. response = flask.make_response(upload.get("payload", "")) if next_byte > 1 and not upload["done"]: response.headers["Range"] = "bytes=0-%d" % (next_byte - 1) response.status_code = 200 if upload["done"] else 308 return response match = re.match("bytes \*/(\\*|[0-9]+)", content_range) if match: if match.group(1) == "*": total = 0 else: total = int(match.group(1)) final_chunk = True else: match = re.match("bytes ([0-9]+)-([0-9]+)/(\\*|[0-9]+)", content_range) if not match: raise error_response.ErrorResponse( "Invalid Content-Range in upload %s" % content_range, status_code=400, ) begin = int(match.group(1)) end = int(match.group(2)) if match.group(3) == "*": total = 0 else: total = int(match.group(3)) final_chunk = True if begin != next_byte: raise error_response.ErrorResponse( "Mismatched data range, expected data at %d, got %d" % (next_byte, begin), status_code=400, ) if len(payload) != end - begin + 1: raise error_response.ErrorResponse( "Mismatched data range (%d) vs. received data (%d)" % (end - begin + 1, len(payload)), status_code=400, ) upload["media"] = upload.get("media", b"") + payload next_byte = len(upload.get("media", "")) upload["next_byte"] = next_byte response_payload = "" if final_chunk and next_byte >= total: expected_bytes = upload["expected_bytes"] if expected_bytes is not None and expected_bytes != total: raise error_response.ErrorResponse( "X-Upload-Content-Length" "validation failed. Expected=%d, got %d." % (expected_bytes, total) ) upload["done"] = True object_name = upload.get("object_name") object_path, blob = testbench_utils.get_object( self.name, object_name, gcs_object.GcsObject(self.name, object_name) ) # Release a few resources to control memory usage. original_metadata = upload.pop("metadata", None) media = upload.pop("media", None) blob.check_preconditions_by_value( upload.get("ifGenerationMatch"), upload.get("ifGenerationNotMatch"), upload.get("ifMetagenerationMatch"), upload.get("ifMetagenerationNotMatch"), ) if upload.pop("instructions", None) == "inject-upload-data-error": media = testbench_utils.corrupt_media(media) revision = blob.insert_resumable(gcs_url, request, media, original_metadata) revision.metadata.setdefault("metadata", {}) revision.metadata["metadata"]["x_testbench_transfer_encoding"] = ":".join( upload["transfer"] ) response_payload = testbench_utils.filter_fields_from_response( upload.get("fields"), revision.metadata ) upload["payload"] = response_payload testbench_utils.insert_object(object_path, blob) response = flask.make_response(response_payload) if next_byte == 0: response.headers["Range"] = "bytes=0-0" else: response.headers["Range"] = "bytes=0-%d" % (next_byte - 1) if upload.get("done", False): response.status_code = 200 else: response.status_code = 308 return response