def InitializeUpload(self, http_request, http=None, client=None): """Initialize this upload from the given http_request.""" if self.strategy is None: raise exceptions.UserError( 'No upload strategy set; did you call ConfigureRequest?') if http is None and client is None: raise exceptions.UserError('Must provide client or http.') if self.strategy != RESUMABLE_UPLOAD: return http = http or client.http if client is not None: http_request.url = client.FinalizeTransferUrl(http_request.url) self.EnsureUninitialized() http_response = http_wrapper.MakeRequest(http, http_request, retries=self.num_retries) if http_response.status_code != httplib.OK: raise exceptions.HttpError.FromResponse(http_response) self.__server_chunk_granularity = http_response.info.get( 'X-Goog-Upload-Chunk-Granularity') self.__ValidateChunksize() url = http_response.info['location'] if client is not None: url = client.FinalizeTransferUrl(url) self._Initialize(http, url) # Unless the user has requested otherwise, we want to just # go ahead and pump the bytes now. if self.auto_transfer: return self.StreamInChunks()
def RefreshResumableUploadState(self): """Talk to the server and refresh the state of this resumable upload. Returns: Response if the upload is complete. """ if self.strategy != RESUMABLE_UPLOAD: return self.EnsureInitialized() refresh_request = http_wrapper.Request( url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) refresh_response = http_wrapper.MakeRequest(self.http, refresh_request, redirections=0, retries=self.num_retries) range_header = self._GetRangeHeaderFromResponse(refresh_response) if refresh_response.status_code in (httplib.OK, httplib.CREATED): self.__complete = True self.__progress = self.total_size self.stream.seek(self.progress) # If we're finished, the refresh response will contain the metadata # originally requested. Cache it so it can be returned in StreamInChunks. self.__final_response = refresh_response elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: if range_header is None: self.__progress = 0 else: self.__progress = self.__GetLastByte(range_header) + 1 self.stream.seek(self.progress) else: raise exceptions.HttpError.FromResponse(refresh_response)
def __SendChunk(self, start, additional_headers=None): """Send the specified chunk.""" self.EnsureInitialized() end = min(start + self.chunksize, self.total_size) body_stream = stream_slice.StreamSlice(self.stream, end - start) # TODO: Think about clearer errors on "no data in stream". request = http_wrapper.Request(url=self.url, http_method='PUT', body=body_stream) request.headers['Content-Type'] = self.mime_type request.headers['Content-Range'] = 'bytes %s-%s/%s' % (start, end - 1, self.total_size) if additional_headers: request.headers.update(additional_headers) response = http_wrapper.MakeRequest(self.bytes_http, request, retry_func=self.retry_func) if response.status_code not in (httplib.OK, httplib.CREATED, http_wrapper.RESUME_INCOMPLETE): # We want to reset our state to wherever the server left us # before this failed request, and then raise. self.RefreshResumableUploadState() raise exceptions.HttpError.FromResponse(response) if response.status_code == http_wrapper.RESUME_INCOMPLETE: last_byte = self.__GetLastByte( self._GetRangeHeaderFromResponse(response)) if last_byte + 1 != end: self.stream.seek(last_byte) return response
def __GetChunk(self, start, end=None, additional_headers=None): """Retrieve a chunk, and return the full response.""" self.EnsureInitialized() end_byte = end if self.total_size and end: end_byte = min(end, self.total_size) request = http_wrapper.Request(url=self.url) self.__SetRangeHeader(request, start, end=end_byte) if additional_headers is not None: request.headers.update(additional_headers) return http_wrapper.MakeRequest(self.bytes_http, request, retry_func=self.retry_func)
def __SendChunk(self, start, additional_headers=None): """Send the specified chunk.""" self.EnsureInitialized() if self.total_size is None: # For the streaming resumable case, we need to detect when we're at the # end of the stream. body_stream = buffered_stream.BufferedStream( self.stream, start, self.chunksize) end = body_stream.stream_end_position if body_stream.stream_exhausted: self.__total_size = end else: end = min(start + self.chunksize, self.total_size) body_stream = stream_slice.StreamSlice(self.stream, end - start) # TODO: Think about clearer errors on "no data in stream". request = http_wrapper.Request(url=self.url, http_method='PUT', body=body_stream) request.headers['Content-Type'] = self.mime_type if self.total_size is None: # Streaming resumable upload case, unknown total size. range_string = 'bytes %s-%s/*' % (start, end - 1) elif end == start: # End of an upload with 0 bytes left to send; just finalize. range_string = 'bytes */%s' % self.total_size else: # Normal resumable upload case with known sizes. range_string = 'bytes %s-%s/%s' % (start, end - 1, self.total_size) request.headers['Content-Range'] = range_string if additional_headers: request.headers.update(additional_headers) response = http_wrapper.MakeRequest(self.bytes_http, request, retry_func=self.retry_func, retries=self.num_retries) if response.status_code not in (httplib.OK, httplib.CREATED, http_wrapper.RESUME_INCOMPLETE): # We want to reset our state to wherever the server left us # before this failed request, and then raise. self.RefreshResumableUploadState() raise exceptions.HttpError.FromResponse(response) if response.status_code == http_wrapper.RESUME_INCOMPLETE: last_byte = self.__GetLastByte( self._GetRangeHeaderFromResponse(response)) if last_byte + 1 != end: self.stream.seek(last_byte) return response
def __SendMediaRequest(self, request, end): """Helper function to make the request for SendMediaBody & SendChunk.""" response = http_wrapper.MakeRequest(self.bytes_http, request, retry_func=self.retry_func, retries=self.num_retries) if response.status_code not in (httplib.OK, httplib.CREATED, http_wrapper.RESUME_INCOMPLETE): # We want to reset our state to wherever the server left us # before this failed request, and then raise. self.RefreshResumableUploadState() raise exceptions.HttpError.FromResponse(response) if response.status_code == http_wrapper.RESUME_INCOMPLETE: last_byte = self.__GetLastByte( self._GetRangeHeaderFromResponse(response)) if last_byte + 1 != end: self.stream.seek(last_byte) return response
def _RunMethod(self, method_config, request, global_params=None, upload=None, upload_config=None, download=None): """Call this method with request.""" if upload is not None and download is not None: # TODO: This just involves refactoring the logic # below into callbacks that we can pass around; in particular, # the order should be that the upload gets the initial request, # and then passes its reply to a download if one exists, and # then that goes to ProcessResponse and is returned. raise exceptions.NotYetImplementedError( 'Cannot yet use both upload and download at once') http_request = self.PrepareHttpRequest(method_config, request, global_params, upload, upload_config, download) # TODO: Make num_retries customizable on Transfer # objects, and pass in self.__client.num_retries when initializing # an upload or download. if download is not None: download.InitializeDownload(http_request, client=self._client) return http_response = None if upload is not None: http_response = upload.InitializeUpload(http_request, client=self._client) if http_response is None: http = self.__client.http if upload and upload.bytes_http: http = upload.bytes_http http_response = http_wrapper.MakeRequest( http, http_request, retries=self.__client.num_retries) return self.ProcessHttpResponse(method_config, http_response)