def initialize_download(self, http_request, http): """Initialize this download. If the instance has :attr:`auto_transfer` enabled, begins the download immediately. :type http_request: :class:`~.streaming.http_wrapper.Request` :param http_request: the request to use to initialize this download. :type http: :class:`httplib2.Http` (or workalike) :param http: Http instance for this request. """ self._ensure_uninitialized() url = http_request.url if self.auto_transfer: end_byte = self._compute_end_byte(0) self._set_range_header(http_request, 0, end_byte) response = make_api_request(self.bytes_http or http, http_request) if response.status_code not in self._ACCEPTABLE_STATUSES: raise HttpError.from_response(response) self._initial_response = response self._set_total(response.info) url = response.info.get('content-location', response.request_url) self._initialize(http, url) # Unless the user has requested otherwise, we want to just # go ahead and pump the bytes now. if self.auto_transfer: self.stream_file(use_chunks=True, headers=http_request.headers)
def _process_response(self, response): """Update attribtes and writing stream, based on response. :type response: :class:`google.cloud.streaming.http_wrapper.Response` :param response: response from a download request. :rtype: :class:`google.cloud.streaming.http_wrapper.Response` :returns: the response :raises: :exc:`google.cloud.streaming.exceptions.HttpError` for missing / unauthorized responses; :exc:`google.cloud.streaming.exceptions.TransferRetryError` for other error responses. """ if response.status_code not in self._ACCEPTABLE_STATUSES: # We distinguish errors that mean we made a mistake in setting # up the transfer versus something we should attempt again. if response.status_code in (http_client.FORBIDDEN, http_client.NOT_FOUND): raise HttpError.from_response(response) else: raise TransferRetryError(response.content) if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): self.stream.write(response.content) self._progress += response.length if response.info and 'content-encoding' in response.info: self._encoding = response.info['content-encoding'] elif response.status_code == http_client.NO_CONTENT: # It's important to write something to the stream for the case # of a 0-byte download to a file, as otherwise python won't # create the file. self.stream.write('') return response
def _send_media_request(self, request, end): """Peform API upload request. Helper for _send_media_body & _send_chunk: :type request: :class:`google.cloud.streaming.http_wrapper.Request` :param request: the request to upload :type end: int :param end: end byte of the to be uploaded :rtype: :class:`google.cloud.streaming.http_wrapper.Response` :returns: the response :raises: :exc:`~.streaming.exceptions.HttpError` if the status code from the response indicates an error. """ response = make_api_request(self.bytes_http, request, retries=self.num_retries) if response.status_code not in (http_client.OK, http_client.CREATED, RESUME_INCOMPLETE): # We want to reset our state to wherever the server left us # before this failed request, and then raise. self.refresh_upload_state() raise HttpError.from_response(response) if response.status_code == RESUME_INCOMPLETE: last_byte = self._last_byte(self._get_range_header(response)) if last_byte + 1 != end: self.stream.seek(last_byte) return response
def initialize_download(self, http_request, http): """Initialize this download. If the instance has :attr:`auto_transfer` enabled, begins the download immediately. :type http_request: :class:`~.streaming.http_wrapper.Request` :param http_request: the request to use to initialize this download. :type http: :class:`httplib2.Http` (or workalike) :param http: Http instance for this request. """ self._ensure_uninitialized() url = http_request.url if self.auto_transfer: end_byte = self._compute_end_byte(0) self._set_range_header(http_request, 0, end_byte) response = make_api_request( self.bytes_http or http, http_request) if response.status_code not in self._ACCEPTABLE_STATUSES: raise HttpError.from_response(response) self._initial_response = response self._set_total(response.info) url = response.info.get('content-location', response.request_url) self._initialize(http, url) # Unless the user has requested otherwise, we want to just # go ahead and pump the bytes now. if self.auto_transfer: self.stream_file(use_chunks=True)
def _send_media_request(self, request, end): """Peform API upload request. Helper for _send_media_body & _send_chunk: :type request: :class:`google.cloud.streaming.http_wrapper.Request` :param request: the request to upload :type end: integer :param end: end byte of the to be uploaded :rtype: :class:`google.cloud.streaming.http_wrapper.Response` :returns: the response :raises: :exc:`~.streaming.exceptions.HttpError` if the status code from the response indicates an error. """ response = make_api_request( self.bytes_http, request, retries=self.num_retries) if response.status_code not in (http_client.OK, http_client.CREATED, RESUME_INCOMPLETE): # We want to reset our state to wherever the server left us # before this failed request, and then raise. self.refresh_upload_state() raise HttpError.from_response(response) if response.status_code == RESUME_INCOMPLETE: last_byte = self._last_byte( self._get_range_header(response)) if last_byte + 1 != end: self.stream.seek(last_byte) return response
def refresh_upload_state(self): """Refresh the state of a resumable upload via query to the back-end. """ if self.strategy != RESUMABLE_UPLOAD: return self._ensure_initialized() # NOTE: Per RFC 2616[1]/7231[2], a 'PUT' request is inappropriate # here: it is intended to be used to replace the entire # resource, not to query for a status. # # If the back-end doesn't provide a way to query for this state # via a 'GET' request, somebody should be spanked. # # The violation is documented[3]. # # [1] http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6 # [2] http://tools.ietf.org/html/rfc7231#section-4.3.4 # [3] # https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resume-upload refresh_request = Request(url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) refresh_response = make_api_request(self.http, refresh_request, redirections=0, retries=self.num_retries) range_header = self._get_range_header(refresh_response) if refresh_response.status_code in (http_client.OK, http_client.CREATED): self._complete = True self._progress = self.total_size self.stream.seek(self.progress) # If we're finished, the refresh response will contain the metadata # originally requested. Cache it so it can be returned in # StreamInChunks. self._final_response = refresh_response elif refresh_response.status_code == RESUME_INCOMPLETE: if range_header is None: self._progress = 0 else: self._progress = self._last_byte(range_header) + 1 self.stream.seek(self.progress) else: raise HttpError.from_response(refresh_response)
def refresh_upload_state(self): """Refresh the state of a resumable upload via query to the back-end. """ if self.strategy != RESUMABLE_UPLOAD: return self._ensure_initialized() # NOTE: Per RFC 2616[1]/7231[2], a 'PUT' request is inappropriate # here: it is intended to be used to replace the entire # resource, not to query for a status. # # If the back-end doesn't provide a way to query for this state # via a 'GET' request, somebody should be spanked. # # The violation is documented[3]. # # [1] http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6 # [2] http://tools.ietf.org/html/rfc7231#section-4.3.4 # [3] # https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resume-upload refresh_request = Request( url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) refresh_response = make_api_request( self.http, refresh_request, redirections=0, retries=self.num_retries) range_header = self._get_range_header(refresh_response) if refresh_response.status_code in (http_client.OK, http_client.CREATED): self._complete = True self._progress = self.total_size self.stream.seek(self.progress) # If we're finished, the refresh response will contain the metadata # originally requested. Cache it so it can be returned in # StreamInChunks. self._final_response = refresh_response elif refresh_response.status_code == RESUME_INCOMPLETE: if range_header is None: self._progress = 0 else: self._progress = self._last_byte(range_header) + 1 self.stream.seek(self.progress) else: raise HttpError.from_response(refresh_response)
def initialize_upload(self, http_request, http): """Initialize this upload from the given http_request. :type http_request: :class:`~.streaming.http_wrapper.Request` :param http_request: the request to be used :type http: :class:`httplib2.Http` (or workalike) :param http: Http instance for this request. :raises: :exc:`ValueError` if the instance has not been configured with a strategy. :rtype: :class:`~google.cloud.streaming.http_wrapper.Response` :returns: The response if the upload is resumable and auto transfer is not used. """ if self.strategy is None: raise ValueError( 'No upload strategy set; did you call configure_request?') if self.strategy != RESUMABLE_UPLOAD: return self._ensure_uninitialized() http_response = make_api_request(http, http_request, retries=self.num_retries) if http_response.status_code != http_client.OK: raise HttpError.from_response(http_response) granularity = http_response.info.get('X-Goog-Upload-Chunk-Granularity') if granularity is not None: granularity = int(granularity) self._server_chunk_granularity = granularity url = http_response.info['location'] self._initialize(http, url) # Unless the user has requested otherwise, we want to just # go ahead and pump the bytes now. if self.auto_transfer: return self.stream_file(use_chunks=True) else: return http_response