Example #1
0
    def initialize_download(self, http_request, http):
        """Initialize this download.

        If the instance has :attr:`auto_transfer` enabled, begins the
        download immediately.

        :type http_request: :class:`~.streaming.http_wrapper.Request`
        :param http_request: the request to use to initialize this download.

        :type http: :class:`httplib2.Http` (or workalike)
        :param http: Http instance for this request.
        """
        self._ensure_uninitialized()
        url = http_request.url
        if self.auto_transfer:
            end_byte = self._compute_end_byte(0)
            self._set_range_header(http_request, 0, end_byte)
            response = make_api_request(self.bytes_http or http, http_request)
            if response.status_code not in self._ACCEPTABLE_STATUSES:
                raise HttpError.from_response(response)
            self._initial_response = response
            self._set_total(response.info)
            url = response.info.get('content-location', response.request_url)
        self._initialize(http, url)
        # Unless the user has requested otherwise, we want to just
        # go ahead and pump the bytes now.
        if self.auto_transfer:
            self.stream_file(use_chunks=True, headers=http_request.headers)
Example #2
0
    def _process_response(self, response):
        """Update attribtes and writing stream, based on response.

        :type response: :class:`google.cloud.streaming.http_wrapper.Response`
        :param response: response from a download request.

        :rtype: :class:`google.cloud.streaming.http_wrapper.Response`
        :returns: the response
        :raises: :exc:`google.cloud.streaming.exceptions.HttpError` for
                 missing / unauthorized responses;
                 :exc:`google.cloud.streaming.exceptions.TransferRetryError`
                 for other error responses.
        """
        if response.status_code not in self._ACCEPTABLE_STATUSES:
            # We distinguish errors that mean we made a mistake in setting
            # up the transfer versus something we should attempt again.
            if response.status_code in (http_client.FORBIDDEN,
                                        http_client.NOT_FOUND):
                raise HttpError.from_response(response)
            else:
                raise TransferRetryError(response.content)
        if response.status_code in (http_client.OK,
                                    http_client.PARTIAL_CONTENT):
            self.stream.write(response.content)
            self._progress += response.length
            if response.info and 'content-encoding' in response.info:
                self._encoding = response.info['content-encoding']
        elif response.status_code == http_client.NO_CONTENT:
            # It's important to write something to the stream for the case
            # of a 0-byte download to a file, as otherwise python won't
            # create the file.
            self.stream.write('')
        return response
Example #3
0
    def _process_response(self, response):
        """Update attribtes and writing stream, based on response.

        :type response: :class:`google.cloud.streaming.http_wrapper.Response`
        :param response: response from a download request.

        :rtype: :class:`google.cloud.streaming.http_wrapper.Response`
        :returns: the response
        :raises: :exc:`google.cloud.streaming.exceptions.HttpError` for
                 missing / unauthorized responses;
                 :exc:`google.cloud.streaming.exceptions.TransferRetryError`
                 for other error responses.
        """
        if response.status_code not in self._ACCEPTABLE_STATUSES:
            # We distinguish errors that mean we made a mistake in setting
            # up the transfer versus something we should attempt again.
            if response.status_code in (http_client.FORBIDDEN,
                                        http_client.NOT_FOUND):
                raise HttpError.from_response(response)
            else:
                raise TransferRetryError(response.content)
        if response.status_code in (http_client.OK,
                                    http_client.PARTIAL_CONTENT):
            self.stream.write(response.content)
            self._progress += response.length
            if response.info and 'content-encoding' in response.info:
                self._encoding = response.info['content-encoding']
        elif response.status_code == http_client.NO_CONTENT:
            # It's important to write something to the stream for the case
            # of a 0-byte download to a file, as otherwise python won't
            # create the file.
            self.stream.write('')
        return response
Example #4
0
    def _send_media_request(self, request, end):
        """Peform API upload request.

        Helper for _send_media_body & _send_chunk:

        :type request: :class:`google.cloud.streaming.http_wrapper.Request`
        :param request: the request to upload

        :type end: int
        :param end: end byte of the to be uploaded

        :rtype: :class:`google.cloud.streaming.http_wrapper.Response`
        :returns: the response
        :raises: :exc:`~.streaming.exceptions.HttpError` if the status
                 code from the response indicates an error.
        """
        response = make_api_request(self.bytes_http,
                                    request,
                                    retries=self.num_retries)
        if response.status_code not in (http_client.OK, http_client.CREATED,
                                        RESUME_INCOMPLETE):
            # We want to reset our state to wherever the server left us
            # before this failed request, and then raise.
            self.refresh_upload_state()
            raise HttpError.from_response(response)
        if response.status_code == RESUME_INCOMPLETE:
            last_byte = self._last_byte(self._get_range_header(response))
            if last_byte + 1 != end:
                self.stream.seek(last_byte)
        return response
Example #5
0
    def initialize_download(self, http_request, http):
        """Initialize this download.

        If the instance has :attr:`auto_transfer` enabled, begins the
        download immediately.

        :type http_request: :class:`~.streaming.http_wrapper.Request`
        :param http_request: the request to use to initialize this download.

        :type http: :class:`httplib2.Http` (or workalike)
        :param http: Http instance for this request.
        """
        self._ensure_uninitialized()
        url = http_request.url
        if self.auto_transfer:
            end_byte = self._compute_end_byte(0)
            self._set_range_header(http_request, 0, end_byte)
            response = make_api_request(
                self.bytes_http or http, http_request)
            if response.status_code not in self._ACCEPTABLE_STATUSES:
                raise HttpError.from_response(response)
            self._initial_response = response
            self._set_total(response.info)
            url = response.info.get('content-location', response.request_url)
        self._initialize(http, url)
        # Unless the user has requested otherwise, we want to just
        # go ahead and pump the bytes now.
        if self.auto_transfer:
            self.stream_file(use_chunks=True)
Example #6
0
    def _send_media_request(self, request, end):
        """Peform API upload request.

        Helper for _send_media_body & _send_chunk:

        :type request: :class:`google.cloud.streaming.http_wrapper.Request`
        :param request: the request to upload

        :type end: integer
        :param end: end byte of the to be uploaded

        :rtype: :class:`google.cloud.streaming.http_wrapper.Response`
        :returns: the response
        :raises: :exc:`~.streaming.exceptions.HttpError` if the status
                 code from the response indicates an error.
        """
        response = make_api_request(
            self.bytes_http, request, retries=self.num_retries)
        if response.status_code not in (http_client.OK, http_client.CREATED,
                                        RESUME_INCOMPLETE):
            # We want to reset our state to wherever the server left us
            # before this failed request, and then raise.
            self.refresh_upload_state()
            raise HttpError.from_response(response)
        if response.status_code == RESUME_INCOMPLETE:
            last_byte = self._last_byte(
                self._get_range_header(response))
            if last_byte + 1 != end:
                self.stream.seek(last_byte)
        return response
Example #7
0
 def refresh_upload_state(self):
     """Refresh the state of a resumable upload via query to the back-end.
     """
     if self.strategy != RESUMABLE_UPLOAD:
         return
     self._ensure_initialized()
     # NOTE: Per RFC 2616[1]/7231[2], a 'PUT' request is inappropriate
     #       here:  it is intended to be used to replace the entire
     #       resource, not to  query for a status.
     #
     #       If the back-end doesn't provide a way to query for this state
     #       via a 'GET' request, somebody should be spanked.
     #
     #       The violation is documented[3].
     #
     # [1] http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6
     # [2] http://tools.ietf.org/html/rfc7231#section-4.3.4
     # [3]
     # https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resume-upload
     refresh_request = Request(url=self.url,
                               http_method='PUT',
                               headers={'Content-Range': 'bytes */*'})
     refresh_response = make_api_request(self.http,
                                         refresh_request,
                                         redirections=0,
                                         retries=self.num_retries)
     range_header = self._get_range_header(refresh_response)
     if refresh_response.status_code in (http_client.OK,
                                         http_client.CREATED):
         self._complete = True
         self._progress = self.total_size
         self.stream.seek(self.progress)
         # If we're finished, the refresh response will contain the metadata
         # originally requested. Cache it so it can be returned in
         # StreamInChunks.
         self._final_response = refresh_response
     elif refresh_response.status_code == RESUME_INCOMPLETE:
         if range_header is None:
             self._progress = 0
         else:
             self._progress = self._last_byte(range_header) + 1
         self.stream.seek(self.progress)
     else:
         raise HttpError.from_response(refresh_response)
Example #8
0
 def refresh_upload_state(self):
     """Refresh the state of a resumable upload via query to the back-end.
     """
     if self.strategy != RESUMABLE_UPLOAD:
         return
     self._ensure_initialized()
     # NOTE: Per RFC 2616[1]/7231[2], a 'PUT' request is inappropriate
     #       here:  it is intended to be used to replace the entire
     #       resource, not to  query for a status.
     #
     #       If the back-end doesn't provide a way to query for this state
     #       via a 'GET' request, somebody should be spanked.
     #
     #       The violation is documented[3].
     #
     # [1] http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6
     # [2] http://tools.ietf.org/html/rfc7231#section-4.3.4
     # [3]
     # https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resume-upload
     refresh_request = Request(
         url=self.url, http_method='PUT',
         headers={'Content-Range': 'bytes */*'})
     refresh_response = make_api_request(
         self.http, refresh_request, redirections=0,
         retries=self.num_retries)
     range_header = self._get_range_header(refresh_response)
     if refresh_response.status_code in (http_client.OK,
                                         http_client.CREATED):
         self._complete = True
         self._progress = self.total_size
         self.stream.seek(self.progress)
         # If we're finished, the refresh response will contain the metadata
         # originally requested. Cache it so it can be returned in
         # StreamInChunks.
         self._final_response = refresh_response
     elif refresh_response.status_code == RESUME_INCOMPLETE:
         if range_header is None:
             self._progress = 0
         else:
             self._progress = self._last_byte(range_header) + 1
         self.stream.seek(self.progress)
     else:
         raise HttpError.from_response(refresh_response)
Example #9
0
    def initialize_upload(self, http_request, http):
        """Initialize this upload from the given http_request.

        :type http_request: :class:`~.streaming.http_wrapper.Request`
        :param http_request: the request to be used

        :type http: :class:`httplib2.Http` (or workalike)
        :param http: Http instance for this request.

        :raises: :exc:`ValueError` if the instance has not been configured
                 with a strategy.
        :rtype: :class:`~google.cloud.streaming.http_wrapper.Response`
        :returns: The response if the upload is resumable and auto transfer
                  is not used.
        """
        if self.strategy is None:
            raise ValueError(
                'No upload strategy set; did you call configure_request?')
        if self.strategy != RESUMABLE_UPLOAD:
            return
        self._ensure_uninitialized()
        http_response = make_api_request(http,
                                         http_request,
                                         retries=self.num_retries)
        if http_response.status_code != http_client.OK:
            raise HttpError.from_response(http_response)

        granularity = http_response.info.get('X-Goog-Upload-Chunk-Granularity')
        if granularity is not None:
            granularity = int(granularity)
        self._server_chunk_granularity = granularity
        url = http_response.info['location']
        self._initialize(http, url)

        # Unless the user has requested otherwise, we want to just
        # go ahead and pump the bytes now.
        if self.auto_transfer:
            return self.stream_file(use_chunks=True)
        else:
            return http_response
Example #10
0
    def initialize_upload(self, http_request, http):
        """Initialize this upload from the given http_request.

        :type http_request: :class:`~.streaming.http_wrapper.Request`
        :param http_request: the request to be used

        :type http: :class:`httplib2.Http` (or workalike)
        :param http: Http instance for this request.

        :raises: :exc:`ValueError` if the instance has not been configured
                 with a strategy.
        :rtype: :class:`~google.cloud.streaming.http_wrapper.Response`
        :returns: The response if the upload is resumable and auto transfer
                  is not used.
        """
        if self.strategy is None:
            raise ValueError(
                'No upload strategy set; did you call configure_request?')
        if self.strategy != RESUMABLE_UPLOAD:
            return
        self._ensure_uninitialized()
        http_response = make_api_request(http, http_request,
                                         retries=self.num_retries)
        if http_response.status_code != http_client.OK:
            raise HttpError.from_response(http_response)

        granularity = http_response.info.get('X-Goog-Upload-Chunk-Granularity')
        if granularity is not None:
            granularity = int(granularity)
        self._server_chunk_granularity = granularity
        url = http_response.info['location']
        self._initialize(http, url)

        # Unless the user has requested otherwise, we want to just
        # go ahead and pump the bytes now.
        if self.auto_transfer:
            return self.stream_file(use_chunks=True)
        else:
            return http_response