def transmit(self, transport, data, metadata, content_type): """Transmit the resource to be uploaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. data (bytes): The resource content to be uploaded. metadata (Mapping[str, str]): The resource metadata, such as an ACL list. content_type (str): The content type of the resource, e.g. a JPEG image has content type ``image/jpeg``. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ method, url, payload, headers = self._prepare_request( data, metadata, content_type) result = _helpers.http_request(transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy) self._process_response(result) return result
def initiate( self, transport, stream, metadata, content_type, total_bytes=None, stream_final=True, ): """Initiate a resumable upload. By default, this method assumes your ``stream`` is in a "final" state ready to transmit. However, ``stream_final=False`` can be used to indicate that the size of the resource is not known. This can happen if bytes are being dynamically fed into ``stream``, e.g. if the stream is attached to application logs. If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be read from the stream every time :meth:`transmit_next_chunk` is called. If one of those reads produces strictly fewer bites than the chunk size, the upload will be concluded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. stream (IO[bytes]): The stream (i.e. file-like object) that will be uploaded. The stream **must** be at the beginning (i.e. ``stream.tell() == 0``). metadata (Mapping[str, str]): The resource metadata, such as an ACL list. content_type (str): The content type of the resource, e.g. a JPEG image has content type ``image/jpeg``. total_bytes (Optional[int]): The total number of bytes to be uploaded. If specified, the upload size **will not** be determined from the stream (even if ``stream_final=True``). stream_final (Optional[bool]): Indicates if the ``stream`` is "final" (i.e. no more bytes will be added to it). In this case we determine the upload size from the size of the stream. If ``total_bytes`` is passed, this argument will be ignored. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ method, url, payload, headers = self._prepare_initiate_request( stream, metadata, content_type, total_bytes=total_bytes, stream_final=stream_final, ) response = _helpers.http_request( transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy, ) self._process_initiate_response(response) return response
def recover(self, transport): """Recover from a failure. This method should be used when a :class:`ResumableUpload` is in an :attr:`~ResumableUpload.invalid` state due to a request failure. This will verify the progress with the server and make sure the current upload is in a valid state before :meth:`transmit_next_chunk` can be used again. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ method, url, payload, headers = self._prepare_recover_request() # NOTE: We assume "payload is None" but pass it along anyway. result = _helpers.http_request(transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy) self._process_recover_response(result) return result
def consume_next_chunk(self, transport): """Consume the next chunk of the resource to be downloaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ValueError: If the current download has finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. result = _helpers.http_request( transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy, ) self._process_response(result) return result
def test_http_request(): transport, responses = _make_transport(http_client.OK) method = u"POST" url = u"http://test.invalid" data = mock.sentinel.data headers = {u"one": u"fish", u"blue": u"fish"} timeout = mock.sentinel.timeout ret_val = _helpers.http_request( transport, method, url, data=data, headers=headers, extra1=b"work", extra2=125.5, timeout=timeout, ) assert ret_val is responses[0] transport.request.assert_called_once_with( method, url, data=data, headers=headers, extra1=b"work", extra2=125.5, timeout=timeout, )
def test_http_request_defaults(): transport, responses = _make_transport(http_client.OK) method = u"POST" url = u"http://test.invalid" ret_val = _helpers.http_request(transport, method, url) assert ret_val is responses[0] transport.request.assert_called_once_with( method, url, data=None, headers=None, timeout=EXPECTED_TIMEOUT )
def test_http_request(): transport, responses = _make_transport(http_client.OK) method = u'POST' url = u'http://test.invalid' data = mock.sentinel.data headers = {u'one': u'fish', u'blue': u'fish'} ret_val = _helpers.http_request( transport, method, url, data=data, headers=headers, extra1=b'work', extra2=125.5) assert ret_val is responses[0] transport.request.assert_called_once_with( method, url, data=data, headers=headers, extra1=b'work', extra2=125.5)
def consume( self, transport, timeout=(_helpers._DEFAULT_CONNECT_TIMEOUT, _helpers._DEFAULT_READ_TIMEOUT), ): """Consume the resource to be downloaded. If a ``stream`` is attached to this download, then the downloaded resource will be written to the stream. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. ValueError: If the current :class:`Download` has already finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. result = _helpers.http_request( transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy, stream=True, timeout=timeout, ) self._process_response(result) if self._stream is not None: self._write_to_stream(result) return result
def transmit( self, transport, data, metadata, content_type, timeout=(_helpers._DEFAULT_CONNECT_TIMEOUT, _helpers._DEFAULT_READ_TIMEOUT), ): """Transmit the resource to be uploaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. data (bytes): The resource content to be uploaded. metadata (Mapping[str, str]): The resource metadata, such as an ACL list. content_type (str): The content type of the resource, e.g. a JPEG image has content type ``image/jpeg``. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ method, url, payload, headers = self._prepare_request( data, metadata, content_type) response = _helpers.http_request( transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy, timeout=timeout, ) self._process_response(response) return response
def consume_next_chunk(self, transport): """Consume the next chunk of the resource to be downloaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ValueError: If the current download has finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. result = _helpers.http_request( transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy) self._process_response(result) return result
def consume_next_chunk( self, transport, timeout=(_helpers._DEFAULT_CONNECT_TIMEOUT, _helpers._DEFAULT_READ_TIMEOUT), ): """Consume the next chunk of the resource to be downloaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ValueError: If the current download has finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. result = _helpers.http_request( transport, method, url, data=payload, headers=headers, stream=True, retry_strategy=self._retry_strategy, timeout=timeout, ) self._process_response(result) return result
def consume(self, transport): """Consume the resource to be downloaded. If a ``stream`` is attached to this download, then the downloaded resource will be written to the stream. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. ValueError: If the current :class:`Download` has already finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. request_kwargs = { u"data": payload, u"headers": headers, u"retry_strategy": self._retry_strategy, } if self._stream is not None: request_kwargs[u"stream"] = True result = _helpers.http_request(transport, method, url, **request_kwargs) self._process_response(result) if self._stream is not None: self._write_to_stream(result) return result
def consume(self, transport): """Consume the resource to be downloaded. If a ``stream`` is attached to this download, then the downloaded resource will be written to the stream. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. ValueError: If the current :class:`Download` has already finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. request_kwargs = { u'data': payload, u'headers': headers, u'retry_strategy': self._retry_strategy, } if self._stream is not None: request_kwargs[u'stream'] = True result = _helpers.http_request( transport, method, url, **request_kwargs) self._process_response(result) if self._stream is not None: self._write_to_stream(result) return result
def consume(self, transport): """Consume the resource to be downloaded. If a ``stream`` is attached to this download, then the downloaded resource will be written to the stream. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ValueError: If the current :class:`Download` has already finished. """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. request_kwargs = { u'data': payload, u'headers': headers, u'retry_strategy': self._retry_strategy, } if self._stream is not None: request_kwargs[u'stream'] = True result = _helpers.http_request(transport, method, url, **request_kwargs) self._process_response(result) if self._stream is not None: self._write_to_stream(result) return result
def transmit_next_chunk(self, transport): """Transmit the next chunk of the resource to be uploaded. If the current upload was initiated with ``stream_final=False``, this method will dynamically determine if the upload has completed. The upload will be considered complete if the stream produces fewer than :attr:`chunk_size` bytes when a chunk is read from it. In the case of failure, an exception is thrown that preserves the failed response: .. testsetup:: bad-response import io import mock import requests from six.moves import http_client from google import resumable_media import google.resumable_media.requests.upload as upload_mod transport = mock.Mock(spec=[u'request']) fake_response = requests.Response() fake_response.status_code = int(http_client.BAD_REQUEST) transport.request.return_value = fake_response upload_url = u'http://test.invalid' upload = upload_mod.ResumableUpload( upload_url, resumable_media.UPLOAD_CHUNK_SIZE) # Fake that the upload has been initiate()-d data = b'data is here' upload._stream = io.BytesIO(data) upload._total_bytes = len(data) upload._resumable_url = u'http://test.invalid?upload_id=nope' .. doctest:: bad-response :options: +NORMALIZE_WHITESPACE >>> error = None >>> try: ... upload.transmit_next_chunk(transport) ... except resumable_media.InvalidResponse as caught_exc: ... error = caught_exc ... >>> error InvalidResponse('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>, 308) >>> error.response <Response [400]> Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.exceptions.InvalidResponse: If the status code is not 200 or 308. """ method, url, payload, headers = self._prepare_request() result = _helpers.http_request(transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy) self._process_response(result) return result
def transmit_next_chunk( self, transport, timeout=(_helpers._DEFAULT_CONNECT_TIMEOUT, _helpers._DEFAULT_READ_TIMEOUT), ): """Transmit the next chunk of the resource to be uploaded. If the current upload was initiated with ``stream_final=False``, this method will dynamically determine if the upload has completed. The upload will be considered complete if the stream produces fewer than :attr:`chunk_size` bytes when a chunk is read from it. In the case of failure, an exception is thrown that preserves the failed response: .. testsetup:: bad-response import io import mock import requests from six.moves import http_client from google import resumable_media import google.resumable_media.requests.upload as upload_mod transport = mock.Mock(spec=['request']) fake_response = requests.Response() fake_response.status_code = int(http_client.BAD_REQUEST) transport.request.return_value = fake_response upload_url = u'http://test.invalid' upload = upload_mod.ResumableUpload( upload_url, resumable_media.UPLOAD_CHUNK_SIZE) # Fake that the upload has been initiate()-d data = b'data is here' upload._stream = io.BytesIO(data) upload._total_bytes = len(data) upload._resumable_url = u'http://test.invalid?upload_id=nope' .. doctest:: bad-response :options: +NORMALIZE_WHITESPACE >>> error = None >>> try: ... upload.transmit_next_chunk(transport) ... except resumable_media.InvalidResponse as caught_exc: ... error = caught_exc ... >>> error InvalidResponse('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>, 308) >>> error.response <Response [400]> Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.common.InvalidResponse: If the status code is not 200 or 308. """ method, url, payload, headers = self._prepare_request() response = _helpers.http_request( transport, method, url, data=payload, headers=headers, retry_strategy=self._retry_strategy, timeout=timeout, ) self._process_response(response, len(payload)) return response