def initialize_download(self, http_request, http):
"""Initialize this download.
If the instance has :attr:`auto_transfer` enabled, begins the
download immediately.
:type http_request: :class:`gcloud.streaming.http_wrapper.Request`
:param http_request: the request to use to initialize this download.
:type http: :class:`httplib2.Http` (or workalike)
:param http: Http instance for this request.
"""
self._ensure_uninitialized()
url = http_request.url
if self.auto_transfer:
end_byte = self._compute_end_byte(0)
self._set_range_header(http_request, 0, end_byte)
response = make_api_request(self.bytes_http or http, http_request)
if response.status_code not in self._ACCEPTABLE_STATUSES:
raise HttpError.from_response(response)
self._initial_response = response
self._set_total(response.info)
url = response.info.get('content-location', response.request_url)
self._initialize(http, url)
# Unless the user has requested otherwise, we want to just
# go ahead and pump the bytes now.
if self.auto_transfer:
self.stream_file(use_chunks=True)
def _process_response(self, response):
"""Update attribtes and writing stream, based on response.
:type response: :class:`gcloud.streaming.http_wrapper.Response`
:param response: response from a download request.
:rtype: :class:`gcloud.streaming.http_wrapper.Response`
:returns: the response
:raises: :exc:`gcloud.streaming.exceptions.HttpError` for
missing / unauthorized responses;
:exc:`gcloud.streaming.exceptions.TransferRetryError`
for other error responses.
"""
if response.status_code not in self._ACCEPTABLE_STATUSES:
# We distinguish errors that mean we made a mistake in setting
# up the transfer versus something we should attempt again.
if response.status_code in (http_client.FORBIDDEN,
http_client.NOT_FOUND):
raise HttpError.from_response(response)
else:
raise TransferRetryError(response.content)
if response.status_code in (http_client.OK,
http_client.PARTIAL_CONTENT):
self.stream.write(response.content)
self._progress += response.length
if response.info and 'content-encoding' in response.info:
self._encoding = response.info['content-encoding']
elif response.status_code == http_client.NO_CONTENT:
# It's important to write something to the stream for the case
# of a 0-byte download to a file, as otherwise python won't
# create the file.
self.stream.write('')
return response
def _process_response(self, response):
"""Update attribtes and writing stream, based on response.
:type response: :class:`gcloud.streaming.http_wrapper.Response`
:param response: response from a download request.
:rtype: :class:`gcloud.streaming.http_wrapper.Response`
:returns: the response
:raises: :exc:`gcloud.streaming.exceptions.HttpError` for
missing / unauthorized responses;
:exc:`gcloud.streaming.exceptions.TransferRetryError`
for other error responses.
"""
if response.status_code not in self._ACCEPTABLE_STATUSES:
# We distinguish errors that mean we made a mistake in setting
# up the transfer versus something we should attempt again.
if response.status_code in (http_client.FORBIDDEN,
http_client.NOT_FOUND):
raise HttpError.from_response(response)
else:
raise TransferRetryError(response.content)
if response.status_code in (http_client.OK,
http_client.PARTIAL_CONTENT):
self.stream.write(response.content)
self._progress += response.length
if response.info and 'content-encoding' in response.info:
self._encoding = response.info['content-encoding']
elif response.status_code == http_client.NO_CONTENT:
# It's important to write something to the stream for the case
# of a 0-byte download to a file, as otherwise python won't
# create the file.
self.stream.write('')
return response
def _send_media_request(self, request, end):
"""Peform API upload request.
Helper for _send_media_body & _send_chunk:
:type request: :class:`gcloud.streaming.http_wrapper.Request`
:param request: the request to upload
:type end: integer
:param end: end byte of the to be uploaded
:rtype: :class:`gcloud.streaming.http_wrapper.Response`
:returns: the response
:raises: :exc:`gcloud.streaming.exceptions.HttpError` if the status
code from the response indicates an error.
"""
response = make_api_request(self.bytes_http,
request,
retries=self.num_retries)
if response.status_code not in (http_client.OK, http_client.CREATED,
RESUME_INCOMPLETE):
# We want to reset our state to wherever the server left us
# before this failed request, and then raise.
self.refresh_upload_state()
raise HttpError.from_response(response)
if response.status_code == RESUME_INCOMPLETE:
last_byte = self._last_byte(self._get_range_header(response))
if last_byte + 1 != end:
self.stream.seek(last_byte)
return response
def initialize_download(self, http_request, http):
"""Initialize this download.
If the instance has :attr:`auto_transfer` enabled, begins the
download immediately.
:type http_request: :class:`gcloud.streaming.http_wrapper.Request`
:param http_request: the request to use to initialize this download.
:type http: :class:`httplib2.Http` (or workalike)
:param http: Http instance for this request.
"""
self._ensure_uninitialized()
url = http_request.url
if self.auto_transfer:
end_byte = self._compute_end_byte(0)
self._set_range_header(http_request, 0, end_byte)
response = make_api_request(
self.bytes_http or http, http_request)
if response.status_code not in self._ACCEPTABLE_STATUSES:
raise HttpError.from_response(response)
self._initial_response = response
self._set_total(response.info)
url = response.info.get('content-location', response.request_url)
self._initialize(http, url)
# Unless the user has requested otherwise, we want to just
# go ahead and pump the bytes now.
if self.auto_transfer:
self.stream_file(use_chunks=True)
def _send_media_request(self, request, end):
"""Peform API upload request.
Helper for _send_media_body & _send_chunk:
:type request: :class:`gcloud.streaming.http_wrapper.Request`
:param request: the request to upload
:type end: integer
:param end: end byte of the to be uploaded
:rtype: :class:`gcloud.streaming.http_wrapper.Response`
:returns: the response
:raises: :exc:`gcloud.streaming.exceptions.HttpError` if the status
code from the response indicates an error.
"""
response = make_api_request(
self.bytes_http, request, retry_func=self.retry_func,
retries=self.num_retries)
if response.status_code not in (http_client.OK, http_client.CREATED,
RESUME_INCOMPLETE):
# We want to reset our state to wherever the server left us
# before this failed request, and then raise.
self.refresh_upload_state()
raise HttpError.from_response(response)
if response.status_code == RESUME_INCOMPLETE:
last_byte = self._last_byte(
self._get_range_header(response))
if last_byte + 1 != end:
self.stream.seek(last_byte)
return response
def refresh_upload_state(self):
"""Refresh the state of a resumable upload via query to the back-end.
"""
if self.strategy != RESUMABLE_UPLOAD:
return
self._ensure_initialized()
# NOTE: Per RFC 2616[1]/7231[2], a 'PUT' request is inappropriate
# here: it is intended to be used to replace the entire
# resource, not to query for a status.
#
# If the back-end doesn't provide a way to query for this state
# via a 'GET' request, somebody should be spanked.
#
# The violation is documented[3].
#
# [1] http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6
# [2] http://tools.ietf.org/html/rfc7231#section-4.3.4
# [3]
# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resume-upload
refresh_request = Request(url=self.url,
http_method='PUT',
headers={'Content-Range': 'bytes */*'})
refresh_response = make_api_request(self.http,
refresh_request,
redirections=0,
retries=self.num_retries)
range_header = self._get_range_header(refresh_response)
if refresh_response.status_code in (http_client.OK,
http_client.CREATED):
self._complete = True
self._progress = self.total_size
self.stream.seek(self.progress)
# If we're finished, the refresh response will contain the metadata
# originally requested. Cache it so it can be returned in
# StreamInChunks.
self._final_response = refresh_response
elif refresh_response.status_code == RESUME_INCOMPLETE:
if range_header is None:
self._progress = 0
else:
self._progress = self._last_byte(range_header) + 1
self.stream.seek(self.progress)
else:
raise HttpError.from_response(refresh_response)
def refresh_upload_state(self):
"""Refresh the state of a resumable upload via query to the back-end.
"""
if self.strategy != RESUMABLE_UPLOAD:
return
self._ensure_initialized()
# NOTE: Per RFC 2616[1]/7231[2], a 'PUT' request is inappropriate
# here: it is intended to be used to replace the entire
# resource, not to query for a status.
#
# If the back-end doesn't provide a way to query for this state
# via a 'GET' request, somebody should be spanked.
#
# The violation is documented[3].
#
# [1] http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6
# [2] http://tools.ietf.org/html/rfc7231#section-4.3.4
# [3]
# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resume-upload
refresh_request = Request(
url=self.url, http_method='PUT',
headers={'Content-Range': 'bytes */*'})
refresh_response = make_api_request(
self.http, refresh_request, redirections=0,
retries=self.num_retries)
range_header = self._get_range_header(refresh_response)
if refresh_response.status_code in (http_client.OK,
http_client.CREATED):
self._complete = True
self._progress = self.total_size
self.stream.seek(self.progress)
# If we're finished, the refresh response will contain the metadata
# originally requested. Cache it so it can be returned in
# StreamInChunks.
self._final_response = refresh_response
elif refresh_response.status_code == RESUME_INCOMPLETE:
if range_header is None:
self._progress = 0
else:
self._progress = self._last_byte(range_header) + 1
self.stream.seek(self.progress)
else:
raise HttpError.from_response(refresh_response)
def initialize_upload(self, http_request, http):
"""Initialize this upload from the given http_request.
:type http_request: :class:`gcloud.streaming.http_wrapper.Request`
:param http_request: the request to be used
:type http: :class:`httplib2.Http` (or workalike)
:param http: Http instance for this request.
:raises: :exc:`ValueError` if the instance has not been configured
with a strategy.
:rtype: :class:`~gcloud.streaming.http_wrapper.Response`
:returns: The response if the upload is resumable and auto transfer
is not used.
"""
if self.strategy is None:
raise ValueError(
'No upload strategy set; did you call configure_request?')
if self.strategy != RESUMABLE_UPLOAD:
return
self._ensure_uninitialized()
http_response = make_api_request(http,
http_request,
retries=self.num_retries)
if http_response.status_code != http_client.OK:
raise HttpError.from_response(http_response)
granularity = http_response.info.get('X-Goog-Upload-Chunk-Granularity')
if granularity is not None:
granularity = int(granularity)
self._server_chunk_granularity = granularity
url = http_response.info['location']
self._initialize(http, url)
# Unless the user has requested otherwise, we want to just
# go ahead and pump the bytes now.
if self.auto_transfer:
return self.stream_file(use_chunks=True)
else:
return http_response
def initialize_upload(self, http_request, http):
"""Initialize this upload from the given http_request.
:type http_request: :class:`gcloud.streaming.http_wrapper.Request`
:param http_request: the request to be used
:type http: :class:`httplib2.Http` (or workalike)
:param http: Http instance for this request.
:raises: :exc:`ValueError` if the instance has not been configured
with a strategy.
:rtype: :class:`~gcloud.streaming.http_wrapper.Response`
:returns: The response if the upload is resumable and auto transfer
is not used.
"""
if self.strategy is None:
raise ValueError(
'No upload strategy set; did you call configure_request?')
if self.strategy != RESUMABLE_UPLOAD:
return
self._ensure_uninitialized()
http_response = make_api_request(http, http_request,
retries=self.num_retries)
if http_response.status_code != http_client.OK:
raise HttpError.from_response(http_response)
granularity = http_response.info.get('X-Goog-Upload-Chunk-Granularity')
if granularity is not None:
granularity = int(granularity)
self._server_chunk_granularity = granularity
url = http_response.info['location']
self._initialize(http, url)
# Unless the user has requested otherwise, we want to just
# go ahead and pump the bytes now.
if self.auto_transfer:
return self.stream_file(use_chunks=True)
else:
return http_response
