def _finish_futures(self, responses):
"""Apply all the batch responses to the futures created.
:type responses: list of (headers, payload) tuples.
:param responses: List of headers and payloads from each response in
the batch.
:raises: :class:`ValueError` if no requests have been deferred.
"""
# If a bad status occurs, we track it, but don't raise an exception
# until all futures have been populated.
exception_args = None
if len(self._target_objects) != len(responses):
raise ValueError("Expected a response for every request.")
for target_object, subresponse in zip(self._target_objects, responses):
if not 200 <= subresponse.status_code < 300:
exception_args = exception_args or subresponse
elif target_object is not None:
try:
target_object._properties = subresponse.json()
except ValueError:
target_object._properties = subresponse.content
if exception_args is not None:
raise exceptions.from_http_response(exception_args)
def _finish_futures(self, responses):
"""Apply all the batch responses to the futures created.
:type responses: list of (headers, payload) tuples.
:param responses: List of headers and payloads from each response in
the batch.
:raises: :class:`ValueError` if no requests have been deferred.
"""
# If a bad status occurs, we track it, but don't raise an exception
# until all futures have been populated.
exception_args = None
if len(self._target_objects) != len(responses): # pragma: NO COVER
raise ValueError("Expected a response for every request.")
for target_object, subresponse in zip(self._target_objects, responses):
if not 200 <= subresponse.status_code < 300:
exception_args = exception_args or subresponse
elif target_object is not None:
try:
target_object._properties = subresponse.json()
except ValueError:
target_object._properties = subresponse.content
if exception_args is not None:
raise exceptions.from_http_response(exception_args)
def finish(self):
"""Submit a single `multipart/mixed` request with deferred requests.
:rtype: list of tuples
:returns: one ``(headers, payload)`` tuple per deferred request.
"""
headers, body, timeout = self._prepare_batch_request()
url = "%s/batch/storage/v1" % self.API_BASE_URL
# Use the private ``_base_connection`` rather than the property
# ``_connection``, since the property may be this
# current batch.
response = self._client._base_connection._make_request("POST",
url,
data=body,
headers=headers,
timeout=timeout)
# Raise exception if the top-level batch request fails
if not 200 <= response.status_code < 300:
raise exceptions.from_http_response(response)
responses = list(_unpack_batch_response(response))
self._finish_futures(responses)
return responses
def test_from_http_response_text_content():
response = make_response(b'message')
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == 'POST https://example.com/: message'
def test_from_http_response_bad_json_content():
response = make_response(json.dumps({'meep': 'moop'}).encode('utf-8'))
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == 'POST https://example.com/: unknown error'
def test_from_http_response_text_content():
response = make_response(b"message")
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == "POST https://example.com/: message"
def test_from_http_response_bad_json_content():
response = make_response(json.dumps({"meep": "moop"}).encode("utf-8"))
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == "POST https://example.com/: unknown error"
def test_from_http_response_no_content():
response = make_response(None)
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == 'POST https://example.com/: unknown error'
assert exception.response == response
def test_from_http_response_no_content():
response = make_response(None)
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == "POST https://example.com/: unknown error"
assert exception.response == response
def test_from_http_response_json_content():
response = make_response(
json.dumps({"error": {"message": "json message", "errors": ["1", "2"]}}).encode(
"utf-8"
)
)
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == "POST https://example.com/: json message"
assert exception.errors == ["1", "2"]
def test_from_http_response_json_content():
response = make_response(json.dumps({
'error': {
'message': 'json message',
'errors': ['1', '2']
}
}).encode('utf-8'))
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == 'POST https://example.com/: json message'
assert exception.errors == ['1', '2']
def test_from_http_response_json_content():
response = make_response(
json.dumps({
"error": {
"message": "json message",
"errors": ["1", "2"]
}
}).encode("utf-8"))
exception = exceptions.from_http_response(response)
assert isinstance(exception, exceptions.NotFound)
assert exception.code == http_client.NOT_FOUND
assert exception.message == "POST https://example.com/: json message"
assert exception.errors == ["1", "2"]
def api_request(
self,
method,
path,
query_params=None,
data=None,
content_type=None,
headers=None,
api_base_url=None,
api_version=None,
expect_json=True,
_target_object=None,
timeout=_DEFAULT_TIMEOUT,
):
"""Make a request over the HTTP transport to the API.
You shouldn't need to use this method, but if you plan to
interact with the API using these primitives, this is the
correct one to use.
:type method: str
:param method: The HTTP method name (ie, ``GET``, ``POST``, etc).
Required.
:type path: str
:param path: The path to the resource (ie, ``'/b/bucket-name'``).
Required.
:type query_params: dict or list
:param query_params: A dictionary of keys and values (or list of
key-value pairs) to insert into the query
string of the URL.
:type data: str
:param data: The data to send as the body of the request. Default is
the empty string.
:type content_type: str
:param content_type: The proper MIME type of the data provided. Default
is None.
:type headers: dict
:param headers: extra HTTP headers to be sent with the request.
:type api_base_url: str
:param api_base_url: The base URL for the API endpoint.
Typically you won't have to provide this.
Default is the standard API base URL.
:type api_version: str
:param api_version: The version of the API to call. Typically
you shouldn't provide this and instead use
the default for the library. Default is the
latest API version supported by
google-cloud-python.
:type expect_json: bool
:param expect_json: If True, this method will try to parse the
response as JSON and raise an exception if
that cannot be done. Default is True.
:type _target_object: :class:`object`
:param _target_object:
(Optional) Protected argument to be used by library callers. This
can allow custom behavior, for example, to defer an HTTP request
and complete initialization of the object at a later time.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
:raises ~google.cloud.exceptions.GoogleCloudError: if the response code
is not 200 OK.
:raises ValueError: if the response content type is not JSON.
:rtype: dict or str
:returns: The API response payload, either as a raw string or
a dictionary if the response is valid JSON.
"""
url = self.build_api_url(
path=path,
query_params=query_params,
api_base_url=api_base_url,
api_version=api_version,
)
# Making the executive decision that any dictionary
# data will be sent properly as JSON.
if data and isinstance(data, dict):
data = json.dumps(data)
content_type = "application/json"
response = self._make_request(
method=method,
url=url,
data=data,
content_type=content_type,
headers=headers,
target_object=_target_object,
timeout=timeout,
)
if not 200 <= response.status_code < 300:
raise exceptions.from_http_response(response)
if expect_json and response.content:
return response.json()
else:
return response.content
def upload_from_file(self,
file_obj,
source_format,
rewind=False,
size=None,
num_retries=_DEFAULT_NUM_RETRIES,
allow_jagged_rows=None,
allow_quoted_newlines=None,
create_disposition=None,
encoding=None,
field_delimiter=None,
ignore_unknown_values=None,
max_bad_records=None,
quote_character=None,
skip_leading_rows=None,
write_disposition=None,
client=None,
job_name=None,
null_marker=None):
"""Upload the contents of this table from a file-like object.
:type file_obj: file
:param file_obj: A file handle opened in binary mode for reading.
:type source_format: str
:param source_format: Any supported format. The full list of supported
formats is documented under the
``configuration.extract.destinationFormat`` property on this page:
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs
:type rewind: bool
:param rewind: If True, seek to the beginning of the file handle before
writing the file.
:type size: int
:param size: The number of bytes to read from the file handle.
If not provided, we'll try to guess the size using
:func:`os.fstat`. (If the file handle is not from the
filesystem this won't be possible.)
:type num_retries: int
:param num_retries: Number of upload retries. Defaults to 6.
:type allow_jagged_rows: bool
:param allow_jagged_rows: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type allow_quoted_newlines: bool
:param allow_quoted_newlines: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type create_disposition: str
:param create_disposition: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type encoding: str
:param encoding: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type field_delimiter: str
:param field_delimiter: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type ignore_unknown_values: bool
:param ignore_unknown_values: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type max_bad_records: int
:param max_bad_records: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type quote_character: str
:param quote_character: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type skip_leading_rows: int
:param skip_leading_rows: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type write_disposition: str
:param write_disposition: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type client: :class:`~google.cloud.bigquery.client.Client`
:param client: (Optional) The client to use. If not passed, falls back
to the ``client`` stored on the current table.
:type job_name: str
:param job_name: Optional. The id of the job. Generated if not
explicitly passed in.
:type null_marker: str
:param null_marker: Optional. A custom null marker (example: "\\N")
:rtype: :class:`~google.cloud.bigquery.jobs.LoadTableFromStorageJob`
:returns: the job instance used to load the data (e.g., for
querying status). Note that the job is already started:
do not call ``job.begin()``.
:raises: :class:`ValueError` if ``size`` is not passed in and can not
be determined, or if the ``file_obj`` can be detected to be
a file opened in text mode.
"""
client = self._require_client(client)
_maybe_rewind(file_obj, rewind=rewind)
_check_mode(file_obj)
metadata = _get_upload_metadata(source_format, self._schema,
self._dataset, self.name)
_configure_job_metadata(metadata, allow_jagged_rows,
allow_quoted_newlines, create_disposition,
encoding, field_delimiter,
ignore_unknown_values, max_bad_records,
quote_character, skip_leading_rows,
write_disposition, job_name, null_marker)
try:
created_json = self._do_upload(client, file_obj, metadata, size,
num_retries)
return client.job_from_resource(created_json)
except resumable_media.InvalidResponse as exc:
raise exceptions.from_http_response(exc.response)
def api_request(self, method, path, query_params=None,
data=None, content_type=None, headers=None,
api_base_url=None, api_version=None,
expect_json=True, _target_object=None):
"""Make a request over the HTTP transport to the API.
You shouldn't need to use this method, but if you plan to
interact with the API using these primitives, this is the
correct one to use.
:type method: str
:param method: The HTTP method name (ie, ``GET``, ``POST``, etc).
Required.
:type path: str
:param path: The path to the resource (ie, ``'/b/bucket-name'``).
Required.
:type query_params: dict or list
:param query_params: A dictionary of keys and values (or list of
key-value pairs) to insert into the query
string of the URL.
:type data: str
:param data: The data to send as the body of the request. Default is
the empty string.
:type content_type: str
:param content_type: The proper MIME type of the data provided. Default
is None.
:type headers: dict
:param headers: extra HTTP headers to be sent with the request.
:type api_base_url: str
:param api_base_url: The base URL for the API endpoint.
Typically you won't have to provide this.
Default is the standard API base URL.
:type api_version: str
:param api_version: The version of the API to call. Typically
you shouldn't provide this and instead use
the default for the library. Default is the
latest API version supported by
google-cloud-python.
:type expect_json: bool
:param expect_json: If True, this method will try to parse the
response as JSON and raise an exception if
that cannot be done. Default is True.
:type _target_object: :class:`object`
:param _target_object:
(Optional) Protected argument to be used by library callers. This
can allow custom behavior, for example, to defer an HTTP request
and complete initialization of the object at a later time.
:raises ~google.cloud.exceptions.GoogleCloudError: if the response code
is not 200 OK.
:raises ValueError: if the response content type is not JSON.
:rtype: dict or str
:returns: The API response payload, either as a raw string or
a dictionary if the response is valid JSON.
"""
url = self.build_api_url(path=path, query_params=query_params,
api_base_url=api_base_url,
api_version=api_version)
# Making the executive decision that any dictionary
# data will be sent properly as JSON.
if data and isinstance(data, dict):
data = json.dumps(data)
content_type = 'application/json'
response = self._make_request(
method=method, url=url, data=data, content_type=content_type,
headers=headers, target_object=_target_object)
if not 200 <= response.status_code < 300:
raise exceptions.from_http_response(response)
if expect_json and response.content:
return response.json()
else:
return response.content
