def validate_bq_table_name(table_name):
"""Validates provided BigQuery table name for maximum character limit and
permitted characters."""
max_characters = 1024
patterns = '^[a-zA-Z0-9_]*$'
error_msg = "Invalid table name {}. Table name must be alphanumeric" \
"(plus underscores) and must be at most 1024 characters" \
" long.".format(table_name)
if len(table_name) > max_characters:
raise exceptions.BadRequest(error_msg)
if not re.search(patterns, table_name):
raise exceptions.BadRequest(error_msg)
def test_check_dataset_google_api_call_error(mock_configured_client):
with patch.object(google.cloud.bigquery.client, 'Client', autospec=True) as mock_client:
mock_client.get_dataset.side_effect = exceptions.BadRequest('bad request')
mock_configured_client.client = mock_client
result = mock_configured_client.check_dataset('test_data_set')
assert result is False
assert mock_configured_client.errors
def test_exists(self):
from google.cloud.bigtable_admin_v2.services.bigtable_instance_admin import (
BigtableInstanceAdminClient,
)
from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2
from google.api_core import exceptions
instance_api = mock.create_autospec(BigtableInstanceAdminClient)
credentials = _make_credentials()
client = self._make_client(
project=self.PROJECT, credentials=credentials, admin=True
)
instance = client.instance(self.INSTANCE_ID)
# Create response_pb
response_pb = data_v2_pb2.AppProfile(name=self.APP_PROFILE_NAME)
client._instance_admin_client = instance_api
# Patch the stub used by the API method.
client._instance_admin_client = instance_api
instance_stub = client._instance_admin_client
instance_stub.get_app_profile.side_effect = [
response_pb,
exceptions.NotFound("testing"),
exceptions.BadRequest("testing"),
]
# Perform the method and check the result.
non_existing_app_profile_id = "other-app-profile-id"
app_profile = self._make_one(self.APP_PROFILE_ID, instance)
alt_app_profile = self._make_one(non_existing_app_profile_id, instance)
self.assertTrue(app_profile.exists())
self.assertFalse(alt_app_profile.exists())
with self.assertRaises(exceptions.BadRequest):
alt_app_profile.exists()
def test_query_api_error(mock_configured_client):
with patch.object(google.cloud.bigquery.client, 'Client', autospec=True) as mock_client:
mock_client.query.side_effect = exceptions.BadRequest('bad request')
mock_configured_client.client = mock_client
result = mock_configured_client.query(" ")
assert result is False
assert mock_configured_client.errors
def test_bigquery_magic_w_table_id_invalid():
ip = IPython.get_ipython()
ip.extension_manager.load_extension("google.cloud.bigquery")
magics.context._project = None
credentials_mock = mock.create_autospec(
google.auth.credentials.Credentials, instance=True
)
default_patch = mock.patch(
"google.auth.default", return_value=(credentials_mock, "general-project")
)
list_rows_patch = mock.patch(
"google.cloud.bigquery.magics.bigquery.Client.list_rows",
autospec=True,
side_effect=exceptions.BadRequest("Not a valid table ID"),
)
table_id = "not-a-real-table"
with list_rows_patch, default_patch, io.capture_output() as captured_io:
ip.run_cell_magic("bigquery", "df", table_id)
output = captured_io.stderr
assert "Could not save output to variable" in output
assert "400 Not a valid table ID" in output
assert "Traceback (most recent call last)" not in output
def test_get_reservation_exception(client_constructor):
grpc_client = setup_mock_(client_constructor)
grpc_client.get_quantum_reservation.side_effect = exceptions.BadRequest('boom')
client = EngineClient()
with pytest.raises(EngineException, match='boom'):
client.get_reservation('proj', 'processor0', 'goog')
def test_create_data_set_exception(mock_configured_client):
mock_configured_client.set_data_set_ref('test_data_set')
with patch.object(google.cloud.bigquery.client, 'Client', autospec=True) as mock_client:
mock_client.create_dataset.side_effect = exceptions.BadRequest('bad request')
mock_configured_client.client = mock_client
result = mock_configured_client.create_data_set()
assert result is False
assert mock_configured_client.errors
def test_get_schema_api_error(mock_configured_client):
mock_configured_client.set_data_set_ref('test_data_set')
mock_configured_client.set_table_ref('test_table')
with patch.object(google.cloud.bigquery.client, 'Client', autospec=True) as mock_client:
mock_client.get_table.side_effect = exceptions.BadRequest('bad request')
mock_configured_client.client = mock_client
result = mock_configured_client.get_schema()
print(mock_configured_client.errors)
assert type(result) is list
assert len(result) == 0
assert mock_configured_client.errors
def test_create_table_api_error(mock_configured_client):
mock_configured_client.set_data_set_ref('test_data_set')
mock_configured_client.set_table_ref('test_table')
schema = [
bigquery.SchemaField("column1", "STRING", mode="REQUIRED"),
bigquery.SchemaField("column2", "INTEGER", mode="REQUIRED"),
]
with patch.object(google.cloud.bigquery.client, 'Client', autospec=True) as mock_client:
mock_client.create_table.side_effect = exceptions.BadRequest('bad request')
mock_configured_client.client = mock_client
result = mock_configured_client.create_table(schema)
assert result is False
assert mock_configured_client.errors
def test_instance_exists_w_error():
from google.api_core import exceptions
credentials = _make_credentials()
client = _make_client(project=PROJECT, credentials=credentials, admin=True)
api = client._instance_admin_client = _make_instance_admin_api()
api.instance_path.return_value = INSTANCE_NAME
api.get_instance.side_effect = exceptions.BadRequest("testing")
instance = _make_instance(INSTANCE_ID, client)
with pytest.raises(exceptions.BadRequest):
instance.exists()
api.get_instance.assert_called_once_with(request={"name": INSTANCE_NAME})
def test_export_table_to_storage_api_error(mock_configured_client):
mock_configured_client.set_data_set_ref('dataset')
with patch.object(google.cloud.bigquery.client, 'Client', autospec=True) as mock_client:
mock_extract_job = Mock(spec=google.cloud.bigquery.job.ExtractJob)
mock_extract_job.done.side_effect = exceptions.BadRequest('bad request')
mock_client.extract_table.return_value = mock_extract_job
mock_configured_client.client = mock_client
mock_table = Mock(spec=google.cloud.bigquery.table.TableReference)
result = mock_configured_client.export_table_to_storage(mock_table, 'destination')
assert result is False
assert mock_configured_client.errors
def test_cluster_exists_w_error():
from google.cloud.bigtable.instance import Instance
from google.api_core import exceptions
credentials = _make_credentials()
client = _make_client(project=PROJECT, credentials=credentials, admin=True)
instance = Instance(INSTANCE_ID, client)
api = client._instance_admin_client = _make_instance_admin_client()
api.get_cluster.side_effect = exceptions.BadRequest("testing")
cluster = _make_cluster(CLUSTER_ID, instance)
with pytest.raises(exceptions.BadRequest):
cluster.exists()
api.get_cluster.assert_called_once_with(request={"name": cluster.name})
def test_get_event_consumer_raises(consumer_config, auth_client,
subscriber_client, caplog, emulator,
exp_topic, exp_sub, metrics):
"""Raise when any other error occured with creating a subscription."""
success_chnl, error_chnl = asyncio.Queue(), asyncio.Queue()
exp = google_exceptions.BadRequest('foo')
sub_inst = subscriber_client.return_value
sub_inst.create_subscription.side_effect = [exp]
with pytest.raises(exceptions.GCPGordonError) as e:
service.get_event_consumer(consumer_config, success_chnl, error_chnl,
metrics)
sub_inst.create_subscription.assert_called_once_with(exp_sub, exp_topic)
e.match(f'Error trying to create subscription "{exp_sub}"')
assert 3 == len(caplog.records)
def test_bigquery_magic_dryrun_option_variable_error_message():
ip = IPython.get_ipython()
ip.extension_manager.load_extension("google.cloud.bigquery")
magics.context.credentials = mock.create_autospec(
google.auth.credentials.Credentials, instance=True)
run_query_patch = mock.patch(
"google.cloud.bigquery.magics._run_query",
autospec=True,
side_effect=exceptions.BadRequest("Syntax error in SQL query"),
)
sql = "SELECT SELECT 17 AS num"
assert "q_job" not in ip.user_ns
with run_query_patch, io.capture_output() as captured:
ip.run_cell_magic("bigquery", "q_job --dry_run", sql)
full_text = captured.stderr
assert "Could not save output to variable 'q_job'." in full_text
def test_exists(self):
from google.cloud.bigtable_admin_v2.gapic import (
bigtable_instance_admin_client)
from google.cloud.bigtable_admin_v2.proto import (
instance_pb2 as data_v2_pb2)
from google.cloud.bigtable.instance import Instance
from google.api_core import exceptions
instance_api = (
bigtable_instance_admin_client.BigtableInstanceAdminClient(
mock.Mock()))
credentials = _make_credentials()
client = self._make_client(project=self.PROJECT,
credentials=credentials, admin=True)
instance = Instance(self.INSTANCE_ID, client)
# Create response_pb
cluster_name = client.instance_admin_client.cluster_path(
self.PROJECT, self.INSTANCE_ID, self.CLUSTER_ID)
response_pb = data_v2_pb2.Cluster(name=cluster_name)
# Patch the stub used by the API method.
client._instance_admin_client = instance_api
instance_admin_client = client._instance_admin_client
instance_stub = instance_admin_client.bigtable_instance_admin_stub
instance_stub.GetCluster.side_effect = [
response_pb,
exceptions.NotFound('testing'),
exceptions.BadRequest('testing')
]
# Perform the method and check the result.
non_existing_cluster_id = 'cluster-id-2'
alt_cluster_1 = self._make_one(self.CLUSTER_ID, instance)
alt_cluster_2 = self._make_one(non_existing_cluster_id, instance)
self.assertTrue(alt_cluster_1.exists())
self.assertFalse(alt_cluster_2.exists())
with self.assertRaises(exceptions.BadRequest):
alt_cluster_1.exists()
def test_exists(self):
from google.cloud.bigtable_admin_v2.services.bigtable_instance_admin import (
BigtableInstanceAdminClient,
)
from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2
from google.cloud.bigtable.instance import Instance
from google.api_core import exceptions
instance_api = mock.create_autospec(BigtableInstanceAdminClient)
credentials = _make_credentials()
client = self._make_client(
project=self.PROJECT, credentials=credentials, admin=True
)
instance = Instance(self.INSTANCE_ID, client)
# Create response_pb
cluster_name = client.instance_admin_client.cluster_path(
self.PROJECT, self.INSTANCE_ID, self.CLUSTER_ID
)
response_pb = data_v2_pb2.Cluster(name=cluster_name)
# Patch the stub used by the API method.
client._instance_admin_client = instance_api
bigtable_instance_stub = client._instance_admin_client
bigtable_instance_stub.get_cluster.side_effect = [
response_pb,
exceptions.NotFound("testing"),
exceptions.BadRequest("testing"),
]
# Perform the method and check the result.
non_existing_cluster_id = "cluster-id-2"
alt_cluster_1 = self._make_one(self.CLUSTER_ID, instance)
alt_cluster_2 = self._make_one(non_existing_cluster_id, instance)
self.assertTrue(alt_cluster_1.exists())
self.assertFalse(alt_cluster_2.exists())
with self.assertRaises(exceptions.BadRequest):
alt_cluster_1.exists()
def test_exists(self):
from google.cloud.bigtable_admin_v2.gapic import (
bigtable_instance_admin_client)
from google.cloud.bigtable_admin_v2.proto import (instance_pb2 as
data_v2_pb2)
from google.api_core import exceptions
instance_api = (
bigtable_instance_admin_client.BigtableInstanceAdminClient(
mock.Mock()))
credentials = _make_credentials()
client = self._make_client(project=self.PROJECT,
credentials=credentials,
admin=True)
instance = client.instance(self.INSTANCE_ID)
# Create response_pb
response_pb = data_v2_pb2.AppProfile(name=self.APP_PROFILE_NAME)
client._instance_admin_client = instance_api
# Patch the stub used by the API method.
client._instance_admin_client = instance_api
instance_stub = (
client._instance_admin_client.bigtable_instance_admin_stub)
instance_stub.GetCluster.side_effect = [
response_pb,
exceptions.NotFound('testing'),
exceptions.BadRequest('testing'),
]
# Perform the method and check the result.
non_existing_app_profile_id = 'other-app-profile-id'
app_profile = self._make_one(self.APP_PROFILE_ID, instance)
alt_app_profile = self._make_one(non_existing_app_profile_id, instance)
self.assertTrue(app_profile.exists())
self.assertFalse(alt_app_profile.exists())
with self.assertRaises(exceptions.BadRequest):
alt_app_profile.exists()
def test_bigquery_magic_omits_tracebacks_from_error_message():
ip = IPython.get_ipython()
ip.extension_manager.load_extension("google.cloud.bigquery")
credentials_mock = mock.create_autospec(
google.auth.credentials.Credentials, instance=True)
default_patch = mock.patch("google.auth.default",
return_value=(credentials_mock,
"general-project"))
run_query_patch = mock.patch(
"google.cloud.bigquery.magics._run_query",
autospec=True,
side_effect=exceptions.BadRequest("Syntax error in SQL query"),
)
with run_query_patch, default_patch, io.capture_output() as captured_io:
ip.run_cell_magic("bigquery", "", "SELECT foo FROM WHERE LIMIT bar")
output = captured_io.stderr
assert "400 Syntax error in SQL query" in output
assert "Traceback (most recent call last)" not in output
assert "Syntax error" not in captured_io.stdout
def test_exists(self):
from google.cloud.bigtable_admin_v2.gapic import bigtable_instance_admin_client
from google.cloud.bigtable_admin_v2.proto import instance_pb2 as data_v2_pb2
from google.api_core import exceptions
api = bigtable_instance_admin_client.BigtableInstanceAdminClient(
mock.Mock())
credentials = _make_credentials()
client = self._make_client(project=self.PROJECT,
credentials=credentials,
admin=True)
# Create response_pb
instance_name = client.instance_admin_client.instance_path(
self.PROJECT, self.INSTANCE_ID)
response_pb = data_v2_pb2.Instance(name=instance_name)
# Patch the stub used by the API method.
client._instance_admin_client = api
instance_admin_client = client._instance_admin_client
instance_stub = instance_admin_client.transport
instance_stub.get_instance.side_effect = [
response_pb,
exceptions.NotFound("testing"),
exceptions.BadRequest("testing"),
]
# Perform the method and check the result.
non_existing_instance_id = "instance-id-2"
alt_instance_1 = self._make_one(self.INSTANCE_ID, client)
alt_instance_2 = self._make_one(non_existing_instance_id, client)
self.assertTrue(alt_instance_1.exists())
self.assertFalse(alt_instance_2.exists())
with self.assertRaises(exceptions.BadRequest):
alt_instance_2.exists()
def test_fetchall_w_bqstorage_client_fetch_error_fallback_on_client(self):
from google.cloud.bigquery import dbapi
from google.cloud.bigquery import table
# use unordered data to also test any non-determenistic key order in dicts
row_data = [
table.Row([1.4, 1.1, 1.3, 1.2], {
"bar": 3,
"baz": 2,
"foo": 1,
"quux": 0
}),
table.Row([2.4, 2.1, 2.3, 2.2], {
"bar": 3,
"baz": 2,
"foo": 1,
"quux": 0
}),
]
bqstorage_streamed_rows = [
{
"bar": 1.2,
"foo": 1.1,
"quux": 1.4,
"baz": 1.3
},
{
"bar": 2.2,
"foo": 2.1,
"quux": 2.4,
"baz": 2.3
},
]
mock_client = self._mock_client(rows=row_data)
mock_bqstorage_client = self._mock_bqstorage_client(
stream_count=1,
rows=bqstorage_streamed_rows,
)
request_error = exceptions.BadRequest("BQ storage what??")
mock_bqstorage_client.create_read_session.side_effect = request_error
connection = dbapi.connect(
client=mock_client,
bqstorage_client=mock_bqstorage_client,
)
cursor = connection.cursor()
cursor.execute("SELECT foo, bar FROM some_table")
logger_patcher = mock.patch(
"google.cloud.bigquery.dbapi.cursor._LOGGER")
with logger_patcher as mock_logger:
rows = cursor.fetchall()
# both client were used
mock_bqstorage_client.create_read_session.assert_called()
mock_client.list_rows.assert_called()
# fallback to default API should have been logged
relevant_calls = [
call for call in mock_logger.debug.call_args_list
if call.args and "tabledata.list API" in call.args[0]
]
self.assertTrue(relevant_calls)
# check the data returned
field_value = op.itemgetter(1)
sorted_row_data = [
sorted(row.items(), key=field_value) for row in rows
]
expected_row_data = [
[("foo", 1.1), ("bar", 1.2), ("baz", 1.3), ("quux", 1.4)],
[("foo", 2.1), ("bar", 2.2), ("baz", 2.3), ("quux", 2.4)],
]
self.assertEqual(sorted_row_data, expected_row_data)
from ddt import data, ddt, unpack
from google.api_core import exceptions, retry
from google.cloud import storage
from google.cloud.bigquery import ExtractJob
from google.cloud.bigquery.job import QueryJobConfig
from google.cloud.bigquery.schema import SchemaField
from mock import MagicMock, PropertyMock, patch
from verily.bigquery_wrapper import bq, bq_shared_tests, bq_test_case
from verily.bigquery_wrapper.bq_base import (DEFAULT_RETRY_FOR_API_CALLS, is_job_done,
validate_query_job)
# Arguments to pass to retry-related tests
EXCEPTION_RETRY_TEST_ARGS = (
dict(
exc=exceptions.BadRequest('Extra comma before FROM clause.'),
should_retry=False
),
dict(
exc=exceptions.BadRequest(
'The job encountered an internal error during execution. (Transient error.)'),
should_retry=True
),
dict(
exc=exceptions.InternalServerError('Transient error.'),
should_retry=True
),
dict(
exc=exceptions.TooManyRequests('Transient error.'),
should_retry=True
),