def test_query_output_json_format(self, mock_client,
mock_kfp_context, mock_dump_json, mock_display):
mock_kfp_context().__enter__().context_id.return_value = 'ctx1'
mock_client().get_job.side_effect = exceptions.NotFound('not found')
mock_dataset = bigquery.DatasetReference('project-1', 'dataset-1')
mock_client().dataset.return_value = mock_dataset
mock_client().get_dataset.side_effect = exceptions.NotFound('not found')
mock_response = {
'configuration': {
'query': {
'query': 'SELECT * FROM table_1'
}
}
}
mock_client().query.return_value.to_api_repr.return_value = mock_response
result = query('SELECT * FROM table_1', 'project-1', 'dataset-1',
output_gcs_path='gs://output/path',
output_destination_format="NEWLINE_DELIMITED_JSON")
self.assertEqual(mock_response, result)
mock_client().create_dataset.assert_called()
extract = mock_client().extract_table.call_args_list[0]
self.assertEqual(extract[0], (mock_dataset.table('query_ctx1'), 'gs://output/path',))
self.assertEqual(extract[1]["job_config"].destination_format, "NEWLINE_DELIMITED_JSON",)
def test_query_succeed(self, mock_client,
mock_kfp_context, mock_dump_json, mock_display):
mock_kfp_context().__enter__().context_id.return_value = 'ctx1'
mock_client().get_job.side_effect = exceptions.NotFound('not found')
mock_dataset = bigquery.DatasetReference('project-1', 'dataset-1')
mock_client().dataset.return_value = mock_dataset
mock_client().get_dataset.side_effect = exceptions.NotFound('not found')
mock_response = {
'configuration': {
'query': {
'query': 'SELECT * FROM table_1'
}
}
}
mock_client().query.return_value.to_api_repr.return_value = mock_response
result = query('SELECT * FROM table_1', 'project-1', 'dataset-1',
output_gcs_path='gs://output/path')
self.assertEqual(mock_response, result)
mock_client().create_dataset.assert_called()
expected_job_config = bigquery.QueryJobConfig()
expected_job_config.create_disposition = bigquery.job.CreateDisposition.CREATE_IF_NEEDED
expected_job_config.write_disposition = bigquery.job.WriteDisposition.WRITE_TRUNCATE
expected_job_config.destination = mock_dataset.table('query_ctx1')
mock_client().query.assert_called_with('SELECT * FROM table_1',mock.ANY,
job_id = 'query_ctx1')
actual_job_config = mock_client().query.call_args_list[0][0][1]
self.assertDictEqual(
expected_job_config.to_api_repr(),
actual_job_config.to_api_repr()
)
extract = mock_client().extract_table.call_args_list[0]
self.assertEqual(extract[0], (mock_dataset.table('query_ctx1'), 'gs://output/path',))
self.assertEqual(extract[1]["job_config"].destination_format, "CSV",)
def test_result_w_retry_wo_state(self):
begun_job_resource = _make_job_resource(job_id=self.JOB_ID,
project_id=self.PROJECT,
location="EU",
started=True)
done_job_resource = _make_job_resource(
job_id=self.JOB_ID,
project_id=self.PROJECT,
location="EU",
started=True,
ended=True,
)
conn = _make_connection(
exceptions.NotFound("not normally retriable"),
begun_job_resource,
exceptions.NotFound("not normally retriable"),
done_job_resource,
)
client = _make_client(project=self.PROJECT, connection=conn)
job = self._make_one(
self._job_reference(self.JOB_ID, self.PROJECT, "EU"), client)
custom_predicate = mock.Mock()
custom_predicate.return_value = True
custom_retry = google.api_core.retry.Retry(
predicate=custom_predicate,
initial=0.001,
maximum=0.001,
deadline=0.1,
)
self.assertIs(job.result(retry=custom_retry), job)
begin_call = mock.call(
method="POST",
path=f"/projects/{self.PROJECT}/jobs",
data={
"jobReference": {
"jobId": self.JOB_ID,
"projectId": self.PROJECT,
"location": "EU",
}
},
timeout=None,
)
reload_call = mock.call(
method="GET",
path=f"/projects/{self.PROJECT}/jobs/{self.JOB_ID}",
query_params={"location": "EU"},
timeout=None,
)
conn.api_request.assert_has_calls(
[begin_call, begin_call, reload_call, reload_call])
def test_exists(self):
from google.cloud.bigtable_admin_v2.services.bigtable_instance_admin import (
BigtableInstanceAdminClient,
)
from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2
from google.api_core import exceptions
instance_api = mock.create_autospec(BigtableInstanceAdminClient)
credentials = _make_credentials()
client = self._make_client(
project=self.PROJECT, credentials=credentials, admin=True
)
instance = client.instance(self.INSTANCE_ID)
# Create response_pb
response_pb = data_v2_pb2.AppProfile(name=self.APP_PROFILE_NAME)
client._instance_admin_client = instance_api
# Patch the stub used by the API method.
client._instance_admin_client = instance_api
instance_stub = client._instance_admin_client
instance_stub.get_app_profile.side_effect = [
response_pb,
exceptions.NotFound("testing"),
exceptions.BadRequest("testing"),
]
# Perform the method and check the result.
non_existing_app_profile_id = "other-app-profile-id"
app_profile = self._make_one(self.APP_PROFILE_ID, instance)
alt_app_profile = self._make_one(non_existing_app_profile_id, instance)
self.assertTrue(app_profile.exists())
self.assertFalse(alt_app_profile.exists())
with self.assertRaises(exceptions.BadRequest):
alt_app_profile.exists()
def find_required(
cls: Type[T],
*,
transaction: Optional[spanner_transaction.Transaction] = None,
**keys: Any,
) -> T:
"""Retrieves an object from Spanner based on the provided key.
Args:
transaction: The existing transaction to use, or None to start a new
transaction
**keys: The keys provided are the complete set of primary keys for this
table and the corresponding values make up the unique identifier of the
object being retrieved
Returns:
The requested object.
Raises:
exceptions.NotFound: The object wasn't found.
"""
result = cls.find(**keys, transaction=transaction)
if result is None:
raise exceptions.NotFound(
f'{cls.__qualname__} has no object with primary key {keys}')
return result
def test_fetch_file_raises_an_exception_when_bucket_is_not_found(self):
self.mock_client.return_value.get_bucket.side_effect = exceptions.NotFound(
'Cloud Storage Bucket not found.')
with self.assertRaises(exceptions.NotFound):
self.cloud_storage_obj.fetch_file(bucket_name=self.bucket_name,
file_name=self.mock_blob_name)
def test_api_doesnt_retry_not_found_errors(client_constructor):
grpc_client = setup_mock_(client_constructor)
grpc_client.get_quantum_program.side_effect = exceptions.NotFound('not found')
client = EngineClient()
with pytest.raises(EngineException, match='not found'):
client.get_program('proj', 'prog', False)
assert grpc_client.get_quantum_program.call_count == 1
def test_missing_latest_calibration(client_constructor):
client = mock.Mock()
client_constructor.return_value = client
client.get_quantum_calibration.side_effect = exceptions.NotFound('')
calibration = cg.Engine(project_id='myproject').get_latest_calibration('x')
assert client.get_quantum_calibration.call_args[0][
0] == 'projects/myproject/processors/x/calibrations/current'
assert not calibration
class TestSubscriber:
@patch.object(SubscriberClient, "create_subscription")
def test_creates_subscription_with_default_ack_deadline_when_none_provided(
self, _mocked_client, project_id, subscriber):
expected_subscription = f"projects/{project_id}/subscriptions/" f"test-topic"
expected_topic = f"projects/{project_id}/topics/" f"{project_id}-test-topic"
subscriber.create_subscription("test-topic",
f"{project_id}-test-topic")
_mocked_client.assert_called_once_with(
ack_deadline_seconds=60,
name=expected_subscription,
topic=expected_topic,
)
assert subscriber._gc_project_id == "rele-test"
@patch.object(SubscriberClient, "create_subscription")
def test_creates_subscription_with_custom_ack_deadline_when_provided(
self, _mocked_client, project_id, subscriber):
expected_subscription = f"projects/{project_id}/subscriptions/" f"test-topic"
expected_topic = f"projects/{project_id}/topics/" f"{project_id}-test-topic"
subscriber._ack_deadline = 100
subscriber.create_subscription(subscription="test-topic",
topic=f"{project_id}-test-topic")
_mocked_client.assert_called_once_with(
ack_deadline_seconds=100,
name=expected_subscription,
topic=expected_topic,
)
@patch.object(
SubscriberClient,
"create_subscription",
side_effect=exceptions.AlreadyExists("Subscription already exists"),
)
def test_does_not_raise_when_subscription_already_exists(
self, _mocked_client, project_id, subscriber):
subscriber.create_subscription(subscription="test-topic",
topic=f"{project_id}-test-topic")
_mocked_client.assert_called()
@patch.object(
SubscriberClient,
"create_subscription",
side_effect=exceptions.NotFound("Subscription topic does not exist"),
)
def test_logs_error_when_subscription_topic_does_not_exist(
self, _mocked_client, project_id, subscriber, caplog):
subscriber.create_subscription(subscription="test-topic",
topic=f"{project_id}-test-topic")
_mocked_client.assert_called()
log = caplog.records[0]
assert log.message == "Cannot subscribe to a topic that does not exist."
assert log.levelname == "ERROR"
def test_get_bucket_on_exception_should_not_leak_error(self, get_bucket):
get_bucket.side_effect = exceptions.NotFound(
'error on retrieving bucket')
storage_client = StorageClientHelper('test_project')
bucket = storage_client.get_bucket('my_bucket')
self.assertIsNone(bucket)
get_bucket.assert_called_once()
def test_api_doesnt_retry_not_found_errors(client_constructor):
client = mock.Mock()
client_constructor.return_value = client
client.get_quantum_program.side_effect = exceptions.NotFound('not found')
engine = cg.Engine(project_id='project-id')
with pytest.raises(cg.engine.engine.EngineException, match='not found'):
engine.get_program('foo')
assert client.get_quantum_program.call_count == 1
def test_delete_metrics_on_exception_should_not_raise_error(self):
monitoring_client = self.__monitoring_client
monitoring_client.delete_metric_descriptor.side_effect = \
exceptions.NotFound('Metric Descriptor does not exist')
self.__monitoring_facade.delete_metrics()
self.assertEqual(3,
monitoring_client.delete_metric_descriptor.call_count)
def test_delete_table_fails_not_found(self, mock_client):
mock_client.return_value.dataset.return_value.table.return_value = (
'table_ref')
mock_client.return_value.delete_table.side_effect = gexc.NotFound(
'test')
with self.assertRaisesRegex(Exception, r'does not exist:.*table_ref'):
utils.delete_bq_table('unused_project', 'unused_dataset',
'unused_table')
def test_create_publisher_topic_not_found(mock_publisher):
client = mock_publisher.return_value
client.get_topic.side_effect = gapi_exceptions.NotFound("foo")
with pytest.raises(SystemExit):
publish._create_publisher("a-topic")
mock_publisher.assert_called_once_with()
client.get_topic.assert_called_once_with("a-topic")
def get_metadata_store_mock_raise_not_found_exception():
with patch.object(
MetadataServiceClient, "get_metadata_store"
) as get_metadata_store_mock:
get_metadata_store_mock.side_effect = [
exceptions.NotFound("Test store not found."),
GapicMetadataStore(name=_TEST_METADATASTORE,),
]
yield get_metadata_store_mock
def test_raise_on_expected_error(self, api_method, mock_spanner_api):
mock_api = MockSpannerApi()
mock_method = mock.Mock()
mock_method.side_effect = exceptions.NotFound('Database not found')
with self.assertRaises(exceptions.NotFound):
getattr(mock_api, api_method)(mock_method)
mock_method.assert_called()
def test_get_reservation_not_found(client_constructor):
grpc_client = setup_mock_(client_constructor)
name = 'projects/proj/processors/processor0/reservations/papar-party-44'
grpc_client.get_quantum_reservation.side_effect = exceptions.NotFound('not found')
client = EngineClient()
assert client.get_reservation('proj', 'processor0', 'papar-party-44') is None
grpc_client.get_quantum_reservation.assert_called_with(
quantum.GetQuantumReservationRequest(name=name)
)
def test_get_current_calibration_does_not_exist(client_constructor):
grpc_client = setup_mock_(client_constructor)
grpc_client.get_quantum_calibration.side_effect = exceptions.NotFound(
'not found')
client = EngineClient()
assert client.get_current_calibration('proj', 'processor0') is None
assert grpc_client.get_quantum_calibration.call_args[0] == (
'projects/proj/processors/processor0/calibrations/current', )
def test_get_or_create_bucket_method_creates_bucket_if_it_does_not_exist(
self):
self.mock_client.return_value.get_bucket.side_effect = exceptions.NotFound(
'Bucket Not Found')
self.mock_client.return_value.create_bucket.return_value = self.mock_bucket
self.cloud_storage_obj._get_or_create_bucket(self.bucket_name)
self.mock_client.return_value.create_bucket.assert_called_once_with(
self.bucket_name)
def test_get_current_calibration_does_not_exist(client_constructor):
grpc_client = setup_mock_(client_constructor)
grpc_client.get_quantum_calibration.side_effect = exceptions.NotFound('not found')
client = EngineClient()
assert client.get_current_calibration('proj', 'processor0') is None
grpc_client.get_quantum_calibration.assert_called_with(
quantum.GetQuantumCalibrationRequest(
name='projects/proj/processors/processor0/calibrations/current'
)
)
def test__create_dataset_if_necessary_not_exist():
project = "project_id"
dataset_id = "dataset_id"
client_patch = mock.patch(
"google.cloud.bigquery.magics.bigquery.Client", autospec=True
)
with client_patch as client_mock:
client = client_mock()
client.location = "us"
client.project = project
client.get_dataset.side_effect = exceptions.NotFound("dataset not found")
magics._create_dataset_if_necessary(client, dataset_id)
client.create_dataset.assert_called_once()
def test_get_reservation_not_found(client_constructor):
grpc_client = setup_mock_(client_constructor)
name = 'projects/proj/processors/processor0/reservations/papar-party-44'
grpc_client.get_quantum_reservation.side_effect = exceptions.NotFound(
'not found')
client = EngineClient()
assert (client.get_reservation('proj', 'processor0',
'papar-party-44') == None)
kwargs = grpc_client.get_quantum_reservation.call_args[1]
assert kwargs == {
'name': name,
}
def test_reconnect_on_expected_error(self, api_method, mock_spanner_api):
mock_api = MockSpannerApi()
mock_method = mock.Mock()
mock_method.side_effect = [
exceptions.NotFound('Session not found'),
'Anything other than an exception'
]
mock_connect = mock_spanner_api.return_value.connect
getattr(mock_api, api_method)(mock_method)
mock_connect.assert_called_once()
mock_method.assert_called()
def test_wrap_method_with_overriding_retry_and_timeout(unusued_sleep):
method = mock.Mock(spec=['__call__'],
side_effect=[exceptions.NotFound(None), 42])
default_retry = retry.Retry()
default_timeout = timeout.ConstantTimeout(60)
wrapped_method = google.api_core.gapic_v1.method.wrap_method(
method, default_retry, default_timeout)
result = wrapped_method(retry=retry.Retry(
retry.if_exception_type(exceptions.NotFound)),
timeout=timeout.ConstantTimeout(22))
assert result == 42
assert method.call_count == 2
method.assert_called_with(timeout=22, metadata=mock.ANY)
def test_ensure_bucket_exists_custom_bucket_name(self):
mock_bucket = mock.Mock()
gcs_client = self.gcs_client(
bucket_name="my-bucket",
client_attrs={
"get_bucket.side_effect": exceptions.NotFound("err"),
"bucket.return_value": mock_bucket,
},
)
returned_bucket_name = gcs_client.ensure_bucket_exists(
"my-project", "us-central1")
gcs_client.client.get_bucket.assert_called_with("my-bucket")
gcs_client.client.bucket.assert_called_with("my-bucket")
mock_bucket.create.assert_called_with(project="my-project",
location="us-central1")
assert returned_bucket_name == "my-bucket"
def test_instance_exists_miss():
from google.api_core import exceptions
credentials = _make_credentials()
client = _make_client(project=PROJECT, credentials=credentials, admin=True)
api = client._instance_admin_client = _make_instance_admin_api()
api.instance_path.return_value = INSTANCE_NAME
api.get_instance.side_effect = exceptions.NotFound("testing")
non_existing_instance_id = "instance-id-2"
instance = _make_instance(non_existing_instance_id, client)
assert not instance.exists()
api.get_instance.assert_called_once_with(request={"name": INSTANCE_NAME})
def test_get_event_consumer_raises_topic(consumer_config, auth_client,
subscriber_client, caplog, emulator,
exp_topic, exp_sub, metrics):
"""Raise when there is no topic to subscribe to."""
success_chnl, error_chnl = asyncio.Queue(), asyncio.Queue()
exp = google_exceptions.NotFound('foo')
sub_inst = subscriber_client.return_value
sub_inst.create_subscription.side_effect = [exp]
with pytest.raises(exceptions.GCPGordonError) as e:
service.get_event_consumer(consumer_config, success_chnl, error_chnl,
metrics)
sub_inst.create_subscription.assert_called_once_with(exp_sub, exp_topic)
e.match(f'Topic "{exp_topic}" does not exist.')
assert 3 == len(caplog.records)
def test_cluster_exists_miss():
from google.cloud.bigtable.instance import Instance
from google.api_core import exceptions
credentials = _make_credentials()
client = _make_client(project=PROJECT, credentials=credentials, admin=True)
instance = Instance(INSTANCE_ID, client)
api = client._instance_admin_client = _make_instance_admin_client()
api.get_cluster.side_effect = exceptions.NotFound("testing")
non_existing_cluster_id = "nonesuch-cluster-2"
cluster = _make_cluster(non_existing_cluster_id, instance)
assert not cluster.exists()
api.get_cluster.assert_called_once_with(request={"name": cluster.name})
async def test_wrap_method_with_overriding_retry_and_timeout(unused_sleep):
fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42)
method = mock.Mock(
spec=aio.UnaryUnaryMultiCallable,
side_effect=[exceptions.NotFound(None), fake_call],
)
default_retry = retry_async.AsyncRetry()
default_timeout = timeout.ConstantTimeout(60)
wrapped_method = gapic_v1.method_async.wrap_method(method, default_retry,
default_timeout)
result = await wrapped_method(
retry=retry_async.AsyncRetry(
retry_async.if_exception_type(exceptions.NotFound)),
timeout=timeout.ConstantTimeout(22),
)
assert result == 42
assert method.call_count == 2
method.assert_called_with(timeout=22, metadata=mock.ANY)
def test_query_dump_locally(self, mock_client, mock_kfp_context,
mock_dump_json, mock_display):
mock_kfp_context().__enter__().context_id.return_value = 'ctx1'
mock_client().get_job.side_effect = exceptions.NotFound('not found')
mock_response = {
'configuration': {
'query': {
'query': 'SELECT * FROM table_1'
}
}
}
mock_client(
).query.return_value.to_api_repr.return_value = mock_response
result = query('SELECT * FROM table_1', 'project-1')
self.assertEqual(mock_response, result)
mock_client().create_dataset.assert_not_called()
mock_client().query.assert_called()
mock_client().extract_table.assert_not_called()
self.assertEqual(3, mock_dump_json.call_count)
