def api_request(self, **kw):
from google.cloud.exceptions import NotFound
self._requested.append(kw)
method = kw.get('method')
path = kw.get('path', '')
if method == 'DELETE' and self._is_bucket_path(path):
self._deleted_buckets.append(kw)
if self._delete_bucket:
return
else:
raise NotFound('miss')
try:
response, self._responses = self._responses[0], self._responses[1:]
except:
raise NotFound('miss')
else:
return response
def get_bucket(self, bucket_or_name):
bucket = self._bucket_arg_to_bucket(bucket_or_name)
# TODO: Use bucket.reload(client=self) when MockBucket class is implemented
if bucket.name in self.backend.buckets.keys():
return self.backend.buckets[bucket.name]
else:
raise NotFound(
"404 GET https://storage.googleapis.com/storage/v1/b/{}?projection=noAcl"
.format(bucket.name))
def test_exists(self, mock_bigquery_client):
"""Test `exists` method invocation."""
mock_bigquery_client.return_value.get_table.side_effect = [
NotFound("NotFound"),
"exists",
]
data_set = GBQTableDataSet(DATASET, TABLE_NAME)
assert not data_set.exists()
assert data_set.exists()
def api_request(self, **kw):
from google.cloud.exceptions import NotFound
self._requested.append(kw)
try:
response, self._responses = self._responses[0], self._responses[1:]
except:
raise NotFound('miss')
else:
return response
def test_delete_nonexisting_topic_failifnotexists(self, mock_service):
mock_service.return_value.delete_topic.side_effect = NotFound(
'Topic does not exists: %s' % EXPANDED_TOPIC)
with self.assertRaises(PubSubException) as e:
self.pubsub_hook.delete_topic(project_id=TEST_PROJECT,
topic=TEST_TOPIC,
fail_if_not_exists=True)
self.assertEqual(str(e.exception),
'Topic does not exist: %s' % EXPANDED_TOPIC)
def transfer(source, destination):
'''Transfers from `source` to `destination`.
'''
blob = get_blob_from_url(source)
if blob is None:
raise NotFound(u'File not found at source: {}'.format(source))
# Retrieve destination bucket and perform rename
bucket = get_bucket_from_url(destination)
return bucket.rename_blob(blob, to_relative_path(destination))
def test_bigquery_matcher_query_error_retry(self, mock_bigquery):
mock_query = mock_bigquery.return_value.query
mock_query.side_effect = NotFound('table not found')
matcher = bq_verifier.BigqueryMatcher('mock_project',
'mock_query',
'mock_checksum')
with self.assertRaises(NotFound):
hc_assert_that(self._mock_result, matcher)
self.assertEqual(bq_verifier.MAX_RETRIES + 1, mock_query.call_count)
def get_bq_table(bq_project_id, bq_dataset_id, table_name):
bigquery_client = bigquery.Client(project=bq_project_id)
dataset = bigquery_client.dataset(bq_dataset_id)
table = dataset.table(table_name)
if table.exists():
return table
else:
raise NotFound('Table {} not found.'.format(table_name))
def get_session(self, name, metadata=None):
from google.api_core.exceptions import NotFound
if self._rpc_error is not None:
raise self._rpc_error
self._get_session_called_with = name, metadata
try:
return self._get_session_response
except AttributeError:
raise NotFound('miss')
def test_delete_nonexisting_subscription_failifnotexists(
self, mock_service):
mock_service.delete_subscription.side_effect = NotFound(
f'Subscription does not exists: {EXPANDED_SUBSCRIPTION}')
with pytest.raises(PubSubException) as ctx:
self.pubsub_hook.delete_subscription(
project_id=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
fail_if_not_exists=True)
assert str(ctx.value
) == f'Subscription does not exist: {EXPANDED_SUBSCRIPTION}'
def reload(self):
"""Reload this table.
Refresh any configured schema into :attr:`schema`.
:raises NotFound: if the table does not exist
"""
with self._database.snapshot() as snapshot:
if not self._exists(snapshot):
raise NotFound("table '{}' does not exist".format(self.table_id))
self._schema = self._get_schema(snapshot)
def update_database_ddl(self, database, statements, operation_id,
metadata=None):
from google.api_core.exceptions import NotFound, Unknown
self._updated_database_ddl = (
database, statements, operation_id, metadata)
if self._rpc_error:
raise Unknown('error')
if self._database_not_found:
raise NotFound('not found')
return self._update_database_ddl_response
def idempotent_transfer(source, destination):
'''Transfers from `source` to `destination`
Raises NotFound if file doesn't exist at source or destination.
'''
try:
transfer(source, destination)
except NotFound:
if get_blob_from_url(destination) is None:
raise NotFound(
u'File not found at source {} or destination {}'.format(
source, destination))
def api_request(self, **kw):
from google.cloud.exceptions import Conflict
from google.cloud.exceptions import NotFound
self._called_with = kw
if self._raise_conflict:
raise Conflict('oops')
try:
response, self._responses = self._responses[0], self._responses[1:]
except IndexError:
raise NotFound('miss')
return response
def make_connection(*responses):
import google.cloud.bigquery._http
import mock
from google.cloud.exceptions import NotFound
mock_conn = mock.create_autospec(google.cloud.bigquery._http.Connection)
mock_conn.user_agent = "testing 1.2.3"
mock_conn.api_request.side_effect = list(responses) + [NotFound("miss")]
mock_conn.API_BASE_URL = "https://bigquery.googleapis.com"
mock_conn.get_api_base_url_for_mtls = mock.Mock(return_value=mock_conn.API_BASE_URL)
return mock_conn
def test_delete_nonexisting_subscription_failifnotexists(
self, mock_service):
mock_service.delete_subscription.side_effect = NotFound(
'Subscription does not exists: %s' % EXPANDED_SUBSCRIPTION)
with self.assertRaises(PubSubException) as e:
self.pubsub_hook.delete_subscription(
project_id=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
fail_if_not_exists=True)
self.assertEqual(
str(e.exception),
'Subscription does not exist: %s' % EXPANDED_SUBSCRIPTION)
def test_bucket_doesnt_exist_raises_by_default(self, monkeypatch, klass):
task = klass(bucket="test")
run_arg = "data" if isinstance(task, GCSUpload) else "blob"
client = MagicMock()
client.return_value = MagicMock(get_bucket=MagicMock(
side_effect=NotFound("no bucket")))
monkeypatch.setattr("prefect.tasks.gcp.storage.get_storage_client",
client)
with pytest.raises(NotFound, match="no bucket"):
task.run(**{run_arg: "empty"}, credentials={})
def test_bigquery_matcher_query_responds_error_code(self, mock_bigquery):
mock_query = Mock()
mock_client = mock_bigquery.return_value
mock_client.run_sync_query.return_value = mock_query
mock_query.run.side_effect = NotFound('table is not found')
matcher = bq_verifier.BigqueryMatcher('mock_project', 'mock_query',
'mock_checksum')
with self.assertRaises(NotFound):
hc_assert_that(self._mock_result, matcher)
self.assertTrue(mock_query.run.called)
self.assertEqual(bq_verifier.MAX_RETRIES + 1,
mock_query.run.call_count)
def create_database(self, parent, create_statement, extra_statements=None,
metadata=None):
from google.api_core.exceptions import AlreadyExists, NotFound, Unknown
self._created_database = (
parent, create_statement, extra_statements, metadata)
if self._rpc_error:
raise Unknown('error')
if self._create_database_conflict:
raise AlreadyExists('conflict')
if self._database_not_found:
raise NotFound('not found')
return self._create_database_response
def api_request(self, **kw):
from google.cloud.exceptions import NotFound
self._requested.append(kw)
try:
response, self._responses = self._responses[0], self._responses[1:]
except IndexError:
raise NotFound("miss")
else:
if issubclass(type(response), Exception):
raise response
return response
def test_backup_exists_w_not_found():
from google.api_core.exceptions import NotFound
client = _Client()
api = client.table_admin_client = _make_table_admin_client()
api.get_backup.side_effect = NotFound("testing")
instance = _Instance(INSTANCE_NAME, client=client)
backup = _make_backup(BACKUP_ID, instance, cluster_id=CLUSTER_ID)
assert not backup.exists()
api.get_backup.assert_called_once_with(request={"name": BACKUP_NAME})
def test_delete_not_found(self):
from google.api_core.exceptions import NotFound
client = _Client()
api = client.table_admin_client = self._make_table_admin_client()
api.delete_backup.side_effect = NotFound("testing")
instance = _Instance(self.INSTANCE_NAME, client=client)
backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID)
with self.assertRaises(NotFound):
backup.delete()
api.delete_backup.assert_called_once_with(request={"name": self.BACKUP_NAME})
def test_creds_are_pulled_from_secret_at_runtime(self, monkeypatch):
task = CreateBigQueryTable(
credentials_secret="GOOGLE_APPLICATION_CREDENTIALS")
client_util = MagicMock(return_value=MagicMock(get_table=MagicMock(
side_effect=NotFound("boy"))))
monkeypatch.setattr("prefect.tasks.gcp.bigquery.get_bigquery_client",
client_util)
with prefect.context(secrets=dict(
GOOGLE_APPLICATION_CREDENTIALS={"key": 42})):
task.run()
assert client_util.call_args[1]["credentials"] == {"key": 42}
def test_cost_usage_source_is_reachable_dataset_not_found(self, mock_auth, mock_discovery, mock_bigquery):
"""Test that cost_usage_source_is_reachable throws appropriate error when dataset is not found."""
mock_bigquery.Client.side_effect = NotFound(message="Not Found")
gcp_creds = MagicMock()
mock_auth.return_value = (gcp_creds, MagicMock())
mock_discovery.build.return_value.projects.return_value.testIamPermissions.return_value.execute.return_value.get.return_value = ( # noqa: E501
REQUIRED_IAM_PERMISSIONS
)
billing_source_param = {"dataset": FAKE.word()}
credentials_param = {"project_id": FAKE.word()}
provider = GCPProvider()
with self.assertRaises(ValidationError):
provider.cost_usage_source_is_reachable(credentials_param, billing_source_param)
def test_exists_not_found(self):
from google.api_core.exceptions import NotFound
client = _Client()
api = client.table_admin_client = self._make_table_admin_client()
api.get_backup.side_effect = NotFound("testing")
instance = _Instance(self.INSTANCE_NAME, client=client)
backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID)
self.assertFalse(backup.exists())
api.get_backup.assert_called_once_with(self.BACKUP_NAME)
def test_create_instance_when_not_exists(self, mock_get_conn):
mock_get_conn.return_value.get_instance.side_effect = [
NotFound("Instance not found"),
Instance(name=TEST_NAME),
]
mock_get_conn.return_value.create_instance.return_value.result.return_value = Instance(
name=TEST_NAME)
result = self.hook.create_instance(
location=TEST_LOCATION,
instance_id=TEST_INSTANCE_ID,
instance=Instance(name=TEST_NAME),
project_id=TEST_PROJECT_ID,
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
)
mock_get_conn.return_value.get_instance.has_calls([
mock.call(
name=
"projects/test-project-id/locations/test-location/instances/test-instance-id",
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
),
mock.call(
name=
"projects/test-project-id/locations/test-location/instances/test-instance-id",
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
metadata=TEST_METADATA,
),
])
mock_get_conn.return_value.create_instance.assert_called_once_with(
request=dict(
parent=TEST_PARENT,
instance=Instance(
name=TEST_NAME,
labels={
"airflow-version":
"v" +
version.version.replace(".", "-").replace("+", "-")
},
),
instance_id=TEST_INSTANCE_ID,
),
metadata=TEST_METADATA,
retry=TEST_RETRY,
timeout=TEST_TIMEOUT,
)
assert Instance(name=TEST_NAME) == result
def test_reload_not_found(self):
from google.api_core.exceptions import NotFound
client = _Client()
api = client.database_admin_api = self._make_database_admin_api()
api.get_backup.side_effect = NotFound("testing")
instance = _Instance(self.INSTANCE_NAME, client=client)
backup = self._make_one(self.BACKUP_ID, instance)
with self.assertRaises(NotFound):
backup.reload()
api.get_backup.assert_called_once_with(
self.BACKUP_NAME,
metadata=[("google-cloud-resource-prefix", backup.name)])
def test_exists_miss(self):
from google.cloud.exceptions import NotFound
client = self._make_client()
bucket = self._make_bucket(client)
notification = self._make_one(bucket, self.TOPIC_NAME)
notification._properties["id"] = self.NOTIFICATION_ID
api_request = client._connection.api_request
api_request.side_effect = NotFound("testing")
self.assertFalse(notification.exists())
api_request.assert_called_once_with(method="GET",
path=self.NOTIFICATION_PATH,
query_params={})
def drop(self):
"""Drop this database.
See
https://cloud.google.com/spanner/reference/rpc/google.spanner.admin.database.v1#google.spanner.admin.database.v1.DatabaseAdmin.DropDatabase
"""
api = self._instance._client.database_admin_api
options = _options_with_prefix(self.name)
try:
api.drop_database(self.name, options=options)
except GaxError as exc:
if exc_to_code(exc.cause) == StatusCode.NOT_FOUND:
raise NotFound(self.name)
raise
def test_bucket_doesnt_exist_can_be_created_on_upload(self, monkeypatch):
task = GCSUpload(bucket="test", create_bucket=True)
client = MagicMock()
client.return_value = MagicMock(
get_bucket=MagicMock(side_effect=NotFound("no bucket"))
)
monkeypatch.setattr("prefect.tasks.gcp.storage.get_storage_client", client)
task.run(data="empty", credentials={})
task.run(data="empty", bucket="run", credentials={})
assert client.return_value.create_bucket.called
assert client.return_value.create_bucket.call_args_list[0][0][0] == "test"
assert client.return_value.create_bucket.call_args_list[1][0][0] == "run"
