def test_execute_integrity_error(self):
from google.api_core import exceptions
from google.cloud.spanner_dbapi.exceptions import IntegrityError
connection = self._make_connection(self.INSTANCE, mock.MagicMock())
cursor = self._make_one(connection)
with mock.patch(
"google.cloud.spanner_dbapi.parse_utils.classify_stmt",
side_effect=exceptions.AlreadyExists("message"),
):
with self.assertRaises(IntegrityError):
cursor.execute(sql="sql")
with mock.patch(
"google.cloud.spanner_dbapi.parse_utils.classify_stmt",
side_effect=exceptions.FailedPrecondition("message"),
):
with self.assertRaises(IntegrityError):
cursor.execute(sql="sql")
with mock.patch(
"google.cloud.spanner_dbapi.parse_utils.classify_stmt",
side_effect=exceptions.OutOfRange("message"),
):
with self.assertRaises(IntegrityError):
cursor.execute("sql")
class TestSubscriber:
@patch.object(SubscriberClient, "create_subscription")
def test_creates_subscription_with_default_ack_deadline_when_none_provided(
self, _mocked_client, project_id, subscriber):
expected_subscription = f"projects/{project_id}/subscriptions/" f"test-topic"
expected_topic = f"projects/{project_id}/topics/" f"{project_id}-test-topic"
subscriber.create_subscription("test-topic",
f"{project_id}-test-topic")
_mocked_client.assert_called_once_with(
ack_deadline_seconds=60,
name=expected_subscription,
topic=expected_topic,
)
assert subscriber._gc_project_id == "rele-test"
@patch.object(SubscriberClient, "create_subscription")
def test_creates_subscription_with_custom_ack_deadline_when_provided(
self, _mocked_client, project_id, subscriber):
expected_subscription = f"projects/{project_id}/subscriptions/" f"test-topic"
expected_topic = f"projects/{project_id}/topics/" f"{project_id}-test-topic"
subscriber._ack_deadline = 100
subscriber.create_subscription(subscription="test-topic",
topic=f"{project_id}-test-topic")
_mocked_client.assert_called_once_with(
ack_deadline_seconds=100,
name=expected_subscription,
topic=expected_topic,
)
@patch.object(
SubscriberClient,
"create_subscription",
side_effect=exceptions.AlreadyExists("Subscription already exists"),
)
def test_does_not_raise_when_subscription_already_exists(
self, _mocked_client, project_id, subscriber):
subscriber.create_subscription(subscription="test-topic",
topic=f"{project_id}-test-topic")
_mocked_client.assert_called()
@patch.object(
SubscriberClient,
"create_subscription",
side_effect=exceptions.NotFound("Subscription topic does not exist"),
)
def test_logs_error_when_subscription_topic_does_not_exist(
self, _mocked_client, project_id, subscriber, caplog):
subscriber.create_subscription(subscription="test-topic",
topic=f"{project_id}-test-topic")
_mocked_client.assert_called()
log = caplog.records[0]
assert log.message == "Cannot subscribe to a topic that does not exist."
assert log.levelname == "ERROR"
def test_create_metrics_on_exception_should_not_raise_error(self):
monitoring_client = self.__monitoring_client
monitoring_client.create_metric_descriptor.side_effect = \
exceptions.AlreadyExists('Metric Descriptor already exists')
self.__monitoring_facade.create_metrics()
self.assertEqual(3,
monitoring_client.create_metric_descriptor.call_count)
def test_create_subscription_already_exist():
"""
Test subscription creation with a subscription that already exists
"""
subscriber = mock.Mock()
subscriber.create_subscription.side_effect = exceptions.AlreadyExists(
"topic already exists")
create_subscription(subscriber, "project", "topic", "subscription")
assert subscriber.create_subscription.call_count == 1
def test_create_topic_already_exist():
"""
Test topic creation with a topic that already exists
"""
publisher = mock.Mock()
publisher.create_topic.side_effect = exceptions.AlreadyExists(
"topic already exists")
create_topic(publisher, "project", "topic")
assert publisher.create_topic.call_count == 1
def test_private_get_publisher_topic_exists(mock_publisher):
client = mock_publisher.return_value
client.create_topic.side_effect = gapi_exceptions.AlreadyExists("foo")
ret_publisher = utils._get_publisher("a-topic")
mock_publisher.assert_called_once_with()
client.create_topic.assert_called_once_with("a-topic")
assert client == ret_publisher
def test_verify_pub_topic_exists(
klio_config, mock_publisher,
):
test_topic = "test"
mock_publisher.create_topic.side_effect = api_ex.AlreadyExists("test")
job = verify.VerifyJob(klio_config, True)
job._publisher_client = mock_publisher
actual = job._verify_pub_topic(test_topic, input)
assert actual is True
def test_existing_entry_group_should_manually_build_entry_group_path(self):
entries = utils.Utils \
.create_assembled_entries_user_defined_types()
datacatalog_facade = self.__datacatalog_facade
datacatalog_facade.create_entry_group.side_effect = \
exceptions.AlreadyExists('Entry Group already exists')
self.__metadata_ingestor.ingest_metadata(entries, {})
self.assertEqual(1, datacatalog_facade.create_entry_group.call_count)
self.assertEqual(2, datacatalog_facade.upsert_entry.call_count)
def test_verify_subscription_and_topic_exists(klio_config, mock_publisher,
mock_sub):
test_sub = "test"
upstream_topic = "Some"
job = verify.VerifyJob(klio_config, True)
job._publisher_client = mock_publisher
job._subscriber_client = mock_sub
mock_sub.create_subscription.side_effect = api_ex.AlreadyExists("test")
actual = job._verify_subscription_and_topic(test_sub, upstream_topic)
expected = True, True
assert expected == actual
def test_get_event_consumer_sub_exists(consumer_config, auth_client,
subscriber_client, emulator, exp_topic,
exp_sub, metrics):
"""Do not raise if topic already exists."""
success_chnl, error_chnl = asyncio.Queue(), asyncio.Queue()
exp = google_exceptions.AlreadyExists('foo')
sub_inst = subscriber_client.return_value
sub_inst.create_subscription.side_effect = [exp]
client = service.get_event_consumer(consumer_config, success_chnl,
error_chnl, metrics)
assert client._subscriber
sub_inst.create_subscription.assert_called_once_with(exp_sub, exp_topic)
def test_get_publisher_topic_exists(config, auth_client, publisher_client,
emulator):
"""Do not raise if topic already exists."""
changes_chnl = asyncio.Queue()
exp = google_exceptions.AlreadyExists('foo')
publisher_client.return_value.create_topic.side_effect = [exp]
short_topic = config['topic']
client = plugins.get_publisher(config, changes_chnl)
exp_topic = f'projects/{config["project"]}/topics/{short_topic}'
assert 60 == client.cleanup_timeout
assert client.publisher is not None
assert not client._messages
assert exp_topic == client.topic
client.publisher.create_topic.assert_called_once_with(exp_topic)
def test_ingest_metadata_existing_template_should_succeed(self):
entries = utils \
.Utils.create_assembled_entries_user_defined_types()
datacatalog_facade = self.__datacatalog_facade
datacatalog_facade.get_entry.return_value = None
datacatalog_facade.create_tag_template.side_effect = \
exceptions.AlreadyExists('Tag Template already exists')
tag_template = self.__create_tag_template()
tag_templates_dict = {'database': tag_template}
self.__metadata_ingestor.ingest_metadata(entries, tag_templates_dict)
self.assertEqual(1, datacatalog_facade.create_entry_group.call_count)
self.assertEqual(2, datacatalog_facade.upsert_entry.call_count)
self.assertEqual(1, datacatalog_facade.create_tag_template.call_count)
def check_bq_write_mode(self, mysql_component, hive_table_model,
bq_table_model):
"""Validates the bq_table_write_mode provided by user.
If the mode is overwrite, drops the tracking table and deletes the
BigQuery table. If the mode is create, checks if the tracking table
and BigQuery table exist. If the mode is append, checks whether the
BigQuery table exists.
Args:
mysql_component (:class:`MySQLComponent`): Instance of
MySQLComponent to connect to MySQL.
hive_table_model (:class:`HiveTableModel`): Wrapper to Hive table
details.
bq_table_model (:class:`BigQueryTableModel`): Wrapper to BigQuery
table details.
Returns:
boolean: True if the write mode is okay to use, else False.
"""
write_mode = PropertiesReader.get('bq_table_write_mode')
if write_mode == "overwrite":
logger.debug("Deleting tracking table and BigQuery table...")
mysql_component.drop_table(hive_table_model.tracking_table_name)
mysql_component.update_tracking_meta_table(hive_table_model,
"DELETE")
self.delete_table(bq_table_model.dataset_id,
bq_table_model.table_name)
hive_table_model.is_first_run = True
elif write_mode == "create":
if not hive_table_model.is_first_run:
raise exceptions.AlreadyExists(
"Tracking Table {} already exist".format(
hive_table_model.tracking_table_name))
if self.check_bq_table_exists(bq_table_model.dataset_id,
bq_table_model.table_name):
raise exceptions.AlreadyExists(
"BigQuery Table {} already exist in {} dataset".format(
bq_table_model.table_name, bq_table_model.dataset_id))
else:
if hive_table_model.is_first_run is False:
query = "SELECT COUNT(*) FROM {} WHERE " \
"bq_job_status='RUNNING' OR " \
"bq_job_status='DONE'".format(
hive_table_model.tracking_table_name)
results = mysql_component.execute_query(query)
if results[0][0] != 0:
if not self.check_bq_table_exists(
bq_table_model.dataset_id,
bq_table_model.table_name):
raise exceptions.NotFound(
"Found the tracking table but BigQuery Table {} "
"doesn't exist in {} dataset. Clean up the "
"resources and try again".format(
bq_table_model.table_name,
bq_table_model.dataset_id))
else:
raise exceptions.NotFound(
"Tracking Table {} doesn't exist".format(
hive_table_model.tracking_table_name))
def test_get_or_create_experiment(self):
api = mock.Mock()
api.create_tensorboard_experiment.side_effect = exceptions.AlreadyExists("test")
tensorboard_api._get_or_create_experiment(api, "test")
api.get_tensorboard_experiment.assert_called_once()
