def test_create_metric(self):
METRIC_NAME = 'test-create-metric%s' % (_RESOURCE_ID,)
metric = Config.CLIENT.metric(
METRIC_NAME, DEFAULT_FILTER, DEFAULT_DESCRIPTION)
self.assertFalse(metric.exists())
retry = RetryErrors(Conflict)
retry(metric.create)()
self.to_delete.append(metric)
self.assertTrue(metric.exists())
def test_create_sink_bigquery_dataset(self):
SINK_NAME = "test-create-sink-dataset%s" % (_RESOURCE_ID,)
retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10)
uri = self._init_bigquery_dataset()
sink = Config.CLIENT.sink(SINK_NAME, filter_=DEFAULT_FILTER, destination=uri)
self.assertFalse(sink.exists())
retry(sink.create)()
self.to_delete.append(sink)
self.assertTrue(sink.exists())
def test_create_sink_storage_bucket(self):
uri = self._init_storage_bucket()
SINK_NAME = "test-create-sink-bucket%s" % (_RESOURCE_ID,)
retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10)
sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri)
self.assertFalse(sink.exists())
retry(sink.create)()
self.to_delete.append(sink)
self.assertTrue(sink.exists())
def delete_blob(blob):
errors = (exceptions.Conflict, exceptions.TooManyRequests)
retry = RetryErrors(errors)
try:
retry(blob.delete)(timeout=120) # seconds
except exceptions.NotFound: # race
pass
except exceptions.Forbidden: # event-based hold
blob.event_based_hold = False
blob.patch()
retry_no_event_based_hold(blob.reload)()
retry(blob.delete)(timeout=120) # seconds
def test_reload_sink(self):
SINK_NAME = 'test-reload-sink%s' % (_RESOURCE_ID, )
retry = RetryErrors(Conflict)
uri = self._init_bigquery_dataset()
sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri)
self.assertFalse(sink.exists())
retry(sink.create)()
self.to_delete.append(sink)
sink.filter_ = 'BOGUS FILTER'
sink.destination = 'BOGUS DESTINATION'
sink.reload()
self.assertEqual(sink.filter_, DEFAULT_FILTER)
self.assertEqual(sink.destination, uri)
def test_reload_metric(self):
METRIC_NAME = 'test-reload-metric%s' % (_RESOURCE_ID, )
retry = RetryErrors(Conflict)
metric = Config.CLIENT.metric(METRIC_NAME, DEFAULT_FILTER,
DEFAULT_DESCRIPTION)
self.assertFalse(metric.exists())
retry(metric.create)()
self.to_delete.append(metric)
metric.filter_ = 'logName:other'
metric.description = 'local changes'
metric.reload()
self.assertEqual(metric.filter_, DEFAULT_FILTER)
self.assertEqual(metric.description, DEFAULT_DESCRIPTION)
def test_update_database_with_default_leader(
capsys,
multi_region_instance,
multi_region_instance_id,
default_leader_database_id,
default_leader,
):
retry_429 = RetryErrors(exceptions.ResourceExhausted, delay=15)
retry_429(snippets.update_database_with_default_leader)(
multi_region_instance_id, default_leader_database_id, default_leader)
out, _ = capsys.readouterr()
assert default_leader_database_id in out
assert default_leader in out
def _query_timeseries_with_retries():
MAX_RETRIES = 6
def _has_timeseries(result):
return len(list(result)) > 0
retry_result = RetryResult(
_has_timeseries,
max_tries=MAX_RETRIES,
backoff=3)(client.query)
return RetryErrors(
BadRequest,
max_tries=MAX_RETRIES,
backoff=3)(retry_result)
def test_list_entry_with_auditlog(self):
"""
Test emitting and listing logs containing a google.cloud.audit.AuditLog proto message
"""
from google.protobuf import descriptor_pool
from google.cloud.logging_v2 import entries
pool = descriptor_pool.Default()
type_name = "google.cloud.audit.AuditLog"
type_url = "type.googleapis.com/" + type_name
# Make sure the descriptor is known in the registry.
# Raises KeyError if unknown
pool.FindMessageTypeByName(type_name)
# create log
audit_dict = {
"@type": type_url,
"methodName": "test",
"requestMetadata": {
"callerIp": "::1",
"callerSuppliedUserAgent": "test"
},
"resourceName": "test",
"serviceName": "test",
"status": {
"code": 0
},
}
audit_struct = self._dict_to_struct(audit_dict)
logger = Config.CLIENT.logger(f"audit-proto-{uuid.uuid1()}")
logger.log_proto(audit_struct)
# retrieve log
retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8)
protobuf_entry = retry(lambda: next(logger.list_entries()))()
self.assertIsInstance(protobuf_entry, entries.ProtobufEntry)
self.assertIsNone(protobuf_entry.payload_pb)
self.assertIsInstance(protobuf_entry.payload_json, dict)
self.assertEqual(protobuf_entry.payload_json["@type"], type_url)
self.assertEqual(protobuf_entry.payload_json["methodName"],
audit_dict["methodName"])
self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["@type"],
type_url)
self.assertEqual(
protobuf_entry.to_api_repr()["protoPayload"]["methodName"],
audit_dict["methodName"],
)
def test_update_sink(self):
SINK_NAME = 'test-update-sink%s' % (_RESOURCE_ID, )
retry = RetryErrors(Conflict, max_tries=10)
bucket_uri = self._init_storage_bucket()
dataset_uri = self._init_bigquery_dataset()
UPDATED_FILTER = 'logName:syslog'
sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, bucket_uri)
self.assertFalse(sink.exists())
retry(sink.create)()
self.to_delete.append(sink)
sink.filter_ = UPDATED_FILTER
sink.destination = dataset_uri
sink.update()
self.assertEqual(sink.filter_, UPDATED_FILTER)
self.assertEqual(sink.destination, dataset_uri)
def test_reload_sink(self):
SINK_NAME = "test-reload-sink%s" % (_RESOURCE_ID,)
retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10)
uri = self._init_bigquery_dataset()
sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri)
self.assertFalse(sink.exists())
retry(sink.create)()
self.to_delete.append(sink)
sink.filter_ = "BOGUS FILTER"
sink.destination = "BOGUS DESTINATION"
sink.reload()
self.assertEqual(sink.filter_, DEFAULT_FILTER)
self.assertEqual(sink.destination, uri)
def _list_entries(logger):
"""Retry-ing list entries in a logger.
Retry until there are actual results and retry on any
failures.
:type logger: :class:`~google.cloud.logging.logger.Logger`
:param logger: A Logger containing entries.
:rtype: list
:returns: List of all entries consumed.
"""
inner = RetryResult(_has_entries, max_tries=9)(_consume_entries)
outer = RetryErrors((ServiceUnavailable, ResourceExhausted), max_tries=9)(inner)
return outer(logger)
def _list_entries(logger):
"""Retry-ing list entries in a logger.
Retry until there are actual results and retry on any
failures.
:type logger: :class:`~google.cloud.logging.logger.Logger`
:param logger: A Logger containing entries.
:rtype: list
:returns: List of all entries consumed.
"""
inner = RetryResult(_has_entries)(_consume_entries)
outer = RetryErrors(GaxError, _retry_on_unavailable)(inner)
return outer(logger)
def test_list_metrics(self):
METRIC_NAME = "test-list-metrics%s" % (_RESOURCE_ID,)
metric = Config.CLIENT.metric(METRIC_NAME, DEFAULT_FILTER, DEFAULT_DESCRIPTION)
self.assertFalse(metric.exists())
before_metrics = list(Config.CLIENT.list_metrics())
before_names = set(before.name for before in before_metrics)
self.assertFalse(metric.name in before_names)
retry = RetryErrors(Conflict)
retry(metric.create)()
self.to_delete.append(metric)
self.assertTrue(metric.exists())
after_metrics = list(Config.CLIENT.list_metrics())
after_names = set(after.name for after in after_metrics)
self.assertTrue(metric.name in after_names)
def test_list_sinks(self):
SINK_NAME = "test-list-sinks%s" % (_RESOURCE_ID,)
uri = self._init_storage_bucket()
retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10)
sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri)
self.assertFalse(sink.exists())
before_sinks = list(Config.CLIENT.list_sinks())
before_names = set(before.name for before in before_sinks)
self.assertFalse(sink.name in before_names)
retry(sink.create)()
self.to_delete.append(sink)
self.assertTrue(sink.exists())
after_sinks = list(Config.CLIENT.list_sinks())
after_names = set(after.name for after in after_sinks)
self.assertTrue(sink.name in after_names)
def setUpModule():
from google.cloud.exceptions import GrpcRendezvous
from google.cloud.bigtable.enums import Instance
# See: https://github.com/googleapis/google-cloud-python/issues/5928
interfaces = table_admin_config.config["interfaces"]
iface_config = interfaces["google.bigtable.admin.v2.BigtableTableAdmin"]
methods = iface_config["methods"]
create_table = methods["CreateTable"]
create_table["timeout_millis"] = 90000
Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
if Config.IN_EMULATOR:
credentials = EmulatorCreds()
Config.CLIENT = Client(admin=True, credentials=credentials)
else:
Config.CLIENT = Client(admin=True)
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, labels=LABELS)
Config.CLUSTER = Config.INSTANCE.cluster(CLUSTER_ID,
location_id=LOCATION_ID,
serve_nodes=SERVE_NODES)
Config.INSTANCE_DATA = Config.CLIENT.instance(
INSTANCE_ID_DATA,
instance_type=Instance.Type.DEVELOPMENT,
labels=LABELS)
Config.CLUSTER_DATA = Config.INSTANCE_DATA.cluster(CLUSTER_ID_DATA,
location_id=LOCATION_ID)
if not Config.IN_EMULATOR:
retry = RetryErrors(GrpcRendezvous,
error_predicate=_retry_on_unavailable)
instances, failed_locations = retry(Config.CLIENT.list_instances)()
if len(failed_locations) != 0:
raise ValueError("List instances failed in module set up.")
EXISTING_INSTANCES[:] = instances
# After listing, create the test instances.
admin_op = Config.INSTANCE.create(clusters=[Config.CLUSTER])
admin_op.result(timeout=10)
data_op = Config.INSTANCE_DATA.create(clusters=[Config.CLUSTER_DATA])
data_op.result(timeout=10)
def test_update_metric(self):
METRIC_NAME = 'test-update-metric%s' % (_RESOURCE_ID, )
retry = RetryErrors(Conflict)
NEW_FILTER = 'logName:other'
NEW_DESCRIPTION = 'updated'
metric = Config.CLIENT.metric(METRIC_NAME, DEFAULT_FILTER,
DEFAULT_DESCRIPTION)
self.assertFalse(metric.exists())
retry(metric.create)()
self.to_delete.append(metric)
metric.filter_ = NEW_FILTER
metric.description = NEW_DESCRIPTION
metric.update()
after_metrics = list(Config.CLIENT.list_metrics())
after_info = {metric.name: metric for metric in after_metrics}
after = after_info[METRIC_NAME]
self.assertEqual(after.filter_, NEW_FILTER)
self.assertEqual(after.description, NEW_DESCRIPTION)
def test_delete_column_family(self):
temp_table_id = "test-delete-column-family"
temp_table = Config.INSTANCE_DATA.table(temp_table_id)
temp_table.create()
self.tables_to_delete.append(temp_table)
self.assertEqual(temp_table.list_column_families(), {})
column_family = temp_table.column_family(COLUMN_FAMILY_ID1)
column_family.create()
# Make sure the family is there before deleting it.
col_fams = temp_table.list_column_families()
self.assertEqual(list(col_fams.keys()), [COLUMN_FAMILY_ID1])
retry_504 = RetryErrors(DeadlineExceeded)
retry_504(column_family.delete)()
# Make sure we have successfully deleted it.
self.assertEqual(temp_table.list_column_families(), {})
def _init_storage_bucket(self):
from google.cloud import storage
BUCKET_NAME = 'g-c-python-testing%s' % (_RESOURCE_ID,)
BUCKET_URI = 'storage.googleapis.com/%s' % (BUCKET_NAME,)
# Create the destination bucket, and set up the ACL to allow
# Stackdriver Logging to write into it.
retry = RetryErrors((Conflict, TooManyRequests, ServiceUnavailable))
storage_client = storage.Client()
bucket = storage_client.bucket(BUCKET_NAME)
retry(bucket.create)()
self.to_delete.append(bucket)
bucket.acl.reload()
logs_group = bucket.acl.group('*****@*****.**')
logs_group.grant_owner()
bucket.acl.add_entity(logs_group)
bucket.acl.save()
return BUCKET_URI
def test_list_entry_with_requestlog(self):
"""
Test emitting and listing logs containing a google.appengine.logging.v1.RequestLog proto message
"""
from google.protobuf import descriptor_pool
from google.cloud.logging_v2 import entries
pool = descriptor_pool.Default()
type_name = "google.appengine.logging.v1.RequestLog"
type_url = "type.googleapis.com/" + type_name
# Make sure the descriptor is known in the registry.
# Raises KeyError if unknown
pool.FindMessageTypeByName(type_name)
# create log
req_dict = {
"@type": type_url,
"ip": "0.0.0.0",
"appId": "test",
"versionId": "test",
"requestId": "12345",
"latency": "500.0s",
"method": "GET",
"status": 500,
"resource": "test",
"httpVersion": "HTTP/1.1",
}
req_struct = self._dict_to_struct(req_dict)
logger = Config.CLIENT.logger(f"req-proto-{uuid.uuid1()}")
logger.log_proto(req_struct)
# retrieve log
retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8)
protobuf_entry = retry(lambda: next(logger.list_entries()))()
self.assertIsInstance(protobuf_entry, entries.ProtobufEntry)
self.assertIsNone(protobuf_entry.payload_pb)
self.assertIsInstance(protobuf_entry.payload_json, dict)
self.assertEqual(protobuf_entry.payload_json["@type"], type_url)
self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["@type"],
type_url)
def setUpModule():
from google.cloud.exceptions import GrpcRendezvous
from google.cloud.bigtable.enums import Instance
Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
if Config.IN_EMULATOR:
credentials = EmulatorCreds()
Config.CLIENT = Client(admin=True, credentials=credentials)
else:
Config.CLIENT = Client(admin=True)
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, labels=LABELS)
Config.CLUSTER = Config.INSTANCE.cluster(CLUSTER_ID,
location_id=LOCATION_ID,
serve_nodes=SERVE_NODES)
Config.INSTANCE_DATA = Config.CLIENT.instance(
INSTANCE_ID_DATA,
instance_type=Instance.Type.DEVELOPMENT,
labels=LABELS)
Config.CLUSTER_DATA = Config.INSTANCE_DATA.cluster(CLUSTER_ID_DATA,
location_id=LOCATION_ID)
if not Config.IN_EMULATOR:
retry = RetryErrors(GrpcRendezvous,
error_predicate=_retry_on_unavailable)
instances, failed_locations = retry(Config.CLIENT.list_instances)()
if len(failed_locations) != 0:
raise ValueError("List instances failed in module set up.")
EXISTING_INSTANCES[:] = instances
# After listing, create the test instances.
created_op = Config.INSTANCE.create(clusters=[Config.CLUSTER])
created_op.result(timeout=10)
created_op = Config.INSTANCE_DATA.create(
clusters=[Config.CLUSTER_DATA])
created_op.result(timeout=10)
def _init_bigquery_dataset(self):
from google.cloud import bigquery
from google.cloud.bigquery.dataset import AccessEntry
dataset_name = ("system_testing_dataset" + _RESOURCE_ID).replace("-", "_")
dataset_uri = "bigquery.googleapis.com/projects/%s/datasets/%s" % (
Config.CLIENT.project,
dataset_name,
)
# Create the destination dataset, and set up the ACL to allow
# Stackdriver Logging to write into it.
retry = RetryErrors((TooManyRequests, BadGateway, ServiceUnavailable))
bigquery_client = bigquery.Client()
dataset_ref = bigquery_client.dataset(dataset_name)
dataset = retry(bigquery_client.create_dataset)(bigquery.Dataset(dataset_ref))
self.to_delete.append((bigquery_client, dataset))
bigquery_client.get_dataset(dataset)
access = AccessEntry("WRITER", "groupByEmail", "*****@*****.**")
dataset.access_entries.append(access)
bigquery_client.update_dataset(dataset, ["access_entries"])
return dataset_uri
def setUpModule():
Config.CLIENT = Client()
retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable)
configs = list(retry(Config.CLIENT.list_instance_configs)())
if len(configs) < 1:
raise ValueError('List instance configs failed in module set up.')
Config.INSTANCE_CONFIG = configs[0]
config_name = configs[0].name
instances = retry(_list_instances)()
EXISTING_INSTANCES[:] = instances
if CREATE_INSTANCE:
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name)
created_op = Config.INSTANCE.create()
created_op.result(30) # block until completion
else:
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
Config.INSTANCE.reload()
def tearDownModule():
# 409 Conflict if the bucket is full.
# 429 Too Many Requests in case API requests rate-limited.
bucket_retry = RetryErrors(
(exceptions.TooManyRequests, exceptions.Conflict))
bucket_retry(Config.TEST_BUCKET.delete)(force=True)
from google.cloud.storage._helpers import _base64_md5hash
from test_utils.retry import RetryErrors
from test_utils.system import unique_resource_id
USER_PROJECT = os.environ.get('GOOGLE_CLOUD_TESTS_USER_PROJECT')
def _bad_copy(bad_request):
"""Predicate: pass only exceptions for a failed copyTo."""
err_msg = bad_request.message
return (err_msg.startswith('No file found in request. (POST')
and 'copyTo' in err_msg)
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
def _empty_bucket(bucket):
"""Empty a bucket of all existing blobs.
This accounts (partially) for the eventual consistency of the
list blobs API call.
"""
for blob in bucket.list_blobs():
try:
blob.delete()
except exceptions.NotFound: # eventual consistency
pass
def tearDownModule():
# 409 Conflict if the bucket is full.
# 429 Too Many Requests in case API requests rate-limited.
bucket_retry = RetryErrors(
(exceptions.TooManyRequests, exceptions.Conflict))
bucket_retry(VisionSystemTestBase.test_bucket.delete)(force=True)
def tearDownModule():
retry = RetryErrors(exceptions.Conflict)
retry(Config.TEST_BUCKET.delete)(force=True)
from test_utils.retry import RetryErrors
from test_utils.system import unique_resource_id
USER_PROJECT = os.environ.get('GOOGLE_CLOUD_TESTS_USER_PROJECT')
def _bad_copy(bad_request):
"""Predicate: pass only exceptions for a failed copyTo."""
err_msg = bad_request.message
return (err_msg.startswith('No file found in request. (POST') and
'copyTo' in err_msg)
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429_503 = RetryErrors([
exceptions.TooManyRequests, exceptions.ServiceUnavailable])
retry_bad_copy = RetryErrors(exceptions.BadRequest,
error_predicate=_bad_copy)
def _empty_bucket(bucket):
"""Empty a bucket of all existing blobs.
This accounts (partially) for the eventual consistency of the
list blobs API call.
"""
for blob in bucket.list_blobs():
try:
blob.delete()
import google.cloud.logging
from google.cloud._helpers import UTC
from google.cloud.logging_v2.handlers import AppEngineHandler
from google.cloud.logging_v2.handlers import CloudLoggingHandler
from google.cloud.logging_v2.handlers.transports import SyncTransport
from google.cloud.logging_v2 import client
from google.cloud.logging_v2.resource import Resource
from test_utils.retry import RetryErrors
from test_utils.retry import RetryResult
from test_utils.system import unique_resource_id
_RESOURCE_ID = unique_resource_id("-")
DEFAULT_FILTER = "logName:syslog AND severity>=INFO"
DEFAULT_DESCRIPTION = "System testing"
retry_429 = RetryErrors(TooManyRequests)
def _consume_entries(logger):
"""Consume all log entries from logger iterator.
:type logger: :class:`~google.cloud.logging.logger.Logger`
:param logger: A Logger containing entries.
:rtype: list
:returns: List of all entries consumed.
"""
return list(logger.list_entries())
def _list_entries(logger):
CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX
APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX
TABLE_ID = "tabl-1" + UNIQUE_SUFFIX
ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY
LOCATION_ID = "us-central1-f"
ALT_LOCATION_ID = "us-central1-a"
PRODUCTION = enums.Instance.Type.PRODUCTION
SERVER_NODES = 3
STORAGE_TYPE = enums.StorageType.SSD
LABEL_KEY = u"python-snippet"
LABEL_STAMP = (datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S"))
LABELS = {LABEL_KEY: str(LABEL_STAMP)}
INSTANCES_TO_DELETE = []
retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9)
retry_504 = RetryErrors(DeadlineExceeded, max_tries=4)
class Config(object):
"""Run-time configuration to be modified at set-up.
This is a mutable stand-in to allow test set-up to modify
global state.
"""
CLIENT = None
INSTANCE = None
TABLE = None
