def test_create_metric(self): METRIC_NAME = 'test-create-metric%s' % (_RESOURCE_ID,) metric = Config.CLIENT.metric( METRIC_NAME, DEFAULT_FILTER, DEFAULT_DESCRIPTION) self.assertFalse(metric.exists()) retry = RetryErrors(Conflict) retry(metric.create)() self.to_delete.append(metric) self.assertTrue(metric.exists())
def test_create_sink_bigquery_dataset(self): SINK_NAME = "test-create-sink-dataset%s" % (_RESOURCE_ID,) retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10) uri = self._init_bigquery_dataset() sink = Config.CLIENT.sink(SINK_NAME, filter_=DEFAULT_FILTER, destination=uri) self.assertFalse(sink.exists()) retry(sink.create)() self.to_delete.append(sink) self.assertTrue(sink.exists())
def test_create_sink_storage_bucket(self): uri = self._init_storage_bucket() SINK_NAME = "test-create-sink-bucket%s" % (_RESOURCE_ID,) retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10) sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri) self.assertFalse(sink.exists()) retry(sink.create)() self.to_delete.append(sink) self.assertTrue(sink.exists())
def delete_blob(blob): errors = (exceptions.Conflict, exceptions.TooManyRequests) retry = RetryErrors(errors) try: retry(blob.delete)(timeout=120) # seconds except exceptions.NotFound: # race pass except exceptions.Forbidden: # event-based hold blob.event_based_hold = False blob.patch() retry_no_event_based_hold(blob.reload)() retry(blob.delete)(timeout=120) # seconds
def test_reload_sink(self): SINK_NAME = 'test-reload-sink%s' % (_RESOURCE_ID, ) retry = RetryErrors(Conflict) uri = self._init_bigquery_dataset() sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri) self.assertFalse(sink.exists()) retry(sink.create)() self.to_delete.append(sink) sink.filter_ = 'BOGUS FILTER' sink.destination = 'BOGUS DESTINATION' sink.reload() self.assertEqual(sink.filter_, DEFAULT_FILTER) self.assertEqual(sink.destination, uri)
def test_reload_metric(self): METRIC_NAME = 'test-reload-metric%s' % (_RESOURCE_ID, ) retry = RetryErrors(Conflict) metric = Config.CLIENT.metric(METRIC_NAME, DEFAULT_FILTER, DEFAULT_DESCRIPTION) self.assertFalse(metric.exists()) retry(metric.create)() self.to_delete.append(metric) metric.filter_ = 'logName:other' metric.description = 'local changes' metric.reload() self.assertEqual(metric.filter_, DEFAULT_FILTER) self.assertEqual(metric.description, DEFAULT_DESCRIPTION)
def test_update_database_with_default_leader( capsys, multi_region_instance, multi_region_instance_id, default_leader_database_id, default_leader, ): retry_429 = RetryErrors(exceptions.ResourceExhausted, delay=15) retry_429(snippets.update_database_with_default_leader)( multi_region_instance_id, default_leader_database_id, default_leader) out, _ = capsys.readouterr() assert default_leader_database_id in out assert default_leader in out
def _query_timeseries_with_retries(): MAX_RETRIES = 6 def _has_timeseries(result): return len(list(result)) > 0 retry_result = RetryResult( _has_timeseries, max_tries=MAX_RETRIES, backoff=3)(client.query) return RetryErrors( BadRequest, max_tries=MAX_RETRIES, backoff=3)(retry_result)
def test_list_entry_with_auditlog(self): """ Test emitting and listing logs containing a google.cloud.audit.AuditLog proto message """ from google.protobuf import descriptor_pool from google.cloud.logging_v2 import entries pool = descriptor_pool.Default() type_name = "google.cloud.audit.AuditLog" type_url = "type.googleapis.com/" + type_name # Make sure the descriptor is known in the registry. # Raises KeyError if unknown pool.FindMessageTypeByName(type_name) # create log audit_dict = { "@type": type_url, "methodName": "test", "requestMetadata": { "callerIp": "::1", "callerSuppliedUserAgent": "test" }, "resourceName": "test", "serviceName": "test", "status": { "code": 0 }, } audit_struct = self._dict_to_struct(audit_dict) logger = Config.CLIENT.logger(f"audit-proto-{uuid.uuid1()}") logger.log_proto(audit_struct) # retrieve log retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8) protobuf_entry = retry(lambda: next(logger.list_entries()))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) self.assertIsNone(protobuf_entry.payload_pb) self.assertIsInstance(protobuf_entry.payload_json, dict) self.assertEqual(protobuf_entry.payload_json["@type"], type_url) self.assertEqual(protobuf_entry.payload_json["methodName"], audit_dict["methodName"]) self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url) self.assertEqual( protobuf_entry.to_api_repr()["protoPayload"]["methodName"], audit_dict["methodName"], )
def test_update_sink(self): SINK_NAME = 'test-update-sink%s' % (_RESOURCE_ID, ) retry = RetryErrors(Conflict, max_tries=10) bucket_uri = self._init_storage_bucket() dataset_uri = self._init_bigquery_dataset() UPDATED_FILTER = 'logName:syslog' sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, bucket_uri) self.assertFalse(sink.exists()) retry(sink.create)() self.to_delete.append(sink) sink.filter_ = UPDATED_FILTER sink.destination = dataset_uri sink.update() self.assertEqual(sink.filter_, UPDATED_FILTER) self.assertEqual(sink.destination, dataset_uri)
def test_reload_sink(self): SINK_NAME = "test-reload-sink%s" % (_RESOURCE_ID,) retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10) uri = self._init_bigquery_dataset() sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri) self.assertFalse(sink.exists()) retry(sink.create)() self.to_delete.append(sink) sink.filter_ = "BOGUS FILTER" sink.destination = "BOGUS DESTINATION" sink.reload() self.assertEqual(sink.filter_, DEFAULT_FILTER) self.assertEqual(sink.destination, uri)
def _list_entries(logger): """Retry-ing list entries in a logger. Retry until there are actual results and retry on any failures. :type logger: :class:`~google.cloud.logging.logger.Logger` :param logger: A Logger containing entries. :rtype: list :returns: List of all entries consumed. """ inner = RetryResult(_has_entries, max_tries=9)(_consume_entries) outer = RetryErrors((ServiceUnavailable, ResourceExhausted), max_tries=9)(inner) return outer(logger)
def _list_entries(logger): """Retry-ing list entries in a logger. Retry until there are actual results and retry on any failures. :type logger: :class:`~google.cloud.logging.logger.Logger` :param logger: A Logger containing entries. :rtype: list :returns: List of all entries consumed. """ inner = RetryResult(_has_entries)(_consume_entries) outer = RetryErrors(GaxError, _retry_on_unavailable)(inner) return outer(logger)
def test_list_metrics(self): METRIC_NAME = "test-list-metrics%s" % (_RESOURCE_ID,) metric = Config.CLIENT.metric(METRIC_NAME, DEFAULT_FILTER, DEFAULT_DESCRIPTION) self.assertFalse(metric.exists()) before_metrics = list(Config.CLIENT.list_metrics()) before_names = set(before.name for before in before_metrics) self.assertFalse(metric.name in before_names) retry = RetryErrors(Conflict) retry(metric.create)() self.to_delete.append(metric) self.assertTrue(metric.exists()) after_metrics = list(Config.CLIENT.list_metrics()) after_names = set(after.name for after in after_metrics) self.assertTrue(metric.name in after_names)
def test_list_sinks(self): SINK_NAME = "test-list-sinks%s" % (_RESOURCE_ID,) uri = self._init_storage_bucket() retry = RetryErrors((Conflict, ServiceUnavailable), max_tries=10) sink = Config.CLIENT.sink(SINK_NAME, DEFAULT_FILTER, uri) self.assertFalse(sink.exists()) before_sinks = list(Config.CLIENT.list_sinks()) before_names = set(before.name for before in before_sinks) self.assertFalse(sink.name in before_names) retry(sink.create)() self.to_delete.append(sink) self.assertTrue(sink.exists()) after_sinks = list(Config.CLIENT.list_sinks()) after_names = set(after.name for after in after_sinks) self.assertTrue(sink.name in after_names)
def setUpModule(): from google.cloud.exceptions import GrpcRendezvous from google.cloud.bigtable.enums import Instance # See: https://github.com/googleapis/google-cloud-python/issues/5928 interfaces = table_admin_config.config["interfaces"] iface_config = interfaces["google.bigtable.admin.v2.BigtableTableAdmin"] methods = iface_config["methods"] create_table = methods["CreateTable"] create_table["timeout_millis"] = 90000 Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None if Config.IN_EMULATOR: credentials = EmulatorCreds() Config.CLIENT = Client(admin=True, credentials=credentials) else: Config.CLIENT = Client(admin=True) Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, labels=LABELS) Config.CLUSTER = Config.INSTANCE.cluster(CLUSTER_ID, location_id=LOCATION_ID, serve_nodes=SERVE_NODES) Config.INSTANCE_DATA = Config.CLIENT.instance( INSTANCE_ID_DATA, instance_type=Instance.Type.DEVELOPMENT, labels=LABELS) Config.CLUSTER_DATA = Config.INSTANCE_DATA.cluster(CLUSTER_ID_DATA, location_id=LOCATION_ID) if not Config.IN_EMULATOR: retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable) instances, failed_locations = retry(Config.CLIENT.list_instances)() if len(failed_locations) != 0: raise ValueError("List instances failed in module set up.") EXISTING_INSTANCES[:] = instances # After listing, create the test instances. admin_op = Config.INSTANCE.create(clusters=[Config.CLUSTER]) admin_op.result(timeout=10) data_op = Config.INSTANCE_DATA.create(clusters=[Config.CLUSTER_DATA]) data_op.result(timeout=10)
def test_update_metric(self): METRIC_NAME = 'test-update-metric%s' % (_RESOURCE_ID, ) retry = RetryErrors(Conflict) NEW_FILTER = 'logName:other' NEW_DESCRIPTION = 'updated' metric = Config.CLIENT.metric(METRIC_NAME, DEFAULT_FILTER, DEFAULT_DESCRIPTION) self.assertFalse(metric.exists()) retry(metric.create)() self.to_delete.append(metric) metric.filter_ = NEW_FILTER metric.description = NEW_DESCRIPTION metric.update() after_metrics = list(Config.CLIENT.list_metrics()) after_info = {metric.name: metric for metric in after_metrics} after = after_info[METRIC_NAME] self.assertEqual(after.filter_, NEW_FILTER) self.assertEqual(after.description, NEW_DESCRIPTION)
def test_delete_column_family(self): temp_table_id = "test-delete-column-family" temp_table = Config.INSTANCE_DATA.table(temp_table_id) temp_table.create() self.tables_to_delete.append(temp_table) self.assertEqual(temp_table.list_column_families(), {}) column_family = temp_table.column_family(COLUMN_FAMILY_ID1) column_family.create() # Make sure the family is there before deleting it. col_fams = temp_table.list_column_families() self.assertEqual(list(col_fams.keys()), [COLUMN_FAMILY_ID1]) retry_504 = RetryErrors(DeadlineExceeded) retry_504(column_family.delete)() # Make sure we have successfully deleted it. self.assertEqual(temp_table.list_column_families(), {})
def _init_storage_bucket(self): from google.cloud import storage BUCKET_NAME = 'g-c-python-testing%s' % (_RESOURCE_ID,) BUCKET_URI = 'storage.googleapis.com/%s' % (BUCKET_NAME,) # Create the destination bucket, and set up the ACL to allow # Stackdriver Logging to write into it. retry = RetryErrors((Conflict, TooManyRequests, ServiceUnavailable)) storage_client = storage.Client() bucket = storage_client.bucket(BUCKET_NAME) retry(bucket.create)() self.to_delete.append(bucket) bucket.acl.reload() logs_group = bucket.acl.group('*****@*****.**') logs_group.grant_owner() bucket.acl.add_entity(logs_group) bucket.acl.save() return BUCKET_URI
def test_list_entry_with_requestlog(self): """ Test emitting and listing logs containing a google.appengine.logging.v1.RequestLog proto message """ from google.protobuf import descriptor_pool from google.cloud.logging_v2 import entries pool = descriptor_pool.Default() type_name = "google.appengine.logging.v1.RequestLog" type_url = "type.googleapis.com/" + type_name # Make sure the descriptor is known in the registry. # Raises KeyError if unknown pool.FindMessageTypeByName(type_name) # create log req_dict = { "@type": type_url, "ip": "0.0.0.0", "appId": "test", "versionId": "test", "requestId": "12345", "latency": "500.0s", "method": "GET", "status": 500, "resource": "test", "httpVersion": "HTTP/1.1", } req_struct = self._dict_to_struct(req_dict) logger = Config.CLIENT.logger(f"req-proto-{uuid.uuid1()}") logger.log_proto(req_struct) # retrieve log retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8) protobuf_entry = retry(lambda: next(logger.list_entries()))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) self.assertIsNone(protobuf_entry.payload_pb) self.assertIsInstance(protobuf_entry.payload_json, dict) self.assertEqual(protobuf_entry.payload_json["@type"], type_url) self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url)
def setUpModule(): from google.cloud.exceptions import GrpcRendezvous from google.cloud.bigtable.enums import Instance Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None if Config.IN_EMULATOR: credentials = EmulatorCreds() Config.CLIENT = Client(admin=True, credentials=credentials) else: Config.CLIENT = Client(admin=True) Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, labels=LABELS) Config.CLUSTER = Config.INSTANCE.cluster(CLUSTER_ID, location_id=LOCATION_ID, serve_nodes=SERVE_NODES) Config.INSTANCE_DATA = Config.CLIENT.instance( INSTANCE_ID_DATA, instance_type=Instance.Type.DEVELOPMENT, labels=LABELS) Config.CLUSTER_DATA = Config.INSTANCE_DATA.cluster(CLUSTER_ID_DATA, location_id=LOCATION_ID) if not Config.IN_EMULATOR: retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable) instances, failed_locations = retry(Config.CLIENT.list_instances)() if len(failed_locations) != 0: raise ValueError("List instances failed in module set up.") EXISTING_INSTANCES[:] = instances # After listing, create the test instances. created_op = Config.INSTANCE.create(clusters=[Config.CLUSTER]) created_op.result(timeout=10) created_op = Config.INSTANCE_DATA.create( clusters=[Config.CLUSTER_DATA]) created_op.result(timeout=10)
def _init_bigquery_dataset(self): from google.cloud import bigquery from google.cloud.bigquery.dataset import AccessEntry dataset_name = ("system_testing_dataset" + _RESOURCE_ID).replace("-", "_") dataset_uri = "bigquery.googleapis.com/projects/%s/datasets/%s" % ( Config.CLIENT.project, dataset_name, ) # Create the destination dataset, and set up the ACL to allow # Stackdriver Logging to write into it. retry = RetryErrors((TooManyRequests, BadGateway, ServiceUnavailable)) bigquery_client = bigquery.Client() dataset_ref = bigquery_client.dataset(dataset_name) dataset = retry(bigquery_client.create_dataset)(bigquery.Dataset(dataset_ref)) self.to_delete.append((bigquery_client, dataset)) bigquery_client.get_dataset(dataset) access = AccessEntry("WRITER", "groupByEmail", "*****@*****.**") dataset.access_entries.append(access) bigquery_client.update_dataset(dataset, ["access_entries"]) return dataset_uri
def setUpModule(): Config.CLIENT = Client() retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) if len(configs) < 1: raise ValueError('List instance configs failed in module set up.') Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances if CREATE_INSTANCE: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name) created_op = Config.INSTANCE.create() created_op.result(30) # block until completion else: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload()
def tearDownModule(): # 409 Conflict if the bucket is full. # 429 Too Many Requests in case API requests rate-limited. bucket_retry = RetryErrors( (exceptions.TooManyRequests, exceptions.Conflict)) bucket_retry(Config.TEST_BUCKET.delete)(force=True)
from google.cloud.storage._helpers import _base64_md5hash from test_utils.retry import RetryErrors from test_utils.system import unique_resource_id USER_PROJECT = os.environ.get('GOOGLE_CLOUD_TESTS_USER_PROJECT') def _bad_copy(bad_request): """Predicate: pass only exceptions for a failed copyTo.""" err_msg = bad_request.message return (err_msg.startswith('No file found in request. (POST') and 'copyTo' in err_msg) retry_429 = RetryErrors(exceptions.TooManyRequests) retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) def _empty_bucket(bucket): """Empty a bucket of all existing blobs. This accounts (partially) for the eventual consistency of the list blobs API call. """ for blob in bucket.list_blobs(): try: blob.delete() except exceptions.NotFound: # eventual consistency pass
def tearDownModule(): # 409 Conflict if the bucket is full. # 429 Too Many Requests in case API requests rate-limited. bucket_retry = RetryErrors( (exceptions.TooManyRequests, exceptions.Conflict)) bucket_retry(VisionSystemTestBase.test_bucket.delete)(force=True)
def tearDownModule(): retry = RetryErrors(exceptions.Conflict) retry(Config.TEST_BUCKET.delete)(force=True)
from test_utils.retry import RetryErrors from test_utils.system import unique_resource_id USER_PROJECT = os.environ.get('GOOGLE_CLOUD_TESTS_USER_PROJECT') def _bad_copy(bad_request): """Predicate: pass only exceptions for a failed copyTo.""" err_msg = bad_request.message return (err_msg.startswith('No file found in request. (POST') and 'copyTo' in err_msg) retry_429 = RetryErrors(exceptions.TooManyRequests) retry_429_503 = RetryErrors([ exceptions.TooManyRequests, exceptions.ServiceUnavailable]) retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) def _empty_bucket(bucket): """Empty a bucket of all existing blobs. This accounts (partially) for the eventual consistency of the list blobs API call. """ for blob in bucket.list_blobs(): try: blob.delete()
import google.cloud.logging from google.cloud._helpers import UTC from google.cloud.logging_v2.handlers import AppEngineHandler from google.cloud.logging_v2.handlers import CloudLoggingHandler from google.cloud.logging_v2.handlers.transports import SyncTransport from google.cloud.logging_v2 import client from google.cloud.logging_v2.resource import Resource from test_utils.retry import RetryErrors from test_utils.retry import RetryResult from test_utils.system import unique_resource_id _RESOURCE_ID = unique_resource_id("-") DEFAULT_FILTER = "logName:syslog AND severity>=INFO" DEFAULT_DESCRIPTION = "System testing" retry_429 = RetryErrors(TooManyRequests) def _consume_entries(logger): """Consume all log entries from logger iterator. :type logger: :class:`~google.cloud.logging.logger.Logger` :param logger: A Logger containing entries. :rtype: list :returns: List of all entries consumed. """ return list(logger.list_entries()) def _list_entries(logger):
CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX TABLE_ID = "tabl-1" + UNIQUE_SUFFIX ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY LOCATION_ID = "us-central1-f" ALT_LOCATION_ID = "us-central1-a" PRODUCTION = enums.Instance.Type.PRODUCTION SERVER_NODES = 3 STORAGE_TYPE = enums.StorageType.SSD LABEL_KEY = u"python-snippet" LABEL_STAMP = (datetime.datetime.utcnow().replace( microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S")) LABELS = {LABEL_KEY: str(LABEL_STAMP)} INSTANCES_TO_DELETE = [] retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) retry_504 = RetryErrors(DeadlineExceeded, max_tries=4) class Config(object): """Run-time configuration to be modified at set-up. This is a mutable stand-in to allow test set-up to modify global state. """ CLIENT = None INSTANCE = None TABLE = None