def tearDown(self): retry = RetryErrors(NotFound, max_tries=10) for doomed in self.to_delete: retry(doomed.delete)() logging.getLogger().handlers = self._handlers_cache[:]
SERVER_NODES = 3 STORAGE_TYPE = enums.StorageType.SSD LABEL_KEY = u"python-snippet" LABEL_STAMP = (datetime.datetime.utcnow().replace( microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S")) LABELS = {LABEL_KEY: str(LABEL_STAMP)} COLUMN_FAMILY_ID = "col_fam_id1" COL_NAME1 = b"col-name1" CELL_VAL1 = b"cell-val" ROW_KEY1 = b"row_key_id1" COLUMN_FAMILY_ID2 = "col_fam_id2" COL_NAME2 = b"col-name2" CELL_VAL2 = b"cell-val2" ROW_KEY2 = b"row_key_id2" retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) class Config(object): """Run-time configuration to be modified at set-up. This is a mutable stand-in to allow test set-up to modify global state. """ CLIENT = None INSTANCE = None TABLE = None def setup_module():
from google.cloud.storage._helpers import _base64_md5hash from test_utils.retry import RetryErrors from test_utils.system import unique_resource_id USER_PROJECT = os.environ.get('GOOGLE_CLOUD_TESTS_USER_PROJECT') def _bad_copy(bad_request): """Predicate: pass only exceptions for a failed copyTo.""" err_msg = bad_request.message return (err_msg.startswith('No file found in request. (POST') and 'copyTo' in err_msg) retry_429 = RetryErrors(exceptions.TooManyRequests) retry_429_503 = RetryErrors( [exceptions.TooManyRequests, exceptions.ServiceUnavailable]) retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) def _empty_bucket(bucket): """Empty a bucket of all existing blobs. This accounts (partially) for the eventual consistency of the list blobs API call. """ for blob in bucket.list_blobs(): try: blob.delete() except exceptions.NotFound: # eventual consistency
from google.cloud.exceptions import NotFound from google.cloud.exceptions import TooManyRequests import google.cloud.logging import google.cloud.logging.handlers.handlers from google.cloud.logging.handlers.handlers import CloudLoggingHandler from google.cloud.logging.handlers.transports import SyncTransport from google.cloud.logging import client from test_utils.retry import RetryErrors from test_utils.retry import RetryResult from test_utils.system import unique_resource_id _RESOURCE_ID = unique_resource_id('-') DEFAULT_FILTER = 'logName:syslog AND severity>=INFO' DEFAULT_DESCRIPTION = 'System testing' retry_429 = RetryErrors(TooManyRequests) def _retry_on_unavailable(exc): """Retry only errors whose status code is 'UNAVAILABLE'. :type exc: :class:`~google.gax.errors.GaxError` :param exc: The exception that was caught. :rtype: bool :returns: Boolean indicating if the exception was UNAVAILABLE. """ return exc_to_code(exc) == StatusCode.UNAVAILABLE def _consume_entries(logger):
def tearDownModule(): retry = RetryErrors(exceptions.Conflict) retry(Config.TEST_BUCKET.delete)(force=True)
CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX TABLE_ID = "tabl-1" + UNIQUE_SUFFIX ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY LOCATION_ID = "us-central1-f" ALT_LOCATION_ID = "us-central1-a" PRODUCTION = enums.Instance.Type.PRODUCTION SERVER_NODES = 3 STORAGE_TYPE = enums.StorageType.SSD LABEL_KEY = u"python-snippet" LABEL_STAMP = (datetime.datetime.utcnow().replace( microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S")) LABELS = {LABEL_KEY: str(LABEL_STAMP)} INSTANCES_TO_DELETE = [] retry_429 = RetryErrors(TooManyRequests, max_tries=9) retry_504 = RetryErrors(DeadlineExceeded, max_tries=4) class Config(object): """Run-time configuration to be modified at set-up. This is a mutable stand-in to allow test set-up to modify global state. """ CLIENT = None INSTANCE = None TABLE = None
def retry_failures(decorated): # no-op wrapped = RetryErrors(AssertionError)(decorated) wrapped.__wrapped__ = decorated return wrapped
def tearDownModule(): # 409 Conflict if the bucket is full. # 429 Too Many Requests in case API requests rate-limited. bucket_retry = RetryErrors( (exceptions.TooManyRequests, exceptions.Conflict)) bucket_retry(VisionSystemTestBase.test_bucket.delete)(force=True)
from google.cloud.environment_vars import PUBSUB_EMULATOR from google.cloud.exceptions import Conflict from google.cloud.pubsub import client from test_utils.retry import RetryInstanceState from test_utils.retry import RetryResult from test_utils.retry import RetryErrors from test_utils.system import EmulatorCreds from test_utils.system import unique_resource_id def _unavailable(exc): return exc_to_code(exc) == StatusCode.UNAVAILABLE retry_unavailable = RetryErrors(GaxError, _unavailable) class Config(object): """Run-time configuration to be modified at set-up. This is a mutable stand-in to allow test set-up to modify global state. """ CLIENT = None IN_EMULATOR = False def setUpModule(): Config.IN_EMULATOR = os.getenv(PUBSUB_EMULATOR) is not None if Config.IN_EMULATOR:
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import six from google.api_core import exceptions from test_utils.retry import RetryErrors from test_utils.retry import RetryInstanceState from test_utils.system import unique_resource_id retry_429 = RetryErrors(exceptions.TooManyRequests) retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10) retry_429_503 = RetryErrors( [exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=10) # Work around https://github.com/googleapis/python-test-utils/issues/36 if six.PY3: retry_failures = RetryErrors(AssertionError) else: def retry_failures(decorated): # no-op wrapped = RetryErrors(AssertionError)(decorated) wrapped.__wrapped__ = decorated return wrapped
# See the License for the specific language governing permissions and # limitations under the License. import unittest from google.cloud.exceptions import BadRequest from google.cloud.exceptions import InternalServerError from google.cloud.exceptions import NotFound from google.cloud.exceptions import ServiceUnavailable from google.cloud import monitoring from test_utils.retry import RetryErrors from test_utils.retry import RetryResult from test_utils.system import unique_resource_id retry_404 = RetryErrors(NotFound, max_tries=5) retry_404_500 = RetryErrors((NotFound, InternalServerError)) retry_500 = RetryErrors(InternalServerError) retry_503 = RetryErrors(ServiceUnavailable) class TestMonitoring(unittest.TestCase): def test_fetch_metric_descriptor(self): METRIC_TYPE = ( 'pubsub.googleapis.com/topic/send_message_operation_count') METRIC_KIND = monitoring.MetricKind.DELTA VALUE_TYPE = monitoring.ValueType.INT64 client = monitoring.Client() descriptor = client.fetch_metric_descriptor(METRIC_TYPE)
def setup_instance(): if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials Config.CLIENT = Client( project=PROJECT_ID, credentials=AnonymousCredentials() ) else: Config.CLIENT = Client() retry = RetryErrors(exceptions.ServiceUnavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances # Delete test instances that are older than an hour. cutoff = int(time.time()) - 1 * 60 * 60 instance_pbs = Config.CLIENT.list_instances( "labels.python-spanner-systests:true" ) for instance_pb in instance_pbs: instance = Instance.from_pb(instance_pb, Config.CLIENT) if "created" not in instance.labels: continue create_time = int(instance.labels["created"]) if create_time > cutoff: continue if not USE_EMULATOR: # Instance cannot be deleted while backups exist. for backup_pb in instance.list_backups(): backup = Backup.from_pb(backup_pb, instance) backup.delete() instance.delete() if CREATE_INSTANCE: if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. configs = [config for config in configs if "-us-" in config.name] if not configs: raise ValueError("List instance configs failed in module set up.") Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name create_time = str(int(time.time())) labels = {"django-spanner-systests": "true", "created": create_time} Config.INSTANCE = Config.CLIENT.instance( INSTANCE_ID, config_name, labels=labels ) if not Config.INSTANCE.exists(): created_op = Config.INSTANCE.create() created_op.result( SPANNER_OPERATION_TIMEOUT_IN_SECONDS ) # block until completion else: Config.INSTANCE.reload() else: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload()
CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX TABLE_ID = "tabl-1" + UNIQUE_SUFFIX ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY LOCATION_ID = "us-central1-f" ALT_LOCATION_ID = "us-central1-a" PRODUCTION = enums.Instance.Type.PRODUCTION SERVER_NODES = 3 STORAGE_TYPE = enums.StorageType.SSD LABEL_KEY = u"python-snippet" LABEL_STAMP = (datetime.datetime.utcnow().replace( microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S")) LABELS = {LABEL_KEY: str(LABEL_STAMP)} INSTANCES_TO_DELETE = [] retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) retry_504 = RetryErrors(DeadlineExceeded, max_tries=4) class Config(object): """Run-time configuration to be modified at set-up. This is a mutable stand-in to allow test set-up to modify global state. """ CLIENT = None INSTANCE = None TABLE = None
with open(json_filename, 'r') as schema_file: return _parse_schema_resource(json.load(schema_file)) def _rate_limit_exceeded(forbidden): """Predicate: pass only exceptions with 'rateLimitExceeded' as reason.""" return any(error['reason'] == 'rateLimitExceeded' for error in forbidden._errors) # We need to wait to stay within the rate limits. # The alternative outcome is a 403 Forbidden response from upstream, which # they return instead of the more appropriate 429. # See https://cloud.google.com/bigquery/quota-policy retry_403 = RetryErrors(Forbidden, error_predicate=_rate_limit_exceeded) class Config(object): """Run-time configuration to be modified at set-up. This is a mutable stand-in to allow test set-up to modify global state. """ CLIENT = None CURSOR = None def setUpModule(): Config.CLIENT = bigquery.Client() Config.CURSOR = dbapi.connect(Config.CLIENT).cursor()
def delete_bucket(bucket): errors = (exceptions.Conflict, exceptions.TooManyRequests) retry = RetryErrors(errors, max_tries=15) retry(empty_bucket)(bucket) retry(bucket.delete)(force=True)
CLIENT = None INSTANCE = None INSTANCE_DATA = None CLUSTER = None CLUSTER_DATA = None IN_EMULATOR = False def _retry_on_unavailable(exc): """Retry only errors whose status code is 'UNAVAILABLE'.""" from grpc import StatusCode return exc.code() == StatusCode.UNAVAILABLE retry_429 = RetryErrors(TooManyRequests, max_tries=9) def setUpModule(): from google.cloud.exceptions import GrpcRendezvous from google.cloud.bigtable.enums import Instance # See: https://github.com/googleapis/google-cloud-python/issues/5928 interfaces = table_admin_config.config["interfaces"] iface_config = interfaces["google.bigtable.admin.v2.BigtableTableAdmin"] methods = iface_config["methods"] create_table = methods["CreateTable"] create_table["timeout_millis"] = 90000 Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
def tearDownModule(): errors = (exceptions.Conflict, exceptions.TooManyRequests) retry = RetryErrors(errors, max_tries=6) retry(Config.TEST_BUCKET.delete)(force=True)
from google.cloud import exceptions from google.cloud import storage from google.cloud.storage._helpers import _base64_md5hash from test_utils.retry import RetryErrors from test_utils.system import unique_resource_id def _bad_copy(bad_request): """Predicate: pass only exceptions for a failed copyTo.""" err_msg = bad_request.message return (err_msg.startswith('No file found in request. (POST') and 'copyTo' in err_msg) retry_429 = RetryErrors(exceptions.TooManyRequests) retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) def _empty_bucket(bucket): """Empty a bucket of all existing blobs. This accounts (partially) for the eventual consistency of the list blobs API call. """ for blob in bucket.list_blobs(): try: blob.delete() except exceptions.NotFound: # eventual consistency pass
SCHEMA = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), ] ROWS = [ ("Phred Phlyntstone", 32), ("Bharney Rhubble", 33), ("Wylma Phlyntstone", 29), ("Bhettye Rhubble", 27), ] QUERY = ("SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " 'WHERE state = "TX"') retry_429 = RetryErrors(TooManyRequests) retry_storage_errors = RetryErrors( (TooManyRequests, InternalServerError, ServiceUnavailable)) @pytest.fixture(scope="module") def client(): return bigquery.Client() @pytest.fixture def to_delete(client): doomed = [] yield doomed for item in doomed: if isinstance(item, (bigquery.Dataset, bigquery.DatasetReference)):
def tearDownModule(): # 409 Conflict if the bucket is full. # 429 Too Many Requests in case API requests rate-limited. bucket_retry = RetryErrors( (exceptions.TooManyRequests, exceptions.Conflict)) bucket_retry(Config.TEST_BUCKET.delete)(force=True)
import os import uuid from google.api_core import exceptions from test_utils.retry import RetryErrors from tableadmin import create_table from tableadmin import delete_table from tableadmin import run_table_operations PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] TABLE_ID_FORMAT = "tableadmin-test-{}" retry_429_503 = RetryErrors(exceptions.TooManyRequests, exceptions.ServiceUnavailable) def test_run_table_operations(capsys): table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, table_id) out, _ = capsys.readouterr() assert "Creating the " + table_id + " table." in out assert "Listing tables in current project." in out assert "Creating column family cf1 with with MaxAge GC Rule" in out assert "Created column family cf1 with MaxAge GC Rule." in out assert "Created column family cf2 with Max Versions GC Rule." in out assert "Created column family cf3 with Union GC rule" in out assert "Created column family cf4 with Intersection GC rule." in out