def tearDown(self):
retry = RetryErrors(NotFound, max_tries=10)
for doomed in self.to_delete:
retry(doomed.delete)()
logging.getLogger().handlers = self._handlers_cache[:]
SERVER_NODES = 3
STORAGE_TYPE = enums.StorageType.SSD
LABEL_KEY = u"python-snippet"
LABEL_STAMP = (datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S"))
LABELS = {LABEL_KEY: str(LABEL_STAMP)}
COLUMN_FAMILY_ID = "col_fam_id1"
COL_NAME1 = b"col-name1"
CELL_VAL1 = b"cell-val"
ROW_KEY1 = b"row_key_id1"
COLUMN_FAMILY_ID2 = "col_fam_id2"
COL_NAME2 = b"col-name2"
CELL_VAL2 = b"cell-val2"
ROW_KEY2 = b"row_key_id2"
retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9)
class Config(object):
"""Run-time configuration to be modified at set-up.
This is a mutable stand-in to allow test set-up to modify
global state.
"""
CLIENT = None
INSTANCE = None
TABLE = None
def setup_module():
from google.cloud.storage._helpers import _base64_md5hash
from test_utils.retry import RetryErrors
from test_utils.system import unique_resource_id
USER_PROJECT = os.environ.get('GOOGLE_CLOUD_TESTS_USER_PROJECT')
def _bad_copy(bad_request):
"""Predicate: pass only exceptions for a failed copyTo."""
err_msg = bad_request.message
return (err_msg.startswith('No file found in request. (POST')
and 'copyTo' in err_msg)
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429_503 = RetryErrors(
[exceptions.TooManyRequests, exceptions.ServiceUnavailable])
retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
def _empty_bucket(bucket):
"""Empty a bucket of all existing blobs.
This accounts (partially) for the eventual consistency of the
list blobs API call.
"""
for blob in bucket.list_blobs():
try:
blob.delete()
except exceptions.NotFound: # eventual consistency
from google.cloud.exceptions import NotFound
from google.cloud.exceptions import TooManyRequests
import google.cloud.logging
import google.cloud.logging.handlers.handlers
from google.cloud.logging.handlers.handlers import CloudLoggingHandler
from google.cloud.logging.handlers.transports import SyncTransport
from google.cloud.logging import client
from test_utils.retry import RetryErrors
from test_utils.retry import RetryResult
from test_utils.system import unique_resource_id
_RESOURCE_ID = unique_resource_id('-')
DEFAULT_FILTER = 'logName:syslog AND severity>=INFO'
DEFAULT_DESCRIPTION = 'System testing'
retry_429 = RetryErrors(TooManyRequests)
def _retry_on_unavailable(exc):
"""Retry only errors whose status code is 'UNAVAILABLE'.
:type exc: :class:`~google.gax.errors.GaxError`
:param exc: The exception that was caught.
:rtype: bool
:returns: Boolean indicating if the exception was UNAVAILABLE.
"""
return exc_to_code(exc) == StatusCode.UNAVAILABLE
def _consume_entries(logger):
def tearDownModule():
retry = RetryErrors(exceptions.Conflict)
retry(Config.TEST_BUCKET.delete)(force=True)
CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX
APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX
TABLE_ID = "tabl-1" + UNIQUE_SUFFIX
ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY
LOCATION_ID = "us-central1-f"
ALT_LOCATION_ID = "us-central1-a"
PRODUCTION = enums.Instance.Type.PRODUCTION
SERVER_NODES = 3
STORAGE_TYPE = enums.StorageType.SSD
LABEL_KEY = u"python-snippet"
LABEL_STAMP = (datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S"))
LABELS = {LABEL_KEY: str(LABEL_STAMP)}
INSTANCES_TO_DELETE = []
retry_429 = RetryErrors(TooManyRequests, max_tries=9)
retry_504 = RetryErrors(DeadlineExceeded, max_tries=4)
class Config(object):
"""Run-time configuration to be modified at set-up.
This is a mutable stand-in to allow test set-up to modify
global state.
"""
CLIENT = None
INSTANCE = None
TABLE = None
def retry_failures(decorated): # no-op
wrapped = RetryErrors(AssertionError)(decorated)
wrapped.__wrapped__ = decorated
return wrapped
def tearDownModule():
# 409 Conflict if the bucket is full.
# 429 Too Many Requests in case API requests rate-limited.
bucket_retry = RetryErrors(
(exceptions.TooManyRequests, exceptions.Conflict))
bucket_retry(VisionSystemTestBase.test_bucket.delete)(force=True)
from google.cloud.environment_vars import PUBSUB_EMULATOR
from google.cloud.exceptions import Conflict
from google.cloud.pubsub import client
from test_utils.retry import RetryInstanceState
from test_utils.retry import RetryResult
from test_utils.retry import RetryErrors
from test_utils.system import EmulatorCreds
from test_utils.system import unique_resource_id
def _unavailable(exc):
return exc_to_code(exc) == StatusCode.UNAVAILABLE
retry_unavailable = RetryErrors(GaxError, _unavailable)
class Config(object):
"""Run-time configuration to be modified at set-up.
This is a mutable stand-in to allow test set-up to modify
global state.
"""
CLIENT = None
IN_EMULATOR = False
def setUpModule():
Config.IN_EMULATOR = os.getenv(PUBSUB_EMULATOR) is not None
if Config.IN_EMULATOR:
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import six
from google.api_core import exceptions
from test_utils.retry import RetryErrors
from test_utils.retry import RetryInstanceState
from test_utils.system import unique_resource_id
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10)
retry_429_503 = RetryErrors(
[exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=10)
# Work around https://github.com/googleapis/python-test-utils/issues/36
if six.PY3:
retry_failures = RetryErrors(AssertionError)
else:
def retry_failures(decorated): # no-op
wrapped = RetryErrors(AssertionError)(decorated)
wrapped.__wrapped__ = decorated
return wrapped
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from google.cloud.exceptions import BadRequest
from google.cloud.exceptions import InternalServerError
from google.cloud.exceptions import NotFound
from google.cloud.exceptions import ServiceUnavailable
from google.cloud import monitoring
from test_utils.retry import RetryErrors
from test_utils.retry import RetryResult
from test_utils.system import unique_resource_id
retry_404 = RetryErrors(NotFound, max_tries=5)
retry_404_500 = RetryErrors((NotFound, InternalServerError))
retry_500 = RetryErrors(InternalServerError)
retry_503 = RetryErrors(ServiceUnavailable)
class TestMonitoring(unittest.TestCase):
def test_fetch_metric_descriptor(self):
METRIC_TYPE = (
'pubsub.googleapis.com/topic/send_message_operation_count')
METRIC_KIND = monitoring.MetricKind.DELTA
VALUE_TYPE = monitoring.ValueType.INT64
client = monitoring.Client()
descriptor = client.fetch_metric_descriptor(METRIC_TYPE)
def setup_instance():
if USE_EMULATOR:
from google.auth.credentials import AnonymousCredentials
Config.CLIENT = Client(
project=PROJECT_ID, credentials=AnonymousCredentials()
)
else:
Config.CLIENT = Client()
retry = RetryErrors(exceptions.ServiceUnavailable)
configs = list(retry(Config.CLIENT.list_instance_configs)())
instances = retry(_list_instances)()
EXISTING_INSTANCES[:] = instances
# Delete test instances that are older than an hour.
cutoff = int(time.time()) - 1 * 60 * 60
instance_pbs = Config.CLIENT.list_instances(
"labels.python-spanner-systests:true"
)
for instance_pb in instance_pbs:
instance = Instance.from_pb(instance_pb, Config.CLIENT)
if "created" not in instance.labels:
continue
create_time = int(instance.labels["created"])
if create_time > cutoff:
continue
if not USE_EMULATOR:
# Instance cannot be deleted while backups exist.
for backup_pb in instance.list_backups():
backup = Backup.from_pb(backup_pb, instance)
backup.delete()
instance.delete()
if CREATE_INSTANCE:
if not USE_EMULATOR:
# Defend against back-end returning configs for regions we aren't
# actually allowed to use.
configs = [config for config in configs if "-us-" in config.name]
if not configs:
raise ValueError("List instance configs failed in module set up.")
Config.INSTANCE_CONFIG = configs[0]
config_name = configs[0].name
create_time = str(int(time.time()))
labels = {"django-spanner-systests": "true", "created": create_time}
Config.INSTANCE = Config.CLIENT.instance(
INSTANCE_ID, config_name, labels=labels
)
if not Config.INSTANCE.exists():
created_op = Config.INSTANCE.create()
created_op.result(
SPANNER_OPERATION_TIMEOUT_IN_SECONDS
) # block until completion
else:
Config.INSTANCE.reload()
else:
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
Config.INSTANCE.reload()
CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX
APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX
TABLE_ID = "tabl-1" + UNIQUE_SUFFIX
ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY
LOCATION_ID = "us-central1-f"
ALT_LOCATION_ID = "us-central1-a"
PRODUCTION = enums.Instance.Type.PRODUCTION
SERVER_NODES = 3
STORAGE_TYPE = enums.StorageType.SSD
LABEL_KEY = u"python-snippet"
LABEL_STAMP = (datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=UTC).strftime("%Y-%m-%dt%H-%M-%S"))
LABELS = {LABEL_KEY: str(LABEL_STAMP)}
INSTANCES_TO_DELETE = []
retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9)
retry_504 = RetryErrors(DeadlineExceeded, max_tries=4)
class Config(object):
"""Run-time configuration to be modified at set-up.
This is a mutable stand-in to allow test set-up to modify
global state.
"""
CLIENT = None
INSTANCE = None
TABLE = None
with open(json_filename, 'r') as schema_file:
return _parse_schema_resource(json.load(schema_file))
def _rate_limit_exceeded(forbidden):
"""Predicate: pass only exceptions with 'rateLimitExceeded' as reason."""
return any(error['reason'] == 'rateLimitExceeded'
for error in forbidden._errors)
# We need to wait to stay within the rate limits.
# The alternative outcome is a 403 Forbidden response from upstream, which
# they return instead of the more appropriate 429.
# See https://cloud.google.com/bigquery/quota-policy
retry_403 = RetryErrors(Forbidden, error_predicate=_rate_limit_exceeded)
class Config(object):
"""Run-time configuration to be modified at set-up.
This is a mutable stand-in to allow test set-up to modify
global state.
"""
CLIENT = None
CURSOR = None
def setUpModule():
Config.CLIENT = bigquery.Client()
Config.CURSOR = dbapi.connect(Config.CLIENT).cursor()
def delete_bucket(bucket):
errors = (exceptions.Conflict, exceptions.TooManyRequests)
retry = RetryErrors(errors, max_tries=15)
retry(empty_bucket)(bucket)
retry(bucket.delete)(force=True)
CLIENT = None
INSTANCE = None
INSTANCE_DATA = None
CLUSTER = None
CLUSTER_DATA = None
IN_EMULATOR = False
def _retry_on_unavailable(exc):
"""Retry only errors whose status code is 'UNAVAILABLE'."""
from grpc import StatusCode
return exc.code() == StatusCode.UNAVAILABLE
retry_429 = RetryErrors(TooManyRequests, max_tries=9)
def setUpModule():
from google.cloud.exceptions import GrpcRendezvous
from google.cloud.bigtable.enums import Instance
# See: https://github.com/googleapis/google-cloud-python/issues/5928
interfaces = table_admin_config.config["interfaces"]
iface_config = interfaces["google.bigtable.admin.v2.BigtableTableAdmin"]
methods = iface_config["methods"]
create_table = methods["CreateTable"]
create_table["timeout_millis"] = 90000
Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
def tearDownModule():
errors = (exceptions.Conflict, exceptions.TooManyRequests)
retry = RetryErrors(errors, max_tries=6)
retry(Config.TEST_BUCKET.delete)(force=True)
from google.cloud import exceptions
from google.cloud import storage
from google.cloud.storage._helpers import _base64_md5hash
from test_utils.retry import RetryErrors
from test_utils.system import unique_resource_id
def _bad_copy(bad_request):
"""Predicate: pass only exceptions for a failed copyTo."""
err_msg = bad_request.message
return (err_msg.startswith('No file found in request. (POST')
and 'copyTo' in err_msg)
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
def _empty_bucket(bucket):
"""Empty a bucket of all existing blobs.
This accounts (partially) for the eventual consistency of the
list blobs API call.
"""
for blob in bucket.list_blobs():
try:
blob.delete()
except exceptions.NotFound: # eventual consistency
pass
SCHEMA = [
bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
]
ROWS = [
("Phred Phlyntstone", 32),
("Bharney Rhubble", 33),
("Wylma Phlyntstone", 29),
("Bhettye Rhubble", 27),
]
QUERY = ("SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` "
'WHERE state = "TX"')
retry_429 = RetryErrors(TooManyRequests)
retry_storage_errors = RetryErrors(
(TooManyRequests, InternalServerError, ServiceUnavailable))
@pytest.fixture(scope="module")
def client():
return bigquery.Client()
@pytest.fixture
def to_delete(client):
doomed = []
yield doomed
for item in doomed:
if isinstance(item, (bigquery.Dataset, bigquery.DatasetReference)):
def tearDownModule():
# 409 Conflict if the bucket is full.
# 429 Too Many Requests in case API requests rate-limited.
bucket_retry = RetryErrors(
(exceptions.TooManyRequests, exceptions.Conflict))
bucket_retry(Config.TEST_BUCKET.delete)(force=True)
import os
import uuid
from google.api_core import exceptions
from test_utils.retry import RetryErrors
from tableadmin import create_table
from tableadmin import delete_table
from tableadmin import run_table_operations
PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"]
TABLE_ID_FORMAT = "tableadmin-test-{}"
retry_429_503 = RetryErrors(exceptions.TooManyRequests,
exceptions.ServiceUnavailable)
def test_run_table_operations(capsys):
table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8])
retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, table_id)
out, _ = capsys.readouterr()
assert "Creating the " + table_id + " table." in out
assert "Listing tables in current project." in out
assert "Creating column family cf1 with with MaxAge GC Rule" in out
assert "Created column family cf1 with MaxAge GC Rule." in out
assert "Created column family cf2 with Max Versions GC Rule." in out
assert "Created column family cf3 with Union GC rule" in out
assert "Created column family cf4 with Intersection GC rule." in out