def test_subscription_iam_policy(self):
from google.cloud.pubsub.iam import PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-sub-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
# Retry / backoff up to 7 seconds (1 + 2 + 4)
retry = RetryResult(lambda result: result, max_tries=4)
retry(topic.exists)()
self.assertTrue(topic.exists())
self.to_delete.append(topic)
SUB_NAME = 'test-sub-iam-policy-sub' + unique_resource_id('-')
subscription = topic.subscription(SUB_NAME)
subscription.create()
# Retry / backoff up to 7 seconds (1 + 2 + 4)
retry = RetryResult(lambda result: result, max_tries=4)
retry(subscription.exists)()
self.assertTrue(subscription.exists())
self.to_delete.insert(0, subscription)
if subscription.check_iam_permissions(
[PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY]):
policy = subscription.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = subscription.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_fetch_delete_subscription_w_deleted_topic(self):
TO_DELETE = 'delete-me' + unique_resource_id('-')
ORPHANED = 'orphaned' + unique_resource_id('-')
topic = Config.CLIENT.topic(TO_DELETE)
topic.create()
subscription = topic.subscription(ORPHANED)
subscription.create()
topic.delete()
all_subs = []
token = None
while True:
subs, token = Config.CLIENT.list_subscriptions(page_token=token)
all_subs.extend(subs)
if token is None:
break
created = [subscription for subscription in all_subs
if subscription.name == ORPHANED]
self.assertEqual(len(created), 1)
orphaned = created[0]
def _no_topic(instance):
return instance.topic is None
retry = RetryInstanceState(_no_topic, max_tries=6)
retry(orphaned.reload)()
self.assertTrue(orphaned.topic is None)
orphaned.delete()
def test_list_subscriptions(self):
TOPIC_NAME = 'list-sub' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME)
topic.create()
self.to_delete.append(topic)
empty, _ = topic.list_subscriptions()
self.assertEqual(len(empty), 0)
subscriptions_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for subscription_name in subscriptions_to_create:
subscription = topic.subscription(subscription_name)
subscription.create()
self.to_delete.append(subscription)
# Retrieve the subscriptions.
def _all_created(result):
return len(result[0]) == len(subscriptions_to_create)
retry = RetryResult(_all_created)
all_subscriptions, _ = retry(topic.list_subscriptions)()
created = [subscription for subscription in all_subscriptions
if subscription.name in subscriptions_to_create]
self.assertEqual(len(created), len(subscriptions_to_create))
def test_list_tables(self):
dataset = Config.CLIENT.dataset(DATASET_NAME)
self.assertFalse(dataset.exists())
dataset.create()
self.to_delete.append(dataset)
# Retrieve tables before any are created for the dataset.
all_tables, token = dataset.list_tables()
self.assertEqual(all_tables, [])
self.assertEqual(token, None)
# Insert some tables to be listed.
tables_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
full_name = bigquery.SchemaField('full_name', 'STRING',
mode='REQUIRED')
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
for table_name in tables_to_create:
table = dataset.table(table_name, schema=[full_name, age])
table.create()
self.to_delete.insert(0, table)
# Retrieve the tables.
all_tables, token = dataset.list_tables()
self.assertTrue(token is None)
created = [table for table in all_tables
if (table.name in tables_to_create and
table.dataset_name == DATASET_NAME)]
self.assertEqual(len(created), len(tables_to_create))
def test_subscription_iam_policy(self):
from google.cloud.pubsub.iam import PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-sub-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
# Retry / backoff up to 7 seconds (1 + 2 + 4)
retry = RetryResult(lambda result: result, max_tries=4)
retry(topic.exists)()
self.to_delete.append(topic)
SUB_NAME = 'test-sub-iam-policy-sub' + unique_resource_id('-')
subscription = topic.subscription(SUB_NAME)
subscription.create()
# Retry / backoff up to 7 seconds (1 + 2 + 4)
retry = RetryResult(lambda result: result, max_tries=4)
retry(subscription.exists)()
self.to_delete.insert(0, subscription)
if subscription.check_iam_permissions(
[PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY]):
policy = subscription.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = subscription.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_fetch_delete_subscription_w_deleted_topic(self):
TO_DELETE = 'delete-me' + unique_resource_id('-')
ORPHANED = 'orphaned' + unique_resource_id('-')
topic = Config.CLIENT.topic(TO_DELETE)
topic.create()
subscription = topic.subscription(ORPHANED)
subscription.create()
topic.delete()
all_subs = []
token = None
while True:
subs, token = Config.CLIENT.list_subscriptions(page_token=token)
all_subs.extend(subs)
if token is None:
break
created = [
subscription for subscription in all_subs
if subscription.name == ORPHANED
]
self.assertEqual(len(created), 1)
orphaned = created[0]
self.assertTrue(orphaned.topic is None)
orphaned.delete()
def test_subscription_iam_policy(self):
from gcloud.pubsub.iam import PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-sub-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
SUB_NAME = 'test-sub-iam-policy-sub' + unique_resource_id('-')
subscription = topic.subscription(SUB_NAME)
subscription.create()
count = 5
while count > 0 and not subscription.exists():
time.sleep(1)
count -= 1
self.assertTrue(subscription.exists())
self.to_delete.insert(0, subscription)
if subscription.check_iam_permissions(
[PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY]):
policy = subscription.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = subscription.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_list_tables(self):
dataset = Config.CLIENT.dataset(DATASET_NAME)
self.assertFalse(dataset.exists())
dataset.create()
self.to_delete.append(dataset)
# Retrieve tables before any are created for the dataset.
all_tables, token = dataset.list_tables()
self.assertEqual(all_tables, [])
self.assertEqual(token, None)
# Insert some tables to be listed.
tables_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
full_name = bigquery.SchemaField('full_name', 'STRING',
mode='REQUIRED')
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
for table_name in tables_to_create:
table = dataset.table(table_name, schema=[full_name, age])
table.create()
self.to_delete.insert(0, table)
# Retrieve the tables.
all_tables, token = dataset.list_tables()
self.assertTrue(token is None)
created = [table for table in all_tables
if (table.name in tables_to_create and
table.dataset_name == DATASET_NAME)]
self.assertEqual(len(created), len(tables_to_create))
def test_list_subscriptions(self):
TOPIC_NAME = 'list-sub' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME)
topic.create()
self.to_delete.append(topic)
empty = _consume_subscriptions(topic)
self.assertEqual(len(empty), 0)
subscriptions_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for subscription_name in subscriptions_to_create:
subscription = topic.subscription(subscription_name)
subscription.create()
self.to_delete.append(subscription)
# Retrieve the subscriptions.
def _all_created(result):
return len(result) == len(subscriptions_to_create)
retry = RetryResult(_all_created)
all_subscriptions = retry(_consume_subscriptions)(topic)
created = [subscription for subscription in all_subscriptions
if subscription.name in subscriptions_to_create]
self.assertEqual(len(created), len(subscriptions_to_create))
def test_subscription_iam_policy(self):
from gcloud.pubsub.iam import PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-sub-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
SUB_NAME = 'test-sub-iam-policy-sub' + unique_resource_id('-')
subscription = topic.subscription(SUB_NAME)
subscription.create()
count = 5
while count > 0 and not subscription.exists():
time.sleep(1)
count -= 1
self.assertTrue(subscription.exists())
self.to_delete.insert(0, subscription)
if subscription.check_iam_permissions(
[PUBSUB_SUBSCRIPTIONS_GET_IAM_POLICY]):
policy = subscription.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = subscription.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_fetch_delete_subscription_w_deleted_topic(self):
from google.cloud.iterator import MethodIterator
TO_DELETE = 'delete-me' + unique_resource_id('-')
ORPHANED = 'orphaned' + unique_resource_id('-')
topic = Config.CLIENT.topic(TO_DELETE)
topic.create()
subscription = topic.subscription(ORPHANED)
subscription.create()
topic.delete()
def _fetch():
return list(MethodIterator(Config.CLIENT.list_subscriptions))
def _found_orphan(result):
names = [subscription.name for subscription in result]
return ORPHANED in names
retry_until_found_orphan = RetryResult(_found_orphan)
all_subs = retry_until_found_orphan(_fetch)()
created = [subscription for subscription in all_subs
if subscription.name == ORPHANED]
self.assertEqual(len(created), 1)
orphaned = created[0]
def _no_topic(instance):
return instance.topic is None
retry_until_no_topic = RetryInstanceState(_no_topic)
retry_until_no_topic(orphaned.reload)()
self.assertTrue(orphaned.topic is None)
orphaned.delete()
def test_fetch_delete_subscription_w_deleted_topic(self):
from google.cloud.iterator import MethodIterator
TO_DELETE = 'delete-me' + unique_resource_id('-')
ORPHANED = 'orphaned' + unique_resource_id('-')
topic = Config.CLIENT.topic(TO_DELETE)
topic.create()
subscription = topic.subscription(ORPHANED)
subscription.create()
topic.delete()
def _fetch():
return list(MethodIterator(Config.CLIENT.list_subscriptions))
def _found_orphan(result):
names = [subscription.name for subscription in result]
return ORPHANED in names
retry_until_found_orphan = RetryResult(_found_orphan)
all_subs = retry_until_found_orphan(_fetch)()
created = [
subscription for subscription in all_subs
if subscription.name == ORPHANED
]
self.assertEqual(len(created), 1)
orphaned = created[0]
def _no_topic(instance):
return instance.topic is None
retry_until_no_topic = RetryInstanceState(_no_topic)
retry_until_no_topic(orphaned.reload)()
self.assertTrue(orphaned.topic is None)
orphaned.delete()
def test_create_subscription_defaults(self):
TOPIC_NAME = 'create-sub-def' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id('-')
subscription = topic.subscription(SUBSCRIPTION_NAME)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
self.assertTrue(subscription.exists())
self.assertEqual(subscription.name, SUBSCRIPTION_NAME)
self.assertIs(subscription.topic, topic)
def test_create_subscription_defaults(self):
TOPIC_NAME = 'create-sub-def' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id('-')
subscription = topic.subscription(SUBSCRIPTION_NAME)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
self.assertTrue(subscription.exists())
self.assertEqual(subscription.name, SUBSCRIPTION_NAME)
self.assertTrue(subscription.topic is topic)
def test_create_subscription_w_ack_deadline(self):
TOPIC_NAME = 'create-sub-ack' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id()
subscription = topic.subscription(SUBSCRIPTION_NAME, ack_deadline=120)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
self.assertTrue(subscription.exists())
self.assertEqual(subscription.name, SUBSCRIPTION_NAME)
self.assertEqual(subscription.ack_deadline, 120)
self.assertTrue(subscription.topic is topic)
def test_message_pull_mode_e2e(self):
topic = Config.CLIENT.topic(DEFAULT_TOPIC_NAME,
timestamp_messages=True)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id('-')
subscription = topic.subscription(SUBSCRIPTION_NAME)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
MESSAGE_1 = b'MESSAGE ONE'
MESSAGE_2 = b'MESSAGE ONE'
EXTRA_1 = 'EXTRA 1'
EXTRA_2 = 'EXTRA 2'
topic.publish(MESSAGE_1, extra=EXTRA_1)
topic.publish(MESSAGE_2, extra=EXTRA_2)
received = subscription.pull(max_messages=2)
ack_ids = [recv[0] for recv in received]
subscription.acknowledge(ack_ids)
messages = [recv[1] for recv in received]
def _by_timestamp(message):
return message.timestamp
message1, message2 = sorted(messages, key=_by_timestamp)
self.assertEqual(message1.data, MESSAGE_1)
self.assertEqual(message1.attributes['extra'], EXTRA_1)
self.assertEqual(message2.data, MESSAGE_2)
self.assertEqual(message2.attributes['extra'], EXTRA_2)
def setUpModule():
Config.CLIENT = storage.Client()
bucket_name = 'new' + unique_resource_id()
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
retry_429(Config.TEST_BUCKET.create)()
def setUpModule():
Config.CLIENT = storage.Client()
bucket_name = 'new' + unique_resource_id()
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
retry_429(Config.TEST_BUCKET.create)()
def test_list_buckets(self):
buckets_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
created_buckets = []
for bucket_name in buckets_to_create:
bucket = Config.CLIENT.create_bucket(bucket_name)
self.case_buckets_to_delete.append(bucket_name)
# Retrieve the buckets.
all_buckets = Config.CLIENT.list_buckets()
created_buckets = [bucket for bucket in all_buckets
if bucket.name in buckets_to_create]
self.assertEqual(len(created_buckets), len(buckets_to_create))
def test_create_bucket(self):
new_bucket_name = 'a-new-bucket' + unique_resource_id('-')
self.assertRaises(exceptions.NotFound, Config.CLIENT.get_bucket,
new_bucket_name)
created = Config.CLIENT.create_bucket(new_bucket_name)
self.case_buckets_to_delete.append(new_bucket_name)
self.assertEqual(created.name, new_bucket_name)
def test_create_bucket(self):
new_bucket_name = 'a-new-bucket' + unique_resource_id('-')
self.assertRaises(exceptions.NotFound,
Config.CLIENT.get_bucket, new_bucket_name)
created = Config.CLIENT.create_bucket(new_bucket_name)
self.case_buckets_to_delete.append(new_bucket_name)
self.assertEqual(created.name, new_bucket_name)
def test_message_pull_mode_e2e(self):
topic = Config.CLIENT.topic(DEFAULT_TOPIC_NAME,
timestamp_messages=True)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id('-')
subscription = topic.subscription(SUBSCRIPTION_NAME)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
MESSAGE_1 = b'MESSAGE ONE'
MESSAGE_2 = b'MESSAGE ONE'
EXTRA_1 = 'EXTRA 1'
EXTRA_2 = 'EXTRA 2'
topic.publish(MESSAGE_1, extra=EXTRA_1)
topic.publish(MESSAGE_2, extra=EXTRA_2)
received = subscription.pull(max_messages=2)
ack_ids = [recv[0] for recv in received]
subscription.acknowledge(ack_ids)
messages = [recv[1] for recv in received]
def _by_timestamp(message):
return message.timestamp
message1, message2 = sorted(messages, key=_by_timestamp)
self.assertEqual(message1.data, MESSAGE_1)
self.assertEqual(message1.attributes['extra'], EXTRA_1)
self.assertEqual(message2.data, MESSAGE_2)
self.assertEqual(message2.attributes['extra'], EXTRA_2)
def test_message_pull_mode_e2e(self):
import operator
TOPIC_NAME = 'message-e2e' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME,
timestamp_messages=True)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id('-')
subscription = topic.subscription(SUBSCRIPTION_NAME)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
MESSAGE_1 = b'MESSAGE ONE'
MESSAGE_2 = b'MESSAGE ONE'
EXTRA_1 = 'EXTRA 1'
EXTRA_2 = 'EXTRA 2'
topic.publish(MESSAGE_1, extra=EXTRA_1)
topic.publish(MESSAGE_2, extra=EXTRA_2)
class Hoover(object):
def __init__(self):
self.received = []
def done(self, *dummy):
return len(self.received) == 2
def suction(self):
with subscription.auto_ack(max_messages=2) as ack:
self.received.extend(ack.values())
hoover = Hoover()
retry = RetryInstanceState(hoover.done)
retry(hoover.suction)()
message1, message2 = sorted(hoover.received,
key=operator.attrgetter('timestamp'))
self.assertEqual(message1.data, MESSAGE_1)
self.assertEqual(message1.attributes['extra'], EXTRA_1)
self.assertIsNotNone(message1.service_timestamp)
self.assertEqual(message2.data, MESSAGE_2)
self.assertEqual(message2.attributes['extra'], EXTRA_2)
self.assertIsNotNone(message2.service_timestamp)
def setUpModule():
Config.CLIENT = vision.Client()
storage_client = storage.Client()
bucket_name = 'new' + unique_resource_id()
Config.TEST_BUCKET = storage_client.bucket(bucket_name)
# 429 Too Many Requests in case API requests rate-limited.
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429(Config.TEST_BUCKET.create)()
def test_create_topic(self):
topic_name = 'a-new-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
self.assertTrue(topic.exists())
self.assertEqual(topic.name, topic_name)
def test_list_buckets(self):
buckets_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
created_buckets = []
for bucket_name in buckets_to_create:
bucket = Config.CLIENT.bucket(bucket_name)
retry_429(bucket.create)()
self.case_buckets_to_delete.append(bucket_name)
# Retrieve the buckets.
all_buckets = Config.CLIENT.list_buckets()
created_buckets = [bucket for bucket in all_buckets
if bucket.name in buckets_to_create]
self.assertEqual(len(created_buckets), len(buckets_to_create))
def test_create_topic(self):
topic_name = 'a-new-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
self.assertTrue(topic.exists())
self.assertEqual(topic.name, topic_name)
def setUpModule():
Config.CLIENT = vision.Client()
storage_client = storage.Client()
bucket_name = 'new' + unique_resource_id()
Config.TEST_BUCKET = storage_client.bucket(bucket_name)
# 429 Too Many Requests in case API requests rate-limited.
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429(Config.TEST_BUCKET.create)()
def setUpModule():
_helpers.PROJECT = TESTS_PROJECT
Config.CLIENT = language.Client()
# Now create a bucket for GCS stored content.
storage_client = storage.Client()
bucket_name = 'new' + unique_resource_id()
Config.TEST_BUCKET = storage_client.bucket(bucket_name)
retry_429(Config.TEST_BUCKET.create)()
def test_list_topics(self):
topics_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for topic_name in topics_to_create:
topic = Config.CLIENT.topic(topic_name)
topic.create()
self.to_delete.append(topic)
# Retrieve the topics.
all_topics, _ = Config.CLIENT.list_topics()
created = [topic for topic in all_topics
if topic.name in topics_to_create and
topic.project == Config.CLIENT.project]
self.assertEqual(len(created), len(topics_to_create))
def setUpModule():
_helpers.PROJECT = TESTS_PROJECT
Config.CLIENT = language.Client()
# Now create a bucket for GCS stored content.
storage_client = storage.Client()
bucket_name = 'new' + unique_resource_id()
Config.TEST_BUCKET = storage_client.bucket(bucket_name)
retry_429(Config.TEST_BUCKET.create)()
def test_list_datasets(self):
datasets_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for dataset_name in datasets_to_create:
dataset = Config.CLIENT.dataset(dataset_name)
dataset.create()
self.to_delete.append(dataset)
# Retrieve the datasets.
all_datasets, token = Config.CLIENT.list_datasets()
self.assertTrue(token is None)
created = [dataset for dataset in all_datasets
if dataset.name in datasets_to_create and
dataset.project == Config.CLIENT.project]
self.assertEqual(len(created), len(datasets_to_create))
def test_list_datasets(self):
datasets_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for dataset_name in datasets_to_create:
dataset = Config.CLIENT.dataset(dataset_name)
dataset.create()
self.to_delete.append(dataset)
# Retrieve the datasets.
all_datasets, token = Config.CLIENT.list_datasets()
self.assertTrue(token is None)
created = [dataset for dataset in all_datasets
if dataset.name in datasets_to_create and
dataset.project == Config.CLIENT.project]
self.assertEqual(len(created), len(datasets_to_create))
def test_list_topics(self):
topics_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for topic_name in topics_to_create:
topic = Config.CLIENT.topic(topic_name)
topic.create()
self.to_delete.append(topic)
# Retrieve the topics.
all_topics, _ = Config.CLIENT.list_topics()
created = [
topic for topic in all_topics if topic.name in topics_to_create
and topic.project == Config.CLIENT.project
]
self.assertEqual(len(created), len(topics_to_create))
def setUpModule():
Config.CLIENT = speech.Client()
Config.USE_GAX = Config.CLIENT._use_gax
# Now create a bucket for GCS stored content.
storage_client = storage.Client()
bucket_name = 'new' + unique_resource_id()
Config.TEST_BUCKET = storage_client.bucket(bucket_name)
# 429 Too Many Requests in case API requests rate-limited.
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429(Config.TEST_BUCKET.create)()
def setUpModule():
Config.CLIENT = speech.Client()
Config.USE_GAX = Config.CLIENT._use_gax
# Now create a bucket for GCS stored content.
storage_client = storage.Client()
bucket_name = 'new' + unique_resource_id()
Config.TEST_BUCKET = storage_client.bucket(bucket_name)
# 429 Too Many Requests in case API requests rate-limited.
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429(Config.TEST_BUCKET.create)()
def test_list_datasets(self):
datasets_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for dataset_name in datasets_to_create:
dataset = Config.CLIENT.dataset(dataset_name)
retry_403(dataset.create)()
self.to_delete.append(dataset)
# Retrieve the datasets.
iterator = Config.CLIENT.list_datasets()
all_datasets = list(iterator)
self.assertIsNone(iterator.next_page_token)
created = [dataset for dataset in all_datasets
if dataset.name in datasets_to_create and
dataset.project == Config.CLIENT.project]
self.assertEqual(len(created), len(datasets_to_create))
def test_list_datasets(self):
datasets_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for dataset_name in datasets_to_create:
dataset = Config.CLIENT.dataset(dataset_name)
retry_403(dataset.create)()
self.to_delete.append(dataset)
# Retrieve the datasets.
iterator = Config.CLIENT.list_datasets()
all_datasets = list(iterator)
self.assertIsNone(iterator.next_page_token)
created = [dataset for dataset in all_datasets
if dataset.name in datasets_to_create and
dataset.project == Config.CLIENT.project]
self.assertEqual(len(created), len(datasets_to_create))
def setUpModule():
emulator_dataset = os.getenv(GCD_DATASET)
# Isolated namespace so concurrent test runs don't collide.
test_namespace = "ns" + unique_resource_id()
if emulator_dataset is None:
Config.CLIENT = datastore.Client(namespace=test_namespace)
else:
credentials = EmulatorCreds()
http = httplib2.Http() # Un-authorized.
Config.CLIENT = datastore.Client(
project=emulator_dataset, namespace=test_namespace, credentials=credentials, http=http
)
def setUpModule():
emulator_dataset = os.getenv(GCD_DATASET)
# Isolated namespace so concurrent test runs don't collide.
test_namespace = 'ns' + unique_resource_id()
if emulator_dataset is None:
Config.CLIENT = datastore.Client(namespace=test_namespace)
else:
credentials = EmulatorCreds()
http = httplib2.Http() # Un-authorized.
Config.CLIENT = datastore.Client(project=emulator_dataset,
namespace=test_namespace,
credentials=credentials,
http=http)
def test_list_subscriptions(self):
topic = Config.CLIENT.topic(DEFAULT_TOPIC_NAME)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
empty, _ = topic.list_subscriptions()
self.assertEqual(len(empty), 0)
subscriptions_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for subscription_name in subscriptions_to_create:
subscription = topic.subscription(subscription_name)
subscription.create()
self.to_delete.append(subscription)
# Retrieve the subscriptions.
all_subscriptions, _ = topic.list_subscriptions()
created = [subscription for subscription in all_subscriptions
if subscription.name in subscriptions_to_create]
self.assertEqual(len(created), len(subscriptions_to_create))
def test_list_topics(self):
before, _ = Config.CLIENT.list_topics()
topics_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for topic_name in topics_to_create:
topic = Config.CLIENT.topic(topic_name)
topic.create()
self.to_delete.append(topic)
# Retrieve the topics.
def _all_created(result):
return len(result[0]) == len(before) + len(topics_to_create)
retry = RetryResult(_all_created)
after, _ = retry(Config.CLIENT.list_topics)()
created = [topic for topic in after
if topic.name in topics_to_create and
topic.project == Config.CLIENT.project]
self.assertEqual(len(created), len(topics_to_create))
def test_list_topics(self):
before = _consume_topics(Config.CLIENT)
topics_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for topic_name in topics_to_create:
topic = Config.CLIENT.topic(topic_name)
topic.create()
self.to_delete.append(topic)
# Retrieve the topics.
def _all_created(result):
return len(result) == len(before) + len(topics_to_create)
retry = RetryResult(_all_created)
after = retry(_consume_topics)(Config.CLIENT)
created = [topic for topic in after
if topic.name in topics_to_create and
topic.project == Config.CLIENT.project]
self.assertEqual(len(created), len(topics_to_create))
def test_create_subscription_w_ack_deadline(self):
topic = Config.CLIENT.topic(DEFAULT_TOPIC_NAME)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id()
subscription = topic.subscription(SUBSCRIPTION_NAME, ack_deadline=120)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
self.assertTrue(subscription.exists())
self.assertEqual(subscription.name, SUBSCRIPTION_NAME)
self.assertEqual(subscription.ack_deadline, 120)
self.assertTrue(subscription.topic is topic)
def test_list_subscriptions(self):
topic = Config.CLIENT.topic(DEFAULT_TOPIC_NAME)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
empty, _ = topic.list_subscriptions()
self.assertEqual(len(empty), 0)
subscriptions_to_create = [
'new' + unique_resource_id(),
'newer' + unique_resource_id(),
'newest' + unique_resource_id(),
]
for subscription_name in subscriptions_to_create:
subscription = topic.subscription(subscription_name)
subscription.create()
self.to_delete.append(subscription)
# Retrieve the subscriptions.
all_subscriptions, _ = topic.list_subscriptions()
created = [
subscription for subscription in all_subscriptions
if subscription.name in subscriptions_to_create
]
self.assertEqual(len(created), len(subscriptions_to_create))
def test_subscription_iam_policy(self):
self._maybe_emulator_skip()
topic_name = 'test-sub-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
SUB_NAME = 'test-sub-iam-policy-sub' + unique_resource_id('-')
subscription = topic.subscription(SUB_NAME)
subscription.create()
count = 5
while count > 0 and not subscription.exists():
time.sleep(1)
count -= 1
self.assertTrue(subscription.exists())
self.to_delete.insert(0, subscription)
policy = subscription.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = subscription.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_subscription_iam_policy(self):
self._maybe_emulator_skip()
topic_name = 'test-sub-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
SUB_NAME = 'test-sub-iam-policy-sub' + unique_resource_id('-')
subscription = topic.subscription(SUB_NAME)
subscription.create()
count = 5
while count > 0 and not subscription.exists():
time.sleep(1)
count -= 1
self.assertTrue(subscription.exists())
self.to_delete.insert(0, subscription)
policy = subscription.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = subscription.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_topic_iam_policy(self):
self._maybe_emulator_skip()
topic_name = 'test-topic-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
policy = topic.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = topic.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_topic_iam_policy(self):
self._maybe_emulator_skip()
topic_name = 'test-topic-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
policy = topic.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = topic.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_create_instance(self):
ALT_INSTANCE_ID = 'new' + unique_resource_id('-')
instance = Config.CLIENT.instance(ALT_INSTANCE_ID, LOCATION_ID)
operation = instance.create()
# Make sure this instance gets deleted after the test case.
self.instances_to_delete.append(instance)
# We want to make sure the operation completes.
self.assertTrue(_operation_wait(operation))
# Create a new instance instance and make sure it is the same.
instance_alt = Config.CLIENT.instance(ALT_INSTANCE_ID, LOCATION_ID)
instance_alt.reload()
self.assertEqual(instance, instance_alt)
self.assertEqual(instance.display_name, instance_alt.display_name)
def test_write_point(self):
METRIC_TYPE = ('custom.googleapis.com/tmp/system_test_example' +
unique_resource_id())
METRIC_KIND = monitoring.MetricKind.GAUGE
VALUE_TYPE = monitoring.ValueType.DOUBLE
DESCRIPTION = 'System test example -- DELETE ME!'
VALUE = 3.14
client = monitoring.Client()
descriptor = client.metric_descriptor(
METRIC_TYPE,
metric_kind=METRIC_KIND,
value_type=VALUE_TYPE,
description=DESCRIPTION,
)
descriptor.create()
metric = client.metric(METRIC_TYPE, {})
resource = client.resource('global', {})
retry_500(client.write_point)(metric, resource, VALUE)
def _query_timeseries_with_retries():
MAX_RETRIES = 7
def _has_timeseries(result):
return len(list(result)) > 0
retry_result = RetryResult(_has_timeseries,
max_tries=MAX_RETRIES)(client.query)
return RetryErrors(BadRequest, max_tries=MAX_RETRIES)(retry_result)
query = _query_timeseries_with_retries()(METRIC_TYPE, minutes=5)
timeseries_list = list(query)
self.assertEqual(len(timeseries_list), 1)
timeseries = timeseries_list[0]
self.assertEqual(timeseries.metric, metric)
# project_id label only exists on output.
del timeseries.resource.labels['project_id']
self.assertEqual(timeseries.resource, resource)
descriptor.delete()
with self.assertRaises(NotFound):
descriptor.delete()
def test_create_and_delete_metric_descriptor(self):
METRIC_TYPE = ('custom.googleapis.com/tmp/system_test_example' +
unique_resource_id())
METRIC_KIND = monitoring.MetricKind.GAUGE
VALUE_TYPE = monitoring.ValueType.DOUBLE
DESCRIPTION = 'System test example -- DELETE ME!'
client = monitoring.Client()
descriptor = client.metric_descriptor(
METRIC_TYPE,
metric_kind=METRIC_KIND,
value_type=VALUE_TYPE,
description=DESCRIPTION,
)
retry_500(descriptor.create)()
retry_404_500(descriptor.delete)()
def test_topic_iam_policy(self):
from google.cloud.pubsub.iam import PUBSUB_TOPICS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-topic-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
# Retry / backoff up to 7 seconds (1 + 2 + 4)
retry = RetryResult(lambda result: result, max_tries=4)
retry(topic.exists)()
self.to_delete.append(topic)
if topic.check_iam_permissions([PUBSUB_TOPICS_GET_IAM_POLICY]):
policy = topic.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = topic.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_topic_iam_policy(self):
from gcloud.pubsub.iam import PUBSUB_TOPICS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-topic-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
if topic.check_iam_permissions([PUBSUB_TOPICS_GET_IAM_POLICY]):
policy = topic.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = topic.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_create_and_delete_metric_descriptor(self):
METRIC_TYPE = ('custom.googleapis.com/tmp/system_test_example' +
unique_resource_id())
METRIC_KIND = monitoring.MetricKind.GAUGE
VALUE_TYPE = monitoring.ValueType.DOUBLE
DESCRIPTION = 'System test example -- DELETE ME!'
client = monitoring.Client()
descriptor = client.metric_descriptor(
METRIC_TYPE,
metric_kind=METRIC_KIND,
value_type=VALUE_TYPE,
description=DESCRIPTION,
)
retry_500(descriptor.create)()
retry_404_500(descriptor.delete)()
def test_topic_iam_policy(self):
from gcloud.pubsub.iam import PUBSUB_TOPICS_GET_IAM_POLICY
self._maybe_emulator_skip()
topic_name = 'test-topic-iam-policy-topic' + unique_resource_id('-')
topic = Config.CLIENT.topic(topic_name)
topic.create()
count = 5
while count > 0 and not topic.exists():
time.sleep(1)
count -= 1
self.assertTrue(topic.exists())
self.to_delete.append(topic)
if topic.check_iam_permissions([PUBSUB_TOPICS_GET_IAM_POLICY]):
policy = topic.get_iam_policy()
policy.viewers.add(policy.user('*****@*****.**'))
new_policy = topic.set_iam_policy(policy)
self.assertEqual(new_policy.viewers, policy.viewers)
def test_write_point(self):
METRIC_TYPE = ('custom.googleapis.com/tmp/system_test_example' +
unique_resource_id())
METRIC_KIND = monitoring.MetricKind.GAUGE
VALUE_TYPE = monitoring.ValueType.DOUBLE
DESCRIPTION = 'System test example -- DELETE ME!'
VALUE = 3.14
client = monitoring.Client()
descriptor = client.metric_descriptor(
METRIC_TYPE,
metric_kind=METRIC_KIND,
value_type=VALUE_TYPE,
description=DESCRIPTION,
)
descriptor.create()
metric = client.metric(METRIC_TYPE, {})
resource = client.resource('global', {})
retry_500(client.write_point)(metric, resource, VALUE)
def _query_timeseries_with_retries():
MAX_RETRIES = 7
def _has_timeseries(result):
return len(list(result)) > 0
retry_result = RetryResult(_has_timeseries,
max_tries=MAX_RETRIES)(client.query)
return RetryErrors(BadRequest, max_tries=MAX_RETRIES)(retry_result)
query = _query_timeseries_with_retries()(METRIC_TYPE, minutes=5)
timeseries_list = list(query)
self.assertEqual(len(timeseries_list), 1)
timeseries = timeseries_list[0]
self.assertEqual(timeseries.metric, metric)
# project_id label only exists on output.
del timeseries.resource.labels['project_id']
self.assertEqual(timeseries.resource, resource)
descriptor.delete()
with self.assertRaises(NotFound):
descriptor.delete()
def test_create_instance(self):
ALT_INSTANCE_ID = 'new' + unique_resource_id('-')
instance = Config.CLIENT.instance(ALT_INSTANCE_ID,
Config.INSTANCE_CONFIG.name)
operation = instance.create()
# Make sure this instance gets deleted after the test case.
self.instances_to_delete.append(instance)
# We want to make sure the operation completes.
operation.result(30) # raises on failure / timeout.
# Create a new instance instance and make sure it is the same.
instance_alt = Config.CLIENT.instance(ALT_INSTANCE_ID,
Config.INSTANCE_CONFIG.name)
instance_alt.reload()
self.assertEqual(instance, instance_alt)
self.assertEqual(instance.display_name, instance_alt.display_name)
