def test_submit_pubsub_server_error(
integration_test: IntegrationTest,
publisher: PublisherClient,
pubsub: Any,
topic: str,
):
if pubsub == "google":
pytest.skip("requires pubsub emulator")
# override pubsub status
publisher.update_topic({"name": topic}, {"paths": ["status_code=internal"]})
integration_test.assert_accepted_and_queued()
# restore pubsub status
publisher.update_topic({"name": topic}, {"paths": ["status_code="]})
integration_test.assert_flushed()
def get_pubsub_publisher() -> PublisherClient:
global publisher
if publisher is None:
publisher = PublisherClient()
return publisher
def init_app(app: Sanic) -> Tuple[PublisherClient, SQLiteAckQueue]:
"""Initialize Sanic app with url rules."""
# Initialize PubSub client
timeout = app.config.get("PUBLISH_TIMEOUT_SECONDS", None)
client = PublisherClient()
client.api.publish = partial(
client.api.publish,
retry=Retry(TRANSIENT_ERRORS, deadline=timeout),
timeout=timeout,
)
client._batch_class = AsyncioBatch
# Use a SQLiteAckQueue because:
# * we use acks to ensure messages only removed on success
# * persist-queue's SQLite*Queue is faster than its Queue
# * SQLite provides thread-safe and process-safe access
queue_config = {
key[6:].lower(): value
for key, value in app.config.items() if key.startswith("QUEUE_")
}
q = SQLiteAckQueue(**queue_config)
# get metadata_headers config
metadata_headers = app.config["METADATA_HEADERS"]
# validate attribute keys
for attribute in metadata_headers.values():
if len(attribute.encode("utf8")) > 256:
# https://cloud.google.com/pubsub/quotas#resource_limits
raise ValueError(
"Metadata attribute exceeds key size limit of 256 bytes")
# add routes for ROUTE_TABLE
handlers: Dict[str, Callable] = {}
for route in app.config["ROUTE_TABLE"]:
if route.topic not in handlers:
# generate one handler per topic
handlers[route.topic] = partial(
submit,
client=client,
q=q,
topic=route.topic,
metadata_headers=metadata_headers,
)
handlers[route.topic].__name__ = f"submit({route.topic})"
app.add_route(
handler=handlers[route.topic],
uri=route.uri,
methods=[method.upper() for method in route.methods],
)
return client, q
def test_set_topic_policy(
publisher_client: pubsub_v1.PublisherClient,
topic_path: str,
) -> CaptureFixture:
iam.set_topic_policy(PROJECT_ID, TOPIC_ID)
policy = publisher_client.get_iam_policy(request={"resource": topic_path})
assert "roles/pubsub.publisher" in str(policy)
assert "allUsers" in str(policy)
def tearDownClass(cls):
super().tearDownClass()
publisher = PublisherClient()
subscriber = SubscriberClient()
for topic in ["test0", "test1"]:
try:
publisher.delete_topic(publisher.topic_path(
cls.project, topic))
except Exception as ex:
raise ex
try:
subscriber.delete_subscription(
subscriber.subscription_path(cls.project, topic))
except Exception as ex:
raise ex
def publish_message(response_text, patient_id, event_attributes):
publisher = PublisherClient()
# pylint: disable=no-member
topic_path = publisher.topic_path(
os.environ.get("PROJECT_NAME"), os.environ.get("TOPIC_NAME")
)
data = response_text.encode("utf-8")
future = publisher.publish(
topic_path,
data=data,
**event_attributes,
patient_id=patient_id,
)
message_id = future.result()
logger.info(f"Successfully published message: {message_id}")
print(f"Successfully published message: {message_id}")
def test_submit_pubsub_permission_denied(
integration_test: IntegrationTest,
publisher: PublisherClient,
pubsub: str,
topic: str,
):
if pubsub == "google":
pytest.skip("not implemented")
else:
publisher.update_topic({"name": topic},
{"paths": ["status_code=permission_denied"]})
integration_test.assert_accepted_and_queued()
if pubsub == "google":
pytest.skip("not implemented")
else:
publisher.update_topic({"name": topic}, {"paths": ["status_code="]})
integration_test.assert_flushed_and_delivered()
def _publisher(service_account_json: Union[str, None]) -> PublisherClient:
try:
return PublisherClient(
credentials=_credentials(service_account_json, 'Publisher'),
publisher_options=PublisherOptions(enable_message_ordering=True))
except DefaultCredentialsError:
logging.error(
'`GOOGLE_APPLICATION_CREDENTIALS` environment variable or service account in `address` field are not set!'
)
def test_submit_pubsub_timeout(
integration_test: IntegrationTest,
publisher: PublisherClient,
pubsub: Any,
topic: str,
):
if pubsub == "google":
pytest.skip("requires pubsub emulator")
# override pubsub response time
publisher.update_topic(
{"name": topic},
{"paths": ["sleep=%.1f" % (PUBLISH_TIMEOUT_SECONDS + 1)]})
integration_test.assert_accepted_and_queued()
# restore pubsub response time
publisher.update_topic({"name": topic}, {"paths": ["sleep="]})
integration_test.assert_flushed()
def __init__(self,
name,
dist,
emulator_host,
emulator_port,
max_bytes,
max_latency,
max_messages,
ordering=False,
client_options=None,
credentials=None,
services_service=None,
**config):
services_service(plugin.Plugin.__init__,
self,
name,
dist,
emulator_host=emulator_host,
emulator_port=emulator_port,
max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages,
ordering=False,
client_options=None,
credentials=None,
**config)
batch_settings = types.BatchSettings(max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages)
publisher_options = types.PublisherOptions(
enable_message_ordering=ordering)
settings = {}
if client_options is not None:
if isinstance(client_options, (str, type(u''))):
client_options = services_service(
reference.load_object(client_options)[0])
settings['client_options'] = client_options
if emulator_host:
channel = grpc.insecure_channel('{}:{}'.format(
emulator_host, emulator_port))
transport = PublisherGrpcTransport(channel=channel)
else:
transport = None
if credentials is not None:
settings['credentials'] = services_service(
reference.load_object(credentials)[0])
self.__class__.proxy_target = PublisherClient(batch_settings,
publisher_options,
transport=transport,
**settings)
def get_conn(self) -> PublisherClient:
"""
Retrieves connection to Google Cloud Pub/Sub.
:return: Google Cloud Pub/Sub client object.
:rtype: google.cloud.pubsub_v1.PublisherClient
"""
if not self._client:
self._client = PublisherClient(credentials=self._get_credentials(), client_info=self.client_info)
return self._client
def proto_topic(publisher_client: pubsub_v1.PublisherClient,
proto_schema: str) -> Generator[str, None, None]:
proto_topic_path = publisher_client.topic_path(PROJECT_ID, PROTO_TOPIC_ID)
try:
proto_topic = publisher_client.get_topic(
request={"topic": proto_topic_path})
except NotFound:
proto_topic = publisher_client.create_topic(
request={
"name": proto_topic_path,
"schema_settings": {
"schema": proto_schema,
"encoding": Encoding.BINARY,
},
})
yield proto_topic.name
publisher_client.delete_topic(request={"topic": proto_topic.name})
class PublishService:
_publisher: PublisherClient
_topic_name: str
_request: Request
def __init__(self, topic_name: str, request: Request):
self._publisher = PublisherClient()
self._topic_name = topic_name
self._request = request
def _publish_message(self, message_name, message):
metadata = Gobits.from_request(request=self._request)
try:
my_gobits = [metadata.to_json()]
except: # noqa: E722
my_gobits = []
message_to_publish = {"gobits": my_gobits, message_name: message}
self._publisher.publish(
self._topic_name,
bytes(json.dumps(message_to_publish).encode("utf-8")))
def delete_topic(
self,
topic: str,
project_id: Optional[str] = None,
fail_if_not_exists: bool = False,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
) -> None:
"""
Deletes a Pub/Sub topic if it exists.
:param topic: the Pub/Sub topic name to delete; do not
include the ``projects/{project}/topics/`` prefix.
:type topic: str
:param project_id: Optional, the GCP project ID in which to delete the topic.
If set to None or missing, the default project_id from the GCP connection is used.
:type project_id: str
:param fail_if_not_exists: if set, raise an exception if the topic
does not exist
:type fail_if_not_exists: bool
:param retry: (Optional) A retry object used to retry requests.
If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: (Optional) The amount of time, in seconds, to wait for the request
to complete. Note that if retry is specified, the timeout applies to each
individual attempt.
:type timeout: float
:param metadata: (Optional) Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]]
"""
if not project_id:
raise ValueError("Project ID should be set.")
publisher = self.get_conn()
topic_path = PublisherClient.topic_path(project_id, topic) # pylint: disable=no-member
self.log.info("Deleting topic (path) %s", topic_path)
try:
# pylint: disable=no-member
publisher.delete_topic(
topic=topic_path,
retry=retry,
timeout=timeout,
metadata=metadata,
)
except NotFound:
self.log.warning('Topic does not exist: %s', topic_path)
if fail_if_not_exists:
raise PubSubException(
'Topic does not exist: {}'.format(topic_path))
except GoogleAPICallError as e:
raise PubSubException('Error deleting topic {}'.format(topic), e)
self.log.info("Deleted topic (path) %s", topic_path)
def avro_topic(publisher_client: pubsub_v1.PublisherClient,
avro_schema: str) -> Generator[str, None, None]:
from google.pubsub_v1.types import Encoding
avro_topic_path = publisher_client.topic_path(PROJECT_ID, AVRO_TOPIC_ID)
try:
avro_topic = publisher_client.get_topic(
request={"topic": avro_topic_path})
except NotFound:
avro_topic = publisher_client.create_topic(
request={
"name": avro_topic_path,
"schema_settings": {
"schema": avro_schema,
"encoding": Encoding.BINARY,
},
})
yield avro_topic.name
publisher_client.delete_topic(request={"topic": avro_topic.name})
def setUpClass(cls):
super().setUpClass()
cls.project = os.environ.get("PUBSUB_PROJECT", "testing")
cls.credentials = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS",
None)
for subscription in ["test0", "test1"]:
try:
publisher = PublisherClient()
publisher.create_topic(
publisher.topic_path(cls.project, subscription))
except AlreadyExists:
pass
try:
subscriber = SubscriberClient()
subscriber.create_subscription(
subscriber.subscription_path(cls.project, subscription),
subscriber.topic_path(cls.project, subscription))
except AlreadyExists:
pass
def test_flush(
integration_test: IntegrationTest,
publisher: PublisherClient,
pubsub: Any,
topic: str,
server_process: subprocess.Popen,
):
if pubsub == "google":
pytest.skip("requires pubsub emulator")
# override pubsub status
publisher.update_topic({"name": topic},
{"paths": ["status_code=internal"]})
integration_test.assert_accepted_and_queued()
# stop server
server_process.kill()
server_process.wait()
# start flush
process = subprocess.Popen(
[sys.executable, "-u", "-m", "ingestion_edge.flush"],
stderr=subprocess.PIPE)
assert process.stderr is not None
for line in process.stderr:
break # wait for an error to be logged
assert process.poll() is None # server still running
# restore pubsub status
publisher.update_topic({"name": topic}, {"paths": ["status_code="]})
try:
process.wait(5)
except subprocess.TimeoutExpired:
# kill after 5 seconds
process.kill()
assert process.wait() == 0
assert json.loads(line)["Fields"]["msg"] == "pubsub unavailable"
integration_test.assert_delivered()
def _initialize_topic(self):
try:
from google.cloud.pubsub_v1 import PublisherClient
except ImportError:
raise unittest.SkipTest("Cannot import pubsub")
self.publisher_client = PublisherClient()
retry_429(self.publisher_client.create_topic)(self.topic_path)
policy = self.publisher_client.get_iam_policy(self.topic_path)
binding = policy.bindings.add()
binding.role = 'roles/pubsub.publisher'
binding.members.append(
'serviceAccount:{}'.format(
Config.CLIENT.get_service_account_email()))
self.publisher_client.set_iam_policy(self.topic_path, policy)
async def submit(
request: Request,
client: PublisherClient,
q: SQLiteAckQueue,
topic: str,
metadata_headers: Dict[str, str],
timeout: Optional[float],
**kwargs
) -> response.HTTPResponse:
"""Deliver request to the pubsub topic.
Deliver to the local queue to be retried on transient errors.
"""
data = request.body
attrs = {
key: value
for key, value in dict(
submission_timestamp=datetime.utcnow().isoformat() + "Z",
uri=request.path,
protocol="HTTP/" + request.version,
method=request.method,
args=request.query_string,
remote_addr=request.ip,
host=request.host,
**{
attr: request.headers.get(header)
for header, attr in metadata_headers.items()
}
).items()
if value is not None
}
# assert valid pubsub message
for value in attrs.values():
if len(value.encode("utf8")) > 1024:
# attribute exceeds value size limit of 1024 bytes
# https://cloud.google.com/pubsub/quotas#resource_limits
return response.text(
"header too large\n", HTTP_STATUS.REQUEST_HEADER_FIELDS_TOO_LARGE
)
try:
future = client.publish(topic, data, **attrs)
await asyncio.wait_for(async_wrap(future), timeout)
except Exception:
# api call failure, write to queue
try:
q.put((topic, data, attrs))
except DatabaseError:
# sqlite queue is probably out of space
return response.text("", HTTP_STATUS.INSUFFICIENT_STORAGE)
return response.text("")
def init_app(app: Sanic) -> Tuple[PublisherClient, SQLiteAckQueue]:
"""Initialize Sanic app with url rules."""
# Get PubSub timeout
timeout = app.config.get("PUBLISH_TIMEOUT_SECONDS")
# Initialize PubSub client
client = PublisherClient()
# Use a SQLiteAckQueue because:
# * we use acks to ensure messages only removed on success
# * persist-queue's SQLite*Queue is faster than its Queue
# * SQLite provides thread-safe and process-safe access
queue_config = {
key[6:].lower(): value
for key, value in app.config.items()
if key.startswith("QUEUE_")
}
q = SQLiteAckQueue(**queue_config)
# get metadata_headers config
metadata_headers = app.config["METADATA_HEADERS"]
# validate attribute keys
for attribute in metadata_headers.values():
if len(attribute.encode("utf8")) > 256:
# https://cloud.google.com/pubsub/quotas#resource_limits
raise ValueError("Metadata attribute exceeds key size limit of 256 bytes")
# generate one view_func per topic
handlers = {
route.topic: partial(
submit,
client=client,
q=q,
topic=route.topic,
metadata_headers=metadata_headers,
timeout=timeout,
)
for route in app.config["ROUTE_TABLE"]
}
# add routes for ROUTE_TABLE
for route in app.config["ROUTE_TABLE"]:
app.add_route(
handler=handlers[route.topic],
uri=route.uri,
methods=[method.upper() for method in route.methods],
# required because handler.__name__ does not exist
# must be a unique name for each handler
name="submit_" + route.topic,
)
return client, q
def publish(
self,
topic: str,
messages: List[Dict],
project_id: Optional[str] = None,
) -> None:
"""
Publishes messages to a Pub/Sub topic.
:param topic: the Pub/Sub topic to which to publish; do not
include the ``projects/{project}/topics/`` prefix.
:type topic: str
:param messages: messages to publish; if the data field in a
message is set, it should be a bytestring (utf-8 encoded)
:type messages: list of PubSub messages; see
http://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage
:param project_id: Optional, the GCP project ID in which to publish.
If set to None or missing, the default project_id from the GCP connection is used.
:type project_id: str
"""
if not project_id:
raise ValueError("Project ID should be set.")
self._validate_messages(messages)
publisher = self.get_conn()
topic_path = PublisherClient.topic_path(project_id, topic) # pylint: disable=no-member
self.log.info("Publish %d messages to topic (path) %s", len(messages),
topic_path)
try:
for message in messages:
publisher.publish(topic=topic_path,
data=message.get("data", b''),
**message.get('attributes', {}))
except GoogleAPICallError as e:
raise PubSubException(
'Error publishing to topic {}'.format(topic_path), e)
self.log.info("Published %d messages to topic (path) %s",
len(messages), topic_path)
def emulator(node_pools: List[str], options: Dict[str, Any]) -> Iterator[str]:
if "emulator" in node_pools:
kube_apply("kube/emulator.deploy.yml", **options)
kube_apply("kube/emulator.svc.yml")
yield "emulator:8000"
elif "server" in node_pools:
pubsub = PublisherClient()
topic = f"projects/{options['project']}/topics/{options['topic']}"
try:
pubsub.create_topic(topic)
except AlreadyExists:
pass
yield ""
pubsub.delete_topic(topic)
else:
yield ""
def ensure_topic_and_subscription():
from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
publisher_client = PublisherClient()
try:
publisher_client.create_topic(
publisher_client.topic_path(PROJECT_ID, TOPIC_NAME))
except AlreadyExists:
pass
subscriber_client = SubscriberClient()
try:
subscriber_client.create_subscription(
subscriber_client.subscription_path(PROJECT_ID, SUBSCRIPTION_NAME),
publisher_client.topic_path(PROJECT_ID, TOPIC_NAME))
except AlreadyExists:
pass
class EmsPublisherClient:
def __init__(self):
self.__client = PublisherClient()
def publish(self, topic: str, data: bytes, **attrs) -> Future:
return self.__client.publish(topic=topic, data=data, **attrs)
def topic_create_if_not_exists(self, project_id: str, topic_name: str):
topic_path = self.__client.api.topic_path(project_id, topic_name)
try:
self.__client.api.create_topic(topic_path)
LOGGER.info("Topic %s created in project %s", topic_name,
project_id)
except AlreadyExists:
LOGGER.info("Topic %s already exists in project %s", topic_name,
project_id)
def delete_topic_if_exists(self, project_id: str, topic_name: str):
topic_path = self.__client.api.topic_path(project_id, topic_name)
try:
self.__client.api.delete_topic(topic_path)
LOGGER.info("Topic %s deleted in project %s", topic_name,
project_id)
except NotFound:
LOGGER.info("Topic %s not found in project %s", topic_name,
project_id)
@staticmethod
def subscription_create(project_id: str, topic_name: str,
subscription_name: str):
subscriber = SubscriberClient()
topic_path = subscriber.api.topic_path(project_id, topic_name)
subscription_path = subscriber.api.subscription_path(
project_id, subscription_name)
subscriber.api.create_subscription(subscription_path, topic_path)
def setup(hass: HomeAssistant, yaml_config: ConfigType) -> bool:
"""Activate Google Pub/Sub component."""
config = yaml_config[DOMAIN]
project_id = config[CONF_PROJECT_ID]
topic_name = config[CONF_TOPIC_NAME]
service_principal_path = hass.config.path(config[CONF_SERVICE_PRINCIPAL])
if not os.path.isfile(service_principal_path):
_LOGGER.error("Path to credentials file cannot be found")
return False
entities_filter = config[CONF_FILTER]
publisher = PublisherClient.from_service_account_json(
service_principal_path)
topic_path = publisher.topic_path( # pylint: disable=no-member
project_id, topic_name)
encoder = DateTimeJSONEncoder()
def send_to_pubsub(event: Event):
"""Send states to Pub/Sub."""
state = event.data.get("new_state")
if (state is None
or state.state in (STATE_UNKNOWN, "", STATE_UNAVAILABLE)
or not entities_filter(state.entity_id)):
return
as_dict = state.as_dict()
data = json.dumps(obj=as_dict, default=encoder.encode).encode("utf-8")
publisher.publish(topic_path, data=data)
hass.bus.listen(EVENT_STATE_CHANGED, send_to_pubsub)
return True
def setUpClass(cls):
cls.publisher = PublisherClient()
def create_topic(
self,
topic: str,
project_id: str,
fail_if_exists: bool = False,
labels: Optional[Dict[str, str]] = None,
message_storage_policy: Union[Dict, MessageStoragePolicy] = None,
kms_key_name: Optional[str] = None,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
) -> None:
"""
Creates a Pub/Sub topic, if it does not already exist.
:param topic: the Pub/Sub topic name to create; do not
include the ``projects/{project}/topics/`` prefix.
:type topic: str
:param project_id: Optional, the Google Cloud project ID in which to create the topic
If set to None or missing, the default project_id from the Google Cloud connection is used.
:type project_id: str
:param fail_if_exists: if set, raise an exception if the topic
already exists
:type fail_if_exists: bool
:param labels: Client-assigned labels; see
https://cloud.google.com/pubsub/docs/labels
:type labels: Dict[str, str]
:param message_storage_policy: Policy constraining the set
of Google Cloud regions where messages published to
the topic may be stored. If not present, then no constraints
are in effect.
:type message_storage_policy:
Union[Dict, google.cloud.pubsub_v1.types.MessageStoragePolicy]
:param kms_key_name: The resource name of the Cloud KMS CryptoKey
to be used to protect access to messages published on this topic.
The expected format is
``projects/*/locations/*/keyRings/*/cryptoKeys/*``.
:type kms_key_name: str
:param retry: (Optional) A retry object used to retry requests.
If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: (Optional) The amount of time, in seconds, to wait for the request
to complete. Note that if retry is specified, the timeout applies to each
individual attempt.
:type timeout: float
:param metadata: (Optional) Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]]
"""
publisher = self.get_conn()
topic_path = PublisherClient.topic_path(project_id, topic) # pylint: disable=no-member
# Add airflow-version label to the topic
labels = labels or {}
labels['airflow-version'] = 'v' + version.replace('.', '-').replace('+', '-')
self.log.info("Creating topic (path) %s", topic_path)
try:
# pylint: disable=no-member
publisher.create_topic(
name=topic_path,
labels=labels,
message_storage_policy=message_storage_policy,
kms_key_name=kms_key_name,
retry=retry,
timeout=timeout,
metadata=metadata,
)
except AlreadyExists:
self.log.warning('Topic already exists: %s', topic)
if fail_if_exists:
raise PubSubException('Topic already exists: {}'.format(topic))
except GoogleAPICallError as e:
raise PubSubException('Error creating topic {}'.format(topic), e)
self.log.info("Created topic (path) %s", topic_path)
class TestStorageNotificationCRUD(unittest.TestCase):
topic = None
TOPIC_NAME = 'notification' + unique_resource_id('-')
CUSTOM_ATTRIBUTES = {
'attr1': 'value1',
'attr2': 'value2',
}
BLOB_NAME_PREFIX = 'blob-name-prefix/'
@property
def topic_path(self):
return 'projects/{}/topics/{}'.format(Config.CLIENT.project,
self.TOPIC_NAME)
def _intialize_topic(self):
try:
from google.cloud.pubsub_v1 import PublisherClient
except ImportError:
raise unittest.SkipTest("Cannot import pubsub")
self.publisher_client = PublisherClient()
retry_429(self.publisher_client.create_topic)(self.topic_path)
policy = self.publisher_client.get_iam_policy(self.topic_path)
binding = policy.bindings.add()
binding.role = 'roles/pubsub.publisher'
binding.members.append(
'serviceAccount:{}'
'@gs-project-accounts.iam.gserviceaccount.com'.format(
Config.CLIENT.project))
self.publisher_client.set_iam_policy(self.topic_path, policy)
def setUp(self):
self.case_buckets_to_delete = []
self._intialize_topic()
def tearDown(self):
retry_429(self.publisher_client.delete_topic)(self.topic_path)
with Config.CLIENT.batch():
for bucket_name in self.case_buckets_to_delete:
bucket = Config.CLIENT.bucket(bucket_name)
retry_429(bucket.delete)()
@staticmethod
def event_types():
from google.cloud.storage.notification import (
OBJECT_FINALIZE_EVENT_TYPE, OBJECT_DELETE_EVENT_TYPE)
return [OBJECT_FINALIZE_EVENT_TYPE, OBJECT_DELETE_EVENT_TYPE]
@staticmethod
def payload_format():
from google.cloud.storage.notification import (
JSON_API_V1_PAYLOAD_FORMAT)
return JSON_API_V1_PAYLOAD_FORMAT
def test_notification_minimal(self):
new_bucket_name = 'notification-minimal' + unique_resource_id('-')
bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name)
self.case_buckets_to_delete.append(new_bucket_name)
self.assertEqual(list(bucket.list_notifications()), [])
notification = bucket.notification(self.TOPIC_NAME)
retry_429(notification.create)()
try:
self.assertTrue(notification.exists())
self.assertIsNotNone(notification.notification_id)
notifications = list(bucket.list_notifications())
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0].topic_name, self.TOPIC_NAME)
finally:
notification.delete()
def test_notification_explicit(self):
new_bucket_name = 'notification-explicit' + unique_resource_id('-')
bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name)
self.case_buckets_to_delete.append(new_bucket_name)
notification = bucket.notification(
self.TOPIC_NAME,
custom_attributes=self.CUSTOM_ATTRIBUTES,
event_types=self.event_types(),
blob_name_prefix=self.BLOB_NAME_PREFIX,
payload_format=self.payload_format(),
)
retry_429(notification.create)()
try:
self.assertTrue(notification.exists())
self.assertIsNotNone(notification.notification_id)
self.assertEqual(notification.custom_attributes,
self.CUSTOM_ATTRIBUTES)
self.assertEqual(notification.event_types, self.event_types())
self.assertEqual(notification.blob_name_prefix,
self.BLOB_NAME_PREFIX)
self.assertEqual(notification.payload_format,
self.payload_format())
finally:
notification.delete()
@unittest.skipUnless(USER_PROJECT, 'USER_PROJECT not set in environment.')
def test_notification_w_user_project(self):
new_bucket_name = 'notification-minimal' + unique_resource_id('-')
bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name,
requester_pays=True)
self.case_buckets_to_delete.append(new_bucket_name)
with_user_project = Config.CLIENT.bucket(new_bucket_name,
user_project=USER_PROJECT)
self.assertEqual(list(with_user_project.list_notifications()), [])
notification = with_user_project.notification(self.TOPIC_NAME)
retry_429(notification.create)()
try:
self.assertTrue(notification.exists())
self.assertIsNotNone(notification.notification_id)
notifications = list(with_user_project.list_notifications())
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0].topic_name, self.TOPIC_NAME)
finally:
notification.delete()
def client() -> PublisherClient:
if "PUBSUB_EMULATOR_HOST" not in os.environ:
return PublisherClient(channel=grpc.insecure_channel(target=""))
else:
return PublisherClient()
def pubsub_topic(pubsub_client: pubsub_v1.PublisherClient, project_id):
topic_id = resource_prefix()
topic_path = pubsub_v1.PublisherClient.topic_path(project_id, topic_id)
pubsub_client.create_topic(name=topic_path)
yield topic_path
pubsub_client.delete_topic(topic=topic_path)
def setUpClass(cls):
cls.publisher = PublisherClient()
cls.subscriber = SubscriberClient()
class TestStorageNotificationCRUD(unittest.TestCase):
topic = None
TOPIC_NAME = 'notification' + unique_resource_id('-')
CUSTOM_ATTRIBUTES = {
'attr1': 'value1',
'attr2': 'value2',
}
BLOB_NAME_PREFIX = 'blob-name-prefix/'
@property
def topic_path(self):
return 'projects/{}/topics/{}'.format(
Config.CLIENT.project, self.TOPIC_NAME)
def _initialize_topic(self):
try:
from google.cloud.pubsub_v1 import PublisherClient
except ImportError:
raise unittest.SkipTest("Cannot import pubsub")
self.publisher_client = PublisherClient()
retry_429(self.publisher_client.create_topic)(self.topic_path)
policy = self.publisher_client.get_iam_policy(self.topic_path)
binding = policy.bindings.add()
binding.role = 'roles/pubsub.publisher'
binding.members.append(
'serviceAccount:{}'.format(
Config.CLIENT.get_service_account_email()))
self.publisher_client.set_iam_policy(self.topic_path, policy)
def setUp(self):
self.case_buckets_to_delete = []
self._initialize_topic()
def tearDown(self):
retry_429(self.publisher_client.delete_topic)(self.topic_path)
with Config.CLIENT.batch():
for bucket_name in self.case_buckets_to_delete:
bucket = Config.CLIENT.bucket(bucket_name)
retry_429(bucket.delete)()
@staticmethod
def event_types():
from google.cloud.storage.notification import (
OBJECT_FINALIZE_EVENT_TYPE,
OBJECT_DELETE_EVENT_TYPE)
return [OBJECT_FINALIZE_EVENT_TYPE, OBJECT_DELETE_EVENT_TYPE]
@staticmethod
def payload_format():
from google.cloud.storage.notification import (
JSON_API_V1_PAYLOAD_FORMAT)
return JSON_API_V1_PAYLOAD_FORMAT
def test_notification_minimal(self):
new_bucket_name = 'notification-minimal' + unique_resource_id('-')
bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name)
self.case_buckets_to_delete.append(new_bucket_name)
self.assertEqual(list(bucket.list_notifications()), [])
notification = bucket.notification(self.TOPIC_NAME)
retry_429_503(notification.create)()
try:
self.assertTrue(notification.exists())
self.assertIsNotNone(notification.notification_id)
notifications = list(bucket.list_notifications())
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0].topic_name, self.TOPIC_NAME)
finally:
notification.delete()
def test_notification_explicit(self):
new_bucket_name = 'notification-explicit' + unique_resource_id('-')
bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name)
self.case_buckets_to_delete.append(new_bucket_name)
notification = bucket.notification(
self.TOPIC_NAME,
custom_attributes=self.CUSTOM_ATTRIBUTES,
event_types=self.event_types(),
blob_name_prefix=self.BLOB_NAME_PREFIX,
payload_format=self.payload_format(),
)
retry_429_503(notification.create)()
try:
self.assertTrue(notification.exists())
self.assertIsNotNone(notification.notification_id)
self.assertEqual(
notification.custom_attributes, self.CUSTOM_ATTRIBUTES)
self.assertEqual(notification.event_types, self.event_types())
self.assertEqual(
notification.blob_name_prefix, self.BLOB_NAME_PREFIX)
self.assertEqual(
notification.payload_format, self.payload_format())
finally:
notification.delete()
@unittest.skipUnless(USER_PROJECT, 'USER_PROJECT not set in environment.')
def test_notification_w_user_project(self):
new_bucket_name = 'notification-minimal' + unique_resource_id('-')
retry_429(Config.CLIENT.create_bucket)(
new_bucket_name, requester_pays=True)
self.case_buckets_to_delete.append(new_bucket_name)
with_user_project = Config.CLIENT.bucket(
new_bucket_name, user_project=USER_PROJECT)
self.assertEqual(list(with_user_project.list_notifications()), [])
notification = with_user_project.notification(self.TOPIC_NAME)
retry_429(notification.create)()
try:
self.assertTrue(notification.exists())
self.assertIsNotNone(notification.notification_id)
notifications = list(with_user_project.list_notifications())
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0].topic_name, self.TOPIC_NAME)
finally:
notification.delete()
