def main(args: dict) -> None:
# batch_settings = pubsub.types.BatchSettings(max_messages=10)
# publisher = pubsub.PublisherClient(batch_settings=batch_settings)
kwargs = {}
emulator = os.environ.get('PUBSUB_EMULATOR_HOST')
if emulator:
channel = grpc.insecure_channel(target=emulator)
else:
channel = grpc_helpers.create_channel(
credentials=None,
target=publisher_client.PublisherClient.SERVICE_ADDRESS,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options=[('grpc.max_send_message_length', -1),
('grpc.max_receive_message_length', -1)])
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
api = publisher_client.PublisherClient(transport=transport)
topic = f'projects/{args["PROJECT"]}/topics/{args["TOPIC"]}'
for xs in batches(range(int(args['--num'])), int(args['--rate'])):
messages = []
for x in xs:
if x is None:
break
# message = PubsubMessage(data=b'foobar', ordering_key='', attributes={})
message = PubsubMessage(data=b'foobar', attributes={})
messages.append(message)
print('.', end='')
# retries, then raises google.api_core.exceptions.GoogleAPIError
# response = publisher.api.publish(topic, messages)
response = api.publish(topic, messages, retry=None, timeout=1)
print()
def test_grpc_request_with_on_demand_jwt_credentials():
credentials, project_id = google.auth.default()
credentials = google.auth.jwt.OnDemandCredentials.from_signing_credentials(
credentials)
transport = publisher_grpc_transport.PublisherGrpcTransport(
address=publisher_client.PublisherClient.SERVICE_ADDRESS,
credentials=credentials,
)
# Create a pub/sub client.
client = pubsub_v1.PublisherClient(transport=transport)
# list the topics and drain the iterator to test that an authorized API
# call works.
list_topics_iter = client.list_topics(
project="projects/{}".format(project_id))
list(list_topics_iter)
def test_grpc_request_with_regular_credentials(http_request):
credentials, project_id = google.auth.default()
credentials = google.auth.credentials.with_scopes_if_required(
credentials, ["https://www.googleapis.com/auth/pubsub"])
transport = publisher_grpc_transport.PublisherGrpcTransport(
address=publisher_client.PublisherClient.SERVICE_ADDRESS,
credentials=credentials,
)
# Create a pub/sub client.
client = pubsub_v1.PublisherClient(transport=transport)
# list the topics and drain the iterator to test that an authorized API
# call works.
list_topics_iter = client.list_topics(
project="projects/{}".format(project_id))
list(list_topics_iter)
def __init__(self, batch_settings=(), **kwargs):
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "transport" not in kwargs:
channel = kwargs.pop("channel", None)
if channel is None:
channel = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# cannot pass both 'channel' and 'credentials'
kwargs.pop("credentials", None)
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
kwargs["transport"] = transport
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
self._batches = {}
def __init__(self,
transport=None,
channel=None,
credentials=None,
client_config=None,
client_info=None):
"""Constructor.
Args:
transport (Union[~.PublisherGrpcTransport,
Callable[[~.Credentials, type], ~.PublisherGrpcTransport]): A transport
instance, responsible for actually making the API calls.
The default transport uses the gRPC protocol.
This argument may also be a callable which returns a
transport instance. Callables will be sent the credentials
as the first argument and the default transport class as
the second argument.
channel (grpc.Channel): DEPRECATED. A ``Channel`` instance
through which to make calls. This argument is mutually exclusive
with ``credentials``; providing both will raise an exception.
credentials (google.auth.credentials.Credentials): The
authorization credentials to attach to requests. These
credentials identify this application to the service. If none
are specified, the client will attempt to ascertain the
credentials from the environment.
This argument is mutually exclusive with providing a
transport instance to ``transport``; doing so will raise
an exception.
client_config (dict): DEPRECATED. A dictionary of call options for
each method. If not specified, the default configuration is used.
client_info (google.api_core.gapic_v1.client_info.ClientInfo):
The client info used to send a user-agent string along with
API requests. If ``None``, then default info will be used.
Generally, you only need to set this if you're developing
your own client library.
"""
# Raise deprecation warnings for things we want to go away.
if client_config is not None:
warnings.warn('The `client_config` argument is deprecated.',
PendingDeprecationWarning,
stacklevel=2)
else:
client_config = publisher_client_config.config
if channel:
warnings.warn(
'The `channel` argument is deprecated; use '
'`transport` instead.',
PendingDeprecationWarning,
stacklevel=2)
# Instantiate the transport.
# The transport is responsible for handling serialization and
# deserialization and actually sending data to the service.
if transport:
if callable(transport):
self.transport = transport(
credentials=credentials,
default_class=publisher_grpc_transport.
PublisherGrpcTransport,
)
else:
if credentials:
raise ValueError(
'Received both a transport instance and '
'credentials; these are mutually exclusive.')
self.transport = transport
else:
self.transport = publisher_grpc_transport.PublisherGrpcTransport(
address=self.SERVICE_ADDRESS,
channel=channel,
credentials=credentials,
)
if client_info is None:
client_info = google.api_core.gapic_v1.client_info.ClientInfo(
gapic_version=_GAPIC_LIBRARY_VERSION, )
else:
client_info.gapic_version = _GAPIC_LIBRARY_VERSION
self._client_info = client_info
# Parse out the default settings for retry and timeout for each RPC
# from the client configuration.
# (Ordinarily, these are the defaults specified in the `*_config.py`
# file next to this one.)
self._method_configs = google.api_core.gapic_v1.config.parse_method_configs(
client_config['interfaces'][self._INTERFACE_NAME], )
# Save a dictionary of cached API call functions.
# These are the actual callables which invoke the proper
# transport methods, wrapped with `wrap_method` to add retry,
# timeout, and the like.
self._inner_api_calls = {}
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
client_options = kwargs.pop("client_options", None)
if (client_options and "api_endpoint" in client_options and isinstance(
client_options["api_endpoint"], six.string_types)):
self._target = client_options["api_endpoint"]
else:
self._target = publisher_client.PublisherClient.SERVICE_ADDRESS
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "transport" not in kwargs:
channel = kwargs.pop("channel", None)
if channel is None:
channel = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# cannot pass both 'channel' and 'credentials'
kwargs.pop("credentials", None)
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
kwargs["transport"] = transport
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
if self._enable_message_ordering:
# Set retry timeout to "infinite" when message ordering is enabled.
# Note that this then also impacts messages added with an empty ordering
# key.
client_config = _set_nested_value(
kwargs.pop("client_config", {}),
2**32,
[
"interfaces",
"google.pubsub.v1.Publisher",
"retry_params",
"messaging",
"total_timeout_millis",
],
)
kwargs["client_config"] = client_config
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
