def test_publish_after_batch_error():
client = create_client()
message = create_message()
batch = client._batch_class(client, "topic_name",
types.BatchSettings(max_latency=float("inf")))
batch._messages.append(
mock.Mock(name="message")) # Make batch truthy (non-empty).
sequencer = unordered_sequencer.UnorderedSequencer(client, "topic_name")
sequencer._set_batch(batch)
with mock.patch.object(batch, "commit") as fake_batch_commit:
sequencer.commit()
fake_batch_commit.assert_called_once()
# Simulate publish RPC failing.
batch._set_status(base.BatchStatus.ERROR)
# Will create a new batch since the old one has been committed. The fact
# that the old batch errored should not matter in the publish of the next
# message.
future = sequencer.publish(message)
assert future is not None
def test_publish_large_messages(publisher, topic_path, cleanup):
# Make sure the topic gets deleted.
cleanup.append((publisher.delete_topic, topic_path))
# Each message should be smaller than 10**7 bytes (the server side limit for
# PublishRequest), but all messages combined in a PublishRequest should
# slightly exceed that threshold to make sure the publish code handles these
# cases well.
# Mind that the total PublishRequest size must still be smaller than
# 10 * 1024 * 1024 bytes in order to not exceed the max request body size limit.
msg_data = b"x" * (2 * 10 ** 6)
publisher.batch_settings = types.BatchSettings(
max_bytes=11 * 1000 * 1000, # more than the server limit of 10 ** 7
max_latency=2.0, # so that autocommit happens after publishing all messages
max_messages=100,
)
publisher.create_topic(topic_path)
futures = [publisher.publish(topic_path, msg_data, num=str(i)) for i in range(5)]
# If the publishing logic correctly split all messages into more than a
# single batch despite a high BatchSettings.max_bytes limit, there should
# be no "InvalidArgument: request_size is too large" error.
for future in futures:
result = future.result(timeout=10)
assert isinstance(result, six.string_types) # the message ID
def __init__(self, batch_settings=(), **kwargs):
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "channel" not in kwargs:
kwargs["channel"] = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
self._batches = {}
def test_will_accept_oversize():
batch = create_batch(
settings=types.BatchSettings(max_bytes=10),
status=BatchStatus.ACCEPTING_MESSAGES,
)
message = types.PubsubMessage(data=b"abcdefghijklmnopqrstuvwxyz")
assert batch.will_accept(message) is True
def test_ordered_sequencer_cleaned_up(creds):
# Max latency is infinite so a commit thread is not created.
# We don't want a commit thread to interfere with this test.
batch_settings = types.BatchSettings(max_latency=float("inf"))
publisher_options = types.PublisherOptions(enable_message_ordering=True)
client = publisher.Client(
batch_settings=batch_settings,
publisher_options=publisher_options,
credentials=creds,
)
topic = "topic"
ordering_key = "ord_key"
sequencer = mock.Mock(spec=ordered_sequencer.OrderedSequencer)
sequencer.is_finished.return_value = False
client._set_sequencer(topic=topic,
sequencer=sequencer,
ordering_key=ordering_key)
assert len(client._sequencers) == 1
# 'sequencer' is not finished yet so don't remove it.
client._commit_sequencers()
assert len(client._sequencers) == 1
sequencer.is_finished.return_value = True
# 'sequencer' is finished so remove it.
client._commit_sequencers()
assert len(client._sequencers) == 0
def create_batch(topic="topic_name",
batch_done_callback=None,
commit_when_full=True,
commit_retry=gapic_v1.method.DEFAULT,
**batch_settings):
"""Return a batch object suitable for testing.
Args:
topic (str): Topic name.
batch_done_callback (Callable[bool]): A callable that is called when
the batch is done, either with a success or a failure flag.
commit_when_full (bool): Whether to commit the batch when the batch
has reached byte-size or number-of-messages limits.
commit_retry (Optional[google.api_core.retry.Retry]): The retry settings
for the batch commit call.
batch_settings (Mapping[str, str]): Arguments passed on to the
:class:``~.pubsub_v1.types.BatchSettings`` constructor.
Returns:
~.pubsub_v1.publisher.batch.thread.Batch: A batch object.
"""
client = create_client()
settings = types.BatchSettings(**batch_settings)
return Batch(
client,
topic,
settings,
batch_done_callback=batch_done_callback,
commit_when_full=commit_when_full,
commit_retry=commit_retry,
)
def test_will_not_accept_number():
batch = create_batch(
settings=types.BatchSettings(max_messages=-1),
status=BatchStatus.ACCEPTING_MESSAGES,
)
message = types.PubsubMessage(data=b"abc")
assert batch.will_accept(message) is False
def __init__(self, batch_settings=(), batch_class=thread.Batch, **kwargs):
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get('PUBSUB_EMULATOR_HOST'):
kwargs['channel'] = grpc.insecure_channel(
target=os.environ.get('PUBSUB_EMULATOR_HOST'),
)
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if 'channel' not in kwargs:
kwargs['channel'] = grpc_helpers.create_channel(
credentials=kwargs.get('credentials', None),
target=self.target,
scopes=publisher_client.PublisherClient._ALL_SCOPES,
options={
'grpc.max_send_message_length': -1,
'grpc.max_receive_message_length': -1,
}.items(),
)
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
kwargs['lib_name'] = 'gccl'
kwargs['lib_version'] = __VERSION__
self.api = publisher_client.PublisherClient(**kwargs)
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_class = batch_class
self._batch_lock = threading.Lock()
self._batches = {}
def test_commit_thread_not_created_on_publish_if_max_latency_is_inf(creds):
# Max latency is infinite so a commit thread is not created.
batch_settings = types.BatchSettings(max_latency=float("inf"))
client = publisher.Client(batch_settings=batch_settings, credentials=creds)
assert client.publish("topic", b"bytestring body",
ordering_key="") is not None
assert client._commit_thread is None
def test_publish_data_too_large():
creds = mock.Mock(spec=credentials.Credentials)
client = publisher.Client(credentials=creds)
topic = 'topic/path'
client.batch_settings = types.BatchSettings(
0, client.batch_settings.max_latency,
client.batch_settings.max_messages)
with pytest.raises(ValueError):
client.publish(topic, b'This is a text string.')
def __init__(self,
name,
dist,
emulator_host,
emulator_port,
max_bytes,
max_latency,
max_messages,
ordering=False,
client_options=None,
credentials=None,
services_service=None,
**config):
services_service(plugin.Plugin.__init__,
self,
name,
dist,
emulator_host=emulator_host,
emulator_port=emulator_port,
max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages,
ordering=False,
client_options=None,
credentials=None,
**config)
batch_settings = types.BatchSettings(max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages)
publisher_options = types.PublisherOptions(
enable_message_ordering=ordering)
settings = {}
if client_options is not None:
if isinstance(client_options, (str, type(u''))):
client_options = services_service(
reference.load_object(client_options)[0])
settings['client_options'] = client_options
if emulator_host:
channel = grpc.insecure_channel('{}:{}'.format(
emulator_host, emulator_port))
transport = PublisherGrpcTransport(channel=channel)
else:
transport = None
if credentials is not None:
settings['credentials'] = services_service(
reference.load_object(credentials)[0])
self.__class__.proxy_target = PublisherClient(batch_settings,
publisher_options,
transport=transport,
**settings)
def __init__(self, batch_settings=(), batch_class=thread.Batch, **kwargs):
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
kwargs['lib_name'] = 'gccl'
kwargs['lib_version'] = __VERSION__
self.api = publisher_client.PublisherClient(**kwargs)
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_class = batch_class
self._batch_lock = threading.Lock()
self._batches = {}
def test_init():
"""Establish that a monitor thread is usually created on init."""
client = create_client()
# Do not actually create a thread, but do verify that one was created;
# it should be running the batch's "monitor" method (which commits the
# batch once time elapses).
with mock.patch.object(threading, 'Thread', autospec=True) as Thread:
batch = Batch(client, 'topic_name', types.BatchSettings())
Thread.assert_called_once_with(target=batch.monitor)
# New batches start able to accept messages by default.
assert batch.status == BatchStatus.ACCEPTING_MESSAGES
def create_batch(status, settings=types.BatchSettings()):
"""Create a batch object, which does not commit.
Args:
status (str): The batch's internal status will be set to the provided status.
Returns:
~.pubsub_v1.publisher.batch.thread.Batch: The batch object
"""
creds = mock.Mock(spec=credentials.Credentials)
client = publisher.Client(credentials=creds)
batch = Batch(client, "topic_name", settings)
batch._status = status
return batch
def create_batch(autocommit=False, **batch_settings):
"""Return a batch object suitable for testing.
Args:
autocommit (bool): Whether the batch should commit after
``max_latency`` seconds. By default, this is ``False``
for unit testing.
kwargs (dict): Arguments passed on to the
:class:``~.pubsub_v1.types.BatchSettings`` constructor.
Returns:
~.pubsub_v1.publisher.batch.thread.Batch: A batch object.
"""
client = create_client()
settings = types.BatchSettings(**batch_settings)
return Batch(client, 'topic_name', settings, autocommit=autocommit)
def test_wait_and_commit_sequencers(creds):
# Max latency is infinite so a commit thread is not created.
# We don't want a commit thread to interfere with this test.
batch_settings = types.BatchSettings(max_latency=float("inf"))
client = publisher.Client(batch_settings=batch_settings, credentials=creds)
# Mock out time so no sleep is actually done.
with mock.patch.object(time, "sleep"):
with mock.patch.object(client,
"_commit_sequencers") as _commit_sequencers:
assert (client.publish("topic",
b"bytestring body",
ordering_key="") is not None)
# Call _wait_and_commit_sequencers to simulate what would happen if a
# commit thread actually ran.
client._wait_and_commit_sequencers()
assert _commit_sequencers.call_count == 1
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["client_options"] = {
"api_endpoint": os.environ.get("PUBSUB_EMULATOR_HOST")
}
kwargs["credentials"] = AnonymousCredentials()
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._target = self.api._transport._host
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
def test_commit_thread_created_on_publish(creds):
# Max latency is not infinite so a commit thread is created.
batch_settings = types.BatchSettings(max_latency=600)
client = publisher.Client(batch_settings=batch_settings, credentials=creds)
with mock.patch.object(client, "_start_commit_thread",
autospec=True) as _start_commit_thread:
# First publish should create a commit thread.
assert client.publish("topic", b"bytestring body",
ordering_key="") is not None
_start_commit_thread.assert_called_once()
# Since _start_commit_thread is a mock, no actual thread has been
# created, so let's put a sentinel there to mimic real behavior.
client._commit_thread = mock.Mock()
# Second publish should not create a commit thread since one (the mock)
# already exists.
assert client.publish("topic", b"bytestring body",
ordering_key="") is not None
# Call count should remain 1.
_start_commit_thread.assert_called_once()
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
client_options = kwargs.pop("client_options", None)
if (client_options and "api_endpoint" in client_options and isinstance(
client_options["api_endpoint"], six.string_types)):
self._target = client_options["api_endpoint"]
else:
self._target = publisher_client.PublisherClient.SERVICE_ADDRESS
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "transport" not in kwargs:
channel = kwargs.pop("channel", None)
if channel is None:
channel = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# cannot pass both 'channel' and 'credentials'
kwargs.pop("credentials", None)
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
kwargs["transport"] = transport
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
if self._enable_message_ordering:
# Set retry timeout to "infinite" when message ordering is enabled.
# Note that this then also impacts messages added with an empty ordering
# key.
client_config = _set_nested_value(
kwargs.pop("client_config", {}),
2**32,
[
"interfaces",
"google.pubsub.v1.Publisher",
"retry_params",
"messaging",
"total_timeout_millis",
],
)
kwargs["client_config"] = client_config
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
def test_client():
client = create_client()
settings = types.BatchSettings()
batch = Batch(client, "topic_name", settings)
assert batch.client is client
def test_client():
client = create_client()
settings = types.BatchSettings()
batch = Batch(client, 'topic_name', settings, autocommit=False)
assert batch.client is client
async def test_batch_reject_message(client: PublisherClient):
client.batch_settings = types.BatchSettings(1, 0, 1)
with pytest.raises(ValueError):
client.publish("", b"..")
for batch in client._batches.values():
await batch.result
class Publisher(plugin.Plugin):
"""
"""
LOAD_PRIORITY = 10
CONFIG_SPEC = dict(plugin.Plugin.CONFIG_SPEC,
emulator_host='string(default="")',
emulator_port='integer(default=8085)',
max_bytes='integer(default={})'.format(
types.BatchSettings().max_bytes),
max_latency='float(default={})'.format(
types.BatchSettings().max_latency),
max_messages='integer(default={})'.format(
types.BatchSettings().max_messages),
ordering='boolean(default=False)',
client_options='string(default=None)',
credentials='string(default=None)')
proxy_target = None
def __init__(self,
name,
dist,
emulator_host,
emulator_port,
max_bytes,
max_latency,
max_messages,
ordering=False,
client_options=None,
credentials=None,
services_service=None,
**config):
services_service(plugin.Plugin.__init__,
self,
name,
dist,
emulator_host=emulator_host,
emulator_port=emulator_port,
max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages,
ordering=False,
client_options=None,
credentials=None,
**config)
batch_settings = types.BatchSettings(max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages)
publisher_options = types.PublisherOptions(
enable_message_ordering=ordering)
settings = {}
if client_options is not None:
if isinstance(client_options, (str, type(u''))):
client_options = services_service(
reference.load_object(client_options)[0])
settings['client_options'] = client_options
if emulator_host:
channel = grpc.insecure_channel('{}:{}'.format(
emulator_host, emulator_port))
transport = PublisherGrpcTransport(channel=channel)
else:
transport = None
if credentials is not None:
settings['credentials'] = services_service(
reference.load_object(credentials)[0])
self.__class__.proxy_target = PublisherClient(batch_settings,
publisher_options,
transport=transport,
**settings)
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
# The GAPIC client has mTLS logic to determine the api endpoint and the
# ssl credentials to use. Here we create a GAPIC client to help compute the
# api endpoint and ssl credentials. The api endpoint will be used to set
# `self._target`, and ssl credentials will be passed to
# `grpc_helpers.create_channel` to establish a mTLS channel (if ssl
# credentials is not None).
client_options = kwargs.get("client_options", None)
credentials = kwargs.get("credentials", None)
client_for_mtls_info = publisher_client.PublisherClient(
credentials=credentials, client_options=client_options)
self._target = client_for_mtls_info._transport._host
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "transport" not in kwargs:
channel = kwargs.pop("channel", None)
if channel is None:
channel = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
ssl_credentials=client_for_mtls_info._transport.
_ssl_channel_credentials,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# cannot pass both 'channel' and 'credentials'
kwargs.pop("credentials", None)
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
kwargs["transport"] = transport
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
def __init__(self, batch_settings: dict = {}):
self._client = pubsub_v1.PublisherClient(
batch_settings=types.BatchSettings(**batch_settings), )
