def test_publish_with_ordering_key():
creds = mock.Mock(spec=credentials.Credentials)
publisher_options = types.PublisherOptions(enable_message_ordering=True)
client = publisher.Client(publisher_options=publisher_options,
credentials=creds)
# Use a mock in lieu of the actual batch class.
batch = mock.Mock(spec=client._batch_class)
# Set the mock up to claim indiscriminately that it accepts all messages.
batch.will_accept.return_value = True
batch.publish.side_effect = (mock.sentinel.future1, mock.sentinel.future2)
topic = "topic/path"
ordering_key = "k1"
client._set_batch(topic, batch, ordering_key=ordering_key)
# Begin publishing.
future1 = client.publish(topic, b"spam", ordering_key=ordering_key)
future2 = client.publish(topic,
b"foo",
bar="baz",
ordering_key=ordering_key)
assert future1 is mock.sentinel.future1
assert future2 is mock.sentinel.future2
# Check mock.
batch.publish.assert_has_calls([
mock.call(types.PubsubMessage(data=b"spam", ordering_key="k1")),
mock.call(
types.PubsubMessage(data=b"foo",
attributes={"bar": "baz"},
ordering_key="k1")),
])
def test_ordered_sequencer_cleaned_up(creds):
# Max latency is infinite so a commit thread is not created.
# We don't want a commit thread to interfere with this test.
batch_settings = types.BatchSettings(max_latency=float("inf"))
publisher_options = types.PublisherOptions(enable_message_ordering=True)
client = publisher.Client(
batch_settings=batch_settings,
publisher_options=publisher_options,
credentials=creds,
)
topic = "topic"
ordering_key = "ord_key"
sequencer = mock.Mock(spec=ordered_sequencer.OrderedSequencer)
sequencer.is_finished.return_value = False
client._set_sequencer(topic=topic,
sequencer=sequencer,
ordering_key=ordering_key)
assert len(client._sequencers) == 1
# 'sequencer' is not finished yet so don't remove it.
client._commit_sequencers()
assert len(client._sequencers) == 1
sequencer.is_finished.return_value = True
# 'sequencer' is finished so remove it.
client._commit_sequencers()
assert len(client._sequencers) == 0
def test_resume_publish_ordering_keys_not_enabled(creds):
publisher_options = types.PublisherOptions(enable_message_ordering=False)
client = publisher.Client(publisher_options=publisher_options,
credentials=creds)
# Throw on calling resume_publish() when enable_message_ordering is False.
with pytest.raises(ValueError):
client.resume_publish("topic", "ord_key")
def test_resume_publish_no_sequencer_found(creds):
publisher_options = types.PublisherOptions(enable_message_ordering=True)
client = publisher.Client(publisher_options=publisher_options,
credentials=creds)
# Check no exception is thrown if a sequencer with the (topic, ordering_key)
# pair does not exist.
client.resume_publish("topic", "ord_key")
def test_publish_empty_ordering_key_when_message_ordering_enabled(creds):
client = publisher.Client(
publisher_options=types.PublisherOptions(enable_message_ordering=True),
credentials=creds,
)
topic = "topic/path"
assert client.publish(topic, b"bytestring body",
ordering_key="") is not None
def test_message_ordering_enabled(creds):
client = publisher.Client(credentials=creds)
assert not client._enable_message_ordering
client = publisher.Client(
publisher_options=types.PublisherOptions(enable_message_ordering=True),
credentials=creds,
)
assert client._enable_message_ordering
def __init__(self,
name,
dist,
emulator_host,
emulator_port,
max_bytes,
max_latency,
max_messages,
ordering=False,
client_options=None,
credentials=None,
services_service=None,
**config):
services_service(plugin.Plugin.__init__,
self,
name,
dist,
emulator_host=emulator_host,
emulator_port=emulator_port,
max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages,
ordering=False,
client_options=None,
credentials=None,
**config)
batch_settings = types.BatchSettings(max_bytes=max_bytes,
max_latency=max_latency,
max_messages=max_messages)
publisher_options = types.PublisherOptions(
enable_message_ordering=ordering)
settings = {}
if client_options is not None:
if isinstance(client_options, (str, type(u''))):
client_options = services_service(
reference.load_object(client_options)[0])
settings['client_options'] = client_options
if emulator_host:
channel = grpc.insecure_channel('{}:{}'.format(
emulator_host, emulator_port))
transport = PublisherGrpcTransport(channel=channel)
else:
transport = None
if credentials is not None:
settings['credentials'] = services_service(
reference.load_object(credentials)[0])
self.__class__.proxy_target = PublisherClient(batch_settings,
publisher_options,
transport=transport,
**settings)
def test_message_ordering_changes_retry_deadline():
creds = mock.Mock(spec=credentials.Credentials)
client = publisher.Client(credentials=creds)
assert client.api._method_configs["Publish"].retry._deadline == 60
client = publisher.Client(
publisher_options=types.PublisherOptions(enable_message_ordering=True),
credentials=creds,
)
assert client.api._method_configs["Publish"].retry._deadline == 2 ** 32 / 1000
def test_resume_publish():
creds = mock.Mock(spec=credentials.Credentials)
publisher_options = types.PublisherOptions(enable_message_ordering=True)
client = publisher.Client(publisher_options=publisher_options, credentials=creds)
topic = "topic"
ordering_key = "ord_key"
sequencer = mock.Mock(spec=ordered_sequencer.OrderedSequencer)
client._set_sequencer(topic=topic, sequencer=sequencer, ordering_key=ordering_key)
client.resume_publish(topic, ordering_key)
assert sequencer.unpause.called_once()
def test_publish_custom_timeout_overrides_configured_timeout(creds):
client = publisher.Client(
credentials=creds,
publisher_options=types.PublisherOptions(
timeout=mock.sentinel.publish_timeout),
)
topic = "topic/path"
client._flow_controller = mock.Mock()
fake_sequencer = mock.Mock()
client._get_or_create_sequencer = mock.Mock(return_value=fake_sequencer)
client.publish(topic, b"hello!", timeout=mock.sentinel.custom_timeout)
fake_sequencer.publish.assert_called_once_with(
mock.ANY, retry=mock.ANY, timeout=mock.sentinel.custom_timeout)
message = fake_sequencer.publish.call_args.args[0]
assert message.data == b"hello!"
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["client_options"] = {
"api_endpoint": os.environ.get("PUBSUB_EMULATOR_HOST")
}
kwargs["credentials"] = AnonymousCredentials()
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._target = self.api._transport._host
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
def test_publish_error_exceeding_flow_control_limits(creds):
publisher_options = types.PublisherOptions(
flow_control=types.PublishFlowControl(
message_limit=10,
byte_limit=150,
limit_exceeded_behavior=types.LimitExceededBehavior.ERROR,
))
client = publisher.Client(credentials=creds,
publisher_options=publisher_options)
mock_batch = mock.Mock(spec=client._batch_class)
topic = "topic/path"
client._set_batch(topic, mock_batch)
future1 = client.publish(topic, b"a" * 100)
future2 = client.publish(topic, b"b" * 100)
future1.result() # no error, still within flow control limits
with pytest.raises(exceptions.FlowControlLimitError):
future2.result()
def test_publish_with_ordering_key_uses_extended_retry_deadline():
creds = mock.Mock(spec=credentials.Credentials)
client = publisher.Client(
credentials=creds,
publisher_options=types.PublisherOptions(enable_message_ordering=True),
)
# Use mocks in lieu of the actual batch class.
batch = mock.Mock(spec=client._batch_class)
future = mock.sentinel.future
future.add_done_callback = mock.Mock(spec=["__call__"])
batch.publish.return_value = future
topic = "topic/path"
client._set_batch(topic, batch)
# Actually mock the batch class now.
batch_class = mock.Mock(spec=(), return_value=batch)
client._set_batch_class(batch_class)
# Publish a message with custom retry settings.
custom_retry = retries.Retry(
initial=1,
maximum=20,
multiplier=3.3,
deadline=999,
predicate=retries.if_exception_type(TimeoutError, KeyboardInterrupt),
)
future = client.publish(topic,
b"foo",
ordering_key="first",
retry=custom_retry)
assert future is mock.sentinel.future
# Check the retry settings used for the batch.
batch_class.assert_called_once()
_, kwargs = batch_class.call_args
batch_commit_retry = kwargs["commit_retry"]
expected_retry = custom_retry.with_deadline(2.0**32)
_assert_retries_equal(batch_commit_retry, expected_retry)
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
# The GAPIC client has mTLS logic to determine the api endpoint and the
# ssl credentials to use. Here we create a GAPIC client to help compute the
# api endpoint and ssl credentials. The api endpoint will be used to set
# `self._target`, and ssl credentials will be passed to
# `grpc_helpers.create_channel` to establish a mTLS channel (if ssl
# credentials is not None).
client_options = kwargs.get("client_options", None)
credentials = kwargs.get("credentials", None)
client_for_mtls_info = publisher_client.PublisherClient(
credentials=credentials, client_options=client_options)
self._target = client_for_mtls_info._transport._host
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "transport" not in kwargs:
channel = kwargs.pop("channel", None)
if channel is None:
channel = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
ssl_credentials=client_for_mtls_info._transport.
_ssl_channel_credentials,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# cannot pass both 'channel' and 'credentials'
kwargs.pop("credentials", None)
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
kwargs["transport"] = transport
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
def __init__(self, batch_settings=(), publisher_options=(), **kwargs):
assert (
type(batch_settings) is types.BatchSettings
or len(batch_settings) == 0
), "batch_settings must be of type BatchSettings or an empty tuple."
assert (
type(publisher_options) is types.PublisherOptions
or len(publisher_options) == 0
), "publisher_options must be of type PublisherOptions or an empty tuple."
# Sanity check: Is our goal to use the emulator?
# If so, create a grpc insecure channel with the emulator host
# as the target.
if os.environ.get("PUBSUB_EMULATOR_HOST"):
kwargs["channel"] = grpc.insecure_channel(
target=os.environ.get("PUBSUB_EMULATOR_HOST"))
client_options = kwargs.pop("client_options", None)
if (client_options and "api_endpoint" in client_options and isinstance(
client_options["api_endpoint"], six.string_types)):
self._target = client_options["api_endpoint"]
else:
self._target = publisher_client.PublisherClient.SERVICE_ADDRESS
# Use a custom channel.
# We need this in order to set appropriate default message size and
# keepalive options.
if "transport" not in kwargs:
channel = kwargs.pop("channel", None)
if channel is None:
channel = grpc_helpers.create_channel(
credentials=kwargs.pop("credentials", None),
target=self.target,
scopes=publisher_client.PublisherClient._DEFAULT_SCOPES,
options={
"grpc.max_send_message_length": -1,
"grpc.max_receive_message_length": -1,
}.items(),
)
# cannot pass both 'channel' and 'credentials'
kwargs.pop("credentials", None)
transport = publisher_grpc_transport.PublisherGrpcTransport(
channel=channel)
kwargs["transport"] = transport
# For a transient failure, retry publishing the message infinitely.
self.publisher_options = types.PublisherOptions(*publisher_options)
self._enable_message_ordering = self.publisher_options[0]
if self._enable_message_ordering:
# Set retry timeout to "infinite" when message ordering is enabled.
# Note that this then also impacts messages added with an empty ordering
# key.
client_config = _set_nested_value(
kwargs.pop("client_config", {}),
2**32,
[
"interfaces",
"google.pubsub.v1.Publisher",
"retry_params",
"messaging",
"total_timeout_millis",
],
)
kwargs["client_config"] = client_config
# Add the metrics headers, and instantiate the underlying GAPIC
# client.
self.api = publisher_client.PublisherClient(**kwargs)
self._batch_class = thread.Batch
self.batch_settings = types.BatchSettings(*batch_settings)
# The batches on the publisher client are responsible for holding
# messages. One batch exists for each topic.
self._batch_lock = self._batch_class.make_lock()
# (topic, ordering_key) => sequencers object
self._sequencers = {}
self._is_stopped = False
# Thread created to commit all sequencers after a timeout.
self._commit_thread = None
# The object controlling the message publishing flow
self._flow_controller = FlowController(
self.publisher_options.flow_control)
