def test_publish_after_batch_error(): client = create_client() message = create_message() batch = client._batch_class(client, "topic_name", types.BatchSettings(max_latency=float("inf"))) batch._messages.append( mock.Mock(name="message")) # Make batch truthy (non-empty). sequencer = unordered_sequencer.UnorderedSequencer(client, "topic_name") sequencer._set_batch(batch) with mock.patch.object(batch, "commit") as fake_batch_commit: sequencer.commit() fake_batch_commit.assert_called_once() # Simulate publish RPC failing. batch._set_status(base.BatchStatus.ERROR) # Will create a new batch since the old one has been committed. The fact # that the old batch errored should not matter in the publish of the next # message. future = sequencer.publish(message) assert future is not None
def test_publish_large_messages(publisher, topic_path, cleanup): # Make sure the topic gets deleted. cleanup.append((publisher.delete_topic, topic_path)) # Each message should be smaller than 10**7 bytes (the server side limit for # PublishRequest), but all messages combined in a PublishRequest should # slightly exceed that threshold to make sure the publish code handles these # cases well. # Mind that the total PublishRequest size must still be smaller than # 10 * 1024 * 1024 bytes in order to not exceed the max request body size limit. msg_data = b"x" * (2 * 10 ** 6) publisher.batch_settings = types.BatchSettings( max_bytes=11 * 1000 * 1000, # more than the server limit of 10 ** 7 max_latency=2.0, # so that autocommit happens after publishing all messages max_messages=100, ) publisher.create_topic(topic_path) futures = [publisher.publish(topic_path, msg_data, num=str(i)) for i in range(5)] # If the publishing logic correctly split all messages into more than a # single batch despite a high BatchSettings.max_bytes limit, there should # be no "InvalidArgument: request_size is too large" error. for future in futures: result = future.result(timeout=10) assert isinstance(result, six.string_types) # the message ID
def __init__(self, batch_settings=(), **kwargs): # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get("PUBSUB_EMULATOR_HOST"): kwargs["channel"] = grpc.insecure_channel( target=os.environ.get("PUBSUB_EMULATOR_HOST")) # Use a custom channel. # We need this in order to set appropriate default message size and # keepalive options. if "channel" not in kwargs: kwargs["channel"] = grpc_helpers.create_channel( credentials=kwargs.pop("credentials", None), target=self.target, scopes=publisher_client.PublisherClient._DEFAULT_SCOPES, options={ "grpc.max_send_message_length": -1, "grpc.max_receive_message_length": -1, }.items(), ) # Add the metrics headers, and instantiate the underlying GAPIC # client. self.api = publisher_client.PublisherClient(**kwargs) self.batch_settings = types.BatchSettings(*batch_settings) # The batches on the publisher client are responsible for holding # messages. One batch exists for each topic. self._batch_lock = self._batch_class.make_lock() self._batches = {}
def test_will_accept_oversize(): batch = create_batch( settings=types.BatchSettings(max_bytes=10), status=BatchStatus.ACCEPTING_MESSAGES, ) message = types.PubsubMessage(data=b"abcdefghijklmnopqrstuvwxyz") assert batch.will_accept(message) is True
def test_ordered_sequencer_cleaned_up(creds): # Max latency is infinite so a commit thread is not created. # We don't want a commit thread to interfere with this test. batch_settings = types.BatchSettings(max_latency=float("inf")) publisher_options = types.PublisherOptions(enable_message_ordering=True) client = publisher.Client( batch_settings=batch_settings, publisher_options=publisher_options, credentials=creds, ) topic = "topic" ordering_key = "ord_key" sequencer = mock.Mock(spec=ordered_sequencer.OrderedSequencer) sequencer.is_finished.return_value = False client._set_sequencer(topic=topic, sequencer=sequencer, ordering_key=ordering_key) assert len(client._sequencers) == 1 # 'sequencer' is not finished yet so don't remove it. client._commit_sequencers() assert len(client._sequencers) == 1 sequencer.is_finished.return_value = True # 'sequencer' is finished so remove it. client._commit_sequencers() assert len(client._sequencers) == 0
def create_batch(topic="topic_name", batch_done_callback=None, commit_when_full=True, commit_retry=gapic_v1.method.DEFAULT, **batch_settings): """Return a batch object suitable for testing. Args: topic (str): Topic name. batch_done_callback (Callable[bool]): A callable that is called when the batch is done, either with a success or a failure flag. commit_when_full (bool): Whether to commit the batch when the batch has reached byte-size or number-of-messages limits. commit_retry (Optional[google.api_core.retry.Retry]): The retry settings for the batch commit call. batch_settings (Mapping[str, str]): Arguments passed on to the :class:``~.pubsub_v1.types.BatchSettings`` constructor. Returns: ~.pubsub_v1.publisher.batch.thread.Batch: A batch object. """ client = create_client() settings = types.BatchSettings(**batch_settings) return Batch( client, topic, settings, batch_done_callback=batch_done_callback, commit_when_full=commit_when_full, commit_retry=commit_retry, )
def test_will_not_accept_number(): batch = create_batch( settings=types.BatchSettings(max_messages=-1), status=BatchStatus.ACCEPTING_MESSAGES, ) message = types.PubsubMessage(data=b"abc") assert batch.will_accept(message) is False
def __init__(self, batch_settings=(), batch_class=thread.Batch, **kwargs): # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get('PUBSUB_EMULATOR_HOST'): kwargs['channel'] = grpc.insecure_channel( target=os.environ.get('PUBSUB_EMULATOR_HOST'), ) # Use a custom channel. # We need this in order to set appropriate default message size and # keepalive options. if 'channel' not in kwargs: kwargs['channel'] = grpc_helpers.create_channel( credentials=kwargs.get('credentials', None), target=self.target, scopes=publisher_client.PublisherClient._ALL_SCOPES, options={ 'grpc.max_send_message_length': -1, 'grpc.max_receive_message_length': -1, }.items(), ) # Add the metrics headers, and instantiate the underlying GAPIC # client. kwargs['lib_name'] = 'gccl' kwargs['lib_version'] = __VERSION__ self.api = publisher_client.PublisherClient(**kwargs) self.batch_settings = types.BatchSettings(*batch_settings) # The batches on the publisher client are responsible for holding # messages. One batch exists for each topic. self._batch_class = batch_class self._batch_lock = threading.Lock() self._batches = {}
def test_commit_thread_not_created_on_publish_if_max_latency_is_inf(creds): # Max latency is infinite so a commit thread is not created. batch_settings = types.BatchSettings(max_latency=float("inf")) client = publisher.Client(batch_settings=batch_settings, credentials=creds) assert client.publish("topic", b"bytestring body", ordering_key="") is not None assert client._commit_thread is None
def test_publish_data_too_large(): creds = mock.Mock(spec=credentials.Credentials) client = publisher.Client(credentials=creds) topic = 'topic/path' client.batch_settings = types.BatchSettings( 0, client.batch_settings.max_latency, client.batch_settings.max_messages) with pytest.raises(ValueError): client.publish(topic, b'This is a text string.')
def __init__(self, name, dist, emulator_host, emulator_port, max_bytes, max_latency, max_messages, ordering=False, client_options=None, credentials=None, services_service=None, **config): services_service(plugin.Plugin.__init__, self, name, dist, emulator_host=emulator_host, emulator_port=emulator_port, max_bytes=max_bytes, max_latency=max_latency, max_messages=max_messages, ordering=False, client_options=None, credentials=None, **config) batch_settings = types.BatchSettings(max_bytes=max_bytes, max_latency=max_latency, max_messages=max_messages) publisher_options = types.PublisherOptions( enable_message_ordering=ordering) settings = {} if client_options is not None: if isinstance(client_options, (str, type(u''))): client_options = services_service( reference.load_object(client_options)[0]) settings['client_options'] = client_options if emulator_host: channel = grpc.insecure_channel('{}:{}'.format( emulator_host, emulator_port)) transport = PublisherGrpcTransport(channel=channel) else: transport = None if credentials is not None: settings['credentials'] = services_service( reference.load_object(credentials)[0]) self.__class__.proxy_target = PublisherClient(batch_settings, publisher_options, transport=transport, **settings)
def __init__(self, batch_settings=(), batch_class=thread.Batch, **kwargs): # Add the metrics headers, and instantiate the underlying GAPIC # client. kwargs['lib_name'] = 'gccl' kwargs['lib_version'] = __VERSION__ self.api = publisher_client.PublisherClient(**kwargs) self.batch_settings = types.BatchSettings(*batch_settings) # The batches on the publisher client are responsible for holding # messages. One batch exists for each topic. self._batch_class = batch_class self._batch_lock = threading.Lock() self._batches = {}
def test_init(): """Establish that a monitor thread is usually created on init.""" client = create_client() # Do not actually create a thread, but do verify that one was created; # it should be running the batch's "monitor" method (which commits the # batch once time elapses). with mock.patch.object(threading, 'Thread', autospec=True) as Thread: batch = Batch(client, 'topic_name', types.BatchSettings()) Thread.assert_called_once_with(target=batch.monitor) # New batches start able to accept messages by default. assert batch.status == BatchStatus.ACCEPTING_MESSAGES
def create_batch(status, settings=types.BatchSettings()): """Create a batch object, which does not commit. Args: status (str): The batch's internal status will be set to the provided status. Returns: ~.pubsub_v1.publisher.batch.thread.Batch: The batch object """ creds = mock.Mock(spec=credentials.Credentials) client = publisher.Client(credentials=creds) batch = Batch(client, "topic_name", settings) batch._status = status return batch
def create_batch(autocommit=False, **batch_settings): """Return a batch object suitable for testing. Args: autocommit (bool): Whether the batch should commit after ``max_latency`` seconds. By default, this is ``False`` for unit testing. kwargs (dict): Arguments passed on to the :class:``~.pubsub_v1.types.BatchSettings`` constructor. Returns: ~.pubsub_v1.publisher.batch.thread.Batch: A batch object. """ client = create_client() settings = types.BatchSettings(**batch_settings) return Batch(client, 'topic_name', settings, autocommit=autocommit)
def test_wait_and_commit_sequencers(creds): # Max latency is infinite so a commit thread is not created. # We don't want a commit thread to interfere with this test. batch_settings = types.BatchSettings(max_latency=float("inf")) client = publisher.Client(batch_settings=batch_settings, credentials=creds) # Mock out time so no sleep is actually done. with mock.patch.object(time, "sleep"): with mock.patch.object(client, "_commit_sequencers") as _commit_sequencers: assert (client.publish("topic", b"bytestring body", ordering_key="") is not None) # Call _wait_and_commit_sequencers to simulate what would happen if a # commit thread actually ran. client._wait_and_commit_sequencers() assert _commit_sequencers.call_count == 1
def __init__(self, batch_settings=(), publisher_options=(), **kwargs): assert ( type(batch_settings) is types.BatchSettings or len(batch_settings) == 0 ), "batch_settings must be of type BatchSettings or an empty tuple." assert ( type(publisher_options) is types.PublisherOptions or len(publisher_options) == 0 ), "publisher_options must be of type PublisherOptions or an empty tuple." # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get("PUBSUB_EMULATOR_HOST"): kwargs["client_options"] = { "api_endpoint": os.environ.get("PUBSUB_EMULATOR_HOST") } kwargs["credentials"] = AnonymousCredentials() # For a transient failure, retry publishing the message infinitely. self.publisher_options = types.PublisherOptions(*publisher_options) self._enable_message_ordering = self.publisher_options[0] # Add the metrics headers, and instantiate the underlying GAPIC # client. self.api = publisher_client.PublisherClient(**kwargs) self._target = self.api._transport._host self._batch_class = thread.Batch self.batch_settings = types.BatchSettings(*batch_settings) # The batches on the publisher client are responsible for holding # messages. One batch exists for each topic. self._batch_lock = self._batch_class.make_lock() # (topic, ordering_key) => sequencers object self._sequencers = {} self._is_stopped = False # Thread created to commit all sequencers after a timeout. self._commit_thread = None # The object controlling the message publishing flow self._flow_controller = FlowController( self.publisher_options.flow_control)
def test_commit_thread_created_on_publish(creds): # Max latency is not infinite so a commit thread is created. batch_settings = types.BatchSettings(max_latency=600) client = publisher.Client(batch_settings=batch_settings, credentials=creds) with mock.patch.object(client, "_start_commit_thread", autospec=True) as _start_commit_thread: # First publish should create a commit thread. assert client.publish("topic", b"bytestring body", ordering_key="") is not None _start_commit_thread.assert_called_once() # Since _start_commit_thread is a mock, no actual thread has been # created, so let's put a sentinel there to mimic real behavior. client._commit_thread = mock.Mock() # Second publish should not create a commit thread since one (the mock) # already exists. assert client.publish("topic", b"bytestring body", ordering_key="") is not None # Call count should remain 1. _start_commit_thread.assert_called_once()
def __init__(self, batch_settings=(), publisher_options=(), **kwargs): assert ( type(batch_settings) is types.BatchSettings or len(batch_settings) == 0 ), "batch_settings must be of type BatchSettings or an empty tuple." assert ( type(publisher_options) is types.PublisherOptions or len(publisher_options) == 0 ), "publisher_options must be of type PublisherOptions or an empty tuple." # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get("PUBSUB_EMULATOR_HOST"): kwargs["channel"] = grpc.insecure_channel( target=os.environ.get("PUBSUB_EMULATOR_HOST")) client_options = kwargs.pop("client_options", None) if (client_options and "api_endpoint" in client_options and isinstance( client_options["api_endpoint"], six.string_types)): self._target = client_options["api_endpoint"] else: self._target = publisher_client.PublisherClient.SERVICE_ADDRESS # Use a custom channel. # We need this in order to set appropriate default message size and # keepalive options. if "transport" not in kwargs: channel = kwargs.pop("channel", None) if channel is None: channel = grpc_helpers.create_channel( credentials=kwargs.pop("credentials", None), target=self.target, scopes=publisher_client.PublisherClient._DEFAULT_SCOPES, options={ "grpc.max_send_message_length": -1, "grpc.max_receive_message_length": -1, }.items(), ) # cannot pass both 'channel' and 'credentials' kwargs.pop("credentials", None) transport = publisher_grpc_transport.PublisherGrpcTransport( channel=channel) kwargs["transport"] = transport # For a transient failure, retry publishing the message infinitely. self.publisher_options = types.PublisherOptions(*publisher_options) self._enable_message_ordering = self.publisher_options[0] if self._enable_message_ordering: # Set retry timeout to "infinite" when message ordering is enabled. # Note that this then also impacts messages added with an empty ordering # key. client_config = _set_nested_value( kwargs.pop("client_config", {}), 2**32, [ "interfaces", "google.pubsub.v1.Publisher", "retry_params", "messaging", "total_timeout_millis", ], ) kwargs["client_config"] = client_config # Add the metrics headers, and instantiate the underlying GAPIC # client. self.api = publisher_client.PublisherClient(**kwargs) self._batch_class = thread.Batch self.batch_settings = types.BatchSettings(*batch_settings) # The batches on the publisher client are responsible for holding # messages. One batch exists for each topic. self._batch_lock = self._batch_class.make_lock() # (topic, ordering_key) => sequencers object self._sequencers = {} self._is_stopped = False # Thread created to commit all sequencers after a timeout. self._commit_thread = None # The object controlling the message publishing flow self._flow_controller = FlowController( self.publisher_options.flow_control)
def test_client(): client = create_client() settings = types.BatchSettings() batch = Batch(client, "topic_name", settings) assert batch.client is client
def test_client(): client = create_client() settings = types.BatchSettings() batch = Batch(client, 'topic_name', settings, autocommit=False) assert batch.client is client
async def test_batch_reject_message(client: PublisherClient): client.batch_settings = types.BatchSettings(1, 0, 1) with pytest.raises(ValueError): client.publish("", b"..") for batch in client._batches.values(): await batch.result
class Publisher(plugin.Plugin): """ """ LOAD_PRIORITY = 10 CONFIG_SPEC = dict(plugin.Plugin.CONFIG_SPEC, emulator_host='string(default="")', emulator_port='integer(default=8085)', max_bytes='integer(default={})'.format( types.BatchSettings().max_bytes), max_latency='float(default={})'.format( types.BatchSettings().max_latency), max_messages='integer(default={})'.format( types.BatchSettings().max_messages), ordering='boolean(default=False)', client_options='string(default=None)', credentials='string(default=None)') proxy_target = None def __init__(self, name, dist, emulator_host, emulator_port, max_bytes, max_latency, max_messages, ordering=False, client_options=None, credentials=None, services_service=None, **config): services_service(plugin.Plugin.__init__, self, name, dist, emulator_host=emulator_host, emulator_port=emulator_port, max_bytes=max_bytes, max_latency=max_latency, max_messages=max_messages, ordering=False, client_options=None, credentials=None, **config) batch_settings = types.BatchSettings(max_bytes=max_bytes, max_latency=max_latency, max_messages=max_messages) publisher_options = types.PublisherOptions( enable_message_ordering=ordering) settings = {} if client_options is not None: if isinstance(client_options, (str, type(u''))): client_options = services_service( reference.load_object(client_options)[0]) settings['client_options'] = client_options if emulator_host: channel = grpc.insecure_channel('{}:{}'.format( emulator_host, emulator_port)) transport = PublisherGrpcTransport(channel=channel) else: transport = None if credentials is not None: settings['credentials'] = services_service( reference.load_object(credentials)[0]) self.__class__.proxy_target = PublisherClient(batch_settings, publisher_options, transport=transport, **settings)
def __init__(self, batch_settings=(), publisher_options=(), **kwargs): assert ( type(batch_settings) is types.BatchSettings or len(batch_settings) == 0 ), "batch_settings must be of type BatchSettings or an empty tuple." assert ( type(publisher_options) is types.PublisherOptions or len(publisher_options) == 0 ), "publisher_options must be of type PublisherOptions or an empty tuple." # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get("PUBSUB_EMULATOR_HOST"): kwargs["channel"] = grpc.insecure_channel( target=os.environ.get("PUBSUB_EMULATOR_HOST")) # The GAPIC client has mTLS logic to determine the api endpoint and the # ssl credentials to use. Here we create a GAPIC client to help compute the # api endpoint and ssl credentials. The api endpoint will be used to set # `self._target`, and ssl credentials will be passed to # `grpc_helpers.create_channel` to establish a mTLS channel (if ssl # credentials is not None). client_options = kwargs.get("client_options", None) credentials = kwargs.get("credentials", None) client_for_mtls_info = publisher_client.PublisherClient( credentials=credentials, client_options=client_options) self._target = client_for_mtls_info._transport._host # Use a custom channel. # We need this in order to set appropriate default message size and # keepalive options. if "transport" not in kwargs: channel = kwargs.pop("channel", None) if channel is None: channel = grpc_helpers.create_channel( credentials=kwargs.pop("credentials", None), target=self.target, ssl_credentials=client_for_mtls_info._transport. _ssl_channel_credentials, scopes=publisher_client.PublisherClient._DEFAULT_SCOPES, options={ "grpc.max_send_message_length": -1, "grpc.max_receive_message_length": -1, }.items(), ) # cannot pass both 'channel' and 'credentials' kwargs.pop("credentials", None) transport = publisher_grpc_transport.PublisherGrpcTransport( channel=channel) kwargs["transport"] = transport # For a transient failure, retry publishing the message infinitely. self.publisher_options = types.PublisherOptions(*publisher_options) self._enable_message_ordering = self.publisher_options[0] # Add the metrics headers, and instantiate the underlying GAPIC # client. self.api = publisher_client.PublisherClient(**kwargs) self._batch_class = thread.Batch self.batch_settings = types.BatchSettings(*batch_settings) # The batches on the publisher client are responsible for holding # messages. One batch exists for each topic. self._batch_lock = self._batch_class.make_lock() # (topic, ordering_key) => sequencers object self._sequencers = {} self._is_stopped = False # Thread created to commit all sequencers after a timeout. self._commit_thread = None # The object controlling the message publishing flow self._flow_controller = FlowController( self.publisher_options.flow_control)
def __init__(self, batch_settings: dict = {}): self._client = pubsub_v1.PublisherClient( batch_settings=types.BatchSettings(**batch_settings), )