def exp_backoff_with_max_retry_count_policy(self):
return RetryPolicy(
ExpBackoffPolicy(initial_delay_secs=0.1,
max_delay_secs=0.5,
backoff_factor=2),
max_retry_count=self.max_retry_count,
)
def __init__(self, producer_position_callback, dry_run=False):
self.producer_position_callback = producer_position_callback
self.dry_run = dry_run
self.kafka_client = KafkaClient(get_config().cluster_config.broker_list)
self.position_data_tracker = PositionDataTracker()
self._reset_message_buffer()
self.skip_messages_with_pii = get_config().skip_messages_with_pii
self._publish_retry_policy = RetryPolicy(
ExpBackoffPolicy(with_jitter=True),
max_retry_count=get_config().producer_max_publish_retry_count
)
self._automatic_flush_enabled = True
def __init__(self,
consumer_name,
team_name,
expected_frequency_seconds,
topic_to_consumer_topic_state_map=None,
consumer_source=None,
force_payload_decode=True,
auto_offset_reset='smallest',
partitioner_cooldown=get_config(
).consumer_partitioner_cooldown_default,
use_group_sha=get_config().consumer_use_group_sha_default,
topic_refresh_frequency_seconds=get_config().
topic_refresh_frequency_seconds,
pre_rebalance_callback=None,
post_rebalance_callback=None,
fetch_offsets_for_topics=None,
pre_topic_refresh_callback=None,
cluster_name=None):
super(BaseConsumer, self).__init__(consumer_name,
team_name,
expected_frequency_seconds,
monitoring_enabled=False)
if ((topic_to_consumer_topic_state_map and consumer_source) or
(not topic_to_consumer_topic_state_map and not consumer_source)):
raise ValueError(
"Exactly one of topic_to_consumer_topic_state_map "
"or consumer_source must be specified")
self.consumer_source = consumer_source
self.topic_to_consumer_topic_state_map = topic_to_consumer_topic_state_map
self.force_payload_decode = force_payload_decode
self.auto_offset_reset = auto_offset_reset
self.partitioner_cooldown = partitioner_cooldown
self.use_group_sha = use_group_sha
self.running = False
self.consumer_group = None
self.pre_rebalance_callback = pre_rebalance_callback
self.post_rebalance_callback = post_rebalance_callback
self.fetch_offsets_for_topics = fetch_offsets_for_topics
self.pre_topic_refresh_callback = pre_topic_refresh_callback
self.cluster_name = self._set_cluster_name(cluster_name)
self._refresh_timer = _ConsumerTick(
refresh_time_seconds=topic_refresh_frequency_seconds)
self._topic_to_reader_schema_map = self._get_topic_to_reader_schema_map(
consumer_source)
self._consumer_retry_policy = RetryPolicy(
ExpBackoffPolicy(with_jitter=True),
max_retry_count=get_config().consumer_max_offset_retry_count)
self._envelope = Envelope()
if self.topic_to_consumer_topic_state_map:
self.cluster_type = self._determine_cluster_type_from_topics(
self.topic_to_consumer_topic_state_map.keys())
def retry_policy(self):
return RetryPolicy(ExpBackoffPolicy(initial_delay_secs=0.1,
max_delay_secs=0.5),
max_retry_count=self.max_retry_count)