def __init__(self, *topics, **configs): self.config = copy.copy(self.DEFAULT_CONFIG) for key in self.config: if key in configs: self.config[key] = configs.pop(key) # Only check for extra config keys in top-level class assert not configs, 'Unrecognized configs: %s' % configs deprecated = {'smallest': 'earliest', 'largest': 'latest' } if self.config['auto_offset_reset'] in deprecated: new_config = deprecated[self.config['auto_offset_reset']] log.warning('use auto_offset_reset=%s (%s is deprecated)', new_config, self.config['auto_offset_reset']) self.config['auto_offset_reset'] = new_config metrics_tags = {'client-id': self.config['client_id']} metric_config = MetricConfig(samples=self.config['metrics_num_samples'], time_window_ms=self.config['metrics_sample_window_ms'], tags=metrics_tags) reporters = [reporter() for reporter in self.config['metric_reporters']] reporters.append(DictReporter('kafka.consumer')) self._metrics = Metrics(metric_config, reporters) metric_group_prefix = 'consumer' # TODO _metrics likely needs to be passed to KafkaClient, etc. self._client = KafkaClient(**self.config) # Check Broker Version if not set explicitly if self.config['api_version'] == 'auto': self.config['api_version'] = self._client.check_version() assert self.config['api_version'] in ('0.9', '0.8.2', '0.8.1', '0.8.0'), 'Unrecognized api version' # Convert api_version config to tuple for easy comparisons self.config['api_version'] = tuple( map(int, self.config['api_version'].split('.'))) self._subscription = SubscriptionState(self.config['auto_offset_reset']) self._fetcher = Fetcher( self._client, self._subscription, self._metrics, metric_group_prefix, **self.config) self._coordinator = ConsumerCoordinator( self._client, self._subscription, self._metrics, metric_group_prefix, assignors=self.config['partition_assignment_strategy'], **self.config) self._closed = False self._iterator = None self._consumer_timeout = float('inf') if topics: self._subscription.subscribe(topics=topics) self._client.set_topics(topics)
def test_autocommit_enable_api_version(conn, api_version): coordinator = ConsumerCoordinator(KafkaClient(), SubscriptionState(), Metrics(), 'consumer', enable_auto_commit=True, group_id='foobar', api_version=api_version) if api_version < (0, 8, 1): assert coordinator._auto_commit_task is None assert coordinator.config['enable_auto_commit'] is False else: assert coordinator._auto_commit_task is not None assert coordinator.config['enable_auto_commit'] is True
def test_autocommit_enable_api_version(client, api_version): coordinator = ConsumerCoordinator( client, SubscriptionState(), Metrics(), enable_auto_commit=True, session_timeout_ms=30000, # session_timeout_ms and max_poll_interval_ms max_poll_interval_ms= 30000, # should be the same to avoid KafkaConfigurationError group_id='foobar', api_version=api_version) if api_version < (0, 8, 1): assert coordinator.config['enable_auto_commit'] is False else: assert coordinator.config['enable_auto_commit'] is True
def __init__(self, *topics, **configs): self.config = copy.copy(self.DEFAULT_CONFIG) for key in self.config: if key in configs: self.config[key] = configs.pop(key) # Only check for extra config keys in top-level class assert not configs, 'Unrecognized configs: %s' % configs deprecated = {'smallest': 'earliest', 'largest': 'latest'} if self.config['auto_offset_reset'] in deprecated: new_config = deprecated[self.config['auto_offset_reset']] log.warning('use auto_offset_reset=%s (%s is deprecated)', new_config, self.config['auto_offset_reset']) self.config['auto_offset_reset'] = new_config self._client = KafkaClient(**self.config) # Check Broker Version if not set explicitly if self.config['api_version'] == 'auto': self.config['api_version'] = self._client.check_version() assert self.config['api_version'] in ( '0.9', '0.8.2', '0.8.1', '0.8.0'), 'Unrecognized api version' # Convert api_version config to tuple for easy comparisons self.config['api_version'] = tuple( map(int, self.config['api_version'].split('.'))) self._subscription = SubscriptionState( self.config['auto_offset_reset']) self._fetcher = Fetcher(self._client, self._subscription, **self.config) self._coordinator = ConsumerCoordinator( self._client, self._subscription, assignors=self.config['partition_assignment_strategy'], **self.config) self._closed = False self._iterator = None self._consumer_timeout = float('inf') #self.metrics = None if topics: self._subscription.subscribe(topics=topics) self._client.set_topics(topics)
def __init__(self, *topics, **configs): self.config = copy.copy(self.DEFAULT_CONFIG) for key in self.config: if key in configs: self.config[key] = configs.pop(key) # Only check for extra config keys in top-level class assert not configs, 'Unrecognized configs: %s' % configs deprecated = {'smallest': 'earliest', 'largest': 'latest'} if self.config['auto_offset_reset'] in deprecated: new_config = deprecated[self.config['auto_offset_reset']] log.warning('use auto_offset_reset=%s (%s is deprecated)', new_config, self.config['auto_offset_reset']) self.config['auto_offset_reset'] = new_config metrics_tags = {'client-id': self.config['client_id']} metric_config = MetricConfig( samples=self.config['metrics_num_samples'], time_window_ms=self.config['metrics_sample_window_ms'], tags=metrics_tags) reporters = [ reporter() for reporter in self.config['metric_reporters'] ] self._metrics = Metrics(metric_config, reporters) # TODO _metrics likely needs to be passed to KafkaClient, etc. # api_version was previously a str. accept old format for now if isinstance(self.config['api_version'], str): str_version = self.config['api_version'] if str_version == 'auto': self.config['api_version'] = None else: self.config['api_version'] = tuple( map(int, str_version.split('.'))) log.warning( 'use api_version=%s [tuple] -- "%s" as str is deprecated', str(self.config['api_version']), str_version) self._client = KafkaClient(metrics=self._metrics, **self.config) # Get auto-discovered version from client if necessary if self.config['api_version'] is None: self.config['api_version'] = self._client.config['api_version'] self._subscription = SubscriptionState( self.config['auto_offset_reset']) self._fetcher = Fetcher(self._client, self._subscription, self._metrics, **self.config) self._coordinator = ConsumerCoordinator( self._client, self._subscription, self._metrics, assignors=self.config['partition_assignment_strategy'], **self.config) self._closed = False self._iterator = None self._consumer_timeout = float('inf') if topics: self._subscription.subscribe(topics=topics) self._client.set_topics(topics)
def coordinator(client): return ConsumerCoordinator(client, SubscriptionState(), Metrics())
def coordinator(conn): return ConsumerCoordinator(KafkaClient(), SubscriptionState(), Metrics(), 'consumer')