def test_Quotas(metrics):
sensor = metrics.sensor('test')
sensor.add(metrics.metric_name('test1.total', 'grp1'), Total(),
MetricConfig(quota=Quota.upper_bound(5.0)))
sensor.add(metrics.metric_name('test2.total', 'grp1'), Total(),
MetricConfig(quota=Quota.lower_bound(0.0)))
sensor.record(5.0)
with pytest.raises(QuotaViolationError):
sensor.record(1.0)
assert abs(6.0 - metrics.metrics.get(metrics.metric_name('test1.total', 'grp1')).value()) \
< EPS
sensor.record(-6.0)
with pytest.raises(QuotaViolationError):
sensor.record(-1.0)
def test_Percentiles(metrics):
buckets = 100
_percentiles = [
Percentile(metrics.metric_name('test.p25', 'grp1'), 25),
Percentile(metrics.metric_name('test.p50', 'grp1'), 50),
Percentile(metrics.metric_name('test.p75', 'grp1'), 75),
]
percs = Percentiles(4 * buckets, BucketSizing.CONSTANT, 100.0, 0.0,
percentiles=_percentiles)
config = MetricConfig(event_window=50, samples=2)
sensor = metrics.sensor('test', config)
sensor.add_compound(percs)
p25 = metrics.metrics.get(metrics.metric_name('test.p25', 'grp1'))
p50 = metrics.metrics.get(metrics.metric_name('test.p50', 'grp1'))
p75 = metrics.metrics.get(metrics.metric_name('test.p75', 'grp1'))
# record two windows worth of sequential values
for i in range(buckets):
sensor.record(i)
assert abs(p25.value() - 25) < 1.0
assert abs(p50.value() - 50) < 1.0
assert abs(p75.value() - 75) < 1.0
for i in range(buckets):
sensor.record(0.0)
assert p25.value() < 1.0
assert p50.value() < 1.0
assert p75.value() < 1.0
def __init__(self, **configs):
log.debug("Starting KafkaAdminClient with configuration: %s", configs)
extra_configs = set(configs).difference(self.DEFAULT_CONFIG)
if extra_configs:
raise KafkaConfigurationError(
"Unrecognized configs: {}".format(extra_configs))
self.config = copy.copy(self.DEFAULT_CONFIG)
self.config.update(configs)
# Configure metrics
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(
samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [
reporter() for reporter in self.config['metric_reporters']
]
self._metrics = Metrics(metric_config, reporters)
self._client = KafkaClient(metrics=self._metrics,
metric_group_prefix='admin',
**self.config)
# Get auto-discovered version from client if necessary
if self.config['api_version'] is None:
self.config['api_version'] = self._client.config['api_version']
self._closed = False
self._refresh_controller_id()
log.debug("KafkaAdminClient started.")
def test_rate_windowing(mocker, time_keeper, metrics):
mocker.patch('time.time', side_effect=time_keeper.time)
# Use the default time window. Set 3 samples
config = MetricConfig(samples=3)
sensor = metrics.sensor('test.sensor', config)
sensor.add(metrics.metric_name('test.rate', 'grp1'),
Rate(TimeUnit.SECONDS))
sum_val = 0
count = config.samples - 1
# Advance 1 window after every record
for i in range(count):
sensor.record(100)
sum_val += 100
time_keeper.sleep(config.time_window_ms / 1000.0)
# Sleep for half the window.
time_keeper.sleep(config.time_window_ms / 2.0 / 1000.0)
# prior to any time passing
elapsed_secs = (config.time_window_ms * (config.samples - 1) +
config.time_window_ms / 2.0) / 1000.0
kafka_metric = metrics.metrics.get(metrics.metric_name(
'test.rate', 'grp1'))
assert abs((sum_val / elapsed_secs) - kafka_metric.value()) < EPS, \
'Rate(0...2) = 2.666'
assert abs(elapsed_secs - (kafka_metric.measurable.window_size(config, time.time() * 1000) / 1000.0)) \
< EPS, 'Elapsed Time = 75 seconds'
def test_event_windowing(mocker, time_keeper):
mocker.patch('time.time', side_effect=time_keeper.time)
count = Count()
config = MetricConfig(event_window=1, samples=2)
count.record(config, 1.0, time_keeper.ms())
count.record(config, 1.0, time_keeper.ms())
assert 2.0 == count.measure(config, time_keeper.ms())
count.record(config, 1.0, time_keeper.ms()) # first event times out
assert 2.0 == count.measure(config, time_keeper.ms())
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, 'Unrecognized configs: %s' % configs
deprecated = {'smallest': 'earliest', 'largest': 'latest'}
if self.config['auto_offset_reset'] in deprecated:
new_config = deprecated[self.config['auto_offset_reset']]
log.warning('use auto_offset_reset=%s (%s is deprecated)',
new_config, self.config['auto_offset_reset'])
self.config['auto_offset_reset'] = new_config
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [reporter() for reporter in self.config['metric_reporters']]
self._metrics = Metrics(metric_config, reporters)
# TODO _metrics likely needs to be passed to KafkaClient, etc.
# api_version was previously a str. Accept old format for now
if isinstance(self.config['api_version'], str):
str_version = self.config['api_version']
if str_version == 'auto':
self.config['api_version'] = None
else:
self.config['api_version'] = tuple(map(int, str_version.split('.')))
log.warning('use api_version=%s [tuple] -- "%s" as str is deprecated',
str(self.config['api_version']), str_version)
self._client = KafkaClient(metrics=self._metrics, **self.config)
# Get auto-discovered version from client if necessary
if self.config['api_version'] is None:
self.config['api_version'] = self._client.config['api_version']
self._subscription = SubscriptionState(self.config['auto_offset_reset'])
self._fetcher = Fetcher(
self._client, self._subscription, self._metrics, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription, self._metrics,
assignors=self.config['partition_assignment_strategy'],
**self.config)
self._closed = False
self._iterator = None
self._consumer_timeout = float('inf')
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)
def test_time_windowing(mocker, time_keeper):
mocker.patch('time.time', side_effect=time_keeper.time)
count = Count()
config = MetricConfig(time_window_ms=1, samples=2)
count.record(config, 1.0, time_keeper.ms())
time_keeper.sleep(.001)
count.record(config, 1.0, time_keeper.ms())
assert 2.0 == count.measure(config, time_keeper.ms())
time_keeper.sleep(.001)
count.record(config, 1.0, time_keeper.ms()) # oldest event times out
assert 2.0 == count.measure(config, time_keeper.ms())
def __init__(self, config):
Thread.__init__(self)
self.config = config
self.timeout_ms = 10000
self.kafka_client = None
self.running = True
self.sc = None
metrics_tags = {"client-id": self.config["client_id"]}
metric_config = MetricConfig(samples=2, time_window_ms=30000, tags=metrics_tags)
self._metrics = Metrics(metric_config, reporters=[])
self.lock = Lock()
self.lock.acquire()
self.log = logging.getLogger("MasterCoordinator")
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, 'Unrecognized configs: %s' % configs
deprecated = {'smallest': 'earliest', 'largest': 'latest' }
if self.config['auto_offset_reset'] in deprecated:
new_config = deprecated[self.config['auto_offset_reset']]
log.warning('use auto_offset_reset=%s (%s is deprecated)',
new_config, self.config['auto_offset_reset'])
self.config['auto_offset_reset'] = new_config
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [reporter() for reporter in self.config['metric_reporters']]
reporters.append(DictReporter('kafka.consumer'))
self._metrics = Metrics(metric_config, reporters)
metric_group_prefix = 'consumer'
# TODO _metrics likely needs to be passed to KafkaClient, etc.
self._client = KafkaClient(**self.config)
# Check Broker Version if not set explicitly
if self.config['api_version'] == 'auto':
self.config['api_version'] = self._client.check_version()
assert self.config['api_version'] in ('0.9', '0.8.2', '0.8.1', '0.8.0'), 'Unrecognized api version'
# Convert api_version config to tuple for easy comparisons
self.config['api_version'] = tuple(
map(int, self.config['api_version'].split('.')))
self._subscription = SubscriptionState(self.config['auto_offset_reset'])
self._fetcher = Fetcher(
self._client, self._subscription, self._metrics, metric_group_prefix, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription, self._metrics, metric_group_prefix,
assignors=self.config['partition_assignment_strategy'],
**self.config)
self._closed = False
self._iterator = None
self._consumer_timeout = float('inf')
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)
def __init__(self, **configs):
log.debug("Starting Kafka administration interface")
extra_configs = set(configs).difference(self.DEFAULT_CONFIG)
if extra_configs:
raise KafkaConfigurationError("Unrecognized configs: %s" %
extra_configs)
self.config = copy.copy(self.DEFAULT_CONFIG)
self.config.update(configs)
# api_version was previously a str. accept old format for now
if isinstance(self.config['api_version'], str):
deprecated = self.config['api_version']
if deprecated == 'auto':
self.config['api_version'] = None
else:
self.config['api_version'] = tuple(
map(int, deprecated.split('.')))
log.warning(
'use api_version=%s [tuple] -- "%s" as str is deprecated',
str(self.config['api_version']), deprecated)
# Configure metrics
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(
samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [
reporter() for reporter in self.config['metric_reporters']
]
self._metrics = Metrics(metric_config, reporters)
self._client = KafkaClient(metrics=self._metrics,
metric_group_prefix='admin',
**self.config)
# Get auto-discovered version from client if necessary
if self.config['api_version'] is None:
self.config['api_version'] = self._client.config['api_version']
self._closed = False
self._refresh_controller_id()
log.debug('Kafka administration interface started')
def __init__(self,
default_config=None,
reporters=None,
enable_expiration=False):
"""
Create a metrics repository with a default config, given metric
reporters and the ability to expire eligible sensors
Arguments:
default_config (MetricConfig, optional): The default config
reporters (list of AbstractMetricsReporter, optional):
The metrics reporters
enable_expiration (bool, optional): true if the metrics instance
can garbage collect inactive sensors, false otherwise
"""
self._lock = threading.RLock()
self._config = default_config or MetricConfig()
self._sensors = {}
self._metrics = {}
self._children_sensors = {}
self._reporters = reporters or []
for reporter in self._reporters:
reporter.init([])
if enable_expiration:
def expire_loop():
while True:
# delay 30 seconds
time.sleep(30)
self.ExpireSensorTask.run(self)
metrics_scheduler = threading.Thread(target=expire_loop)
# Creating a daemon thread to not block shutdown
metrics_scheduler.daemon = True
metrics_scheduler.start()
self.add_metric(
self.metric_name("count", "kafka-metrics-count",
"total number of registered metrics"),
AnonMeasurable(lambda config, now: len(self._metrics)),
)
def test_old_data_has_no_effect(mocker, time_keeper):
mocker.patch('time.time', side_effect=time_keeper.time)
max_stat = Max()
min_stat = Min()
avg_stat = Avg()
count_stat = Count()
window_ms = 100
samples = 2
config = MetricConfig(time_window_ms=window_ms, samples=samples)
max_stat.record(config, 50, time_keeper.ms())
min_stat.record(config, 50, time_keeper.ms())
avg_stat.record(config, 50, time_keeper.ms())
count_stat.record(config, 50, time_keeper.ms())
time_keeper.sleep(samples * window_ms / 1000.0)
assert float('-inf') == max_stat.measure(config, time_keeper.ms())
assert float(sys.maxsize) == min_stat.measure(config, time_keeper.ms())
assert 0.0 == avg_stat.measure(config, time_keeper.ms())
assert 0 == count_stat.measure(config, time_keeper.ms())
def __init__(self, **configs):
log.debug("Starting the Kafka producer") # trace
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, 'Unrecognized configs: %s' % (configs,)
if self.config['client_id'] is None:
self.config['client_id'] = 'kafka-python-producer-%s' % \
(PRODUCER_CLIENT_ID_SEQUENCE.increment(),)
if self.config['acks'] == 'all':
self.config['acks'] = -1
# api_version was previously a str. accept old format for now
if isinstance(self.config['api_version'], str):
deprecated = self.config['api_version']
if deprecated == 'auto':
self.config['api_version'] = None
else:
self.config['api_version'] = tuple(map(int, deprecated.split('.')))
log.warning('use api_version=%s [tuple] -- "%s" as str is deprecated',
str(self.config['api_version']), deprecated)
# Configure metrics
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [reporter() for reporter in self.config['metric_reporters']]
self._metrics = Metrics(metric_config, reporters)
client = KafkaClient(metrics=self._metrics, metric_group_prefix='producer',
wakeup_timeout_ms=self.config['max_block_ms'],
**self.config)
# Get auto-discovered version from client if necessary
if self.config['api_version'] is None:
self.config['api_version'] = client.config['api_version']
if self.config['compression_type'] == 'lz4':
assert self.config['api_version'] >= (0, 8, 2), 'LZ4 Requires >= Kafka 0.8.2 Brokers'
# Check compression_type for library support
ct = self.config['compression_type']
if ct not in self._COMPRESSORS:
raise ValueError("Not supported codec: {}".format(ct))
else:
checker, compression_attrs = self._COMPRESSORS[ct]
assert checker(), "Libraries for {} compression codec not found".format(ct)
self.config['compression_attrs'] = compression_attrs
message_version = self._max_usable_produce_magic()
self._accumulator = RecordAccumulator(message_version=message_version, metrics=self._metrics, **self.config)
self._metadata = client.cluster
guarantee_message_order = bool(self.config['max_in_flight_requests_per_connection'] == 1)
self._sender = Sender(client, self._metadata,
self._accumulator, self._metrics,
guarantee_message_order=guarantee_message_order,
**self.config)
self._sender.daemon = True
self._sender.start()
self._closed = False
self._cleanup = self._cleanup_factory()
atexit.register(self._cleanup)
log.debug("Kafka producer started")
def config():
return MetricConfig()