def test_heartbeat_timeout(conn, mocker):
mocker.patch('kafka.client_async.KafkaClient.check_version',
return_value=(0, 9))
mocker.patch('time.time', return_value=1234)
consumer = KafkaConsumer('foobar')
mocker.patch.object(consumer._coordinator.heartbeat, 'ttl', return_value=0)
assert consumer._next_timeout() == 1234
def test_heartbeat_thread(kafka_broker, topic):
group_id = 'test-group-' + random_string(6)
consumer = KafkaConsumer(topic,
bootstrap_servers=get_connect_str(kafka_broker),
group_id=group_id,
heartbeat_interval_ms=500)
# poll until we have joined group / have assignment
while not consumer.assignment():
consumer.poll(timeout_ms=100)
assert consumer._coordinator.state is MemberState.STABLE
last_poll = consumer._coordinator.heartbeat.last_poll
last_beat = consumer._coordinator.heartbeat.last_send
timeout = time.time() + 30
while True:
if time.time() > timeout:
raise RuntimeError('timeout waiting for heartbeat')
if consumer._coordinator.heartbeat.last_send > last_beat:
break
time.sleep(0.5)
assert consumer._coordinator.heartbeat.last_poll == last_poll
consumer.poll(timeout_ms=100)
assert consumer._coordinator.heartbeat.last_poll > last_poll
consumer.close()
def fetch_alert(request):
consumer = KafkaConsumer(bootstrap_servers='en1:9092,en2:9092,en3:9092',
auto_offset_reset='earliest')
consumer.subscribe(['ApacheLogAlert'])
time = int(request.GET.get('time', '0'))
test = consumer.poll(10000)
result = []
for aa in test:
cc = test.get(aa)
for msg in cc:
print msg
try:
int(msg.value) + 2
int(msg.key) + 2
except:
continue
if int(msg.key) <= time or int(msg.value) <= 0:
continue
t = datetime.datetime.fromtimestamp(
int(msg.key) / 1e3).strftime('%Y-%m-%d %H:%M:%S')
c = msg.value
tmp = {t: c}
result.append(tmp)
consumer.close()
return render('fetch_alert.mako', request, dict(data=result))
def test_consumer_topics(kafka_broker, topic, version):
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
# Necessary to drive the IO
consumer.poll(500)
consumer_topics = consumer.topics()
assert topic in consumer_topics
assert len(consumer.partitions_for_topic(topic)) > 0
consumer.close()
def __init__(self):
self.KAFKA_HOST_NAME = os.environ.get('KAFKA_HOST_NAME')
self.KAFKA_PORT = os.environ.get('KAFKA_PORT')
self.KAFKA_TOPIC_NAME = os.environ.get('KAFKA_TOPIC_NAME')
self.kafka_twitter_consumer = KafkaConsumer(
self.KAFKA_TOPIC_NAME,
bootstrap_servers=f'{self.KAFKA_HOST_NAME}:{self.KAFKA_PORT}')
def test_consumer(kafka_broker, topic, version):
# The `topic` fixture is included because
# 0.8.2 brokers need a topic to function well
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
consumer.poll(500)
assert len(consumer._client._conns) > 0
node_id = list(consumer._client._conns.keys())[0]
assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
consumer.close()
def test_consumer(kafka_broker, version):
# 0.8.2 brokers need a topic to function well
if version >= (0, 8, 2) and version < (0, 9):
topic(simple_client(kafka_broker))
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
consumer.poll(500)
assert len(consumer._client._conns) > 0
node_id = list(consumer._client._conns.keys())[0]
assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
def consumer_thread(i):
assert i not in consumers
assert i not in stop
stop[i] = threading.Event()
consumers[i] = KafkaConsumer(topic,
bootstrap_servers=connect_str,
heartbeat_interval_ms=500)
while not stop[i].is_set():
for tp, records in six.itervalues(consumers[i].poll()):
messages[i][tp].extend(records)
consumers[i].close()
del consumers[i]
del stop[i]
def test_paused(kafka_broker, topic):
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
topics = [TopicPartition(topic, 1)]
consumer.assign(topics)
assert set(topics) == consumer.assignment()
assert set() == consumer.paused()
consumer.pause(topics[0])
assert set([topics[0]]) == consumer.paused()
consumer.resume(topics[0])
assert set() == consumer.paused()
consumer.unsubscribe()
assert set() == consumer.paused()
def fetch_timeline(request):
consumer = KafkaConsumer('ApacheLogAlert',
bootstrap_servers='en1:9092,en2:9092,en3:9092',
enable_auto_commit=True,
auto_offset_reset='earliest')
time = int(request.GET.get('time', 0))
threshold = int(request.GET.get('threshold', -1))
test = consumer.poll(5000)
i = 0
temp = {}
result = []
threshold_list = []
for topic in test:
cc = test.get(topic)
for msg in cc:
print msg
x = msg.key.split(' ')[0]
y = msg.value.split(',')[2]
key = msg.value.split(',')[1]
if int(x) > time and threshold > -1 and int(y) >= threshold:
print x, time, y, threshold
threshold_list.append({
'x': int(x),
'y': int(y),
'key': int(key)
})
if key in temp:
temp[key].append({'x': int(x), 'y': int(y)})
else:
temp[key] = []
temp[key].append({'x': int(x), 'y': int(y)})
for key in temp:
result.append({'name': key, 'data': temp.get(key)})
for key in result:
key['data'] = sorted(key['data'], key=lambda a: a['x'])
consumer.close()
return render('fetch_timeline.mako', request,
dict(data=result, threshold=threshold_list))
def history(self, topic, channel, records):
messages = []
if self.uid == None:
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=64,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=64,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT):
records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT
kfkTopic = self.__subKey + "." + topic
x = TopicPartition(kfkTopic, channel)
self.consumer.assign([x])
self.consumer.seek_to_end()
last = self.consumer.position(x)
pos = last - records if last > records else 0
self.consumer.seek(x, pos)
data = self.consumer.poll(256)
endIsNotReached = True
while endIsNotReached:
if len(data.values()) == 0:
return messages
records = list(data.values())
for record in records[0]:
index = record[2]
if index >= last - 1: endIsNotReached = False
message = Message(record[0][41:], record[1], record[6], index,
record[3])
messages.append(message)
if endIsNotReached == False:
self.consumer.close()
return messages
pos = pos + len(messages)
self.consumer.seek(x, pos)
data = self.consumer.poll(256)
self.consumer.close()
return messages
def historyForTimePeriod(self, topic, channel, start, end, limit=-1):
out = []
try:
kfkTopic = self.__subKey + "." + topic
x = TopicPartition(kfkTopic, channel)
if self.uid == None:
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=30000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=30000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token +
'/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
self.consumer = KafkaConsumer(bootstrap_servers=self.__bootstrap)
self.consumer.assign([x])
self.consumer.seek_to_end()
last = self.consumer.position(x)
position = last - 1000
found = False
while found == False:
self.consumer.seek(x, position)
data = self.consumer.poll(500)
if x not in data.keys() or len(data[x]) == 0: break
record = data[x][0]
timestamp = record[3]
if timestamp < start:
found = True
break
position = position - 1000
self.consumer.close()
if self.uid == None:
self.с = KafkaConsumer(bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.с = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token +
'/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
self.с.assign([x])
self.c.seek(x, position)
data = self.с.poll(256)
while (x in data.keys() and len(data[x]) > 0):
for record in data[x]:
timestamp = record[3]
if timestamp < start: continue
index = record[2]
if timestamp > end or index >= last - 1:
self.с.close()
return out
message = Message(record[0][41:], record[1], record[6],
index, timestamp)
out.append(message)
if limit is not None and limit > 0 and len(out) >= limit:
self.с.close()
return out
self.с.seek(x, position + len(data[x]))
data = self.с.poll(256)
return out
except:
raise MagistralException(
"Exception during history invocation occurred")
