def test_heartbeat_timeout(conn, mocker):
mocker.patch('kafka.client_async.KafkaClient.check_version',
return_value=(0, 9))
mocker.patch('time.time', return_value=1234)
consumer = KafkaConsumer('foobar')
mocker.patch.object(consumer._coordinator.heartbeat, 'ttl', return_value=0)
assert consumer._next_timeout() == 1234
def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
threading.Thread.__init__(self)
self.threadId = threadId
self.name = name
self.group = groupId;
self.subKey = sKey;
self.cipher = None if cipher is None else cipher;
configs = Configs.consumerConfigs();
configs["bootstrap_servers"] = bootstrapServers.split(',');
configs["group_id"] = groupId;
configs['enable_auto_commit'] = False;
self.__isAlive = True
home = expanduser("~")
if uid == None:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10),
group_id = groupId);
else:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + token + '/key.pem',
ssl_cafile = home + '/magistral/' + token + '/ca.pem',
ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
api_version = (0, 10),
group_id = groupId);
self.permissions = permissions;
self.map = {}
self.__offsets = {}
def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
threading.Thread.__init__(self)
self.threadId = threadId
self.name = name
self.group = groupId;
self.subKey = sKey;
self.cipher = None if cipher is None else cipher;
configs = Configs.consumerConfigs();
configs["bootstrap_servers"] = bootstrapServers.split(',');
configs["group_id"] = groupId;
configs['enable_auto_commit'] = False;
self.__isAlive = True
home = expanduser("~")
if uid == None:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10),
group_id = groupId);
else:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + token + '/key.pem',
ssl_cafile = home + '/magistral/' + token + '/ca.pem',
ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
api_version = (0, 10),
group_id = groupId);
self.permissions = permissions;
self.map = {}
self.__offsets = {}
def test_consumer(kafka_broker, topic, version):
# The `topic` fixture is included because
# 0.8.2 brokers need a topic to function well
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
consumer.poll(500)
assert len(consumer._client._conns) > 0
node_id = list(consumer._client._conns.keys())[0]
assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
consumer.close()
def test_heartbeat_thread(kafka_broker, topic):
group_id = 'test-group-' + random_string(6)
consumer = KafkaConsumer(topic,
bootstrap_servers=get_connect_str(kafka_broker),
group_id=group_id,
heartbeat_interval_ms=500)
# poll until we have joined group / have assignment
while not consumer.assignment():
consumer.poll(timeout_ms=100)
assert consumer._coordinator.state is MemberState.STABLE
last_poll = consumer._coordinator.heartbeat.last_poll
last_beat = consumer._coordinator.heartbeat.last_send
timeout = time.time() + 30
while True:
if time.time() > timeout:
raise RuntimeError('timeout waiting for heartbeat')
if consumer._coordinator.heartbeat.last_send > last_beat:
break
time.sleep(0.5)
assert consumer._coordinator.heartbeat.last_poll == last_poll
consumer.poll(timeout_ms=100)
assert consumer._coordinator.heartbeat.last_poll > last_poll
consumer.close()
def test_consumer(kafka_broker, version):
# 0.8.2 brokers need a topic to function well
if version >= (0, 8, 2) and version < (0, 9):
topic(simple_client(kafka_broker))
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
consumer.poll(500)
assert len(consumer._client._conns) > 0
node_id = list(consumer._client._conns.keys())[0]
assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
def test_consumer(kafka_broker, version):
# 0.8.2 brokers need a topic to function well
if version >= (0, 8, 2) and version < (0, 9):
topic(simple_client(kafka_broker))
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
consumer.poll(500)
assert len(consumer._client._conns) > 0
node_id = list(consumer._client._conns.keys())[0]
assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
def test_consumer_topics(kafka_broker, topic, version):
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
# Necessary to drive the IO
consumer.poll(500)
assert topic in consumer.topics()
assert len(consumer.partitions_for_topic(topic)) > 0
consumer.close()
def test_heartbeat_thread(kafka_broker, topic):
group_id = 'test-group-' + random_string(6)
consumer = KafkaConsumer(topic,
bootstrap_servers=get_connect_str(kafka_broker),
group_id=group_id,
heartbeat_interval_ms=500)
# poll until we have joined group / have assignment
while not consumer.assignment():
consumer.poll(timeout_ms=100)
assert consumer._coordinator.state is MemberState.STABLE
last_poll = consumer._coordinator.heartbeat.last_poll
last_beat = consumer._coordinator.heartbeat.last_send
timeout = time.time() + 30
while True:
if time.time() > timeout:
raise RuntimeError('timeout waiting for heartbeat')
if consumer._coordinator.heartbeat.last_send > last_beat:
break
time.sleep(0.5)
assert consumer._coordinator.heartbeat.last_poll == last_poll
consumer.poll(timeout_ms=100)
assert consumer._coordinator.heartbeat.last_poll > last_poll
consumer.close()
def __init__(self):
self.KAFKA_HOST_NAME = os.environ.get('KAFKA_HOST_NAME')
self.KAFKA_PORT = os.environ.get('KAFKA_PORT')
self.KAFKA_TOPIC_NAME = os.environ.get('KAFKA_TOPIC_NAME')
self.kafka_twitter_consumer = KafkaConsumer(
self.KAFKA_TOPIC_NAME,
bootstrap_servers=f'{self.KAFKA_HOST_NAME}:{self.KAFKA_PORT}')
def fetch_alert(request):
consumer = KafkaConsumer(bootstrap_servers='en1:9092,en2:9092,en3:9092',
auto_offset_reset='earliest')
consumer.subscribe(['ApacheLogAlert'])
time = int(request.GET.get('time', '0'))
test = consumer.poll(10000)
result = []
for aa in test:
cc = test.get(aa)
for msg in cc:
print msg
try:
int(msg.value) + 2
int(msg.key) + 2
except:
continue
if int(msg.key) <= time or int(msg.value) <= 0:
continue
t = datetime.datetime.fromtimestamp(
int(msg.key) / 1e3).strftime('%Y-%m-%d %H:%M:%S')
c = msg.value
tmp = {t: c}
result.append(tmp)
consumer.close()
return render('fetch_alert.mako', request, dict(data=result))
def fetch_timeline(request):
consumer = KafkaConsumer('ApacheLogAlert',
bootstrap_servers='en1:9092,en2:9092,en3:9092',
enable_auto_commit=True,
auto_offset_reset='earliest')
time = int(request.GET.get('time', 0))
threshold = int(request.GET.get('threshold', -1))
test = consumer.poll(5000)
i = 0
temp = {}
result = []
threshold_list = []
for topic in test:
cc = test.get(topic)
for msg in cc:
print msg
x = msg.key.split(' ')[0]
y = msg.value.split(',')[2]
key = msg.value.split(',')[1]
if int(x) > time and threshold > -1 and int(y) >= threshold:
print x, time, y, threshold
threshold_list.append({
'x': int(x),
'y': int(y),
'key': int(key)
})
if key in temp:
temp[key].append({'x': int(x), 'y': int(y)})
else:
temp[key] = []
temp[key].append({'x': int(x), 'y': int(y)})
for key in temp:
result.append({'name': key, 'data': temp.get(key)})
for key in result:
key['data'] = sorted(key['data'], key=lambda a: a['x'])
consumer.close()
return render('fetch_timeline.mako', request,
dict(data=result, threshold=threshold_list))
def consumer_thread(i):
assert i not in consumers
assert i not in stop
stop[i] = threading.Event()
consumers[i] = KafkaConsumer(topic,
bootstrap_servers=connect_str,
heartbeat_interval_ms=500)
while not stop[i].is_set():
for tp, records in six.itervalues(consumers[i].poll()):
messages[i][tp].extend(records)
consumers[i].close()
del consumers[i]
del stop[i]
def test_consumer(kafka_broker, topic, version):
# The `topic` fixture is included because
# 0.8.2 brokers need a topic to function well
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
consumer.poll(500)
assert len(consumer._client._conns) > 0
node_id = list(consumer._client._conns.keys())[0]
assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
consumer.close()
def historyForTimePeriod(self, topic, channel, start, end, limit = -1):
out = []
try:
kfkTopic = self.__subKey + "." + topic;
x = TopicPartition(kfkTopic, channel);
if self.uid == None:
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 30000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10));
else:
home = expanduser("~")
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 30000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
api_version = (0, 10));
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap);
self.consumer.assign([x]);
self.consumer.seek_to_end();
last = self.consumer.position(x);
position = last - 1000;
found = False;
while found == False:
self.consumer.seek(x, position);
data = self.consumer.poll(500);
if x not in data.keys() or len(data[x]) == 0: break;
record = data[x][0];
timestamp = record[3];
if timestamp < start:
found = True;
break;
position = position - 1000;
self.consumer.close();
if self.uid == None:
self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10));
else:
home = expanduser("~")
self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
api_version = (0, 10));
self.с.assign([x]);
self.c.seek(x, position);
data = self.с.poll(256);
while (x in data.keys() and len(data[x]) > 0):
for record in data[x] :
timestamp = record[3];
if timestamp < start: continue;
index = record[2];
if timestamp > end or index >= last - 1:
self.с.close();
return out;
message = Message(record[0][41:], record[1], record[6], index, timestamp);
out.append(message);
if limit is not None and limit > 0 and len(out) >= limit:
self.с.close();
return out;
self.с.seek(x, position + len(data[x]));
data = self.с.poll(256);
return out;
except:
raise MagistralException("Exception during history invocation occurred");
def test_paused(kafka_broker, topic):
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
topics = [TopicPartition(topic, 1)]
consumer.assign(topics)
assert set(topics) == consumer.assignment()
assert set() == consumer.paused()
consumer.pause(topics[0])
assert set([topics[0]]) == consumer.paused()
consumer.resume(topics[0])
assert set() == consumer.paused()
consumer.unsubscribe()
assert set() == consumer.paused()
def historyForTimePeriod(self, topic, channel, start, end, limit=-1):
out = []
try:
kfkTopic = self.__subKey + "." + topic
x = TopicPartition(kfkTopic, channel)
if self.uid == None:
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=30000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=30000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token +
'/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
self.consumer = KafkaConsumer(bootstrap_servers=self.__bootstrap)
self.consumer.assign([x])
self.consumer.seek_to_end()
last = self.consumer.position(x)
position = last - 1000
found = False
while found == False:
self.consumer.seek(x, position)
data = self.consumer.poll(500)
if x not in data.keys() or len(data[x]) == 0: break
record = data[x][0]
timestamp = record[3]
if timestamp < start:
found = True
break
position = position - 1000
self.consumer.close()
if self.uid == None:
self.с = KafkaConsumer(bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.с = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token +
'/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
self.с.assign([x])
self.c.seek(x, position)
data = self.с.poll(256)
while (x in data.keys() and len(data[x]) > 0):
for record in data[x]:
timestamp = record[3]
if timestamp < start: continue
index = record[2]
if timestamp > end or index >= last - 1:
self.с.close()
return out
message = Message(record[0][41:], record[1], record[6],
index, timestamp)
out.append(message)
if limit is not None and limit > 0 and len(out) >= limit:
self.с.close()
return out
self.с.seek(x, position + len(data[x]))
data = self.с.poll(256)
return out
except:
raise MagistralException(
"Exception during history invocation occurred")
class GroupConsumer(threading.Thread):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
threading.Thread.__init__(self)
self.threadId = threadId
self.name = name
self.group = groupId;
self.subKey = sKey;
self.cipher = None if cipher is None else cipher;
configs = Configs.consumerConfigs();
configs["bootstrap_servers"] = bootstrapServers.split(',');
configs["group_id"] = groupId;
configs['enable_auto_commit'] = False;
self.__isAlive = True
home = expanduser("~")
if uid == None:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10),
group_id = groupId);
else:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + token + '/key.pem',
ssl_cafile = home + '/magistral/' + token + '/ca.pem',
ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
api_version = (0, 10),
group_id = groupId);
self.permissions = permissions;
self.map = {}
self.__offsets = {}
def recordsTotally(self, data):
size = 0;
for val in list(data.values()):
if len(val) > 0: size = size + len(val);
return size;
def consumerRecord2Message(self, record):
payload = record[6]
if self.cipher is not None:
try:
payload = self.cipher.decrypt(payload)
except:
pass
msg = Message(record[0][41:], record[1], payload, record[2], record[3])
return msg
def run(self):
threadLock.acquire(False)
while self.__isAlive:
try:
data = self.__consumer.poll(512);
for values in list(data.values()):
for value in values:
msg = self.consumerRecord2Message(value);
listener = self.map[value[0]][msg.channel()];
if listener is not None: listener(msg);
if len(list(data.values())) > 0: self.__consumer.commit_async();
except:
pass
threadLock.release()
# ////////////////////////////////////////////////////////////////////////////////////
def subscribe(self, topic, channel = -1, listener = None, callback = None):
assert channel is not None and isinstance(channel, int), "Channel expected as int argument"
if (channel < -1): channel = -1;
etopic = self.subKey + "." + topic;
self.logger.debug("Subscribe -> %s : %s | key = %s", topic, channel, self.subKey);
if (self.permissions == None or len(self.permissions) == 0):
raise MagistralException("User has no permissions for topic [" + topic + "].");
self.fch = [];
for meta in self.permissions:
if (meta.topic() != topic): continue;
if channel == -1:
self.fch = meta.channels();
elif channel in meta.channels():
self.fch = [ channel ];
if (len(self.fch) == 0):
npgex = "No permissions for topic [" + topic + "] granted";
self.logger.error(npgex);
raise MagistralException(npgex);
if (self.map == None or etopic not in self.map):
self.map[etopic] = {}
# // Assign Topic-partition pairs to listen
tpas = [];
for ch in self.fch:
tpas.append(TopicPartition(etopic, ch));
if (listener is not None): self.map[etopic][ch] = listener
ca = self.__consumer.assignment()
if (ca is not None):
for tp in ca: tpas.append(tp)
self.__consumer.assign(tpas);
if callback is not None:
callback(self.__consumer.assignment());
return self.__consumer.assignment();
def unsubscribe(self, topic):
self.consumer.assign([]);
self.map.remove(topic);
def close(self):
self.__isAlive = False
self.__consumer.pause()
self.__consumer.close()
logging.getLogger('kafka.conn').setLevel(logging.FATAL)
logging.getLogger('kafka.cluster').setLevel(logging.FATAL)
logging.getLogger('kafka.consumer.group').setLevel(logging.INFO)
logging.getLogger('kafka.consumer.fetcher').setLevel(logging.INFO)
logging.getLogger('kafka.coordinator.consumer').setLevel(logging.INFO)
logging.getLogger('kafka.producer.record_accumulator').setLevel(logging.INFO)
class MagistralConsumer(object):
__HISTORY_DATA_FETCH_SIZE_LIMIT = 10000
def __init__(self,
pubKey,
subKey,
secretKey,
bootstrap,
token,
cipher=None,
uid=None):
self.__pubKey = pubKey
self.__subKey = subKey
self.__secretKey = secretKey
self.__token = token
self.uid = uid
self.__bootstrap = bootstrap.split(',')
if cipher is not None: self.__cipher = cipher
def history(self, topic, channel, records):
messages = []
if self.uid == None:
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=64,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=64,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT):
records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT
kfkTopic = self.__subKey + "." + topic
x = TopicPartition(kfkTopic, channel)
self.consumer.assign([x])
self.consumer.seek_to_end()
last = self.consumer.position(x)
pos = last - records if last > records else 0
self.consumer.seek(x, pos)
data = self.consumer.poll(256)
endIsNotReached = True
while endIsNotReached:
if len(data.values()) == 0:
return messages
records = list(data.values())
for record in records[0]:
index = record[2]
if index >= last - 1: endIsNotReached = False
message = Message(record[0][41:], record[1], record[6], index,
record[3])
messages.append(message)
if endIsNotReached == False:
self.consumer.close()
return messages
pos = pos + len(messages)
self.consumer.seek(x, pos)
data = self.consumer.poll(256)
self.consumer.close()
return messages
def historyForTimePeriod(self, topic, channel, start, end, limit=-1):
out = []
try:
kfkTopic = self.__subKey + "." + topic
x = TopicPartition(kfkTopic, channel)
if self.uid == None:
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=30000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=30000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token +
'/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
self.consumer = KafkaConsumer(bootstrap_servers=self.__bootstrap)
self.consumer.assign([x])
self.consumer.seek_to_end()
last = self.consumer.position(x)
position = last - 1000
found = False
while found == False:
self.consumer.seek(x, position)
data = self.consumer.poll(500)
if x not in data.keys() or len(data[x]) == 0: break
record = data[x][0]
timestamp = record[3]
if timestamp < start:
found = True
break
position = position - 1000
self.consumer.close()
if self.uid == None:
self.с = KafkaConsumer(bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.с = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=32,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_partition_fetch_bytes=65536,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token +
'/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
self.с.assign([x])
self.c.seek(x, position)
data = self.с.poll(256)
while (x in data.keys() and len(data[x]) > 0):
for record in data[x]:
timestamp = record[3]
if timestamp < start: continue
index = record[2]
if timestamp > end or index >= last - 1:
self.с.close()
return out
message = Message(record[0][41:], record[1], record[6],
index, timestamp)
out.append(message)
if limit is not None and limit > 0 and len(out) >= limit:
self.с.close()
return out
self.с.seek(x, position + len(data[x]))
data = self.с.poll(256)
return out
except:
raise MagistralException(
"Exception during history invocation occurred")
def test_heartbeat_timeout(conn, mocker):
mocker.patch('kafka.client_async.KafkaClient.check_version', return_value = '0.9')
mocker.patch('time.time', return_value = 1234)
consumer = KafkaConsumer('foobar')
mocker.patch.object(consumer._coordinator.heartbeat, 'ttl', return_value = 0)
assert consumer._next_timeout() == 1234
def history(self, topic, channel, records):
messages = []
if self.uid == None:
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10));
else:
home = expanduser("~")
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
api_version = (0, 10));
if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT): records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT;
kfkTopic = self.__subKey + "." + topic;
x = TopicPartition(kfkTopic, channel);
self.consumer.assign([x]);
self.consumer.seek_to_end();
last = self.consumer.position(x);
pos = last - records if last > records else 0;
self.consumer.seek(x, pos);
data = self.consumer.poll(256);
endIsNotReached = True;
while endIsNotReached:
if len(data.values()) == 0:
return messages;
records = list(data.values())
for record in records[0]:
index = record[2];
if index >= last - 1: endIsNotReached = False;
message = Message(record[0][41:], record[1], record[6], index, record[3]);
messages.append(message);
if endIsNotReached == False:
self.consumer.close();
return messages;
pos = pos + len(messages)
self.consumer.seek(x, pos);
data = self.consumer.poll(256);
self.consumer.close();
return messages;
def history(self, topic, channel, records):
messages = []
if self.uid == None:
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=64,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_in_flight_requests_per_connection=4,
api_version=(0, 10))
else:
home = expanduser("~")
self.consumer = KafkaConsumer(
bootstrap_servers=self.__bootstrap,
check_crcs=False,
exclude_internal_topics=True,
session_timeout_ms=10000,
reconnect_backoff_ms=10000,
heartbeat_interval_ms=2000,
retry_backoff_ms=500,
fetch_min_bytes=64,
fetch_max_wait_ms=96,
enable_auto_commit=False,
max_in_flight_requests_per_connection=4,
security_protocol='SSL',
ssl_check_hostname=False,
ssl_keyfile=home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile=home + '/magistral/' + self.__token +
'/certificate.pem',
api_version=(0, 10))
if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT):
records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT
kfkTopic = self.__subKey + "." + topic
x = TopicPartition(kfkTopic, channel)
self.consumer.assign([x])
self.consumer.seek_to_end()
last = self.consumer.position(x)
pos = last - records if last > records else 0
self.consumer.seek(x, pos)
data = self.consumer.poll(256)
endIsNotReached = True
while endIsNotReached:
if len(data.values()) == 0:
return messages
records = list(data.values())
for record in records[0]:
index = record[2]
if index >= last - 1: endIsNotReached = False
message = Message(record[0][41:], record[1], record[6], index,
record[3])
messages.append(message)
if endIsNotReached == False:
self.consumer.close()
return messages
pos = pos + len(messages)
self.consumer.seek(x, pos)
data = self.consumer.poll(256)
self.consumer.close()
return messages
def test_paused(kafka_broker, topic):
consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
topics = [TopicPartition(topic, 1)]
consumer.assign(topics)
assert set(topics) == consumer.assignment()
assert set() == consumer.paused()
consumer.pause(topics[0])
assert set([topics[0]]) == consumer.paused()
consumer.resume(topics[0])
assert set() == consumer.paused()
consumer.unsubscribe()
assert set() == consumer.paused()
class MagistralConsumer(object):
__HISTORY_DATA_FETCH_SIZE_LIMIT = 10000;
def __init__(self, pubKey, subKey, secretKey, bootstrap, token, cipher = None, uid = None):
self.__pubKey = pubKey
self.__subKey = subKey
self.__secretKey = secretKey
self.__token = token
self.uid = uid
self.__bootstrap = bootstrap.split(',')
if cipher is not None: self.__cipher = cipher
def history(self, topic, channel, records):
messages = []
if self.uid == None:
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10));
else:
home = expanduser("~")
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
api_version = (0, 10));
if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT): records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT;
kfkTopic = self.__subKey + "." + topic;
x = TopicPartition(kfkTopic, channel);
self.consumer.assign([x]);
self.consumer.seek_to_end();
last = self.consumer.position(x);
pos = last - records if last > records else 0;
self.consumer.seek(x, pos);
data = self.consumer.poll(256);
endIsNotReached = True;
while endIsNotReached:
if len(data.values()) == 0:
return messages;
records = list(data.values())
for record in records[0]:
index = record[2];
if index >= last - 1: endIsNotReached = False;
message = Message(record[0][41:], record[1], record[6], index, record[3]);
messages.append(message);
if endIsNotReached == False:
self.consumer.close();
return messages;
pos = pos + len(messages)
self.consumer.seek(x, pos);
data = self.consumer.poll(256);
self.consumer.close();
return messages;
def historyForTimePeriod(self, topic, channel, start, end, limit = -1):
out = []
try:
kfkTopic = self.__subKey + "." + topic;
x = TopicPartition(kfkTopic, channel);
if self.uid == None:
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 30000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10));
else:
home = expanduser("~")
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 30000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
api_version = (0, 10));
self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap);
self.consumer.assign([x]);
self.consumer.seek_to_end();
last = self.consumer.position(x);
position = last - 1000;
found = False;
while found == False:
self.consumer.seek(x, position);
data = self.consumer.poll(500);
if x not in data.keys() or len(data[x]) == 0: break;
record = data[x][0];
timestamp = record[3];
if timestamp < start:
found = True;
break;
position = position - 1000;
self.consumer.close();
if self.uid == None:
self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10));
else:
home = expanduser("~")
self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 32,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_partition_fetch_bytes = 65536,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
api_version = (0, 10));
self.с.assign([x]);
self.c.seek(x, position);
data = self.с.poll(256);
while (x in data.keys() and len(data[x]) > 0):
for record in data[x] :
timestamp = record[3];
if timestamp < start: continue;
index = record[2];
if timestamp > end or index >= last - 1:
self.с.close();
return out;
message = Message(record[0][41:], record[1], record[6], index, timestamp);
out.append(message);
if limit is not None and limit > 0 and len(out) >= limit:
self.с.close();
return out;
self.с.seek(x, position + len(data[x]));
data = self.с.poll(256);
return out;
except:
raise MagistralException("Exception during history invocation occurred");
class GroupConsumer(threading.Thread):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
threading.Thread.__init__(self)
self.threadId = threadId
self.name = name
self.group = groupId;
self.subKey = sKey;
self.cipher = None if cipher is None else cipher;
configs = Configs.consumerConfigs();
configs["bootstrap_servers"] = bootstrapServers.split(',');
configs["group_id"] = groupId;
configs['enable_auto_commit'] = False;
self.__isAlive = True
home = expanduser("~")
if uid == None:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
api_version = (0, 10),
group_id = groupId);
else:
self.__consumer = KafkaConsumer(
bootstrap_servers = configs["bootstrap_servers"],
check_crcs = False,
exclude_internal_topics = True,
session_timeout_ms = 10000,
reconnect_backoff_ms = 10000,
heartbeat_interval_ms = 2000,
retry_backoff_ms = 500,
fetch_min_bytes = 64,
fetch_max_wait_ms = 96,
enable_auto_commit = False,
max_in_flight_requests_per_connection = 4,
security_protocol = 'SSL',
ssl_check_hostname = False,
ssl_keyfile = home + '/magistral/' + token + '/key.pem',
ssl_cafile = home + '/magistral/' + token + '/ca.pem',
ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
api_version = (0, 10),
group_id = groupId);
self.permissions = permissions;
self.map = {}
self.__offsets = {}
def recordsTotally(self, data):
size = 0;
for val in data.values():
if len(val) > 0: size = size + len(val);
return size;
def consumerRecord2Message(self, record):
payload = record[6]
if self.cipher is not None:
try:
payload = self.cipher.decrypt(payload)
except:
pass
msg = Message(record[0][41:], record[1], payload, record[2], record[3])
return msg
def run(self):
threadLock.acquire(False)
while self.__isAlive:
try:
data = self.__consumer.poll(512);
for values in data.values():
for value in values:
msg = self.consumerRecord2Message(value);
listener = self.map[value[0]][msg.channel()];
if listener is not None: listener(msg);
if len(data.values()) > 0: self.__consumer.commit_async();
except:
pass
threadLock.release()
# ////////////////////////////////////////////////////////////////////////////////////
def subscribe(self, topic, channel = -1, listener = None, callback = None):
assert channel is not None and isinstance(channel, int), "Channel expected as int argument"
if (channel < -1): channel = -1;
etopic = self.subKey + "." + topic;
self.logger.debug("Subscribe -> %s : %s | key = %s", topic, channel, self.subKey);
if (self.permissions == None or len(self.permissions) == 0):
raise MagistralException("User has no permissions for topic [" + topic + "].");
self.fch = [];
for meta in self.permissions:
if (meta.topic() != topic): continue;
if channel == -1:
self.fch = meta.channels();
elif channel in meta.channels():
self.fch = [ channel ];
if (len(self.fch) == 0):
npgex = "No permissions for topic [" + topic + "] granted";
self.logger.error(npgex);
raise MagistralException(npgex);
if (self.map == None or etopic not in self.map):
self.map[etopic] = {}
# // Assign Topic-partition pairs to listen
tpas = [];
for ch in self.fch:
tpas.append(TopicPartition(etopic, ch));
if (listener is not None): self.map[etopic][ch] = listener
ca = self.__consumer.assignment()
if (ca is not None):
for tp in ca: tpas.append(tp)
self.__consumer.assign(tpas);
if callback is not None:
callback(self.__consumer.assignment());
return self.__consumer.assignment();
def unsubscribe(self, topic):
self.consumer.assign([]);
self.map.remove(topic);
def close(self):
self.__isAlive = False
self.__consumer.pause()
self.__consumer.close()
logging.getLogger('kafka.conn').setLevel(logging.FATAL)
logging.getLogger('kafka.cluster').setLevel(logging.FATAL)
logging.getLogger('kafka.consumer.group').setLevel(logging.INFO)
logging.getLogger('kafka.consumer.fetcher').setLevel(logging.INFO)
logging.getLogger('kafka.coordinator.consumer').setLevel(logging.INFO)
logging.getLogger('kafka.producer.record_accumulator').setLevel(logging.INFO)