def init_consumer(cls, hosts, topic, group, offset_reset):
zookeeper = KazooClient(hosts)
zookeeper.start()
cluster = Cluster(zookeeper)
topic = cluster.topics[topic]
consumer = topic.subscribe(group=group, offset_reset=offset_reset)
return consumer
def setUpClass(cls):
cls.zk_harness = KazooTestHarness()
cls.zk_harness.setup_zookeeper()
cls.client = cls.zk_harness.client
cls.hosts = cls.zk_harness.hosts
cls.kafka_broker = cls.start_broker(cls.client, cls.hosts)
cls.kafka_cluster = Cluster(cls.client)
def consumer():
zookeeper = KazooClient()
zookeeper.start()
cluster = Cluster(zookeeper)
topic = cluster.topics['topicname']
for msg in topic.subscribe('groupname'):
print msg
def comsumer(topic):
zookeeper = KazooClient('172.16.252.201', 2181)
zookeeper.start()
cluster = Cluster(zookeeper)
cluster.brokers.keys()
topic = cluster.topics[topic]
c = topic.subscribe('group-name')
for i in c:
print i
def producer(topic):
zookeeper = KazooClient('172.16.252.201:2181', '172.16.252.202:2181', '172.16.252.203:2181')
zookeeper.start()
cluster = Cluster(zookeeper)
topic = cluster.topics[topic]
i = 0
while(True):
i += 1
topic.publish("hhworld_%s::%s" %(i, current_time()))
if i > 100:
break
def test_multiclient_rebalance(self, *args):
"""Test rebalancing with many connected clients
This test is primarily good at ferreting out concurrency bugs
and therefore doesn't test one specific thing, since such
bugs are fundamentally hard to trap.
To test, it simulates 10 consumer connecting over time, rebalancing,
and eventually disconnecting one by one. In order to accomplish
this, it creates a separate kazoo client for each consumer. This
is required because otherwise there is too much thread contention
over the single kazoo client. Some callbacks won't happen until
much too late because there aren't enough threads to go around.
"""
n_partitions = 10
n_consumers = 10
t = Topic(self.c, 'testtopic')
# with self.client, new clients don't see the brokers -- unsure why
from kazoo.client import KazooClient
zk_hosts = ','.join(
['%s:%s' % (h, p) for h, p in self.client.hosts.hosts])
zkclient = KazooClient(hosts=zk_hosts)
zkclient.start()
self._register_fake_brokers(n_partitions, client=zkclient)
zkclient.ensure_path('/brokers/topics')
# bring up consumers
consumers = []
for i in xrange(n_consumers):
newclient = KazooClient(hosts=zk_hosts)
newclient.start()
cluster = Cluster(newclient)
topic = Topic(cluster, 'testtopic')
consumers.append((newclient, topic.subscribe('group1')))
time.sleep(1)
time.sleep(5) # let things settle
# bring down consumers
for client, consumer in consumers:
consumer.stop_partitions()
client.stop()
time.sleep(
2) # a little more time so we don't kill during rebalance
newclient.stop()
def setUp(self):
"""Set up kafka and zookeeper."""
try:
self.setup_zookeeper()
except kazoo.handlers.threading.TimeoutError:
logging.warning('Zookeeper failed to start. Trying again.')
if self.cluster[0].running:
self.cluster.stop()
self.setup_zookeeper() # try again if travis-ci is being slow
# Keep track of processes started
self._consumers = []
self._producers = []
self.kafka_broker = self.start_broker(self.client, self.hosts)
self.kafka_cluster = Cluster(self.client)
def setUp(self, bm, *args):
super(TestPartitionOwnerRegistry, self).setUp()
self.c = Cluster(self.client)
broker = mock.Mock()
broker.id = 1
self.c.brokers.__getitem__.return_value = broker
self.consumer = mock.Mock()
self.consumer.id = '1234'
self.topic = mock.Mock()
self.topic.name = 'topic'
self.por = PartitionOwnerRegistry(self.consumer, self.c, self.topic,
'group')
# Create 5 partitions with on the same topic and broker
self.partitions = []
self.message_set_queue = Queue.Queue()
for i in xrange(5):
self.partitions.append(
OwnedPartition(Partition(self.c, self.topic, broker, i),
'group', self.message_set_queue))
#! /usr/local/Python-2.7.5/bin/python
# coding:utf-8
from kazoo.client import KazooClient
from samsa.cluster import Cluster
import json
import time
import redis
zookeeper = KazooClient(hosts='localhost:2181') #...(1)
zookeeper.start() #...(2)
cluster = Cluster(zookeeper) #...(3)
topic = cluster.topics['topic1'] #...(4)
consumer = topic.subscribe('group-name1') #...(5)
r = redis.Redis(host='localhost', port=6379, db=0) #...(6)
for message in consumer: #...(7)
jmes = json.loads(message) #...(8)
shohinlist = jmes['shohinlist']
for shohin in shohinlist: #...(9)
name = shohin['name']
count = shohin['count']
r.zincrby('ranking', name, count) #...(10)
help='log level',
default='DEBUG')
options, args = parser.parse_args()
zookeeper_kwargs = {}
if options.zookeeper is not None:
zookeeper_kwargs['hosts'] = options.zookeeper
logger = logging.getLogger()
logger.setLevel(getattr(logging, options.loglevel.upper()))
logger.addHandler(logging.StreamHandler())
zookeeper = KazooClient(**zookeeper_kwargs)
zookeeper.connect()
kafka = Cluster(zookeeper)
exposed = {
'kafka': kafka,
'zookeeper': zookeeper,
}
banner = """---
Welcome to Samsa! We've provided some local variables for you:
%s
---""" % '\n'.join('> %s: %s' % (key, repr(value))
for key, value in exposed.iteritems())
code.interact(banner, local=exposed)
def producer():
zookeeper = KazooClient()
zookeeper.start()
cluster = Cluster(zookeeper)
topic = cluster.topics['topicname']
topic.publish('msg')
def setUp(self):
super(TestConsumer, self).setUp()
self.client.ensure_path("/brokers/ids")
self.client.ensure_path('/brokers/topics')
self.c = Cluster(self.client)