def __init__(self, messagebus):
self._conn = messagebus.conn
self._general_group = messagebus.general_group
self._topic = messagebus.topic_todo
self._max_next_requests = messagebus.max_next_requests
self._hostname_partitioning = messagebus.hostname_partitioning
self._offset_fetcher = Fetcher(self._conn, self._topic,
self._general_group)
def __init__(self, messagebus):
self._conn = messagebus.conn
self._general_group = messagebus.general_group
self._topic = messagebus.topic_todo
self._max_next_requests = messagebus.max_next_requests
self._hostname_partitioning = messagebus.hostname_partitioning
self._offset_fetcher = Fetcher(self._conn, self._topic, self._general_group)
class SpiderFeedStream(BaseSpiderFeedStream):
def __init__(self, messagebus):
self._conn = messagebus.conn
self._general_group = messagebus.general_group
self._topic = messagebus.topic_todo
self._max_next_requests = messagebus.max_next_requests
self._hostname_partitioning = messagebus.hostname_partitioning
self._offset_fetcher = Fetcher(self._conn, self._topic,
self._general_group)
def consumer(self, partition_id):
return Consumer(self._conn, self._topic, self._general_group,
partition_id)
def available_partitions(self):
partitions = []
lags = self._offset_fetcher.get()
for partition, lag in lags.iteritems():
if lag < self._max_next_requests:
partitions.append(partition)
return partitions
def producer(self):
partitioner = Crc32NamePartitioner if self._hostname_partitioning else FingerprintPartitioner
return KeyedProducer(self._conn, self._topic, partitioner)
class SpiderFeedStream(BaseSpiderFeedStream):
def __init__(self, messagebus):
self._conn = messagebus.conn
self._general_group = messagebus.general_group
self._topic = messagebus.topic_todo
self._max_next_requests = messagebus.max_next_requests
self._hostname_partitioning = messagebus.hostname_partitioning
self._offset_fetcher = Fetcher(self._conn, self._topic, self._general_group)
def consumer(self, partition_id):
return Consumer(self._conn, self._topic, self._general_group, partition_id)
def available_partitions(self):
partitions = []
lags = self._offset_fetcher.get()
for partition, lag in lags.iteritems():
if lag < self._max_next_requests:
partitions.append(partition)
return partitions
def producer(self):
partitioner = Crc32NamePartitioner if self._hostname_partitioning else FingerprintPartitioner
return KeyedProducer(self._conn, self._topic, partitioner)