def __init__(self, messagebus): self._conn = messagebus.conn self._general_group = messagebus.general_group self._topic = messagebus.topic_todo self._max_next_requests = messagebus.max_next_requests self._hostname_partitioning = messagebus.hostname_partitioning self._offset_fetcher = Fetcher(self._conn, self._topic, self._general_group)
class SpiderFeedStream(BaseSpiderFeedStream): def __init__(self, messagebus): self._conn = messagebus.conn self._general_group = messagebus.general_group self._topic = messagebus.topic_todo self._max_next_requests = messagebus.max_next_requests self._hostname_partitioning = messagebus.hostname_partitioning self._offset_fetcher = Fetcher(self._conn, self._topic, self._general_group) def consumer(self, partition_id): return Consumer(self._conn, self._topic, self._general_group, partition_id) def available_partitions(self): partitions = [] lags = self._offset_fetcher.get() for partition, lag in lags.iteritems(): if lag < self._max_next_requests: partitions.append(partition) return partitions def producer(self): partitioner = Crc32NamePartitioner if self._hostname_partitioning else FingerprintPartitioner return KeyedProducer(self._conn, self._topic, partitioner)