def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = b'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers) cls.consumer = cls.client.topics[cls.topic_name].get_simple_consumer( consumer_timeout_ms=1000)
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = b'test-rdkafka-consumer' cls.n_partitions = 3 cls.kafka.create_topic(cls.topic_name, cls.n_partitions, 2) cls.partition_ids = list(range(cls.n_partitions)) cls.start_offsets = cls.n_partitions * [0]
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = 'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers) prod = cls.client.topics[cls.topic_name].get_producer(batch_size=5) prod.produce('msg {num}'.format(num=i) for i in xrange(1000))
def mk_topics(num_partitions_range=NUM_PARTITIONS_RANGE, msg_size_bytes_range=MSG_SIZE_BYTES_RANGE, msg_sum_bytes=10**9, max_num_msgs=10**6): """ Pre-populate an existing testinstances cluster with test messages A topic is created for each combination of partition-count and message-size (passed in iterables). Note that it's easy to eat lots of disk space with this! :param msg_sum_bytes: Limit the number of messages written to a topic so that topic will not exceed this number of bytes :param max_num_msgs: Limit to number of messages written to any topic """ cluster = get_cluster() # make this in advance, like in travis.yml topics = [(np, sz, get_topic_name(np, sz)) for np in num_partitions_range for sz in msg_size_bytes_range] for num_partitions, msg_size_bytes, topic_name in topics: cluster.create_topic(topic_name, num_partitions, replication_factor=1) print "Producing into {}".format(topic_name) client = KafkaClient(cluster.brokers) n_msgs = min(msg_sum_bytes // msg_size_bytes, max_num_msgs) prod = client.topics[topic_name].get_producer() prod.produce(msg_size_bytes * b" " for _ in xrange(n_msgs))
def setUpClass(cls): cls._logger = logging.getLogger(__name__) cls._logger.setLevel(logging.DEBUG) cls._kafka = get_cluster() topics = ["in", "out", "fail"] list(map(lambda topic: cls._kafka.create_topic(topic, 3, 2), topics)) cls._connection = cls._kafka.connection cls._client = KafkaClient(cls._kafka.brokers) if "in" not in cls._client.topics: cls._logger.debug("in not found in topics, waiting") time.sleep(1) cls._client.update_cluster() if "in" not in cls._client.topics: msg = ( "Client topics {0} do not include required input topic".format( cls._client.topics)) cls._logger.error(msg) raise Exception(msg) cls._transformer = PureKafkaValueTransformer( brokers=cls._kafka.brokers, consumer_kwargs={'zookeeper_connect': cls._kafka.zookeeper}, producer_kwargs={}, transform_function=transform, in_topic="in", out_topic="out", backoff=1.0, failure_topic="fail")
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = b'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers, use_greenlets=cls.USE_GEVENT, broker_version=kafka_version)
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = 'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.kafka.produce_messages( cls.topic_name, ('msg {}'.format(i) for i in xrange(1000)) ) cls.client = KafkaClient(cls.kafka.brokers)
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = 'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers) cls.producer = cls.client.topics[cls.topic_name].get_producer() cls.total_messages = 99 for i in range(cls.total_messages): cls.producer.produce(["message %s" % i])
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = 'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers) topic = cls.client.topics[cls.topic_name] cls.producer = topic.get_producer(min_queued_messages=1) cls.total_messages = 99 for i in range(cls.total_messages): cls.producer.produce("message %s" % i)
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.n_partitions = 3 cls.kafka.create_topic(cls.topic_name, cls.n_partitions, 2) cls.client = KafkaClient(cls.kafka.brokers) cls.prod = cls.client.topics[cls.topic_name].get_producer( min_queued_messages=1) for i in range(1000): cls.prod.produce('msg {num}'.format(num=i).encode())
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = b'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers) topic = cls.client.topics[cls.topic_name] cls.producer = topic.get_producer(min_queued_messages=1) cls.total_messages = 99 for i in range(cls.total_messages): cls.producer.produce("message {}".format(i).encode())
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = b'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers) cls.prod = cls.client.topics[cls.topic_name].get_producer( min_queued_messages=1 ) for i in range(1000): cls.prod.produce('msg {num}'.format(num=i).encode())
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.n_partitions = 3 cls.kafka.create_topic(cls.topic_name, cls.n_partitions, 2) cls.client = KafkaClient(cls.kafka.brokers, use_greenlets=cls.USE_GEVENT) cls.prod = cls.client.topics[cls.topic_name].get_producer( min_queued_messages=1 ) for i in range(1000): cls.prod.produce('msg {num}'.format(num=i).encode())
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.kafka.create_topic(cls.topic_name, 3, 2) cls.total_msgs = 1000 cls.client = KafkaClient(cls.kafka.brokers, broker_version=kafka_version) cls.prod = cls.client.topics[cls.topic_name].get_producer( min_queued_messages=1) for i in range(cls.total_msgs): cls.prod.produce('msg {i}'.format(i=i).encode())
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.kafka.create_topic(cls.topic_name, 3, 2) cls.total_msgs = 1000 cls.client = KafkaClient(cls.kafka.brokers, broker_version=kafka_version) cls.prod = cls.client.topics[cls.topic_name].get_producer( min_queued_messages=1 ) for i in range(cls.total_msgs): cls.prod.produce('msg {i}'.format(i=i).encode())
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.n_partitions = 3 cls.kafka.create_topic(cls.topic_name, cls.n_partitions, 2) cls.total_msgs = 1000 cls.client = KafkaClient(cls.kafka.brokers, use_greenlets=cls.USE_GEVENT, broker_version=kafka_version_string) cls.prod = cls.client.topics[cls.topic_name].get_producer( min_queued_messages=1) for i in range(cls.total_msgs): cls.prod.produce('msg {num}'.format(num=i).encode())
def run_bench(consumer_type, num_partitions, msg_size_bytes, num_consumer_fetchers, queued_max_messages, num_iterations, filename_append="consumer_bench.json"): """ Run a single timeit-benchmark for num_iterations of consume() calls This is a somewhat awkward-looking runner, because we currently have an issue where consumers don't get garbage-collected, and so their worker threads keep going even after timeit is done with them (and even if we do slip in a stop() call, they don't release the memory for their message queues - I tried that). As a workaround, we must exit the interpreter after every call to run_bench (cf run_bench_in_shell), and then aggregate the running times in the analysis script instead. :param consumer_type: Either "pure-py" or "rdkafka" :param num_partitions: Number of partitions in topic. You need to prepare a test-topic with this parameter beforehand, see mk_topics() :param msg_size_bytes: Size of messages in topic. You need to prepare a test-topic with this parameter beforehand, see mk_topics() :param num_consumer_fetchers: Passed to consumer init :param queued_max_messages: Passed to consumer init :param num_iterations: Number of iterations in timeit :param filename_append: A line of JSONified benchmark-data is appended to the specified file """ setup = (SETUP.format(topic_name=get_topic_name(num_partitions, msg_size_bytes), num_consumer_fetchers=num_consumer_fetchers, queued_max_messages=queued_max_messages) + SETUP_FIN[consumer_type]) timer = timeit.Timer("cons.consume().value", setup) runtime_secs = timer.timeit(num_iterations) data = {k: v for k, v in vars().items() if k in ("consumer_type", "num_partitions", "msg_size_bytes", "num_consumer_fetchers", "queued_max_messages", "num_iterations", "runtime_secs")} data["num_brokers"] = len(get_cluster().brokers.split(',')) with open(filename_append, 'a') as f: f.write(json.dumps(data) + '\n') print data
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.kafka.create_topic(cls.topic_name, 3, 2) # It turns out that the underlying producer used by KafkaInstance will # write all messages in a batch to a single partition, though not the # same partition every time. We try to attain some spread here by # sending more than one batch: batch = 300 cls.total_msgs = 3 * batch for _ in range(3): cls.kafka.produce_messages(cls.topic_name, ('msg {i}'.format(i=i) for i in range(batch))) cls.client = KafkaClient(cls.kafka.brokers)
def setUpClass(cls): cls.kafka = get_cluster() if not isinstance(cls.kafka, ManagedInstance): pytest.skip("Only test on ManagedInstance (run locally)") cls.client = KafkaClient(cls.kafka.brokers) # BrokerConnection ports = cls.kafka._port_generator(9092) cls.dest_port = next(ports) cls.src_port = next(ports) cls.conn = BrokerConnection('localhost', cls.dest_port, cls.client._handler, buffer_size=1024 * 1024, source_host='localhost', source_port=cls.src_port, ssl_config=None)
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = uuid4().hex.encode() cls.kafka.create_topic(cls.topic_name, 3, 2) # It turns out that the underlying producer used by KafkaInstance will # write all messages in a batch to a single partition, though not the # same partition every time. We try to attain some spread here by # sending more than one batch: batch = 300 cls.total_msgs = 3 * batch for _ in range(3): cls.kafka.produce_messages( cls.topic_name, ('msg {i}'.format(i=i) for i in range(batch))) cls.client = KafkaClient(cls.kafka.brokers)
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = 'test-data' cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers)
def setUpClass(cls): cls.kafka = get_cluster() cls.client = KafkaClient(cls.kafka.brokers)
def setUpClass(cls): cls.kafka = get_cluster() if cls.kafka.brokers_ssl is None: pytest.skip("Test-cluster doesn't advertise ssl ports.")
def setUpClass(cls): cls.kafka = get_cluster() cls.topic_name = "test-data" cls.kafka.create_topic(cls.topic_name, 3, 2) cls.client = KafkaClient(cls.kafka.brokers)