def __init__(self, context, num_nodes, kafka, topic, new_consumer=False, message_validator=None, from_beginning=True, consumer_timeout_ms=None, version=TRUNK, client_id="console-consumer", jmx_object_names=None, jmx_attributes=[]): """ Args: context: standard context num_nodes: number of nodes to use (this should be 1) kafka: kafka service topic: consume from this topic new_consumer: use new Kafka consumer if True message_validator: function which returns message or None from_beginning: consume from beginning if True, else from the end consumer_timeout_ms: corresponds to consumer.timeout.ms. consumer process ends if time between successively consumed messages exceeds this timeout. Setting this and waiting for the consumer to stop is a pretty good way to consume all messages in a topic. """ JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) BackgroundThreadService.__init__(self, context, num_nodes) self.kafka = kafka self.new_consumer = new_consumer self.args = { 'topic': topic, } self.consumer_timeout_ms = consumer_timeout_ms for node in self.nodes: node.version = version self.from_beginning = from_beginning self.message_validator = message_validator self.messages_consumed = {idx: [] for idx in range(1, num_nodes + 1)} self.client_id = client_id
def __init__(self, context, num_nodes, kafka, topic, num_records, record_size, throughput, settings={}, intermediate_stats=False, client_id="producer-performance", jmx_object_names=None, jmx_attributes=[]): JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) PerformanceService.__init__(self, context, num_nodes) self.kafka = kafka self.security_config = kafka.security_config.client_config() self.args = { 'topic': topic, 'kafka_opts': self.security_config.kafka_opts, 'num_records': num_records, 'record_size': record_size, 'throughput': throughput } self.settings = settings self.intermediate_stats = intermediate_stats self.client_id = client_id
def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAINTEXT, interbroker_security_protocol=SecurityConfig.PLAINTEXT, sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, topics=None, version=TRUNK, quota_config=None, jmx_object_names=None, jmx_attributes=[]): """ :type context :type zk: ZookeeperService :type topics: dict """ Service.__init__(self, context, num_nodes) JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) self.log_level = "DEBUG" self.zk = zk self.quota_config = quota_config self.security_protocol = security_protocol self.interbroker_security_protocol = interbroker_security_protocol self.sasl_mechanism = sasl_mechanism self.topics = topics for node in self.nodes: node.version = version node.config = KafkaConfig( **{config_property.BROKER_ID: self.idx(node)})
def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-group", new_consumer=True, message_validator=None, from_beginning=True, consumer_timeout_ms=None, version=DEV_BRANCH, client_id="console-consumer", print_key=False, jmx_object_names=None, jmx_attributes=None, enable_systest_events=False, stop_timeout_sec=30, print_timestamp=False, isolation_level="read_uncommitted", jaas_override_variables=None): """ Args: context: standard context num_nodes: number of nodes to use (this should be 1) kafka: kafka service topic: consume from this topic new_consumer: use new Kafka consumer if True message_validator: function which returns message or None from_beginning: consume from beginning if True, else from the end consumer_timeout_ms: corresponds to consumer.timeout.ms. consumer process ends if time between successively consumed messages exceeds this timeout. Setting this and waiting for the consumer to stop is a pretty good way to consume all messages in a topic. print_key if True, print each message's key as well enable_systest_events if True, console consumer will print additional lifecycle-related information only available in 0.10.0 and later. stop_timeout_sec After stopping a node, wait up to stop_timeout_sec for the node to stop, and the corresponding background thread to finish successfully. print_timestamp if True, print each message's timestamp as well isolation_level How to handle transactional messages. jaas_override_variables A dict of variables to be used in the jaas.conf template file """ JmxMixin.__init__(self, num_nodes=num_nodes, jmx_object_names=jmx_object_names, jmx_attributes=(jmx_attributes or []), root=ConsoleConsumer.PERSISTENT_ROOT) BackgroundThreadService.__init__(self, context, num_nodes) self.kafka = kafka self.new_consumer = new_consumer self.group_id = group_id self.args = { 'topic': topic, } self.consumer_timeout_ms = consumer_timeout_ms for node in self.nodes: node.version = version self.from_beginning = from_beginning self.message_validator = message_validator self.messages_consumed = {idx: [] for idx in range(1, num_nodes + 1)} self.clean_shutdown_nodes = set() self.client_id = client_id self.print_key = print_key self.log_level = "TRACE" self.stop_timeout_sec = stop_timeout_sec self.isolation_level = isolation_level self.enable_systest_events = enable_systest_events if self.enable_systest_events: # Only available in 0.10.0 and up assert version >= V_0_10_0_0 self.print_timestamp = print_timestamp self.jaas_override_variables = jaas_override_variables or {}
def __init__(self, context, num_nodes, kafka, topic, num_records, record_size, throughput, version=TRUNK, settings=None, intermediate_stats=False, client_id="producer-performance", jmx_object_names=None, jmx_attributes=None): JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes or []) PerformanceService.__init__(self, context, num_nodes) self.logs = { "producer_performance_stdout": { "path": ProducerPerformanceService.STDOUT_CAPTURE, "collect_default": True }, "producer_performance_stderr": { "path": ProducerPerformanceService.STDERR_CAPTURE, "collect_default": True }, "producer_performance_log": { "path": ProducerPerformanceService.LOG_FILE, "collect_default": True }, "jmx_log": { "path": "/mnt/jmx_tool.log", "collect_default": jmx_object_names is not None } } self.kafka = kafka self.security_config = kafka.security_config.client_config() security_protocol = self.security_config.security_protocol assert version >= V_0_9_0_0 or security_protocol == SecurityConfig.PLAINTEXT, \ "Security protocol %s is only supported if version >= 0.9.0.0, version %s" % (self.security_config, str(version)) self.args = { 'topic': topic, 'kafka_opts': self.security_config.kafka_opts, 'num_records': num_records, 'record_size': record_size, 'throughput': throughput } self.settings = settings or {} self.intermediate_stats = intermediate_stats self.client_id = client_id for node in self.nodes: node.version = version
def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAINTEXT, interbroker_security_protocol=SecurityConfig.PLAINTEXT, client_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, authorizer_class_name=None, topics=None, version=DEV_BRANCH, jmx_object_names=None, jmx_attributes=None, zk_connect_timeout=5000, zk_session_timeout=6000, server_prop_overides=None, zk_chroot=None): """ :type context :type zk: ZookeeperService :type topics: dict """ Service.__init__(self, context, num_nodes) JmxMixin.__init__(self, num_nodes=num_nodes, jmx_object_names=jmx_object_names, jmx_attributes=(jmx_attributes or []), root=KafkaService.PERSISTENT_ROOT) self.zk = zk self.security_protocol = security_protocol self.interbroker_security_protocol = interbroker_security_protocol self.client_sasl_mechanism = client_sasl_mechanism self.interbroker_sasl_mechanism = interbroker_sasl_mechanism self.topics = topics self.minikdc = None self.authorizer_class_name = authorizer_class_name self.zk_set_acl = False if server_prop_overides is None: self.server_prop_overides = [] else: self.server_prop_overides = server_prop_overides self.log_level = "DEBUG" self.zk_chroot = zk_chroot # # In a heavily loaded and not very fast machine, it is # sometimes necessary to give more time for the zk client # to have its session established, especially if the client # is authenticating and waiting for the SaslAuthenticated # in addition to the SyncConnected event. # # The default value for zookeeper.connect.timeout.ms is # 2 seconds and here we increase it to 5 seconds, but # it can be overridden by setting the corresponding parameter # for this constructor. self.zk_connect_timeout = zk_connect_timeout # Also allow the session timeout to be provided explicitly, # primarily so that test cases can depend on it when waiting # e.g. brokers to deregister after a hard kill. self.zk_session_timeout = zk_session_timeout self.port_mappings = { 'PLAINTEXT': Port('PLAINTEXT', 9092, False), 'SSL': Port('SSL', 9093, False), 'SASL_PLAINTEXT': Port('SASL_PLAINTEXT', 9094, False), 'SASL_SSL': Port('SASL_SSL', 9095, False) } for node in self.nodes: node.version = version node.config = KafkaConfig(**{config_property.BROKER_ID: self.idx(node)})
def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-group", new_consumer=True, message_validator=None, from_beginning=True, consumer_timeout_ms=None, version=DEV_BRANCH, client_id="console-consumer", print_key=False, jmx_object_names=None, jmx_attributes=None, enable_systest_events=False, stop_timeout_sec=15, print_timestamp=False, isolation_level="read_uncommitted"): """ Args: context: standard context num_nodes: number of nodes to use (this should be 1) kafka: kafka service topic: consume from this topic new_consumer: use new Kafka consumer if True message_validator: function which returns message or None from_beginning: consume from beginning if True, else from the end consumer_timeout_ms: corresponds to consumer.timeout.ms. consumer process ends if time between successively consumed messages exceeds this timeout. Setting this and waiting for the consumer to stop is a pretty good way to consume all messages in a topic. print_key if True, print each message's key as well enable_systest_events if True, console consumer will print additional lifecycle-related information only available in 0.10.0 and later. stop_timeout_sec After stopping a node, wait up to stop_timeout_sec for the node to stop, and the corresponding background thread to finish successfully. print_timestamp if True, print each message's timestamp as well isolation_level How to handle transactional messages. """ JmxMixin.__init__(self, num_nodes=num_nodes, jmx_object_names=jmx_object_names, jmx_attributes=(jmx_attributes or []), root=ConsoleConsumer.PERSISTENT_ROOT) BackgroundThreadService.__init__(self, context, num_nodes) self.kafka = kafka self.new_consumer = new_consumer self.group_id = group_id self.args = { 'topic': topic, } self.consumer_timeout_ms = consumer_timeout_ms for node in self.nodes: node.version = version self.from_beginning = from_beginning self.message_validator = message_validator self.messages_consumed = {idx: [] for idx in range(1, num_nodes + 1)} self.clean_shutdown_nodes = set() self.client_id = client_id self.print_key = print_key self.log_level = "TRACE" self.stop_timeout_sec = stop_timeout_sec self.isolation_level = isolation_level self.enable_systest_events = enable_systest_events if self.enable_systest_events: # Only available in 0.10.0 and up assert version >= V_0_10_0_0 self.print_timestamp = print_timestamp
def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAINTEXT, interbroker_security_protocol=SecurityConfig.PLAINTEXT, sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, authorizer_class_name=None, topics=None, version=TRUNK, quota_config=None, jmx_object_names=None, jmx_attributes=[], zk_connect_timeout=5000): """ :type context :type zk: ZookeeperService :type topics: dict """ Service.__init__(self, context, num_nodes) JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) self.zk = zk self.quota_config = quota_config self.security_protocol = security_protocol self.interbroker_security_protocol = interbroker_security_protocol self.sasl_mechanism = sasl_mechanism self.topics = topics self.minikdc = None self.authorizer_class_name = authorizer_class_name # # In a heavily loaded and not very fast machine, it is # sometimes necessary to give more time for the zk client # to have its session established, especially if the client # is authenticating and waiting for the SaslAuthenticated # in addition to the SyncConnected event. # # The defaut value for zookeeper.connect.timeout.ms is # 2 seconds and here we increase it to 5 seconds, but # it can be overriden by setting the corresponding parameter # for this constructor. self.zk_connect_timeout = zk_connect_timeout self.port_mappings = { 'PLAINTEXT': Port('PLAINTEXT', 9092, False), 'SSL': Port('SSL', 9093, False), 'SASL_PLAINTEXT': Port('SASL_PLAINTEXT', 9094, False), 'SASL_SSL': Port('SASL_SSL', 9095, False) } for node in self.nodes: node.version = version node.config = KafkaConfig( **{config_property.BROKER_ID: self.idx(node)})
def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAINTEXT, interbroker_security_protocol=SecurityConfig.PLAINTEXT, client_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, authorizer_class_name=None, topics=None, version=TRUNK, quota_config=None, jmx_object_names=None, jmx_attributes=[], zk_connect_timeout=5000): """ :type context :type zk: ZookeeperService :type topics: dict """ Service.__init__(self, context, num_nodes) JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) self.zk = zk self.quota_config = quota_config self.security_protocol = security_protocol self.interbroker_security_protocol = interbroker_security_protocol self.client_sasl_mechanism = client_sasl_mechanism self.interbroker_sasl_mechanism = interbroker_sasl_mechanism self.topics = topics self.minikdc = None self.authorizer_class_name = authorizer_class_name # # In a heavily loaded and not very fast machine, it is # sometimes necessary to give more time for the zk client # to have its session established, especially if the client # is authenticating and waiting for the SaslAuthenticated # in addition to the SyncConnected event. # # The defaut value for zookeeper.connect.timeout.ms is # 2 seconds and here we increase it to 5 seconds, but # it can be overriden by setting the corresponding parameter # for this constructor. self.zk_connect_timeout = zk_connect_timeout self.port_mappings = { 'PLAINTEXT': Port('PLAINTEXT', 9092, False), 'SSL': Port('SSL', 9093, False), 'SASL_PLAINTEXT': Port('SASL_PLAINTEXT', 9094, False), 'SASL_SSL': Port('SASL_SSL', 9095, False) } for node in self.nodes: node.version = version node.config = KafkaConfig(**{config_property.BROKER_ID: self.idx(node)})
def __init__(self, context, num_nodes, kafka, topic, num_records, record_size, throughput, version=TRUNK, settings={}, intermediate_stats=False, client_id="producer-performance", jmx_object_names=None, jmx_attributes=[]): JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) PerformanceService.__init__(self, context, num_nodes) self.logs = { "producer_performance_stdout": { "path": ProducerPerformanceService.STDOUT_CAPTURE, "collect_default": True}, "producer_performance_stderr": { "path": ProducerPerformanceService.STDERR_CAPTURE, "collect_default": True}, "producer_performance_log": { "path": ProducerPerformanceService.LOG_FILE, "collect_default": True}, "jmx_log": { "path": "/mnt/jmx_tool.log", "collect_default": jmx_object_names is not None } } self.kafka = kafka self.security_config = kafka.security_config.client_config() security_protocol = self.security_config.security_protocol assert version >= V_0_9_0_0 or security_protocol == SecurityConfig.PLAINTEXT, \ "Security protocol %s is only supported if version >= 0.9.0.0, version %s" % (self.security_config, str(version)) self.args = { 'topic': topic, 'kafka_opts': self.security_config.kafka_opts, 'num_records': num_records, 'record_size': record_size, 'throughput': throughput } self.settings = settings self.intermediate_stats = intermediate_stats self.client_id = client_id for node in self.nodes: node.version = version
def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAINTEXT, interbroker_security_protocol=SecurityConfig.PLAINTEXT, sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, topics=None, version=TRUNK, quota_config=None, jmx_object_names=None, jmx_attributes=[]): """ :type context :type zk: ZookeeperService :type topics: dict """ Service.__init__(self, context, num_nodes) JmxMixin.__init__(self, num_nodes, jmx_object_names, jmx_attributes) self.log_level = "DEBUG" self.zk = zk self.quota_config = quota_config self.security_protocol = security_protocol self.interbroker_security_protocol = interbroker_security_protocol self.sasl_mechanism = sasl_mechanism self.topics = topics for node in self.nodes: node.version = version node.config = KafkaConfig(**{config_property.BROKER_ID: self.idx(node)})
def __init__(self, test_context, kafka, numrecs, load_phase, test_name, num_threads): super(StreamsSimpleBenchmarkService, self).__init__(test_context, kafka, "org.apache.kafka.streams.perf.SimpleBenchmark", numrecs, load_phase, test_name, num_threads) self.load_phase = load_phase if self.load_phase == "false": JmxMixin.__init__( self, num_nodes=1, jmx_object_names=[ 'kafka.streams:type=stream-metrics,client-id=simple-benchmark-StreamThread-%d' % (i + 1) for i in range(num_threads) ], jmx_attributes=[ 'process-latency-avg', 'process-rate', 'commit-latency-avg', 'commit-rate', 'poll-latency-avg', 'poll-rate' ], root=StreamsTestBaseService.PERSISTENT_ROOT)
def __init__(self, test_context, kafka, test_name, num_threads, num_recs_or_wait_ms, key_skew, value_size): super(StreamsSimpleBenchmarkService, self).__init__(test_context, kafka, "org.apache.kafka.streams.perf.SimpleBenchmark", test_name, num_recs_or_wait_ms, key_skew, value_size) self.jmx_option = "" if test_name.startswith('stream') or test_name.startswith('table'): self.jmx_option = "stream-jmx" JmxMixin.__init__( self, num_nodes=1, jmx_object_names=[ 'kafka.streams:type=stream-thread-metrics,thread-id=simple-benchmark-StreamThread-%d' % (i + 1) for i in range(num_threads) ], jmx_attributes=[ 'process-latency-avg', 'process-rate', 'commit-latency-avg', 'commit-rate', 'poll-latency-avg', 'poll-rate' ], root=StreamsTestBaseService.PERSISTENT_ROOT) if test_name.startswith('consume'): self.jmx_option = "consumer-jmx" JmxMixin.__init__( self, num_nodes=1, jmx_object_names=[ 'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=simple-benchmark-consumer' ], jmx_attributes=['records-consumed-rate'], root=StreamsTestBaseService.PERSISTENT_ROOT) self.num_threads = num_threads
def __init__( self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAINTEXT, interbroker_security_protocol=SecurityConfig.PLAINTEXT, client_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SecurityConfig.SASL_MECHANISM_GSSAPI, authorizer_class_name=None, topics=None, version=DEV_BRANCH, jmx_object_names=None, jmx_attributes=None, zk_connect_timeout=5000, zk_session_timeout=6000, server_prop_overides=None, zk_chroot=None, listener_security_config=ListenerSecurityConfig(), per_node_server_prop_overrides={}): """ :param context: test context :param ZookeeperService zk: :param dict topics: which topics to create automatically :param str security_protocol: security protocol for clients to use :param str interbroker_security_protocol: security protocol to use for broker-to-broker communication :param str client_sasl_mechanism: sasl mechanism for clients to use :param str interbroker_sasl_mechanism: sasl mechanism to use for broker-to-broker communication :param str authorizer_class_name: which authorizer class to use :param str version: which kafka version to use. Defaults to "dev" branch :param jmx_object_names: :param jmx_attributes: :param int zk_connect_timeout: :param int zk_session_timeout: :param dict server_prop_overides: overrides for kafka.properties file :param zk_chroot: :param ListenerSecurityConfig listener_security_config: listener config to use """ Service.__init__(self, context, num_nodes) JmxMixin.__init__(self, num_nodes=num_nodes, jmx_object_names=jmx_object_names, jmx_attributes=(jmx_attributes or []), root=KafkaService.PERSISTENT_ROOT) self.zk = zk self.security_protocol = security_protocol self.client_sasl_mechanism = client_sasl_mechanism self.topics = topics self.minikdc = None self.authorizer_class_name = authorizer_class_name self.zk_set_acl = False if server_prop_overides is None: self.server_prop_overides = [] else: self.server_prop_overides = server_prop_overides if per_node_server_prop_overrides is None: self.per_node_server_prop_overrides = {} else: self.per_node_server_prop_overrides = per_node_server_prop_overrides self.log_level = "DEBUG" self.zk_chroot = zk_chroot self.listener_security_config = listener_security_config # # In a heavily loaded and not very fast machine, it is # sometimes necessary to give more time for the zk client # to have its session established, especially if the client # is authenticating and waiting for the SaslAuthenticated # in addition to the SyncConnected event. # # The default value for zookeeper.connect.timeout.ms is # 2 seconds and here we increase it to 5 seconds, but # it can be overridden by setting the corresponding parameter # for this constructor. self.zk_connect_timeout = zk_connect_timeout # Also allow the session timeout to be provided explicitly, # primarily so that test cases can depend on it when waiting # e.g. brokers to deregister after a hard kill. self.zk_session_timeout = zk_session_timeout self.port_mappings = { 'PLAINTEXT': KafkaListener('PLAINTEXT', 9092, 'PLAINTEXT', False), 'SSL': KafkaListener('SSL', 9093, 'SSL', False), 'SASL_PLAINTEXT': KafkaListener('SASL_PLAINTEXT', 9094, 'SASL_PLAINTEXT', False), 'SASL_SSL': KafkaListener('SASL_SSL', 9095, 'SASL_SSL', False), KafkaService.INTERBROKER_LISTENER_NAME: KafkaListener(KafkaService.INTERBROKER_LISTENER_NAME, 9099, None, False) } self.interbroker_listener = None self.setup_interbroker_listener( interbroker_security_protocol, self.listener_security_config.use_separate_interbroker_listener) self.interbroker_sasl_mechanism = interbroker_sasl_mechanism for node in self.nodes: node.version = version node.config = KafkaConfig( **{config_property.BROKER_ID: self.idx(node)})
def __init__(self, text_context, *args, **kwargs): JmxMixin.__init__(self, num_nodes=1, *args, **kwargs) self.context = text_context