def setUpClass(cls): cls.harness = KafkaHarness.start(replicas=1, partitions=2) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.warning("WARNING: KAFKA_VERSION not found in environment") return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = 2 cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas)] # server is used by our superclass when creating the client... cls.server = cls.kafka_brokers[0] # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.warning("WARNING: KAFKA_VERSION not found in environment") return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = PARTITION_COUNT cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas) ] # server is used by our superclass when creating the client... cls.server = cls.kafka_brokers[0] # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setupClass(cls): # start twisted and get root handle reactor, reactorThread = threaded_reactor() cls.reactorWrapper = ReactorWrapper(reactor, reactorThread) cls.connect() # restart remote zfs - there could be update available try: cls.reactorWrapper.setTimeout(5) restart = cls.remoteControlWrapper.call('restart') raise RemoteException("Restart returned (not expected): " + str(restart)) except KeyboardInterrupt: raise except Exception: pass finally: cls.reactorWrapper.setTimeout(cls.defaultTimeout) cls.connect() # configure cls.setupRemoteZfs() cls.setupLocalZfs() #start cls.remoteZfs.call('start') cls.localZfs.runZfs() cls.globalState = TestGlobalState()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING zk_chroot = random_string(10) replicas = 2 partitions = 7 # mini zookeeper, 2 kafka brokers cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas) ] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] # We want a short timeout on message sending for this test, since # we are expecting failures when we take down the brokers cls.client = KafkaClient(hosts, timeout=1000, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING zk_chroot = random_string(10) replicas = 2 partitions = 7 # mini zookeeper, 2 kafka brokers cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas)] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] # We want a short timeout on message sending for this test, since # we are expecting failures when we take down the brokers cls.client = KafkaClient(hosts, timeout=1000, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = 2 max_bytes = 12 * 1048576 # 12 MB cls.zk = ZookeeperFixture.instance() kk_args = [ cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions, max_bytes ] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas) ] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] cls.client = KafkaClient(hosts, timeout=2500, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.warning("WARNING: KAFKA_VERSION not found in environment") return cls.zk = ZookeeperFixture.instance() cls.server = KafkaFixture.instance(0, cls.zk.host, cls.zk.port) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if 'TRAVIS' in os.environ: raise SkipTest("not run on Travis due to flakiness") cls.harness = KafkaHarness.start( replicas=3, partitions=PARTITION_COUNT, ) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.warning("WARNING: KAFKA_VERSION not found in environment") return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING cls.zk = ZookeeperFixture.instance() cls.server = KafkaFixture.instance(0, cls.zk.host, cls.zk.port) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): cls.harness = KafkaHarness.start( replicas=3, partitions=7, ) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor() # We want a short timeout on message sending for this test, since # we are expecting failures when we take down the brokers cls.client = KafkaClient(cls.harness.bootstrap_hosts, timeout=1000, clientId=__name__, reactor=cls.reactor)
def setUpClass(cls): replicas = 3 partitions = 2 max_bytes = 12 * 1048576 # 12 MB cls.harness = KafkaHarness.start( replicas=replicas, partitions=partitions, message_max_bytes=max_bytes, ) cls.client = KafkaClient(cls.harness.bootstrap_hosts, timeout=2500, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = 2 cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas)] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] cls.client = KafkaClient(hosts, timeout=1500, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def wrapper(*args, **kwargs): reactor, reactor_thread = threaded_reactor() if reactor is None: raise ImportError("twisted is not available or could not be imported") q = Queue() def g(): try: fun(*args, **kwargs) except: q.put(sys.exc_info()) else: q.put(None) reactor.callFromThread(g) try: error = q.get(timeout=10) except Empty: raise RuntimeError('Timeout expired') if error is not None: exc_type, exc_value, tb = error raise exc_type, exc_value, tb
def wrapper(*args, **kwargs): reactor, reactor_thread = threaded_reactor() if reactor is None: raise ImportError( "twisted is not available or could not be imported") q = Queue() def g(): try: fun(*args, **kwargs) except: q.put(sys.exc_info()) else: q.put(None) reactor.callFromThread(g) try: error = q.get(timeout=10) except Empty: raise RuntimeError('Timeout expired') if error is not None: exc_type, exc_value, tb = error raise exc_type, exc_value, tb
def test_switch_leader(self): """ Produce messages while killing the coordinator broker. Note that in order to avoid loss of acknowledged writes the producer must request acks of -1 (`afkak.common.PRODUCER_ACK_ALL_REPLICAS`). """ self.harness = KafkaHarness.start( replicas=3, partitions=7, ) self.addCleanup(self.harness.halt) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection self.reactor, self.thread = threaded_reactor() # We want a short timeout on message sending for this test, since # we are expecting failures when we take down the brokers self.client = KafkaClient( self.harness.bootstrap_hosts, timeout=1000, clientId=__name__, reactor=self.reactor, ) producer = Producer( self.client, req_acks=PRODUCER_ACK_ALL_REPLICAS, max_req_attempts=100, ) topic = self.id() try: for index in range(1, 3): # cause the client to establish connections to all the brokers log.debug("Pass: %d. Sending 10 random messages", index) yield self._send_random_messages(producer, topic, 10) # kill leader for partition 0 log.debug("Killing leader of partition 0") broker, kill_time = self._kill_leader(topic, 0) log.debug("Sending 1 more message: 'part 1'") yield producer.send_messages(topic, msgs=[b'part 1']) log.debug("Sending 1 more message: 'part 2'") yield producer.send_messages(topic, msgs=[b'part 2']) # send to new leader log.debug("Sending 10 more messages") yield self._send_random_messages(producer, topic, 10) # Make sure the ZK ephemeral time (~6 seconds) has elapsed wait_time = (kill_time + 6.5) - time.time() if wait_time > 0: log.debug("Waiting: %4.2f for ZK timeout", wait_time) yield async_delay(wait_time) # restart the kafka broker log.debug("Restarting leader broker %r", broker) broker.restart() # count number of messages log.debug("Getting message count") count = yield self._count_messages(topic) self.assertGreaterEqual(count, 22 * index) finally: log.debug("Stopping the producer") yield producer.stop() log.debug("Producer stopped") log.debug("Closing client") yield self.client.close() assertNoDelayedCalls(self.reactor) log.debug("Test complete.")
def setUpClass(cls): from nose.twistedtools import threaded_reactor threaded_reactor()
def setup(self): threaded_reactor() services.youtube.test_hook_exception = None
def setup_class(cls): threaded_reactor()
def setup_module(): threaded_reactor()