def _start_brokers(self):
"""Start all brokers and return used ports."""
self._broker_procs = []
ports = self._port_generator(9092)
used_ports = []
for i in range(self._num_instances):
port = next(ports)
used_ports.append(port)
log.info('Starting Kafka on port %i.', port)
conf = os.path.join(
self._conf_dir,
'kafka_{instance}.properties'.format(instance=i))
with open(conf, 'w') as f:
f.write(
_kafka_properties.format(
broker_id=i,
port=port,
zk_connstr=self.zookeeper,
data_dir=self._data_dir +
'_{instance}'.format(instance=i),
))
binfile = os.path.join(self._bin_dir, 'bin/kafka-server-start.sh')
logfile = os.path.join(self._log_dir,
'kafka_{instance}.log'.format(instance=i))
self._broker_procs.append(
utils.Popen(args=[binfile, conf],
stderr=utils.STDOUT,
stdout=open(logfile, 'w'),
use_gevent=self.use_gevent))
return used_ports
def _start_zookeeper(self):
port = next(self._port_generator(2181))
log.info('Starting zookeeper on port %i.', port)
conf = os.path.join(self._conf_dir, 'zk.properties')
with open(conf, 'w') as f:
f.write(
_zookeeper_properties.format(zk_dir=self._zk_dir,
zk_port=port))
binfile = os.path.join(self._bin_dir, 'bin/zookeeper-server-start.sh')
logfile = os.path.join(self._log_dir, 'zk.log')
self._zk_proc = utils.Popen(args=[binfile, conf],
stderr=utils.STDOUT,
stdout=open(logfile, 'w'),
use_gevent=self.use_gevent)
return port
def _start_process(self):
"""Start the instance process"""
log.info('Starting mongod on port %i...', self.port)
self._process = utils.Popen(
args=[
"mongod",
'--port',
str(self.port),
'--bind_ip',
'127.0.0.1',
'--dbpath',
self._root_dir,
'--nojournal',
],
stderr=utils.STDOUT,
stdout=open(self.logfile, 'w'),
use_gevent=self.use_gevent,
)
# Connect to the shiny new instance
self.conn = None
fails = 0
while self.conn is None:
try:
conn = pymongo.MongoClient(port=self.port,
use_greenlets=self.use_gevent)
if conn.alive():
self.conn = conn
except:
if fails >= self.timeout:
break
fails += 1
time.sleep(1)
if self.conn is None or self._process.poll() is not None:
log.info('Unable to start process. Trying to get process output.')
try:
log.info(open(self.logfile).read())
except:
pass # file doesn't exist or whatever
raise ProcessNotStartingError(
"Unable to start mongod in {} seconds.".format(self.timeout))
def _start_process(self):
"""Start the instance process"""
log.info('Starting redis-server on port %i...', self.port)
if self.dumpfile:
log.info('Loading dumpfile %s...', self.dumpfile)
shutil.copyfile(
self.dumpfile,
os.path.join(self._root_dir, os.path.basename(self.dumpfile)))
self._process = utils.Popen(
args=[
"redis-server",
'--port',
str(self.port),
'--bind',
'127.0.0.1',
'--dir',
self._root_dir,
],
stderr=utils.STDOUT,
stdout=open(self.logfile, 'w'),
use_gevent=self.use_gevent,
)
# Connect to the shiny new instance
self.conn = None
fails = 0
while self.conn is None:
try:
conn = redis.StrictRedis(port=self.port)
if conn.info()['loading'] == 0: # make sure dump is loaded
self.conn = conn
except:
if fails == 10:
break
fails += 1
time.sleep(1)
if self.conn is None or self._process.poll() is not None:
log.critical('Unable to start redis-server in 10 seconds')
raise ProcessNotStartingError("Unable to start redis-server")