def _start_kafka(self, log, port=9092, zookeeper_port=2181):
kdata = os.path.abspath(os.path.join(self._data, 'kdata'))
if not os.path.isdir(kdata):
fileutils.makedirs(kdata, exist_ok=True)
config = os.path.join(self._data, 'kafka.properties')
with open(config, 'w') as cout:
cout.write('broker.id=0\n')
cout.write('listeners=PLAINTEXT://0.0.0.0:{port}\n'.format(port=port))
cout.write('log.dirs={data}\n'.format(data=kdata))
cout.write('num.partitions=1\n')
cout.write('log.retention.hours=8760000\n')
cout.write('zookeeper.connect=localhost:{port}\n'.format(port=zookeeper_port))
command = [self._kafka_bin, config]
kafka = subprocess.Popen(command, stdout=log, stderr=log, shell=False).pid
for i in range(1, 5):
with open(log.name, 'r') as rlog:
for line in rlog:
if 'INFO [Kafka Server 0], started (kafka.server.KafkaServer)' in line:
return kafka
time.sleep(1)
daemon.kill(kafka)
return None
def start(self):
if self.is_running():
raise IllegalStateException('process is already running')
self._log_file = self._engine.get_logfile('embedded-kafka', ensure=True)
success = False
zpid, kpid = 0, 0
log = open(self._log_file, 'w')
try:
zpid = self._start_zookeeper(log)
if zpid is None:
raise IllegalStateException(
'failed to start zookeeper, check log file for more details: ' + self._log_file)
kpid = self._start_kafka(log)
if kpid is None:
raise IllegalStateException(
'failed to start kafka, check log file for more details: ' + self._log_file)
self._set_pids(kpid, zpid)
success = True
except:
if not success:
daemon.kill(kpid)
daemon.kill(zpid)
log.close()
raise
def _start_zookeeper(self, log, port=2181):
zdata = os.path.abspath(os.path.join(self._data, 'zdata'))
if not os.path.isdir(zdata):
fileutils.makedirs(zdata, exist_ok=True)
config = os.path.join(self._data, 'zookeeper.properties')
with open(config, 'w') as cout:
cout.write('dataDir={data}\n'.format(data=zdata))
cout.write('clientPort={port}\n'.format(port=port))
cout.write('maxClientCnxns=0\n')
command = [self._zookeeper_bin, config]
zookeeper = subprocess.Popen(command, stdout=log, stderr=log, shell=False).pid
for i in range(1, 5):
try:
msg = fileutils.netcat('127.0.0.1', port, 'ruok', timeout=2)
except:
msg = None
if 'imok' == msg:
return zookeeper
else:
time.sleep(1)
daemon.kill(zookeeper)
return None
def _start_zookeeper(self, log, port=2181):
zdata = os.path.abspath(os.path.join(self._data, 'zdata'))
if not os.path.isdir(zdata):
fileutils.makedirs(zdata, exist_ok=True)
config = os.path.join(self._data, 'zookeeper.properties')
with open(config, 'w') as cout:
cout.write('dataDir={data}\n'.format(data=zdata))
cout.write('clientPort={port}\n'.format(port=port))
cout.write('maxClientCnxns=0\n')
command = [self._zookeeper_bin, config]
zookeeper = subprocess.Popen(command,
stdout=log,
stderr=log,
shell=False).pid
for i in range(1, 5):
try:
msg = fileutils.netcat('127.0.0.1', port, 'ruok', timeout=2)
except:
msg = None
if 'imok' == msg:
return zookeeper
else:
time.sleep(1)
daemon.kill(zookeeper)
return None
def _start_kafka(self, log, port=9092, zookeeper_port=2181):
kdata = os.path.abspath(os.path.join(self._data, 'kdata'))
if not os.path.isdir(kdata):
fileutils.makedirs(kdata, exist_ok=True)
config = os.path.join(self._data, 'kafka.properties')
with open(config, 'w') as cout:
cout.write('broker.id=0\n')
cout.write(
'listeners=PLAINTEXT://0.0.0.0:{port}\n'.format(port=port))
cout.write('log.dirs={data}\n'.format(data=kdata))
cout.write('num.partitions=1\n')
cout.write('log.retention.hours=8760000\n')
cout.write('zookeeper.connect=localhost:{port}\n'.format(
port=zookeeper_port))
command = [self._kafka_bin, config]
kafka = subprocess.Popen(command, stdout=log, stderr=log,
shell=False).pid
for i in range(1, 5):
with open(log.name, 'r') as rlog:
for line in rlog:
if 'INFO [Kafka Server 0], started (kafka.server.KafkaServer)' in line:
return kafka
time.sleep(1)
daemon.kill(kafka)
return None
def start(self):
if self.is_running():
raise IllegalStateException('process is already running')
self._log_file = self._engine.get_logfile('embedded-kafka',
ensure=True)
success = False
zpid, kpid = 0, 0
log = open(self._log_file, 'w')
try:
zpid = self._start_zookeeper(log)
if zpid is None:
raise IllegalStateException(
'failed to start zookeeper, check log file for more details: '
+ self._log_file)
kpid = self._start_kafka(log)
if kpid is None:
raise IllegalStateException(
'failed to start kafka, check log file for more details: '
+ self._log_file)
self._set_pids(kpid, zpid)
success = True
except:
if not success:
daemon.kill(kpid)
daemon.kill(zpid)
log.close()
raise
def stop(self):
kpid, zpid = self._get_pids()
if not self.is_running():
raise IllegalStateException('process is not running')
daemon.kill(kpid, 5)
daemon.kill(zpid)
def stop(self):
kpid, zpid = self._get_pids()
if not self.is_running():
raise IllegalStateException('process is not running')
daemon.kill(kpid, 5)
daemon.kill(zpid)
def stop(self):
pid = self._get_pid()
if not self.is_running():
raise IllegalStateException('process is not running')
daemon.kill(pid, ClusterNode.__SIGTERM_TIMEOUT)
if self._kafka:
self._kafka.stop()
def stop(self):
pid = self._get_pid()
if not self.is_running():
raise IllegalStateException('process is not running')
daemon.kill(pid, ClusterNode.__SIGTERM_TIMEOUT)
if self._kafka:
self._kafka.stop()
def stop(self):
pid = self._get_pid()
if self.is_running():
daemon.kill(pid, ClusterNode.__SIGTERM_TIMEOUT)
if self._kafka:
self._kafka.stop()
if self._cassandra:
self._cassandra.stop()
def stop(self):
pid = self._get_pid()
self._update_properties()
if self.is_running():
daemon.kill(pid, ClusterNode.__SIGTERM_TIMEOUT)
if self._properties is not None and "embedded_services" in self._properties:
for service_pid in self._properties["embedded_services"]:
daemon.kill(service_pid, ignore_errors=True)
os.remove(self._status_file)
os.remove(self._pidfile)
def start(self):
if self.is_running():
raise IllegalStateException(
'Cannot start Kafka process. Kafka process is already running')
if not netutils.is_free(self.port):
raise IllegalStateException(
'port %d is already in use, please specify another port with --datastream-port'
% self.port)
self._log_file = self._engine.get_logfile('embedded-kafka',
ensure=True)
shutil.rmtree(self._runtime, ignore_errors=True)
fileutils.makedirs(self._runtime, exist_ok=True)
success = False
zpid, kpid = 0, 0
log = open(self._log_file, 'w')
try:
zookeeper_port = netutils.get_free_tcp_port()
zpid = self._start_zookeeper(log, zookeeper_port)
if zpid is None:
raise IllegalStateException(
'failed to start zookeeper, check log file for more details: '
+ self._log_file)
kpid = self._start_kafka(log, zookeeper_port)
if kpid is None:
raise IllegalStateException(
'failed to start kafka, check log file for more details: '
+ self._log_file)
self._set_pids(kpid, zpid)
success = True
except:
if not success:
daemon.kill(kpid)
daemon.kill(zpid)
log.close()
raise
def start(self):
if self.is_running():
raise IllegalStateException(
'Cannot start Cassandra process. Cassandra process is already running'
)
if not netutils.is_free(self.port):
raise IllegalStateException(
'port %d is already in use, please specify another port with --db-port'
% self.port)
self._log_file = self._engine.get_logfile('embedded-cassandra',
ensure=True)
shutil.rmtree(self._runtime, ignore_errors=True)
fileutils.makedirs(self._runtime, exist_ok=True)
success = False
cpid = 0
log = open(self._log_file, 'w')
try:
cpid = self._start_cassandra(log)
if cpid is None:
raise IllegalStateException(
'failed to start Cassandra, check log file for more details: '
+ self._log_file)
self._set_pid(cpid)
success = True
except:
if not success:
daemon.kill(cpid)
log.close()
raise
def stop(self):
if self.is_running():
daemon.kill(self._get_pid(), 5)
def stop(self):
kpid, zpid = self._get_pids()
if self.is_running():
daemon.kill(kpid, 5)
daemon.kill(zpid)