def run_zookeeper(dist_directory: str = MC_DIST_DIR,
listen: str = MC_ZOOKEEPER_LISTEN,
port: int = MC_ZOOKEEPER_PORT,
data_dir: str = MC_SOLR_BASE_DATA_DIR,
zookeeper_version: str = MC_ZOOKEEPER_VERSION,
solr_version: str = MC_SOLR_VERSION) -> None:
"""Run ZooKeeper, install if needed too."""
if not __zookeeper_is_installed():
log.info("ZooKeeper is not installed, installing...")
__install_zookeeper()
data_dir = resolve_absolute_path_under_mc_root(path=data_dir,
must_exist=True)
zookeeper_data_dir = os.path.join(data_dir, "mediacloud-cluster-zookeeper")
if not os.path.isdir(zookeeper_data_dir):
log.info("Creating data directory at %s..." % zookeeper_data_dir)
mkdir_p(zookeeper_data_dir)
if tcp_port_is_open(port=port):
raise McZooKeeperRunException(
"Port %d is already open on this machine." % port)
zookeeper_path = __zookeeper_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
zkserver_path = os.path.join(zookeeper_path, "bin", "zkServer.sh")
if not os.path.isfile(zkserver_path):
raise McZooKeeperRunException("zkServer.sh at '%s' was not found." %
zkserver_path)
log4j_properties_path = os.path.join(zookeeper_path, "conf",
"log4j.properties")
if not os.path.isfile(log4j_properties_path):
raise McZooKeeperRunException(
"log4j.properties at '%s' was not found.")
zoo_cnf_path = os.path.join(zookeeper_data_dir, "zoo.cfg")
log.info("Creating zoo.cfg in '%s'..." % zoo_cnf_path)
with open(zoo_cnf_path, 'w') as zoo_cnf:
zoo_cnf.write("""
#
# This file is autogenerated. Please do not modify it!
#
clientPortAddress=%(listen)s
clientPort=%(port)d
dataDir=%(data_dir)s
# Must be between zkClientTimeout / 2 and zkClientTimeout / 20
tickTime=30000
initLimit=10
syncLimit=10
""" % {
"listen": listen,
"port": port,
"data_dir": zookeeper_data_dir,
})
zookeeper_env = os.environ.copy()
zookeeper_env[
"ZOOCFGDIR"] = zookeeper_data_dir # Serves as configuration dir too
zookeeper_env["ZOOCFG"] = "zoo.cfg"
zookeeper_env["ZOO_LOG_DIR"] = zookeeper_data_dir
zookeeper_env[
"SERVER_JVMFLAGS"] = "-Dlog4j.configuration=file://" + os.path.abspath(
log4j_properties_path)
args = [zkserver_path, "start-foreground"]
log.info("Starting ZooKeeper on %s:%d..." % (listen, port))
log.debug("Running command: %s" % str(args))
log.debug("Environment variables: %s" % str(zookeeper_env))
process = subprocess.Popen(args, env=zookeeper_env)
global __zookeeper_pid
__zookeeper_pid = process.pid
# Declare that we don't care about the exit code of the child process so
# it doesn't become a zombie when it gets killed in signal handler
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
signal.signal(signal.SIGTERM, __kill_zookeeper_process
) # SIGTERM is handled differently for whatever reason
atexit.register(__kill_zookeeper_process)
log.info("ZooKeeper PID: %d" % __zookeeper_pid)
log.info("Waiting for ZooKeeper to start at port %d..." % port)
zookeeper_started = wait_for_tcp_port_to_open(
port=port, retries=MC_ZOOKEEPER_CONNECT_RETRIES)
if not zookeeper_started:
raise McZooKeeperRunException(
"Unable to connect to ZooKeeper at port %d" % port)
log.info(
"Uploading initial Solr collection configurations to ZooKeeper...")
update_zookeeper_solr_configuration(zookeeper_host="localhost",
zookeeper_port=port,
dist_directory=dist_directory,
solr_version=solr_version)
log.info("ZooKeeper is ready on port %d!" % port)
while True:
time.sleep(1)
def run_zookeeper(dist_directory: str = MC_DIST_DIR,
listen: str = MC_ZOOKEEPER_LISTEN,
port: int = MC_ZOOKEEPER_PORT,
data_dir: str = MC_SOLR_BASE_DATA_DIR,
zookeeper_version: str = MC_ZOOKEEPER_VERSION,
solr_version: str = MC_SOLR_VERSION) -> None:
"""Run ZooKeeper, install if needed too."""
if not __zookeeper_is_installed():
log.info("ZooKeeper is not installed, installing...")
__install_zookeeper()
data_dir = resolve_absolute_path_under_mc_root(path=data_dir, must_exist=True)
zookeeper_data_dir = os.path.join(data_dir, "mediacloud-cluster-zookeeper")
if not os.path.isdir(zookeeper_data_dir):
log.info("Creating data directory at %s..." % zookeeper_data_dir)
mkdir_p(zookeeper_data_dir)
if tcp_port_is_open(port=port):
raise McZooKeeperRunException("Port %d is already open on this machine." % port)
zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version)
zkserver_path = os.path.join(zookeeper_path, "bin", "zkServer.sh")
if not os.path.isfile(zkserver_path):
raise McZooKeeperRunException("zkServer.sh at '%s' was not found." % zkserver_path)
log4j_properties_path = os.path.join(zookeeper_path, "conf", "log4j.properties")
if not os.path.isfile(log4j_properties_path):
raise McZooKeeperRunException("log4j.properties at '%s' was not found.")
zoo_cnf_path = os.path.join(zookeeper_data_dir, "zoo.cfg")
log.info("Creating zoo.cfg in '%s'..." % zoo_cnf_path)
with open(zoo_cnf_path, 'w') as zoo_cnf:
zoo_cnf.write("""
#
# This file is autogenerated. Please do not modify it!
#
clientPortAddress=%(listen)s
clientPort=%(port)d
dataDir=%(data_dir)s
# Must be between zkClientTimeout / 2 and zkClientTimeout / 20
tickTime=30000
initLimit=10
syncLimit=10
""" % {
"listen": listen,
"port": port,
"data_dir": zookeeper_data_dir,
})
zookeeper_env = os.environ.copy()
zookeeper_env["ZOOCFGDIR"] = zookeeper_data_dir # Serves as configuration dir too
zookeeper_env["ZOOCFG"] = "zoo.cfg"
zookeeper_env["ZOO_LOG_DIR"] = zookeeper_data_dir
zookeeper_env["SERVER_JVMFLAGS"] = "-Dlog4j.configuration=file://" + os.path.abspath(log4j_properties_path)
args = [
zkserver_path,
"start-foreground"
]
log.info("Starting ZooKeeper on %s:%d..." % (listen, port))
log.debug("Running command: %s" % str(args))
log.debug("Environment variables: %s" % str(zookeeper_env))
process = subprocess.Popen(args, env=zookeeper_env)
global __zookeeper_pid
__zookeeper_pid = process.pid
# Declare that we don't care about the exit code of the child process so
# it doesn't become a zombie when it gets killed in signal handler
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
signal.signal(signal.SIGTERM, __kill_zookeeper_process) # SIGTERM is handled differently for whatever reason
atexit.register(__kill_zookeeper_process)
log.info("ZooKeeper PID: %d" % __zookeeper_pid)
log.info("Waiting for ZooKeeper to start at port %d..." % port)
zookeeper_started = wait_for_tcp_port_to_open(port=port, retries=MC_ZOOKEEPER_CONNECT_RETRIES)
if not zookeeper_started:
raise McZooKeeperRunException("Unable to connect to ZooKeeper at port %d" % port)
log.info("Uploading initial Solr collection configurations to ZooKeeper...")
update_zookeeper_solr_configuration(zookeeper_host="localhost",
zookeeper_port=port,
dist_directory=dist_directory,
solr_version=solr_version)
log.info("ZooKeeper is ready on port %d!" % port)
while True:
time.sleep(1)
#!/usr/bin/env python3
import argparse
from mediawords.solr.run.constants import *
from mediawords.solr.run.solr import update_zookeeper_solr_configuration
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Update Solr's configuration on ZooKeeper.",
epilog="This script does not reload Solr shards! " +
"Run 'reload_solr_shards.py' afterwards.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-zh", "--zookeeper_host", type=str, required=False, default=MC_SOLR_CLUSTER_ZOOKEEPER_HOST,
help="ZooKeeper host to connect to.")
parser.add_argument("-zp", "--zookeeper_port", type=int, required=False, default=MC_SOLR_CLUSTER_ZOOKEEPER_PORT,
help="ZooKeeper port to connect to.")
args = parser.parse_args()
update_zookeeper_solr_configuration(zookeeper_host=args.zookeeper_host, zookeeper_port=args.zookeeper_port)
#!/usr/bin/env python3
import argparse
from mediawords.solr.run.constants import *
from mediawords.solr.run.solr import update_zookeeper_solr_configuration
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Update Solr's configuration on ZooKeeper.",
epilog="This script does not reload Solr shards! " +
"Run 'reload_solr_shards.py' afterwards.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-zh",
"--zookeeper_host",
type=str,
required=False,
default=MC_SOLR_CLUSTER_ZOOKEEPER_HOST,
help="ZooKeeper host to connect to.")
parser.add_argument("-zp",
"--zookeeper_port",
type=int,
required=False,
default=MC_SOLR_CLUSTER_ZOOKEEPER_PORT,
help="ZooKeeper port to connect to.")
args = parser.parse_args()
update_zookeeper_solr_configuration(zookeeper_host=args.zookeeper_host,
zookeeper_port=args.zookeeper_port)
