def run_zookeeper(dist_directory: str = MC_DIST_DIR, listen: str = MC_ZOOKEEPER_LISTEN, port: int = MC_ZOOKEEPER_PORT, data_dir: str = MC_SOLR_BASE_DATA_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION, solr_version: str = MC_SOLR_VERSION) -> None: """Run ZooKeeper, install if needed too.""" if not __zookeeper_is_installed(): log.info("ZooKeeper is not installed, installing...") __install_zookeeper() data_dir = resolve_absolute_path_under_mc_root(path=data_dir, must_exist=True) zookeeper_data_dir = os.path.join(data_dir, "mediacloud-cluster-zookeeper") if not os.path.isdir(zookeeper_data_dir): log.info("Creating data directory at %s..." % zookeeper_data_dir) mkdir_p(zookeeper_data_dir) if tcp_port_is_open(port=port): raise McZooKeeperRunException( "Port %d is already open on this machine." % port) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) zkserver_path = os.path.join(zookeeper_path, "bin", "zkServer.sh") if not os.path.isfile(zkserver_path): raise McZooKeeperRunException("zkServer.sh at '%s' was not found." % zkserver_path) log4j_properties_path = os.path.join(zookeeper_path, "conf", "log4j.properties") if not os.path.isfile(log4j_properties_path): raise McZooKeeperRunException( "log4j.properties at '%s' was not found.") zoo_cnf_path = os.path.join(zookeeper_data_dir, "zoo.cfg") log.info("Creating zoo.cfg in '%s'..." % zoo_cnf_path) with open(zoo_cnf_path, 'w') as zoo_cnf: zoo_cnf.write(""" # # This file is autogenerated. Please do not modify it! # clientPortAddress=%(listen)s clientPort=%(port)d dataDir=%(data_dir)s # Must be between zkClientTimeout / 2 and zkClientTimeout / 20 tickTime=30000 initLimit=10 syncLimit=10 """ % { "listen": listen, "port": port, "data_dir": zookeeper_data_dir, }) zookeeper_env = os.environ.copy() zookeeper_env[ "ZOOCFGDIR"] = zookeeper_data_dir # Serves as configuration dir too zookeeper_env["ZOOCFG"] = "zoo.cfg" zookeeper_env["ZOO_LOG_DIR"] = zookeeper_data_dir zookeeper_env[ "SERVER_JVMFLAGS"] = "-Dlog4j.configuration=file://" + os.path.abspath( log4j_properties_path) args = [zkserver_path, "start-foreground"] log.info("Starting ZooKeeper on %s:%d..." % (listen, port)) log.debug("Running command: %s" % str(args)) log.debug("Environment variables: %s" % str(zookeeper_env)) process = subprocess.Popen(args, env=zookeeper_env) global __zookeeper_pid __zookeeper_pid = process.pid # Declare that we don't care about the exit code of the child process so # it doesn't become a zombie when it gets killed in signal handler signal.signal(signal.SIGCHLD, signal.SIG_IGN) signal.signal(signal.SIGTERM, __kill_zookeeper_process ) # SIGTERM is handled differently for whatever reason atexit.register(__kill_zookeeper_process) log.info("ZooKeeper PID: %d" % __zookeeper_pid) log.info("Waiting for ZooKeeper to start at port %d..." % port) zookeeper_started = wait_for_tcp_port_to_open( port=port, retries=MC_ZOOKEEPER_CONNECT_RETRIES) if not zookeeper_started: raise McZooKeeperRunException( "Unable to connect to ZooKeeper at port %d" % port) log.info( "Uploading initial Solr collection configurations to ZooKeeper...") update_zookeeper_solr_configuration(zookeeper_host="localhost", zookeeper_port=port, dist_directory=dist_directory, solr_version=solr_version) log.info("ZooKeeper is ready on port %d!" % port) while True: time.sleep(1)
def run_zookeeper(dist_directory: str = MC_DIST_DIR, listen: str = MC_ZOOKEEPER_LISTEN, port: int = MC_ZOOKEEPER_PORT, data_dir: str = MC_SOLR_BASE_DATA_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION, solr_version: str = MC_SOLR_VERSION) -> None: """Run ZooKeeper, install if needed too.""" if not __zookeeper_is_installed(): log.info("ZooKeeper is not installed, installing...") __install_zookeeper() data_dir = resolve_absolute_path_under_mc_root(path=data_dir, must_exist=True) zookeeper_data_dir = os.path.join(data_dir, "mediacloud-cluster-zookeeper") if not os.path.isdir(zookeeper_data_dir): log.info("Creating data directory at %s..." % zookeeper_data_dir) mkdir_p(zookeeper_data_dir) if tcp_port_is_open(port=port): raise McZooKeeperRunException("Port %d is already open on this machine." % port) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) zkserver_path = os.path.join(zookeeper_path, "bin", "zkServer.sh") if not os.path.isfile(zkserver_path): raise McZooKeeperRunException("zkServer.sh at '%s' was not found." % zkserver_path) log4j_properties_path = os.path.join(zookeeper_path, "conf", "log4j.properties") if not os.path.isfile(log4j_properties_path): raise McZooKeeperRunException("log4j.properties at '%s' was not found.") zoo_cnf_path = os.path.join(zookeeper_data_dir, "zoo.cfg") log.info("Creating zoo.cfg in '%s'..." % zoo_cnf_path) with open(zoo_cnf_path, 'w') as zoo_cnf: zoo_cnf.write(""" # # This file is autogenerated. Please do not modify it! # clientPortAddress=%(listen)s clientPort=%(port)d dataDir=%(data_dir)s # Must be between zkClientTimeout / 2 and zkClientTimeout / 20 tickTime=30000 initLimit=10 syncLimit=10 """ % { "listen": listen, "port": port, "data_dir": zookeeper_data_dir, }) zookeeper_env = os.environ.copy() zookeeper_env["ZOOCFGDIR"] = zookeeper_data_dir # Serves as configuration dir too zookeeper_env["ZOOCFG"] = "zoo.cfg" zookeeper_env["ZOO_LOG_DIR"] = zookeeper_data_dir zookeeper_env["SERVER_JVMFLAGS"] = "-Dlog4j.configuration=file://" + os.path.abspath(log4j_properties_path) args = [ zkserver_path, "start-foreground" ] log.info("Starting ZooKeeper on %s:%d..." % (listen, port)) log.debug("Running command: %s" % str(args)) log.debug("Environment variables: %s" % str(zookeeper_env)) process = subprocess.Popen(args, env=zookeeper_env) global __zookeeper_pid __zookeeper_pid = process.pid # Declare that we don't care about the exit code of the child process so # it doesn't become a zombie when it gets killed in signal handler signal.signal(signal.SIGCHLD, signal.SIG_IGN) signal.signal(signal.SIGTERM, __kill_zookeeper_process) # SIGTERM is handled differently for whatever reason atexit.register(__kill_zookeeper_process) log.info("ZooKeeper PID: %d" % __zookeeper_pid) log.info("Waiting for ZooKeeper to start at port %d..." % port) zookeeper_started = wait_for_tcp_port_to_open(port=port, retries=MC_ZOOKEEPER_CONNECT_RETRIES) if not zookeeper_started: raise McZooKeeperRunException("Unable to connect to ZooKeeper at port %d" % port) log.info("Uploading initial Solr collection configurations to ZooKeeper...") update_zookeeper_solr_configuration(zookeeper_host="localhost", zookeeper_port=port, dist_directory=dist_directory, solr_version=solr_version) log.info("ZooKeeper is ready on port %d!" % port) while True: time.sleep(1)
#!/usr/bin/env python3 import argparse from mediawords.solr.run.constants import * from mediawords.solr.run.solr import update_zookeeper_solr_configuration if __name__ == "__main__": parser = argparse.ArgumentParser(description="Update Solr's configuration on ZooKeeper.", epilog="This script does not reload Solr shards! " + "Run 'reload_solr_shards.py' afterwards.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-zh", "--zookeeper_host", type=str, required=False, default=MC_SOLR_CLUSTER_ZOOKEEPER_HOST, help="ZooKeeper host to connect to.") parser.add_argument("-zp", "--zookeeper_port", type=int, required=False, default=MC_SOLR_CLUSTER_ZOOKEEPER_PORT, help="ZooKeeper port to connect to.") args = parser.parse_args() update_zookeeper_solr_configuration(zookeeper_host=args.zookeeper_host, zookeeper_port=args.zookeeper_port)
#!/usr/bin/env python3 import argparse from mediawords.solr.run.constants import * from mediawords.solr.run.solr import update_zookeeper_solr_configuration if __name__ == "__main__": parser = argparse.ArgumentParser( description="Update Solr's configuration on ZooKeeper.", epilog="This script does not reload Solr shards! " + "Run 'reload_solr_shards.py' afterwards.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-zh", "--zookeeper_host", type=str, required=False, default=MC_SOLR_CLUSTER_ZOOKEEPER_HOST, help="ZooKeeper host to connect to.") parser.add_argument("-zp", "--zookeeper_port", type=int, required=False, default=MC_SOLR_CLUSTER_ZOOKEEPER_PORT, help="ZooKeeper port to connect to.") args = parser.parse_args() update_zookeeper_solr_configuration(zookeeper_host=args.zookeeper_host, zookeeper_port=args.zookeeper_port)