def test_load(self): plugin_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "plugins") plugins = load_plugins_in_dir(plugin_path, None) # Check that the plugin loaded successfully self.assertEqual(plugins[0].__class__.__name__, "PluginLoaderTestPlugin")
def main(args): syslog.openlog("config-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise local_ip = arguments['--local-ip'] local_site = arguments['--local-site'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) stdout_err_log = os.path.join(log_dir, "config-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) logging_config.configure_logging(log_level, log_dir, "config-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("config-manager") # Drop a pidfile. pid = os.getpid() with open(arguments['--pidfile'], "w") as pidfile: pidfile.write(str(pid) + "\n") plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir) plugins.sort(key=lambda x: x.key()) threads = [] files = [p.file() for p in plugins] alarm = ConfigAlarm(files) for plugin in plugins: syncer = EtcdSynchronizer(plugin, local_ip, local_site, alarm) thread = Thread(target=syncer.main, name=plugin.__class__.__name__) thread.start() threads.append(thread) _log.info("Loaded plugin %s" % plugin) while any([thread.isAlive() for thread in threads]): for thread in threads: if thread.isAlive(): thread.join(1) _log.info("Clearwater Configuration Manager shutting down") pdlogs.EXITING.log() syslog.closelog()
# as those licenses appear in the file LICENSE-OPENSSL. from metaswitch.clearwater.etcd_shared.plugin_loader import load_plugins_in_dir from metaswitch.clearwater.config_manager.plugin_base import FileStatus import etcd import os import sys etcd_ip = sys.argv[1] site = sys.argv[2] etcd_key = sys.argv[3] client = etcd.Client(etcd_ip, 4000) plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir) rc = 0 for plugin in plugins: try: result = client.get("/" + etcd_key + "/" + site + "/configuration/" + plugin.key()) value = result.value except etcd.EtcdKeyNotFound: value = "" state = plugin.status(value) if state == FileStatus.UP_TO_DATE: print " - {} is up to date".format(plugin.file())
# Metaswitch Networks in a separate written agreement. from metaswitch.clearwater.etcd_shared.plugin_loader import load_plugins_in_dir from metaswitch.clearwater.config_manager.plugin_base import FileStatus import etcd import os import sys etcd_ip = sys.argv[1] site = sys.argv[2] etcd_key = sys.argv[3] client = etcd.Client(etcd_ip, 4000) plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir) rc = 0 for plugin in plugins: try: result = client.get("/" + etcd_key + "/" + site + "/configuration/" + plugin.key()) value = result.value except etcd.EtcdKeyNotFound: value = "" state = plugin.status(value) if state == FileStatus.UP_TO_DATE: print " - {} is up to date".format(plugin.file()) elif state == FileStatus.OUT_OF_SYNC:
def main(args): syslog.openlog("cluster-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise mgmt_ip = arguments['--mgmt-local-ip'] sig_ip = arguments['--sig-local-ip'] local_site_name = arguments['--local-site'] remote_site_name = arguments['--remote-site'] remote_cassandra_seeds = arguments['--remote-cassandra-seeds'] if remote_cassandra_seeds: remote_cassandra_seeds = remote_cassandra_seeds.split(',') else: remote_cassandra_seeds = [] signaling_namespace = arguments.get('--signaling-namespace') local_uuid = UUID(arguments['--uuid']) etcd_key = arguments.get('--etcd-key') etcd_cluster_key = arguments.get('--etcd-cluster-key') cluster_manager_enabled = arguments['--cluster-manager-enabled'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log") # Check that there's an etcd_cluster_key value passed to the cluster # manager if etcd_cluster_key == "": # The etcd_cluster_key isn't valid, and possibly get weird entries in # the etcd database if we allow the cluster_manager to start pdlogs.EXITING_MISSING_ETCD_CLUSTER_KEY.log() exit(1) if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-cluster-mgr") logging_config.configure_logging(log_level, log_dir, "cluster-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("cluster-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir, PluginParams(ip=sig_ip, mgmt_ip=mgmt_ip, local_site=local_site_name, remote_site=remote_site_name, remote_cassandra_seeds=remote_cassandra_seeds, signaling_namespace=signaling_namespace, uuid=local_uuid, etcd_key=etcd_key, etcd_cluster_key=etcd_cluster_key)) plugins.sort(key=lambda x: x.key()) plugins_to_use = [] files = [] skip = False for plugin in plugins: for plugin_file in plugin.files(): if plugin_file in files: _log.info("Skipping plugin {} because {} " "is already managed by another plugin" .format(plugin, plugin_file)) skip = True if not skip: plugins_to_use.append(plugin) files.extend(plugin.files()) synchronizers = [] threads = [] if cluster_manager_enabled == "N": # Don't start any threads as we don't want the cluster manager to run pdlogs.DO_NOT_START.log() elif etcd_cluster_key == "DO_NOT_CLUSTER": # Don't start any threads as we don't want this box to cluster pdlogs.DO_NOT_CLUSTER.log() else: for plugin in plugins_to_use: syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip) syncer.start_thread() synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) install_sigquit_handler(synchronizers) utils.install_sigterm_handler(synchronizers) while any([thread.isAlive() for thread in threads]): for thread in threads: if thread.isAlive(): thread.join(1) _log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT") while not utils.should_quit and not should_quit: sleep(1) _log.info("Quitting") _log.debug("%d threads outstanding at exit" % activeCount()) pdlogs.EXITING.log() syslog.closelog()
def main(args): syslog.openlog("queue-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise local_ip = arguments['--local-ip'] local_site = arguments['--local-site'] etcd_key = arguments['--etcd-key'] node_type = arguments['--node-type'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) wait_plugin_complete = arguments['--wait-plugin-complete'] stdout_err_log = os.path.join(log_dir, "queue-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-queue-mgr") logging_config.configure_logging(log_level, log_dir, "queue-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("queue-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir, PluginParams(wait_plugin_complete=wait_plugin_complete)) plugins.sort(key=lambda x: x.key()) synchronizers = [] threads = [] for plugin in plugins: syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key, node_type) syncer.start_thread() synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) utils.install_sigterm_handler(synchronizers) while any([thr.isAlive() for thr in threads]): for thr in threads: if thr.isAlive(): thr.join(1) while not utils.should_quit: sleep(1) _log.info("Clearwater Queue Manager shutting down") pdlogs.EXITING.log() syslog.closelog()
def main(args): syslog.openlog("cluster-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise mgmt_ip = arguments['--mgmt-local-ip'] sig_ip = arguments['--sig-local-ip'] local_site_name = arguments['--local-site'] remote_site_name = arguments['--remote-site'] remote_cassandra_seeds = arguments['--remote-cassandra-seeds'] if remote_cassandra_seeds: remote_cassandra_seeds = remote_cassandra_seeds.split(',') else: remote_cassandra_seeds = [] signaling_namespace = arguments.get('--signaling-namespace') local_uuid = UUID(arguments['--uuid']) etcd_key = arguments.get('--etcd-key') etcd_cluster_key = arguments.get('--etcd-cluster-key') cluster_manager_enabled = arguments['--cluster-manager-enabled'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log") # Check that there's an etcd_cluster_key value passed to the cluster # manager if etcd_cluster_key == "": # The etcd_cluster_key isn't valid, and possibly get weird entries in # the etcd database if we allow the cluster_manager to start pdlogs.EXITING_MISSING_ETCD_CLUSTER_KEY.log() exit(1) if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-cluster-mgr") logging_config.configure_logging(log_level, log_dir, "cluster-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("cluster-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file( arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/" plugins = load_plugins_in_dir( plugins_dir, PluginParams(ip=sig_ip, mgmt_ip=mgmt_ip, local_site=local_site_name, remote_site=remote_site_name, remote_cassandra_seeds=remote_cassandra_seeds, signaling_namespace=signaling_namespace, uuid=local_uuid, etcd_key=etcd_key, etcd_cluster_key=etcd_cluster_key)) plugins.sort(key=lambda x: x.key()) plugins_to_use = [] files = [] skip = False for plugin in plugins: for plugin_file in plugin.files(): if plugin_file in files: _log.info("Skipping plugin {} because {} " "is already managed by another plugin".format( plugin, plugin_file)) skip = True if not skip: plugins_to_use.append(plugin) files.extend(plugin.files()) synchronizers = [] threads = [] if cluster_manager_enabled == "N": # Don't start any threads as we don't want the cluster manager to run pdlogs.DO_NOT_START.log() elif etcd_cluster_key == "DO_NOT_CLUSTER": # Don't start any threads as we don't want this box to cluster pdlogs.DO_NOT_CLUSTER.log() else: for plugin in plugins_to_use: syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip) syncer.start_thread() synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) install_sigquit_handler(synchronizers) install_sigterm_handler(synchronizers) while any([thread.isAlive() for thread in threads]): for thread in threads: if thread.isAlive(): thread.join(1) _log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT") while not should_quit: sleep(1) _log.info("Quitting") _log.debug("%d threads outstanding at exit" % activeCount()) pdlogs.EXITING.log() syslog.closelog()
def main(args): syslog.openlog("cluster-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise mgmt_ip = arguments['--mgmt-local-ip'] sig_ip = arguments['--sig-local-ip'] local_site_name = arguments['--local-site'] remote_site_name = arguments['--remote-site'] signaling_namespace = arguments.get('--signaling-namespace') log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) logging_config.configure_logging(log_level, log_dir, "cluster-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("cluster-manager") # Drop a pidfile. pid = os.getpid() with open(arguments['--pidfile'], "w") as pidfile: pidfile.write(str(pid) + "\n") plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir, PluginParams(ip=sig_ip, mgmt_ip=mgmt_ip, local_site=local_site_name, remote_site=remote_site_name, signaling_namespace=signaling_namespace)) plugins.sort(key=lambda x: x.key()) plugins_to_use = [] files = [] skip = False for plugin in plugins: for plugin_file in plugin.files(): if plugin_file in files: _log.info("Skipping plugin {} because {} " "is already managed by another plugin" .format(plugin, plugin_file)) skip = True if not skip: plugins_to_use.append(plugin) files.extend(plugin.files()) synchronizers = [] threads = [] for plugin in plugins_to_use: syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip) syncer.start_thread() synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) install_sigquit_handler(synchronizers) install_sigterm_handler(synchronizers) while any([thread.isAlive() for thread in threads]): for thread in threads: if thread.isAlive(): thread.join(1) _log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT") while not should_quit: sleep(1) _log.info("Quitting") _log.debug("%d threads outstanding at exit" % activeCount()) pdlogs.EXITING.log() syslog.closelog() # Use os.exit to skip exit handlers - otherwise the concurrent.futures exit # handler waits for an infinite wait to end os._exit(0)
def main(args): syslog.openlog("queue-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise local_ip = arguments['--local-ip'] local_site = arguments['--local-site'] etcd_key = arguments['--etcd-key'] node_type = arguments['--node-type'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) wait_plugin_complete = arguments['--wait-plugin-complete'] stdout_err_log = os.path.join(log_dir, "queue-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-queue-mgr") logging_config.configure_logging(log_level, log_dir, "queue-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("queue-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file( arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/" plugins = load_plugins_in_dir( plugins_dir, PluginParams(wait_plugin_complete=wait_plugin_complete)) plugins.sort(key=lambda x: x.key()) synchronizers = [] threads = [] # Load the plugins, but don't start them until we've installed the SIGTERM # handler, as that handler will gracefully shut down any running # synchronizers on receiving a SIGTERM for plugin in plugins: syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key, node_type) synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) utils.install_sigterm_handler(synchronizers) # Now start the plugin threads for syncer in synchronizers: syncer.start_thread() _log.info("Started thread for plugin %s" % syncer._plugin) while any([thr.isAlive() for thr in threads]): for thr in threads: if thr.isAlive(): thr.join(1) while not utils.should_quit: sleep(1) _log.info("Clearwater Queue Manager shutting down") pdlogs.EXITING.log() syslog.closelog()
def main(args): syslog.openlog("config-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise local_ip = arguments['--local-ip'] local_site = arguments['--local-site'] etcd_key = arguments['--etcd-key'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) stdout_err_log = os.path.join(log_dir, "config-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-config-mgr") logging_config.configure_logging(log_level, log_dir, "config-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("config-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir) plugins.sort(key=lambda x: x.key()) threads = [] files = [p.file() for p in plugins] alarm = ConfigAlarm(files) for plugin in plugins: syncer = EtcdSynchronizer(plugin, local_ip, local_site, alarm, etcd_key) syncer.start_thread() threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) while any([thr.isAlive() for thr in threads]): for thr in threads: if thr.isAlive(): thr.join(1) _log.info("Clearwater Configuration Manager shutting down") pdlogs.EXITING.log() syslog.closelog()