def test_load(self):
plugin_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"plugins")
plugins = load_plugins_in_dir(plugin_path, None)
# Check that the plugin loaded successfully
self.assertEqual(plugins[0].__class__.__name__,
"PluginLoaderTestPlugin")
def test_load(self):
plugin_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"plugins")
plugins = load_plugins_in_dir(plugin_path, None)
# Check that the plugin loaded successfully
self.assertEqual(plugins[0].__class__.__name__,
"PluginLoaderTestPlugin")
def main(args):
syslog.openlog("config-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
local_ip = arguments['--local-ip']
local_site = arguments['--local-site']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
stdout_err_log = os.path.join(log_dir, "config-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
logging_config.configure_logging(log_level, log_dir, "config-manager", show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("config-manager")
# Drop a pidfile.
pid = os.getpid()
with open(arguments['--pidfile'], "w") as pidfile:
pidfile.write(str(pid) + "\n")
plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir)
plugins.sort(key=lambda x: x.key())
threads = []
files = [p.file() for p in plugins]
alarm = ConfigAlarm(files)
for plugin in plugins:
syncer = EtcdSynchronizer(plugin, local_ip, local_site, alarm)
thread = Thread(target=syncer.main, name=plugin.__class__.__name__)
thread.start()
threads.append(thread)
_log.info("Loaded plugin %s" % plugin)
while any([thread.isAlive() for thread in threads]):
for thread in threads:
if thread.isAlive():
thread.join(1)
_log.info("Clearwater Configuration Manager shutting down")
pdlogs.EXITING.log()
syslog.closelog()
# as those licenses appear in the file LICENSE-OPENSSL.
from metaswitch.clearwater.etcd_shared.plugin_loader import load_plugins_in_dir
from metaswitch.clearwater.config_manager.plugin_base import FileStatus
import etcd
import os
import sys
etcd_ip = sys.argv[1]
site = sys.argv[2]
etcd_key = sys.argv[3]
client = etcd.Client(etcd_ip, 4000)
plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir)
rc = 0
for plugin in plugins:
try:
result = client.get("/" + etcd_key + "/" + site + "/configuration/" +
plugin.key())
value = result.value
except etcd.EtcdKeyNotFound:
value = ""
state = plugin.status(value)
if state == FileStatus.UP_TO_DATE:
print " - {} is up to date".format(plugin.file())
# Metaswitch Networks in a separate written agreement.
from metaswitch.clearwater.etcd_shared.plugin_loader import load_plugins_in_dir
from metaswitch.clearwater.config_manager.plugin_base import FileStatus
import etcd
import os
import sys
etcd_ip = sys.argv[1]
site = sys.argv[2]
etcd_key = sys.argv[3]
client = etcd.Client(etcd_ip, 4000)
plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir)
rc = 0
for plugin in plugins:
try:
result = client.get("/" + etcd_key + "/" + site + "/configuration/" + plugin.key())
value = result.value
except etcd.EtcdKeyNotFound:
value = ""
state = plugin.status(value)
if state == FileStatus.UP_TO_DATE:
print " - {} is up to date".format(plugin.file())
elif state == FileStatus.OUT_OF_SYNC:
def main(args):
syslog.openlog("cluster-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
mgmt_ip = arguments['--mgmt-local-ip']
sig_ip = arguments['--sig-local-ip']
local_site_name = arguments['--local-site']
remote_site_name = arguments['--remote-site']
remote_cassandra_seeds = arguments['--remote-cassandra-seeds']
if remote_cassandra_seeds:
remote_cassandra_seeds = remote_cassandra_seeds.split(',')
else:
remote_cassandra_seeds = []
signaling_namespace = arguments.get('--signaling-namespace')
local_uuid = UUID(arguments['--uuid'])
etcd_key = arguments.get('--etcd-key')
etcd_cluster_key = arguments.get('--etcd-cluster-key')
cluster_manager_enabled = arguments['--cluster-manager-enabled']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log")
# Check that there's an etcd_cluster_key value passed to the cluster
# manager
if etcd_cluster_key == "":
# The etcd_cluster_key isn't valid, and possibly get weird entries in
# the etcd database if we allow the cluster_manager to start
pdlogs.EXITING_MISSING_ETCD_CLUSTER_KEY.log()
exit(1)
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-cluster-mgr")
logging_config.configure_logging(log_level, log_dir, "cluster-manager", show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("cluster-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir,
PluginParams(ip=sig_ip,
mgmt_ip=mgmt_ip,
local_site=local_site_name,
remote_site=remote_site_name,
remote_cassandra_seeds=remote_cassandra_seeds,
signaling_namespace=signaling_namespace,
uuid=local_uuid,
etcd_key=etcd_key,
etcd_cluster_key=etcd_cluster_key))
plugins.sort(key=lambda x: x.key())
plugins_to_use = []
files = []
skip = False
for plugin in plugins:
for plugin_file in plugin.files():
if plugin_file in files:
_log.info("Skipping plugin {} because {} "
"is already managed by another plugin"
.format(plugin, plugin_file))
skip = True
if not skip:
plugins_to_use.append(plugin)
files.extend(plugin.files())
synchronizers = []
threads = []
if cluster_manager_enabled == "N":
# Don't start any threads as we don't want the cluster manager to run
pdlogs.DO_NOT_START.log()
elif etcd_cluster_key == "DO_NOT_CLUSTER":
# Don't start any threads as we don't want this box to cluster
pdlogs.DO_NOT_CLUSTER.log()
else:
for plugin in plugins_to_use:
syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip)
syncer.start_thread()
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
install_sigquit_handler(synchronizers)
utils.install_sigterm_handler(synchronizers)
while any([thread.isAlive() for thread in threads]):
for thread in threads:
if thread.isAlive():
thread.join(1)
_log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT")
while not utils.should_quit and not should_quit:
sleep(1)
_log.info("Quitting")
_log.debug("%d threads outstanding at exit" % activeCount())
pdlogs.EXITING.log()
syslog.closelog()
def main(args):
syslog.openlog("queue-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
local_ip = arguments['--local-ip']
local_site = arguments['--local-site']
etcd_key = arguments['--etcd-key']
node_type = arguments['--node-type']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
wait_plugin_complete = arguments['--wait-plugin-complete']
stdout_err_log = os.path.join(log_dir, "queue-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-queue-mgr")
logging_config.configure_logging(log_level, log_dir, "queue-manager", show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("queue-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir,
PluginParams(wait_plugin_complete=wait_plugin_complete))
plugins.sort(key=lambda x: x.key())
synchronizers = []
threads = []
for plugin in plugins:
syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key, node_type)
syncer.start_thread()
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
utils.install_sigterm_handler(synchronizers)
while any([thr.isAlive() for thr in threads]):
for thr in threads:
if thr.isAlive():
thr.join(1)
while not utils.should_quit:
sleep(1)
_log.info("Clearwater Queue Manager shutting down")
pdlogs.EXITING.log()
syslog.closelog()
def main(args):
syslog.openlog("cluster-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
mgmt_ip = arguments['--mgmt-local-ip']
sig_ip = arguments['--sig-local-ip']
local_site_name = arguments['--local-site']
remote_site_name = arguments['--remote-site']
remote_cassandra_seeds = arguments['--remote-cassandra-seeds']
if remote_cassandra_seeds:
remote_cassandra_seeds = remote_cassandra_seeds.split(',')
else:
remote_cassandra_seeds = []
signaling_namespace = arguments.get('--signaling-namespace')
local_uuid = UUID(arguments['--uuid'])
etcd_key = arguments.get('--etcd-key')
etcd_cluster_key = arguments.get('--etcd-cluster-key')
cluster_manager_enabled = arguments['--cluster-manager-enabled']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log")
# Check that there's an etcd_cluster_key value passed to the cluster
# manager
if etcd_cluster_key == "":
# The etcd_cluster_key isn't valid, and possibly get weird entries in
# the etcd database if we allow the cluster_manager to start
pdlogs.EXITING_MISSING_ETCD_CLUSTER_KEY.log()
exit(1)
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-cluster-mgr")
logging_config.configure_logging(log_level,
log_dir,
"cluster-manager",
show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("cluster-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(
arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/"
plugins = load_plugins_in_dir(
plugins_dir,
PluginParams(ip=sig_ip,
mgmt_ip=mgmt_ip,
local_site=local_site_name,
remote_site=remote_site_name,
remote_cassandra_seeds=remote_cassandra_seeds,
signaling_namespace=signaling_namespace,
uuid=local_uuid,
etcd_key=etcd_key,
etcd_cluster_key=etcd_cluster_key))
plugins.sort(key=lambda x: x.key())
plugins_to_use = []
files = []
skip = False
for plugin in plugins:
for plugin_file in plugin.files():
if plugin_file in files:
_log.info("Skipping plugin {} because {} "
"is already managed by another plugin".format(
plugin, plugin_file))
skip = True
if not skip:
plugins_to_use.append(plugin)
files.extend(plugin.files())
synchronizers = []
threads = []
if cluster_manager_enabled == "N":
# Don't start any threads as we don't want the cluster manager to run
pdlogs.DO_NOT_START.log()
elif etcd_cluster_key == "DO_NOT_CLUSTER":
# Don't start any threads as we don't want this box to cluster
pdlogs.DO_NOT_CLUSTER.log()
else:
for plugin in plugins_to_use:
syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip)
syncer.start_thread()
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
install_sigquit_handler(synchronizers)
install_sigterm_handler(synchronizers)
while any([thread.isAlive() for thread in threads]):
for thread in threads:
if thread.isAlive():
thread.join(1)
_log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT")
while not should_quit:
sleep(1)
_log.info("Quitting")
_log.debug("%d threads outstanding at exit" % activeCount())
pdlogs.EXITING.log()
syslog.closelog()
def main(args):
syslog.openlog("cluster-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
mgmt_ip = arguments['--mgmt-local-ip']
sig_ip = arguments['--sig-local-ip']
local_site_name = arguments['--local-site']
remote_site_name = arguments['--remote-site']
signaling_namespace = arguments.get('--signaling-namespace')
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
logging_config.configure_logging(log_level, log_dir, "cluster-manager", show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("cluster-manager")
# Drop a pidfile.
pid = os.getpid()
with open(arguments['--pidfile'], "w") as pidfile:
pidfile.write(str(pid) + "\n")
plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir,
PluginParams(ip=sig_ip,
mgmt_ip=mgmt_ip,
local_site=local_site_name,
remote_site=remote_site_name,
signaling_namespace=signaling_namespace))
plugins.sort(key=lambda x: x.key())
plugins_to_use = []
files = []
skip = False
for plugin in plugins:
for plugin_file in plugin.files():
if plugin_file in files:
_log.info("Skipping plugin {} because {} "
"is already managed by another plugin"
.format(plugin, plugin_file))
skip = True
if not skip:
plugins_to_use.append(plugin)
files.extend(plugin.files())
synchronizers = []
threads = []
for plugin in plugins_to_use:
syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip)
syncer.start_thread()
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
install_sigquit_handler(synchronizers)
install_sigterm_handler(synchronizers)
while any([thread.isAlive() for thread in threads]):
for thread in threads:
if thread.isAlive():
thread.join(1)
_log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT")
while not should_quit:
sleep(1)
_log.info("Quitting")
_log.debug("%d threads outstanding at exit" % activeCount())
pdlogs.EXITING.log()
syslog.closelog()
# Use os.exit to skip exit handlers - otherwise the concurrent.futures exit
# handler waits for an infinite wait to end
os._exit(0)
def main(args):
syslog.openlog("queue-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
local_ip = arguments['--local-ip']
local_site = arguments['--local-site']
etcd_key = arguments['--etcd-key']
node_type = arguments['--node-type']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
wait_plugin_complete = arguments['--wait-plugin-complete']
stdout_err_log = os.path.join(log_dir, "queue-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-queue-mgr")
logging_config.configure_logging(log_level,
log_dir,
"queue-manager",
show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("queue-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(
arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/"
plugins = load_plugins_in_dir(
plugins_dir, PluginParams(wait_plugin_complete=wait_plugin_complete))
plugins.sort(key=lambda x: x.key())
synchronizers = []
threads = []
# Load the plugins, but don't start them until we've installed the SIGTERM
# handler, as that handler will gracefully shut down any running
# synchronizers on receiving a SIGTERM
for plugin in plugins:
syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key,
node_type)
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
utils.install_sigterm_handler(synchronizers)
# Now start the plugin threads
for syncer in synchronizers:
syncer.start_thread()
_log.info("Started thread for plugin %s" % syncer._plugin)
while any([thr.isAlive() for thr in threads]):
for thr in threads:
if thr.isAlive():
thr.join(1)
while not utils.should_quit:
sleep(1)
_log.info("Clearwater Queue Manager shutting down")
pdlogs.EXITING.log()
syslog.closelog()
def main(args):
syslog.openlog("config-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
local_ip = arguments['--local-ip']
local_site = arguments['--local-site']
etcd_key = arguments['--etcd-key']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
stdout_err_log = os.path.join(log_dir, "config-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-config-mgr")
logging_config.configure_logging(log_level, log_dir, "config-manager", show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("config-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-config-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir)
plugins.sort(key=lambda x: x.key())
threads = []
files = [p.file() for p in plugins]
alarm = ConfigAlarm(files)
for plugin in plugins:
syncer = EtcdSynchronizer(plugin, local_ip, local_site, alarm, etcd_key)
syncer.start_thread()
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
while any([thr.isAlive() for thr in threads]):
for thr in threads:
if thr.isAlive():
thr.join(1)
_log.info("Clearwater Configuration Manager shutting down")
pdlogs.EXITING.log()
syslog.closelog()
