def test_remote_plugin_write_config(self, mock_safely_write,
mock_run_command):
"""Test remote_memcached_plugin writes settings correctly"""
# Create a plugin with dummy parameters
plugin = RemoteMemcachedPlugin(
PluginParams(
ip='10.0.0.1',
mgmt_ip='10.0.1.1',
local_site='local_site',
remote_site='remote_site',
remote_cassandra_seeds='',
signaling_namespace='',
uuid=uuid.UUID('92a674aa-a64b-4549-b150-596fd466923f'),
etcd_key='etcd_key',
etcd_cluster_key='etcd_cluster_key'))
# Config writing is properly tested above, so run with simple cluster
cluster_view = {"10.0.0.5": "normal"}
# Call the plugin to write the settings itself
plugin.write_cluster_settings(cluster_view)
mock_safely_write.assert_called_once()
# Save off the arguments the plugin called our mock with
args = mock_safely_write.call_args
# Catch the call to reload memcached
mock_run_command.assert_called_once_with(
"/usr/share/clearwater/bin/reload_memcached_users")
# Check the plugin is attempting to write to the correct location
self.assertEqual("/etc/clearwater/remote_cluster_settings", args[0][0])
def get_plugin(self, mock_get_alarm):
# Create a plugin with dummy parameters
# Do this here to separate out the check for the alarm call
plugin = CassandraPlugin(
PluginParams(
ip='10.0.0.1',
mgmt_ip='10.0.1.1',
local_site='local_site',
remote_site='remote_site',
remote_cassandra_seeds='10.2.2.1',
signaling_namespace='',
uuid=uuid.UUID('92a674aa-a64b-4549-b150-596fd466923f'),
etcd_key='etcd_key',
etcd_cluster_key='etcd_cluster_key'))
# We expect this alarm to be called on creation of the plugin
mock_get_alarm.assert_called_once_with(
'cluster-manager', alarm_constants.CASSANDRA_NOT_YET_CLUSTERED)
return plugin
def test_write_config_all_node_states(self, mock_get_alarm,
mock_safely_write, mock_run_command):
"""Test memcached_plugin writes settings correctly when given all possible server states"""
# Create a plugin with dummy parameters
plugin = MemcachedPlugin(
PluginParams(
ip='10.0.0.1',
mgmt_ip='10.0.1.1',
local_site='local_site',
remote_site='remote_site',
remote_cassandra_seeds='',
signaling_namespace='',
uuid=uuid.UUID('92a674aa-a64b-4549-b150-596fd466923f'),
etcd_key='etcd_key',
etcd_cluster_key='etcd_cluster_key'))
# We expect this alarm to be called on creation of the plugin
mock_get_alarm.assert_called_once_with(
'cluster-manager', alarm_constants.MEMCACHED_NOT_YET_CLUSTERED)
# Build a cluster_view that includes all possible node states
cluster_view = {
"10.0.0.1": "waiting to join",
"10.0.0.2": "joining",
"10.0.0.3": "joining, acknowledged change",
"10.0.0.4": "joining, config changed",
"10.0.0.5": "normal",
"10.0.0.6": "normal, acknowledged change",
"10.0.0.7": "normal, config changed",
"10.0.0.8": "waiting to leave",
"10.0.0.9": "leaving",
"10.0.0.10": "leaving, acknowledged change",
"10.0.0.11": "leaving, config changed",
"10.0.0.12": "finished",
"10.0.0.13": "error"
}
# Call the plugin to write the settings itself
plugin.write_cluster_settings(cluster_view)
mock_safely_write.assert_called_once()
# Save off the arguments the plugin called our mock with
args = mock_safely_write.call_args
# Catch the call to reload memcached
mock_run_command.assert_called_once_with(
"/usr/share/clearwater/bin/reload_memcached_users")
# Check the plugin is attempting to write to the correct location
self.assertEqual("/etc/clearwater/cluster_settings", args[0][0])
# Save off the file contents sent to the mock safely_write call
# The file is not a proper config file structure, so we do string
# based parsing, rather than using python ConfigParser
config_string = args[0][1]
config_lines = config_string.splitlines()
# Assert there is only one 'servers' line, and parse out the ips.
server_list = [s for s in config_lines if s.startswith('servers')]
self.assertTrue(len(server_list) == 1)
server_ips_with_ports = [
s for s in (str(server_list[0]).strip('servers=')).split(',')
]
server_ips = Counter(
[ip.split(':')[0] for ip in server_ips_with_ports])
# Assert there is only one 'new_servers' line, and parse out the ips.
new_server_list = [
s for s in config_lines if s.startswith('new_servers')
]
self.assertTrue(len(new_server_list) == 1)
new_server_ips_with_ports = [
s
for s in (str(new_server_list[0]).strip('new_servers=')).split(',')
]
new_server_ips = Counter(
[ip.split(':')[0] for ip in new_server_ips_with_ports])
# Set expectations, and assert that the correct ips made it into each list
expected_server_ips = Counter(
['10.0.0.5', '10.0.0.6', '10.0.0.7', '10.0.0.10', '10.0.0.11'])
expected_new_server_ips = Counter(
['10.0.0.3', '10.0.0.4', '10.0.0.5', '10.0.0.6', '10.0.0.7'])
self.assertTrue(server_ips == expected_server_ips)
self.assertTrue(new_server_ips == expected_new_server_ips)
def main(args):
syslog.openlog("cluster-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
mgmt_ip = arguments['--mgmt-local-ip']
sig_ip = arguments['--sig-local-ip']
local_site_name = arguments['--local-site']
remote_site_name = arguments['--remote-site']
remote_cassandra_seeds = arguments['--remote-cassandra-seeds']
if remote_cassandra_seeds:
remote_cassandra_seeds = remote_cassandra_seeds.split(',')
else:
remote_cassandra_seeds = []
signaling_namespace = arguments.get('--signaling-namespace')
local_uuid = UUID(arguments['--uuid'])
etcd_key = arguments.get('--etcd-key')
etcd_cluster_key = arguments.get('--etcd-cluster-key')
cluster_manager_enabled = arguments['--cluster-manager-enabled']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
stdout_err_log = os.path.join(log_dir, "cluster-manager.output.log")
# Check that there's an etcd_cluster_key value passed to the cluster
# manager
if etcd_cluster_key == "":
# The etcd_cluster_key isn't valid, and possibly get weird entries in
# the etcd database if we allow the cluster_manager to start
pdlogs.EXITING_MISSING_ETCD_CLUSTER_KEY.log()
exit(1)
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-cluster-mgr")
logging_config.configure_logging(log_level,
log_dir,
"cluster-manager",
show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("cluster-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(
arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-cluster-manager/plugins/"
plugins = load_plugins_in_dir(
plugins_dir,
PluginParams(ip=sig_ip,
mgmt_ip=mgmt_ip,
local_site=local_site_name,
remote_site=remote_site_name,
remote_cassandra_seeds=remote_cassandra_seeds,
signaling_namespace=signaling_namespace,
uuid=local_uuid,
etcd_key=etcd_key,
etcd_cluster_key=etcd_cluster_key))
plugins.sort(key=lambda x: x.key())
plugins_to_use = []
files = []
skip = False
for plugin in plugins:
for plugin_file in plugin.files():
if plugin_file in files:
_log.info("Skipping plugin {} because {} "
"is already managed by another plugin".format(
plugin, plugin_file))
skip = True
if not skip:
plugins_to_use.append(plugin)
files.extend(plugin.files())
synchronizers = []
threads = []
if cluster_manager_enabled == "N":
# Don't start any threads as we don't want the cluster manager to run
pdlogs.DO_NOT_START.log()
elif etcd_cluster_key == "DO_NOT_CLUSTER":
# Don't start any threads as we don't want this box to cluster
pdlogs.DO_NOT_CLUSTER.log()
else:
for plugin in plugins_to_use:
syncer = EtcdSynchronizer(plugin, sig_ip, etcd_ip=mgmt_ip)
syncer.start_thread()
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
install_sigquit_handler(synchronizers)
install_sigterm_handler(synchronizers)
while any([thread.isAlive() for thread in threads]):
for thread in threads:
if thread.isAlive():
thread.join(1)
_log.info("No plugin threads running, waiting for a SIGTERM or SIGQUIT")
while not should_quit:
sleep(1)
_log.info("Quitting")
_log.debug("%d threads outstanding at exit" % activeCount())
pdlogs.EXITING.log()
syslog.closelog()
def test_write_config(self, mock_get_alarm, mock_safely_write,
mock_run_command):
"""Test chronos_plugin writes settings correctly with all possible server states"""
# Create a plugin with dummy parameters
plugin = ChronosPlugin(
PluginParams(
ip='10.0.0.1',
mgmt_ip='10.0.1.1',
local_site='local_site',
remote_site='remote_site',
remote_cassandra_seeds='',
signaling_namespace='',
uuid=uuid.UUID('92a674aa-a64b-4549-b150-596fd466923f'),
etcd_key='etcd_key',
etcd_cluster_key='etcd_cluster_key'))
# We expect this alarm to be called on creation of the plugin
mock_get_alarm.assert_called_once_with(
'cluster-manager', alarm_constants.CHRONOS_NOT_YET_CLUSTERED)
# Build a cluster_view that includes all possible node states
cluster_view = {
"10.0.0.1": "waiting to join",
"10.0.0.2": "joining",
"10.0.0.3": "joining, acknowledged change",
"10.0.0.4": "joining, config changed",
"10.0.0.5": "normal",
"10.0.0.6": "normal, acknowledged change",
"10.0.0.7": "normal, config changed",
"10.0.0.8": "waiting to leave",
"10.0.0.9": "leaving",
"10.0.0.10": "leaving, acknowledged change",
"10.0.0.11": "leaving, config changed",
"10.0.0.12": "finished",
"10.0.0.13": "error"
}
# Call the plugin to write the settings itself
plugin.write_cluster_settings(cluster_view)
mock_safely_write.assert_called_once()
# Save off the arguments the plugin called our mock with
args = mock_safely_write.call_args
# Catch the call to reload chronos
mock_run_command.assert_called_once_with(
['service', 'chronos', 'reload'])
# Check the plugin is attempting to write to the correct location
self.assertEqual("/etc/chronos/chronos_cluster.conf", args[0][0])
# ConfigParser can't parse plain strings in python 2.7
# Load the config into a buffer and pass it in as a string like object
buf = StringIO(args[0][1])
config = RawConfigParser(dict_type=MultiOrderedDict)
config.readfp(buf)
# Check identity section
self.assertEqual(config.get('identity', 'instance_id'), '18')
self.assertEqual(config.get('identity', 'deployment_id'), '6')
# Check cluster section
self.assertEqual(config.get('cluster', 'localhost'), '10.0.0.1')
self.assertTrue(
all(ip in config.get('cluster', 'joining')
for ip in ("10.0.0.3", "10.0.0.4")))
self.assertTrue(
all(ip in config.get('cluster', 'node')
for ip in ("10.0.0.5", "10.0.0.6", "10.0.0.7")))
self.assertTrue(
all(ip in config.get('cluster', 'leaving')
for ip in ("10.0.0.10", "10.0.0.11")))