class WaitActionTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
@patch.object(amulet.actions, "get_action_output")
def test_returns_true_if_completed(self, get_action_output):
"""JSON output is parsed and returns True if the action completed."""
get_action_output.return_value = {"status": "completed"}
self.assertTrue(self.utils.wait_on_action("action-id"))
get_action_output.assert_called_with("action-id", full_output=True)
@patch.object(amulet.actions, "get_action_output")
def test_returns_false_if_still_running(self, get_action_output):
"""
JSON output is parsed and returns False if the action is still running.
"""
get_action_output.return_value = {"status": "running"}
self.assertFalse(self.utils.wait_on_action("action-id"))
get_action_output.assert_called_with("action-id", full_output=True)
@patch.object(amulet.actions, "get_action_output")
def test_returns_false_if_no_status(self, get_action_output):
"""
JSON output is parsed and returns False if there is no action status.
"""
get_action_output.return_value = {}
self.assertFalse(self.utils.wait_on_action("action-id"))
get_action_output.assert_called_with("action-id", full_output=True)
def ubuntu_basic_deployment(series):
""" Common test routines to run per-series. """
# Initialize
seconds = 900
u = AmuletUtils(logging.DEBUG)
d = amulet.Deployment(series=series)
d.add('ubuntu')
# Deploy services, wait for started state. Fail or skip on timeout.
try:
d.setup(timeout=seconds)
sentry_unit = d.sentry.unit['ubuntu/0']
except amulet.helpers.TimeoutError:
message = 'Deployment timed out ({}s)'.format(seconds)
amulet.raise_status(amulet.FAIL, msg=message)
except:
raise
# Confirm Ubuntu release name from the unit.
release, ret = u.get_ubuntu_release_from_sentry(sentry_unit)
if ret:
# Something went wrong trying to query the unit, or it is an
# unknown/alien release name based on distro-info validation.
amulet.raise_status(amulet.FAIL, msg=ret)
if release == series:
u.log.info('Release/series check: OK')
else:
msg = 'Release/series check: FAIL ({} != {})'.format(release, series)
u.log.error(msg)
amulet.raise_status(amulet.FAIL, msg=msg)
class StatusGetTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_status_get(self):
"""
We can get the status of a unit.
"""
self.sentry_unit.commands[
"status-get --format=json --include-data"] = (
"""{"status": "active", "message": "foo"}""", 0)
self.assertEqual(self.utils.status_get(self.sentry_unit),
(u"active", u"foo"))
def test_status_get_missing_command(self):
"""
Older releases of Juju have no status-get command. In those
cases we should return the "unknown" status.
"""
self.sentry_unit.commands[
"status-get --format=json --include-data"] = (
"status-get: command not found", 127)
self.assertEqual(self.utils.status_get(self.sentry_unit),
(u"unknown", u""))
class GetUnitProcessIdsTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_returns_map(self):
"""
Normal execution returns a dictionary mapping process names to
PIDs for each unit.
"""
second_sentry = FakeSentry(name="bar")
self.sentry_unit.commands['pidof -x "foo"'] = ("123 124", 0)
second_sentry.commands['pidof -x "bar"'] = ("456 457", 0)
result = self.utils.get_unit_process_ids({
self.sentry_unit: ["foo"],
second_sentry: ["bar"]
})
self.assertEqual(
{
self.sentry_unit: {
"foo": ["123", "124"]
},
second_sentry: {
"bar": ["456", "457"]
}
}, result)
def test_expect_failure(self):
"""
Expected failures return empty lists.
"""
second_sentry = FakeSentry(name="bar")
self.sentry_unit.commands['pidof -x "foo" || exit 0 && exit 1'] = ("",
0)
second_sentry.commands['pidof -x "bar" || exit 0 && exit 1'] = ("", 0)
result = self.utils.get_unit_process_ids(
{
self.sentry_unit: ["foo"],
second_sentry: ["bar"]
},
expect_success=False)
self.assertEqual(
{
self.sentry_unit: {
"foo": []
},
second_sentry: {
"bar": []
}
}, result)
class GetProcessIdListTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_returns_pids(self):
"""
Normal execution returns a list of pids
"""
self.sentry_unit.commands['pidof -x "foo"'] = ("123 124 125", 0)
result = self.utils.get_process_id_list(self.sentry_unit, "foo")
self.assertEqual(["123", "124", "125"], result)
def test_fails_if_no_process_found(self):
"""
By default, the expectation is that a process is running. Failure
to find a given process results in an amulet.FAIL being
raised.
"""
self.sentry_unit.commands['pidof -x "foo"'] = ("", 1)
with self.assertRaises(SystemExit) as cm, captured_output() as (out,
err):
self.utils.get_process_id_list(self.sentry_unit, "foo")
the_exception = cm.exception
self.assertEqual(1, the_exception.code)
self.assertEqual('foo `pidof -x "foo"` returned 1',
out.getvalue().rstrip())
def test_looks_for_scripts(self):
"""
pidof command uses -x to return a list of pids of scripts
"""
self.sentry_unit.commands["pidof foo"] = ("", 1)
self.sentry_unit.commands['pidof -x "foo"'] = ("123 124 125", 0)
result = self.utils.get_process_id_list(self.sentry_unit, "foo")
self.assertEqual(["123", "124", "125"], result)
def test_expect_no_pid(self):
"""
By setting expectation that there are no pids running the logic
about when to fail is reversed.
"""
self.sentry_unit.commands['pidof -x "foo" || exit 0 && exit 1'] = ("",
0)
self.sentry_unit.commands['pidof -x "bar" || exit 0 && exit 1'] = ("",
1)
result = self.utils.get_process_id_list(self.sentry_unit,
"foo",
expect_success=False)
self.assertEqual([], result)
with self.assertRaises(SystemExit) as cm, captured_output() as (out,
err):
self.utils.get_process_id_list(self.sentry_unit,
"bar",
expect_success=False)
the_exception = cm.exception
self.assertEqual(1, the_exception.code)
self.assertEqual('foo `pidof -x "bar" || exit 0 && exit 1` returned 1',
out.getvalue().rstrip())
class RunActionTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_returns_action_id(self):
"""Returns action_id."""
self.assertEqual("action-id",
self.utils.run_action(self.sentry_unit, "foo"))
class GetProcessIdListTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_returns_pids(self):
"""
Normal execution returns a list of pids
"""
self.sentry_unit.commands['pidof -x "foo"'] = ("123 124 125", 0)
result = self.utils.get_process_id_list(self.sentry_unit, "foo")
self.assertEqual(["123", "124", "125"], result)
@patch("amulet.FAIL", new="failed")
@patch("amulet.raise_status")
def test_fails_if_no_process_found(self, mock_raise_status):
"""
By default, the expectation is that a process is running. Failure
to find a given process results in an amulet.FAIL being
raised.
"""
self.sentry_unit.commands['pidof -x "foo"'] = ("", 1)
self.utils.get_process_id_list(self.sentry_unit, "foo")
mock_raise_status.assert_called_once_with(
"failed", msg='foo `pidof -x "foo"` returned 1 ')
def test_looks_for_scripts(self):
"""
pidof command uses -x to return a list of pids of scripts
"""
self.sentry_unit.commands["pidof foo"] = ("", 1)
self.sentry_unit.commands['pidof -x "foo"'] = ("123 124 125", 0)
result = self.utils.get_process_id_list(self.sentry_unit, "foo")
self.assertEqual(["123", "124", "125"], result)
@patch("amulet.FAIL", new="failed")
@patch("amulet.raise_status")
def test_expect_no_pid(self, mock_raise_status):
"""
By setting expectation that there are no pids running the logic
about when to fail is reversed.
"""
self.sentry_unit.commands['pidof -x "foo" || exit 0 && exit 1'] = ("",
0)
self.sentry_unit.commands['pidof -x "bar" || exit 0 && exit 1'] = ("",
1)
result = self.utils.get_process_id_list(self.sentry_unit,
"foo",
expect_success=False)
self.assertEqual([], result)
self.utils.get_process_id_list(self.sentry_unit,
"bar",
expect_success=False)
mock_raise_status.assert_called_once_with(
"failed",
msg='foo `pidof -x "bar" || exit 0 && exit 1` returned 1 ')
class ValidateServicesByProcessIDTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_accepts_list_wrong(self):
"""
Validates that it can accept a list
"""
expected = {self.sentry_unit: {"foo": [3, 4]}}
actual = {self.sentry_unit: {"foo": [12345, 67890]}}
result = self.utils.validate_unit_process_ids(expected, actual)
self.assertIsNotNone(result)
def test_accepts_list(self):
"""
Validates that it can accept a list
"""
expected = {self.sentry_unit: {"foo": [2, 3]}}
actual = {self.sentry_unit: {"foo": [12345, 67890]}}
result = self.utils.validate_unit_process_ids(expected, actual)
self.assertIsNone(result)
def test_accepts_string(self):
"""
Validates that it can accept a string
"""
expected = {self.sentry_unit: {"foo": 2}}
actual = {self.sentry_unit: {"foo": [12345, 67890]}}
result = self.utils.validate_unit_process_ids(expected, actual)
self.assertIsNone(result)
def test_accepts_string_wrong(self):
"""
Validates that it can accept a string
"""
expected = {self.sentry_unit: {"foo": 3}}
actual = {self.sentry_unit: {"foo": [12345, 67890]}}
result = self.utils.validate_unit_process_ids(expected, actual)
self.assertIsNotNone(result)
def test_accepts_bool(self):
"""
Validates that it can accept a boolean
"""
expected = {self.sentry_unit: {"foo": True}}
actual = {self.sentry_unit: {"foo": [12345, 67890]}}
result = self.utils.validate_unit_process_ids(expected, actual)
self.assertIsNone(result)
def test_accepts_bool_wrong(self):
"""
Validates that it can accept a boolean
"""
expected = {self.sentry_unit: {"foo": True}}
actual = {self.sentry_unit: {"foo": []}}
result = self.utils.validate_unit_process_ids(expected, actual)
self.assertIsNotNone(result)
class BasicDeployment(OpenStackAmuletDeployment):
utils = AmuletUtils()
def __init__(self, vip=None, units=1, series="trusty", openstack=None,
source=None, stable=False):
super(BasicDeployment, self).__init__(series, openstack, source,
stable)
self.units = units
self.master_unit = None
self.vip = None
if units > 1:
if vip:
self.vip = vip
elif 'AMULET_OS_VIP' in os.environ:
self.vip = os.environ.get('AMULET_OS_VIP')
elif os.path.isfile('local.yaml'):
with open('local.yaml', 'rb') as f:
self.cfg = yaml.safe_load(f.read())
self.vip = self.cfg.get('vip')
else:
amulet.raise_status(amulet.SKIP,
("Please set the vip in local.yaml or "
"env var AMULET_OS_VIP to run this test "
"suite"))
self.log = self.utils.get_logger()
def _add_services(self):
"""Add services
Add the services that we're testing, where percona-cluster is local,
and the rest of the service are from lp branches that are
compatible with the local charm (e.g. stable or next).
"""
this_service = {'name': 'percona-cluster',
'units': self.units}
other_services = []
if self.units > 1:
other_services.append({'name': 'hacluster'})
super(BasicDeployment, self)._add_services(this_service,
other_services)
def _add_relations(self):
"""Add all of the relations for the services."""
if self.units > 1:
relations = {'percona-cluster:ha': 'hacluster:ha'}
super(BasicDeployment, self)._add_relations(relations)
def _get_configs(self):
"""Configure all of the services."""
cfg_percona = {'min-cluster-size': self.units,
'vip': self.vip}
cfg_ha = {'debug': True,
'corosync_key': ('xZP7GDWV0e8Qs0GxWThXirNNYlScgi3sRTdZk/IXKD'
'qkNFcwdCWfRQnqrHU/6mb6sz6OIoZzX2MtfMQIDcXu'
'PqQyvKuv7YbRyGHmQwAWDUA4ed759VWAO39kHkfWp9'
'y5RRk/wcHakTcWYMwm70upDGJEP00YT3xem3NQy27A'
'C1w=')}
configs = {}
if self.units > 1:
cfg_ha['cluster_count'] = str(self.units)
configs['hacluster'] = cfg_ha
configs['percona-cluster'] = cfg_percona
return configs
def _configure_services(self):
super(BasicDeployment, self)._configure_services(self._get_configs())
def run(self):
self._add_services()
self._add_relations()
self._configure_services()
self._deploy()
self.d.sentry.wait()
self.test_deployment()
def test_deployment(self):
'''Top level test function executor'''
self.test_pacemaker()
self.test_pxc_running()
self.test_bootstrapped_and_clustered()
self.test_bootstrap_uuid_set_in_the_relation()
self.test_pause_resume()
self.test_kill_master()
def test_pacemaker(self):
'''
Ensure that pacemaker and corosync are correctly configured in
clustered deployments.
side effect: self.master_unit should be set after execution
'''
if self.units > 1:
i = 0
while i < 30 and not self.master_unit:
self.master_unit = self.find_master()
i += 1
time.sleep(10)
msg = 'percona-cluster vip not found'
assert self.master_unit is not None, msg
_, code = self.master_unit.run('sudo crm_verify --live-check')
assert code == 0, "'crm_verify --live-check' failed"
resources = ['res_mysql_vip']
resources += ['res_mysql_monitor:%d' %
m for m in range(self.units)]
assert sorted(self.get_pcmkr_resources()) == sorted(resources)
else:
self.master_unit = self.find_master(ha=False)
def test_pxc_running(self):
'''
Ensure PXC is running on all units
'''
for unit in self.d.sentry['percona-cluster']:
assert self.is_mysqld_running(unit), 'mysql not running: %s' % unit
def test_bootstrapped_and_clustered(self):
'''
Ensure PXC is bootstrapped and that peer units are clustered
'''
self.log.info('Ensuring PXC is bootstrapped')
msg = "Percona cluster failed to bootstrap"
assert self.is_pxc_bootstrapped(), msg
self.log.info('Checking PXC cluster size == {}'.format(self.units))
got = int(self.get_cluster_size())
msg = ("Percona cluster unexpected size"
" (wanted=%s, got=%s)" % (self.units, got))
assert got == self.units, msg
def test_bootstrap_uuid_set_in_the_relation(self):
"""Verify that the bootstrap-uuid attribute was set by the leader and
all the peers where notified.
"""
(leader_uuid, code) = self.master_unit.run("leader-get bootstrap-uuid")
assert leader_uuid
cmd_rel_get = ("relation-get -r `relation-ids cluster` "
"bootstrap-uuid %s")
units = self.d.sentry['percona-cluster']
for unit in units:
for peer in units:
cmd = cmd_rel_get % peer.info['unit_name']
self.log.debug(cmd)
(output, code) = unit.run(cmd)
assert code == 0
assert output == leader_uuid, "%s != %s" % (output,
leader_uuid)
def test_pause_resume(self):
'''
Ensure pasue/resume actions stop/start mysqld on units
'''
self.log.info('Testing pause/resume actions')
self.log.info('Pausing service on first PXC unit')
unit = self.d.sentry['percona-cluster'][0]
assert self.is_mysqld_running(unit), 'mysql not running'
assert self.utils.status_get(unit)[0] == "active"
action_id = self.utils.run_action(unit, "pause")
assert self.utils.wait_on_action(action_id), "Pause action failed."
# Note that is_mysqld_running will print an error message when
# mysqld is not running. This is by design but it looks odd
# in the output.
assert not self.is_mysqld_running(unit=unit), \
"mysqld is still running!"
self.log.info('Resuming service on first PXC unit')
assert self.utils.status_get(unit)[0] == "maintenance"
action_id = self.utils.run_action(unit, "resume")
assert self.utils.wait_on_action(action_id), "Resume action failed"
assert self.utils.status_get(unit)[0] == "active"
assert self.is_mysqld_running(unit=unit), \
"mysqld not running after resume."
def test_kill_master(self):
'''
Ensure that killing the mysqld on the master unit results
in a VIP failover
'''
self.log.info('Testing failover of master unit on mysqld failure')
# we are going to kill the master
old_master = self.master_unit
self.log.info(
'kill -9 mysqld on {}'.format(self.master_unit.info['unit_name'])
)
self.master_unit.run('sudo killall -9 mysqld')
self.log.info('looking for the new master')
i = 0
changed = False
while i < 10 and not changed:
i += 1
time.sleep(5) # give some time to pacemaker to react
new_master = self.find_master()
if (new_master and new_master.info['unit_name'] !=
old_master.info['unit_name']):
self.log.info(
'New master unit detected'
' on {}'.format(new_master.info['unit_name'])
)
changed = True
assert changed, "The master didn't change"
assert self.is_port_open(address=self.vip), 'cannot connect to vip'
def find_master(self, ha=True):
for unit in self.d.sentry['percona-cluster']:
if not ha:
return unit
# is the vip running here?
output, code = unit.run('sudo ip a | grep "inet %s/"' % self.vip)
self.log.info("Checking {}".format(unit.info['unit_name']))
self.log.debug(output)
if code == 0:
self.log.info('vip ({}) running in {}'.format(
self.vip,
unit.info['unit_name'])
)
return unit
def get_pcmkr_resources(self, unit=None):
if unit:
u = unit
else:
u = self.master_unit
output, code = u.run('sudo crm_resource -l')
assert code == 0, 'could not get "crm resource list"'
return output.split('\n')
def is_mysqld_running(self, unit=None):
if unit:
u = unit
else:
u = self.master_unit
_, code = u.run('pidof mysqld')
if code != 0:
self.log.debug("command returned non-zero '%s'" % (code))
return False
return True
def get_wsrep_value(self, attr, unit=None):
if unit:
u = unit
else:
u = self.master_unit
root_password, _ = u.run('leader-get root-password')
cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| "
"grep {}".format(root_password, attr, attr))
output, code = u.run(cmd)
if code != 0:
self.log.debug("command returned non-zero '%s'" % (code))
return ""
value = re.search(r"^.+?\s+(.+)", output).group(1)
self.log.info("%s = %s" % (attr, value))
return value
def is_pxc_bootstrapped(self, unit=None):
value = self.get_wsrep_value('wsrep_ready', unit)
return value.lower() in ['on', 'ready']
def get_cluster_size(self, unit=None):
return self.get_wsrep_value('wsrep_cluster_size', unit)
def is_port_open(self, unit=None, port='3306', address=None):
if unit:
addr = unit.info['public-address']
elif address:
addr = address
else:
raise Exception('Please provide a unit or address')
try:
telnetlib.Telnet(addr, port)
return True
except socket.error as e:
if e.errno == 113:
self.log.error("could not connect to %s:%s" % (addr, port))
if e.errno == 111:
self.log.error("connection refused connecting"
" to %s:%s" % (addr,
port))
return False
#!/usr/bin/python3
# test percona-cluster pause and resum
import basic_deployment
from charmhelpers.contrib.amulet.utils import AmuletUtils
utils = AmuletUtils()
class PauseResume(basic_deployment.BasicDeployment):
def run(self):
super(PauseResume, self).run()
uid = 'percona-cluster/0'
unit = self.d.sentry.unit[uid]
assert self.is_mysqld_running(unit), 'mysql not running: %s' % uid
assert utils.status_get(unit)[0] == "active"
action_id = utils.run_action(unit, "pause")
assert utils.wait_on_action(action_id), "Pause action failed."
# Note that is_mysqld_running will print an error message when
# mysqld is not running. This is by design but it looks odd
# in the output.
assert not self.is_mysqld_running(unit=unit), \
"mysqld is still running!"
assert utils.status_get(unit)[0] == "maintenance"
action_id = utils.run_action(unit, "resume")
assert utils.wait_on_action(action_id), "Resume action failed"
assert utils.status_get(unit)[0] == "active"
assert self.is_mysqld_running(unit=unit), \
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
class ValidateServicesByNameTestCase(unittest.TestCase):
def setUp(self):
self.utils = AmuletUtils()
self.sentry_unit = FakeSentry()
def test_errors_for_unknown_upstart_service(self):
"""
Returns a message if the Upstart service is unknown.
"""
self.sentry_unit.commands["lsb_release -cs"] = "trusty", 0
self.sentry_unit.commands["sudo status foo"] = (
"status: Unknown job: foo", 1)
result = self.utils.validate_services_by_name(
{self.sentry_unit: ["foo"]})
self.assertIsNotNone(result)
def test_none_for_started_upstart_service(self):
"""
Returns None if the Upstart service is running.
"""
self.sentry_unit.commands["lsb_release -cs"] = "trusty", 0
self.sentry_unit.commands["sudo status foo"] = (
"foo start/running, process 42", 0)
result = self.utils.validate_services_by_name(
{self.sentry_unit: ["foo"]})
self.assertIsNone(result)
def test_errors_for_stopped_upstart_service(self):
"""
Returns a message if the Upstart service is stopped.
"""
self.sentry_unit.commands["lsb_release -cs"] = "trusty", 0
self.sentry_unit.commands["sudo status foo"] = "foo stop/waiting", 0
result = self.utils.validate_services_by_name(
{self.sentry_unit: ["foo"]})
self.assertIsNotNone(result)
def test_errors_for_unknown_systemd_service(self):
"""
Returns a message if a systemd service is unknown.
"""
self.sentry_unit.commands["lsb_release -cs"] = "vivid", 0
self.sentry_unit.commands["sudo service foo status"] = (u"""\
\u25cf foo.service
Loaded: not-found (Reason: No such file or directory)
Active: inactive (dead)
""", 3)
result = self.utils.validate_services_by_name(
{self.sentry_unit: ["foo"]})
self.assertIsNotNone(result)
def test_none_for_started_systemd_service(self):
"""
Returns None if a systemd service is running.
"""
self.sentry_unit.commands["lsb_release -cs"] = "vivid", 0
self.sentry_unit.commands["sudo service foo status"] = (u"""\
\u25cf foo.service - Foo
Loaded: loaded (/lib/systemd/system/foo.service; enabled)
Active: active (exited) since Thu 1970-01-01 00:00:00 UTC; 42h 42min ago
Main PID: 3 (code=exited, status=0/SUCCESS)
CGroup: /system.slice/foo.service
""", 0)
result = self.utils.validate_services_by_name(
{self.sentry_unit: ["foo"]})
self.assertIsNone(result)
def test_errors_for_stopped_systemd_service(self):
"""
Returns a message if a systemd service is stopped.
"""
self.sentry_unit.commands["lsb_release -cs"] = "vivid", 0
self.sentry_unit.commands["sudo service foo status"] = (u"""\
\u25cf foo.service - Foo
Loaded: loaded (/lib/systemd/system/foo.service; disabled)
Active: inactive (dead)
""", 3)
result = self.utils.validate_services_by_name(
{self.sentry_unit: ["foo"]})
self.assertIsNotNone(result)
def setUp(self):
self.utils = AmuletUtils()