def background_chaos(env, client, log_dir, time):
monkey = MonkeyRunner(env, client, enablement_timeout=time)
monkey.deploy_chaos_monkey()
monkey_ids = monkey.unleash_once()
monkey.wait_for_chaos(state='start')
try:
yield
monkey.wait_for_chaos(state='complete', timeout=time)
except BaseException as e:
logging.exception(e)
sys.exit(1)
finally:
# Copy the chaos logs to the log directory.
# Get the remote machine. Currently the remote machine will always be
# ubuntu/0. IF background_chaos() is enhanced to take a target service,
# then log collection will also need to be updated.
remote = remote_from_unit(client, "ubuntu/0")
for id in monkey_ids:
monkey_log = ['chaos-monkey/chaos_monkey.{}/log/*'.format(id)]
dest_dir = '{}/chaos-monkey-{}'.format(log_dir, id)
os.mkdir(dest_dir)
try:
remote.copy(dest_dir, monkey_log)
except subprocess.CalledProcessError as e:
logging.warning(
'Could not retrieve Chaos Monkey log for {}:'.format(id))
logging.warning(e.output)
def test_run_with_unit_fallback(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status") as st:
st.return_value = Status.from_text(self.precise_status_output)
remote = remote_from_unit(client, unit)
with patch.object(client, "get_juju_output") as mock_gjo:
mock_gjo.side_effect = subprocess.CalledProcessError(255,
"ssh",
output="")
with patch.object(remote, "_run_subprocess") as mock_run:
mock_run.return_value = "contents of /a/file"
output = remote.run("cat /a/file")
self.assertEqual(output, "contents of /a/file")
mock_gjo.assert_called_once_with("ssh",
unit,
"cat /a/file",
timeout=120)
mock_run.assert_called_once_with([
"ssh",
"-o",
"User ubuntu",
"-o",
"UserKnownHostsFile /dev/null",
"-o",
"StrictHostKeyChecking no",
"-o",
"PasswordAuthentication no",
"10.55.60.1",
"cat /a/file",
])
self.assertRegexpMatches(
self.log_stream.getvalue(),
"(?m)^WARNING juju ssh to 'a-application/0' failed, .*")
def test_run_with_unit_fallback(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status") as st:
st.return_value = Status.from_text(self.precise_status_output)
remote = remote_from_unit(client, unit)
with patch.object(client, "get_juju_output") as mock_gjo:
mock_gjo.side_effect = subprocess.CalledProcessError(
255, "ssh", output="")
with patch.object(remote, "_run_subprocess") as mock_run:
mock_run.return_value = "contents of /a/file"
output = remote.run("cat /a/file")
self.assertEqual(output, "contents of /a/file")
mock_gjo.assert_called_once_with("ssh", unit, "cat /a/file",
timeout=120)
mock_run.assert_called_once_with([
"ssh",
"-o", "User ubuntu",
"-o", "UserKnownHostsFile /dev/null",
"-o", "StrictHostKeyChecking no",
"-o", "PasswordAuthentication no",
"10.55.60.1",
"cat /a/file",
])
self.assertRegexpMatches(
self.log_stream.getvalue(),
"(?m)^WARNING juju ssh to 'a-application/0' failed, .*")
def test_remote_from_unit_with_series(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
remote = remote_from_unit(client, unit, series="trusty")
self.assertEqual(repr(remote),
"<SSHRemote env='an-env' unit='a-application/0'>")
self.assertIs(False, remote.is_windows())
def test_remote_from_unit_with_series(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
remote = remote_from_unit(client, unit, series="trusty")
self.assertEqual(
repr(remote),
"<SSHRemote env='an-env' unit='a-application/0'>")
self.assertIs(False, remote.is_windows())
def test_remote_from_unit(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.precise_status_output)
remote = remote_from_unit(client, unit)
self.assertEqual(repr(remote),
"<SSHRemote env='an-env' unit='a-application/0'>")
self.assertIs(False, remote.is_windows())
def test_remote_from_unit_with_status(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
status = Status.from_text(self.win2012hvr2_status_output)
remote = remote_from_unit(client, unit, status=status)
self.assertEqual(
repr(remote), "<WinRmRemote env='an-env' unit='a-application/0'"
" addr='10.55.60.2'>")
self.assertIs(True, remote.is_windows())
def test_remote_from_unit(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.precise_status_output)
remote = remote_from_unit(client, unit)
self.assertEqual(
repr(remote),
"<SSHRemote env='an-env' unit='a-application/0'>")
self.assertIs(False, remote.is_windows())
def apply_condition(client, condition):
"""Apply an adverse condition to the given unit."""
unit, action = condition.split(':', 1)
logging.info('Applying {} to unit {}'.format(action, unit))
remote = remote_from_unit(client, unit)
if not remote.is_windows():
if action == 'clock_skew':
result = remote.run(CLOCK_SKEW_SCRIPT)
logging.info('Clock on {} set to: {}'.format(unit, result))
else:
raise ErrUnitCondition("%s: Unknown condition type." % action)
def test_run_with_unit(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
remote = remote_from_unit(client, unit, series="trusty")
with patch.object(client, "get_juju_output") as mock_cmd:
mock_cmd.return_value = "contents of /a/file"
output = remote.run("cat /a/file")
self.assertEqual(output, "contents of /a/file")
mock_cmd.assert_called_once_with("ssh", unit, "cat /a/file",
timeout=120)
def test_remote_from_unit_with_status(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
status = Status.from_text(self.win2012hvr2_status_output)
remote = remote_from_unit(client, unit, status=status)
self.assertEqual(
repr(remote),
"<WinRmRemote env='an-env' unit='a-application/0'"
" addr='10.55.60.2'>")
self.assertIs(True, remote.is_windows())
def test_run_with_unit(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
remote = remote_from_unit(client, unit, series="trusty")
with patch.object(client, "get_juju_output") as mock_cmd:
mock_cmd.return_value = "contents of /a/file"
output = remote.run("cat /a/file")
self.assertEqual(output, "contents of /a/file")
mock_cmd.assert_called_once_with("ssh",
unit,
"cat /a/file",
timeout=120)
def test_cat_on_windows(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.win2012hvr2_status_output)
response = winrm.Response(("contents of /a/file", "", 0))
remote = remote_from_unit(client, unit)
with patch.object(remote.session, "run_cmd", autospec=True,
return_value=response) as mock_run:
output = remote.cat("/a/file")
self.assertEqual(output, "contents of /a/file")
st.assert_called_once_with()
mock_run.assert_called_once_with("type", ["/a/file"])
def test_run_cmd(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.win2012hvr2_status_output)
response = winrm.Response(("some out", "some err", 0))
remote = remote_from_unit(client, unit)
with patch.object(remote.session, "run_cmd", autospec=True,
return_value=response) as mock_run:
output = remote.run_cmd(
["C:\\Program Files\\bin.exe", "/IN", "Bob's Stuff"])
self.assertEqual(output, response)
st.assert_called_once_with()
mock_run.assert_called_once_with(
'"C:\\Program Files\\bin.exe"', ['/IN "Bob\'s Stuff"'])
def test_cat_on_windows(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.win2012hvr2_status_output)
response = winrm.Response(("contents of /a/file", "", 0))
remote = remote_from_unit(client, unit)
with patch.object(remote.session,
"run_cmd",
autospec=True,
return_value=response) as mock_run:
output = remote.cat("/a/file")
self.assertEqual(output, "contents of /a/file")
st.assert_called_once_with()
mock_run.assert_called_once_with("type", ["/a/file"])
def test_run_default_command_error_fallback(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
error = subprocess.CalledProcessError(1, "ssh", output="bad command")
with patch.object(client, "get_status") as st:
st.return_value = Status.from_text(self.precise_status_output)
remote = remote_from_unit(client, unit)
with patch.object(client, "get_juju_output") as mock_gjo:
mock_gjo.side_effect = error
with self.assertRaises(subprocess.CalledProcessError) as c:
remote.run("cat /a/file")
self.assertIs(c.exception, error)
mock_gjo.assert_called_once_with("ssh", unit, "cat /a/file",
timeout=120)
self.assertRegexpMatches(
self.log_stream.getvalue(),
"(?m)^WARNING juju ssh to 'a-application/0' failed, .*")
def test_run_cmd(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.win2012hvr2_status_output)
response = winrm.Response(("some out", "some err", 0))
remote = remote_from_unit(client, unit)
with patch.object(remote.session,
"run_cmd",
autospec=True,
return_value=response) as mock_run:
output = remote.run_cmd(
["C:\\Program Files\\bin.exe", "/IN", "Bob's Stuff"])
self.assertEqual(output, response)
st.assert_called_once_with()
mock_run.assert_called_once_with('"C:\\Program Files\\bin.exe"',
['/IN "Bob\'s Stuff"'])
def deploy_charm_and_verify(client, series="xenial", charm_app="dummy-source"):
"""
Deploy dummy charm from local repository and
verify it uses the specified agent-metadata-url option
:param client: Juju client
:param series: The charm series to deploy
:param charm_app: Juju charm application
"""
charm_source = local_charm_path(
charm=charm_app, juju_ver=client.version, series=series)
client.deploy(charm_source)
client.wait_for_started()
client.set_config(charm_app, {'token': 'one'})
client.wait_for_workloads()
remote = remote_from_unit(client, "{}/0".format(charm_app))
verify_deployed_charm(client, remote)
log.info(
"Successfully deployed charm {} of series {} and verified".format(
"dummy-source", series))
def test_copy_on_windows(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
dest = "/local/path"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.win2012hvr2_status_output)
response = winrm.Response(("fake output", "", 0))
remote = remote_from_unit(client, unit)
with patch.object(remote.session, "run_ps", autospec=True,
return_value=response) as mock_run:
with patch.object(remote, "_encoded_copy_to_dir",
autospec=True) as mock_cpdir:
remote.copy(dest, ["C:\\logs\\*", "%APPDATA%\\*.log"])
mock_cpdir.assert_called_once_with(dest, "fake output")
st.assert_called_once_with()
self.assertEquals(mock_run.call_count, 1)
self.assertRegexpMatches(
mock_run.call_args[0][0],
r'.*"C:\\logs\\[*]","%APPDATA%\\[*].log".*')
def test_run_default_command_error_fallback(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
error = subprocess.CalledProcessError(1, "ssh", output="bad command")
with patch.object(client, "get_status") as st:
st.return_value = Status.from_text(self.precise_status_output)
remote = remote_from_unit(client, unit)
with patch.object(client, "get_juju_output") as mock_gjo:
mock_gjo.side_effect = error
with self.assertRaises(subprocess.CalledProcessError) as c:
remote.run("cat /a/file")
self.assertIs(c.exception, error)
mock_gjo.assert_called_once_with("ssh",
unit,
"cat /a/file",
timeout=120)
self.assertRegexpMatches(
self.log_stream.getvalue(),
"(?m)^WARNING juju ssh to 'a-application/0' failed, .*")
def test_copy_on_windows(self):
env = JujuData("an-env", {"type": "nonlocal"})
client = ModelClient(env, None, None)
unit = "a-application/0"
dest = "/local/path"
with patch.object(client, "get_status", autospec=True) as st:
st.return_value = Status.from_text(self.win2012hvr2_status_output)
response = winrm.Response(("fake output", "", 0))
remote = remote_from_unit(client, unit)
with patch.object(remote.session,
"run_ps",
autospec=True,
return_value=response) as mock_run:
with patch.object(remote,
"_encoded_copy_to_dir",
autospec=True) as mock_cpdir:
remote.copy(dest, ["C:\\logs\\*", "%APPDATA%\\*.log"])
mock_cpdir.assert_called_once_with(dest, "fake output")
st.assert_called_once_with()
self.assertEquals(mock_run.call_count, 1)
self.assertRegexpMatches(mock_run.call_args[0][0],
r'.*"C:\\logs\\[*]","%APPDATA%\\[*].log".*')
def check_token(client, token, timeout=120):
"""Check the token found on dummy-sink/0 or raise ValueError."""
logging.info('Waiting for applications to reach ready.')
client.wait_for_workloads()
logging.info('Retrieving token.')
remote = remote_from_unit(client, "dummy-sink/0")
# Update remote with real address if needed.
resolve_remote_dns_names(client.env, [remote])
# By this point the workloads should be ready and token will have been
# sent successfully, but fallback to timeout as previously for now.
start = time.time()
while True:
is_winclient1x = (isinstance(client, EnvJujuClient1X) and
sys.platform == "win32")
if remote.is_windows() or is_winclient1x:
result = get_token_from_status(client)
if not result:
result = _get_token(remote, "%ProgramData%\\dummy-sink\\token")
else:
result = _get_token(remote)
if result == token:
logging.info("Token matches expected %r", result)
return
if time.time() - start > timeout:
if remote.use_juju_ssh and _can_run_ssh():
# 'juju ssh' didn't error, but try raw ssh to verify
# the result is the same.
remote.get_address()
remote.use_juju_ssh = False
result = _get_token(remote)
if result == token:
logging.info("Token matches expected %r", result)
logging.error("juju ssh to unit is broken.")
raise ValueError('Token is %r' % result)
logging.info("Found token %r expected %r", result, token)
time.sleep(5)