def reboot_machines(self, client):
log.info("Starting reboot of all containers.")
try:
for machine, m_info in client.get_status().iter_machines():
cont_ids = []
try:
cont_ids.extend([c['instance-id'] for c in
m_info.get('containers').values()])
except KeyError:
log.info('No containers for machine: {}'.format(machine))
if cont_ids:
log.info('Restarting containers: {0} on '
'machine: {1}'.format(cont_ids, machine))
self.ssh(client, machine,
'sudo lxc restart {}'.format(' '.join(cont_ids)))
log.info("Restarting machine: {}".format(machine))
client.juju('run', ('--machine', machine,
'sudo shutdown -r now'))
hostname = client.get_status().get_machine_dns_name(machine)
wait_for_port(hostname, 22, timeout=240)
except subprocess.CalledProcessError as e:
logging.info(
"Error running shutdown:\nstdout: {}\nstderr: {}".format(
e.output, getattr(e, 'stderr', None)))
client.wait_for_started()
def reboot_machines(self, client):
log.info("Starting reboot of all containers.")
try:
for machine, m_info in client.get_status().iter_machines():
cont_ids = []
try:
cont_ids.extend([
c['instance-id']
for c in m_info.get('containers').values()
])
except KeyError:
log.info('No containers for machine: {}'.format(machine))
if cont_ids:
log.info('Restarting containers: {0} on '
'machine: {1}'.format(cont_ids, machine))
self.ssh(client, machine,
'sudo lxc restart {}'.format(' '.join(cont_ids)))
log.info("Restarting machine: {}".format(machine))
client.juju('run',
('--machine', machine, 'sudo shutdown -r now'))
hostname = client.get_status().get_machine_dns_name(machine)
wait_for_port(hostname, 22, timeout=240)
except subprocess.CalledProcessError as e:
logging.info(
"Error running shutdown:\nstdout: {}\nstderr: {}".format(
e.output, getattr(e, 'stderr', None)))
client.wait_for_started()
def existing_bootstrap_context(self, machines, omit_config=None):
""" Context for bootstrapping a state server that shares the
environment with an existing bootstrap environment.
Using this context makes it possible to boot multiple simultaneous
environments that share a JUJU_HOME.
"""
bootstrap_host = self.known_hosts.get('0')
kwargs = dict(
series=self.series, bootstrap_host=bootstrap_host,
agent_url=self.agent_url, agent_stream=self.agent_stream,
region=self.region)
if omit_config is not None:
for key in omit_config:
kwargs.pop(key.replace('-', '_'), None)
update_env(self.client.env, self.temp_env_name, **kwargs)
ssh_machines = list(machines)
if bootstrap_host is not None:
ssh_machines.append(bootstrap_host)
for machine in ssh_machines:
logging.info('Waiting for port 22 on %s' % machine)
wait_for_port(machine, 22, timeout=120)
with self.handle_bootstrap_exceptions():
self.has_controller = True
yield
def test_wait_for_port_no_address_closed(self):
error = socket.gaierror(socket.EAI_NODATA, 'What address?')
with patch('socket.getaddrinfo', autospec=True,
side_effect=error) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('asdf', 26, closed=True)
gai_mock.assert_called_once_with('asdf', 26, socket.AF_INET,
socket.SOCK_STREAM)
self.assertEqual(socket_mock.call_count, 0)
def test_wait_for_port_no_address_closed(self):
error = socket.gaierror(socket.EAI_NODATA, 'What address?')
with patch('socket.getaddrinfo', autospec=True,
side_effect=error) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('asdf', 26, closed=True)
gai_mock.assert_called_once_with('asdf', 26, socket.AF_INET,
socket.SOCK_STREAM)
self.assertEqual(socket_mock.call_count, 0)
def test_wait_for_port_0000_closed(self):
with patch(
'socket.getaddrinfo', autospec=True,
return_value=[('foo', 'bar', 'baz', 'qux', ('0.0.0.0', 27))]
) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('asdf', 26, closed=True)
gai_mock.assert_called_once_with('asdf', 26, socket.AF_INET,
socket.SOCK_STREAM)
self.assertEqual(socket_mock.call_count, 0)
def test_wait_for_port_0000_closed(self):
with patch('socket.getaddrinfo',
autospec=True,
return_value=[('foo', 'bar', 'baz', 'qux', ('0.0.0.0', 27))
]) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('asdf', 26, closed=True)
gai_mock.assert_called_once_with('asdf', 26, socket.AF_INET,
socket.SOCK_STREAM)
self.assertEqual(socket_mock.call_count, 0)
def test_ipv6_open(self):
gai_result = [(23, 0, 0, '', ('2001:db8::2', 22, 0, 0))]
with patch('socket.getaddrinfo', autospec=True,
return_value=gai_result) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('2001:db8::2', 22, closed=False)
gai_mock.assert_called_once_with(
'2001:db8::2', 22, socket.AF_INET6, socket.SOCK_STREAM)
socket_mock.assert_called_once_with(23, 0, 0)
connect_mock = socket_mock.return_value.connect
connect_mock.assert_called_once_with(('2001:db8::2', 22, 0, 0))
def test_ipv6_open(self):
gai_result = [(23, 0, 0, '', ('2001:db8::2', 22, 0, 0))]
with patch('socket.getaddrinfo',
autospec=True,
return_value=gai_result) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('2001:db8::2', 22, closed=False)
gai_mock.assert_called_once_with('2001:db8::2', 22, socket.AF_INET6,
socket.SOCK_STREAM)
socket_mock.assert_called_once_with(23, 0, 0)
connect_mock = socket_mock.return_value.connect
connect_mock.assert_called_once_with(('2001:db8::2', 22, 0, 0))
def test_wait_for_port(self):
with patch('socket.getaddrinfo',
autospec=True,
return_value=[('foo', 'bar', 'baz', 'qux',
('192.168.8.3', 27))]) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('asdf', 26, closed=False)
gai_mock.assert_called_once_with('asdf', 26, socket.AF_INET,
socket.SOCK_STREAM),
socket_mock.assert_called_once_with('foo', 'bar', 'baz')
connect_mock = socket_mock.return_value.connect
connect_mock.assert_called_once_with(('192.168.8.3', 27))
def test_wait_for_port(self):
with patch(
'socket.getaddrinfo', autospec=True, return_value=[
('foo', 'bar', 'baz', 'qux', ('192.168.8.3', 27))
]) as gai_mock:
with patch('socket.socket') as socket_mock:
wait_for_port('asdf', 26, closed=False)
gai_mock.assert_called_once_with(
'asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
socket_mock.assert_called_once_with('foo', 'bar', 'baz')
connect_mock = socket_mock.return_value.connect
connect_mock.assert_called_once_with(('192.168.8.3', 27))
def assess_container_networking(client, types):
"""Runs _assess_address_allocation, reboots hosts, repeat.
:param client: Juju client
:param types: Container types to test
:return: None
"""
log.info("Setting up test.")
hosts, containers = make_machines(client, types)
status = client.wait_for_started().status
log.info("Setup complete.")
log.info("Test started.")
_assess_container_networking(client, types, hosts, containers)
# Reboot all hosted modelled machines then the controller.
log.info("Instrumenting reboot of all machines.")
try:
for host in hosts:
log.info("Restarting hosted machine: {}".format(host))
client.juju(
'run', ('--machine', host, 'sudo shutdown -r now'))
client.juju('show-action-status', ('--name', 'juju-run'))
log.info("Restarting controller machine 0")
controller_client = client.get_controller_client()
controller_status = controller_client.get_status()
controller_host = controller_status.status['machines']['0']['dns-name']
first_uptime = get_uptime(controller_client, '0')
ssh(controller_client, '0', 'sudo shutdown -r now')
except subprocess.CalledProcessError as e:
logging.info(
"Error running shutdown:\nstdout: %s\nstderr: %s",
e.output, getattr(e, 'stderr', None))
raise
# Wait for the controller to shut down if it has not yet restarted.
# This ensure the call to wait_for_started happens after each host
# has restarted.
second_uptime = get_uptime(controller_client, '0')
if second_uptime > first_uptime:
wait_for_port(controller_host, 22, closed=True, timeout=300)
client.wait_for_started()
# Once Juju is up it can take a little while before ssh responds.
for host in hosts:
hostname = status['machines'][host]['dns-name']
wait_for_port(hostname, 22, timeout=240)
log.info("Reboot complete and all hosts ready for retest.")
_assess_container_networking(client, types, hosts, containers)
log.info("PASS")
def copy_remote_logs(remote, directory):
"""Copy as many logs from the remote host as possible to the directory."""
# This list of names must be in the order of creation to ensure they
# are retrieved.
if remote.is_windows():
log_paths = [
"%ProgramFiles(x86)%\\Cloudbase Solutions\\Cloudbase-Init\\log\\*",
"C:\\Juju\\log\\juju\\*.log",
]
else:
log_paths = [
'/var/log/cloud-init*.log',
'/var/log/juju/*.log',
# TODO(gz): Also capture kvm container logs?
'/var/lib/juju/containers/juju-*-lxc-*/',
'/var/log/lxd/juju-*',
'/var/log/lxd/lxd.log',
'/var/log/syslog',
'/var/log/mongodb/mongodb.log',
'/etc/network/interfaces',
'/etc/environment',
'/home/ubuntu/ifconfig.log',
]
try:
wait_for_port(remote.address, 22, timeout=60)
except PortTimeoutError:
logging.warning("Could not dump logs because port 22 was closed.")
return
try:
remote.run('sudo chmod -Rf go+r ' + ' '.join(log_paths))
except subprocess.CalledProcessError as e:
# The juju log dir is not created until after cloud-init succeeds.
logging.warning("Could not allow access to the juju logs:")
logging.warning(e.output)
try:
remote.run('ifconfig > /home/ubuntu/ifconfig.log')
except subprocess.CalledProcessError as e:
logging.warning("Could not capture ifconfig state:")
logging.warning(e.output)
try:
remote.copy(directory, log_paths)
except (subprocess.CalledProcessError,
winrm.exceptions.WinRMTransportError) as e:
# The juju logs will not exist if cloud-init failed.
logging.warning("Could not retrieve some or all logs:")
if getattr(e, 'output', None):
logging.warning(e.output)
else:
logging.warning(repr(e))
def wait_for_state_server_to_shutdown(host, client, instance_id, timeout=60):
print_now("Waiting for port to close on %s" % host)
wait_for_port(host, 17070, closed=True, timeout=timeout)
print_now("Closed.")
try:
provider_type = client.env.provider
except NoProvider:
provider_type = None
if provider_type == 'openstack':
for ignored in until_timeout(300):
if not has_nova_instance(client.env, instance_id):
print_now('{} was removed from nova list'.format(instance_id))
break
else:
raise Exception(
'{} was not deleted:'.format(instance_id))
def bootstrap_context(self, machines, omit_config=None):
"""Context for bootstrapping a state server."""
bootstrap_host = self.known_hosts.get('0')
kwargs = dict(
series=self.series, bootstrap_host=bootstrap_host,
agent_url=self.agent_url, agent_stream=self.agent_stream,
region=self.region)
if omit_config is not None:
for key in omit_config:
kwargs.pop(key.replace('-', '_'), None)
update_env(self.client.env, self.temp_env_name, **kwargs)
ssh_machines = list(machines)
if bootstrap_host is not None:
ssh_machines.append(bootstrap_host)
for machine in ssh_machines:
logging.info('Waiting for port 22 on %s' % machine)
wait_for_port(machine, 22, timeout=120)
jenv_path = get_jenv_path(self.client.env.juju_home,
self.client.env.environment)
torn_down = False
if os.path.isfile(jenv_path):
# An existing .jenv implies JES was not used, because when JES is
# enabled, cache.yaml is enabled.
self.tear_down_client.kill_controller()
torn_down = True
else:
jes_home = jes_home_path(
self.client.env.juju_home, self.client.env.environment)
with temp_juju_home(self.client, jes_home):
cache_path = self.client.get_cache_path()
if os.path.isfile(cache_path):
# An existing .jenv implies JES was used, because when JES
# is enabled, cache.yaml is enabled.
self.controller_strategy.prepare()
torn_down = True
ensure_deleted(jenv_path)
with temp_bootstrap_env(self.client.env.juju_home, self.client,
permanent=self.permanent, set_home=False):
with self.handle_bootstrap_exceptions():
if not torn_down:
self.controller_strategy.prepare()
self.has_controller = True
yield
def test_wait_for_port_0000_open(self):
stub_called = False
loc = locals()
def gai_stub(host, port, family, socktype):
if loc['stub_called']:
raise ValueError()
loc['stub_called'] = True
return [('foo', 'bar', 'baz', 'qux', ('0.0.0.0', 27))]
with patch('socket.getaddrinfo', autospec=True, side_effect=gai_stub,
) as gai_mock:
with patch('socket.socket') as socket_mock:
with self.assertRaises(ValueError):
wait_for_port('asdf', 26, closed=False)
self.assertEqual(gai_mock.mock_calls, [
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
])
self.assertEqual(socket_mock.call_count, 0)
def test_wait_for_port_no_address_open(self):
stub_called = False
loc = locals()
def gai_stub(host, port, family, socktype):
if loc['stub_called']:
raise ValueError()
loc['stub_called'] = True
raise socket.error(socket.EAI_NODATA, 'Err, address?')
with patch('socket.getaddrinfo', autospec=True, side_effect=gai_stub,
) as gai_mock:
with patch('socket.socket') as socket_mock:
with self.assertRaises(ValueError):
wait_for_port('asdf', 26, closed=False)
self.assertEqual(gai_mock.mock_calls, [
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
])
self.assertEqual(socket_mock.call_count, 0)
def test_wait_for_port_no_address_open(self):
stub_called = False
loc = locals()
def gai_stub(host, port, family, socktype):
if loc['stub_called']:
raise ValueError()
loc['stub_called'] = True
raise socket.error(socket.EAI_NODATA, 'Err, address?')
with patch(
'socket.getaddrinfo',
autospec=True,
side_effect=gai_stub,
) as gai_mock:
with patch('socket.socket') as socket_mock:
with self.assertRaises(ValueError):
wait_for_port('asdf', 26, closed=False)
self.assertEqual(gai_mock.mock_calls, [
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
])
self.assertEqual(socket_mock.call_count, 0)
def test_wait_for_port_0000_open(self):
stub_called = False
loc = locals()
def gai_stub(host, port, family, socktype):
if loc['stub_called']:
raise ValueError()
loc['stub_called'] = True
return [('foo', 'bar', 'baz', 'qux', ('0.0.0.0', 27))]
with patch(
'socket.getaddrinfo',
autospec=True,
side_effect=gai_stub,
) as gai_mock:
with patch('socket.socket') as socket_mock:
with self.assertRaises(ValueError):
wait_for_port('asdf', 26, closed=False)
self.assertEqual(gai_mock.mock_calls, [
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
call('asdf', 26, socket.AF_INET, socket.SOCK_STREAM),
])
self.assertEqual(socket_mock.call_count, 0)