def test_transient_provider_error_on_get_machines(self):
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.get_machines()
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.get_machines()
self.mocker.passthrough()
self.mocker.replay()
try:
yield self.agent.process_machines([machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
yield self.agent.process_machines([machine_state0.id])
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
self.assertIn("Cannot get machine list", self.output.getvalue())
def convert_unknown_error(failure):
"""Convert any non-juju errors to a provider interaction error.
Supports both usage from within an except clause, and as an
errback handler ie. both the following forms are supported.
...
try:
something()
except Exception, e:
convert_unknown_errors(e)
...
d.addErrback(convert_unknown_errors)
"""
if isinstance(failure, Failure):
error = failure.value
else:
error = failure
if not isinstance(error, JujuError):
message = ("Unexpected %s interacting with provider: %s" %
(type(error).__name__, str(error)))
error = ProviderInteractionError(message)
if isinstance(failure, Failure):
return Failure(error)
raise error
def test_transient_provider_error_on_shutdown_machine(self):
"""
A transient provider error on shutdown will be ignored
and the shutdown will be reattempted (assuming similiar
state conditions) on the next execution of process machines.
"""
yield self.agent.provider.start_machine({"machine-id": 1})
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.shutdown_machine(MATCH_MACHINE)
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.shutdown_machine(MATCH_MACHINE)
self.mocker.passthrough()
self.mocker.replay()
try:
yield self.agent.process_machines([])
except:
self.fail("Should not raise")
machines = yield self.agent.provider.get_machines()
self.assertTrue(machines)
yield self.agent.process_machines([])
machines = yield self.agent.provider.get_machines()
self.assertFalse(machines)
self.assertIn("Cannot shutdown machine 0", self.output.getvalue())
def _ensure_groups(self, machine_id):
"""Ensure the juju group is the machine launch groups.
Machines launched by juju are tagged with a group so they
can be distinguished from other machines that might be running
on an EC2 account. This group can be specified explicitly or
implicitly defined by the environment name. In addition, a
specific machine security group is created for each machine,
so that its firewall rules can be configured per machine.
:param machine_id: The juju machine ID of the new machine
"""
juju_group = "juju-%s" % self._provider.environment_name
juju_machine_group = "juju-%s-%s" % (self._provider.environment_name,
machine_id)
security_groups = yield self._provider.ec2.describe_security_groups()
group_ids = [group.name for group in security_groups]
# Create the provider group if doesn't exist.
if not juju_group in group_ids:
log.debug("Creating juju provider group %s", juju_group)
yield self._provider.ec2.create_security_group(
juju_group,
"juju group for %s" % self._provider.environment_name)
# Authorize SSH.
yield self._provider.ec2.authorize_security_group(
juju_group,
ip_protocol="tcp",
from_port="22",
to_port="22",
cidr_ip="0.0.0.0/0")
# We need to describe the group to pickup the owner_id for auth.
groups_info = yield self._provider.ec2.describe_security_groups(
juju_group)
# Authorize Internal ZK Traffic
yield self._provider.ec2.authorize_security_group(
juju_group,
source_group_name=juju_group,
source_group_owner_id=groups_info.pop().owner_id)
# Create the machine-specific group, but first see if there's
# one already existing from a previous machine launch;
# if so, delete it, since it can have the wrong firewall setup
if juju_machine_group in group_ids:
try:
yield self._provider.ec2.delete_security_group(
juju_machine_group)
log.debug("Deleted existing machine group %s, will replace",
juju_machine_group)
except EC2Error, e:
log.debug("Cannot delete security group %s: %s",
juju_machine_group, e)
raise ProviderInteractionError(
"Unexpected EC2Error deleting security group %s: %s" %
(juju_machine_group, e.get_error_messages()))
def _delete_security_group(provider, group):
"""Wrap EC2 delete_security_group."""
try:
yield provider.ec2.delete_security_group(group)
log.debug("Deleted security group %r", group)
except EC2Error, e:
raise ProviderInteractionError(
"EC2 error when attempting to delete group %s: %s" % (group, e))
def get_provider_opened_ports(provider, machine, machine_id):
"""Gets the opened ports for `machine`.
Retrieves the IP permissions associated with the machine
security group, then parses them to return a set of (port,
proto) pairs.
"""
try:
security_groups = yield provider.ec2.describe_security_groups(
_get_machine_group_name(provider, machine_id))
except EC2Error, e:
raise ProviderInteractionError(
"Unexpected EC2Error getting open ports on machine %s: %s"
% (machine.instance_id, e.get_error_messages()))
def close_provider_port(provider, machine, machine_id, port, protocol):
"""Revoke `port`/`proto` for the machine security group."""
try:
yield provider.ec2.revoke_security_group(
_get_machine_group_name(provider, machine_id),
ip_protocol=protocol,
from_port=str(port), to_port=str(port),
cidr_ip="0.0.0.0/0")
log.debug("Closed %s/%s on provider machine %r",
port, protocol, machine.instance_id)
except EC2Error, e:
raise ProviderInteractionError(
"Unexpected EC2Error closing %s/%s on machine %s: %s"
% (port, protocol, machine.instance_id, e.get_error_messages()))
def test_open_close_ports_on_machine_will_retry(self):
"""Verify port mgmt for a machine will retry if there's a failure."""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(
fail(ProviderInteractionError("Some sort of EC2 problem")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.passthrough()
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Retries will now happen in the periodic recheck. First one
# still fails due to simulated error.
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("ProviderInteractionError: Some sort of EC2 problem",
self.output.getvalue())
# Third time is the charm in the mock setup, the recheck succeeds
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
def get_machines(self, instance_ids=()):
"""List machines running in the provider.
:param list instance_ids: ids of instances you want to get. Leave empty
to list every
:class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by
this provider.
:return: a list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
:raises: :exc:`juju.errors.MachinesNotFound`
"""
group_name = "juju-%s" % self.environment_name
try:
instances = yield self.ec2.describe_instances(*instance_ids)
except EC2Error as error:
code = error.get_error_codes()
message = error.get_error_messages()
if code == "InvalidInstanceID.NotFound":
message = error.get_error_messages()
raise MachinesNotFound(
re.findall(r"\bi-[0-9a-f]{3,15}\b", message))
raise ProviderInteractionError(
"Unexpected EC2Error getting machines %s: %s"
% (", ".join(instance_ids), message))
machines = []
for instance in instances:
if instance.instance_state not in ("running", "pending"):
continue
if group_name not in instance.reservation.groups:
continue
machines.append(machine_from_instance(instance))
if instance_ids:
# We were asked for a specific list of machines, and if we can't
# completely fulfil that request we should blow up.
found_instance_ids = set(m.instance_id for m in machines)
missing = set(instance_ids) - found_instance_ids
if missing:
raise MachinesNotFound(missing)
returnValue(machines)
def test_transient_provider_error_on_start_machine(self):
"""
If there's an error when processing changes, the agent should log
the error and continue.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.start_machine({"machine-id": 0})
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.start_machine({"machine-id": 1})
self.mocker.passthrough()
self.mocker.replay()
yield self.agent.watch_machine_changes(
[], [machine_state0.id, machine_state1.id])
machine1_instance_id = yield machine_state1.get_instance_id()
self.assertEqual(machine1_instance_id, 0)
self.assertIn("Cannot process machine 0", self.output.getvalue())
def test_ProviderInteractionError(self):
error = ProviderInteractionError("Bad Stuff")
self.assertIsJujuError(error)
self.assertEquals(str(error), "Bad Stuff")