def test_remove_security_groups_raises_http_timeout(self):
"""Verify that a timeout in txaws raises an appropiate error.
This is an example of a scenario where txaws may not raise an
EC2Error, but will instead raise some other error.
"""
system_state = MockInstanceState(
self, ["i-amkillable"], [0],
[["shutting-down"],
["shutting-down"],
["shutting-down"],
["terminated"]])
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.mocker.count(4)
self.ec2.delete_security_group("juju-moon-0")
self.mocker.result(fail(
TimeoutError(
"Getting https://x.example.com?Action=DeleteSecurityGroup"
"&etc=andsoforth took longer than 30 seconds.")))
self.mocker.replay()
provider = self.get_provider()
ex = yield self.assertFailure(
remove_security_groups(provider, ["i-amkillable"]),
TimeoutError)
self.assertEquals(
str(ex),
"Getting https://x.example.com?Action=DeleteSecurityGroup"
"&etc=andsoforth took longer than 30 seconds.")
def test_remove_security_groups(self):
"""Test normal running seen in polling instances for their state."""
system_state = MockInstanceState(
self,
["i-amkillable", "i-amkillabletoo"], [0, 2],
[["shutting-down", "shutting-down"],
["shutting-down", "shutting-down"],
["shutting-down", "terminated"],
["terminated"]])
self.ec2.describe_instances("i-amkillable", "i-amkillabletoo")
self.mocker.call(system_state.get_round)
self.mocker.count(3)
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.ec2.delete_security_group(MATCH_GROUP)
deleted_groups = Observed()
self.mocker.call(deleted_groups.add)
self.mocker.count(2)
self.mocker.replay()
provider = self.get_provider()
yield remove_security_groups(
provider, ["i-amkillable", "i-amkillabletoo"])
self.assertEquals(
deleted_groups.items,
set(["juju-moon-0", "juju-moon-2"]))
def test_remove_security_groups_raises_http_timeout(self):
"""Verify that a timeout in txaws raises an appropiate error.
This is an example of a scenario where txaws may not raise an
EC2Error, but will instead raise some other error.
"""
system_state = MockInstanceState(self, ["i-amkillable"], [0],
[["shutting-down"], ["shutting-down"],
["shutting-down"], ["terminated"]])
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.mocker.count(4)
self.ec2.delete_security_group("juju-moon-0")
self.mocker.result(
fail(
TimeoutError(
"Getting https://x.example.com?Action=DeleteSecurityGroup"
"&etc=andsoforth took longer than 30 seconds.")))
self.mocker.replay()
provider = self.get_provider()
ex = yield self.assertFailure(
remove_security_groups(provider, ["i-amkillable"]), TimeoutError)
self.assertEquals(
str(ex), "Getting https://x.example.com?Action=DeleteSecurityGroup"
"&etc=andsoforth took longer than 30 seconds.")
def test_cannot_delete_machine_security_group(self):
"""Verify this raises error.
Note it is possible for a malicious admin to start other
machines with an existing machine security group, so need to
test this removal is robust in this case. Another scenario
that might cause this is that there's an EC2 timeout.
"""
system_state = MockInstanceState(self, ["i-amkillable"], [0],
[["shutting-down"], ["shutting-down"],
["shutting-down"], ["terminated"]])
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.mocker.count(4)
self.ec2.delete_security_group("juju-moon-0")
self.mocker.result(
fail(
self.get_ec2_error(
"juju-moon-0",
format="There are active instances using security group %r"
)))
self.mocker.replay()
provider = self.get_provider()
ex = yield self.assertFailure(
remove_security_groups(provider, ["i-amkillable"]),
ProviderInteractionError)
self.assertEquals(
str(ex), "EC2 error when attempting to delete group juju-moon-0: "
"Error Message: There are active instances using security group "
"'juju-moon-0'")
def test_remove_security_groups(self):
"""Test normal running seen in polling instances for their state."""
system_state = MockInstanceState(
self, ["i-amkillable", "i-amkillabletoo"],
[0, 2], [["shutting-down", "shutting-down"],
["shutting-down", "shutting-down"],
["shutting-down", "terminated"], ["terminated"]])
self.ec2.describe_instances("i-amkillable", "i-amkillabletoo")
self.mocker.call(system_state.get_round)
self.mocker.count(3)
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.ec2.delete_security_group(MATCH_GROUP)
deleted_groups = Observed()
self.mocker.call(deleted_groups.add)
self.mocker.count(2)
self.mocker.replay()
provider = self.get_provider()
yield remove_security_groups(provider,
["i-amkillable", "i-amkillabletoo"])
self.assertEquals(deleted_groups.items,
set(["juju-moon-0", "juju-moon-2"]))
def test_remove_security_groups_takes_too_long(self):
"""Verify that removing security groups doesn't continue indefinitely.
This scenario might happen if the instance is taking too long
to report as going into the terminated state, although not
likely after 1000 polls!
"""
log = self.capture_logging()
states = [["shutting-down", "shutting-down"],
["shutting-down", "shutting-down"],
["shutting-down", "terminated"]]
keeps_on_going = [["shutting-down"] for i in xrange(1000)]
states.extend(keeps_on_going)
system_state = MockInstanceState(
self, ["i-amkillable", "i-amkillabletoo"], [0, 2], states)
self.ec2.describe_instances("i-amkillable", "i-amkillabletoo")
self.mocker.call(system_state.get_round)
self.mocker.count(3)
# Keep the rounds going until it exits with the error message
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.mocker.count(197)
# Verify that at least this machine security group was deleted
deleted_groups = Observed()
self.ec2.delete_security_group("juju-moon-2")
self.mocker.call(deleted_groups.add)
self.mocker.replay()
provider = self.get_provider()
yield remove_security_groups(
provider, ["i-amkillable", "i-amkillabletoo"])
self.assertEquals(deleted_groups.items, set(["juju-moon-2"]))
self.assertIn(
"Instance shutdown taking too long, "
"could not delete groups juju-moon-0",
log.getvalue())
def test_remove_security_groups_takes_too_long(self):
"""Verify that removing security groups doesn't continue indefinitely.
This scenario might happen if the instance is taking too long
to report as going into the terminated state, although not
likely after 1000 polls!
"""
log = self.capture_logging()
states = [["shutting-down", "shutting-down"],
["shutting-down", "shutting-down"],
["shutting-down", "terminated"]]
keeps_on_going = [["shutting-down"] for i in xrange(1000)]
states.extend(keeps_on_going)
system_state = MockInstanceState(self,
["i-amkillable", "i-amkillabletoo"],
[0, 2], states)
self.ec2.describe_instances("i-amkillable", "i-amkillabletoo")
self.mocker.call(system_state.get_round)
self.mocker.count(3)
# Keep the rounds going until it exits with the error message
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.mocker.count(197)
# Verify that at least this machine security group was deleted
deleted_groups = Observed()
self.ec2.delete_security_group("juju-moon-2")
self.mocker.call(deleted_groups.add)
self.mocker.replay()
provider = self.get_provider()
yield remove_security_groups(provider,
["i-amkillable", "i-amkillabletoo"])
self.assertEquals(deleted_groups.items, set(["juju-moon-2"]))
self.assertIn(
"Instance shutdown taking too long, "
"could not delete groups juju-moon-0", log.getvalue())
def test_machine_not_in_machine_security_group(self):
"""Old environments do not have machines in security groups.
TODO It might be desirable to change this behavior to log as an
error, or even raise as a provider exception.
"""
log = self.capture_logging()
# Instance is not in a machine security group, just its env
# security group (juju-moon)
self.ec2.describe_instances("i-amkillable")
self.mocker.result(succeed([
self.get_instance("i-amkillable", "terminated", groups=[])]))
self.mocker.replay()
provider = self.get_provider()
yield remove_security_groups(provider, ["i-amkillable"])
self.assertIn(
"Ignoring missing machine security group for instance "
"'i-amkillable'",
log.getvalue())
def test_machine_not_in_machine_security_group(self):
"""Old environments do not have machines in security groups.
TODO It might be desirable to change this behavior to log as an
error, or even raise as a provider exception.
"""
log = self.capture_logging()
# Instance is not in a machine security group, just its env
# security group (juju-moon)
self.ec2.describe_instances("i-amkillable")
self.mocker.result(
succeed(
[self.get_instance("i-amkillable", "terminated", groups=[])]))
self.mocker.replay()
provider = self.get_provider()
yield remove_security_groups(provider, ["i-amkillable"])
self.assertIn(
"Ignoring missing machine security group for instance "
"'i-amkillable'", log.getvalue())
def test_cannot_delete_machine_security_group(self):
"""Verify this raises error.
Note it is possible for a malicious admin to start other
machines with an existing machine security group, so need to
test this removal is robust in this case. Another scenario
that might cause this is that there's an EC2 timeout.
"""
system_state = MockInstanceState(
self, ["i-amkillable"], [0],
[["shutting-down"],
["shutting-down"],
["shutting-down"],
["terminated"]])
self.ec2.describe_instances("i-amkillable")
self.mocker.call(system_state.get_round)
self.mocker.count(4)
self.ec2.delete_security_group("juju-moon-0")
self.mocker.result(fail(
self.get_ec2_error(
"juju-moon-0",
format="There are active instances using security group %r"
)))
self.mocker.replay()
provider = self.get_provider()
ex = yield self.assertFailure(
remove_security_groups(provider, ["i-amkillable"]),
ProviderInteractionError)
self.assertEquals(
str(ex),
"EC2 error when attempting to delete group juju-moon-0: "
"Error Message: There are active instances using security group "
"'juju-moon-0'")
