def test_invalid_source_resource(self):
""" Tests that NonRecoverableError: Instance NotFound is
raised when a source instance that is not in the user's
EC2 account is provided to the detach function NEW
"""
ctx = self.mock_relationship_context('test_invalid_source_resource')
current_ctx.set(ctx=ctx)
ctx.target.instance.runtime_properties['aws_resource_id'] = '0.0.0.0'
ctx.source.instance.runtime_properties['public_ip_address'] = '0.0.0.0'
with mock.patch(
'cloudify_aws.base.AwsBase.execute') \
as mock_execute:
mock_execute.side_effect = EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'InvalidInstanceID.NotFound')
with mock.patch(
'boto.ec2.connection.EC2Connection.get_all_instances') \
as mock_get_all_instances:
with self.assertRaisesRegexp(
EC2ResponseError,
'InvalidInstanceID.NotFound'):
mock_get_all_instances.side_effect = EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'InvalidInstanceID.NotFound')
output = elasticip.ElasticIPInstanceConnection()\
.get_source_resource()
self.assertIsNone(output)
def test_ec2_terminate_instances_keeps_running(self):
"""
Test out case where all instances keep running.
boto.terminate_instances throws InstanceIDInvalid.NotFound
We recover and keep waiting for the other 2 instances, which keep running
and do *not* terminate.
Will take about 120 seconds to run (timeout)
Should return False
"""
ec2 = self.factory.create_agent('ec2')
instance_ids = ['i-aabbccdd', 'i-aabbccee', 'i-aabbccff']
self.full_params['instance_ids'] = instance_ids
# First instance ID doesn't exist, second call will succeed
(self.fake_ec2.should_receive('terminate_instances')
.and_raise(EC2ResponseError(400, 'no reason',
self.instance_notfound_body.format(instance_ids[0])))
.and_return(True))
# _wait_for_status should return true.
(self.fake_ec2.should_receive('get_all_reservations')
.and_return(self.empty_reservations))
instance_ids = set(self.full_params[ec2.PARAM_INSTANCE_IDS])
status_filters = {"instance-state-name": 'terminated',
"key-name": self.full_params[ec2.PARAM_KEYNAME]}
conn = ec2.open_connection(self.full_params)
result = ec2._EC2Agent__terminate_instances(instance_ids, conn, status_filters, max_attempts=1)
self.assertFalse(result)
def test_ec2_terminate_instances_all_not_found(self):
"""
Test out all not found, should return true
"""
# Uncomment to get logging from the agent, helpful for debugging
logging.basicConfig(level=logging.DEBUG)
l = logging.getLogger('appscale.agents.ec_agent')
l.setLevel(logging.DEBUG)
ec2 = self.factory.create_agent('ec2')
instance_ids = ['i-aabbccdd', 'i-aabbccee', 'i-aabbccff']
self.full_params['instance_ids'] = instance_ids
# First instance ID doesn't exist, second call will succeed
(self.fake_ec2.should_receive('terminate_instances')
.and_raise(EC2ResponseError(400, 'no reason',
self.multiple_instance_notfound_body.format(instance_ids[:])))
.and_return(True))
# _wait_for_status should return true.
(self.fake_ec2.should_receive('get_all_reservations')
.and_return(self.terminated_reservations))
instance_ids = set(self.full_params[ec2.PARAM_INSTANCE_IDS])
status_filters = {"instance-state-name": 'terminated',
"key-name": self.full_params[ec2.PARAM_KEYNAME]}
conn = ec2.open_connection(self.full_params)
result = ec2._EC2Agent__terminate_instances(instance_ids, conn, status_filters, max_attempts=1)
self.assertTrue(result)
def test_ec2_wait_for_status_change_keeps_on_running(self):
"""
Case where all instances keep running and never reach 'terminated'
Should return False
"""
filters = {'instance-state-name': 'terminated',
'key-name': self.full_params['keyname']}
ec2 = self.factory.create_agent('ec2')
conn = ec2.open_connection(self.full_params)
# First element is missing
instance_ids = ['i-aabbccdd', 'i-aabbccee', 'i-aabbccff']
(self.fake_ec2.should_receive('get_all_reservations')
.and_raise(EC2ResponseError(400, 'no reason',
self.instance_notfound_body.format(instance_ids[1])))
.and_return(self.empty_reservations)
.and_return(self.empty_reservations)
.and_return(self.empty_reservations)
.and_return(self.empty_reservations)
)
# Last two won't reach terminated state
result = ec2.wait_for_status_change(instance_ids, conn, filters, max_wait_time=3, poll_interval=1)
self.assertFalse(result)
def test_add_tags_created_tg(self):
""" Tests if add tags is called for new tg"""
self.elb2.describe_target_groups.side_effect = EC2ResponseError(
status="mockstatus", reason="mockreason")
self.elb2.create_target_group.return_value = {
'TargetGroups': [{
'TargetGroupArn': "fake-tg-arn"
}]
}
self.disco_elb.get_or_create_target_group(
environment=TEST_ENV_NAME,
hostclass=TEST_HOSTCLASS,
vpc_id=TEST_VPC_ID,
tags={"fake-key": "fake-value"})
self.elb2.create_target_group.assert_called_with(
Name="unittestenv-mhcunit",
Protocol='HTTP',
Port=80,
VpcId=TEST_VPC_ID,
HealthCheckProtocol="HTTP",
HealthCheckPort="80",
HealthCheckPath="/")
self.elb2.add_tags.assert_called_with(ResourceArns=["fake-tg-arn"],
Tags=[{
'Key': "fake-key",
'Value': "fake-value"
}])
def test_disassociate_response_error(self, *_):
"""this checks that disassociate raises an error
when disassociate_address fails due
to response error.
"""
ctx = self.mock_relationship_context(
'test_disassociate_response_error')
current_ctx.set(ctx=ctx)
address = self.get_address()
with mock.patch(
'cloudify_aws.ec2.elasticip.ElasticIPInstanceConnection'
'.get_target_resource') \
as mock_get_target_resource:
mock_get_target_resource.return_value = \
self.get_client().get_all_addresses(addresses=[address])
with mock.patch(
'cloudify_aws.base.AwsBase.execute') \
as mock_execute_disassociate_address:
mock_execute_disassociate_address.side_effect = \
EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'error')
ex = self.assertRaises(
NonRecoverableError,
elasticip.disassociate)
self.assertIn(
'error', ex.message)
def test_create_target_group_ssl(self):
"""Test creating a group that is SSL"""
self.elb2.describe_target_groups.side_effect = EC2ResponseError(
status="mockstatus", reason="mockreason")
instance_protocols = ('SSL', )
instance_ports = (80, )
elb_protocols = ('SSL', )
elb_ports = (80, )
self.disco_elb.get_or_create_target_group(
environment=TEST_ENV_NAME,
hostclass=TEST_HOSTCLASS,
vpc_id=TEST_VPC_ID,
port_config=DiscoELBPortConfig([
DiscoELBPortMapping(internal_port, internal_protocol,
external_port, external_protocol)
for (internal_port,
internal_protocol), (external_port, external_protocol) in
zip(zip(instance_ports, instance_protocols),
zip(elb_ports, elb_protocols))
]))
self.elb2.create_target_group.assert_called_with(
Name="unittestenv-mhcunit",
Protocol='TLS',
Port=80,
VpcId=TEST_VPC_ID,
HealthCheckProtocol="TCP",
HealthCheckPort="80")
def test_ec2_run_instances(self):
self.run_instances('ec2', True)
self.run_instances('ec2', False)
e = EC2ResponseError('Error', 'Mock error')
e.error_message = 'Mock error'
self.fake_ec2.should_receive('run_instances').and_raise(e)
self.run_instances('ec2', True, False)
self.run_instances('ec2', False, False)
def get_image(self, image_id, retry=False):
image = self.images.get(image_id)
if image:
return image
else:
e = EC2ResponseError(None, None)
e.error_code = 'InvalidAMIID.NotFound'
raise e
def test_ec2_run_instances(self):
self.run_instances('ec2', True)
self.run_instances('ec2', False)
e = EC2ResponseError('Error', 'Mock error')
e.error_message = 'Mock error'
(flexmock(EC2Connection).should_receive('run_instances').and_raise(e))
self.run_instances('ec2', True, False)
self.run_instances('ec2', False, False)
def test_smoketest_once_no_instance(self, mock_config, **kwargs):
'''smoketest_once Converts instance not found to TimeoutError'''
aws = DiscoAWS(config=mock_config, environment_name=TEST_ENV_NAME)
self.instance.update = MagicMock(side_effect=EC2ResponseError(
400, "Bad Request",
body={
"RequestID": "df218052-63f2-4a11-820f-542d97d078bd",
"Error": {"Code": "InvalidInstanceID.NotFound", "Message": "test"}}))
self.assertRaises(TimeoutError, aws.smoketest_once, self.instance)
def test_smoketest_once_passes_exception(self, mock_config, **kwargs):
'''smoketest_once passes random EC2ResponseErrors'''
aws = DiscoAWS(config=mock_config, environment_name=TEST_ENV_NAME)
self.instance.update = MagicMock(side_effect=EC2ResponseError(
400, "Bad Request",
body={
"RequestID": "df218052-63f2-4a11-820f-542d97d078bd",
"Error": {"Code": "Throttled", "Message": "test"}}))
self.assertRaises(EC2ResponseError, aws.smoketest_once, self.instance)
def run_instance_callback(args):
if args.image_id == encryptor_image.id:
self.call_count += 1
if self.call_count < 3:
# Simulate eventual consistency error while creating
# security group.
e = EC2ResponseError(None, None)
e.error_code = 'InvalidGroup.NotFound'
raise e
def _fail_for_n_calls(self, n, status=400):
""" Raise EC2ResponseError the first n times that the method is
called.
"""
self.num_calls += 1
if self.num_calls <= n:
e = EC2ResponseError(status, None)
e.error_code = 'InvalidInstanceID.NotFound'
raise e
def test_wait_for_state_ec2error(self, mock_sleep, mock_resource):
"""Test wait_for_state using EC2ResponseError and timeout"""
setattr(mock_resource, 'status', 'mystatus')
mock_resource.update.side_effect = EC2ResponseError("mystatus", "test")
self.assertRaises(TimeoutError,
wait_for_state,
mock_resource,
'available',
state_attr='status',
timeout=30)
def _get_first_element(list, error_status):
""" Return the first element in the list. If the list is empty, raise
an EC2ResponseError with the given error status. This is a workaround
for the case where the AWS API erroneously returns an empty list instead
of an error.
"""
if list:
return list[0]
else:
raise EC2ResponseError(
error_status, 'AWS API returned an empty response')
def test_wait_for_state_boto3_ec2error(self, mock_sleep):
"""Test wait_for_state_boto3 with EC2ResponseError and returned Timeout"""
mock_describe_func = MagicMock()
mock_describe_func.side_effect = EC2ResponseError("mystatus", "test")
self.assertRaises(TimeoutError,
wait_for_state_boto3,
mock_describe_func, {"param1": "p1"},
"myresource",
'available',
state_attr='status',
timeout=30)
def test_ec2_wait_for_status_change_all_not_found_terminated(self):
"""
Case where all instances are not found
Should return True
"""
filters = {'instance-state-name': 'terminated',
'key-name': self.full_params['keyname']}
ec2 = self.factory.create_agent('ec2')
conn = ec2.open_connection(self.full_params)
instance_ids = ['i-aabbccdd', 'i-aabbccee', 'i-aabbccff']
# Raise an exception for each instance id in serial
(self.fake_ec2.should_receive('get_all_reservations')
.and_raise(EC2ResponseError(400, "no reason", self.instance_notfound_body.format(instance_ids[0])))
.and_raise(EC2ResponseError(400, "no reason", self.instance_notfound_body.format(instance_ids[1])))
.and_raise(EC2ResponseError(400, "no reason", self.instance_notfound_body.format(instance_ids[2]))))
result = ec2.wait_for_status_change(instance_ids, conn, filters, max_wait_time=5, poll_interval=1)
self.assertTrue(result)
def test_create_route_exists(self):
""" Tests that create_route
raises an error when the
route already exists.
"""
ctx = self.get_mock_ctx('test_create_route_exists')
ctx.node.properties['name'] = 'test_create_route'
current_ctx.set(ctx=ctx)
vpc_client = self.create_vpc_client()
vpc = vpc_client.create_vpc('10.10.10.0/16')
new_route_table = vpc_client.create_route_table(vpc.id)
route_table_id = new_route_table.id
ctx.instance.runtime_properties['aws_resource_id'] = route_table_id
ctx.operation._operation_context['name'] = 'start'
routetable.start_route_table(ctx=ctx)
route = dict(
destination_cidr_block='0.0.0.0/0',
gateway_id=ctx.instance.runtime_properties[
constants.EXTERNAL_RESOURCE_ID]
)
resource = RouteMixin()
resource.client = vpc_client
with mock.patch(
'moto.ec2.models.RouteBackend.create_route') \
as mock_create_route:
mock_create_route.side_effect = \
EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'<Code>RouteAlreadyExists</Code>')
self.assertEqual(True, resource.create_route(route_table_id,
route, ctx.instance))
mock_create_route.side_effect = \
EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'some error')
self.assertRaises(
RecoverableError,
resource.create_route,
route_table_id,
route)
def _process_response(self, response, marker_elems=None):
"""
Helper to process the xml response from AWS
"""
body = response.read()
#print body
if '<Errors>' not in body:
rs = ResultSet(marker_elems)
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
else:
raise EC2ResponseError(response.status, response.reason, body)
def test_ec2_wait_for_status_change_already_terminated(self):
"""
Terminate an instance and the cloud returns InvalidInstanceID.NotFound
This should be reported as a success
"""
filters = {'instance-state-name': 'terminated',
'key-name': self.full_params['keyname']}
ec2 = self.factory.create_agent('ec2')
conn = ec2.open_connection(self.full_params)
instance_ids = ['i-aabbccdd']
self.fake_ec2.should_receive('get_all_reservations').and_raise(
EC2ResponseError(400, "no reason", self.instance_notfound_body.format(instance_ids[0])))
result = ec2.wait_for_status_change(instance_ids, conn, filters, max_wait_time=10, poll_interval=1)
self.assertTrue(result)
def test_create_tg_with_health_check(self):
"""Test creating a group with health check"""
self.elb2.describe_target_groups.side_effect = EC2ResponseError(
status="mockstatus", reason="mockreason")
self.disco_elb.get_or_create_target_group(
environment=TEST_ENV_NAME,
hostclass=TEST_HOSTCLASS,
vpc_id=TEST_VPC_ID,
health_check_path="/mockpath")
self.elb2.create_target_group.assert_called_with(
Name="unittestenv-mhcunit",
Protocol='HTTP',
Port=80,
VpcId=TEST_VPC_ID,
HealthCheckProtocol="HTTP",
HealthCheckPort="80",
HealthCheckPath="/mockpath")
def test_ec2_wait_for_status_change_stopped_not_found(self):
"""
When waiting for the 'stopped' state, if an instance is not found
it is considered to be an error and the method will throw
an InstanceIDNotFound exception.
"""
filters = {'instance-state-name': 'stopped',
'key-name': self.full_params['keyname']}
ec2 = self.factory.create_agent('ec2')
conn = ec2.open_connection(self.full_params)
# Throw an invalid id when stopping an instance. wait_for_status should raise an
# exception.
(self.fake_ec2.should_receive('get_all_reservations')
.and_raise(EC2ResponseError(400, "no reason",
self.instance_notfound_body.format('i-aabbccdd'))))
with self.assertRaises(InstanceIDNotFound):
ec2.wait_for_status_change(['i-aabbccdd'], conn, filters, 5, 1)
def test_ec2_wait_for_status_change_one_not_found_terminated(self):
"""
The first instance id in the list is marked as not found.
Note: self.terminated_reservations does return the instance id, which shouldn't have an impact
"""
filters = {'instance-state-name': 'terminated',
'key-name': self.full_params['keyname']}
ec2 = self.factory.create_agent('ec2')
conn = ec2.open_connection(self.full_params)
instance_ids = ['i-aabbccdd', 'i-aabbccee', 'i-aabbccff']
(self.fake_ec2.should_receive('get_all_reservations')
.and_raise(EC2ResponseError(400, "no reason",
self.instance_notfound_body.format(instance_ids[0])))
.and_return(self.terminated_reservations))
result = ec2.wait_for_status_change(instance_ids, conn, filters, max_wait_time=10, poll_interval=1)
self.assertTrue(result)
def test_allocate_response_error(self):
"""this checks that create raises an error
when release_address fails due
to response error.
"""
ctx = self.mock_ctx('test_allocate_response_error')
current_ctx.set(ctx=ctx)
with mock.patch(
'cloudify_aws.base.AwsBase.execute') \
as mock_execute_release_address:
mock_execute_release_address.side_effect = EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'error')
ex = self.assertRaises(
NonRecoverableError,
elasticip.create)
self.assertIn(
'error', ex.message)
def test_ec2_wait_for_status_change_multiple_not_found_different_order(self):
"""
Multiple instance ids not found, first one is found
Should return True
"""
filters = {'instance-state-name': 'terminated',
'key-name': self.full_params['keyname']}
ec2 = self.factory.create_agent('ec2')
conn = ec2.open_connection(self.full_params)
instance_ids = ['i-aabbccdd', 'i-aabbccee', 'i-aabbccff']
# Raise exception for the first two instance ids.
missing_ids = ','.join(instance_ids[1:])
(self.fake_ec2.should_receive('get_all_reservations')
.and_raise(EC2ResponseError(400, "no reason",
self.multiple_instances_not_found.format(missing_ids)))
.and_return(self.terminated_reservations))
result = ec2.wait_for_status_change(instance_ids, conn, filters, max_wait_time=5, poll_interval=1)
self.assertTrue(result)
def test_invalid_target_resource(self):
""" Tests that NonRecoverableError: Address NotFound is
raised when a target address that is not in the user's
EC2 account is provided to the detach function
"""
ctx = self.mock_relationship_context('test_invalid_target_resource')
current_ctx.set(ctx=ctx)
ctx.target.instance.runtime_properties['aws_resource_id'] = '0.0.0.0'
ctx.source.instance.runtime_properties['public_ip_address'] = '0.0.0.0'
with mock.patch(
'cloudify_aws.base.AwsBase.execute') \
as mock_get_target_resource:
mock_get_target_resource.side_effect = EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'InvalidAddress.NotFound')
ex = self.assertRaises(
NonRecoverableError,
elasticip.disassociate)
self.assertIn(
'no matching elastic ip in account', ex.message)
def test_tag_resource_raises_error(self):
""" Tests that when add_tags fails
tag_resource raises the right error.
"""
ctx = self.get_mock_ctx('test_tag_resource_raises_error')
current_ctx.set(ctx=ctx)
resource = AwsBaseNode('root', [], resource_states=[])
ctx.node.properties['name'] = 'root'
test_resource = TaggedEC2Object()
tags = []
with mock.patch(
'boto.ec2.ec2object.TaggedEC2Object.add_tags') \
as mock_add_tags:
mock_add_tags.side_effect = EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'error')
ex = self.assertRaises(
NonRecoverableError,
resource._tag_resource,
test_resource,
tags)
self.assertIn(
'unable to tag resource name', ex.message)
def test_execute_response_error(self):
""" Tests that execute raises
an error when the execution fails.
"""
ctx = self.get_mock_ctx('test_execute_response_error')
current_ctx.set(ctx=ctx)
ec2_client = connection.EC2ConnectionClient().client()
resource = AwsBase(client=ec2_client)
with mock.patch(
'boto.ec2.connection.EC2Connection.run_instances') \
as mock_run_instances:
mock_run_instances.side_effect = \
EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'error')
ex = self.assertRaises(
NonRecoverableError,
resource.execute,
resource.client.run_instances,
dict(image_id='ami-e214778a',
instance_type='t1.micro'))
self.assertIn(
'error', ex.message)
def test_release_response_error(self):
"""this checks that delete raises an error
when release_address fails due
to response error.
"""
ctx = self.mock_ctx('test_release_response_error')
current_ctx.set(ctx=ctx)
address = self.get_address()
ctx.instance.runtime_properties['aws_resource_id'] = \
address.public_ip
ctx.instance.runtime_properties['allocation_id'] = 'random'
with mock.patch(
'cloudify_aws.base.AwsBase.execute') \
as mock_execute_release_address:
mock_execute_release_address.side_effect = EC2ResponseError(
mock.Mock(return_value={'status': 404}),
'error')
ex = self.assertRaises(
NonRecoverableError,
elasticip.delete)
self.assertIn(
'error', ex.message)