def _fetch_wrapper(self, *args, **kwargs):
"""Wrap the superclass _fetch method to catch known Rollbar errors.
https://rollbar.com/docs/api_overview/
"""
try:
res = yield self._fetch(*args, **kwargs)
except httpclient.HTTPError as e:
# These are HTTPErrors that we know about, and can log specific
# error messages for.
if e.code in (401, 403):
raise exceptions.InvalidCredentials(
'The "ROLLBAR_TOKEN" is invalid')
elif e.code == 422:
raise exceptions.RecoverableActorFailure(
'Unprocessable Entity - the request was parseable (i.e. '
'valid JSON), but some parameters were missing or '
'otherwise invalid.')
elif e.code == 429:
raise exceptions.RecoverableActorFailure(
'Too Many Requests - If rate limiting is enabled for '
'your access token, this return code signifies that the '
'rate limit has been reached and the item was not '
'processed.')
else:
# We ran into a problem we can't handle. Also, keep in mind
# that @utils.retry() was used, so this error happened several
# times before getting here. Raise it.
raise exceptions.RecoverableActorFailure(
'Unexpected error from Rollbar API: %s' % e)
raise gen.Return(res)
def test_get_exc_type_with_both(self):
exc_list = [
exceptions.RecoverableActorFailure(),
exceptions.UnrecoverableActorFailure(),
exceptions.RecoverableActorFailure()
]
actor = group.Async('Unit Test Action', {'acts': []})
ret = actor._get_exc_type(exc_list)
self.assertEquals(ret, exceptions.UnrecoverableActorFailure)
def _is_service_updated(self, service_name, task_definition_name):
"""Checks if service's state updates successfully.
Meant to be called in a wait-loop.
Args:
service_name: name of the service to wait for.
task_definition_name: Task Definition string.
Returns:
A boolean indicating whether the service is completely updated.
"""
service = yield self._describe_service(service_name)
deployments = service['deployments']
primary_deployment = self._get_primary_deployment(service)
if not primary_deployment:
# There should always be one 'PRIMARY' deployment returned.
raise exceptions.RecoverableActorFailure(
'No primary deployment.')
# Verify that the primary deployment has the correct task definition.
if not self._is_task_in_deployment(
primary_deployment, task_definition_name):
raise exceptions.RecoverableActorFailure(
'Primary deployment was for {}, not {}.'.format(
self._arn_to_name(primary_deployment['taskDefinition']),
task_definition_name))
service_timestamp = primary_deployment['updatedAt']
sorted_new_events = self._get_sorted_new_log_events(
events=service['events'],
start_timestamp=service_timestamp)
for event in sorted_new_events:
event_timestamp, event_message = event
self.log.info('Event [{}]: {}'.format(event_timestamp,
event_message))
running_count = primary_deployment['runningCount']
desired_count = primary_deployment['desiredCount']
missing_count = desired_count - running_count
extra_deployment_count = len(deployments) - 1
if missing_count == 0 and extra_deployment_count == 0:
raise gen.Return(True)
self.log.info(
'{} tasks running out of {}, '
'and {} deployments waiting on termination.'.format(
running_count, desired_count, extra_deployment_count))
raise gen.Return(False)
def _create_entity(self, name):
"""Creates an IAM Entity.
If the entity exists, we just warn and move on.
args:
name: The IAM Entity Name
"""
if self._dry:
self.log.warning('Would create %s %s' % (self.entity_name, name))
raise gen.Return()
try:
ret = yield self.thread(self.create_entity, name)
except BotoServerError as e:
if e.status != 409:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
self.log.warning('%s %s already exists, skipping creation.' %
(self.entity_name, name))
raise gen.Return()
arn = (
ret['create_%s_response' %
self.entity_name]['create_%s_result' %
self.entity_name][self.entity_name]['arn'])
self.log.info('%s %s created' % (self.entity_name, arn))
def _get_mci(self, name):
"""Searches RightScale for an MCI by name.
Wrapper around our find_by_name_and_keys() mechanism so that we return
either the proper MCI, or None if one isn't found.
args:
name: MCI name to search for
"""
self.log.debug('Searching for MCI "%s"' % name)
mci = yield self._client.find_by_name_and_keys(
collection=self._client._client.multi_cloud_images,
name=name,
revision=0)
# Default searches return us an empty list if there are no matching
# resources, or return us the exact resource we're looking for. Thus,
# if the return value is not a list, then we know we got the MCI back.
# got a list bac
if not isinstance(mci, list):
raise gen.Return(mci)
# If we got a list back, return None because that means that the MCI
# doesn't already exist.
if isinstance(mci, list) and len(mci) == 0:
raise gen.Return(None)
# On anything else, raise an exception. Something really strange
# happened.
raise exceptions.RecoverableActorFailure(
'Found too many matching MCI images with the same name, '
'this shouldn\'t be possible ... so something bad has happened.')
def _ensure_role(self, name, role):
"""Ensures that an Instance Profile role is set correctly.
Adds, Deletes or Changes the Role assigned to an Instance Profile.
args:
name: The IAM Instance Profile we're managing
role: The desired role (or None)
"""
existing = None
try:
raw = yield self.thread(self.iam_conn.get_instance_profile, name)
existing = (raw['get_instance_profile_response']
['get_instance_profile_result']['instance_profile']
['roles']['member']['role_name'])
except BotoServerError as e:
if e.status != 404:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
except KeyError:
# Profile is not a member of any roles
pass
if not existing and not role:
raise gen.Return()
elif existing and not role:
yield self._remove_role(name, existing)
elif not existing and role:
yield self._add_role(name, role)
elif existing != role:
yield self._remove_role(name, existing)
yield self._add_role(name, role)
def _delete_entity(self, name):
"""Deletes and IAM Entity.
If the entity doesn't exist, we just warn and move on.
args:
name: The IAM Entity Name
"""
if self._dry:
self.log.warning('Would delete %s %s' % (self.entity_name, name))
raise gen.Return()
try:
# Get the entities policies. They have to be deleted before we can
# possibly move forward and delete the entity.
existing_policies = yield self._get_entity_policies(name)
tasks = []
for policy in existing_policies:
tasks.append(self._delete_entity_policy(name, policy))
yield tasks
# Now delete the entity
yield self.thread(self.delete_entity, name)
self.log.info('%s %s deleted' % (self.entity_name, name))
except BotoServerError as e:
if e.status != 404:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
self.log.warning('%s %s doesn\'t exist' % (self.entity_name, name))
def _get_group_users(self, name):
"""Returns a list of users assigned to the group.
args:
name: the name of the group
returns:
a list of user name strings
"""
users = []
try:
raw = yield self.thread(self.iam_conn.get_group, name)
users = [
user['user_name'] for user in raw['get_group_response']
['get_group_result']['users']
]
except BotoServerError as e:
if e.status != 404:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
except KeyError:
# No users!
users = []
raise gen.Return(users)
def _execute(self):
"""Executes an actor and yields the results when its finished.
raises: gen.Return()
"""
self.log.info('Sending message "%s" to Hipchat room "%s"' %
(self.option('message'), self.option('room')))
res = yield self._post_message(self.option('room'),
self.option('message'))
# If we get 'None' or 'False' back, the actor failed.
if not res:
raise exceptions.RecoverableActorFailure(
'Failed to send message to HipChat: %s' % res)
# If we got here, the result is supposed to include 'success' as a key
# and inside that key we can dig for the actual message. If the
# response code is 202, we know that we didn't actually execute the
# message send, but just validated the API token against the API.
if 'success' in res:
if res['success']['code'] == 202:
self.log.info('API Token Validated: %s' %
res['success']['message'])
raise gen.Return()
def thread(self, function, *args, **kwargs):
"""Execute `function` in a concurrent thread.
Example:
>>> zones = yield thread(ec2_conn.get_all_zones)
This allows execution of any function in a thread without having
to write a wrapper method that is decorated with run_on_executor()
"""
try:
return function(*args, **kwargs)
except boto_exception.BotoServerError as e:
# If we're using temporary IAM credentials, when those expire we
# can get back a blank 400 from Amazon. This is confusing, but it
# happens because of https://github.com/boto/boto/issues/898. In
# most cases, these temporary IAM creds can be re-loaded by
# reaching out to the AWS API (for example, if we're using an IAM
# Instance Profile role), so thats what Boto tries to do. However,
# if you're using short-term creds (say from SAML auth'd logins),
# then this fails and Boto returns a blank 400.
if (e.status == 400 and e.reason == 'Bad Request'
and e.error_code is None):
msg = 'Access credentials have expired'
raise exceptions.InvalidCredentials(msg)
msg = '%s: %s' % (e.error_code, e.message)
if e.status == 403:
raise exceptions.InvalidCredentials(msg)
raise
except boto3_exceptions.Boto3Error as e:
raise exceptions.RecoverableActorFailure(
'Boto3 had a failure: %s' % e)
def _delete_bucket(self):
bucket = self.option('name')
try:
self.log.info('Deleting bucket %s' % bucket)
yield self.api_call(self.s3_conn.delete_bucket, Bucket=bucket)
except ClientError as e:
raise exceptions.RecoverableActorFailure(
'Cannot delete bucket: %s' % e.message)
def __init__(self, *args, **kwargs):
super(ECSBaseActor, self).__init__(*args, **kwargs)
count = self.option('count')
if type(count) is str or type(count) is unicode:
try:
self._options['count'] = int(count)
except ValueError:
raise exceptions.RecoverableActorFailure(
'Could not parse option \'count\' as int: %s' % count)
def _create_spec(self):
try:
self.existing_spec = yield self._client.create_resource(
self._client._client.alert_specs, self.desired_params)
except requests.exceptions.HTTPError as e:
if e.response.status_code in (422, 400):
msg = ('Invalid parameters supplied to Alert Spec "%s": %s'
% (self.option('href'), self.desired_params))
raise exceptions.RecoverableActorFailure(msg)
raise
self.log.info('Alert spec has been created')
def _verify_can_delete_bucket(self):
# Find out if there are any files in the bucket before we go to delete
# it. We cannot delete a bucket with files in it -- nor do we want to.
bucket = self.option('name')
keys = yield self.api_call(self.s3_conn.list_objects, Bucket=bucket)
if 'Contents' not in keys:
raise gen.Return()
if len(keys['Contents']) > 0:
raise exceptions.RecoverableActorFailure(
'Cannot delete bucket with keys: %s files found' % len(keys))
def _get_entity(self, name):
"""Returns an IAM Entity JSON Blob.
Searches for an IAM Entity and either returns None, or a JSON blob that
describes the Entity.
args:
name: The IAM Entity Name
"""
self.log.debug('Searching for %s %s' % (self.entity_name, name))
# Get a list of all of our entities.
try:
entities = yield self.thread(self.get_all_entities)
except BotoServerError as e:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
# Now search for the entity
entity = [
entity
for entity in entities['list_%ss_response' % self.entity_name][
'list_%ss_result' % self.entity_name]['%ss' % self.entity_name]
if entity['%s_name' % self.entity_name] == name
]
# If there aren't any entities, return None.
if not entity:
raise gen.Return()
# If there is more than one entities, something went really wrong.
# Raise an exception.
if len(entity) > 1:
raise exceptions.RecoverableActorFailure(
'More than one %s found matching %s! Am I crazy?!' %
(self.entity_name, name))
# Finally, return the result!
self.log.debug('Found %s %s' % (self.entity_name, entity[0]['arn']))
raise gen.Return(entity[0])
def _get_entity(self, name):
"""Returns an IAM Entity JSON Blob.
Searches for an IAM Entity and either returns None, or a JSON blob that
describes the Entity.
args:
name: The IAM Entity Name
"""
self.log.debug('Searching for %s %s' % (self.entity_name, name))
# Get a list of all of the entities - return 100 results at a time, and
# paginate the results.
is_truncated = True
marker = None
while is_truncated:
# Get the list back - if the marker has been set, then we pass it
# in and we start from where the last results told us we should.
try:
response = yield self.api_call(self.get_all_entities,
max_items=MAX_ITEMS,
marker=marker)
except BotoServerError as e:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
# Get the result object from the response...
result = (response['list_%ss_response' %
self.entity_name]['list_%ss_result' %
self.entity_name])
# If the results indicate they were truncated, they'll include
# a 'marker'. Setting these two variables will cause this to
# loop again, in the event that we don't find the response in
# the first set of results.
is_truncated = self.str2bool(result.get('is_truncated', False))
marker = result.get('marker', None)
# Check our result for the entity.. if its there, great.
# Otherwise, we'll move on.
entity = [
entity for entity in result['%ss' % self.entity_name]
if entity['%s_name' % self.entity_name] == name
]
if len(entity) > 0:
self.log.debug('Found %s %s' %
(self.entity_name, entity[0]['arn']))
raise gen.Return(entity[0])
# If there aren't any entities, return None.
raise gen.Return()
def _execute(self):
# Find the array we're adding an alert spec to. Specifically, we need
# the servers HREF.
array = yield self._find_server_arrays(
self.option('array'),
raise_on=self._array_raise_on,
allow_mock=self._array_allow_mock)
self.log.info('Found %s (%s)' % (array.soul['name'], array.href))
# Add all of the required parameters to a dictionary
params = {
'condition': self.option('condition'),
'description': self.option('description'),
'duration': int(self.option('duration')),
'file': self.option('file'),
'name': self.option('name'),
'subject_href': array.href,
'threshold': self.option('threshold'),
'variable': self.option('variable'),
}
# Generate the RightScale parameters that we need to pass in when
# creating the alert. The optional parameters should not be passed in
# if their option value came in as None.
_optional_params = [
'description', 'escalation_name', 'vote_tag', 'vote_type'
]
for optional in _optional_params:
if self.option(optional):
params[optional] = self.option(optional)
params = self._generate_rightscale_params('alert_spec', params)
self.log.debug('Generated params: %s' % params)
if self._dry:
# In dry run mode, just log out what we would have done.
self.log.info('Would have created the alert spec \"%s\" on %s' %
(self.option('name'), array.soul['name']))
raise gen.Return()
# We're really doin this. If we get a known exception back, handle
# it. Otherwise, raise it.
try:
yield self._client.create_resource(
self._client._client.alert_specs, params)
self.log.info('Alert spec has been created')
except requests.exceptions.HTTPError as e:
if e.response.status_code in (422, 400):
msg = ('Invalid parameters supplied to Alert Spec "%s": %s'
% (self.option('name'), params))
raise exceptions.RecoverableActorFailure(msg)
raise
def _update_spec(self):
try:
self.existing_spec = yield self._client.update(
self.existing_spec, self.desired_params)
except requests.exceptions.HTTPError as e:
if e.response.status_code in (422, 400):
msg = ('Invalid parameters supplied to Alert Spec "%s": %s'
% (self.existing_spec.soul['name'],
self.desired_params))
raise exceptions.RecoverableActorFailure(msg)
raise
self.log.info('Alert spec has been updated')
def _push_policy(self):
self.log.info('Pushing bucket policy %s' % self.option('policy'))
self.log.debug('Policy doc: %s' % self.policy)
try:
yield self.api_call(self.s3_conn.put_bucket_policy,
Bucket=self.option('name'),
Policy=json.dumps(self.policy))
except ClientError as e:
if 'MalformedPolicy' in e.message:
raise base.InvalidPolicy(e.message)
raise exceptions.RecoverableActorFailure(
'An unexpected error occurred: %s' % e)
def _use_cert(self, elb, arn):
"""Assign an ssl cert to a given ELB.
Args:
elb: boto elb object.
arn: ARN for server certificate to use.
"""
self.log.info('Setting ELB "%s" to use cert arn: %s' % (elb, arn))
try:
yield self.thread(
elb.set_listener_SSL_certificate, self.option('port'), arn)
except BotoServerError as e:
raise exceptions.RecoverableActorFailure(
'Applying new SSL cert to %s failed: %s' % (elb, e))
def _add_user_to_group(self, name, group):
"""Quick helper method to add a user to a group.
args:
name: user name
group: group name
"""
if self._dry:
self.log.warning('Would have added %s to %s' % (name, group))
raise gen.Return()
try:
self.log.info('Adding %s to %s' % (name, group))
yield self.thread(self.iam_conn.add_user_to_group, group, name)
except BotoServerError as e:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
def _remove_user_from_group(self, name, group):
"""Quick helper method to remove a user from a group.
args:
name: user name
group: group name
"""
if self._dry:
self.log.warning('Would have removed %s from %s' % (name, group))
raise gen.Return()
try:
self.log.info('Removing %s from %s' % (name, group))
yield self.api_call(self.iam_conn.remove_user_from_group, group,
name)
except BotoServerError as e:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
def _remove_role(self, name, role):
"""Removes a role assigned to an Instance Profile.
args:
name: The name of the InstanceProfile we're managing
role: The name of the role to remove
"""
if self._dry:
self.log.warning('Would remove role %s from %s' % (role, name))
raise gen.Return()
try:
self.log.info('Removing role %s from %s' % (role, name))
yield self.api_call(
self.iam_conn.remove_role_from_instance_profile, name, role)
except BotoServerError as e:
if e.status != 404:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
def _add_role(self, name, role):
"""Adds a role to an Instance Profile.
args:
name: The name of the Instance Profile we're managing
role: The name of the role to assign to the profile
"""
if self._dry:
self.log.warning('Would add role %s from %s' % (role, name))
raise gen.Return()
try:
self.log.info('Adding role %s to %s' % (role, name))
yield self.thread(self.iam_conn.add_role_to_instance_profile, name,
role)
except BotoServerError as e:
if e.status != 409:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
def _ensure_service_present(self, service_name, existing_service):
"""Ensures a service is present.
Args:
service_name: service name to use.
existing_service:
Result of _describe_service on the given service name.
"""
service_is_active = (existing_service and
existing_service['status'] != 'INACTIVE')
if not service_is_active and not self.task_definition:
# Without a task definition,
# we can only update an existing service.
self.log.error(
'Could not find service with name {} to update '
'in {}.'.format(
service_name, self._format_location()))
raise exceptions.RecoverableActorFailure(
'Cannot update non-existant service.')
if service_is_active:
# Service already exists - update it.
# Ensure no immutable fields were mutated.
self._check_immutable_field_errors(
old_params=existing_service,
new_params=self.service_definition,
immutable_fields=['loadBalancers', 'role'])
task_definition_name = yield self._update_service(service_name,
existing_service)
else:
task_definition_name = yield self._create_service(service_name)
if self.option('wait'):
yield self._wait_for_service_update(
service_name, task_definition_name)
self.log.info(
'Service {} updated successfully in {}.'.format(
service_name, self._format_location()))
else:
self.log.info(
'Not waiting for service {} to finish updating '
'in {}.'.format(
service_name, self._format_location()))
def _ensure_groups(self, name, groups):
"""Ensure that this user is a member of specific groups.
args:
name: The user we're managing
groups: The list (or single) of groups to join be members of
"""
if isinstance(groups, str):
groups = [groups]
current_groups = set()
try:
res = yield self.api_call(self.iam_conn.get_groups_for_user, name)
current_groups = {
g['group_name']
for g in res['list_groups_for_user_response']
['list_groups_for_user_result']['groups']
}
except BotoServerError as e:
# If the error is a 404, then the user doesn't exist and we can
# assume that the mappings don't exist at all. We leave the
# existin_mappings list alone. For any other error, raise.
if e.status != 404:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
# Find any groups that we're not already a member of, and add us
tasks = []
try:
for new_group in set(groups) - current_groups:
tasks.append(self._add_user_to_group(name, new_group))
except StopIteration:
pass
yield tasks
# Find any group memberships we didn't know about, and purge them
tasks = []
for bad_group in current_groups - set(groups):
tasks.append(self._remove_user_from_group(name, bad_group))
yield tasks
def _check_immutable_field_errors(self, old_params, new_params,
immutable_fields):
"""Compares an old service definition to a new one
to ensure that all of the specified immutable fields
are the same between them.
If there are any errors,
this logs them and raises RecoverableActorFailure.
Args:
old_params: old parameters to use.
new_params: new parameters to compare with.
immutable_fields: list of immutable fields.
"""
# API does not return role name, only the role arn.
role_arn = old_params.get('roleArn')
role = None
if role_arn:
role = self._arn_to_name(role_arn)
old_params['role'] = role
has_error = False
for immutable_field_name in immutable_fields:
new_field = new_params.get(immutable_field_name)
old_field = old_params.get(immutable_field_name)
if new_field != old_field:
has_error = True
self.log.error(
"Field \'{field}\' cannot be updated.\n"
'Old value: {old}\n'
'New value: {new}'.format(
field=immutable_field_name,
old=old_field,
new=new_field))
if has_error:
raise exceptions.RecoverableActorFailure(
'Immutable fields cannot be updated. '
'A new service must be created.')
def _wrap_boto_exception(self, e):
if isinstance(e, boto_exception.BotoServerError):
# If we're using temporary IAM credentials, when those expire we
# can get back a blank 400 from Amazon. This is confusing, but it
# happens because of https://github.com/boto/boto/issues/898. In
# most cases, these temporary IAM creds can be re-loaded by
# reaching out to the AWS API (for example, if we're using an IAM
# Instance Profile role), so thats what Boto tries to do. However,
# if you're using short-term creds (say from SAML auth'd logins),
# then this fails and Boto returns a blank 400.
if (e.status == 400 and e.reason == 'Bad Request'
and e.error_code is None):
msg = 'Access credentials have expired'
return exceptions.InvalidCredentials(msg)
msg = '%s: %s' % (e.error_code, str(e))
if e.status == 403:
return exceptions.InvalidCredentials(msg)
elif isinstance(e, boto3_exceptions.Boto3Error):
return exceptions.RecoverableActorFailure(
'Boto3 had a failure: %s' % e)
return e
def _delete_entity_policy(self, name, policy_name):
"""Optionally pushes a policy to an IAM entity.
args:
name: The IAM Entity Name
policy_name: The entity policy name
"""
if self._dry:
self.log.warning('Would delete policy %s from %s %s' %
(policy_name, self.entity_name, name))
raise gen.Return()
self.log.info('Deleting policy %s from %s %s' %
(policy_name, self.entity_name, name))
try:
ret = yield self.thread(self.delete_entity_policy, name,
policy_name)
self.log.debug('Policy %s deleted: %s' % (policy_name, ret))
except BotoServerError as e:
if e.error_code != 404:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
def _put_entity_policy(self, name, policy_name, policy_doc):
"""Optionally pushes a policy to an IAM Entity.
args:
name: The IAM Entity Name
policy_name: The entity policy name
policy_doc: The ploicy document object itself
"""
if self._dry:
self.log.warning('Would push policy %s to %s %s' %
(policy_name, self.entity_name, name))
raise gen.Return()
self.log.info('Pushing policy %s to %s %s' %
(policy_name, self.entity_name, name))
try:
ret = yield self.thread(self.put_entity_policy, name, policy_name,
json.dumps(policy_doc))
self.log.debug('Policy %s pushed: %s' % (policy_name, ret))
except BotoServerError as e:
raise exceptions.RecoverableActorFailure(
'An unexpected API error occurred: %s' % e)
