def _fetch_wrapper(self, *args, **kwargs): """Wrap the superclass _fetch method to catch known Rollbar errors. https://rollbar.com/docs/api_overview/ """ try: res = yield self._fetch(*args, **kwargs) except httpclient.HTTPError as e: # These are HTTPErrors that we know about, and can log specific # error messages for. if e.code in (401, 403): raise exceptions.InvalidCredentials( 'The "ROLLBAR_TOKEN" is invalid') elif e.code == 422: raise exceptions.RecoverableActorFailure( 'Unprocessable Entity - the request was parseable (i.e. ' 'valid JSON), but some parameters were missing or ' 'otherwise invalid.') elif e.code == 429: raise exceptions.RecoverableActorFailure( 'Too Many Requests - If rate limiting is enabled for ' 'your access token, this return code signifies that the ' 'rate limit has been reached and the item was not ' 'processed.') else: # We ran into a problem we can't handle. Also, keep in mind # that @utils.retry() was used, so this error happened several # times before getting here. Raise it. raise exceptions.RecoverableActorFailure( 'Unexpected error from Rollbar API: %s' % e) raise gen.Return(res)
def test_get_exc_type_with_both(self): exc_list = [ exceptions.RecoverableActorFailure(), exceptions.UnrecoverableActorFailure(), exceptions.RecoverableActorFailure() ] actor = group.Async('Unit Test Action', {'acts': []}) ret = actor._get_exc_type(exc_list) self.assertEquals(ret, exceptions.UnrecoverableActorFailure)
def _is_service_updated(self, service_name, task_definition_name): """Checks if service's state updates successfully. Meant to be called in a wait-loop. Args: service_name: name of the service to wait for. task_definition_name: Task Definition string. Returns: A boolean indicating whether the service is completely updated. """ service = yield self._describe_service(service_name) deployments = service['deployments'] primary_deployment = self._get_primary_deployment(service) if not primary_deployment: # There should always be one 'PRIMARY' deployment returned. raise exceptions.RecoverableActorFailure( 'No primary deployment.') # Verify that the primary deployment has the correct task definition. if not self._is_task_in_deployment( primary_deployment, task_definition_name): raise exceptions.RecoverableActorFailure( 'Primary deployment was for {}, not {}.'.format( self._arn_to_name(primary_deployment['taskDefinition']), task_definition_name)) service_timestamp = primary_deployment['updatedAt'] sorted_new_events = self._get_sorted_new_log_events( events=service['events'], start_timestamp=service_timestamp) for event in sorted_new_events: event_timestamp, event_message = event self.log.info('Event [{}]: {}'.format(event_timestamp, event_message)) running_count = primary_deployment['runningCount'] desired_count = primary_deployment['desiredCount'] missing_count = desired_count - running_count extra_deployment_count = len(deployments) - 1 if missing_count == 0 and extra_deployment_count == 0: raise gen.Return(True) self.log.info( '{} tasks running out of {}, ' 'and {} deployments waiting on termination.'.format( running_count, desired_count, extra_deployment_count)) raise gen.Return(False)
def _create_entity(self, name): """Creates an IAM Entity. If the entity exists, we just warn and move on. args: name: The IAM Entity Name """ if self._dry: self.log.warning('Would create %s %s' % (self.entity_name, name)) raise gen.Return() try: ret = yield self.thread(self.create_entity, name) except BotoServerError as e: if e.status != 409: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) self.log.warning('%s %s already exists, skipping creation.' % (self.entity_name, name)) raise gen.Return() arn = ( ret['create_%s_response' % self.entity_name]['create_%s_result' % self.entity_name][self.entity_name]['arn']) self.log.info('%s %s created' % (self.entity_name, arn))
def _get_mci(self, name): """Searches RightScale for an MCI by name. Wrapper around our find_by_name_and_keys() mechanism so that we return either the proper MCI, or None if one isn't found. args: name: MCI name to search for """ self.log.debug('Searching for MCI "%s"' % name) mci = yield self._client.find_by_name_and_keys( collection=self._client._client.multi_cloud_images, name=name, revision=0) # Default searches return us an empty list if there are no matching # resources, or return us the exact resource we're looking for. Thus, # if the return value is not a list, then we know we got the MCI back. # got a list bac if not isinstance(mci, list): raise gen.Return(mci) # If we got a list back, return None because that means that the MCI # doesn't already exist. if isinstance(mci, list) and len(mci) == 0: raise gen.Return(None) # On anything else, raise an exception. Something really strange # happened. raise exceptions.RecoverableActorFailure( 'Found too many matching MCI images with the same name, ' 'this shouldn\'t be possible ... so something bad has happened.')
def _ensure_role(self, name, role): """Ensures that an Instance Profile role is set correctly. Adds, Deletes or Changes the Role assigned to an Instance Profile. args: name: The IAM Instance Profile we're managing role: The desired role (or None) """ existing = None try: raw = yield self.thread(self.iam_conn.get_instance_profile, name) existing = (raw['get_instance_profile_response'] ['get_instance_profile_result']['instance_profile'] ['roles']['member']['role_name']) except BotoServerError as e: if e.status != 404: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) except KeyError: # Profile is not a member of any roles pass if not existing and not role: raise gen.Return() elif existing and not role: yield self._remove_role(name, existing) elif not existing and role: yield self._add_role(name, role) elif existing != role: yield self._remove_role(name, existing) yield self._add_role(name, role)
def _delete_entity(self, name): """Deletes and IAM Entity. If the entity doesn't exist, we just warn and move on. args: name: The IAM Entity Name """ if self._dry: self.log.warning('Would delete %s %s' % (self.entity_name, name)) raise gen.Return() try: # Get the entities policies. They have to be deleted before we can # possibly move forward and delete the entity. existing_policies = yield self._get_entity_policies(name) tasks = [] for policy in existing_policies: tasks.append(self._delete_entity_policy(name, policy)) yield tasks # Now delete the entity yield self.thread(self.delete_entity, name) self.log.info('%s %s deleted' % (self.entity_name, name)) except BotoServerError as e: if e.status != 404: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) self.log.warning('%s %s doesn\'t exist' % (self.entity_name, name))
def _get_group_users(self, name): """Returns a list of users assigned to the group. args: name: the name of the group returns: a list of user name strings """ users = [] try: raw = yield self.thread(self.iam_conn.get_group, name) users = [ user['user_name'] for user in raw['get_group_response'] ['get_group_result']['users'] ] except BotoServerError as e: if e.status != 404: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) except KeyError: # No users! users = [] raise gen.Return(users)
def _execute(self): """Executes an actor and yields the results when its finished. raises: gen.Return() """ self.log.info('Sending message "%s" to Hipchat room "%s"' % (self.option('message'), self.option('room'))) res = yield self._post_message(self.option('room'), self.option('message')) # If we get 'None' or 'False' back, the actor failed. if not res: raise exceptions.RecoverableActorFailure( 'Failed to send message to HipChat: %s' % res) # If we got here, the result is supposed to include 'success' as a key # and inside that key we can dig for the actual message. If the # response code is 202, we know that we didn't actually execute the # message send, but just validated the API token against the API. if 'success' in res: if res['success']['code'] == 202: self.log.info('API Token Validated: %s' % res['success']['message']) raise gen.Return()
def thread(self, function, *args, **kwargs): """Execute `function` in a concurrent thread. Example: >>> zones = yield thread(ec2_conn.get_all_zones) This allows execution of any function in a thread without having to write a wrapper method that is decorated with run_on_executor() """ try: return function(*args, **kwargs) except boto_exception.BotoServerError as e: # If we're using temporary IAM credentials, when those expire we # can get back a blank 400 from Amazon. This is confusing, but it # happens because of https://github.com/boto/boto/issues/898. In # most cases, these temporary IAM creds can be re-loaded by # reaching out to the AWS API (for example, if we're using an IAM # Instance Profile role), so thats what Boto tries to do. However, # if you're using short-term creds (say from SAML auth'd logins), # then this fails and Boto returns a blank 400. if (e.status == 400 and e.reason == 'Bad Request' and e.error_code is None): msg = 'Access credentials have expired' raise exceptions.InvalidCredentials(msg) msg = '%s: %s' % (e.error_code, e.message) if e.status == 403: raise exceptions.InvalidCredentials(msg) raise except boto3_exceptions.Boto3Error as e: raise exceptions.RecoverableActorFailure( 'Boto3 had a failure: %s' % e)
def _delete_bucket(self): bucket = self.option('name') try: self.log.info('Deleting bucket %s' % bucket) yield self.api_call(self.s3_conn.delete_bucket, Bucket=bucket) except ClientError as e: raise exceptions.RecoverableActorFailure( 'Cannot delete bucket: %s' % e.message)
def __init__(self, *args, **kwargs): super(ECSBaseActor, self).__init__(*args, **kwargs) count = self.option('count') if type(count) is str or type(count) is unicode: try: self._options['count'] = int(count) except ValueError: raise exceptions.RecoverableActorFailure( 'Could not parse option \'count\' as int: %s' % count)
def _create_spec(self): try: self.existing_spec = yield self._client.create_resource( self._client._client.alert_specs, self.desired_params) except requests.exceptions.HTTPError as e: if e.response.status_code in (422, 400): msg = ('Invalid parameters supplied to Alert Spec "%s": %s' % (self.option('href'), self.desired_params)) raise exceptions.RecoverableActorFailure(msg) raise self.log.info('Alert spec has been created')
def _verify_can_delete_bucket(self): # Find out if there are any files in the bucket before we go to delete # it. We cannot delete a bucket with files in it -- nor do we want to. bucket = self.option('name') keys = yield self.api_call(self.s3_conn.list_objects, Bucket=bucket) if 'Contents' not in keys: raise gen.Return() if len(keys['Contents']) > 0: raise exceptions.RecoverableActorFailure( 'Cannot delete bucket with keys: %s files found' % len(keys))
def _get_entity(self, name): """Returns an IAM Entity JSON Blob. Searches for an IAM Entity and either returns None, or a JSON blob that describes the Entity. args: name: The IAM Entity Name """ self.log.debug('Searching for %s %s' % (self.entity_name, name)) # Get a list of all of our entities. try: entities = yield self.thread(self.get_all_entities) except BotoServerError as e: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) # Now search for the entity entity = [ entity for entity in entities['list_%ss_response' % self.entity_name][ 'list_%ss_result' % self.entity_name]['%ss' % self.entity_name] if entity['%s_name' % self.entity_name] == name ] # If there aren't any entities, return None. if not entity: raise gen.Return() # If there is more than one entities, something went really wrong. # Raise an exception. if len(entity) > 1: raise exceptions.RecoverableActorFailure( 'More than one %s found matching %s! Am I crazy?!' % (self.entity_name, name)) # Finally, return the result! self.log.debug('Found %s %s' % (self.entity_name, entity[0]['arn'])) raise gen.Return(entity[0])
def _get_entity(self, name): """Returns an IAM Entity JSON Blob. Searches for an IAM Entity and either returns None, or a JSON blob that describes the Entity. args: name: The IAM Entity Name """ self.log.debug('Searching for %s %s' % (self.entity_name, name)) # Get a list of all of the entities - return 100 results at a time, and # paginate the results. is_truncated = True marker = None while is_truncated: # Get the list back - if the marker has been set, then we pass it # in and we start from where the last results told us we should. try: response = yield self.api_call(self.get_all_entities, max_items=MAX_ITEMS, marker=marker) except BotoServerError as e: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) # Get the result object from the response... result = (response['list_%ss_response' % self.entity_name]['list_%ss_result' % self.entity_name]) # If the results indicate they were truncated, they'll include # a 'marker'. Setting these two variables will cause this to # loop again, in the event that we don't find the response in # the first set of results. is_truncated = self.str2bool(result.get('is_truncated', False)) marker = result.get('marker', None) # Check our result for the entity.. if its there, great. # Otherwise, we'll move on. entity = [ entity for entity in result['%ss' % self.entity_name] if entity['%s_name' % self.entity_name] == name ] if len(entity) > 0: self.log.debug('Found %s %s' % (self.entity_name, entity[0]['arn'])) raise gen.Return(entity[0]) # If there aren't any entities, return None. raise gen.Return()
def _execute(self): # Find the array we're adding an alert spec to. Specifically, we need # the servers HREF. array = yield self._find_server_arrays( self.option('array'), raise_on=self._array_raise_on, allow_mock=self._array_allow_mock) self.log.info('Found %s (%s)' % (array.soul['name'], array.href)) # Add all of the required parameters to a dictionary params = { 'condition': self.option('condition'), 'description': self.option('description'), 'duration': int(self.option('duration')), 'file': self.option('file'), 'name': self.option('name'), 'subject_href': array.href, 'threshold': self.option('threshold'), 'variable': self.option('variable'), } # Generate the RightScale parameters that we need to pass in when # creating the alert. The optional parameters should not be passed in # if their option value came in as None. _optional_params = [ 'description', 'escalation_name', 'vote_tag', 'vote_type' ] for optional in _optional_params: if self.option(optional): params[optional] = self.option(optional) params = self._generate_rightscale_params('alert_spec', params) self.log.debug('Generated params: %s' % params) if self._dry: # In dry run mode, just log out what we would have done. self.log.info('Would have created the alert spec \"%s\" on %s' % (self.option('name'), array.soul['name'])) raise gen.Return() # We're really doin this. If we get a known exception back, handle # it. Otherwise, raise it. try: yield self._client.create_resource( self._client._client.alert_specs, params) self.log.info('Alert spec has been created') except requests.exceptions.HTTPError as e: if e.response.status_code in (422, 400): msg = ('Invalid parameters supplied to Alert Spec "%s": %s' % (self.option('name'), params)) raise exceptions.RecoverableActorFailure(msg) raise
def _update_spec(self): try: self.existing_spec = yield self._client.update( self.existing_spec, self.desired_params) except requests.exceptions.HTTPError as e: if e.response.status_code in (422, 400): msg = ('Invalid parameters supplied to Alert Spec "%s": %s' % (self.existing_spec.soul['name'], self.desired_params)) raise exceptions.RecoverableActorFailure(msg) raise self.log.info('Alert spec has been updated')
def _push_policy(self): self.log.info('Pushing bucket policy %s' % self.option('policy')) self.log.debug('Policy doc: %s' % self.policy) try: yield self.api_call(self.s3_conn.put_bucket_policy, Bucket=self.option('name'), Policy=json.dumps(self.policy)) except ClientError as e: if 'MalformedPolicy' in e.message: raise base.InvalidPolicy(e.message) raise exceptions.RecoverableActorFailure( 'An unexpected error occurred: %s' % e)
def _use_cert(self, elb, arn): """Assign an ssl cert to a given ELB. Args: elb: boto elb object. arn: ARN for server certificate to use. """ self.log.info('Setting ELB "%s" to use cert arn: %s' % (elb, arn)) try: yield self.thread( elb.set_listener_SSL_certificate, self.option('port'), arn) except BotoServerError as e: raise exceptions.RecoverableActorFailure( 'Applying new SSL cert to %s failed: %s' % (elb, e))
def _add_user_to_group(self, name, group): """Quick helper method to add a user to a group. args: name: user name group: group name """ if self._dry: self.log.warning('Would have added %s to %s' % (name, group)) raise gen.Return() try: self.log.info('Adding %s to %s' % (name, group)) yield self.thread(self.iam_conn.add_user_to_group, group, name) except BotoServerError as e: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e)
def _remove_user_from_group(self, name, group): """Quick helper method to remove a user from a group. args: name: user name group: group name """ if self._dry: self.log.warning('Would have removed %s from %s' % (name, group)) raise gen.Return() try: self.log.info('Removing %s from %s' % (name, group)) yield self.api_call(self.iam_conn.remove_user_from_group, group, name) except BotoServerError as e: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e)
def _remove_role(self, name, role): """Removes a role assigned to an Instance Profile. args: name: The name of the InstanceProfile we're managing role: The name of the role to remove """ if self._dry: self.log.warning('Would remove role %s from %s' % (role, name)) raise gen.Return() try: self.log.info('Removing role %s from %s' % (role, name)) yield self.api_call( self.iam_conn.remove_role_from_instance_profile, name, role) except BotoServerError as e: if e.status != 404: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e)
def _add_role(self, name, role): """Adds a role to an Instance Profile. args: name: The name of the Instance Profile we're managing role: The name of the role to assign to the profile """ if self._dry: self.log.warning('Would add role %s from %s' % (role, name)) raise gen.Return() try: self.log.info('Adding role %s to %s' % (role, name)) yield self.thread(self.iam_conn.add_role_to_instance_profile, name, role) except BotoServerError as e: if e.status != 409: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e)
def _ensure_service_present(self, service_name, existing_service): """Ensures a service is present. Args: service_name: service name to use. existing_service: Result of _describe_service on the given service name. """ service_is_active = (existing_service and existing_service['status'] != 'INACTIVE') if not service_is_active and not self.task_definition: # Without a task definition, # we can only update an existing service. self.log.error( 'Could not find service with name {} to update ' 'in {}.'.format( service_name, self._format_location())) raise exceptions.RecoverableActorFailure( 'Cannot update non-existant service.') if service_is_active: # Service already exists - update it. # Ensure no immutable fields were mutated. self._check_immutable_field_errors( old_params=existing_service, new_params=self.service_definition, immutable_fields=['loadBalancers', 'role']) task_definition_name = yield self._update_service(service_name, existing_service) else: task_definition_name = yield self._create_service(service_name) if self.option('wait'): yield self._wait_for_service_update( service_name, task_definition_name) self.log.info( 'Service {} updated successfully in {}.'.format( service_name, self._format_location())) else: self.log.info( 'Not waiting for service {} to finish updating ' 'in {}.'.format( service_name, self._format_location()))
def _ensure_groups(self, name, groups): """Ensure that this user is a member of specific groups. args: name: The user we're managing groups: The list (or single) of groups to join be members of """ if isinstance(groups, str): groups = [groups] current_groups = set() try: res = yield self.api_call(self.iam_conn.get_groups_for_user, name) current_groups = { g['group_name'] for g in res['list_groups_for_user_response'] ['list_groups_for_user_result']['groups'] } except BotoServerError as e: # If the error is a 404, then the user doesn't exist and we can # assume that the mappings don't exist at all. We leave the # existin_mappings list alone. For any other error, raise. if e.status != 404: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e) # Find any groups that we're not already a member of, and add us tasks = [] try: for new_group in set(groups) - current_groups: tasks.append(self._add_user_to_group(name, new_group)) except StopIteration: pass yield tasks # Find any group memberships we didn't know about, and purge them tasks = [] for bad_group in current_groups - set(groups): tasks.append(self._remove_user_from_group(name, bad_group)) yield tasks
def _check_immutable_field_errors(self, old_params, new_params, immutable_fields): """Compares an old service definition to a new one to ensure that all of the specified immutable fields are the same between them. If there are any errors, this logs them and raises RecoverableActorFailure. Args: old_params: old parameters to use. new_params: new parameters to compare with. immutable_fields: list of immutable fields. """ # API does not return role name, only the role arn. role_arn = old_params.get('roleArn') role = None if role_arn: role = self._arn_to_name(role_arn) old_params['role'] = role has_error = False for immutable_field_name in immutable_fields: new_field = new_params.get(immutable_field_name) old_field = old_params.get(immutable_field_name) if new_field != old_field: has_error = True self.log.error( "Field \'{field}\' cannot be updated.\n" 'Old value: {old}\n' 'New value: {new}'.format( field=immutable_field_name, old=old_field, new=new_field)) if has_error: raise exceptions.RecoverableActorFailure( 'Immutable fields cannot be updated. ' 'A new service must be created.')
def _wrap_boto_exception(self, e): if isinstance(e, boto_exception.BotoServerError): # If we're using temporary IAM credentials, when those expire we # can get back a blank 400 from Amazon. This is confusing, but it # happens because of https://github.com/boto/boto/issues/898. In # most cases, these temporary IAM creds can be re-loaded by # reaching out to the AWS API (for example, if we're using an IAM # Instance Profile role), so thats what Boto tries to do. However, # if you're using short-term creds (say from SAML auth'd logins), # then this fails and Boto returns a blank 400. if (e.status == 400 and e.reason == 'Bad Request' and e.error_code is None): msg = 'Access credentials have expired' return exceptions.InvalidCredentials(msg) msg = '%s: %s' % (e.error_code, str(e)) if e.status == 403: return exceptions.InvalidCredentials(msg) elif isinstance(e, boto3_exceptions.Boto3Error): return exceptions.RecoverableActorFailure( 'Boto3 had a failure: %s' % e) return e
def _delete_entity_policy(self, name, policy_name): """Optionally pushes a policy to an IAM entity. args: name: The IAM Entity Name policy_name: The entity policy name """ if self._dry: self.log.warning('Would delete policy %s from %s %s' % (policy_name, self.entity_name, name)) raise gen.Return() self.log.info('Deleting policy %s from %s %s' % (policy_name, self.entity_name, name)) try: ret = yield self.thread(self.delete_entity_policy, name, policy_name) self.log.debug('Policy %s deleted: %s' % (policy_name, ret)) except BotoServerError as e: if e.error_code != 404: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e)
def _put_entity_policy(self, name, policy_name, policy_doc): """Optionally pushes a policy to an IAM Entity. args: name: The IAM Entity Name policy_name: The entity policy name policy_doc: The ploicy document object itself """ if self._dry: self.log.warning('Would push policy %s to %s %s' % (policy_name, self.entity_name, name)) raise gen.Return() self.log.info('Pushing policy %s to %s %s' % (policy_name, self.entity_name, name)) try: ret = yield self.thread(self.put_entity_policy, name, policy_name, json.dumps(policy_doc)) self.log.debug('Policy %s pushed: %s' % (policy_name, ret)) except BotoServerError as e: raise exceptions.RecoverableActorFailure( 'An unexpected API error occurred: %s' % e)