def node_feed_req(lb_id, node_id, response): """ Return (intent, performer) sequence for getting clb node's feed that wrapped with retry intent. :param lb_id: Lodbalancer ID :param node_id: LB node ID :param response: The response returned when getting CLB node feed. It is either string containing feed or Exception object that will be raised when getting the feed :return: (intent, performer) tuple """ if isinstance(response, Exception): def handler(i): raise response else: def handler(i): return response return ( Retry( effect=mock.ANY, should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=exponential_backoff_interval(2)) ), nested_sequence([(("gcnf", lb_id, node_id), handler)]) )
def lb_req(url, json_response, response): """ Return a SequenceDispatcher two-tuple that matches a service request to a particular load balancer endpoint (using GET), and returns the given ``response`` as the content in an HTTP 200 ``StubResponse``. """ if isinstance(response, Exception): def handler(i): raise response log_seq = [] else: def handler(i): return (StubResponse(200, {}), response) log_seq = [(Log(mock.ANY, mock.ANY), lambda i: None)] return ( Retry( effect=mock.ANY, should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=exponential_backoff_interval(2)) ), nested_sequence([ (service_request( ServiceType.CLOUD_LOAD_BALANCERS, 'GET', url, json_response=json_response).intent, handler) ] + log_seq) )
def node_feed_req(lb_id, node_id, response): """ Return (intent, performer) sequence for getting clb node's feed that wrapped with retry intent. :param lb_id: Lodbalancer ID :param node_id: LB node ID :param response: The response returned when getting CLB node feed. It is either string containing feed or Exception object that will be raised when getting the feed :return: (intent, performer) tuple """ if isinstance(response, Exception): def handler(i): raise response else: def handler(i): return response return (Retry(effect=mock.ANY, should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=exponential_backoff_interval(2))), nested_sequence([(("gcnf", lb_id, node_id), handler)]))
def _perform_add_event(self, response_sequence): """ Given a sequence of functions that take an intent and returns a response (or raises an exception), perform :func:`add_event` and return the result. """ log = object() eff = add_event(self.event, 'tid', 'ord', log) uid = '00000000-0000-0000-0000-000000000000' svrq = service_request( ServiceType.CLOUD_FEEDS, 'POST', 'autoscale/events', headers={'content-type': ['application/vnd.rackspace.atom+json']}, data=self._get_request('INFO', uid, 'tid'), log=log, success_pred=has_code(201), json_response=False) seq = [ (TenantScope(mock.ANY, 'tid'), nested_sequence([ retry_sequence( Retry(effect=svrq, should_retry=ShouldDelayAndRetry( can_retry=mock.ANY, next_interval=exponential_backoff_interval(2))), response_sequence) ])) ] return perform_sequence(seq, eff)
def _perform_add_event(self, response_sequence): """ Given a sequence of functions that take an intent and returns a response (or raises an exception), perform :func:`add_event` and return the result. """ log = object() eff = add_event(self.event, 'tid', 'ord', log) uid = '00000000-0000-0000-0000-000000000000' svrq = service_request( ServiceType.CLOUD_FEEDS, 'POST', 'autoscale/events', headers={ 'content-type': ['application/vnd.rackspace.atom+json']}, data=self._get_request('INFO', uid, 'tid'), log=log, success_pred=has_code(201), json_response=False) seq = [ (TenantScope(mock.ANY, 'tid'), nested_sequence([ retry_sequence( Retry(effect=svrq, should_retry=ShouldDelayAndRetry( can_retry=mock.ANY, next_interval=exponential_backoff_interval(2))), response_sequence ) ])) ] return perform_sequence(seq, eff)
def convergence_remove_server_from_group(log, transaction_id, server_id, replace, purge, group, state): """ Remove a specific server from the group, optionally decrementing the desired capacity. The server may just be scheduled for deletion, or it may be evicted from the group by removing otter-specific metdata from the server. :param log: A bound logger :param bytes trans_id: The transaction id for this operation. :param bytes server_id: The id of the server to be removed. :param bool replace: Should the server be replaced? :param bool purge: Should the server be deleted from Nova? :param group: The scaling group to remove a server from. :type group: :class:`~otter.models.interface.IScalingGroup` :param state: The current state of the group. :type state: :class:`~otter.models.interface.GroupState` :return: The updated state. :rtype: Effect of :class:`~otter.models.interface.GroupState` :raise: :class:`CannotDeleteServerBelowMinError` if the server cannot be deleted without replacement, and :class:`ServerNotFoundError` if there is no such server to be deleted. """ effects = [_is_server_in_group(group, server_id)] if not replace: effects.append(_can_scale_down(group, server_id)) # the (possibly) two checks can happen in parallel, but we want # ServerNotFoundError to take precedence over # CannotDeleteServerBelowMinError both_checks = yield parallel_all_errors(effects) for is_error, result in both_checks: if is_error: reraise(*result) # Remove the server if purge: eff = set_nova_metadata_item(server_id, *DRAINING_METADATA) else: eff = Effect( EvictServerFromScalingGroup(log=log, transaction_id=transaction_id, scaling_group=group, server_id=server_id)) yield Effect( TenantScope( retry_effect(eff, retry_times(3), exponential_backoff_interval(2)), group.tenant_id)) if not replace: yield do_return(assoc_obj(state, desired=state.desired - 1)) else: yield do_return(state)
def test_exp_backoff_interval(self): """ ``exponential_backoff_interval`` returns previous interval * 2 every time it is called """ err = DummyException() next_interval = exponential_backoff_interval(3) self.assertEqual(next_interval(err), 3) self.assertEqual(next_interval(err), 6) self.assertEqual(next_interval(err), 12)
def convergence_remove_server_from_group( log, transaction_id, server_id, replace, purge, group, state): """ Remove a specific server from the group, optionally decrementing the desired capacity. The server may just be scheduled for deletion, or it may be evicted from the group by removing otter-specific metdata from the server. :param log: A bound logger :param bytes trans_id: The transaction id for this operation. :param bytes server_id: The id of the server to be removed. :param bool replace: Should the server be replaced? :param bool purge: Should the server be deleted from Nova? :param group: The scaling group to remove a server from. :type group: :class:`~otter.models.interface.IScalingGroup` :param state: The current state of the group. :type state: :class:`~otter.models.interface.GroupState` :return: The updated state. :rtype: Effect of :class:`~otter.models.interface.GroupState` :raise: :class:`CannotDeleteServerBelowMinError` if the server cannot be deleted without replacement, and :class:`ServerNotFoundError` if there is no such server to be deleted. """ effects = [_is_server_in_group(group, server_id)] if not replace: effects.append(_can_scale_down(group, server_id)) # the (possibly) two checks can happen in parallel, but we want # ServerNotFoundError to take precedence over # CannotDeleteServerBelowMinError both_checks = yield parallel_all_errors(effects) for is_error, result in both_checks: if is_error: reraise(*result) # Remove the server if purge: eff = set_nova_metadata_item(server_id, *DRAINING_METADATA) else: eff = Effect( EvictServerFromScalingGroup(log=log, transaction_id=transaction_id, scaling_group=group, server_id=server_id)) yield Effect(TenantScope( retry_effect(eff, retry_times(3), exponential_backoff_interval(2)), group.tenant_id)) if not replace: yield do_return(assoc_obj(state, desired=state.desired - 1)) else: yield do_return(state)
def add_event(event, admin_tenant_id, region, log): """ Add event to cloud feeds """ event, error, timestamp, event_tenant_id, event_id = sanitize_event(event) req = prepare_request(request_format, event, error, timestamp, region, event_tenant_id, event_id) eff = retry_effect( publish_autoscale_event(req, log=log), compose_retries(lambda f: (not f.check(APIError) or f.value.code < 400 or f.value.code >= 500), retry_times(5)), exponential_backoff_interval(2), ) return Effect(TenantScope(tenant_id=admin_tenant_id, effect=eff))
def as_effect(self): """Produce a :obj:`Effect` to delete a server.""" eff = retry_effect( delete_and_verify(self.server_id), can_retry=retry_times(3), next_interval=exponential_backoff_interval(2)) def report_success(result): return StepResult.RETRY, [ ErrorReason.String( 'must re-gather after deletion in order to update the ' 'active cache')] return eff.on(success=report_success)
def add_event(event, admin_tenant_id, region, log): """ Add event to cloud feeds """ event, error, timestamp, event_tenant_id, event_id = sanitize_event(event) req = prepare_request(request_format, event, error, timestamp, region, event_tenant_id, event_id) eff = retry_effect( publish_autoscale_event(req, log=log), compose_retries( lambda f: (not f.check(APIError) or f.value.code < 400 or f.value.code >= 500), retry_times(5)), exponential_backoff_interval(2)) return Effect(TenantScope(tenant_id=admin_tenant_id, effect=eff))
def verified_delete(log, server_endpoint, request_bag, server_id, exp_start=2, max_retries=10, clock=None): """ Attempt to delete a server from the server endpoint, and ensure that it is deleted by trying again until deleting/getting the server results in a 404 or until ``OS-EXT-STS:task_state`` in server details is 'deleting', indicating that Nova has acknowledged that the server is to be deleted as soon as possible. Time out attempting to verify deletes after a period of time and log an error. :param log: A bound logger. :param str server_endpoint: Server endpoint URI. :param str auth_token: Keystone Auth token. :param str server_id: Opaque nova server id. :param int exp_start: Exponential backoff interval start seconds. Default 2 :param int max_retries: Maximum number of retry attempts :return: Deferred that fires when the expected status has been seen. """ serv_log = log.bind(server_id=server_id) serv_log.msg('Deleting server') if clock is None: # pragma: no cover from twisted.internet import reactor clock = reactor d = retry( partial(delete_and_verify, serv_log, server_endpoint, request_bag, server_id, clock), can_retry=retry_times(max_retries), next_interval=exponential_backoff_interval(exp_start), clock=clock) d.addCallback(log_with_time, clock, serv_log, clock.seconds(), ('Server deleted successfully (or acknowledged by Nova as ' 'to-be-deleted) : {time_delete} seconds.'), 'time_delete') return d
def _is_server_in_group(group, server_id): """ Given a group and server ID, determines if the server is a member of the group. If it isn't, it raises a :class:`ServerNotFoundError`. """ try: response, server_info = yield Effect( TenantScope( retry_effect(get_server_details(server_id), retry_times(3), exponential_backoff_interval(2)), group.tenant_id)) except NoSuchServerError: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id) group_id = group_id_from_metadata( get_in(('server', 'metadata'), server_info, {})) if group_id != group.uuid: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id)
def verified_delete(log, server_endpoint, request_bag, server_id, exp_start=2, max_retries=10, clock=None): """ Attempt to delete a server from the server endpoint, and ensure that it is deleted by trying again until deleting/getting the server results in a 404 or until ``OS-EXT-STS:task_state`` in server details is 'deleting', indicating that Nova has acknowledged that the server is to be deleted as soon as possible. Time out attempting to verify deletes after a period of time and log an error. :param log: A bound logger. :param str server_endpoint: Server endpoint URI. :param str auth_token: Keystone Auth token. :param str server_id: Opaque nova server id. :param int exp_start: Exponential backoff interval start seconds. Default 2 :param int max_retries: Maximum number of retry attempts :return: Deferred that fires when the expected status has been seen. """ serv_log = log.bind(server_id=server_id) serv_log.msg('Deleting server') if clock is None: # pragma: no cover from twisted.internet import reactor clock = reactor d = retry(partial(delete_and_verify, serv_log, server_endpoint, request_bag, server_id, clock), can_retry=retry_times(max_retries), next_interval=exponential_backoff_interval(exp_start), clock=clock) d.addCallback(log_with_time, clock, serv_log, clock.seconds(), ('Server deleted successfully (or acknowledged by Nova as ' 'to-be-deleted) : {time_delete} seconds.'), 'time_delete') return d
def test_delete_server(self, mock_dav): """ :obj:`DeleteServer.as_effect` calls `delete_and_verify` with retries. It returns SUCCESS on completion and RETRY on failure """ mock_dav.side_effect = lambda sid: Effect(sid) eff = DeleteServer(server_id='abc123').as_effect() self.assertIsInstance(eff.intent, Retry) self.assertEqual( eff.intent.should_retry, ShouldDelayAndRetry(can_retry=retry_times(3), next_interval=exponential_backoff_interval(2))) self.assertEqual(eff.intent.effect.intent, 'abc123') self.assertEqual( resolve_effect(eff, (None, {})), (StepResult.RETRY, [ErrorReason.String('must re-gather after deletion in order to ' 'update the active cache')]))
def _is_server_in_group(group, server_id): """ Given a group and server ID, determines if the server is a member of the group. If it isn't, it raises a :class:`ServerNotFoundError`. """ try: response, server_info = yield Effect( TenantScope( retry_effect(get_server_details(server_id), retry_times(3), exponential_backoff_interval(2)), group.tenant_id, ) ) except NoSuchServerError: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id) group_id = group_id_from_metadata(get_in(("server", "metadata"), server_info, {})) if group_id != group.uuid: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id)
def get_all_server_details(tenant_id, authenticator, service_name, region, limit=100, clock=None, _treq=None): """ Return all servers of a tenant TODO: service_name is possibly internal to this function but I don't want to pass config here? NOTE: This really screams to be a independent txcloud-type API """ token, catalog = yield authenticator.authenticate_tenant(tenant_id, log=default_log) endpoint = public_endpoint_url(catalog, service_name, region) url = append_segments(endpoint, 'servers', 'detail') query = {'limit': limit} all_servers = [] if clock is None: # pragma: no cover from twisted.internet import reactor as clock if _treq is None: # pragma: no cover _treq = treq def fetch(url, headers): d = _treq.get(url, headers=headers) d.addCallback(check_success, [200], _treq=_treq) d.addCallback(_treq.json_content) return d while True: # sort based on query name to make the tests predictable urlparams = sorted(query.items(), key=lambda e: e[0]) d = retry(partial(fetch, '{}?{}'.format(url, urlencode(urlparams)), headers(token)), can_retry=retry_times(5), next_interval=exponential_backoff_interval(2), clock=clock) servers = (yield d)['servers'] all_servers.extend(servers) if len(servers) < limit: break query.update({'marker': servers[-1]['id']}) defer.returnValue(all_servers)
def _retry(eff): """Retry an effect with a common policy.""" return retry_effect( eff, retry_times(5), exponential_backoff_interval(2))
def _retry(eff): """Retry an effect with a common policy.""" return retry_effect(eff, retry_times(5), exponential_backoff_interval(2))