Esempio n. 1
0
    def test_retry_and_timeout_get_the_same_default_clock(self):
        """
        If no clock is passed to ``retry_and_timeout``, both ``retry`` and
        ``timeout`` nevertheless get the same clock.
        """
        retry_and_timeout('do_work', 'timeout')

        retry_clock = self.retry.call_args[1]['clock']
        timeout_clock = self.timeout.call_args[1]['clock']

        self.assertIs(retry_clock, timeout_clock)
Esempio n. 2
0
    def test_both_called_with_all_args(self):
        """
        Both ``retry`` and ``timeout`` gets called with the args passed to
        ``retry_and_timeout``, including the same clock
        """
        clock = mock.MagicMock()
        retry_and_timeout('do_work', 'timeout', can_retry='can_retry',
                          next_interval='next_interval', clock=clock)

        self.retry.assert_called_once_with('do_work', can_retry='can_retry',
                                           next_interval='next_interval',
                                           clock=clock)
        self.timeout.assert_called_once_with(self.retry.return_value,
                                             'timeout', clock=clock)
Esempio n. 3
0
    def verify(_):
        def check_status():
            check_d = treq.head(
                append_segments(server_endpoint, 'servers', server_id),
                headers=headers(auth_token))
            check_d.addCallback(check_success, [404])
            return check_d

        start_time = clock.seconds()

        # this is treating all errors as transient, so the only error that can
        # occur is a CancelledError from timing out
        verify_d = retry_and_timeout(check_status, timeout,
                                     next_interval=repeating_interval(interval),
                                     clock=clock)

        def on_success(_):
            time_delete = clock.seconds() - start_time
            del_log.msg('Server deleted successfully: {time_delete} seconds.',
                        time_delete=time_delete)

        verify_d.addCallback(on_success)

        def on_timeout(_):
            time_delete = clock.seconds() - start_time
            del_log.err(None, timeout=timeout, time_delete=time_delete,
                        why=('Server {instance_id} failed to be deleted within '
                             'a {timeout} second timeout (it has been '
                             '{time_delete} seconds).'))

        verify_d.addErrback(on_timeout)
Esempio n. 4
0
    def wait_for_stack_list(self, expected_states, timeout=180, period=10):
        def check(content):
            states = pbag([s['stack_status'] for s in content['stacks']])
            if not (states == expected_states):
                msg("Waiting for group {} to reach desired group state.\n"
                    "{} (actual) {} (expected)".format(self.group.group_id,
                                                       states,
                                                       expected_states))
                raise TransientRetryError(
                    "Group states of {} did not match expected {})".format(
                        states, expected_states))

            msg("Success: desired group state reached:\n{}".format(
                expected_states))
            return self.rcs

        def poll():
            return self.get_stack_list().addCallback(check)

        expected_states = pbag(expected_states)

        return retry_and_timeout(
            poll,
            timeout,
            can_retry=terminal_errors_except(TransientRetryError),
            next_interval=repeating_interval(period),
            clock=reactor,
            deferred_description=(
                "Waiting for group {} to reach state {}".format(
                    self.group.group_id, str(expected_states))))
Esempio n. 5
0
    def wait_for_stack_list(self, expected_states, timeout=180, period=10):
        def check(content):
            states = pbag([s['stack_status'] for s in content['stacks']])
            if not (states == expected_states):
                msg("Waiting for group {} to reach desired group state.\n"
                    "{} (actual) {} (expected)"
                    .format(self.group.group_id, states, expected_states))
                raise TransientRetryError(
                    "Group states of {} did not match expected {})"
                    .format(states, expected_states))

            msg("Success: desired group state reached:\n{}"
                .format(expected_states))
            return self.rcs

        def poll():
            return self.get_stack_list().addCallback(check)

        expected_states = pbag(expected_states)

        return retry_and_timeout(
            poll, timeout,
            can_retry=terminal_errors_except(TransientRetryError),
            next_interval=repeating_interval(period),
            clock=reactor,
            deferred_description=(
                "Waiting for group {} to reach state {}".format(
                    self.group.group_id, str(expected_states))))
    def verify(_):
        def check_status():
            check_d = treq.head(
                append_segments(server_endpoint, 'servers', server_id),
                headers=headers(auth_token))
            check_d.addCallback(check_success, [404])
            return check_d

        start_time = clock.seconds()

        timeout_description = (
            "Waiting for Nova to actually delete server {0}".format(server_id))

        verify_d = retry_and_timeout(check_status, timeout,
                                     next_interval=repeating_interval(interval),
                                     clock=clock,
                                     deferred_description=timeout_description)

        def on_success(_):
            time_delete = clock.seconds() - start_time
            del_log.msg('Server deleted successfully: {time_delete} seconds.',
                        time_delete=time_delete)

        verify_d.addCallback(on_success)
        verify_d.addErrback(del_log.err)
Esempio n. 7
0
    def wait_for_state(self, rcs, matcher, timeout=600, period=10, clock=None):
        """
        Wait for the state on the scaling group to match the provided matchers,
        specified by matcher.

        :param rcs: a :class:`otter.integration.lib.resources.TestResources`
            instance
        :param matcher: A :mod:`testtool.matcher`, as specified in
            module: testtools.matchers in
            http://testtools.readthedocs.org/en/latest/api.html.
        :param timeout: The amount of time to wait until this step is
            considered failed.
        :param period: How long to wait before polling again.
        :param clock: a :class:`twisted.internet.interfaces.IReactorTime`
            provider

        :return: None, if the state is reached
        :raises: :class:`TimedOutError` if the state is never reached within
            the requisite amount of time.

        Example usage:

        ```
        matcher = MatchesAll(
            IncludesServers(included_server_ids),
            ExcludesServers(exclude_server_ids),
            ContainsDict({
                'pending': Equals(0),
                'desired': Equals(5),
                'status': Equals('ACTIVE')
            })
        )

        ..wait_for_state(rcs, matchers, timeout=60)
        ```
        """
        def check(result):
            response, group_state = result
            mismatch = matcher.match(group_state['group'])
            if mismatch:
                msg("Waiting for group {} to reach desired group state.\n"
                    "Mismatch: {}"
                    .format(self.group_id, mismatch.describe()))
                raise TransientRetryError(mismatch.describe())
            msg("Success: desired group state reached:\n{}\nmatches:\n{}"
                .format(group_state['group'], matcher))
            return rcs

        def poll():
            return self.get_scaling_group_state(rcs, [200]).addCallback(check)

        return retry_and_timeout(
            poll, timeout,
            can_retry=terminal_errors_except(TransientRetryError),
            next_interval=repeating_interval(period),
            clock=clock or reactor,
            deferred_description=(
                "Waiting for group {} to reach state {}"
                .format(self.group_id, str(matcher)))
        )
Esempio n. 8
0
    def wait_for_state(self, rcs, matcher, timeout=600, period=10, clock=None):
        """
        Wait for the state on the scaling group to match the provided matchers,
        specified by matcher.

        :param rcs: a :class:`otter.integration.lib.resources.TestResources`
            instance
        :param matcher: A :mod:`testtool.matcher`, as specified in
            module: testtools.matchers in
            http://testtools.readthedocs.org/en/latest/api.html.
        :param timeout: The amount of time to wait until this step is
            considered failed.
        :param period: How long to wait before polling again.
        :param clock: a :class:`twisted.internet.interfaces.IReactorTime`
            provider

        :return: None, if the state is reached
        :raises: :class:`TimedOutError` if the state is never reached within
            the requisite amount of time.

        Example usage:

        ```
        matcher = MatchesAll(
            IncludesServers(included_server_ids),
            ExcludesServers(exclude_server_ids),
            ContainsDict({
                'pending': Equals(0),
                'desired': Equals(5),
                'status': Equals('ACTIVE')
            })
        )

        ..wait_for_state(rcs, matchers, timeout=60)
        ```
        """
        def check(result):
            response, group_state = result
            mismatch = matcher.match(group_state['group'])
            if mismatch:
                msg("Waiting for group {} to reach desired group state.\n"
                    "Mismatch: {}".format(self.group_id, mismatch.describe()))
                raise TransientRetryError(mismatch.describe())
            msg("Success: desired group state reached:\n{}\nmatches:\n{}".
                format(group_state['group'], matcher))
            return rcs

        def poll():
            return self.get_scaling_group_state(rcs, [200]).addCallback(check)

        return retry_and_timeout(
            poll,
            timeout,
            can_retry=terminal_errors_except(TransientRetryError),
            next_interval=repeating_interval(period),
            clock=clock or reactor,
            deferred_description=(
                "Waiting for group {} to reach state {}".format(
                    self.group_id, str(matcher))))
Esempio n. 9
0
 def retrier(*args, **kwargs):
     return retry_and_timeout(
         partial(f, *args, **kwargs),
         timeout,
         can_retry=terminal_errors_except(TransientRetryError),
         next_interval=repeating_interval(period),
         clock=clock,
         deferred_description=reason)
Esempio n. 10
0
 def retrier(*args, **kwargs):
     return retry_and_timeout(
         partial(f, *args, **kwargs), timeout,
         can_retry=terminal_errors_except(TransientRetryError),
         next_interval=repeating_interval(period),
         clock=clock,
         deferred_description=reason
     )
Esempio n. 11
0
def verified_delete(log,
                    server_endpoint,
                    auth_token,
                    server_id,
                    interval=10,
                    timeout=3660,
                    clock=None):
    """
    Attempt to delete a server from the server endpoint, and ensure that it is
    deleted by trying again until deleting/getting the server results in a 404
    or until ``OS-EXT-STS:task_state`` in server details is 'deleting',
    indicating that Nova has acknowledged that the server is to be deleted
    as soon as possible.

    Time out attempting to verify deletes after a period of time and log an
    error.

    :param log: A bound logger.
    :param str server_endpoint: Server endpoint URI.
    :param str auth_token: Keystone Auth token.
    :param str server_id: Opaque nova server id.
    :param int interval: Deletion interval in seconds - how long until
        verifying a delete is retried. Default: 5.
    :param int timeout: Seconds after which the deletion will be logged as a
        failure, if Nova fails to return a 404.  Default is 3660, because if
        the server is building, the delete will not happen until immediately
        after it has finished building.

    :return: Deferred that fires when the expected status has been seen.
    """
    serv_log = log.bind(server_id=server_id)
    serv_log.msg('Deleting server')

    if clock is None:  # pragma: no cover
        from twisted.internet import reactor
        clock = reactor

    timeout_description = (
        "Waiting for Nova to actually delete server {0} (or acknowledge delete)"
        .format(server_id))

    d = retry_and_timeout(
        partial(delete_and_verify, serv_log, server_endpoint, auth_token, server_id),
        timeout,
        next_interval=repeating_interval(interval),
        clock=clock,
        deferred_description=timeout_description)

    d.addCallback(log_with_time, clock, serv_log, clock.seconds(),
                  ('Server deleted successfully (or acknowledged by Nova as '
                   'to-be-deleted) : {time_delete} seconds.'), 'time_delete')
    d.addErrback(serv_log.err)
    return d
Esempio n. 12
0
def wait_for_servers(rcs,
                     pool,
                     matcher,
                     group=None,
                     timeout=600,
                     period=10,
                     clock=None,
                     _treq=treq):
    """
    Wait until Nova reaches a particular state (as described by the given
    matcher) - if a group is provided, then match only the servers for the
    given group.

    :param rcs: an instance of
        :class:`otter.integration.lib.resources.TestResources`
    :param pool: a :class:`twisted.web.client.HTTPConnectionPool`
    :param matcher: a :mod:`testtools.matcher` matcher that describes the
        desired state of the servers belonging to the autoscaling group.
    :param group: a :class:`otter.integration.lib.autoscale.ScalingGroup` that
        specifies which autoscaling group's servers we are looking at.  This
        group should already exist, and have a `group_id` attribute.  If not
        provided, the matcher will apply to all servers.
    """
    message = "Waiting for {0} Nova servers".format(
        "all" if group is None else "group {0} 's".format(group.group_id))

    @inlineCallbacks
    def do_work():
        servers = yield list_servers(rcs, pool, _treq=_treq)
        servers = servers['servers']
        if group is not None:
            servers = [
                server for server in servers
                if (group.group_id == server['metadata'].get(
                    "rax:autoscale:group:id", None))
            ]
        mismatch = matcher.match(servers)
        if mismatch:
            msg("{0}.\nMismatch: {1}".format(message, mismatch.describe()))
            raise TransientRetryError(mismatch.describe())
        returnValue(servers)

    return retry_and_timeout(
        do_work,
        timeout,
        can_retry=terminal_errors_except(TransientRetryError),
        next_interval=repeating_interval(period),
        clock=clock or reactor,
        deferred_description=("{0} to reach state {1}".format(
            message, str(matcher))))
Esempio n. 13
0
def wait_for_servers(rcs, pool, matcher, group=None, timeout=600, period=10,
                     clock=None, _treq=treq):
    """
    Wait until Nova reaches a particular state (as described by the given
    matcher) - if a group is provided, then match only the servers for the
    given group.

    :param rcs: an instance of
        :class:`otter.integration.lib.resources.TestResources`
    :param pool: a :class:`twisted.web.client.HTTPConnectionPool`
    :param matcher: a :mod:`testtools.matcher` matcher that describes the
        desired state of the servers belonging to the autoscaling group.
    :param group: a :class:`otter.integration.lib.autoscale.ScalingGroup` that
        specifies which autoscaling group's servers we are looking at.  This
        group should already exist, and have a `group_id` attribute.  If not
        provided, the matcher will apply to all servers.
    """
    message = "Waiting for {0} Nova servers".format(
        "all" if group is None else "group {0} 's".format(group.group_id))

    @inlineCallbacks
    def do_work():
        servers = yield list_servers(rcs, pool, _treq=_treq)
        servers = servers['servers']
        if group is not None:
            servers = [
                server for server in servers
                if (group.group_id ==
                    server['metadata'].get("rax:autoscale:group:id", None))
            ]
        mismatch = matcher.match(servers)
        if mismatch:
            msg("{0}.\nMismatch: {1}".format(message, mismatch.describe()))
            raise TransientRetryError(mismatch.describe())
        returnValue(servers)

    return retry_and_timeout(
        do_work, timeout,
        can_retry=terminal_errors_except(TransientRetryError),
        next_interval=repeating_interval(period),
        clock=clock or reactor,
        deferred_description=(
            "{0} to reach state {1}".format(message, str(matcher)))
    )
Esempio n. 14
0
    def delete(self, rcs):
        """
        Delete the server.

        :param rcs: an instance of
            :class:`otter.integration.lib.resources.TestResources`
        """
        def try_delete():
            d = self.treq.delete(
                "{}/servers/{}".format(rcs.endpoints["nova"], self.id),
                headers=headers(str(rcs.token)),
                pool=self.pool)
            d.addCallback(check_success, [404], _treq=self.treq)
            d.addCallback(self.treq.content)
            return d

        return retry_and_timeout(
            try_delete, 120,
            can_retry=terminal_errors_except(APIError),
            next_interval=repeating_interval(5),
            clock=self.clock,
            deferred_description=(
                "Waiting for server {} to get deleted".format(self.id)))
Esempio n. 15
0
    def verify(_):
        def check_status():
            check_d = treq.head(append_segments(server_endpoint, 'servers',
                                                server_id),
                                headers=headers(auth_token))
            check_d.addCallback(check_success, [404])
            return check_d

        start_time = clock.seconds()

        # this is treating all errors as transient, so the only error that can
        # occur is a CancelledError from timing out
        verify_d = retry_and_timeout(
            check_status,
            timeout,
            next_interval=repeating_interval(interval),
            clock=clock)

        def on_success(_):
            time_delete = clock.seconds() - start_time
            del_log.msg('Server deleted successfully: {time_delete} seconds.',
                        time_delete=time_delete)

        verify_d.addCallback(on_success)

        def on_timeout(_):
            time_delete = clock.seconds() - start_time
            del_log.err(
                None,
                timeout=timeout,
                time_delete=time_delete,
                why=('Server {instance_id} failed to be deleted within '
                     'a {timeout} second timeout (it has been '
                     '{time_delete} seconds).'))

        verify_d.addErrback(on_timeout)
Esempio n. 16
0
    def delete(self, rcs):
        """
        Delete the server.

        :param rcs: an instance of
            :class:`otter.integration.lib.resources.TestResources`
        """
        def try_delete():
            d = self.treq.delete("{}/servers/{}".format(
                rcs.endpoints["nova"], self.id),
                                 headers=headers(str(rcs.token)),
                                 pool=self.pool)
            d.addCallback(check_success, [404], _treq=self.treq)
            d.addCallback(self.treq.content)
            return d

        return retry_and_timeout(
            try_delete,
            120,
            can_retry=terminal_errors_except(APIError),
            next_interval=repeating_interval(5),
            clock=self.clock,
            deferred_description=(
                "Waiting for server {} to get deleted".format(self.id)))
Esempio n. 17
0
def wait_for_active(log,
                    server_endpoint,
                    auth_token,
                    server_id,
                    interval=20,
                    timeout=7200,
                    clock=None):
    """
    Wait until the server specified by server_id's status is 'ACTIVE'

    :param log: A bound logger.
    :param str server_endpoint: Server endpoint URI.
    :param str auth_token: Keystone Auth token.
    :param str server_id: Opaque nova server id.
    :param int interval: Polling interval in seconds.  Default: 20.
    :param int timeout: timeout to poll for the server status in seconds.
        Default 7200 (2 hours).

    :return: Deferred that fires when the expected status has been seen.
    """
    log.msg("Checking instance status every {interval} seconds",
            interval=interval)

    if clock is None:  # pragma: no cover
        from twisted.internet import reactor
        clock = reactor

    start_time = clock.seconds()

    def poll():
        def check_status(server):
            status = server['server']['status']
            time_building = clock.seconds() - start_time

            if status == 'ACTIVE':
                log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
                         "{time_building} seconds"),
                        time_building=time_building)
                return server

            elif status != 'BUILD':
                log.msg(
                    "Server changed to '{status}' in {time_building} seconds",
                    time_building=time_building,
                    status=status)
                raise UnexpectedServerStatus(server_id, status, 'ACTIVE')

            else:
                raise TransientRetryError()  # just poll again

        sd = server_details(server_endpoint, auth_token, server_id, log=log)
        sd.addCallback(check_status)
        return sd

    timeout_description = ("Waiting for server <{0}> to change from BUILD "
                           "state to ACTIVE state").format(server_id)

    return retry_and_timeout(poll,
                             timeout,
                             can_retry=transient_errors_except(
                                 UnexpectedServerStatus, ServerDeleted),
                             next_interval=repeating_interval(interval),
                             clock=clock,
                             deferred_description=timeout_description)
Esempio n. 18
0
def wait_for_active(log,
                    server_endpoint,
                    auth_token,
                    server_id,
                    interval=20,
                    timeout=7200,
                    clock=None):
    """
    Wait until the server specified by server_id's status is 'ACTIVE'

    :param log: A bound logger.
    :param str server_endpoint: Server endpoint URI.
    :param str auth_token: Keystone Auth token.
    :param str server_id: Opaque nova server id.
    :param int interval: Polling interval in seconds.  Default: 20.
    :param int timeout: timeout to poll for the server status in seconds.
        Default 7200 (2 hours).

    :return: Deferred that fires when the expected status has been seen.
    """
    log.msg("Checking instance status every {interval} seconds",
            interval=interval)

    if clock is None:  # pragma: no cover
        from twisted.internet import reactor
        clock = reactor

    start_time = clock.seconds()

    def poll():
        def check_status(server):
            status = server['server']['status']
            time_building = clock.seconds() - start_time

            if status == 'ACTIVE':
                log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
                         "{time_building} seconds"),
                        time_building=time_building)
                return server

            elif status != 'BUILD':
                log.msg("Server changed to '{status}' in {time_building} seconds",
                        time_building=time_building, status=status)
                raise UnexpectedServerStatus(
                    server_id,
                    status,
                    'ACTIVE')

            else:
                raise TransientRetryError()  # just poll again

        sd = server_details(server_endpoint, auth_token, server_id, log=log)
        sd.addCallback(check_status)
        return sd

    timeout_description = ("Waiting for server <{0}> to change from BUILD "
                           "state to ACTIVE state").format(server_id)

    return retry_and_timeout(
        poll, timeout,
        can_retry=transient_errors_except(UnexpectedServerStatus, ServerDeleted),
        next_interval=repeating_interval(interval),
        clock=clock,
        deferred_description=timeout_description)
Esempio n. 19
0
def verified_delete(log,
                    server_endpoint,
                    auth_token,
                    server_id,
                    interval=10,
                    timeout=3660,
                    clock=None):
    """
    Attempt to delete a server from the server endpoint, and ensure that it is
    deleted by trying again until deleting the server results in a 404.

    Time out attempting to verify deletes after a period of time and log an
    error.

    :param log: A bound logger.
    :param str server_endpoint: Server endpoint URI.
    :param str auth_token: Keystone Auth token.
    :param str server_id: Opaque nova server id.
    :param int interval: Deletion interval in seconds - how long until
        verifying a delete is retried. Default: 5.
    :param int timeout: Seconds after which the deletion will be logged as a
        failure, if Nova fails to return a 404.  Default is 3660, because if
        the server is building, the delete will not happen until immediately
        after it has finished building.

    :return: Deferred that fires when the expected status has been seen.
    """
    serv_log = log.bind(server_id=server_id)
    serv_log.msg('Deleting server')

    path = append_segments(server_endpoint, 'servers', server_id)

    if clock is None:  # pragma: no cover
        from twisted.internet import reactor
        clock = reactor

    # just delete over and over until a 404 is received
    def delete():
        del_d = treq.delete(path, headers=headers(auth_token), log=serv_log)
        del_d.addCallback(check_success, [404])
        del_d.addCallback(treq.content)
        return del_d

    start_time = clock.seconds()

    timeout_description = (
        "Waiting for Nova to actually delete server {0}".format(server_id))

    d = retry_and_timeout(delete, timeout,
                          next_interval=repeating_interval(interval),
                          clock=clock,
                          deferred_description=timeout_description)

    def on_success(_):
        time_delete = clock.seconds() - start_time
        serv_log.msg('Server deleted successfully: {time_delete} seconds.',
                     time_delete=time_delete)

    d.addCallback(on_success)
    d.addErrback(serv_log.err)
    return d
Esempio n. 20
0
def wait_for_active(log,
                    server_endpoint,
                    auth_token,
                    server_id,
                    interval=5,
                    timeout=3600,
                    clock=None):
    """
    Wait until the server specified by server_id's status is 'ACTIVE'

    :param log: A bound logger.
    :param str server_endpoint: Server endpoint URI.
    :param str auth_token: Keystone Auth token.
    :param str server_id: Opaque nova server id.
    :param int interval: Polling interval in seconds.  Default: 5.
    :param int timeout: timeout to poll for the server status in seconds.
        Default 3600 (1 hour)

    :return: Deferred that fires when the expected status has been seen.
    """
    log.msg("Checking instance status every {interval} seconds",
            interval=interval)

    if clock is None:  # pragma: no cover
        from twisted.internet import reactor
        clock = reactor

    start_time = clock.seconds()

    def poll():
        def check_status(server):
            status = server['server']['status']

            if status == 'ACTIVE':
                time_building = clock.seconds() - start_time
                log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
                         "{time_building} seconds"),
                        time_building=time_building)
                return server

            elif status != 'BUILD':
                raise UnexpectedServerStatus(server_id, status, 'ACTIVE')

            else:
                raise TransientRetryError()  # just poll again

        sd = server_details(server_endpoint, auth_token, server_id)
        sd.addCallback(check_status)
        return sd

    d = retry_and_timeout(
        poll,
        timeout,
        can_retry=transient_errors_except(UnexpectedServerStatus),
        next_interval=repeating_interval(interval),
        clock=clock)

    def on_error(f):
        if f.check(CancelledError):
            time_building = clock.seconds() - start_time
            log.msg(
                ('Server {instance_id} failed to change from BUILD state '
                 'to ACTIVE within a {timeout} second timeout (it has been '
                 '{time_building} seconds).'),
                timeout=timeout,
                time_building=time_building)
        return f

    d.addErrback(on_error)

    return d
Esempio n. 21
0
def wait_for_active(log,
                    server_endpoint,
                    auth_token,
                    server_id,
                    interval=5,
                    timeout=3600,
                    clock=None):
    """
    Wait until the server specified by server_id's status is 'ACTIVE'

    :param log: A bound logger.
    :param str server_endpoint: Server endpoint URI.
    :param str auth_token: Keystone Auth token.
    :param str server_id: Opaque nova server id.
    :param int interval: Polling interval in seconds.  Default: 5.
    :param int timeout: timeout to poll for the server status in seconds.
        Default 3600 (1 hour)

    :return: Deferred that fires when the expected status has been seen.
    """
    log.msg("Checking instance status every {interval} seconds",
            interval=interval)

    if clock is None:  # pragma: no cover
        from twisted.internet import reactor
        clock = reactor

    start_time = clock.seconds()

    def poll():
        def check_status(server):
            status = server['server']['status']

            if status == 'ACTIVE':
                time_building = clock.seconds() - start_time
                log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
                         "{time_building} seconds"),
                        time_building=time_building)
                return server

            elif status != 'BUILD':
                raise UnexpectedServerStatus(
                    server_id,
                    status,
                    'ACTIVE')

            else:
                raise TransientRetryError()  # just poll again

        sd = server_details(server_endpoint, auth_token, server_id)
        sd.addCallback(check_status)
        return sd

    d = retry_and_timeout(
        poll, timeout,
        can_retry=transient_errors_except(UnexpectedServerStatus),
        next_interval=repeating_interval(interval),
        clock=clock)

    def on_error(f):
        if f.check(CancelledError):
            time_building = clock.seconds() - start_time
            log.msg(('Server {instance_id} failed to change from BUILD state '
                     'to ACTIVE within a {timeout} second timeout (it has been '
                     '{time_building} seconds).'),
                    timeout=timeout, time_building=time_building)
        return f

    d.addErrback(on_error)

    return d