Пример #1
0
def wait_server_state(step, serv_as, state):
    """
    Wait old server in selected state
    """
    server = getattr(world, serv_as, None)
    if not server:
        LOG.info('Wait new server %s in state %s' % (serv_as, state))
        server = world.wait_server_bootstrapping(
            status=ServerStatus.from_code(state))
        setattr(world, serv_as, server)
    else:
        LOG.info('Wait server %s in state %s' % (server.id, state))
        world.wait_server_bootstrapping(status=ServerStatus.from_code(state),
                                        server=server)
Пример #2
0
def wait_server_state(step, serv_as, state):
    """
    Wait old server in selected state
    """
    server = getattr(world, serv_as)
    LOG.info('Wait server %s in state %s' % (server.id, state))
    world.wait_server_bootstrapping(status=ServerStatus.from_code(state), server=server)
Пример #3
0
def farm_servers_state(state):
    world.farm.servers.reload()
    for server in world.farm.servers:
        if server.status == ServerStatus.from_code(state):
            LOG.info('Servers is in %s state' % state)
            continue
        else:
            return False
    return True
Пример #4
0
def having_server(step, state, serv_as):
    server = getattr(world, serv_as)
    world.assert_not_equal(server.status, ServerStatus.from_code(state),
                           "Server %s is not in state %s" % (server.id, state))
Пример #5
0
def having_server(step, state, serv_as):
    server = getattr(world, serv_as)
    world.assert_not_equal(server.status, ServerStatus.from_code(state), "Server %s is not in state %s" % (server.id, state))
Пример #6
0
def assert_server_status(server: Server, status: str):
    expected_status = ServerStatus.from_code(status)
    assert server.status == expected_status, \
        f'Server {server.id}: invalid status. ' \
        f'Actual: {server.status}, expected: {expected_status}'
Пример #7
0
def wait_server_bootstrapping(role=None, status=ServerStatus.RUNNING, timeout=2100, server=None):
    """
    Wait a moment when new server starting in the pointed role and wait server will in selected state.
    Moreover this function remember all previous started servers.

    :param class:Role role: Show in which role lookup a new server
    :return class:Server: Return a new Server
    """
    # NOTE: migrated (wait_status)
    platform = CONF.feature.platform
    status = ServerStatus.from_code(status)

    LOG.info('Launch process looking for new server in farm %s for role %s, wait status %s' %
             (world.farm.id, role, status))

    previous_servers = getattr(world, '_previous_servers', [])
    if not previous_servers:
        world._previous_servers = previous_servers

    LOG.debug('Previous servers: %s' % previous_servers)

    lookup_server = server or None
    lookup_node = None
    azure_failed = 0

    start_time = time.time()

    while time.time() - start_time < timeout:
        if not lookup_server:
            LOG.debug('Reload servers in role')
            if not role:
                world.farm.servers.reload()
                servers = world.farm.servers
            else:
                role.servers.reload()
                servers = role.servers
            for server in servers:
                LOG.debug('Work with server: %s - %s' % (server.id, server.status))
                if not server in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
                                                                        ServerStatus.PENDING,
                                                                        ServerStatus.INIT,
                                                                        ServerStatus.RUNNING]:
                    LOG.debug('I found a server: %s' % server.id)
                    lookup_server = server
        if lookup_server:
            LOG.debug('Reload lookup_server')
            previous_state = lookup_server.status
            lookup_server.reload()

            LOG.debug('Check lookup server terminated?')
            if lookup_server.status in [ServerStatus.TERMINATED,
                                        ServerStatus.PENDING_TERMINATE,
                                        ServerStatus.MISSING] \
                and not status in [ServerStatus.TERMINATED,
                                   ServerStatus.PENDING_TERMINATE,
                                   ServerStatus.MISSING]:
                raise ServerTerminated('Server %s change status to %s (was %s)' %
                                       (lookup_server.id, lookup_server.status,
                                        previous_state))

            LOG.debug('Check lookup server launch failed')
            if lookup_server.is_launch_failed:
                err_msg = (
                    'Can not decode json response data',
                    'Cannot establish connection with CloudStack server. (Server returned nothing )'
                )
                failed_message = lookup_server.get_failed_status_message()
                if platform.is_cloudstack and any(msg in failed_message for msg in err_msg):
                    time.sleep(90)
                    lookup_server = None
                    lookup_node = None
                    continue
                # if platform.is_azure and azure_failed != 2:
                #     LOG.warning('Server %s in Azure and failed %s attempt with message: "%s"' % (
                #         lookup_server.id,
                #         azure_failed + 1,
                #         lookup_server.get_failed_status_message()))
                #     azure_failed += 1
                #     time.sleep(15)
                #     continue
                if status == ServerStatus.FAILED:
                    LOG.debug('Return server because we wait a failed state')
                    return lookup_server
                raise ServerFailed('Server %s failed in %s. Reason: %s'
                                   % (lookup_server.id, ServerStatus.PENDING_LAUNCH,
                                      failed_message))

            LOG.debug('Check lookup server init failed')
            if lookup_server.is_init_failed:
                if status == ServerStatus.FAILED:
                    LOG.debug('Return server because we wait a failed state')
                    return lookup_server
                raise ServerFailed('Server %s failed in %s. Failed (Why?): %s' %
                                   (lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message()))

            LOG.debug('Try get node')
            if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
                                                                ServerStatus.PENDING_TERMINATE,
                                                                ServerStatus.TERMINATED,
                                                                ServerStatus.PENDING_SUSPEND,
                                                                ServerStatus.SUSPENDED] \
                    and status != ServerStatus.PENDING:
                LOG.debug('Try to get node object for lookup server')
                lookup_node = world.cloud.get_node(lookup_server)

            LOG.debug('Verify update log in node')
            if lookup_node and lookup_server.status == ServerStatus.PENDING and status != ServerStatus.PENDING:
                LOG.debug('Check scalarizr update log in lookup server')
                if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
                    verify_scalarizr_log(lookup_node, log_type='update')
                else:
                    if platform != Platform.RACKSPACENGUS:
                        verify_scalarizr_log(lookup_node, log_type='update', windows=True, server=lookup_server)

            LOG.debug('Verify debug log in node')
            if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
                                                            ServerStatus.PENDING_TERMINATE,
                                                            ServerStatus.TERMINATED,
                                                            ServerStatus.PENDING_SUSPEND,
                                                            ServerStatus.SUSPENDED]\
                    and not status == ServerStatus.FAILED:
                LOG.debug('Check scalarizr debug log in lookup server')
                if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
                    verify_scalarizr_log(lookup_node)
                else:
                    verify_scalarizr_log(lookup_node, windows=True, server=lookup_server)

            LOG.debug('If server Running and we wait Initializing, return server')
            if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Initializing but server already Running')
                status = ServerStatus.RUNNING
            if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Resuming but server already Running')
                status = ServerStatus.RUNNING

            LOG.debug('Compare server status "%s" == "%s"' % (lookup_server.status, status))
            if lookup_server.status == status:
                LOG.info('Lookup server in right status now: %s' % lookup_server.status)
                if status == ServerStatus.RUNNING:
                    lookup_server.messages.reload()
                    if platform.is_azure \
                            and not Dist(lookup_server.role.dist).is_windows \
                            and not ('ResumeComplete' in map(lambda m: m.name, lookup_server.messages)) \
                            and lookup_server.is_scalarized:
                        LOG.debug('Wait update ssh authorized keys on azure %s server' % lookup_server.id)
                        wait_server_message(
                            lookup_server,
                            'UpdateSshAuthorizedKeys',
                            timeout=2400)
                    LOG.debug('Insert server to previous servers')
                    previous_servers.append(lookup_server)
                LOG.debug('Return server %s' % lookup_server)
                return lookup_server
        LOG.debug('Sleep 10 seconds')
        time.sleep(10)
    else:
        if lookup_server:
            raise TimeoutError('Server %s not in state "%s" it has status: "%s"'
                               % (lookup_server.id, status, lookup_server.status))
        raise TimeoutError('New server in role "%s" was not founding' % role)
Пример #8
0
def wait_server_bootstrapping(role, status=ServerStatus.RUNNING, timeout=2100):
    """
    Wait new server in role and remember which servers was bootstrapping, return Server
    """
    status = ServerStatus.from_code(status)

    LOG.info('Launch process looking for new server in farm %s for role %s, wait status %s' %
             (world.farm.id, role.role.name, status))

    previous_servers = getattr(world, '_previous_servers', [])
    if not previous_servers:
        world._previous_servers = previous_servers

    LOG.debug('Previous servers: %s' % previous_servers)

    lookup_server = None
    lookup_node = None

    start_time = time.time()

    while time.time() - start_time < timeout:
        if not lookup_server:
            LOG.debug('Reload servers in role')
            role.servers.reload()
            for server in role.servers:
                LOG.debug('Work with server: %s - %s' % (server.id, server.status))
                if not server in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
                                                                        ServerStatus.PENDING,
                                                                        ServerStatus.INIT,
                                                                        ServerStatus.RUNNING]:
                    LOG.debug('I found a server: %s' % server.id)
                    lookup_server = server
        if lookup_server:
            LOG.debug('Reload lookup_server')
            lookup_server.reload()

            LOG.debug('Check lookup server terminated?')
            if lookup_server.status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE]\
                and not status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE]:
                raise ServerTerminated('Server %s change status to %s' % (lookup_server.id, lookup_server.status))

            LOG.debug('Check lookup server launch failed')
            if lookup_server.is_launch_failed:
                raise ServerFailed('Server %s failed in %s' % (lookup_server.id, ServerStatus.PENDING_LAUNCH))

            LOG.debug('Check lookup server init failed')
            if lookup_server.is_init_failed:
                raise ServerFailed('Server %s failed in %s. Failed (Why?): %s' %
                                   (lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message()))

            LOG.debug('Try get node')
            if not lookup_node and not lookup_server.status in [ServerStatus.PENDING_LAUNCH,
                                                                ServerStatus.PENDING_TERMINATE,
                                                                ServerStatus.TERMINATED]:
                LOG.debug('Try to get node object for lookup server')
                lookup_node = world.cloud.get_node(lookup_server)

            LOG.debug('Verify debug log')
            if lookup_node:
                LOG.debug('Check scalarizr log in lookup server')
                verify_scalarizr_log(lookup_node)

            LOG.debug('If server Running and we wait Initializing, return server')
            if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Initializing but server already Running')
                status == ServerStatus.RUNNING

            LOG.debug('Compare server status')
            if lookup_server.status == status:
                LOG.info('Lookup server in right status now: %s' % lookup_server.status)
                if status == ServerStatus.RUNNING:
                    LOG.debug('Insert server to previous servers')
                    previous_servers.append(lookup_server)
                LOG.debug('Return server %s' % lookup_server)
                return lookup_server
        LOG.debug('Sleep 10 seconds')
        time.sleep(10)
    else:
        if lookup_server:
            raise TimeoutError('Server %s not in state "%s" it has status: "%s"' % (lookup_server.id, status, lookup_server.status))
        raise TimeoutError('New server in role "%s" was not founding' % role)
Пример #9
0
def wait_server_bootstrapping(role=None,
                              status=ServerStatus.RUNNING,
                              timeout=2100,
                              server=None):
    """
    Wait a moment when new server starting in the pointed role and wait server will in selected state.
    Moreover this function remember all previous started servers.

    :param class:Role role: Show in which role lookup a new server
    :return class:Server: Return a new Server
    """
    status = ServerStatus.from_code(status)

    LOG.info(
        'Launch process looking for new server in farm %s for role %s, wait status %s'
        % (world.farm.id, role, status))

    previous_servers = getattr(world, '_previous_servers', [])
    if not previous_servers:
        world._previous_servers = previous_servers

    LOG.debug('Previous servers: %s' % previous_servers)

    lookup_server = server or None
    lookup_node = None

    start_time = time.time()

    while time.time() - start_time < timeout:
        if not lookup_server:
            LOG.debug('Reload servers in role')
            if not role:
                world.farm.servers.reload()
                servers = world.farm.servers
            else:
                role.servers.reload()
                servers = role.servers
            for server in servers:
                LOG.debug('Work with server: %s - %s' %
                          (server.id, server.status))
                if not server in previous_servers and server.status in [
                        ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING,
                        ServerStatus.INIT, ServerStatus.RUNNING
                ]:
                    LOG.debug('I found a server: %s' % server.id)
                    lookup_server = server
        if lookup_server:
            LOG.debug('Reload lookup_server')
            previous_state = lookup_server.status
            lookup_server.reload()

            LOG.debug('Check lookup server terminated?')
            if lookup_server.status in [ServerStatus.TERMINATED,
                                        ServerStatus.PENDING_TERMINATE,
                                        ServerStatus.MISSING] \
                and not status in [ServerStatus.TERMINATED,
                                   ServerStatus.PENDING_TERMINATE,
                                   ServerStatus.MISSING]:
                raise ServerTerminated(
                    'Server %s change status to %s (was %s)' %
                    (lookup_server.id, lookup_server.status, previous_state))

            LOG.debug('Check lookup server launch failed')
            if lookup_server.is_launch_failed:
                failed_message = lookup_server.get_failed_status_message()
                if CONF.feature.driver.cloud_family == Platform.CLOUDSTACK \
                and ('Can not decode json response data' in failed_message
                     or 'Cannot establish connection with CloudStack server. (Server returned nothing )' in failed_message):
                    time.sleep(90)
                    lookup_server = None
                    lookup_node = None
                    continue
                if status == ServerStatus.FAILED:
                    LOG.debug('Return server because we wait a failed state')
                    return lookup_server
                raise ServerFailed(
                    'Server %s failed in %s. Reason: %s' %
                    (lookup_server.id, ServerStatus.PENDING_LAUNCH,
                     failed_message))

            LOG.debug('Check lookup server init failed')
            if lookup_server.is_init_failed:
                if status == ServerStatus.FAILED:
                    LOG.debug('Return server because we wait a failed state')
                    return lookup_server
                raise ServerFailed(
                    'Server %s failed in %s. Failed (Why?): %s' %
                    (lookup_server.id, ServerStatus.INIT,
                     lookup_server.get_failed_status_message()))

            LOG.debug('Try get node')
            if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
                                                                ServerStatus.PENDING_TERMINATE,
                                                                ServerStatus.TERMINATED,
                                                                ServerStatus.PENDING_SUSPEND,
                                                                ServerStatus.SUSPENDED] \
                    and CONF.feature.driver.current_cloud != Platform.AZURE:
                LOG.debug('Try to get node object for lookup server')
                lookup_node = world.cloud.get_node(lookup_server)

            LOG.debug('Verify update log in node')
            if lookup_node and lookup_server.status in ServerStatus.PENDING:
                LOG.debug('Check scalarizr update log in lookup server')
                if not Dist(lookup_server.role.dist).is_windows:
                    verify_scalarizr_log(lookup_node, log_type='update')
                else:
                    verify_scalarizr_log(lookup_node,
                                         log_type='update',
                                         windows=True,
                                         server=lookup_server)

            LOG.debug('Verify debug log in node')
            if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
                                                            ServerStatus.PENDING_TERMINATE,
                                                            ServerStatus.TERMINATED,
                                                            ServerStatus.PENDING_SUSPEND,
                                                            ServerStatus.SUSPENDED]\
                    and not status == ServerStatus.FAILED:
                LOG.debug('Check scalarizr debug log in lookup server')
                if not Dist(lookup_server.role.dist).is_windows:
                    verify_scalarizr_log(lookup_node)
                else:
                    verify_scalarizr_log(lookup_node,
                                         windows=True,
                                         server=lookup_server)

            LOG.debug(
                'If server Running and we wait Initializing, return server')
            if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Initializing but server already Running')
                status = ServerStatus.RUNNING
            if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Resuming but server already Running')
                status = ServerStatus.RUNNING

            LOG.debug('Compare server status')
            if lookup_server.status == status:
                LOG.info('Lookup server in right status now: %s' %
                         lookup_server.status)
                if status == ServerStatus.RUNNING:
                    LOG.debug('Insert server to previous servers')
                    previous_servers.append(lookup_server)
                LOG.debug('Return server %s' % lookup_server)
                return lookup_server
        LOG.debug('Sleep 10 seconds')
        time.sleep(10)
    else:
        if lookup_server:
            raise TimeoutError(
                'Server %s not in state "%s" it has status: "%s"' %
                (lookup_server.id, status, lookup_server.status))
        raise TimeoutError('New server in role "%s" was not founding' % role)
Пример #10
0
def wait_server_status(context: dict,
                       cloud: Cloud,
                       farm: Farm,
                       role: FarmRole = None,
                       status: str = ServerStatus.RUNNING,
                       timeout: int = 2100,
                       server: Server = None) -> Server:
    platform = CONF.feature.platform
    status = ServerStatus.from_code(status)

    LOG.info(f'Launch process looking for new server in farm {farm.id} for role {role}, wait status {status}')

    previous_servers = context.get('_previous_servers', [])
    if not previous_servers:
        context['_previous_servers'] = previous_servers

    LOG.debug(f'Previous servers: {previous_servers}')

    lookup_server = server or None
    lookup_node = None

    start_time = time.time()

    while time.time() - start_time < timeout:
        if not lookup_server:
            LOG.debug('Reload servers in role')
            if not role:
                farm.servers.reload()
                servers = farm.servers
            else:
                role.servers.reload()
                servers = role.servers
            for server in servers:
                LOG.debug(f'Work with server: {server.id} - {server.status}')
                if server not in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
                                                                        ServerStatus.PENDING,
                                                                        ServerStatus.INIT,
                                                                        ServerStatus.RUNNING]:
                    LOG.debug(f'I found a server: {server.id}')
                    lookup_server = server
        if lookup_server:
            LOG.debug('Reload lookup_server')
            previous_state = lookup_server.status
            lookup_server.reload()

            LOG.debug('Check lookup server terminated?')
            if lookup_server.status in [ServerStatus.TERMINATED,
                                        ServerStatus.PENDING_TERMINATE,
                                        ServerStatus.MISSING] \
                    and status not in [ServerStatus.TERMINATED,
                                       ServerStatus.PENDING_TERMINATE,
                                       ServerStatus.MISSING]:
                raise ServerTerminated(f'Server {lookup_server.id} change status to {lookup_server.status} '
                                       f'(was {previous_state})')

            LOG.debug('Check lookup server launch failed')
            if lookup_server.is_launch_failed:
                err_msg = (
                    'Can not decode json response data',
                    'Cannot establish connection with CloudStack server. (Server returned nothing )'
                )
                failed_message = lookup_server.get_failed_status_message()
                if platform.is_cloudstack and any(msg in failed_message for msg in err_msg):
                    time.sleep(90)
                    lookup_server = None
                    lookup_node = None
                    continue
                if status == ServerStatus.FAILED:
                    LOG.debug('Return server because we wait a failed state')
                    return lookup_server
                raise ServerFailed(f'Server {lookup_server.id} failed in {ServerStatus.PENDING_LAUNCH}. '
                                   f'Reason: {failed_message}')

            LOG.debug('Check lookup server init failed')
            if lookup_server.is_init_failed:
                if status == ServerStatus.FAILED:
                    LOG.debug('Return server because we wait a failed state')
                    return lookup_server
                raise ServerFailed(f'Server {lookup_server.id} failed in {ServerStatus.INIT}. Failed (Why?): '
                                   f'{lookup_server.get_failed_status_message()}')

            LOG.debug('Try get node')
            if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
                                                                ServerStatus.PENDING_TERMINATE,
                                                                ServerStatus.TERMINATED,
                                                                ServerStatus.PENDING_SUSPEND,
                                                                ServerStatus.SUSPENDED] \
                    and status != ServerStatus.PENDING:
                LOG.debug('Try to get node object for lookup server')
                lookup_node = cloud.get_node(lookup_server)

            LOG.debug('Verify update log in node')
            if lookup_node and lookup_server.status == ServerStatus.PENDING and status != ServerStatus.PENDING:
                LOG.debug('Check scalarizr update log in lookup server')
                if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
                    # TODO: verify_scalarizr_log(lookup_node, log_type='update')
                    pass
                else:
                    if platform != Platform.RACKSPACENGUS:
                        # TODO: verify_scalarizr_log(lookup_node, log_type='update', windows=True,
                        # server=lookup_server)
                        pass

            LOG.debug('Verify debug log in node')
            if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
                                                            ServerStatus.PENDING_TERMINATE,
                                                            ServerStatus.TERMINATED,
                                                            ServerStatus.PENDING_SUSPEND,
                                                            ServerStatus.SUSPENDED] \
                    and not status == ServerStatus.FAILED:
                LOG.debug('Check scalarizr debug log in lookup server')
                if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
                    # TODO: verify_scalarizr_log(lookup_node)
                    pass
                else:
                    # TODO: verify_scalarizr_log(lookup_node, windows=True, server=lookup_server)
                    pass

            LOG.debug('If server Running and we wait Initializing, return server')
            if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Initializing but server already Running')
                status = ServerStatus.RUNNING
            if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
                LOG.info('We wait Resuming but server already Running')
                status = ServerStatus.RUNNING

            LOG.debug(f'Compare server status "{lookup_server.status}" == "{status}"')
            if lookup_server.status == status:
                LOG.info(f'Lookup server in right status now: {lookup_server.status}')
                if status == ServerStatus.RUNNING:
                    lookup_server.messages.reload()
                    if platform.is_azure \
                            and not Dist(lookup_server.role.dist).is_windows \
                            and not ('ResumeComplete' in map(lambda m: m.name, lookup_server.messages)) \
                            and lookup_server.is_scalarized:
                        LOG.debug(f'Wait update ssh authorized keys on azure {lookup_server.id} server')
                        # TODO: wait_server_message(
                        #     lookup_server,
                        #     'UpdateSshAuthorizedKeys',
                        #     timeout=2400)
                    LOG.debug('Insert server to previous servers')
                    previous_servers.append(lookup_server)
                LOG.debug(f'Return server {lookup_server}')
                return lookup_server
        LOG.debug('Sleep 10 seconds')
        time.sleep(10)
    else:
        if lookup_server:
            raise TimeoutError(f'Server {lookup_server.id} not in state "{status}" '
                               f'it has status: "{lookup_server.status}"')
        raise TimeoutError(f'New server in role "{role}" was not founding')