def wait_server_state(step, serv_as, state):
"""
Wait old server in selected state
"""
server = getattr(world, serv_as, None)
if not server:
LOG.info('Wait new server %s in state %s' % (serv_as, state))
server = world.wait_server_bootstrapping(
status=ServerStatus.from_code(state))
setattr(world, serv_as, server)
else:
LOG.info('Wait server %s in state %s' % (server.id, state))
world.wait_server_bootstrapping(status=ServerStatus.from_code(state),
server=server)
def wait_server_state(step, serv_as, state):
"""
Wait old server in selected state
"""
server = getattr(world, serv_as)
LOG.info('Wait server %s in state %s' % (server.id, state))
world.wait_server_bootstrapping(status=ServerStatus.from_code(state), server=server)
def farm_servers_state(state):
world.farm.servers.reload()
for server in world.farm.servers:
if server.status == ServerStatus.from_code(state):
LOG.info('Servers is in %s state' % state)
continue
else:
return False
return True
def having_server(step, state, serv_as):
server = getattr(world, serv_as)
world.assert_not_equal(server.status, ServerStatus.from_code(state),
"Server %s is not in state %s" % (server.id, state))
def having_server(step, state, serv_as):
server = getattr(world, serv_as)
world.assert_not_equal(server.status, ServerStatus.from_code(state), "Server %s is not in state %s" % (server.id, state))
def assert_server_status(server: Server, status: str):
expected_status = ServerStatus.from_code(status)
assert server.status == expected_status, \
f'Server {server.id}: invalid status. ' \
f'Actual: {server.status}, expected: {expected_status}'
def wait_server_bootstrapping(role=None, status=ServerStatus.RUNNING, timeout=2100, server=None):
"""
Wait a moment when new server starting in the pointed role and wait server will in selected state.
Moreover this function remember all previous started servers.
:param class:Role role: Show in which role lookup a new server
:return class:Server: Return a new Server
"""
# NOTE: migrated (wait_status)
platform = CONF.feature.platform
status = ServerStatus.from_code(status)
LOG.info('Launch process looking for new server in farm %s for role %s, wait status %s' %
(world.farm.id, role, status))
previous_servers = getattr(world, '_previous_servers', [])
if not previous_servers:
world._previous_servers = previous_servers
LOG.debug('Previous servers: %s' % previous_servers)
lookup_server = server or None
lookup_node = None
azure_failed = 0
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
if not role:
world.farm.servers.reload()
servers = world.farm.servers
else:
role.servers.reload()
servers = role.servers
for server in servers:
LOG.debug('Work with server: %s - %s' % (server.id, server.status))
if not server in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING,
ServerStatus.INIT,
ServerStatus.RUNNING]:
LOG.debug('I found a server: %s' % server.id)
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
previous_state = lookup_server.status
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING] \
and not status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING]:
raise ServerTerminated('Server %s change status to %s (was %s)' %
(lookup_server.id, lookup_server.status,
previous_state))
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
err_msg = (
'Can not decode json response data',
'Cannot establish connection with CloudStack server. (Server returned nothing )'
)
failed_message = lookup_server.get_failed_status_message()
if platform.is_cloudstack and any(msg in failed_message for msg in err_msg):
time.sleep(90)
lookup_server = None
lookup_node = None
continue
# if platform.is_azure and azure_failed != 2:
# LOG.warning('Server %s in Azure and failed %s attempt with message: "%s"' % (
# lookup_server.id,
# azure_failed + 1,
# lookup_server.get_failed_status_message()))
# azure_failed += 1
# time.sleep(15)
# continue
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed('Server %s failed in %s. Reason: %s'
% (lookup_server.id, ServerStatus.PENDING_LAUNCH,
failed_message))
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed('Server %s failed in %s. Failed (Why?): %s' %
(lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message()))
LOG.debug('Try get node')
if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED] \
and status != ServerStatus.PENDING:
LOG.debug('Try to get node object for lookup server')
lookup_node = world.cloud.get_node(lookup_server)
LOG.debug('Verify update log in node')
if lookup_node and lookup_server.status == ServerStatus.PENDING and status != ServerStatus.PENDING:
LOG.debug('Check scalarizr update log in lookup server')
if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
verify_scalarizr_log(lookup_node, log_type='update')
else:
if platform != Platform.RACKSPACENGUS:
verify_scalarizr_log(lookup_node, log_type='update', windows=True, server=lookup_server)
LOG.debug('Verify debug log in node')
if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED]\
and not status == ServerStatus.FAILED:
LOG.debug('Check scalarizr debug log in lookup server')
if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
verify_scalarizr_log(lookup_node)
else:
verify_scalarizr_log(lookup_node, windows=True, server=lookup_server)
LOG.debug('If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status = ServerStatus.RUNNING
if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Resuming but server already Running')
status = ServerStatus.RUNNING
LOG.debug('Compare server status "%s" == "%s"' % (lookup_server.status, status))
if lookup_server.status == status:
LOG.info('Lookup server in right status now: %s' % lookup_server.status)
if status == ServerStatus.RUNNING:
lookup_server.messages.reload()
if platform.is_azure \
and not Dist(lookup_server.role.dist).is_windows \
and not ('ResumeComplete' in map(lambda m: m.name, lookup_server.messages)) \
and lookup_server.is_scalarized:
LOG.debug('Wait update ssh authorized keys on azure %s server' % lookup_server.id)
wait_server_message(
lookup_server,
'UpdateSshAuthorizedKeys',
timeout=2400)
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug('Return server %s' % lookup_server)
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError('Server %s not in state "%s" it has status: "%s"'
% (lookup_server.id, status, lookup_server.status))
raise TimeoutError('New server in role "%s" was not founding' % role)
def wait_server_bootstrapping(role, status=ServerStatus.RUNNING, timeout=2100):
"""
Wait new server in role and remember which servers was bootstrapping, return Server
"""
status = ServerStatus.from_code(status)
LOG.info('Launch process looking for new server in farm %s for role %s, wait status %s' %
(world.farm.id, role.role.name, status))
previous_servers = getattr(world, '_previous_servers', [])
if not previous_servers:
world._previous_servers = previous_servers
LOG.debug('Previous servers: %s' % previous_servers)
lookup_server = None
lookup_node = None
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
role.servers.reload()
for server in role.servers:
LOG.debug('Work with server: %s - %s' % (server.id, server.status))
if not server in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING,
ServerStatus.INIT,
ServerStatus.RUNNING]:
LOG.debug('I found a server: %s' % server.id)
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE]\
and not status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE]:
raise ServerTerminated('Server %s change status to %s' % (lookup_server.id, lookup_server.status))
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
raise ServerFailed('Server %s failed in %s' % (lookup_server.id, ServerStatus.PENDING_LAUNCH))
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
raise ServerFailed('Server %s failed in %s. Failed (Why?): %s' %
(lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message()))
LOG.debug('Try get node')
if not lookup_node and not lookup_server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED]:
LOG.debug('Try to get node object for lookup server')
lookup_node = world.cloud.get_node(lookup_server)
LOG.debug('Verify debug log')
if lookup_node:
LOG.debug('Check scalarizr log in lookup server')
verify_scalarizr_log(lookup_node)
LOG.debug('If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status == ServerStatus.RUNNING
LOG.debug('Compare server status')
if lookup_server.status == status:
LOG.info('Lookup server in right status now: %s' % lookup_server.status)
if status == ServerStatus.RUNNING:
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug('Return server %s' % lookup_server)
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError('Server %s not in state "%s" it has status: "%s"' % (lookup_server.id, status, lookup_server.status))
raise TimeoutError('New server in role "%s" was not founding' % role)
def wait_server_bootstrapping(role=None,
status=ServerStatus.RUNNING,
timeout=2100,
server=None):
"""
Wait a moment when new server starting in the pointed role and wait server will in selected state.
Moreover this function remember all previous started servers.
:param class:Role role: Show in which role lookup a new server
:return class:Server: Return a new Server
"""
status = ServerStatus.from_code(status)
LOG.info(
'Launch process looking for new server in farm %s for role %s, wait status %s'
% (world.farm.id, role, status))
previous_servers = getattr(world, '_previous_servers', [])
if not previous_servers:
world._previous_servers = previous_servers
LOG.debug('Previous servers: %s' % previous_servers)
lookup_server = server or None
lookup_node = None
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
if not role:
world.farm.servers.reload()
servers = world.farm.servers
else:
role.servers.reload()
servers = role.servers
for server in servers:
LOG.debug('Work with server: %s - %s' %
(server.id, server.status))
if not server in previous_servers and server.status in [
ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING,
ServerStatus.INIT, ServerStatus.RUNNING
]:
LOG.debug('I found a server: %s' % server.id)
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
previous_state = lookup_server.status
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING] \
and not status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING]:
raise ServerTerminated(
'Server %s change status to %s (was %s)' %
(lookup_server.id, lookup_server.status, previous_state))
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
failed_message = lookup_server.get_failed_status_message()
if CONF.feature.driver.cloud_family == Platform.CLOUDSTACK \
and ('Can not decode json response data' in failed_message
or 'Cannot establish connection with CloudStack server. (Server returned nothing )' in failed_message):
time.sleep(90)
lookup_server = None
lookup_node = None
continue
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(
'Server %s failed in %s. Reason: %s' %
(lookup_server.id, ServerStatus.PENDING_LAUNCH,
failed_message))
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(
'Server %s failed in %s. Failed (Why?): %s' %
(lookup_server.id, ServerStatus.INIT,
lookup_server.get_failed_status_message()))
LOG.debug('Try get node')
if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED] \
and CONF.feature.driver.current_cloud != Platform.AZURE:
LOG.debug('Try to get node object for lookup server')
lookup_node = world.cloud.get_node(lookup_server)
LOG.debug('Verify update log in node')
if lookup_node and lookup_server.status in ServerStatus.PENDING:
LOG.debug('Check scalarizr update log in lookup server')
if not Dist(lookup_server.role.dist).is_windows:
verify_scalarizr_log(lookup_node, log_type='update')
else:
verify_scalarizr_log(lookup_node,
log_type='update',
windows=True,
server=lookup_server)
LOG.debug('Verify debug log in node')
if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED]\
and not status == ServerStatus.FAILED:
LOG.debug('Check scalarizr debug log in lookup server')
if not Dist(lookup_server.role.dist).is_windows:
verify_scalarizr_log(lookup_node)
else:
verify_scalarizr_log(lookup_node,
windows=True,
server=lookup_server)
LOG.debug(
'If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status = ServerStatus.RUNNING
if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Resuming but server already Running')
status = ServerStatus.RUNNING
LOG.debug('Compare server status')
if lookup_server.status == status:
LOG.info('Lookup server in right status now: %s' %
lookup_server.status)
if status == ServerStatus.RUNNING:
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug('Return server %s' % lookup_server)
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError(
'Server %s not in state "%s" it has status: "%s"' %
(lookup_server.id, status, lookup_server.status))
raise TimeoutError('New server in role "%s" was not founding' % role)
def wait_server_status(context: dict,
cloud: Cloud,
farm: Farm,
role: FarmRole = None,
status: str = ServerStatus.RUNNING,
timeout: int = 2100,
server: Server = None) -> Server:
platform = CONF.feature.platform
status = ServerStatus.from_code(status)
LOG.info(f'Launch process looking for new server in farm {farm.id} for role {role}, wait status {status}')
previous_servers = context.get('_previous_servers', [])
if not previous_servers:
context['_previous_servers'] = previous_servers
LOG.debug(f'Previous servers: {previous_servers}')
lookup_server = server or None
lookup_node = None
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
if not role:
farm.servers.reload()
servers = farm.servers
else:
role.servers.reload()
servers = role.servers
for server in servers:
LOG.debug(f'Work with server: {server.id} - {server.status}')
if server not in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING,
ServerStatus.INIT,
ServerStatus.RUNNING]:
LOG.debug(f'I found a server: {server.id}')
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
previous_state = lookup_server.status
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING] \
and status not in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING]:
raise ServerTerminated(f'Server {lookup_server.id} change status to {lookup_server.status} '
f'(was {previous_state})')
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
err_msg = (
'Can not decode json response data',
'Cannot establish connection with CloudStack server. (Server returned nothing )'
)
failed_message = lookup_server.get_failed_status_message()
if platform.is_cloudstack and any(msg in failed_message for msg in err_msg):
time.sleep(90)
lookup_server = None
lookup_node = None
continue
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(f'Server {lookup_server.id} failed in {ServerStatus.PENDING_LAUNCH}. '
f'Reason: {failed_message}')
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(f'Server {lookup_server.id} failed in {ServerStatus.INIT}. Failed (Why?): '
f'{lookup_server.get_failed_status_message()}')
LOG.debug('Try get node')
if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED] \
and status != ServerStatus.PENDING:
LOG.debug('Try to get node object for lookup server')
lookup_node = cloud.get_node(lookup_server)
LOG.debug('Verify update log in node')
if lookup_node and lookup_server.status == ServerStatus.PENDING and status != ServerStatus.PENDING:
LOG.debug('Check scalarizr update log in lookup server')
if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
# TODO: verify_scalarizr_log(lookup_node, log_type='update')
pass
else:
if platform != Platform.RACKSPACENGUS:
# TODO: verify_scalarizr_log(lookup_node, log_type='update', windows=True,
# server=lookup_server)
pass
LOG.debug('Verify debug log in node')
if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED] \
and not status == ServerStatus.FAILED:
LOG.debug('Check scalarizr debug log in lookup server')
if not Dist(lookup_server.role.dist).is_windows and not platform.is_azure:
# TODO: verify_scalarizr_log(lookup_node)
pass
else:
# TODO: verify_scalarizr_log(lookup_node, windows=True, server=lookup_server)
pass
LOG.debug('If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status = ServerStatus.RUNNING
if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Resuming but server already Running')
status = ServerStatus.RUNNING
LOG.debug(f'Compare server status "{lookup_server.status}" == "{status}"')
if lookup_server.status == status:
LOG.info(f'Lookup server in right status now: {lookup_server.status}')
if status == ServerStatus.RUNNING:
lookup_server.messages.reload()
if platform.is_azure \
and not Dist(lookup_server.role.dist).is_windows \
and not ('ResumeComplete' in map(lambda m: m.name, lookup_server.messages)) \
and lookup_server.is_scalarized:
LOG.debug(f'Wait update ssh authorized keys on azure {lookup_server.id} server')
# TODO: wait_server_message(
# lookup_server,
# 'UpdateSshAuthorizedKeys',
# timeout=2400)
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug(f'Return server {lookup_server}')
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError(f'Server {lookup_server.id} not in state "{status}" '
f'it has status: "{lookup_server.status}"')
raise TimeoutError(f'New server in role "{role}" was not founding')
