def check_instance_configs(step, instance_port, search_condition, serv_as):
redis_path = {
'debian': {
'conf': '/etc/redis',
'data': '/mnt/redisstorage'
},
'centos': {
'conf': '/etc',
'data': '/mnt/redisstorage'
},
}
server = getattr(world, serv_as)
node = world.cloud.get_node(server)
# Get redis instance search condition
if search_condition == 'configuration':
path = redis_path.get(Dist(node.os).family)['conf']
file = 'redis.{0}.conf'.format(instance_port)
else:
path = redis_path.get(Dist(node.os).family)['data']
file = 'appendonly.{0}.aof'.format(instance_port)
command = 'find {0} -name {1}'.format(path, file)
LOG.debug(
'Search condition: ({0}/{1}) to find config for redis instance {2}'.
format(path, file, instance_port))
res = node.run(command)
LOG.debug(
'The result: ({0}) of the find command for redis instance {1}'.format(
res[0], instance_port))
assertion_message = 'Redis instance {0} config file ({1}) was not deleted.'.format(
instance_port, res[0])
assert not (res[0]), assertion_message
LOG.debug(
'Redis instance {0} config file ({1}/{2}) was successfully deleted.'.
format(instance_port, path, file))
def assert_scalarizr_version(step, branch, serv_as):
"""
Argument branch can be system or role.
System branch - CONF.feature.branch
Role branch - CONF.feature.to_branch
"""
#FIXME: Rewrite this ugly code!
server = getattr(world, serv_as)
if branch == 'system' or not branch:
branch = CONF.feature.branch
elif branch == 'role':
branch = CONF.feature.to_branch
# Get custom repo url
os_family = Dist(server.role.dist).family
if '.' in branch and branch.replace('.', '').isdigit():
last_version = branch
else:
if branch in ['stable', 'latest']:
default_repo = DEFAULT_SCALARIZR_RELEASE_REPOS[os_family]
else:
url = DEFAULT_SCALARIZR_DEVEL_REPOS['url'][CONF.feature.ci_repo]
path = DEFAULT_SCALARIZR_DEVEL_REPOS['path'][os_family]
default_repo = url.format(path=path)
# Get last scalarizr version from custom repo
index_url = default_repo.format(branch=branch)
LOG.debug('Check package from index_url: %s' % index_url)
repo_data = parser_for_os_family(server.role.dist)(branch=branch,
index_url=index_url)
versions = [
package['version'] for package in repo_data
if package['name'] == 'scalarizr'
]
versions.sort(reverse=True)
last_version = versions[0]
if last_version.strip().endswith('-1'):
last_version = last_version.strip()[:-2]
LOG.debug('Last scalarizr version %s for branch %s' %
(last_version, branch))
# Get installed scalarizr version
for _ in range(5):
try:
update_status = server.upd_api.status(cached=False)
installed_version = update_status['installed']
if installed_version.strip().endswith('-1'):
installed_version = installed_version.strip()[:-2]
break
except urllib2.URLError:
time.sleep(3)
else:
raise AssertionError(
'Can\'t get access to update client 5 times (15 seconds)')
LOG.debug('Last scalarizr version from update client status: %s' %
update_status['installed'])
assert update_status['state'] == 'completed', \
'Update client not in normal state. Status = "%s", Previous state = "%s"' % \
(update_status['state'], update_status['prev_state'])
assert last_version == installed_version, \
'Server not has last build of scalarizr package, installed: %s last_version: %s' % (installed_version, last_version)
def change_branch_in_sources(step, serv_as, branch):
if 'system' in branch:
branch = CONF.feature.branch
elif not branch.strip():
branch = CONF.feature.to_branch
else:
branch = branch.replace('/', '-').replace('.', '').strip()
server = getattr(world, serv_as)
LOG.info('Change branches in sources list in server %s to %s' %
(server.id, branch))
if Dist(server.role.dist).is_debian:
LOG.debug('Change in debian')
node = world.cloud.get_node(server)
for repo_file in [
'/etc/apt/sources.list.d/scalr-stable.list',
'/etc/apt/sources.list.d/scalr-latest.list'
]:
LOG.info("Change branch in %s to %s" % (repo_file, branch))
node.run(
'echo "deb http://buildbot.scalr-labs.com/apt/debian %s/" > %s'
% (branch, repo_file))
elif Dist(server.role.dist).is_centos:
LOG.debug('Change in centos')
node = world.cloud.get_node(server)
for repo_file in ['/etc/yum.repos.d/scalr-stable.repo']:
LOG.info("Change branch in %s to %s" % (repo_file, branch))
node.run(
'echo "[scalr-branch]\nname=scalr-branch\nbaseurl=http://buildbot.scalr-labs.com/rpm/%s/rhel/\$releasever/\$basearch\nenabled=1\ngpgcheck=0" > %s'
% (branch, repo_file))
node.run('echo > /etc/yum.repos.d/scalr-latest.repo')
elif Dist(server.role.dist).is_windows:
# LOG.debug('Change in windows')
import winrm
console = winrm.Session('http://%s:5985/wsman' % server.public_ip,
auth=("Administrator",
server.windows_password))
for repo_file in [
'C:\Program Files\Scalarizr\etc\scalr-latest.winrepo',
'C:\Program Files\Scalarizr\etc\scalr-stable.winrepo'
]:
# LOG.info("Change branch in %s to %s" % (repo_file, branch))
console.run_cmd(
'echo http://buildbot.scalr-labs.com/win/%s/x86_64/ > "%s"' %
(branch, repo_file))
def _verify_app(server, port):
LOG.info('Verify apache (%s) work in server %s' % (port, server.id))
node = world.cloud.get_node(server)
results = [
VerifyProcessWork._verify_process_running(
server, DEFAULT_SERVICES_CONFIG['app'][Dist(
node.os).family]['service_name']),
VerifyProcessWork._verify_open_port(server, port)
]
return all(results)
def check_process_options(step, process, options, serv_as):
server = getattr(world, serv_as)
LOG.debug('Want check process %s and options %s' % (process, options))
node = world.cloud.get_node(server)
for attempt in range(3):
out = node.run('ps aux | grep %s' % process)
LOG.debug('Grep for ps aux: %s' % out[0])
for line in out[0].splitlines():
if 'grep' in line:
continue
LOG.info('Work with line: %s' % line)
if options not in line and not CONF.feature.dist == Dist(
'amzn1609') and not CONF.feature.dist.is_systemd:
raise AssertionError('Options %s not in process, %s' %
(options, ' '.join(line.split()[10:])))
else:
return True
raise AssertionError('Not found process: %s' % process)
def wait_server_bootstrapping(role=None,
status=ServerStatus.RUNNING,
timeout=2100,
server=None):
"""
Wait a moment when new server starting in the pointed role and wait server will in selected state.
Moreover this function remember all previous started servers.
:param class:Role role: Show in which role lookup a new server
:return class:Server: Return a new Server
"""
status = ServerStatus.from_code(status)
LOG.info(
'Launch process looking for new server in farm %s for role %s, wait status %s'
% (world.farm.id, role, status))
previous_servers = getattr(world, '_previous_servers', [])
if not previous_servers:
world._previous_servers = previous_servers
LOG.debug('Previous servers: %s' % previous_servers)
lookup_server = server or None
lookup_node = None
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
if not role:
world.farm.servers.reload()
servers = world.farm.servers
else:
role.servers.reload()
servers = role.servers
for server in servers:
LOG.debug('Work with server: %s - %s' %
(server.id, server.status))
if not server in previous_servers and server.status in [
ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING,
ServerStatus.INIT, ServerStatus.RUNNING
]:
LOG.debug('I found a server: %s' % server.id)
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
previous_state = lookup_server.status
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING] \
and not status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING]:
raise ServerTerminated(
'Server %s change status to %s (was %s)' %
(lookup_server.id, lookup_server.status, previous_state))
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
failed_message = lookup_server.get_failed_status_message()
if CONF.feature.driver.cloud_family == Platform.CLOUDSTACK \
and ('Can not decode json response data' in failed_message
or 'Cannot establish connection with CloudStack server. (Server returned nothing )' in failed_message):
time.sleep(90)
lookup_server = None
lookup_node = None
continue
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(
'Server %s failed in %s. Reason: %s' %
(lookup_server.id, ServerStatus.PENDING_LAUNCH,
failed_message))
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(
'Server %s failed in %s. Failed (Why?): %s' %
(lookup_server.id, ServerStatus.INIT,
lookup_server.get_failed_status_message()))
LOG.debug('Try get node')
if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED] \
and CONF.feature.driver.current_cloud != Platform.AZURE:
LOG.debug('Try to get node object for lookup server')
lookup_node = world.cloud.get_node(lookup_server)
LOG.debug('Verify update log in node')
if lookup_node and lookup_server.status in ServerStatus.PENDING:
LOG.debug('Check scalarizr update log in lookup server')
if not Dist(lookup_server.role.dist).is_windows:
verify_scalarizr_log(lookup_node, log_type='update')
else:
verify_scalarizr_log(lookup_node,
log_type='update',
windows=True,
server=lookup_server)
LOG.debug('Verify debug log in node')
if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED]\
and not status == ServerStatus.FAILED:
LOG.debug('Check scalarizr debug log in lookup server')
if not Dist(lookup_server.role.dist).is_windows:
verify_scalarizr_log(lookup_node)
else:
verify_scalarizr_log(lookup_node,
windows=True,
server=lookup_server)
LOG.debug(
'If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status = ServerStatus.RUNNING
if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Resuming but server already Running')
status = ServerStatus.RUNNING
LOG.debug('Compare server status')
if lookup_server.status == status:
LOG.info('Lookup server in right status now: %s' %
lookup_server.status)
if status == ServerStatus.RUNNING:
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug('Return server %s' % lookup_server)
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError(
'Server %s not in state "%s" it has status: "%s"' %
(lookup_server.id, status, lookup_server.status))
raise TimeoutError('New server in role "%s" was not founding' % role)
def get_all_logs_and_info(scenario, outline='', outline_failed=None):
if CONF.feature.driver.current_cloud == Platform.AZURE:
return
# Get Farm
LOG.warning('Get scalarizr logs after scenario %s' % scenario.name)
farm = getattr(world, 'farm', None)
if not farm:
LOG.error("Farm does not exists. Can't get logs. Exit from step.")
return
farm.servers.reload()
# Get servers
servers = farm.servers
# Get test
test_name = scenario.described_at.file.split('/')[-1].split('.')[0]
LOG.debug('Test name: %s' % test_name)
# Get path
start_time = world.test_start_time
path = os.path.realpath(
os.path.join(CONF.main.log_path, 'scalarizr', test_name,
start_time.strftime('%m%d-%H:%M'),
scenario.name.replace('/', '-'), outline))
LOG.debug('Path to save log: %s' % path)
if not os.path.exists(path):
os.makedirs(path, 0755)
# Get logs && configs
for server in servers:
if not server.is_scalarized: continue
logs = [
# debug log
{
'file':
os.path.join(path, '_'.join(
(server.id, 'scalarizr_debug.log'))),
'log_type':
'debug',
'compress':
True
},
# update log
{
'file':
os.path.join(path, '_'.join(
(server.id, 'scalarizr_update.log'))),
'log_type':
'update',
'compress':
True
}
]
if server.status in [
ServerStatus.PENDING, ServerStatus.INIT, ServerStatus.RUNNING
]:
try:
#Get log from remote host
for log in logs:
server.get_log_by_api(**log)
LOG.info(
'Save {log_type} log from server {} to {file}'.format(
server.id, **log))
#Get configs and role behavior from remote host only for linux family
if not Dist(server.role.dist).is_windows:
file = os.path.join(
path, '_'.join(
(server.id, 'scalr_configs.tar.gz')))
server.get_configs(file, compress=True)
LOG.info(
'Download archive with scalr directory and behavior to: {}'
.format(file))
except BaseException, e:
LOG.error('Error in downloading configs: %s' % e)
continue
if server.status == ServerStatus.RUNNING and not CONF.feature.dist.is_windows:
node = world.cloud.get_node(server)
out = node.run("ps aux | grep 'bin/scal'")[0]
for line in out.splitlines():
ram = line.split()[5]
if len(ram) > 3:
ram = '%sMB' % ram[:-3]
if 'bin/scalr-upd-client' in line:
LOG.info('Server %s use %s RAM for update-client' %
(server.id, ram))
world.wrt(
etree.Element('meta',
name='szrupdateram',
value=ram,
serverid=server.id))
elif 'bin/scalarizr' in line:
LOG.info('Server %s use %s RAM for scalarizr' %
(server.id, ram))
world.wrt(
etree.Element('meta',
name='szrram',
value=ram,
serverid=server.id))
