def check_instance_configs(step, instance_port, search_condition, serv_as):
redis_path = {
'debian': {
'conf': '/etc/redis',
'data': '/mnt/redisstorage'
},
'centos': {
'conf': '/etc',
'data': '/mnt/redisstorage'
},
}
server = getattr(world, serv_as)
node = world.cloud.get_node(server)
# Get redis instance search condition
if search_condition == 'configuration':
path = redis_path.get(Dist(node.os).family)['conf']
file = 'redis.{0}.conf'.format(instance_port)
else:
path = redis_path.get(Dist(node.os).family)['data']
file = 'appendonly.{0}.aof'.format(instance_port)
command = 'find {0} -name {1}'.format(path, file)
LOG.debug(
'Search condition: ({0}/{1}) to find config for redis instance {2}'.
format(path, file, instance_port))
res = node.run(command)
LOG.debug(
'The result: ({0}) of the find command for redis instance {1}'.format(
res[0], instance_port))
assertion_message = 'Redis instance {0} config file ({1}) was not deleted.'.format(
instance_port, res[0])
assert not (res[0]), assertion_message
LOG.debug(
'Redis instance {0} config file ({1}/{2}) was successfully deleted.'.
format(instance_port, path, file))
def restore(self, src_path, db=None):
#Kill redis-server
LOG.info('Stopping Redis server.')
out = self.node.run("pgrep -l redis-server | awk {print'$1'} | xargs -i{} kill {} && sleep 5 && pgrep -l redis-server | awk {print'$1'}")[0]
if out:
raise AssertionError('Redis server, pid:%s was not properly killed on remote host %s' % (out, self.server.public_ip))
LOG.info('Redis server was successfully stopped. Getting backups and moving to redis storage.')
#Move dump to redis storage
out = self.node.run("find %s -name '*%s*' -print0 | xargs -i{} -0 -r cp -v {} /mnt/redisstorage/" %
(src_path, self.snapshotting_type))
if not out[0]:
raise AssertionError("Can't move dump to redis-server storage. Error is: %s" % out[1])
LOG.info('Available backups in server: %s. Backups was successfully moved to redis storage.' % out[0].split()[0])
#Run redis-server
LOG.info('Running Redis server.')
out = self.node.run("/bin/su redis -s /bin/bash -c \"%(bin)s %(conf)s\" && sleep 5 && pgrep -l redis-server | awk {print'$1'}" %
{
'bin': os.path.join(self.redis_path.get(Dist.get_os_family(self.node.os[0]))['bin'], 'redis-server'),
'conf': os.path.join(self.redis_path.get(Dist.get_os_family(self.node.os[0]))['conf'], 'redis.6379.conf')
})
if out[2]:
raise AssertionError("Redis server was not properly started on remote host %s. Error is: %s %s"
% (self.server.public_ip, out[0], out[1]))
LOG.info('Redis server was successfully run.')
def broke_scalarizr_branch(step, comment):
LOG.debug('Git work dir: %s' % SCALARIZR_REPO_PATH)
os.chdir(SCALARIZR_REPO_PATH)
if Dist.is_centos_family(Dist.from_name(CONF.feature.dist)):
tag = '@rpm'
elif Dist.is_debian_family(Dist.from_name(CONF.feature.dist)):
tag = '@deb'
elif CONF.feature.dist.startswith('win'):
tag = '@win'
comment = comment.split()
comment.insert(-1, tag)
comment = ' '.join(comment)
command = ['git', 'log', '--pretty=oneline', "--grep=Revert '%s'" % comment]
LOG.debug('Execute grep by git: %s' % command)
git_log = subprocess.check_output(command, stderr=subprocess.PIPE).strip().splitlines()
if not git_log:
command = ['git', 'log', '--pretty=oneline', '--grep=Revert "%s"' % comment]
LOG.debug('Execute grep by git: %s' % command)
git_log = subprocess.check_output(command, stderr=subprocess.PIPE).strip().splitlines()
LOG.info('Get latest commits from git history (in broke step): %s' % git_log)
splitted_log = git_log[0].split(' ', 2)
commit = splitted_log[0]
message = splitted_log[-1].replace('"', '').replace("'", '')
LOG.info('Revert commit "%s %s" for brake repository' % (commit, message))
subprocess.call(['git', 'revert', commit, '-n', '--no-edit'])
subprocess.call(['git', 'commit', '-a', '-m', '%s' % message])
last_revision = subprocess.check_output(['git', 'log', '--pretty=oneline', '-1']).splitlines()[0].split()[0].strip()
LOG.info('Last pushed revision: %s' % last_revision)
setattr(world, 'last_scalarizr_revision', last_revision)
subprocess.call(['git', 'push'])
time.sleep(60)
def verify_repository_is_working(step):
if Dist.is_centos_family(Dist.from_name(CONF.feature.dist)):
branch = 'test/update-system-rpm'
commits = ["@update-system @rpm @postinst", "@update-system @rpm @fatal-error"]
elif Dist.is_debian_family(Dist.from_name(CONF.feature.dist)):
branch = 'test/update-system-deb'
commits = ["@update-system @deb @postinst", "@update-system @deb @fatal-error"]
elif CONF.feature.dist.startswith('win'):
branch = 'test/update-system-win'
commits = ["@update-system @win @postinst", "@update-system @win @fatal-error"]
else:
raise AssertionError('Don\'t know what branch use!')
LOG.info('Used scalarizr branch is %s' % branch)
setattr(world, 'scalarizr_branch', branch)
if not os.path.isdir(SCALARIZR_REPO_PATH):
LOG.info('Clone scalarizr repo')
exit_code = subprocess.call(['git', 'clone', '-b', branch, SCALARIZR_GITHUB_PATH, SCALARIZR_REPO_PATH],
stderr=subprocess.PIPE)
if not exit_code == 0:
raise AssertionError('Error in git clone!')
os.chdir(SCALARIZR_REPO_PATH)
LOG.info('Merge feature/update-system to this repository')
merge_proc = subprocess.Popen(['git', 'merge', '-m', 'merge parent branch', 'origin/feature/update-system'],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
merge_proc.wait()
LOG.debug('Merge process result: %s' % merge_proc.stdout.read())
if merge_proc.returncode != 0:
raise AssertionError('Something wrong in git merge, please see log: %s' % merge_proc.stdout.read())
last_revision = subprocess.check_output(['git', 'log', '--pretty=oneline', '-1']).splitlines()[0].split()[0].strip()
LOG.info('Last pushed revision: %s' % last_revision)
setattr(world, 'last_scalarizr_revision', last_revision)
git_log = subprocess.check_output(['git', 'log', '--pretty=oneline', '-20']).splitlines()
LOG.info('Get latest commits from git history: %s' % git_log)
flags = {}
for log in git_log:
commit, message = log.split(' ', 1)
message = message.strip()
for m in commits:
if message.startswith('Revert "%s"' % m) or message.startswith("Revert '%s'" % m):
flags[m] = True
elif message.startswith(m) and not flags.get(m, False):
# revert last commit
LOG.info('Revert broken commit "%s %s"' % (commit, message))
subprocess.call(['git', 'revert', commit, '-n', '--no-edit'])
subprocess.call(['git', 'commit', '-a', '-m', "Revert '%s'" % message])
last_revision = subprocess.check_output(['git', 'log', '--pretty=oneline', '-1']).splitlines()[0].split()[0].strip()
LOG.info('Last pushed revision: %s' % last_revision)
setattr(world, 'last_scalarizr_revision', last_revision)
flags[m] = True
if len(flags) == len(commits):
break
LOG.debug('Push changes to working branch')
subprocess.call(['git', 'push'])
def start_rolebuild(step, behaviors):
behaviors = behaviors.strip().split(',')
if not 'chef' in behaviors:
behaviors.append('chef')
location = CONF.platforms[CONF.feature.platform]['location']
if CONF.feature.driver.current_cloud == Platform.GCE:
location = 'all'
platform = CONF.feature.driver.scalr_cloud
os_dist, os_ver = Dist.get_scalr_dist_info(CONF.feature.dist)
if CONF.feature.driver.current_cloud == Platform.GCE:
image = filter(lambda x: x['os_family']==os_dist and x['os_version'].startswith(os_ver),
images(Platform.to_scalr(CONF.feature.driver.current_cloud)).all()['images'])[0]
else:
image = filter(lambda x: x['cloud_location']==CONF.platforms[CONF.feature.platform]['location'] and
x['os_family']==os_dist and x['os_version'].startswith(os_ver),
images(CONF.feature.driver.scalr_cloud).all()['images'])[0]
bundle_id = IMPL.rolebuilder.build2(platform=platform,
location=location,
arch='x86_64',
behaviors=behaviors,
os_family=image['os_family'],
os_version=image['os_version'],
name='tmp-%s-%s-%s' % (CONF.feature.platform, CONF.feature.dist,
datetime.now().strftime('%m%d-%H%M')),
scalarizr=CONF.feature.branch,
mysqltype='percona' if 'percona' in behaviors else 'mysql')
setattr(world, 'role_type', CONF.feature.behaviors[0])
setattr(world, 'bundle_id', bundle_id)
def assert_scalarizr_version(step, branch, serv_as):
"""
Argument branch can be system or role.
System branch - CONF.feature.branch
Role branch - CONF.feature.to_branch
"""
#FIXME: Rewrite this ugly code!
server = getattr(world, serv_as)
if branch == 'system' or not branch:
branch = CONF.feature.branch
elif branch == 'role':
branch = CONF.feature.to_branch
# Get custom repo url
os_family = Dist(server.role.dist).family
if '.' in branch and branch.replace('.', '').isdigit():
last_version = branch
else:
if branch in ['stable', 'latest']:
default_repo = DEFAULT_SCALARIZR_RELEASE_REPOS[os_family]
else:
url = DEFAULT_SCALARIZR_DEVEL_REPOS['url'][CONF.feature.ci_repo]
path = DEFAULT_SCALARIZR_DEVEL_REPOS['path'][os_family]
default_repo = url.format(path=path)
# Get last scalarizr version from custom repo
index_url = default_repo.format(branch=branch)
LOG.debug('Check package from index_url: %s' % index_url)
repo_data = parser_for_os_family(server.role.dist)(branch=branch,
index_url=index_url)
versions = [
package['version'] for package in repo_data
if package['name'] == 'scalarizr'
]
versions.sort(reverse=True)
last_version = versions[0]
if last_version.strip().endswith('-1'):
last_version = last_version.strip()[:-2]
LOG.debug('Last scalarizr version %s for branch %s' %
(last_version, branch))
# Get installed scalarizr version
for _ in range(5):
try:
update_status = server.upd_api.status(cached=False)
installed_version = update_status['installed']
if installed_version.strip().endswith('-1'):
installed_version = installed_version.strip()[:-2]
break
except urllib2.URLError:
time.sleep(3)
else:
raise AssertionError(
'Can\'t get access to update client 5 times (15 seconds)')
LOG.debug('Last scalarizr version from update client status: %s' %
update_status['installed'])
assert update_status['state'] == 'completed', \
'Update client not in normal state. Status = "%s", Previous state = "%s"' % \
(update_status['state'], update_status['prev_state'])
assert last_version == installed_version, \
'Server not has last build of scalarizr package, installed: %s last_version: %s' % (installed_version, last_version)
def change_branch_in_sources(step, serv_as, branch):
if 'system' in branch:
branch = CONF.feature.branch
elif not branch.strip():
branch = CONF.feature.to_branch
else:
branch = branch.replace('/', '-').replace('.', '').strip()
server = getattr(world, serv_as)
LOG.info('Change branches in sources list in server %s to %s' %
(server.id, branch))
if Dist(server.role.dist).is_debian:
LOG.debug('Change in debian')
node = world.cloud.get_node(server)
for repo_file in [
'/etc/apt/sources.list.d/scalr-stable.list',
'/etc/apt/sources.list.d/scalr-latest.list'
]:
LOG.info("Change branch in %s to %s" % (repo_file, branch))
node.run(
'echo "deb http://buildbot.scalr-labs.com/apt/debian %s/" > %s'
% (branch, repo_file))
elif Dist(server.role.dist).is_centos:
LOG.debug('Change in centos')
node = world.cloud.get_node(server)
for repo_file in ['/etc/yum.repos.d/scalr-stable.repo']:
LOG.info("Change branch in %s to %s" % (repo_file, branch))
node.run(
'echo "[scalr-branch]\nname=scalr-branch\nbaseurl=http://buildbot.scalr-labs.com/rpm/%s/rhel/\$releasever/\$basearch\nenabled=1\ngpgcheck=0" > %s'
% (branch, repo_file))
node.run('echo > /etc/yum.repos.d/scalr-latest.repo')
elif Dist(server.role.dist).is_windows:
# LOG.debug('Change in windows')
import winrm
console = winrm.Session('http://%s:5985/wsman' % server.public_ip,
auth=("Administrator",
server.windows_password))
for repo_file in [
'C:\Program Files\Scalarizr\etc\scalr-latest.winrepo',
'C:\Program Files\Scalarizr\etc\scalr-stable.winrepo'
]:
# LOG.info("Change branch in %s to %s" % (repo_file, branch))
console.run_cmd(
'echo http://buildbot.scalr-labs.com/win/%s/x86_64/ > "%s"' %
(branch, repo_file))
def _verify_app(server, port):
LOG.info('Verify apache (%s) work in server %s' % (port, server.id))
node = world.cloud.get_node(server)
results = [
VerifyProcessWork._verify_process_running(
server, DEFAULT_SERVICES_CONFIG['app'][Dist(
node.os).family]['service_name']),
VerifyProcessWork._verify_open_port(server, port)
]
return all(results)
def start_redis_process(step, process, serv_as):
"""Start redis-server process"""
redis_bin_path = {
'debian': {'bin': '/usr/bin',
'conf': '/etc/redis'},
'centos': {'bin': '/usr/sbin',
'conf': '/etc'}
}
server = getattr(world, serv_as)
node = world.cloud.get_node(server)
LOG.info('Start %s on remote host: %s' % (process, server.public_ip))
node_result = node.run("/bin/su redis -s /bin/bash -c \"%(bin)s %(conf)s\" && sleep 5 && pgrep -l %(process)s | awk {print'$1'}" %
{
'bin': os.path.join(redis_bin_path.get(Dist.get_os_family(node.os[0]))['bin'], process),
'process': process,
'conf': os.path.join(redis_bin_path.get(Dist.get_os_family(node.os[0]))['conf'], 'redis.6379.conf')
})
if node_result[2]:
raise AssertionError("%s was not properly started on remote host %s. Error is: %s %s"
% (process, server.public_ip, node_result[0], node_result[1]))
LOG.info('%s was successfully started on remote host: %s' % (process, server.public_ip))
def check_process_options(step, process, options, serv_as):
server = getattr(world, serv_as)
LOG.debug('Want check process %s and options %s' % (process, options))
node = world.cloud.get_node(server)
for attempt in range(3):
out = node.run('ps aux | grep %s' % process)
LOG.debug('Grep for ps aux: %s' % out[0])
for line in out[0].splitlines():
if 'grep' in line:
continue
LOG.info('Work with line: %s' % line)
if options not in line and not CONF.feature.dist == Dist(
'amzn1609') and not CONF.feature.dist.is_systemd:
raise AssertionError('Options %s not in process, %s' %
(options, ' '.join(line.split()[10:])))
else:
return True
raise AssertionError('Not found process: %s' % process)
def start_rolebuild(step):
location = CONF.platforms[CONF.feature.platform]['location']
if CONF.feature.platform == 'rackspaceng':
platform = 'rackspacengus'
else:
platform = CONF.feature.platform
os_dist, os_ver = Dist.get_scalr_dist_info(CONF.feature.dist)
image = filter(lambda x: x['cloud_location']==CONF.platforms[CONF.feature.platform]['location'] and
x['os_family']==os_dist and x['os_version'].startswith(os_ver),
images(CONF.feature.driver.scalr_cloud).all()['images'])[0]
bundle_id = IMPL.rolebuilder.build2(platform=platform,
location=location,
arch='x86_64',
behaviors=CONF.feature.behaviors,
os_family=image['os_family'],
os_version=image['os_version'],
name='tmp-%s-%s-%s' % (CONF.feature.platform, CONF.feature.dist,
datetime.now().strftime('%m%d-%H%M')),
scalarizr=CONF.feature.branch,)
setattr(world, 'role_type', CONF.feature.behaviors[0])
setattr(world, 'bundle_id', bundle_id)
def wait_server_bootstrapping(role=None, status=ServerStatus.RUNNING, timeout=2100, server=None):
"""
Wait a moment when new server starting in the pointed role and wait server will in selected state.
Moreover this function remember all previous started servers.
:param class:Role role: Show in which role lookup a new server
:return class:Server: Return a new Server
"""
status = ServerStatus.from_code(status)
LOG.info('Launch process looking for new server in farm %s for role %s, wait status %s' %
(world.farm.id, role, status))
previous_servers = getattr(world, '_previous_servers', [])
if not previous_servers:
world._previous_servers = previous_servers
LOG.debug('Previous servers: %s' % previous_servers)
lookup_server = server or None
lookup_node = None
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
if not role:
world.farm.servers.reload()
servers = world.farm.servers
else:
role.servers.reload()
servers = role.servers
for server in servers:
LOG.debug('Work with server: %s - %s' % (server.id, server.status))
if not server in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING,
ServerStatus.INIT,
ServerStatus.RUNNING]:
LOG.debug('I found a server: %s' % server.id)
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE] \
and not status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE]:
raise ServerTerminated('Server %s change status to %s' % (lookup_server.id, lookup_server.status))
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
raise ServerFailed('Server %s failed in %s. Reason: %s'
% (lookup_server.id, ServerStatus.PENDING_LAUNCH,
lookup_server.get_failed_status_message()))
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
raise ServerFailed('Server %s failed in %s. Failed (Why?): %s' %
(lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message()))
LOG.debug('Try get node')
if not lookup_node and not lookup_server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED]:
LOG.debug('Try to get node object for lookup server')
lookup_node = world.cloud.get_node(lookup_server)
LOG.debug('Verify debug log in node')
if lookup_node and not lookup_server.status in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED]:
if not Dist.is_windows_family(lookup_server.role.dist):
LOG.debug('Check scalarizr log in lookup server')
verify_scalarizr_log(lookup_node)
LOG.debug('If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status = ServerStatus.RUNNING
LOG.debug('Compare server status')
if lookup_server.status == status:
LOG.info('Lookup server in right status now: %s' % lookup_server.status)
if status == ServerStatus.RUNNING:
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug('Return server %s' % lookup_server)
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError('Server %s not in state "%s" it has status: "%s"'
% (lookup_server.id, status, lookup_server.status))
raise TimeoutError('New server in role "%s" was not founding' % role)
def wait_server_bootstrapping(role=None,
status=ServerStatus.RUNNING,
timeout=2100,
server=None):
"""
Wait a moment when new server starting in the pointed role and wait server will in selected state.
Moreover this function remember all previous started servers.
:param class:Role role: Show in which role lookup a new server
:return class:Server: Return a new Server
"""
status = ServerStatus.from_code(status)
LOG.info(
'Launch process looking for new server in farm %s for role %s, wait status %s'
% (world.farm.id, role, status))
previous_servers = getattr(world, '_previous_servers', [])
if not previous_servers:
world._previous_servers = previous_servers
LOG.debug('Previous servers: %s' % previous_servers)
lookup_server = server or None
lookup_node = None
start_time = time.time()
while time.time() - start_time < timeout:
if not lookup_server:
LOG.debug('Reload servers in role')
if not role:
world.farm.servers.reload()
servers = world.farm.servers
else:
role.servers.reload()
servers = role.servers
for server in servers:
LOG.debug('Work with server: %s - %s' %
(server.id, server.status))
if not server in previous_servers and server.status in [
ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING,
ServerStatus.INIT, ServerStatus.RUNNING
]:
LOG.debug('I found a server: %s' % server.id)
lookup_server = server
if lookup_server:
LOG.debug('Reload lookup_server')
previous_state = lookup_server.status
lookup_server.reload()
LOG.debug('Check lookup server terminated?')
if lookup_server.status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING] \
and not status in [ServerStatus.TERMINATED,
ServerStatus.PENDING_TERMINATE,
ServerStatus.MISSING]:
raise ServerTerminated(
'Server %s change status to %s (was %s)' %
(lookup_server.id, lookup_server.status, previous_state))
LOG.debug('Check lookup server launch failed')
if lookup_server.is_launch_failed:
failed_message = lookup_server.get_failed_status_message()
if CONF.feature.driver.cloud_family == Platform.CLOUDSTACK \
and ('Can not decode json response data' in failed_message
or 'Cannot establish connection with CloudStack server. (Server returned nothing )' in failed_message):
time.sleep(90)
lookup_server = None
lookup_node = None
continue
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(
'Server %s failed in %s. Reason: %s' %
(lookup_server.id, ServerStatus.PENDING_LAUNCH,
failed_message))
LOG.debug('Check lookup server init failed')
if lookup_server.is_init_failed:
if status == ServerStatus.FAILED:
LOG.debug('Return server because we wait a failed state')
return lookup_server
raise ServerFailed(
'Server %s failed in %s. Failed (Why?): %s' %
(lookup_server.id, ServerStatus.INIT,
lookup_server.get_failed_status_message()))
LOG.debug('Try get node')
if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED] \
and CONF.feature.driver.current_cloud != Platform.AZURE:
LOG.debug('Try to get node object for lookup server')
lookup_node = world.cloud.get_node(lookup_server)
LOG.debug('Verify update log in node')
if lookup_node and lookup_server.status in ServerStatus.PENDING:
LOG.debug('Check scalarizr update log in lookup server')
if not Dist(lookup_server.role.dist).is_windows:
verify_scalarizr_log(lookup_node, log_type='update')
else:
verify_scalarizr_log(lookup_node,
log_type='update',
windows=True,
server=lookup_server)
LOG.debug('Verify debug log in node')
if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH,
ServerStatus.PENDING_TERMINATE,
ServerStatus.TERMINATED,
ServerStatus.PENDING_SUSPEND,
ServerStatus.SUSPENDED]\
and not status == ServerStatus.FAILED:
LOG.debug('Check scalarizr debug log in lookup server')
if not Dist(lookup_server.role.dist).is_windows:
verify_scalarizr_log(lookup_node)
else:
verify_scalarizr_log(lookup_node,
windows=True,
server=lookup_server)
LOG.debug(
'If server Running and we wait Initializing, return server')
if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Initializing but server already Running')
status = ServerStatus.RUNNING
if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING:
LOG.info('We wait Resuming but server already Running')
status = ServerStatus.RUNNING
LOG.debug('Compare server status')
if lookup_server.status == status:
LOG.info('Lookup server in right status now: %s' %
lookup_server.status)
if status == ServerStatus.RUNNING:
LOG.debug('Insert server to previous servers')
previous_servers.append(lookup_server)
LOG.debug('Return server %s' % lookup_server)
return lookup_server
LOG.debug('Sleep 10 seconds')
time.sleep(10)
else:
if lookup_server:
raise TimeoutError(
'Server %s not in state "%s" it has status: "%s"' %
(lookup_server.id, status, lookup_server.status))
raise TimeoutError('New server in role "%s" was not founding' % role)
def get_all_logs_and_info(scenario, outline='', outline_failed=None):
if CONF.feature.driver.current_cloud == Platform.AZURE:
return
# Get Farm
LOG.warning('Get scalarizr logs after scenario %s' % scenario.name)
farm = getattr(world, 'farm', None)
if not farm:
LOG.error("Farm does not exists. Can't get logs. Exit from step.")
return
farm.servers.reload()
# Get servers
servers = farm.servers
# Get test
test_name = scenario.described_at.file.split('/')[-1].split('.')[0]
LOG.debug('Test name: %s' % test_name)
# Get path
start_time = world.test_start_time
path = os.path.realpath(
os.path.join(CONF.main.log_path, 'scalarizr', test_name,
start_time.strftime('%m%d-%H:%M'),
scenario.name.replace('/', '-'), outline))
LOG.debug('Path to save log: %s' % path)
if not os.path.exists(path):
os.makedirs(path, 0755)
# Get logs && configs
for server in servers:
if not server.is_scalarized: continue
logs = [
# debug log
{
'file':
os.path.join(path, '_'.join(
(server.id, 'scalarizr_debug.log'))),
'log_type':
'debug',
'compress':
True
},
# update log
{
'file':
os.path.join(path, '_'.join(
(server.id, 'scalarizr_update.log'))),
'log_type':
'update',
'compress':
True
}
]
if server.status in [
ServerStatus.PENDING, ServerStatus.INIT, ServerStatus.RUNNING
]:
try:
#Get log from remote host
for log in logs:
server.get_log_by_api(**log)
LOG.info(
'Save {log_type} log from server {} to {file}'.format(
server.id, **log))
#Get configs and role behavior from remote host only for linux family
if not Dist(server.role.dist).is_windows:
file = os.path.join(
path, '_'.join(
(server.id, 'scalr_configs.tar.gz')))
server.get_configs(file, compress=True)
LOG.info(
'Download archive with scalr directory and behavior to: {}'
.format(file))
except BaseException, e:
LOG.error('Error in downloading configs: %s' % e)
continue
if server.status == ServerStatus.RUNNING and not CONF.feature.dist.is_windows:
node = world.cloud.get_node(server)
out = node.run("ps aux | grep 'bin/scal'")[0]
for line in out.splitlines():
ram = line.split()[5]
if len(ram) > 3:
ram = '%sMB' % ram[:-3]
if 'bin/scalr-upd-client' in line:
LOG.info('Server %s use %s RAM for update-client' %
(server.id, ram))
world.wrt(
etree.Element('meta',
name='szrupdateram',
value=ram,
serverid=server.id))
elif 'bin/scalarizr' in line:
LOG.info('Server %s use %s RAM for scalarizr' %
(server.id, ram))
world.wrt(
etree.Element('meta',
name='szrram',
value=ram,
serverid=server.id))
