def check_instance_configs(step, instance_port, search_condition, serv_as): redis_path = { 'debian': { 'conf': '/etc/redis', 'data': '/mnt/redisstorage' }, 'centos': { 'conf': '/etc', 'data': '/mnt/redisstorage' }, } server = getattr(world, serv_as) node = world.cloud.get_node(server) # Get redis instance search condition if search_condition == 'configuration': path = redis_path.get(Dist(node.os).family)['conf'] file = 'redis.{0}.conf'.format(instance_port) else: path = redis_path.get(Dist(node.os).family)['data'] file = 'appendonly.{0}.aof'.format(instance_port) command = 'find {0} -name {1}'.format(path, file) LOG.debug( 'Search condition: ({0}/{1}) to find config for redis instance {2}'. format(path, file, instance_port)) res = node.run(command) LOG.debug( 'The result: ({0}) of the find command for redis instance {1}'.format( res[0], instance_port)) assertion_message = 'Redis instance {0} config file ({1}) was not deleted.'.format( instance_port, res[0]) assert not (res[0]), assertion_message LOG.debug( 'Redis instance {0} config file ({1}/{2}) was successfully deleted.'. format(instance_port, path, file))
def restore(self, src_path, db=None): #Kill redis-server LOG.info('Stopping Redis server.') out = self.node.run("pgrep -l redis-server | awk {print'$1'} | xargs -i{} kill {} && sleep 5 && pgrep -l redis-server | awk {print'$1'}")[0] if out: raise AssertionError('Redis server, pid:%s was not properly killed on remote host %s' % (out, self.server.public_ip)) LOG.info('Redis server was successfully stopped. Getting backups and moving to redis storage.') #Move dump to redis storage out = self.node.run("find %s -name '*%s*' -print0 | xargs -i{} -0 -r cp -v {} /mnt/redisstorage/" % (src_path, self.snapshotting_type)) if not out[0]: raise AssertionError("Can't move dump to redis-server storage. Error is: %s" % out[1]) LOG.info('Available backups in server: %s. Backups was successfully moved to redis storage.' % out[0].split()[0]) #Run redis-server LOG.info('Running Redis server.') out = self.node.run("/bin/su redis -s /bin/bash -c \"%(bin)s %(conf)s\" && sleep 5 && pgrep -l redis-server | awk {print'$1'}" % { 'bin': os.path.join(self.redis_path.get(Dist.get_os_family(self.node.os[0]))['bin'], 'redis-server'), 'conf': os.path.join(self.redis_path.get(Dist.get_os_family(self.node.os[0]))['conf'], 'redis.6379.conf') }) if out[2]: raise AssertionError("Redis server was not properly started on remote host %s. Error is: %s %s" % (self.server.public_ip, out[0], out[1])) LOG.info('Redis server was successfully run.')
def broke_scalarizr_branch(step, comment): LOG.debug('Git work dir: %s' % SCALARIZR_REPO_PATH) os.chdir(SCALARIZR_REPO_PATH) if Dist.is_centos_family(Dist.from_name(CONF.feature.dist)): tag = '@rpm' elif Dist.is_debian_family(Dist.from_name(CONF.feature.dist)): tag = '@deb' elif CONF.feature.dist.startswith('win'): tag = '@win' comment = comment.split() comment.insert(-1, tag) comment = ' '.join(comment) command = ['git', 'log', '--pretty=oneline', "--grep=Revert '%s'" % comment] LOG.debug('Execute grep by git: %s' % command) git_log = subprocess.check_output(command, stderr=subprocess.PIPE).strip().splitlines() if not git_log: command = ['git', 'log', '--pretty=oneline', '--grep=Revert "%s"' % comment] LOG.debug('Execute grep by git: %s' % command) git_log = subprocess.check_output(command, stderr=subprocess.PIPE).strip().splitlines() LOG.info('Get latest commits from git history (in broke step): %s' % git_log) splitted_log = git_log[0].split(' ', 2) commit = splitted_log[0] message = splitted_log[-1].replace('"', '').replace("'", '') LOG.info('Revert commit "%s %s" for brake repository' % (commit, message)) subprocess.call(['git', 'revert', commit, '-n', '--no-edit']) subprocess.call(['git', 'commit', '-a', '-m', '%s' % message]) last_revision = subprocess.check_output(['git', 'log', '--pretty=oneline', '-1']).splitlines()[0].split()[0].strip() LOG.info('Last pushed revision: %s' % last_revision) setattr(world, 'last_scalarizr_revision', last_revision) subprocess.call(['git', 'push']) time.sleep(60)
def verify_repository_is_working(step): if Dist.is_centos_family(Dist.from_name(CONF.feature.dist)): branch = 'test/update-system-rpm' commits = ["@update-system @rpm @postinst", "@update-system @rpm @fatal-error"] elif Dist.is_debian_family(Dist.from_name(CONF.feature.dist)): branch = 'test/update-system-deb' commits = ["@update-system @deb @postinst", "@update-system @deb @fatal-error"] elif CONF.feature.dist.startswith('win'): branch = 'test/update-system-win' commits = ["@update-system @win @postinst", "@update-system @win @fatal-error"] else: raise AssertionError('Don\'t know what branch use!') LOG.info('Used scalarizr branch is %s' % branch) setattr(world, 'scalarizr_branch', branch) if not os.path.isdir(SCALARIZR_REPO_PATH): LOG.info('Clone scalarizr repo') exit_code = subprocess.call(['git', 'clone', '-b', branch, SCALARIZR_GITHUB_PATH, SCALARIZR_REPO_PATH], stderr=subprocess.PIPE) if not exit_code == 0: raise AssertionError('Error in git clone!') os.chdir(SCALARIZR_REPO_PATH) LOG.info('Merge feature/update-system to this repository') merge_proc = subprocess.Popen(['git', 'merge', '-m', 'merge parent branch', 'origin/feature/update-system'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) merge_proc.wait() LOG.debug('Merge process result: %s' % merge_proc.stdout.read()) if merge_proc.returncode != 0: raise AssertionError('Something wrong in git merge, please see log: %s' % merge_proc.stdout.read()) last_revision = subprocess.check_output(['git', 'log', '--pretty=oneline', '-1']).splitlines()[0].split()[0].strip() LOG.info('Last pushed revision: %s' % last_revision) setattr(world, 'last_scalarizr_revision', last_revision) git_log = subprocess.check_output(['git', 'log', '--pretty=oneline', '-20']).splitlines() LOG.info('Get latest commits from git history: %s' % git_log) flags = {} for log in git_log: commit, message = log.split(' ', 1) message = message.strip() for m in commits: if message.startswith('Revert "%s"' % m) or message.startswith("Revert '%s'" % m): flags[m] = True elif message.startswith(m) and not flags.get(m, False): # revert last commit LOG.info('Revert broken commit "%s %s"' % (commit, message)) subprocess.call(['git', 'revert', commit, '-n', '--no-edit']) subprocess.call(['git', 'commit', '-a', '-m', "Revert '%s'" % message]) last_revision = subprocess.check_output(['git', 'log', '--pretty=oneline', '-1']).splitlines()[0].split()[0].strip() LOG.info('Last pushed revision: %s' % last_revision) setattr(world, 'last_scalarizr_revision', last_revision) flags[m] = True if len(flags) == len(commits): break LOG.debug('Push changes to working branch') subprocess.call(['git', 'push'])
def start_rolebuild(step, behaviors): behaviors = behaviors.strip().split(',') if not 'chef' in behaviors: behaviors.append('chef') location = CONF.platforms[CONF.feature.platform]['location'] if CONF.feature.driver.current_cloud == Platform.GCE: location = 'all' platform = CONF.feature.driver.scalr_cloud os_dist, os_ver = Dist.get_scalr_dist_info(CONF.feature.dist) if CONF.feature.driver.current_cloud == Platform.GCE: image = filter(lambda x: x['os_family']==os_dist and x['os_version'].startswith(os_ver), images(Platform.to_scalr(CONF.feature.driver.current_cloud)).all()['images'])[0] else: image = filter(lambda x: x['cloud_location']==CONF.platforms[CONF.feature.platform]['location'] and x['os_family']==os_dist and x['os_version'].startswith(os_ver), images(CONF.feature.driver.scalr_cloud).all()['images'])[0] bundle_id = IMPL.rolebuilder.build2(platform=platform, location=location, arch='x86_64', behaviors=behaviors, os_family=image['os_family'], os_version=image['os_version'], name='tmp-%s-%s-%s' % (CONF.feature.platform, CONF.feature.dist, datetime.now().strftime('%m%d-%H%M')), scalarizr=CONF.feature.branch, mysqltype='percona' if 'percona' in behaviors else 'mysql') setattr(world, 'role_type', CONF.feature.behaviors[0]) setattr(world, 'bundle_id', bundle_id)
def assert_scalarizr_version(step, branch, serv_as): """ Argument branch can be system or role. System branch - CONF.feature.branch Role branch - CONF.feature.to_branch """ #FIXME: Rewrite this ugly code! server = getattr(world, serv_as) if branch == 'system' or not branch: branch = CONF.feature.branch elif branch == 'role': branch = CONF.feature.to_branch # Get custom repo url os_family = Dist(server.role.dist).family if '.' in branch and branch.replace('.', '').isdigit(): last_version = branch else: if branch in ['stable', 'latest']: default_repo = DEFAULT_SCALARIZR_RELEASE_REPOS[os_family] else: url = DEFAULT_SCALARIZR_DEVEL_REPOS['url'][CONF.feature.ci_repo] path = DEFAULT_SCALARIZR_DEVEL_REPOS['path'][os_family] default_repo = url.format(path=path) # Get last scalarizr version from custom repo index_url = default_repo.format(branch=branch) LOG.debug('Check package from index_url: %s' % index_url) repo_data = parser_for_os_family(server.role.dist)(branch=branch, index_url=index_url) versions = [ package['version'] for package in repo_data if package['name'] == 'scalarizr' ] versions.sort(reverse=True) last_version = versions[0] if last_version.strip().endswith('-1'): last_version = last_version.strip()[:-2] LOG.debug('Last scalarizr version %s for branch %s' % (last_version, branch)) # Get installed scalarizr version for _ in range(5): try: update_status = server.upd_api.status(cached=False) installed_version = update_status['installed'] if installed_version.strip().endswith('-1'): installed_version = installed_version.strip()[:-2] break except urllib2.URLError: time.sleep(3) else: raise AssertionError( 'Can\'t get access to update client 5 times (15 seconds)') LOG.debug('Last scalarizr version from update client status: %s' % update_status['installed']) assert update_status['state'] == 'completed', \ 'Update client not in normal state. Status = "%s", Previous state = "%s"' % \ (update_status['state'], update_status['prev_state']) assert last_version == installed_version, \ 'Server not has last build of scalarizr package, installed: %s last_version: %s' % (installed_version, last_version)
def change_branch_in_sources(step, serv_as, branch): if 'system' in branch: branch = CONF.feature.branch elif not branch.strip(): branch = CONF.feature.to_branch else: branch = branch.replace('/', '-').replace('.', '').strip() server = getattr(world, serv_as) LOG.info('Change branches in sources list in server %s to %s' % (server.id, branch)) if Dist(server.role.dist).is_debian: LOG.debug('Change in debian') node = world.cloud.get_node(server) for repo_file in [ '/etc/apt/sources.list.d/scalr-stable.list', '/etc/apt/sources.list.d/scalr-latest.list' ]: LOG.info("Change branch in %s to %s" % (repo_file, branch)) node.run( 'echo "deb http://buildbot.scalr-labs.com/apt/debian %s/" > %s' % (branch, repo_file)) elif Dist(server.role.dist).is_centos: LOG.debug('Change in centos') node = world.cloud.get_node(server) for repo_file in ['/etc/yum.repos.d/scalr-stable.repo']: LOG.info("Change branch in %s to %s" % (repo_file, branch)) node.run( 'echo "[scalr-branch]\nname=scalr-branch\nbaseurl=http://buildbot.scalr-labs.com/rpm/%s/rhel/\$releasever/\$basearch\nenabled=1\ngpgcheck=0" > %s' % (branch, repo_file)) node.run('echo > /etc/yum.repos.d/scalr-latest.repo') elif Dist(server.role.dist).is_windows: # LOG.debug('Change in windows') import winrm console = winrm.Session('http://%s:5985/wsman' % server.public_ip, auth=("Administrator", server.windows_password)) for repo_file in [ 'C:\Program Files\Scalarizr\etc\scalr-latest.winrepo', 'C:\Program Files\Scalarizr\etc\scalr-stable.winrepo' ]: # LOG.info("Change branch in %s to %s" % (repo_file, branch)) console.run_cmd( 'echo http://buildbot.scalr-labs.com/win/%s/x86_64/ > "%s"' % (branch, repo_file))
def _verify_app(server, port): LOG.info('Verify apache (%s) work in server %s' % (port, server.id)) node = world.cloud.get_node(server) results = [ VerifyProcessWork._verify_process_running( server, DEFAULT_SERVICES_CONFIG['app'][Dist( node.os).family]['service_name']), VerifyProcessWork._verify_open_port(server, port) ] return all(results)
def start_redis_process(step, process, serv_as): """Start redis-server process""" redis_bin_path = { 'debian': {'bin': '/usr/bin', 'conf': '/etc/redis'}, 'centos': {'bin': '/usr/sbin', 'conf': '/etc'} } server = getattr(world, serv_as) node = world.cloud.get_node(server) LOG.info('Start %s on remote host: %s' % (process, server.public_ip)) node_result = node.run("/bin/su redis -s /bin/bash -c \"%(bin)s %(conf)s\" && sleep 5 && pgrep -l %(process)s | awk {print'$1'}" % { 'bin': os.path.join(redis_bin_path.get(Dist.get_os_family(node.os[0]))['bin'], process), 'process': process, 'conf': os.path.join(redis_bin_path.get(Dist.get_os_family(node.os[0]))['conf'], 'redis.6379.conf') }) if node_result[2]: raise AssertionError("%s was not properly started on remote host %s. Error is: %s %s" % (process, server.public_ip, node_result[0], node_result[1])) LOG.info('%s was successfully started on remote host: %s' % (process, server.public_ip))
def check_process_options(step, process, options, serv_as): server = getattr(world, serv_as) LOG.debug('Want check process %s and options %s' % (process, options)) node = world.cloud.get_node(server) for attempt in range(3): out = node.run('ps aux | grep %s' % process) LOG.debug('Grep for ps aux: %s' % out[0]) for line in out[0].splitlines(): if 'grep' in line: continue LOG.info('Work with line: %s' % line) if options not in line and not CONF.feature.dist == Dist( 'amzn1609') and not CONF.feature.dist.is_systemd: raise AssertionError('Options %s not in process, %s' % (options, ' '.join(line.split()[10:]))) else: return True raise AssertionError('Not found process: %s' % process)
def start_rolebuild(step): location = CONF.platforms[CONF.feature.platform]['location'] if CONF.feature.platform == 'rackspaceng': platform = 'rackspacengus' else: platform = CONF.feature.platform os_dist, os_ver = Dist.get_scalr_dist_info(CONF.feature.dist) image = filter(lambda x: x['cloud_location']==CONF.platforms[CONF.feature.platform]['location'] and x['os_family']==os_dist and x['os_version'].startswith(os_ver), images(CONF.feature.driver.scalr_cloud).all()['images'])[0] bundle_id = IMPL.rolebuilder.build2(platform=platform, location=location, arch='x86_64', behaviors=CONF.feature.behaviors, os_family=image['os_family'], os_version=image['os_version'], name='tmp-%s-%s-%s' % (CONF.feature.platform, CONF.feature.dist, datetime.now().strftime('%m%d-%H%M')), scalarizr=CONF.feature.branch,) setattr(world, 'role_type', CONF.feature.behaviors[0]) setattr(world, 'bundle_id', bundle_id)
def wait_server_bootstrapping(role=None, status=ServerStatus.RUNNING, timeout=2100, server=None): """ Wait a moment when new server starting in the pointed role and wait server will in selected state. Moreover this function remember all previous started servers. :param class:Role role: Show in which role lookup a new server :return class:Server: Return a new Server """ status = ServerStatus.from_code(status) LOG.info('Launch process looking for new server in farm %s for role %s, wait status %s' % (world.farm.id, role, status)) previous_servers = getattr(world, '_previous_servers', []) if not previous_servers: world._previous_servers = previous_servers LOG.debug('Previous servers: %s' % previous_servers) lookup_server = server or None lookup_node = None start_time = time.time() while time.time() - start_time < timeout: if not lookup_server: LOG.debug('Reload servers in role') if not role: world.farm.servers.reload() servers = world.farm.servers else: role.servers.reload() servers = role.servers for server in servers: LOG.debug('Work with server: %s - %s' % (server.id, server.status)) if not server in previous_servers and server.status in [ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING, ServerStatus.INIT, ServerStatus.RUNNING]: LOG.debug('I found a server: %s' % server.id) lookup_server = server if lookup_server: LOG.debug('Reload lookup_server') lookup_server.reload() LOG.debug('Check lookup server terminated?') if lookup_server.status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE] \ and not status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE]: raise ServerTerminated('Server %s change status to %s' % (lookup_server.id, lookup_server.status)) LOG.debug('Check lookup server launch failed') if lookup_server.is_launch_failed: raise ServerFailed('Server %s failed in %s. Reason: %s' % (lookup_server.id, ServerStatus.PENDING_LAUNCH, lookup_server.get_failed_status_message())) LOG.debug('Check lookup server init failed') if lookup_server.is_init_failed: raise ServerFailed('Server %s failed in %s. Failed (Why?): %s' % (lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message())) LOG.debug('Try get node') if not lookup_node and not lookup_server.status in [ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING_TERMINATE, ServerStatus.TERMINATED, ServerStatus.PENDING_SUSPEND, ServerStatus.SUSPENDED]: LOG.debug('Try to get node object for lookup server') lookup_node = world.cloud.get_node(lookup_server) LOG.debug('Verify debug log in node') if lookup_node and not lookup_server.status in [ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING_TERMINATE, ServerStatus.TERMINATED, ServerStatus.PENDING_SUSPEND, ServerStatus.SUSPENDED]: if not Dist.is_windows_family(lookup_server.role.dist): LOG.debug('Check scalarizr log in lookup server') verify_scalarizr_log(lookup_node) LOG.debug('If server Running and we wait Initializing, return server') if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING: LOG.info('We wait Initializing but server already Running') status = ServerStatus.RUNNING LOG.debug('Compare server status') if lookup_server.status == status: LOG.info('Lookup server in right status now: %s' % lookup_server.status) if status == ServerStatus.RUNNING: LOG.debug('Insert server to previous servers') previous_servers.append(lookup_server) LOG.debug('Return server %s' % lookup_server) return lookup_server LOG.debug('Sleep 10 seconds') time.sleep(10) else: if lookup_server: raise TimeoutError('Server %s not in state "%s" it has status: "%s"' % (lookup_server.id, status, lookup_server.status)) raise TimeoutError('New server in role "%s" was not founding' % role)
def wait_server_bootstrapping(role=None, status=ServerStatus.RUNNING, timeout=2100, server=None): """ Wait a moment when new server starting in the pointed role and wait server will in selected state. Moreover this function remember all previous started servers. :param class:Role role: Show in which role lookup a new server :return class:Server: Return a new Server """ status = ServerStatus.from_code(status) LOG.info( 'Launch process looking for new server in farm %s for role %s, wait status %s' % (world.farm.id, role, status)) previous_servers = getattr(world, '_previous_servers', []) if not previous_servers: world._previous_servers = previous_servers LOG.debug('Previous servers: %s' % previous_servers) lookup_server = server or None lookup_node = None start_time = time.time() while time.time() - start_time < timeout: if not lookup_server: LOG.debug('Reload servers in role') if not role: world.farm.servers.reload() servers = world.farm.servers else: role.servers.reload() servers = role.servers for server in servers: LOG.debug('Work with server: %s - %s' % (server.id, server.status)) if not server in previous_servers and server.status in [ ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING, ServerStatus.INIT, ServerStatus.RUNNING ]: LOG.debug('I found a server: %s' % server.id) lookup_server = server if lookup_server: LOG.debug('Reload lookup_server') previous_state = lookup_server.status lookup_server.reload() LOG.debug('Check lookup server terminated?') if lookup_server.status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE, ServerStatus.MISSING] \ and not status in [ServerStatus.TERMINATED, ServerStatus.PENDING_TERMINATE, ServerStatus.MISSING]: raise ServerTerminated( 'Server %s change status to %s (was %s)' % (lookup_server.id, lookup_server.status, previous_state)) LOG.debug('Check lookup server launch failed') if lookup_server.is_launch_failed: failed_message = lookup_server.get_failed_status_message() if CONF.feature.driver.cloud_family == Platform.CLOUDSTACK \ and ('Can not decode json response data' in failed_message or 'Cannot establish connection with CloudStack server. (Server returned nothing )' in failed_message): time.sleep(90) lookup_server = None lookup_node = None continue if status == ServerStatus.FAILED: LOG.debug('Return server because we wait a failed state') return lookup_server raise ServerFailed( 'Server %s failed in %s. Reason: %s' % (lookup_server.id, ServerStatus.PENDING_LAUNCH, failed_message)) LOG.debug('Check lookup server init failed') if lookup_server.is_init_failed: if status == ServerStatus.FAILED: LOG.debug('Return server because we wait a failed state') return lookup_server raise ServerFailed( 'Server %s failed in %s. Failed (Why?): %s' % (lookup_server.id, ServerStatus.INIT, lookup_server.get_failed_status_message())) LOG.debug('Try get node') if not lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING_TERMINATE, ServerStatus.TERMINATED, ServerStatus.PENDING_SUSPEND, ServerStatus.SUSPENDED] \ and CONF.feature.driver.current_cloud != Platform.AZURE: LOG.debug('Try to get node object for lookup server') lookup_node = world.cloud.get_node(lookup_server) LOG.debug('Verify update log in node') if lookup_node and lookup_server.status in ServerStatus.PENDING: LOG.debug('Check scalarizr update log in lookup server') if not Dist(lookup_server.role.dist).is_windows: verify_scalarizr_log(lookup_node, log_type='update') else: verify_scalarizr_log(lookup_node, log_type='update', windows=True, server=lookup_server) LOG.debug('Verify debug log in node') if lookup_node and lookup_server.status not in [ServerStatus.PENDING_LAUNCH, ServerStatus.PENDING_TERMINATE, ServerStatus.TERMINATED, ServerStatus.PENDING_SUSPEND, ServerStatus.SUSPENDED]\ and not status == ServerStatus.FAILED: LOG.debug('Check scalarizr debug log in lookup server') if not Dist(lookup_server.role.dist).is_windows: verify_scalarizr_log(lookup_node) else: verify_scalarizr_log(lookup_node, windows=True, server=lookup_server) LOG.debug( 'If server Running and we wait Initializing, return server') if status == ServerStatus.INIT and lookup_server.status == ServerStatus.RUNNING: LOG.info('We wait Initializing but server already Running') status = ServerStatus.RUNNING if status == ServerStatus.RESUMING and lookup_server.status == ServerStatus.RUNNING: LOG.info('We wait Resuming but server already Running') status = ServerStatus.RUNNING LOG.debug('Compare server status') if lookup_server.status == status: LOG.info('Lookup server in right status now: %s' % lookup_server.status) if status == ServerStatus.RUNNING: LOG.debug('Insert server to previous servers') previous_servers.append(lookup_server) LOG.debug('Return server %s' % lookup_server) return lookup_server LOG.debug('Sleep 10 seconds') time.sleep(10) else: if lookup_server: raise TimeoutError( 'Server %s not in state "%s" it has status: "%s"' % (lookup_server.id, status, lookup_server.status)) raise TimeoutError('New server in role "%s" was not founding' % role)
def get_all_logs_and_info(scenario, outline='', outline_failed=None): if CONF.feature.driver.current_cloud == Platform.AZURE: return # Get Farm LOG.warning('Get scalarizr logs after scenario %s' % scenario.name) farm = getattr(world, 'farm', None) if not farm: LOG.error("Farm does not exists. Can't get logs. Exit from step.") return farm.servers.reload() # Get servers servers = farm.servers # Get test test_name = scenario.described_at.file.split('/')[-1].split('.')[0] LOG.debug('Test name: %s' % test_name) # Get path start_time = world.test_start_time path = os.path.realpath( os.path.join(CONF.main.log_path, 'scalarizr', test_name, start_time.strftime('%m%d-%H:%M'), scenario.name.replace('/', '-'), outline)) LOG.debug('Path to save log: %s' % path) if not os.path.exists(path): os.makedirs(path, 0755) # Get logs && configs for server in servers: if not server.is_scalarized: continue logs = [ # debug log { 'file': os.path.join(path, '_'.join( (server.id, 'scalarizr_debug.log'))), 'log_type': 'debug', 'compress': True }, # update log { 'file': os.path.join(path, '_'.join( (server.id, 'scalarizr_update.log'))), 'log_type': 'update', 'compress': True } ] if server.status in [ ServerStatus.PENDING, ServerStatus.INIT, ServerStatus.RUNNING ]: try: #Get log from remote host for log in logs: server.get_log_by_api(**log) LOG.info( 'Save {log_type} log from server {} to {file}'.format( server.id, **log)) #Get configs and role behavior from remote host only for linux family if not Dist(server.role.dist).is_windows: file = os.path.join( path, '_'.join( (server.id, 'scalr_configs.tar.gz'))) server.get_configs(file, compress=True) LOG.info( 'Download archive with scalr directory and behavior to: {}' .format(file)) except BaseException, e: LOG.error('Error in downloading configs: %s' % e) continue if server.status == ServerStatus.RUNNING and not CONF.feature.dist.is_windows: node = world.cloud.get_node(server) out = node.run("ps aux | grep 'bin/scal'")[0] for line in out.splitlines(): ram = line.split()[5] if len(ram) > 3: ram = '%sMB' % ram[:-3] if 'bin/scalr-upd-client' in line: LOG.info('Server %s use %s RAM for update-client' % (server.id, ram)) world.wrt( etree.Element('meta', name='szrupdateram', value=ram, serverid=server.id)) elif 'bin/scalarizr' in line: LOG.info('Server %s use %s RAM for scalarizr' % (server.id, ram)) world.wrt( etree.Element('meta', name='szrram', value=ram, serverid=server.id))