def fio_gen_jobs(client_targets, fill=0): """ generate fio job files for 'runfio.sh' then upload to clients return: jobdesc, fio_job_dir jobdesc: the string that summarize the fio job fio_job_dir: directory that contains fio job files on client servers """ jobdesc = None; jobfile_names = [] # write fio job files for each client for t in client_targets: jobdesc, fio_filljob_content = fio_build_job_contents(t, fill) jobdesc, fio_jobfile_content = fio_build_job_contents(t) jobfile_name = "%s_%s.fio" % (jobdesc, t.address) f = open(jobfile_name, "w"); f.write(fio_jobfile_content); f.close() jobfile_names.append(jobfile_name) jobfile_name = "fill%d_%s_%s.fio" % (fill, jobdesc, t.address) f = open(jobfile_name, "w"); f.write(fio_filljob_content); f.close() jobfile_names.append(jobfile_name) # upload fio job files to all clients fio_job_dir = g_runtime_dir + "/fiorun/%s" % (jobdesc) for t in client_targets: t.exe( "mkdir -p %s" % (fio_job_dir) ) for jobfile_name in jobfile_names: if not t.upload( jobfile_name, fio_job_dir ): return None, None for jobfile_name in jobfile_names: me.exe("rm -f %s" % (jobfile_name)) return jobdesc, fio_job_dir
def get_iops(logpath): iops, iops_str = None, None rt, path_results, err = me.exe( "tail -100 {log} | grep 'log location' | awk '{{print $3}}'".format( log=logpath)) rt, iops_str, err = me.exe( "tail -100 {log} | grep 'Total: IOPS:' | awk '{{print $3}}' ".format( log=logpath)) iops = int(iops_str.split("@")[0]) if iops_str else None return iops, iops_str, path_results
def replace_rpm(federation_targets, build_server, force=True): ''' build the latest uniio on build_server and replace rpms on federation nodes ''' if not build_server: common.log("failed replace rpms. build server is None.", 1) return False if not federation_targets: common.log("failed replace rpms. uniio servers are None.", 1) return False cos_build = build(build_server, wait=False) if not cos_build: return False # reinitialize backend if not shutdown_cluster(federation_targets, force=True, wait=True): return False # wait for build job to end for co in cos_build: if not co.succ(): common.log("failed when build.") return False # download from build server and upload rpm packages to federation nodes: me.exe("rm -rf /tmp/rpms && mkdir /tmp/rpms") if not build_server.download("/tmp/rpms/", "%s/uniio/build/object-array-*.rpm" % (g_runtime_dir)): # download uniio rpms return False if not g_uioonly: if not build_server.download("/tmp/rpms/", "%s/nasmgmt/build/object-array-nasmgmt-*.rpm" % (g_runtime_dir)): # download nasmgmt rpms return False if not build_server.download("/tmp/rpms/", "%s/sysmgmt/build/object-array-sysmgmt-*.rpm" % (g_runtime_dir)): # download sysmgmt rpms return False if not build_server.download("/tmp/rpms/", "%s/uniio-ui/build_debug/object-array-uniio-ui-*.rpm" % (g_runtime_dir)): # download uniio-ui rpms return False for t in federation_targets: t.exe("rm -rf /tmp/rpms && mkdir -p /tmp/rpms") if not t.upload("/tmp/rpms/*.rpm", "/tmp/rpms/"): return False cos = [] for t in federation_targets: cmd = "%s/uio_scripts/server/init_cluster.sh %s --replace=/tmp/rpms" % (g_runtime_dir, '-f' if force else "") cos.append(t.exe(cmd, wait=False)) for co in cos: if not co.succ(): common.log("failed when replacing uniio packages.") return False return True
def showfio(path): if not path: return False uiodir = "{0}/..".format(os.path.dirname(os.path.realpath(__file__))) path_showfio = "{0}/client/showfio.py".format(uiodir) cmd = "{showfio} $(ls {path}/fio_output/[^fill]*)".format(showfio=path_showfio, path=path) rt, out, err = me.exe(cmd) print ("\n%s\n" % (out)) return True
def perf_test(client_targets, federation_targets, fill=0): ''' run performance test. start fio workload from clients, collect counter logs at the same time, fill: fill the luns first, for a given time return when all jobs are done. ''' # print threadtable.ini contents on all federation servers for t in federation_targets: t.exe("cat /etc/objblk/threadtable.ini") status_str = "" jobdesc, fio_job_dir, fio_cos, fio_driver = fio_run(client_targets, fill) if not fio_cos: return False counter_log_dir, counter_log_path, counter_cos = counter_log(jobdesc, federation_targets) # wait for jobs to end fio_fail = None for fio_co in fio_cos: if not fio_co.succ(): fio_fail = fio_co if fio_fail: status_str += ".FIO_FAIL_on_%s" % (fio_driver.address) counter_fail = None for co in counter_cos: if not co.succ(): counter_fail = co if counter_fail: status_str += ".COUNTER_FAIL_on_%s" % (counter_fail.shell.t.address) iscsi_out(client_targets) # download fio logs from fio driver node localtime = time.localtime() date_str = "%d-%02d-%02d" % (localtime.tm_year, localtime.tm_mon, localtime.tm_mday) time_str = "%02d.%02d.%02d" % (localtime.tm_hour, localtime.tm_min, localtime.tm_sec) logdir = "perflogs/%s/%s.%s_%s%s" % (date_str, jobdesc, date_str, time_str, status_str) me.exe("rm -rf %s" % (logdir)) me.exe("mkdir -p %s" % (logdir)) if not fio_driver.download(logdir, "%s/*" % (fio_job_dir)): return False # download counter logs and cpu data svg files from federation nodes for t in federation_targets: counterdir = "%s/counter_%s" % (logdir, t.address) svgdir = "%s/cpudata_%s" % (logdir, t.address) me.exe("mkdir -p %s" % (counterdir)) if g_cpudata: me.exe("mkdir -p %s" % (svgdir)) if not t.download(counterdir, counter_log_path): return False if g_cpudata and not t.download(svgdir, counter_log_dir+"/*%s*.svg" % (jobdesc)): return False json.dump(g_conf, open("%s/settings.json" % (logdir), 'w'), indent=2) # dump a copy of config file to the logdir common.log("DONE EEPERFTEST.\n%s\nlog location: %s" % ("-"*60, os.path.join(os.getcwd(), logdir))) showfio (logdir) return True
def run_commit(commit): """run a single perf test on the given hash Args: commit (str): a git commit: [hash, committer date, author name, desc] """ path_perfauto = "{0}/perfauto.py".format( os.path.dirname(os.path.realpath(__file__))) hash = commit[0] logpath = "{0}.{1}.out".format(hash, str(uuid.uuid1())) print("-" * 80) sys.stdout.write("testing {0} log: {1}\n".format("|".join(commit), logpath)) sys.stdout.flush() cmd = "{0} -c {1} -u --ref={2} -p --fill 600 --fullmap --uioonly > {3} ".format( path_perfauto, g_conf_file, hash, logpath) iops = None path = None start = time.time() succ = me.succ(cmd) dur = time.time() - start if not succ: # execution fail sys.stdout.write( "FAIL!! {0} log: {1}\nREALTIME: {2:.0f} seconds.\n".format( "|".join(commit), logpath, dur)) else: iops, iops_str, path = get_iops(logpath) sys.stdout.write( "IOPS: {2} ref: {0} log: {1}\nREALTIME: {3:.0f} seconds.\n".format( "|".join(commit), logpath, iops_str, dur)) if path and os.path.exists(path): me.exe("mv {0} {1}/".format(logpath, path)) if iops: result = [hash, iops] else: # can't get iops for this round, retry with the closest commits result = [hash, 0] print("-" * 80) print("") return result