def fio_gen_jobs(client_targets, fill=0):
"""
generate fio job files for 'runfio.sh' then upload to clients
return: jobdesc, fio_job_dir
jobdesc: the string that summarize the fio job
fio_job_dir: directory that contains fio job files on client servers
"""
jobdesc = None; jobfile_names = []
# write fio job files for each client
for t in client_targets:
jobdesc, fio_filljob_content = fio_build_job_contents(t, fill)
jobdesc, fio_jobfile_content = fio_build_job_contents(t)
jobfile_name = "%s_%s.fio" % (jobdesc, t.address)
f = open(jobfile_name, "w"); f.write(fio_jobfile_content); f.close()
jobfile_names.append(jobfile_name)
jobfile_name = "fill%d_%s_%s.fio" % (fill, jobdesc, t.address)
f = open(jobfile_name, "w"); f.write(fio_filljob_content); f.close()
jobfile_names.append(jobfile_name)
# upload fio job files to all clients
fio_job_dir = g_runtime_dir + "/fiorun/%s" % (jobdesc)
for t in client_targets:
t.exe( "mkdir -p %s" % (fio_job_dir) )
for jobfile_name in jobfile_names:
if not t.upload( jobfile_name, fio_job_dir ):
return None, None
for jobfile_name in jobfile_names:
me.exe("rm -f %s" % (jobfile_name))
return jobdesc, fio_job_dir
def get_iops(logpath):
iops, iops_str = None, None
rt, path_results, err = me.exe(
"tail -100 {log} | grep 'log location' | awk '{{print $3}}'".format(
log=logpath))
rt, iops_str, err = me.exe(
"tail -100 {log} | grep 'Total: IOPS:' | awk '{{print $3}}' ".format(
log=logpath))
iops = int(iops_str.split("@")[0]) if iops_str else None
return iops, iops_str, path_results
def replace_rpm(federation_targets, build_server, force=True):
'''
build the latest uniio on build_server and replace rpms on federation nodes
'''
if not build_server:
common.log("failed replace rpms. build server is None.", 1)
return False
if not federation_targets:
common.log("failed replace rpms. uniio servers are None.", 1)
return False
cos_build = build(build_server, wait=False)
if not cos_build:
return False
# reinitialize backend
if not shutdown_cluster(federation_targets, force=True, wait=True):
return False
# wait for build job to end
for co in cos_build:
if not co.succ():
common.log("failed when build.")
return False
# download from build server and upload rpm packages to federation nodes:
me.exe("rm -rf /tmp/rpms && mkdir /tmp/rpms")
if not build_server.download("/tmp/rpms/", "%s/uniio/build/object-array-*.rpm" % (g_runtime_dir)): # download uniio rpms
return False
if not g_uioonly:
if not build_server.download("/tmp/rpms/", "%s/nasmgmt/build/object-array-nasmgmt-*.rpm" % (g_runtime_dir)): # download nasmgmt rpms
return False
if not build_server.download("/tmp/rpms/", "%s/sysmgmt/build/object-array-sysmgmt-*.rpm" % (g_runtime_dir)): # download sysmgmt rpms
return False
if not build_server.download("/tmp/rpms/", "%s/uniio-ui/build_debug/object-array-uniio-ui-*.rpm" % (g_runtime_dir)): # download uniio-ui rpms
return False
for t in federation_targets:
t.exe("rm -rf /tmp/rpms && mkdir -p /tmp/rpms")
if not t.upload("/tmp/rpms/*.rpm", "/tmp/rpms/"):
return False
cos = []
for t in federation_targets:
cmd = "%s/uio_scripts/server/init_cluster.sh %s --replace=/tmp/rpms" % (g_runtime_dir, '-f' if force else "")
cos.append(t.exe(cmd, wait=False))
for co in cos:
if not co.succ():
common.log("failed when replacing uniio packages.")
return False
return True
def showfio(path):
if not path: return False
uiodir = "{0}/..".format(os.path.dirname(os.path.realpath(__file__)))
path_showfio = "{0}/client/showfio.py".format(uiodir)
cmd = "{showfio} $(ls {path}/fio_output/[^fill]*)".format(showfio=path_showfio, path=path)
rt, out, err = me.exe(cmd)
print ("\n%s\n" % (out))
return True
def perf_test(client_targets, federation_targets, fill=0):
'''
run performance test.
start fio workload from clients,
collect counter logs at the same time,
fill: fill the luns first, for a given time
return when all jobs are done.
'''
# print threadtable.ini contents on all federation servers
for t in federation_targets:
t.exe("cat /etc/objblk/threadtable.ini")
status_str = ""
jobdesc, fio_job_dir, fio_cos, fio_driver = fio_run(client_targets, fill)
if not fio_cos:
return False
counter_log_dir, counter_log_path, counter_cos = counter_log(jobdesc, federation_targets)
# wait for jobs to end
fio_fail = None
for fio_co in fio_cos:
if not fio_co.succ():
fio_fail = fio_co
if fio_fail: status_str += ".FIO_FAIL_on_%s" % (fio_driver.address)
counter_fail = None
for co in counter_cos:
if not co.succ():
counter_fail = co
if counter_fail: status_str += ".COUNTER_FAIL_on_%s" % (counter_fail.shell.t.address)
iscsi_out(client_targets)
# download fio logs from fio driver node
localtime = time.localtime()
date_str = "%d-%02d-%02d" % (localtime.tm_year, localtime.tm_mon, localtime.tm_mday)
time_str = "%02d.%02d.%02d" % (localtime.tm_hour, localtime.tm_min, localtime.tm_sec)
logdir = "perflogs/%s/%s.%s_%s%s" % (date_str, jobdesc, date_str, time_str, status_str)
me.exe("rm -rf %s" % (logdir))
me.exe("mkdir -p %s" % (logdir))
if not fio_driver.download(logdir, "%s/*" % (fio_job_dir)):
return False
# download counter logs and cpu data svg files from federation nodes
for t in federation_targets:
counterdir = "%s/counter_%s" % (logdir, t.address)
svgdir = "%s/cpudata_%s" % (logdir, t.address)
me.exe("mkdir -p %s" % (counterdir))
if g_cpudata: me.exe("mkdir -p %s" % (svgdir))
if not t.download(counterdir, counter_log_path):
return False
if g_cpudata and not t.download(svgdir, counter_log_dir+"/*%s*.svg" % (jobdesc)):
return False
json.dump(g_conf, open("%s/settings.json" % (logdir), 'w'), indent=2) # dump a copy of config file to the logdir
common.log("DONE EEPERFTEST.\n%s\nlog location: %s" % ("-"*60, os.path.join(os.getcwd(), logdir)))
showfio (logdir)
return True
def run_commit(commit):
"""run a single perf test on the given hash
Args:
commit (str): a git commit: [hash, committer date, author name, desc]
"""
path_perfauto = "{0}/perfauto.py".format(
os.path.dirname(os.path.realpath(__file__)))
hash = commit[0]
logpath = "{0}.{1}.out".format(hash, str(uuid.uuid1()))
print("-" * 80)
sys.stdout.write("testing {0} log: {1}\n".format("|".join(commit),
logpath))
sys.stdout.flush()
cmd = "{0} -c {1} -u --ref={2} -p --fill 600 --fullmap --uioonly > {3} ".format(
path_perfauto, g_conf_file, hash, logpath)
iops = None
path = None
start = time.time()
succ = me.succ(cmd)
dur = time.time() - start
if not succ: # execution fail
sys.stdout.write(
"FAIL!! {0} log: {1}\nREALTIME: {2:.0f} seconds.\n".format(
"|".join(commit), logpath, dur))
else:
iops, iops_str, path = get_iops(logpath)
sys.stdout.write(
"IOPS: {2} ref: {0} log: {1}\nREALTIME: {3:.0f} seconds.\n".format(
"|".join(commit), logpath, iops_str, dur))
if path and os.path.exists(path):
me.exe("mv {0} {1}/".format(logpath, path))
if iops:
result = [hash, iops]
else: # can't get iops for this round, retry with the closest commits
result = [hash, 0]
print("-" * 80)
print("")
return result