def create_result(self):
#method.check_none(self.parallels, self.times)
labels = ("Copy", 'Scale', 'Add', 'Triad')
for parallel in self.parallels:
sum_dic = {}
for iter in range(self.times):
iter_dic = {}
#tmp_result_file = os.path.join(self.tmp_dir, "stream_%d_%d.out" %(parallel, iter+1))
tmp_result_file = os.path.join(
self.resultsdir, "stream_%d_%d.out" % (parallel, iter + 1))
if not os.path.isfile(tmp_result_file):
lptlog.warning("测试数据 %s 不存在" % tmp_result_file)
continue
result_tuple = self.__search_result(tmp_result_file)
for key, value in zip(labels, result_tuple):
iter_dic[key] = "%.4f" % value
if sum_dic:
sum_dic = method.append_sum_dict(sum_dic, iter_dic)
else:
sum_dic = iter_dic.copy()
self.result_list.append([
self.create_result_node_attrib(iter + 1, self.times,
parallel, self.parallels),
iter_dic
])
if sum_dic:
average_dic = method.append_average_dict(sum_dic, self.times)
self.result_list.append([
self.create_result_node_attrib("Average", self.times,
parallel, self.parallels),
average_dic
])
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
self.parallels = self.get_config_array(tool_node, "parallel", [1])
lptlog.info("测试并行组: %s" % utils.list_to_str(self.parallels))
self.times = self.get_config_value(tool_node, "times", 10, valueType=int)
if self.times < 3:
self.times = 10
lptlog.warning("测试次数必须大于3， 将采用默认值10")
lptlog.info("测试次数: %d" % self.times)
cmd = "./Run"
#运行unixbench程序，进入unixbench 根目录, 清理环境
os.chdir(self.tar_src_dir)
utils.system("rm -rf results/*")
#执行测试程序
#执行测试脚本
lptlog.info("---------运行测试脚本")
#添加测试次数
args_list=["-i", "%d" %self.times]
#添加并行数
for parallel in self.parallels:
args_list.append("-c")
args_list.append("%d" % parallel)
self.mainParameters["parameters"] = " ".join([cmd]+args_list)
utils.run_shell2(cmd, args_list=args_list, file=os.devnull)
#utils.system_output(cmd, args=args_list)
#返回根目录
os.chdir(self.lpt_root)
def getopts(argv=sys.argv):
'''
自定义参数
'''
tools_list = []
try:
opts, args = getopt.getopt(argv[1:], "ht:g:",
["help", 'test', 'group'])
except getopt.GetoptError:
usage()
if len(argv) <= 1:
usage()
for opt, value in opts:
if opt in ('-h', '--help'):
usage()
if opt == '-t':
if value not in TOOLS:
lptlog.error('请核对测试工具，不支持您所输入的测试工具 %s' % value)
sys.exit()
else:
tools_list.append(value)
if opt in ('-g', '--group'):
value_list = value.split()
for tools in value_list:
if tools in TOOLS:
tools_list.append(tools)
else:
lptlog.warning('%s不是有效的测试工具，请核对工具名称' % tools)
return tools_list
def create_result(self):
#method.check_none(self.parallels, self.times)
labels = ("Copy", 'Scale', 'Add', 'Triad')
for parallel in self.parallels:
sum_dic = {}
for iter in range(self.times):
iter_dic = {}
#tmp_result_file = os.path.join(self.tmp_dir, "stream_%d_%d.out" %(parallel, iter+1))
tmp_result_file = os.path.join(self.resultsdir, "stream_%d_%d.out" %(parallel, iter+1))
if not os.path.isfile(tmp_result_file):
lptlog.warning("测试数据 %s 不存在" % tmp_result_file)
continue
result_tuple = self.__search_result(tmp_result_file)
for key, value in zip(labels, result_tuple):
iter_dic[key] = "%.4f" %value
if sum_dic:
sum_dic = method.append_sum_dict(sum_dic, iter_dic)
else:
sum_dic = iter_dic.copy()
self.result_list.append([self.create_result_node_attrib(iter+1, self.times, parallel, self.parallels), iter_dic])
if sum_dic:
average_dic = method.append_average_dict(sum_dic, self.times)
self.result_list.append([self.create_result_node_attrib("Average", self.times, parallel, self.parallels), average_dic])
def getopts(argv=sys.argv):
'''
自定义参数
'''
tools_list = []
try:
opts, args = getopt.getopt(argv[1:],"ht:g:",["help",'test', 'group'])
except getopt.GetoptError:
usage()
if len(argv) <= 1:
usage()
for opt,value in opts:
if opt in ('-h','--help'):
usage()
if opt == '-t':
if value not in TOOLS:
lptlog.error('请核对测试工具，不支持您所输入的测试工具 %s' % value)
sys.exit()
else:
tools_list.append(value)
if opt in ('-g', '--group'):
value_list = value.split()
for tools in value_list:
if tools in TOOLS:
tools_list.append(tools)
else:
lptlog.warning('%s不是有效的测试工具，请核对工具名称' % tools)
return tools_list
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
self.parallels = self.get_config_array(tool_node, "parallel", [1])
lptlog.info("测试并行组: %s" % utils.list_to_str(self.parallels))
self.times = self.get_config_value(tool_node, "times", 10, valueType=int)
if self.times < 3:
self.times = 10
lptlog.warning("测试次数必须大于3， 将采用默认值10")
lptlog.info("测试次数: %d" % self.times)
cmd = "./Run"
#运行unixbench程序，进入unixbench 根目录, 清理环境
os.chdir(self.tar_src_dir)
utils.system("rm -rf results/*")
#执行测试程序
#执行测试脚本
lptlog.info("---------运行测试脚本")
#添加测试次数
args_list=["-i", "%d" %self.times]
#添加并行数
for parallel in self.parallels:
args_list.append("-c")
args_list.append("%d" % parallel)
self.mainParameters["parameters"] = " ".join([cmd]+args_list)
utils.run_shell2(cmd, args_list=args_list, file=os.devnull)
#utils.system_output(cmd, args=args_list)
#返回根目录
os.chdir(self.lpt_root)
def compare(xmls_list,
resultDir=os.getcwd(),
reportFile=None,
names=[],
tools=[],
reportType="xls",
chart=False):
'''对比多个result xml文件， 如果指定tools，那么只对比tools,但tools必须包含在所有xmls文件中
@param xmls: results xml ,type list
@param names: 自定义对比名称，用于输出显示，type list, default []
@param tools: 指定对比测试工具，type list, default [] '''
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
开始创建对比测试报告
~~~~~~~~~~~~~~~~~~~~''')
#检查resultDir
if resultDir is None:
resultDir = os.getcwd()
if not os.path.isdir(resultDir):
os.makedirs(resultDir, mode=644)
if reportFile is None:
reportFile = "LPTCmpResult_%s.%s" % (
datetime.datetime.now().strftime('%y-%m-%d_%H:%M:%S'), reportType)
elif reportFile.split(".")[-1] in ('txt', 'xls', 'doc', 'pdf'):
reportFile = reportFile
else:
reportFile = "%s.%s" % (reportFile, reportType)
report_abs_file = os.path.join(resultDir, reportFile)
lptlog.info("@@@@@@@--指定对比工具: %s" % utils.list_to_str(tools))
lptlog.info("@@@@@@@--对比对象分别为: %s " % utils.list_to_str(names))
lptlog.info("@@@@@@@--对比测试报告类型: %s" % reportType)
lptlog.info("@@@@@@@--对比测试报告: %s" % report_abs_file)
if reportType == "xls":
cmpobject = lptreport.XlsCompare(xmls_list,
report_abs_file,
input_tools=tools,
input_name_list=names)
cmpobject.cmp_tools(chart=chart)
cmpobject.save()
else:
lptlog.warning(" %s 对比测试报告正在开发中..." % reportType)
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
创建对比测试报告结束
~~~~~~~~~~~~~~~~~~~~''')
def create_result(self):
'''创建result_list
'''
#labels = ("Throughtput", "clients", "max_latency")
labels = ("Throughtput", "max_latency")
parallelstring = ",".join(map(str, self.parallels))
r = re.compile(
r"Throughput\s+(\d+.\d+)\s+MB/sec\s+(\d+)\s+clients\s+\d+\s+procs\s+max_latency=(\d+.\d+)\s",
re.I)
for parallel in self.parallels:
sum_dic = {}
for iter in range(self.times):
tmp_result_file = os.path.join(
self.tmp_dir,
"%s_%s_%s.out" % (self.tool, parallel, iter + 1))
if not os.path.isfile(tmp_result_file):
lptlog.warning("%s 不存在" % tmp_result_file)
continue
result_lines = utils.read_all_lines(tmp_result_file)
for line in result_lines:
key_dic = {}
if r.match(line):
m = r.match(line)
#result_list = [m.group(1), m.group(2), m.group(3)]
result_list = [m.group(1), m.group(3)]
result_tuple = tuple(
[utils.change_type(i) for i in result_list])
for l, v in zip(labels, result_tuple):
key_dic[l] = "%d" % v
if not sum_dic:
sum_dic = key_dic.copy()
else:
sum_dic = method.append_sum_dict(sum_dic, key_dic)
self.result_list.append([
self.create_result_node_attrib(
iter + 1, self.times, parallel,
self.parallels), key_dic
])
if sum_dic:
parallel_average_dic = method.append_average_dict(
sum_dic, self.times)
lptlog.debug("%d 并行求平均值:PASS" % parallel)
self.result_list.append([
self.create_result_node_attrib("Average", self.times,
parallel, self.parallels),
parallel_average_dic
])
def get_config_testdir(self, tool_node):
"""获取IO测试目录
"""
testdir = self.get_config_value(tool_node, "testdir", os.path.join(self.lpttmpdir, "testdir"), valueType=str)
if os.path.exists(testdir):
if not os.path.isdir(testdir):
lptlog.warning("%s 不是有效目录，将采用 /home/%u/testdir 目录" % testdir)
testdir = "/home/%s/testdir" % getpass.getuser()
os.makedirs(testdir, stat.S_IRWXU)
else:
os.makedirs(testdir, stat.S_IRWXU)
testdir = os.path.abspath(testdir)
lptlog.info("测试目录: %s" % testdir)
return testdir
def get_config_testdir(self, tool_node):
'''获取IO测试目录
'''
testdir = self.get_config_value(tool_node, "testdir", os.path.join(self.lpttmpdir, "testdir"), valueType=str)
if os.path.exists(testdir):
if not os.path.isdir(testdir):
lptlog.warning("%s 不是有效目录，将采用 /home/%u/testdir 目录" % testdir)
testdir = "/home/%s/testdir" % getpass.getuser()
os.makedirs(testdir, stat.S_IRWXU)
else:
os.makedirs(testdir, stat.S_IRWXU)
testdir = os.path.abspath(testdir)
lptlog.info("测试目录: %s" % testdir)
return testdir
def run_once(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
self.parallels = self.get_config_array(tool_node, "parallel", [1])
lptlog.info("测试并行组: %s" % lutils.list_to_str(self.parallels))
self.times = self.get_config_value(tool_node,
"times",
10,
valueType=int)
if self.times < 3:
self.times = 10
lptlog.warning("测试次数必须大于3， 将采用默认值10")
lptlog.info("测试次数: %d" % self.times)
cmd = "./Run"
#执行测试脚本
lptlog.info("---------运行测试脚本")
#shutil.rmtree(self.resultsdir)
os.chdir(self.srcdir)
#添加测试次数
args_list = ["-i", "%d" % self.times]
#添加并行数
for parallel in self.parallels:
args_list.append("-c")
args_list.append("%d" % parallel)
self.mainParameters["parameters"] = " ".join([cmd] + args_list)
#utils.run_shell2(cmd, args_list=args_list, file=os.devnull)
#utils.system_output(cmd, args=args_list)
#返回根目录
vars = 'UB_TMPDIR="%s" UB_RESULTDIR="%s"' % (self.tmpdir,
self.resultsdir)
self.report_data = utils.system_output(vars + ' %s ' % cmd +
' '.join(args_list))
self.results_path = os.path.join(self.resultsdir,
'raw_output_%s' % self.iteration)
utils.open_write_close(self.results_path, self.report_data)
#数据处理
self.create_result()
self.save_results_to_xml()
#create txt report
self.txt_report()
def report(result_xml,
resultDir,
tools_list=None,
reportname=None,
format='xls',
chart=False):
'''report 方法'''
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
开始创建测试报告
~~~~~~~~~~~~~~~~~~~~''')
if reportname is None:
reportname = "LPTResult_%s.%s" % (
datetime.datetime.now().strftime('%y-%m-%d_%H:%M:%S'), format)
elif reportname.split(".")[-1] in ('txt', 'xls', 'doc', 'pdf'):
reportname = reportname
else:
reportname = "%s.%s" % (reportname, format)
report_file = os.path.join(resultDir, reportname)
if not tools_list:
tools_list = lptxml.get_result_tools(result_xml)
if tools_list is None:
lptlog.warning("%s 中无测试数据" % result_xml)
raise ValueError("result.xml: %s" % result_xml)
lptlog.info("测试报告工具集: %s" % utils.list_to_str(tools_list))
lptlog.info("测试报告格式: %s" % format)
lptlog.info("测试报告名称: %s" % reportname)
#lptreport.report(result_xml, tools_list, report_file, format)
if format == "xls":
lptreport.xls_report(result_xml, tools_list, report_file, chart=chart)
elif format == "txt":
lptreport.txt_report(result_xml, tools_list, report_file)
else:
pass
if format == 'txt':
lptlog.info("Report File: %s*.txt" % report_file.split(".txt")[0])
else:
lptlog.info("Report File: %s" % report_file)
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
创建测试报告结束
~~~~~~~~~~~~~~~~~~~~''')
def get_config_array(self, tool_node, key, defaultValue):
'''转换 1，2，4字符类型为list
'''
try:
getList = lptxml.get_tool_parameter(tool_node, key)
lptlog.debug('获取 %s :%s' % (key, getList))
if getList is None:
getList = defaultValue
getList = utils.check_int_list(getList)
except Exception:
lptlog.warning('获取 %s error,将采用默认值' % key)
getList = defaultValue
finally:
if not isinstance(getList, list):
raise TypeError(getList)
return getList
def get_config_array(self, tool_node, key, defaultValue):
"""转换 1，2，4字符类型为list
"""
try:
getList = lptxml.get_tool_parameter(tool_node, key)
lptlog.debug("获取 %s :%s" % (key, getList))
if getList is None:
getList = defaultValue
getList = utils.check_int_list(getList)
except Exception:
lptlog.warning("获取 %s error,将采用默认值" % key)
getList = defaultValue
finally:
if not isinstance(getList, list):
raise TypeError, getList
return getList
def clean(self):
'''清理测试环境
'''
try:
if self.tar_src_dir:
shutil.rmtree(self.tar_src_dir)
lptlog.info("清理源目录 %s :PASS" % self.tar_src_dir)
if self.processBin is not None and os.path.exists(self.processBin):
os.remove(self.processBin)
lptlog.info("清理Bin文件 %s :PASS" % self.processBin)
if self.processBin2 is not None and os.path.exists(self.processBin2):
os.remove(self.processBin2)
lptlog.info("清理Bin文件 %s :PASS" % self.processBin2)
except Exception, e:
lptlog.warning('清理临时目录或文件：FAIL')
lptlog.debug(e)
def get_config_value(self, tool_node, key, defaultValue, valueType=str):
"""从parameters.conf文件中读取Value, 如果读取失败，赋予key, defaultValue
"""
try:
getValue = lptxml.get_tool_parameter(tool_node, key)
lptlog.debug("获取 %s : %s" % (key, getValue))
if getValue is None:
getValue = defaultValue
except Exception:
lptlog.warning("获取 %s error,将采用默认值: %s" % (key, defaultValue))
getValue = defaultValue
finally:
try:
getValue = valueType(getValue)
except Exception:
raise FormatterError, getValue
return getValue
def get_config_value(self, tool_node, key, defaultValue, valueType=str):
'''从parameters.conf文件中读取Value, 如果读取失败，赋予key, defaultValue
'''
try:
getValue = lptxml.get_tool_parameter(tool_node, key)
lptlog.debug('获取 %s : %s' % (key, getValue))
if getValue is None:
getValue = defaultValue
except Exception:
lptlog.warning("获取 %s error,将采用默认值: %s" %(key, defaultValue))
getValue = defaultValue
finally:
try:
getValue = valueType(getValue)
except Exception:
raise FormatterError(getValue)
return getValue
def clean(self):
'''清理测试环境
'''
try:
if self.tar_src_dir:
shutil.rmtree(self.tar_src_dir)
lptlog.info("清理源目录 %s :PASS" % self.tar_src_dir)
if self.processBin is not None and os.path.exists(self.processBin):
os.remove(self.processBin)
lptlog.info("清理Bin文件 %s :PASS" % self.processBin)
if self.processBin2 is not None and os.path.exists(
self.processBin2):
os.remove(self.processBin2)
lptlog.info("清理Bin文件 %s :PASS" % self.processBin2)
except Exception, e:
lptlog.warning('清理临时目录或文件：FAIL')
lptlog.debug(e)
def run_once(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
self.parallels = self.get_config_array(tool_node, "parallel", [1])
lptlog.info("测试并行组: %s" % lutils.list_to_str(self.parallels))
self.times = self.get_config_value(tool_node, "times", 10, valueType=int)
if self.times < 3:
self.times = 10
lptlog.warning("测试次数必须大于3， 将采用默认值10")
lptlog.info("测试次数: %d" % self.times)
cmd = "./Run"
#执行测试脚本
lptlog.info("---------运行测试脚本")
#shutil.rmtree(self.resultsdir)
os.chdir(self.srcdir)
#添加测试次数
args_list=["-i", "%d" %self.times]
#添加并行数
for parallel in self.parallels:
args_list.append("-c")
args_list.append("%d" % parallel)
self.mainParameters["parameters"] = " ".join([cmd]+args_list)
#utils.run_shell2(cmd, args_list=args_list, file=os.devnull)
#utils.system_output(cmd, args=args_list)
#返回根目录
vars = 'UB_TMPDIR="%s" UB_RESULTDIR="%s"' % (self.tmpdir,
self.resultsdir)
self.report_data = utils.system_output(vars + ' %s ' % cmd + ' '.join(args_list))
self.results_path = os.path.join(self.resultsdir,
'raw_output_%s' % self.iteration)
utils.open_write_close(self.results_path, self.report_data)
#数据处理
self.create_result()
self.save_results_to_xml()
#create txt report
self.txt_report()
def get_config_devices(self, tool_node, testdir):
'''获取测试设备， 并挂载到testdir
'''
devices = self.get_config_value(tool_node, "devices", "Nodevice", valueType=str)
lptlog.info("测试分区: %s " % devices)
if not os.path.exists(devices):
lptlog.debug("%s 不存在" % devices)
return False
else:
try:
if not os.path.ismount(testdir):
utils.system("mount %s %s" % (devices, testdir))
else:
lptlog.debug("%s 已经挂载到 %s 目录" % (devices, testdir))
return True
except Exception:
lptlog.warning("mount %s %s 失败，请确认分区是否已经格式化！！" % (devices, testdir))
return False
def report(result_xml, resultDir, tools_list=None,
reportname=None, format='xls', chart=False):
'''report 方法'''
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
开始创建测试报告
~~~~~~~~~~~~~~~~~~~~''')
if reportname is None:
reportname = "LPTResult_%s.%s" %(datetime.datetime.now().strftime('%y-%m-%d_%H:%M:%S'), format)
elif reportname.split(".")[-1] in ('txt', 'xls', 'doc', 'pdf'):
reportname = reportname
else:
reportname = "%s.%s" %(reportname, format)
report_file = os.path.join(resultDir, reportname)
if not tools_list:
tools_list = lptxml.get_result_tools(result_xml)
if tools_list is None :
lptlog.warning("%s 中无测试数据" % result_xml)
raise ValueError, "result.xml: %s" % result_xml
lptlog.info("测试报告工具集: %s" % utils.list_to_str(tools_list))
lptlog.info("测试报告格式: %s" % format)
lptlog.info("测试报告名称: %s" % reportname)
#lptreport.report(result_xml, tools_list, report_file, format)
if format == "xls":
lptreport.xls_report(result_xml, tools_list, report_file, chart=chart)
elif format == "txt":
lptreport.txt_report(result_xml, tools_list, report_file)
else:
pass
if format=='txt':
lptlog.info("Report File: %s*.txt" % report_file.split(".txt")[0])
else:
lptlog.info("Report File: %s" % report_file)
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
创建测试报告结束
~~~~~~~~~~~~~~~~~~~~''')
def get_config_devices(self, tool_node, testdir):
"""获取测试设备， 并挂载到testdir
"""
devices = self.get_config_value(tool_node, "devices", "Nodevice", valueType=str)
lptlog.info("测试分区: %s " % devices)
if not os.path.exists(devices):
lptlog.debug("%s 不存在" % devices)
return False
else:
try:
if not os.path.ismount(testdir):
utils.system("mount %s %s" % (devices, testdir))
else:
lptlog.debug("%s 已经挂载到 %s 目录" % (devices, testdir))
return True
except Exception:
lptlog.warning("mount %s %s 失败，请确认分区是否已经格式化！！" % (devices, testdir))
return False
def getopts(argv=sys.argv):
tests_list = []
xmls_list = []
names_list = []
report_file = None
try:
opts, args = getopt.getopt(argv[1:], "ht:f:r:n:",
["help", 'test', 'file', 'report', 'name'])
except getopt.GetoptError:
usage()
if len(argv) <= 1:
usage()
for opt, value in opts:
if opt in ('-h', '--help'):
usage()
if opt in ('-t', '--test'):
if value not in TOOLS:
lptlog.warning('请核对测试工具，不支持您所输入的测试工具 %s' % value)
else:
tests_list.append(value)
if opt in ('-f', '--file'):
if os.path.isfile(value):
xmls_list.append(value)
if opt in ('-n', '--name'):
names_list.append(value)
if opt in ('-r', '--report'):
report_file = value
if len(xmls_list) < 2:
lptlog.warning("期望result.xml > 2")
sys.exit()
else:
compare(xmls_list,
names=names_list,
reportFile=report_file,
tools=tests_list)
def compare(xmls_list, resultDir=os.getcwd() ,reportFile=None, names=[], tools=[], reportType="xls", chart=False):
'''对比多个result xml文件， 如果指定tools，那么只对比tools,但tools必须包含在所有xmls文件中
@param xmls: results xml ,type list
@param names: 自定义对比名称，用于输出显示，type list, default []
@param tools: 指定对比测试工具，type list, default [] '''
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
开始创建对比测试报告
~~~~~~~~~~~~~~~~~~~~''')
#检查resultDir
if resultDir is None:
resultDir = os.getcwd()
if not os.path.isdir(resultDir):
os.makedirs(resultDir, mode=644)
if reportFile is None:
reportFile = "LPTCmpResult_%s.%s" %(datetime.datetime.now().strftime('%y-%m-%d_%H:%M:%S'), reportType)
elif reportFile.split(".")[-1] in ('txt', 'xls', 'doc', 'pdf'):
reportFile = reportFile
else:
reportFile = "%s.%s" %(reportFile, reportType)
report_abs_file = os.path.join(resultDir, reportFile)
lptlog.info("@@@@@@@--指定对比工具: %s" % utils.list_to_str(tools))
lptlog.info("@@@@@@@--对比对象分别为: %s " % utils.list_to_str(names))
lptlog.info("@@@@@@@--对比测试报告类型: %s" % reportType)
lptlog.info("@@@@@@@--对比测试报告: %s" % report_abs_file)
if reportType == "xls":
cmpobject = lptreport.XlsCompare(xmls_list, report_abs_file, input_tools=tools, input_name_list=names)
cmpobject.cmp_tools(chart=chart)
cmpobject.save()
else:
lptlog.warning(" %s 对比测试报告正在开发中..." % reportType)
lptlog.info('''
~~~~~~~~~~~~~~~~~~~~
创建对比测试报告结束
~~~~~~~~~~~~~~~~~~~~''')
def clean(self):
'''清理测试环境
'''
try:
if self.tar_src_dir:
lptlog.info(self.tar_src_dir)
shutil.rmtree(self.tar_src_dir)
lptlog.info("清理源目录 %s :PASS" % self.tar_src_dir)
if self.processBin is not None and os.path.exists(self.processBin):
os.remove(self.processBin)
lptlog.info("清理Bin文件 %s :PASS" % self.processBin)
if self.processBin2 is not None and os.path.exists(
self.processBin2):
os.remove(self.processBin2)
lptlog.info("清理Bin文件 %s :PASS" % self.processBin2)
except Exception as e:
lptlog.warning('清理临时目录或文件：FAIL')
lptlog.debug(e)
#raise CleanError, e
finally:
os.chdir(self.lpt_root)
def getopts(argv=sys.argv):
tests_list = []
xmls_list = []
names_list = []
report_file = None
try:
opts, args = getopt.getopt(argv[1:],"ht:f:r:n:",["help",'test', 'file', 'report', 'name'])
except getopt.GetoptError:
usage()
if len(argv) <= 1:
usage()
for opt,value in opts:
if opt in ('-h','--help'):
usage()
if opt in ('-t', '--test'):
if value not in TOOLS:
lptlog.warning('请核对测试工具，不支持您所输入的测试工具 %s' % value)
else:
tests_list.append(value)
if opt in ('-f', '--file'):
if os.path.isfile(value):
xmls_list.append(value)
if opt in ('-n', '--name'):
names_list.append(value)
if opt in ('-r', '--report'):
report_file = value
if len(xmls_list) < 2:
lptlog.warning("期望result.xml > 2")
sys.exit()
else:
compare(xmls_list, names=names_list, reportFile=report_file, tools=tests_list)
def create_result(self):
'''创建result_list
'''
#labels = ("Throughtput", "clients", "max_latency")
labels = ("Throughtput", "max_latency")
parallelstring = ",".join(map(str, self.parallels))
r = re.compile(r"Throughput\s+(\d+.\d+)\s+MB/sec\s+(\d+)\s+clients\s+\d+\s+procs\s+max_latency=(\d+.\d+)\s", re.I)
for parallel in self.parallels:
sum_dic = {}
for iter in range(self.times):
tmp_result_file = os.path.join(self.tmp_dir, "%s_%s_%s.out" % (self.tool, parallel, iter+1))
if not os.path.isfile(tmp_result_file):
lptlog.warning("%s 不存在" % tmp_result_file)
continue
result_lines = utils.read_all_lines(tmp_result_file)
for line in result_lines:
key_dic = {}
if r.match(line):
m = r.match(line)
#result_list = [m.group(1), m.group(2), m.group(3)]
result_list = [m.group(1), m.group(3)]
result_tuple = tuple([utils.change_type(i)for i in result_list])
for l, v in zip(labels, result_tuple):
key_dic[l] = "%d" % v
if not sum_dic:
sum_dic = key_dic.copy()
else:
sum_dic = method.append_sum_dict(sum_dic, key_dic)
self.result_list.append([self.create_result_node_attrib(iter+1, self.times, parallel, self.parallels), key_dic])
if sum_dic:
parallel_average_dic = method.append_average_dict(sum_dic, self.times)
lptlog.debug("%d 并行求平均值:PASS" % parallel)
self.result_list.append([self.create_result_node_attrib("Average", self.times, parallel, self.parallels), parallel_average_dic])
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
speed_mount_status = False
args = ["-i", "0", "-i", "1", "-i", "2"]
self.testmode = self.get_config_value(tool_node,
"testmode",
os.path.join(
self.tmp_dir, "speed"),
valueType=str)
if self.testmode not in ("speed", "throughput"):
self.testmode = "speed"
lptlog.info("测试模式: %s" % self.testmode)
testdir = self.get_config_testdir(tool_node)
#获取设备,并尝试挂载到testdir,返回True或者False
devices_status = self.get_config_devices(tool_node, testdir)
filesize = self.get_config_value(tool_node, "filesize", "10g")
if not utils.check_size_format(filesize):
lptlog.warning("%s 格式 error,将采用默认大小10g" % filesize)
filesize = "10g"
lptlog.info("测试文件大小： %s" % filesize)
args.append("-s")
args.append(filesize)
blocksize = self.get_config_value(tool_node, "blocksize", "4k")
if not utils.check_size_format(blocksize, match="[\d]+k?"):
lptlog.warning("blocksize=%s 格式 error,将采用默认大小4k" % blocksize)
lptlog.info("测试块大小: %s" % blocksize)
args.append("-r")
args.append(blocksize)
self.times = self.get_config_value(tool_node,
"times",
5,
valueType=int)
direct_status = self.get_config_value(tool_node,
"directio",
"False",
valueType=str)
if direct_status == "True":
args.append("-I")
lptlog.info("DirectIO: True")
self.parallel_type = self.get_config_value(tool_node,
"parallel_type",
"process",
valueType=str)
if self.parallel_type not in ("threads", "process"):
self.parallel_type = "process"
lptlog.info("测试并行方式: %s" % self.parallel_type)
self.parallels = self.get_config_array(tool_node, "parallel", [4])
lptlog.info("测试并行: %s" % utils.list_to_str(self.parallels))
if self.testmode == "speed":
self.parallels = [1]
#运行测试程序，要求保存结果到tmp目录，result_file命令为iozone_$parallel_type_$iter.out
self.mainParameters["parameters"] = " ".join(["iozone"] + args)
lptlog.info("----------运行测试脚本")
for parallel in self.parallels:
parallel_args = []
lptlog.info("%s 并行测试" % parallel)
if self.testmode == 'throughput':
parallel_args.append("-t")
parallel_args.append(str(parallel))
parallel_args.append("-F")
for num in range(parallel):
parallel_args.append(
os.path.join(testdir, "iozone_%s_%s_%s") %
(self.testmode, parallel, num + 1))
if self.parallel_type == "threads":
parallel_args.append("-T")
else:
# if devices_status:
# parallel_args.append("-U")
# parallel_args.append(testdir)
parallel_args.append("-f")
parallel_args.append("%s/iozone_test_file" % testdir)
for iter in range(self.times):
lptlog.info("第 %d 次测试" % (iter + 1))
iozone_iter_result_file = os.path.join(
self.tmp_dir,
"iozone_%s_%d_%d.out" % (self.testmode, parallel,
(iter + 1)))
#清除缓冲
method.clean_buffer()
utils.run_shell2(self.processBin,
args_list=args + parallel_args,
file=iozone_iter_result_file)
lptlog.info("%s %s方式, %s并行, 第%d次测试数据保存在 %s 中" %
(self.tool, self.testmode, parallel,
(iter + 1), iozone_iter_result_file))
def clean_buffer():
try:
utils.system('echo "3" > /proc/sys/vm/drop_caches')
except Exception:
lptlog.warning("非root用户, 清除缓冲失败...")
def check_root(self):
if os.getuid() <>0:
lptlog.warning("请使用root用户执行测试用例")
sys.exit()
def clean_buffer():
try:
utils.system('echo "3" > /proc/sys/vm/drop_caches')
except Exception:
lptlog.warning("非root用户, 清除缓冲失败...")
def parser_opts(self, argv=sys.argv):
opts = self.parser(argv)
#检查是否root用户
# self.check_root()
#设定log格式
if opts.verbose:
lptlog.update_logger(quiet=opts.verbose)
#
if opts.loglevel:
lptlog.update_logger(level=opts.loglevel)
if opts.parameter:
if os.path.isfile(opts.parameter):
parameterConfig = opts.parameter
else:
parameterConfig = defaultParameter
TOOLS_LIST = readconfig.para_conf(config=parameterConfig).get_sections()
#获取工具列表
if opts.tools_list:
#self.tools_list.extend(filter(lambda x: TOOLS_LIST.count(x)>0, opts.tools_list))
self.tools_list.extend([ tool for tool in opts.tools_list if tool in TOOLS_LIST])
if opts.tools_group:
#self.tools_list.extend(filter(lambda x: TOOLS_LIST.count(x)>0, opts.tools_group.split(',')))
self.tools_list.extend([ tool for tool in opts.tools_group.split(',') if tool in TOOLS_LIST])
if self.tools_list:
self.tools_list = {}.fromkeys(self.tools_list).keys()
#lptlog.info("工具集合: %s" % ",".join(self.tools_list ))
#jobs_xml
self.jobs_xml = os.path.realpath(opts.jobs_xml)
try:
#创建job，可以指定jobs_xml,指定测试工具，指定job名称
if opts.mode == 0:
if opts.JobName:
self.jobName = opts.JobName
jobs.add_job(self.tools_list, jobs_xml=self.jobs_xml, parameter=parameterConfig, job_attrib={'name':'%s' % self.jobName})
#定义执行任务的方法
elif opts.mode == 1:
#开始测试
lptlog.info(START_MSG)
if not os.path.abspath(self.jobs_xml):
#lptlog.warning("缺失jobs文件，请核对jobs文件或者重新创建job")
raise optparse.OptionValueError, "缺失jobs文件，请核对jobs文件或者重新创建job"
else:
Jobrun.run(job_id=opts.job_id, tools_list=self.tools_list,
jobs_xml=self.jobs_xml, format='txt', clean=opts.clean)
lptlog.info(STOP_MSG)
elif opts.mode == 2:
if not opts.resultdb or not os.path.abspath(opts.resultdb):
#lptlog.warning("请指定result databases(xml)")
raise optparse.OptionValueError, "请指定result databases(xml)"
if not os.path.isdir(opts.reportdir):
lptlog.warning("%s 不是有效的目录" % opts.reportdir)
os.makedirs(opts.reportdir, mode=777)
Testreport.report(os.path.realpath(opts.resultdb), opts.reportdir, tools_list=self.tools_list,
reportname=opts.reportname, format=opts.format, chart=opts.genchart)
elif opts.mode == 3:
if opts.compareFiles:
xmls_list = [ os.path.realpath(str(xml)) for xml in opts.compareFiles.split(',') if os.path.abspath(xml)]
else:
xmls_list = []
if opts.compareNames:
names_list = [ str(name).strip() for name in opts.compareNames.split(',')]
else:
names_list = []
compare.compare(xmls_list, resultDir=opts.reportdir ,reportFile=opts.reportname, names=names_list,
tools=self.tools_list, reportType=opts.format, chart=opts.genchart)
else:
if opts.jobs_list:
#检查是否存在jobs xml文件
if not os.path.isfile(self.jobs_xml):
#lptlog.warning("缺失jobs文件，请核对jobs文件或者重新创建job")
#raise NameError("")
raise ValueError, "缺失jobs文件，请核对jobs文件或者重新创建job"
else:
self.list_jobs(self.jobs_xml)
#else:
# lptlog.warning("-h or --help show help message...")
except KeyboardInterrupt:
lptlog.warning("按下CTRL+C,将停止测试程序")
sys.exit()
except optparse.OptionValueError, e:
lptlog.error("Bad option or value: %s" % e)
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
speed_mount_status=False
args = ["-i", "0", "-i" ,"1", "-i", "2"]
self.testmode = self.get_config_value(tool_node, "testmode", os.path.join(self.tmp_dir, "speed"), valueType=str)
if self.testmode not in ("speed", "throughput"):
self.testmode = "speed"
lptlog.info("测试模式: %s" % self.testmode)
testdir = self.get_config_testdir(tool_node)
#获取设备,并尝试挂载到testdir,返回True或者False
devices_status = self.get_config_devices(tool_node, testdir)
filesize = self.get_config_value(tool_node, "filesize", "10g")
if not utils.check_size_format(filesize):
lptlog.warning("%s 格式 error,将采用默认大小10g" % filesize)
filesize = "10g"
lptlog.info("测试文件大小： %s" % filesize)
args.append("-s")
args.append(filesize)
blocksize = self.get_config_value(tool_node, "blocksize", "4k")
if not utils.check_size_format(blocksize, match="[\d]+k?"):
lptlog.warning("blocksize=%s 格式 error,将采用默认大小4k" % blocksize)
lptlog.info("测试块大小: %s" % blocksize)
args.append("-r")
args.append(blocksize)
self.times = self.get_config_value(tool_node, "times", 5, valueType=int)
direct_status = self.get_config_value(tool_node, "directio", "False", valueType=str)
if direct_status == "True":
args.append("-I")
lptlog.info("DirectIO: True")
self.parallel_type = self.get_config_value(tool_node, "parallel_type", "process", valueType=str)
if self.parallel_type not in ("threads", "process"):
self.parallel_type = "process"
lptlog.info("测试并行方式: %s" % self.parallel_type)
self.parallels = self.get_config_array(tool_node, "parallel", [4])
lptlog.info("测试并行: %s" % utils.list_to_str(self.parallels))
if self.testmode == "speed":
self.parallels = [1]
#运行测试程序，要求保存结果到tmp目录，result_file命令为iozone_$parallel_type_$iter.out
self.mainParameters["parameters"] = " ".join(["iozone"]+args)
lptlog.info("----------运行测试脚本")
for parallel in self.parallels:
parallel_args = []
lptlog.info("%s 并行测试" % parallel)
if self.testmode == 'throughput':
parallel_args.append("-t")
parallel_args.append(str(parallel))
parallel_args.append("-F")
for num in range(parallel):
parallel_args.append(os.path.join(testdir, "iozone_%s_%s_%s") % (self.testmode, parallel, num+1))
if self.parallel_type == "threads":
parallel_args.append("-T")
else:
# if devices_status:
# parallel_args.append("-U")
# parallel_args.append(testdir)
parallel_args.append("-f")
parallel_args.append("%s/iozone_test_file" % testdir)
for iter in range(self.times):
lptlog.info("第 %d 次测试" % (iter+1))
iozone_iter_result_file = os.path.join(self.tmp_dir, "iozone_%s_%d_%d.out" % (self.testmode, parallel, (iter+1)) )
#清除缓冲
method.clean_buffer()
utils.run_shell2(self.processBin, args_list=args+parallel_args, file=iozone_iter_result_file)
lptlog.info("%s %s方式, %s并行, 第%d次测试数据保存在 %s 中" % (self.tool, self.testmode, parallel, (iter+1), iozone_iter_result_file))
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------获取测试参数")
cmd = self.processBin
args = [
'-B', 'fileio', '-c',
os.path.join(self.tmp_dir, 'loadfiles/client.txt'), '-R',
'999999.99'
]
#获取测试目录
testdir = self.get_config_testdir(tool_node)
args.append("-D")
args.append(testdir)
#获取设备,并尝试挂载到testdir
devices_status = self.get_config_devices(tool_node, testdir)
self.times = self.get_config_value(tool_node,
"times",
5,
valueType=int)
lptlog.info("测试次数: %d " % self.times)
self.parallel_type = self.get_config_value(tool_node,
"parallel_type",
"process",
valueType=str)
lptlog.info("测试并行方式: %s" % self.parallel_type)
self.parallels = self.get_config_array(tool_node, "parallel", [4])
lptlog.info("测试并行: %s" % utils.list_to_str(self.parallels))
if self.parallel_type not in ("threads", "process"):
self.parallel_type = "process"
runtime = self.get_config_value(tool_node,
'runtime',
300,
valueType=int)
lptlog.info("测试时长: %s s" % runtime)
args.append("-t")
args.append("%d" % runtime)
warnuptime = self.get_config_value(tool_node,
'warnuptime',
120,
valueType=int)
if warnuptime > runtime:
lptlog.warning("warnuptime 大于 runtime, warnuptime = runtime/5")
warnuptime = runtime / 5
lptlog.info("预热时长: %s s" % warnuptime)
args.append("--warmup=%d" % warnuptime)
self.mainParameters["parameters"] = " ".join(["dbench"] + args)
lptlog.info("----------运行测试脚本")
for parallel in self.parallels:
lptlog.info("运行 %s 并行" % parallel)
parallel_args = []
if self.parallel_type == "threads":
parallel_args.append("--clients-per-process=%d" % parallel)
parallel_args.append("1")
else:
parallel_args.append(str(parallel))
for iter in range(self.times):
lptlog.info("第 %s 次测试" % (iter + 1))
tmp_file = os.path.join(
self.tmp_dir,
"%s_%s_%s.out" % (self.tool, parallel, iter + 1))
#清除buffer
method.clean_buffer()
utils.run_shell2(self.processBin,
args_list=args + parallel_args,
file=tmp_file)
lptlog.info("%d 并行 第 %d 次测试，测试数据保存在: %s " %
(parallel, iter + 1, tmp_file))
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------获取测试参数")
cmd = self.processBin
args = ['-B', 'fileio', '-c', os.path.join(self.tmp_dir, 'loadfiles/client.txt'), '-R', '999999.99']
#获取测试目录
testdir = self.get_config_testdir(tool_node)
args.append("-D")
args.append(testdir)
#获取设备,并尝试挂载到testdir
devices_status = self.get_config_devices(tool_node, testdir)
self.times = self.get_config_value(tool_node, "times", 5, valueType=int)
lptlog.info("测试次数: %d " % self.times)
self.parallel_type = self.get_config_value(tool_node, "parallel_type", "process", valueType=str)
lptlog.info("测试并行方式: %s" % self.parallel_type)
self.parallels = self.get_config_array(tool_node, "parallel", [4])
lptlog.info("测试并行: %s" % utils.list_to_str(self.parallels))
if self.parallel_type not in ("threads", "process"):
self.parallel_type = "process"
runtime = self.get_config_value(tool_node, 'runtime', 300, valueType=int)
lptlog.info("测试时长: %s s" % runtime)
args.append("-t")
args.append("%d" % runtime)
warnuptime = self.get_config_value(tool_node, 'warnuptime', 120, valueType=int)
if warnuptime > runtime:
lptlog.warning("warnuptime 大于 runtime, warnuptime = runtime/5")
warnuptime = runtime/5
lptlog.info("预热时长: %s s" % warnuptime)
args.append("--warmup=%d" % warnuptime)
self.mainParameters["parameters"] = " ".join(["dbench"]+args)
lptlog.info("----------运行测试脚本")
for parallel in self.parallels:
lptlog.info("运行 %s 并行" % parallel)
parallel_args = []
if self.parallel_type == "threads":
parallel_args.append("--clients-per-process=%d" % parallel)
parallel_args.append("1")
else:
parallel_args.append(str(parallel))
for iter in range(self.times):
lptlog.info("第 %s 次测试" % (iter+1))
tmp_file = os.path.join(self.tmp_dir, "%s_%s_%s.out" % (self.tool, parallel, iter+1))
#清除buffer
method.clean_buffer()
utils.run_shell2(self.processBin, args_list=args+parallel_args, file=tmp_file)
lptlog.info("%d 并行 第 %d 次测试，测试数据保存在: %s " % (parallel, iter+1, tmp_file))
def run_once(self, dir=None, args=None):
"""
Runs IOzone with appropriate parameters, record raw results in a per
iteration raw output file as well as in the results attribute
:param dir: IOzone file generation dir.
:param args: Arguments to the iozone program.
"""
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
speed_mount_status=False
args = ["-i", "0", "-i" ,"1", "-i", "2"]
self.testmode = self.get_config_value(tool_node, "testmode", os.path.join(self.lpttmpdir, "speed"), valueType=str)
if self.testmode not in ("speed", "throughput"):
self.testmode = "speed"
lptlog.info("测试模式: %s" % self.testmode)
testdir = self.get_config_testdir(tool_node)
#获取设备,并尝试挂载到testdir,返回True或者False
devices_status = self.get_config_devices(tool_node, testdir)
filesize = self.get_config_value(tool_node, "filesize", "10g")
if not lutils.check_size_format(filesize):
lptlog.warning("%s 格式 error,将采用默认大小10g" % filesize)
filesize = "10g"
lptlog.info("测试文件大小： %s" % filesize)
args.append("-s")
args.append(filesize)
blocksize = self.get_config_value(tool_node, "blocksize", "4k")
if not lutils.check_size_format(blocksize, match="[\d]+k?"):
lptlog.warning("blocksize=%s 格式 error,将采用默认大小4k" % blocksize)
lptlog.info("测试块大小: %s" % blocksize)
args.append("-r")
args.append(blocksize)
self.times = self.get_config_value(tool_node, "times", 5, valueType=int)
direct_status = self.get_config_value(tool_node, "directio", "False", valueType=str)
if direct_status == "True":
args.append("-I")
lptlog.info("DirectIO: True")
self.parallel_type = self.get_config_value(tool_node, "parallel_type", "process", valueType=str)
if self.parallel_type not in ("threads", "process"):
self.parallel_type = "process"
lptlog.info("测试并行方式: %s" % self.parallel_type)
self.parallels = self.get_config_array(tool_node, "parallel", [4])
lptlog.info("测试并行: %s" % lutils.list_to_str(self.parallels))
if self.testmode == "speed":
self.parallels = [1]
#运行测试程序，要求保存结果到tmp目录，result_file命令为iozone_$parallel_type_$iter.out
self.mainParameters["parameters"] = " ".join(["iozone"]+args)
lptlog.info("----------运行测试脚本")
cmd = os.path.join(self.srcdir, 'src', 'current', 'iozone')
for parallel in self.parallels:
parallel_args = []
lptlog.info("%s 并行测试" % parallel)
if self.testmode == 'throughput':
parallel_args.append("-t")
parallel_args.append(str(parallel))
parallel_args.append("-F")
for num in range(parallel):
parallel_args.append(os.path.join(testdir, "iozone_%s_%s_%s") % (self.testmode, parallel, num+1))
if self.parallel_type == "threads":
parallel_args.append("-T")
else:
# if devices_status:
# parallel_args.append("-U")
# parallel_args.append(testdir)
parallel_args.append("-f")
parallel_args.append("%s/iozone_test_file" % testdir)
for iter in range(self.times):
lptlog.info("第 %d 次测试" % (iter+1))
iozone_iter_result_file = os.path.join(self.resultsdir, "iozone_%s_%d_%d.out" % (self.testmode, parallel, (iter+1)) )
#清除缓冲
method.clean_buffer()
lutils.run_shell2(cmd, args_list=args+parallel_args, file=iozone_iter_result_file)
lptlog.info("%s %s方式, %s并行, 第%d次测试数据保存在 %s 中" % (self.tool, self.testmode, parallel, (iter+1), iozone_iter_result_file))
self.create_result()
#save to result.xml
self.save_results_to_xml()
#create txt report
self.txt_report()
def check_root(self):
if os.getuid() <> 0:
lptlog.warning("请使用root用户执行测试用例")
sys.exit()
def parser_opts(self, argv=sys.argv):
opts = self.parser(argv)
#检查是否root用户
# self.check_root()
#设定log格式
if opts.verbose:
lptlog.update_logger(quiet=opts.verbose)
#
if opts.loglevel:
lptlog.update_logger(level=opts.loglevel)
if opts.parameter:
if os.path.isfile(opts.parameter):
parameterConfig = opts.parameter
else:
parameterConfig = defaultParameter
TOOLS_LIST = readconfig.para_conf(
config=parameterConfig).get_sections()
#获取工具列表
if opts.tools_list:
#self.tools_list.extend(filter(lambda x: TOOLS_LIST.count(x)>0, opts.tools_list))
self.tools_list.extend(
[tool for tool in opts.tools_list if tool in TOOLS_LIST])
if opts.tools_group:
#self.tools_list.extend(filter(lambda x: TOOLS_LIST.count(x)>0, opts.tools_group.split(',')))
self.tools_list.extend([
tool for tool in opts.tools_group.split(',')
if tool in TOOLS_LIST
])
if self.tools_list:
self.tools_list = {}.fromkeys(self.tools_list).keys()
#lptlog.info("工具集合: %s" % ",".join(self.tools_list ))
#jobs_xml
self.jobs_xml = os.path.realpath(opts.jobs_xml)
try:
#创建job，可以指定jobs_xml,指定测试工具，指定job名称
if opts.mode == 0:
if opts.JobName:
self.jobName = opts.JobName
jobs.add_job(self.tools_list,
jobs_xml=self.jobs_xml,
parameter=parameterConfig,
job_attrib={'name': '%s' % self.jobName})
#定义执行任务的方法
elif opts.mode == 1:
#开始测试
lptlog.info(START_MSG)
if not os.path.abspath(self.jobs_xml):
#lptlog.warning("缺失jobs文件，请核对jobs文件或者重新创建job")
raise optparse.OptionValueError, "缺失jobs文件，请核对jobs文件或者重新创建job"
else:
Jobrun.run(job_id=opts.job_id,
tools_list=self.tools_list,
jobs_xml=self.jobs_xml,
format='txt',
clean=opts.clean)
lptlog.info(STOP_MSG)
elif opts.mode == 2:
if not opts.resultdb or not os.path.abspath(opts.resultdb):
#lptlog.warning("请指定result databases(xml)")
raise optparse.OptionValueError, "请指定result databases(xml)"
if not os.path.isdir(opts.reportdir):
lptlog.warning("%s 不是有效的目录" % opts.reportdir)
os.makedirs(opts.reportdir, mode=0777)
Testreport.report(os.path.realpath(opts.resultdb),
opts.reportdir,
tools_list=self.tools_list,
reportname=opts.reportname,
format=opts.format,
chart=opts.genchart)
elif opts.mode == 3:
if opts.compareFiles:
xmls_list = [
os.path.realpath(str(xml))
for xml in opts.compareFiles.split(',')
if os.path.abspath(xml)
]
else:
xmls_list = []
if opts.compareNames:
names_list = [
str(name).strip()
for name in opts.compareNames.split(',')
]
else:
names_list = []
compare.compare(xmls_list,
resultDir=opts.reportdir,
reportFile=opts.reportname,
names=names_list,
tools=self.tools_list,
reportType=opts.format,
chart=opts.genchart)
else:
if opts.jobs_list:
#检查是否存在jobs xml文件
if not os.path.isfile(self.jobs_xml):
#lptlog.warning("缺失jobs文件，请核对jobs文件或者重新创建job")
#raise NameError("")
#raise ValueError, "缺失jobs文件，请核对jobs文件或者重新创建job"
pass
else:
self.list_jobs(self.jobs_xml)
#else:
# lptlog.warning("-h or --help show help message...")
except KeyboardInterrupt:
lptlog.warning("按下CTRL+C,将停止测试程序")
sys.exit()
except optparse.OptionValueError, e:
lptlog.error("Bad option or value: %s" % e)
def run(self):
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
cmd = self.processBin
args = ['-u', os.getuid(), '-m', 'lpt', '-q']
#获取测试目录
testdir = self.get_config_value(tool_node, "testdir", os.path.join(self.tmp_dir, "testdir"), valueType=str)
if os.path.exists(testdir):
if not os.path.isdir(testdir):
lptlog.warning("%s 不是有效目录，将采用 /home/%u/testdir 目录" % testdir)
testdir = "/home/%s/testdir" % getpass.getuser()
os.makedirs(testdir, stat.S_IRWXU)
else:
os.makedirs(testdir, stat.S_IRWXU)
testdir = os.path.abspath(testdir)
args.append("-d")
args.append(testdir)
lptlog.info("测试目录: %s" % testdir)
#获取设备
devices = self.get_config_value(tool_node, "devices", "Nodevice", valueType=str)
lptlog.info("测试分区: %s " % devices)
if not os.path.exists(devices):
lptlog.debug("%s 不存在" % devices)
else:
try:
if not os.path.ismount(testdir):
util.system("mount %s %s" % (devices, testdir))
else:
lptlog.debug("%s 已经挂载到 %s 目录" % (devices, testdir))
except Exception:
lptlog.warning("mount %s %s 失败，请确认分区是否已经格式化！！" % (devices, testdir))
#获取测试内存大小
memory = self.get_config_value(tool_node, "memory", sysinfo.get_memory_size(), valueType=str)
if not utils.check_size_format(memory, match="\d+[kKmMgGtT]?"):
lptlog.warning("测试内存配置error, 将采用系统内存")
memory = sysinfo.get_memory_size()
lptlog.debug("系统内存大小：%s" % memory)
#获取测试文件大小
filesize = self.get_config_value(tool_node, "filesize", "10g", valueType=str)
if not utils.check_size_format(filesize, match="\d+[kKmMgGtT]?"):
lptlog.warning("%s 格式 error,将采用默认大小10g" % filesize)
filesize = "10g"
if float(utils.custom_format(memory)) * 2 > float(utils.custom_format(filesize)):
lptlog.warning("测试需求:测试内存*2 小于 文件大小，但实际相反, 将降低测试内存大小为测试文件的1/2" )
memory = float(utils.custom_format(filesize))/2
memory = utils.custom_format(memory, auto=True)
lptlog.info("测试内存大小：%s" % memory)
lptlog.info("测试文件大小： %s" % filesize)
args.append("-r")
args.append(memory)
#获取block大小
blocksize = self.get_config_value(tool_node, "blocksize", "4k", valueType=str)
if not utils.check_size_format(blocksize, match="\d+k?"):
lptlog.warning("blocksize=%s 格式 error,将采用默认大小8k" % blocksize)
blocksize = "8k"
args.append("-s")
args.append("%s:%s" %(filesize, blocksize))
lptlog.info("测试块大小： %s" % blocksize)
small_files_num = self.get_config_value(tool_node, "small_files_num", 0, valueType=int)
small_file_size = self.get_config_value(tool_node, "small_file_size", "1k", valueType=str)
if not small_file_size in ("1k", "2k", "4k", "8k", "16k", "32k", "64k", "128k", "256k"):
lptlog.warning("small_file_size=%s 格式error,请输入整型数字, 将采用默认值1k" % small_file_size)
else:
small_file_size = "1k"
small_files_dirs = self.get_config_value(tool_node, "small_files_dirs", 0, valueType=int)
if small_files_num == "0":
args.append("-n")
args.append("0")
else:
args.append("-n")
args.append("%s:%s:%s:%d" %(small_files_num, small_file_size, small_file_size, small_files_dirs))
lptlog.info("小文件数: %s k, 小文件大小: %s, 测试目录: %s" % (small_files_num, small_file_size, small_files_dirs))
self.times = self.get_config_value(tool_node, "times", 5, valueType=int)
lptlog.info("测试次数: %d " % self.times)
args.append("-x")
args.append("%d" % self.times)
no_buffer = self.get_config_value(tool_node, "no_buffer", "False", valueType=str)
if no_buffer == "True":
lptlog.info("no_buffer=True")
args.append("-b")
direct_io = self.get_config_value(tool_node, "direct_io", "False")
if direct_io == "True":
args.append("-D")
lptlog.info("direct_io=True")
#运行测试程序，要求保存结果到tmp目录，result_file命令为iozone_$parallel_type_$iter.out
self.mainParameters["parameters"] = utils.list_to_str(["bonnie++"]+args, ops=" ")
#清除缓冲
method.clean_buffer()
lptlog.info("----------运行测试脚本")
utils.run_shell2(self.processBin, args_list=args, file=os.path.join(self.tmp_dir, "bonnie.out"))
lptlog.info("%s 测试数据保存在 %s 中" % (self.tool, os.path.join(self.tmp_dir, "bonnie.out")))
def run_once(self, dir=None, args=None):
"""
Runs IOzone with appropriate parameters, record raw results in a per
iteration raw output file as well as in the results attribute
:param dir: IOzone file generation dir.
:param args: Arguments to the iozone program.
"""
tool_node = self.check_tool_result_node()
lptlog.info("----------开始获取测试参数")
speed_mount_status = False
args = ["-i", "0", "-i", "1", "-i", "2"]
self.testmode = self.get_config_value(tool_node,
"testmode",
os.path.join(
self.lpttmpdir, "speed"),
valueType=str)
if self.testmode not in ("speed", "throughput"):
self.testmode = "speed"
lptlog.info("测试模式: %s" % self.testmode)
testdir = self.get_config_testdir(tool_node)
#获取设备,并尝试挂载到testdir,返回True或者False
devices_status = self.get_config_devices(tool_node, testdir)
filesize = self.get_config_value(tool_node, "filesize", "10g")
if not lutils.check_size_format(filesize):
lptlog.warning("%s 格式 error,将采用默认大小10g" % filesize)
filesize = "10g"
lptlog.info("测试文件大小： %s" % filesize)
args.append("-s")
args.append(filesize)
blocksize = self.get_config_value(tool_node, "blocksize", "4k")
if not lutils.check_size_format(blocksize, match="[\d]+k?"):
lptlog.warning("blocksize=%s 格式 error,将采用默认大小4k" % blocksize)
lptlog.info("测试块大小: %s" % blocksize)
args.append("-r")
args.append(blocksize)
self.times = self.get_config_value(tool_node,
"times",
5,
valueType=int)
direct_status = self.get_config_value(tool_node,
"directio",
"False",
valueType=str)
if direct_status == "True":
args.append("-I")
lptlog.info("DirectIO: True")
self.parallel_type = self.get_config_value(tool_node,
"parallel_type",
"process",
valueType=str)
if self.parallel_type not in ("threads", "process"):
self.parallel_type = "process"
lptlog.info("测试并行方式: %s" % self.parallel_type)
self.parallels = self.get_config_array(tool_node, "parallel", [4])
lptlog.info("测试并行: %s" % lutils.list_to_str(self.parallels))
if self.testmode == "speed":
self.parallels = [1]
#运行测试程序，要求保存结果到tmp目录，result_file命令为iozone_$parallel_type_$iter.out
self.mainParameters["parameters"] = " ".join(["iozone"] + args)
lptlog.info("----------运行测试脚本")
cmd = os.path.join(self.srcdir, 'src', 'current', 'iozone')
for parallel in self.parallels:
parallel_args = []
lptlog.info("%s 并行测试" % parallel)
if self.testmode == 'throughput':
parallel_args.append("-t")
parallel_args.append(str(parallel))
parallel_args.append("-F")
for num in range(parallel):
parallel_args.append(
os.path.join(testdir, "iozone_%s_%s_%s") %
(self.testmode, parallel, num + 1))
if self.parallel_type == "threads":
parallel_args.append("-T")
else:
# if devices_status:
# parallel_args.append("-U")
# parallel_args.append(testdir)
parallel_args.append("-f")
parallel_args.append("%s/iozone_test_file" % testdir)
for iter in range(self.times):
lptlog.info("第 %d 次测试" % (iter + 1))
iozone_iter_result_file = os.path.join(
self.resultsdir,
"iozone_%s_%d_%d.out" % (self.testmode, parallel,
(iter + 1)))
#清除缓冲
method.clean_buffer()
lutils.run_shell2(cmd,
args_list=args + parallel_args,
file=iozone_iter_result_file)
lptlog.info("%s %s方式, %s并行, 第%d次测试数据保存在 %s 中" %
(self.tool, self.testmode, parallel,
(iter + 1), iozone_iter_result_file))
self.create_result()
#save to result.xml
self.save_results_to_xml()
#create txt report
self.txt_report()
