def handle_log_record(self, level, msg, record, attachments):
'''处理一个日志记录
:param level: 日志级别，参考EnumLogLevel
:type level: string
:param msg: 日志消息
:type msg: string
:param record: 日志记录
:type record: dict
:param attachments: 附件
:type attachments: dict
'''
self._write("%s: %s\n" % (levelname[level], msg))
if level == EnumLogLevel.ASSERT:
if "actual" in record:
actual = record["actual"]
self._write(" 实际值：%s%s\n" % (actual.__class__, actual))
if "expect" in record:
expect = record["expect"]
self._write(" 期望值：%s%s\n" % (expect.__class__, expect))
if "code_location" in record:
self._write(
smart_text(' File "%s", line %s, in %s\n' %
record["code_location"]))
if "traceback" in record:
self._write(smart_text_by_lines("%s\n" % record["traceback"]))
for name in attachments:
file_path = smart_text(attachments[name])
if path_exists(file_path):
file_path = os.path.abspath(file_path)
self._write(" %s:%s\n" % (smart_text(name), file_path))
def abs_path(self, relative_path):
"""get resource absolute path
: type relative_path:string
:param relative_path: ，资源文件相对描述符，相对于setting下的资源目录的路径,支持多级目录
:return:返回资源文件的绝对路径
"""
relative_path = self._adjust_path(relative_path)
if relative_path.startswith(os.sep):
relative_path = relative_path[1:]
found_paths = []
for it in self._resources_dirs:
file_path = self._adjust_path(os.path.join(it, relative_path))
file_path = smart_text(file_path)
file_link = smart_text(file_path + '.link')
if os.path.exists(file_path):
found_paths.append(file_path)
elif os.path.exists(file_link):
with codecs_open(file_link, encoding="utf-8") as f:
remote_path = f.read()
file_path = self._resolve_link_file(remote_path, file_path)
found_paths.append(file_path)
if len(found_paths) == 0:
raise Exception("relative_path=%s not found" % relative_path)
if len(found_paths) > 1:
raise Exception("relative_path=%s got multiple results:\n%s" %
(relative_path, "\n".join(found_paths)))
return os.path.abspath(file_path)
def handle_test_begin(self, testcase):
'''处理一个测试用例执行的开始
:param testcase: 测试用例
:type testcase: TestCase
'''
self._xmldoc.appendChild(
self._xmldoc.createProcessingInstruction(
"xml-stylesheet", 'type="text/xsl" href="TestResult.xsl"'))
owner = getattr(testcase, 'owner', None)
priority = getattr(testcase, 'priority', None)
timeout = getattr(testcase, 'timeout', None)
self._testnode = self._xmldoc.createElement('TEST')
self._testnode.setAttribute(
"name", smart_text(saxutils.escape(testcase.test_name)))
self._testnode.setAttribute("owner",
smart_text(saxutils.escape(str(owner))))
self._testnode.setAttribute("priority", str(priority))
self._testnode.setAttribute("timeout", str(timeout))
self._testnode.setAttribute(
'begintime',
time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(self.begin_time)))
self._xmldoc.appendChild(self._testnode)
self.begin_step('测试用例初始步骤')
def get_file(self, relative_path):
"""查找某个文件
:type relative_path:string
:param relative_path: ，资源文件相对描述符，相对于setting下的资源目录的路径,支持多级目录
:return:返回资源文件的绝对路径
"""
result = []
relative_path = self._adjust_path(relative_path)
if relative_path.startswith(os.sep):
relative_path = relative_path[1:]
for it in self._resources_dirs:
file_path = self._adjust_path(os.path.join(it, relative_path))
file_path = smart_text(file_path)
file_link = smart_text(file_path + '.link')
if os.path.isfile(file_path):
result.append(file_path)
elif os.path.isfile(file_link):
with codecs_open(file_link, encoding="utf-8") as f:
remote_path = f.read()
file_path = self._resolve_link_file(remote_path, file_path)
result.append(file_path)
if len(result) > 1:
raise Exception("存在多个%s文件" % relative_path)
elif len(result) < 1:
raise Exception("%s文件不存在" % relative_path)
return result[0]
def _log_assert_failed(self, message, back_count=2):
"""记录断言失败的信息
"""
stack = get_last_frame_stack(back_count)
msg = "检查点不通过\n%s%s\n" % (smart_text(stack), smart_text(message))
self.__testresult.log_record(EnumLogLevel.ASSERT, msg)
if not settings.get("QTAF_ASSERT_CONTINUE", True):
raise RuntimeError("testcase assert failed:%s" % message)
def test_class_name(self):
'''返回测试用例名字（不同测试用例的名字不同）
:rtype: str
'''
cls = type(self)
if cls.__module__ == '__main__':
type_name = smart_text(cls.__name__)
else:
type_name = smart_text(cls.__module__ + '.' + cls.__name__)
return type_name
def log_filtered_test(self, loader, testcase, reason):
'''记录一个被过滤的测试用例
:param loader: 用例加载器
:type loader: TestLoader
:param testcase: 测试用例
:type testcase: TestCase
:param reason: 过滤原因
:type reason: str
'''
nodestr = """<FilterTest name="%s" reason="%s"></FilterTest>
""" % (smart_text(saxutils.escape(
testcase.test_name)), smart_text(saxutils.escape(reason)))
doc2 = dom.parseString(nodestr)
filterNode = doc2.childNodes[0]
self._runrstnode.appendChild(filterNode)
def _get_translated_in_datadrive(name, dd):
if isinstance(name, six.string_types):
name = smart_text(name)
else:
name = str(name)
translated_name = translate_bad_char(name)
for item in dd:
if isinstance(item, six.string_types):
item_string = smart_text(item)
else:
item_string = str(item)
if translated_name == translate_bad_char(item_string):
return dd[item]
else:
raise ValueError("找不到对应名字'%s'的数据驱动用例" % name)
def log_test_result(self, testcase, testresult):
'''记录一个测试结果
:param testcase: 测试用例
:type testcase: TestCase
:param testresult: 测试结果
:type testresult: XmlResult
'''
super(XMLTestReport, self).log_test_result(testcase, testresult)
casemark = saxutils.escape(testcase.test_doc)
nodestr = """<TestResult result="%s" log="%s" status="%s">%s</TestResult>
""" % (testresult.passed, testresult.file_path, testcase.status, casemark)
doc2 = dom.parseString(smart_binary(nodestr))
resultNode = doc2.childNodes[0]
resultNode.setAttribute("name", smart_text(saxutils.escape(testcase.test_name)))
resultNode.setAttribute("owner", smart_text(saxutils.escape(testcase.owner)))
self._runrstnode.appendChild(resultNode)
def translate_bad_char4exclude_keys(data_key):
if isinstance(data_key, six.string_types):
data_key = smart_text(data_key)
else:
data_key = str(data_key)
data_key = translate_bad_char(data_key)
return data_key
def __record_assert_failed(self, message, actual, expect):
'''记录Assert失败信息
:param message: 提示信息
:type message: string
:param actual: 实际值
:type actual: string
:param expect: 期望值
:type expect: string
'''
# 得到上一个函数调用帧所在的文件路径，行号，函数名
stack = get_last_frame_stack(3)
msg = "检查点不通过\n%s%s\n期望值：%s%s\n实际值：%s%s" % (
smart_text(stack), smart_text(message), expect.__class__, expect,
actual.__class__, actual)
self.__testresult.log_record(EnumLogLevel.ASSERT, msg)
def list_dir(self, relative_path):
"""列出某个目录下的文件
:type relative_path:string
:param relative_path: ，资源文件目录相对路径，相对于setting下的资源目录的路径,支持多级目录
:return:返回一个包含资源目录下所有文件或者文件下的绝对路径列表
"""
result = []
relative_path = self._adjust_path(relative_path)
if relative_path.startswith(os.sep):
relative_path = relative_path[1:]
for it in self._resources_dirs:
dir_path = self._adjust_path(os.path.join(it, relative_path))
dir_path = smart_text(dir_path)
if os.path.isdir(dir_path):
result.append(dir_path)
if len(result) > 1:
logger.error("找到多个目录:")
for item in result:
logger.error("%s" % item)
raise Exception("存在多个%s目录" % relative_path)
if len(result) < 1:
raise Exception("%s目录不存在" % relative_path)
paths = []
for path in os.listdir(result[0]):
paths.append(os.path.join(result[0], path))
return paths
def smart_text_by_lines(s):
'''将任意字符串转换为UTF-8编码
'''
lines = []
for line in s.split('\n'):
lines.append(smart_text(line))
return '\n'.join(lines)
def _load_from_class(self,
cls,
data_key=None,
exclude_data_key=None,
attrs=None):
'''加载用例类
:param cls: Python类
:type cls: Type
:returns list - 测试用例对象列表
'''
if exclude_data_key is None:
exclude_data_key = []
exclude_data_key = [smart_text(i) for i in exclude_data_key]
tests = []
if datadrive.is_datadrive(cls) or settings.DATA_DRIVE:
tests = datadrive.load_datadrive_tests(cls, data_key, attrs)
else:
tests = [cls(attrs=attrs)]
if self._filter:
final_tests = []
for it in tests:
filter_reason = self._filter(it)
if filter_reason:
self._filtered_tests[it] = filter_reason
else:
final_tests.append(it)
tests = final_tests
return tests
def test_xml_report(self):
test_pairs = [("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),]
old_cwd = os.getcwd()
for test_name, reason in test_pairs:
try:
test_report = report.report_types["xml"]()
test_runner = runner.runner_types["basic"](test_report)
test_name = "test.sampletest.hellotest.%s" % test_name
working_dir = test_name + "_" + get_time_str()
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
print("xml report test for test: %s, wokring dir=%s" % (test_name, working_dir))
test_runner.run(test_name)
report_path = os.path.join(os.getcwd(), "TestReport.xml")
xml_report = minidom.parse(report_path)
test_results = xml_report.getElementsByTagName("TestResult")
self.assertEqual(len(test_results), 1)
attrs = test_results[0].attributes
self.assertEqual(attrs["name"].value, test_name)
self.assertEqual(attrs["result"].value, "False")
result_path = os.path.join(os.getcwd(), attrs["log"].value)
result_xml = minidom.parse(result_path)
error_nodes = result_xml.getElementsByTagName("ERROR")
self.assertEqual(len(error_nodes), 1)
failed_reason = smart_text(error_nodes[0].childNodes[0].nodeValue)
self.assertRegexpMatches(failed_reason, reason)
finally:
os.chdir(old_cwd)
def assert_equal(self, message, actual, expect=True):
'''检查实际值和期望值是否相等，不能则测试用例失败
:param message: 检查信息
:param actual: 实际值
:param expect: 期望值(默认：True)
:return: True or False
'''
if isinstance(actual, six.string_types):
actual = smart_text(actual)
if isinstance(expect, six.string_types):
expect = smart_text(expect)
if expect != actual:
self.__record_assert_failed(message, actual, expect)
return False
else:
return True
def load(self, testname):
'''通过名字加载测试用例
:param name: 用例或用例名称
:type name: string/list
:returns list - 测试用例对象列表
'''
if isinstance(testname, list):
testnames = testname
else:
testnames = [testname]
self._module_errs = {}
testcases = []
for testname in testnames:
testname = smart_text(testname)
if settings.DATA_DRIVE:
self._dataset = TestDataLoader().load()
if '/' in testname:
testname, datakeyname = testname.split('/', 1)
else:
datakeyname = None
obj = self._load(testname)
if isinstance(obj, types.ModuleType):
if hasattr(obj, '__path__'):
testcases += self._load_from_package(obj)
else:
testcases += self._load_from_module(obj)
elif isinstance(obj, type):
testcases += self._load_from_class(obj)
#过滤掉重复的用例
testcase_dict = {}
for testcase in testcases:
if datakeyname and smart_text(
testcase.casedataname) != datakeyname:
continue
testcase_dict[testcase.test_name] = testcase
return list(testcase_dict.values())
def test_json_report(self):
def _clean_json_report(json_report_file, results=None):
if results is None:
with codecs_open(json_report_file, "r", encoding="utf-8") as fd:
content = fd.read()
results = json.loads(content)["results"]
json_files = results[:]
json_files.append(json_report_file)
for json_file in json_files:
os.remove(json_file)
test_pairs = [("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),]
for test_name, reason in test_pairs:
time_str = get_time_str()
test_report_name = "%s_%s.json" % (time_str, test_name)
with codecs_open(test_report_name, "w", encoding="utf-8") as fd:
test_report = report.report_types["json"](fd=fd)
test_runner = runner.runner_types["basic"](test_report)
test_name = "test.sampletest.hellotest.%s" % test_name
print("json report test for test: " + test_name)
test_runner.run(test_name)
with codecs_open(test_report_name, "r", encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
self.assertEqual(report_json["loaded_testcases"][0]["name"], test_name)
test_results = report_json["results"]
self.addCleanup(_clean_json_report, test_report_name, test_results)
self.assertEqual(len(test_results), 1)
with codecs_open(test_results[0], "r", encoding="utf-8") as fd2:
content = fd2.read()
result_json = json.loads(content)
self.assertEqual(result_json["succeed"], False)
failed_step = result_json["steps"][-1]
self.assertEqual(failed_step["succeed"], False)
actual_reson = smart_text(failed_step["logs"][0]["message"])
self.assertRegexpMatches(actual_reson, reason)
test_name = "test.sampletest.hellotest.HelloTest test.sampletest.hellotest.TimeoutTest"
time_str = get_time_str()
test_report_name = "test_json_report_%s.json" % time_str
retry_count = 2
with codecs_open(test_report_name, "w", encoding="utf-8") as fd:
test_report = report.report_types["json"](fd=fd, title="test_json_report")
test_runner = runner.runner_types["multithread"](test_report, retries=retry_count)
test_runner.run(test_name)
with codecs_open(test_report_name, "r", encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
summary = report_json["summary"]
self.assertEqual(summary["testcase_total_run"], (retry_count + 1) * 2)
self.assertEqual(summary["testcase_total_count"], 2)
self.assertTrue("hostname" in summary)
self.assertTrue("os" in summary)
self.addCleanup(_clean_json_report, test_report_name)
def __record_assert_failed(self, message, actual, expect):
'''记录Assert失败信息
:param message: 提示信息
:type message: string
:param actual: 实际值
:type actual: string
:param expect: 期望值
:type expect: string
'''
# 得到上一个函数调用帧所在的文件路径，行号，函数名
stack = get_last_frame_stack(3)
msg = "检查点不通过\n%s%s\n期望值：%s%s\n实际值：%s%s" % (
smart_text(stack), smart_text(message), expect.__class__, expect,
actual.__class__, actual)
self.__testresult.log_record(EnumLogLevel.ASSERT, msg)
if not settings.get("QTAF_ASSERT_CONTINUE", True):
raise RuntimeError("testcase assert failed:%s" % message)
def load_datadrive_tests(cls, name=None, exclude_data_key=None, attrs=None):
'''加载对应数据驱动测试用例类的数据驱动用例
'''
if is_datadrive(cls):
dd = get_datadrive(cls)
else:
if not settings.DATA_DRIVE:
raise RuntimeError("DATA_DRIVE is not set to True")
from testbase.loader import TestDataLoader
dd = TestDataLoader().load()
if name:
if name in dd:
drive_data = {name : dd[name]}
else:
drive_value = _get_translated_in_datadrive(name, dd)
drive_data = {name : drive_value}
else:
drive_data = dd
if exclude_data_key is None:
exclude_data_key = []
exclude_data_key = [_translate_bad_char4exclude_keys(item) for item in exclude_data_key]
tests = []
for item in drive_data:
# 如果有排除标签
exclude_item = _translate_bad_char4exclude_keys(item)
if exclude_data_key is not None and exclude_item in exclude_data_key:
continue
testdata = drive_data[item]
if isinstance(item, six.string_types):
item = smart_text(item)
else:
item = str(item)
casedata_name = item
if has_bad_char(item):
casedata_name = translate_bad_char(item)
warn_msg = "[WARNING]%r's drive data key should use \"%s\" instead of \"%s\"" % (cls, casedata_name, item)
logger.warn(warn_msg)
if isinstance(testdata, dict) and "__attrs__" in testdata:
new_attrs = testdata.get("__attrs__")
else:
new_attrs = None
if attrs:
if not new_attrs:
new_attrs = attrs
else:
new_attrs.update(attrs)
tests.append(cls(testdata, casedata_name, new_attrs))
return tests
def test_name(self):
'''返回测试用例实例的名字
:rtype: str
'''
if self.casedataname is not None:
casedataname = smart_text(self.casedataname)
return '%s/%s' % (self.test_class_name, casedataname)
else:
return self.test_class_name
def get_debugger(self, url=None, title=None):
'''获取Web调试器
'''
if url is None and title is None:
raise ValueError('url或title参数不能同时为空')
page_list = self.get_page_list()
ws_addr = None
if len(page_list) == 1:
ws_addr = page_list[0]['websocketurl']
else:
for page in page_list:
if (url and smart_text(page['url']) == smart_text(url) ) or (title and smart_text(page['title']) == smart_text(title) ):
ws_addr = page['websocketurl']
break
else:
raise RuntimeError('page %s not found' % (url or title))
if not ws_addr in self.__class__.inst_dict:
self.__class__.inst_dict[ws_addr] = WebkitDebugger(ws_addr)
return self.__class__.inst_dict[ws_addr]
def test_get_local_file(self):
fm = resource.TestResourceManager(
resource.LocalResourceManagerBackend()).create_session()
self.assertEqual(self.local_file, fm.get_file(test_file_name))
self.assertEqual(self.local_file, resource.get_file(test_file_name))
paths = []
for it in os.listdir(self.local_dir):
paths.append(smart_text(os.path.join(self.local_dir, it)))
self.assertEqual(paths, fm.list_dir(''))
self.assertEqual(paths, resource.list_dir(''))
def assert_match(self, message, actual, expect):
'''检查actual和expect是否模式匹配，不匹配则记录一个检查失败
:type message: string
:param message: 失败时记录的消息
:type actual: string
:param actual: 需要匹配的字符串
:type expect: string
:param expect: 要匹配的正则表达式
:return: 匹配成果
'''
if isinstance(actual, six.string_types):
actual = smart_text(actual)
if isinstance(expect, six.string_types):
expect = smart_text(expect)
if re.search(expect, actual):
return True
else:
self.__record_assert_failed(message, actual, expect)
return False
def load(self, testname):
'''通过名字加载测试用例
:param name: 用例或用例名称
:type name: string/list
:returns list - 测试用例对象列表
'''
if isinstance(testname, list):
testnames = testname
else:
testnames = [testname]
self._module_errs = {}
testcases = []
for testname in testnames:
parameters = {}
exclude_data_keys = None
if isinstance(testname, dict):
parameters = testname.get("parameters", {})
exclude_data_keys = testname.get("exclude_data_keys")
testname = testname.get("name")
testname = smart_text(testname)
if settings.DATA_DRIVE:
self._dataset = TestDataLoader().load()
if '/' in testname:
testname, data_key = testname.split('/', 1)
else:
data_key = None
obj = self._load(testname)
if isinstance(obj, types.ModuleType):
if hasattr(obj, '__path__'):
testcases += self._load_from_package(
obj, data_key, exclude_data_keys, parameters)
else:
testcases += self._load_from_module(
obj, data_key, exclude_data_keys, parameters)
elif isinstance(obj, type):
testcases += self._load_from_class(obj, data_key,
exclude_data_keys,
parameters)
# 过滤掉重复的用例
testcase_dict = {}
for testcase in testcases:
testcase_dict[testcase.test_name] = testcase
return list(testcase_dict.values())
def handle_test_end(self, passed):
'''处理一个测试用例执行的结束
:param passed: 测试用例是否通过
:type passed: boolean
'''
self._testnode.setAttribute('result', str(passed))
self._testnode.setAttribute('endtime', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(self.end_time)))
self._testnode.setAttribute('duration', "%02d:%02d:%02.2f\n" % _convert_timelength(self.end_time - self.begin_time))
if self._file_path:
with codecs_open(smart_text(self._file_path), 'wb') as fd:
fd.write(to_pretty_xml(self._xmldoc))
def handle_step_begin(self, msg):
'''处理一个测试步骤的开始
:param msg: 测试步骤名称
:type msg: string
'''
if not isinstance(msg, six.string_types):
raise ValueError("msg='%r'必须是string类型" % msg)
self._stepnode = self._xmldoc.createElement("STEP")
self._stepnode.setAttribute('title', smart_text(msg))
self._stepnode.setAttribute('time', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
self._testnode.appendChild(self._stepnode)
def loads(self, data, confs, serializer=None):
'''load from binary
'''
if confs.has_key('charset'):
try:
data = data.decode(confs['charset']).encode('utf8')
except UnicodeDecodeError:
data = smart_text(data)
root = html.fromstring(data)
else:
root = html.fromstring(data)
if root.tag == 'html':
self._data = data
self._root = root
def test_html_report_content(self):
test_pairs = [
("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),
]
old_cwd = os.getcwd()
for test_name, reason in test_pairs:
try:
working_dir = test_name + "_" + get_time_str()
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report = report_types["html"](title="test html report")
test_runner = runner_types["basic"](test_report)
test_name = "tests.sampletest.hellotest.%s" % test_name
print("html report test for test: " + test_name)
test_runner.run(test_name)
html_report_file = os.path.join(os.getcwd(), "qta-report.js")
with codecs_open(html_report_file, encoding="utf-8") as fd:
content = fd.read()
index = content.find("{")
qta_report_data = content[index:]
qta_report = json.loads(qta_report_data)
failed_test_names = list(qta_report["failed_tests"].keys())
self.assertEqual(failed_test_names[0], test_name)
failed_tests = qta_report["failed_tests"]
self.assertEqual(len(failed_tests), 1)
with codecs_open(failed_tests[test_name]["records"][0],
"r",
encoding="utf-8") as fd2:
content = fd2.read()
index = content.find("{")
result_json_data = content[index:]
result_json = json.loads(result_json_data)
self.assertEqual(result_json["succeed"], False)
failed_step = result_json["steps"][-1]
self.assertEqual(failed_step["succeed"], False)
actual_reson = smart_text(
failed_step["logs"][0]["message"])
self.assertRegexpMatches(actual_reson, reason)
finally:
os.chdir(old_cwd)
def log_record(self, level, msg, record=None, attachments=None):
'''处理一个日志记录
:param level: 日志级别，参考EnumLogLevel
:type level: string
:param msg: 日志消息
:type msg: string
:param record: 日志记录
:type record: dict
:param attachments: 附件
:type attachments: dict
'''
if record is None:
record = {}
if attachments is None:
attachments = {}
if not isinstance(msg, six.string_types):
raise ValueError("msg='%r'必须是string类型" % msg)
msg = smart_text(msg)
if level >= EnumLogLevel.ERROR:
self.__steps_passed[self.__curr_step] = False
if level > self.__error_level:
self.__error_level = level
extra_record, extra_attachments = self._get_extra_fail_record_safe(
)
record.update(extra_record)
attachments.update(extra_attachments)
if self.__failed_priority <= 3 and level == EnumLogLevel.APPCRASH:
self.__failed_info, self.__failed_priority = "Crash", 3
if self.__failed_priority <= 2 and level == EnumLogLevel.TESTTIMEOUT:
self.__failed_info, self.__failed_priority = "用例执行超时", 2
if self.__failed_priority <= 1 and level == EnumLogLevel.ASSERT:
self.__failed_info, self.__failed_priority = msg, 1
if self.__failed_priority <= 1 and "traceback" in record:
if not self.__failed_info: # 优先记录第一个异常，第一个异常往往比较大可能是问题的原因
self.__failed_info, self.__failed_priority = record[
"traceback"].split('\n')[-2], 1
with self.__lock:
if not self.__accept_result:
return
self.handle_log_record(level, msg, record, attachments)
