def parse_validator(validator):
""" parse validator, validator maybe in two format
@param (dict) validator
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
@return (dict) validator info
{
"check": "status_code",
"expect": 201,
"comparator": "eq"
}
"""
if not isinstance(validator, dict):
raise exceptions.ParamsError("invalid validator: {}".format(validator))
if "check" in validator and len(validator) > 1:
# format1
check_item = validator.get("check")
if "expect" in validator:
expect_value = validator.get("expect")
elif "expected" in validator:
expect_value = validator.get("expected")
else:
raise exceptions.ParamsError(
"invalid validator: {}".format(validator))
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
# 有断言备注的情况下，舍弃备注信息
if isinstance(compare_values, list) and len(compare_values) > 2:
compare_values.pop()
if not isinstance(compare_values, list) or len(compare_values) != 2:
raise exceptions.ParamsError(
"invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
return {
"check": check_item,
"expect": expect_value,
"comparator": comparator
}
def __getattr__(self, key):
try:
if key == "json":
value = self.resp_obj.json()
else:
value = getattr(self.resp_obj, key)
self.__dict__[key] = value
return value
except AttributeError:
err_msg = "ResponseObject does not have attribute: {}".format(key)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
def extract_field(self, field, key=None, context_obj=None):
""" extract value from requests.Response.
"""
if not isinstance(field, basestring):
err_msg = u"Invalid extractor! => {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
# msg = "extract: {}".format(field)
if key:
msg = "【提取变量】: {}".format(field)
else:
msg = "【提取】: {}".format(field)
sub_str_exp = None
original_value = None
# if text_extractor_regexp_compile.match(field):
# value = self._extract_field_with_regex(field)
# else:
# original_value, sub_str_exp = self._extract_field_with_delimiter(field, context_obj=context_obj)
try:
original_value, sub_str_exp = self._extract_field_with_delimiter(
field, context_obj=context_obj)
except Exception as err:
raise exceptions.ExtractFailure(err)
if sub_str_exp:
value = eval('original_value' + sub_str_exp)
else:
value = original_value
if is_py2 and isinstance(value, unicode):
value = value.encode("utf-8")
if key:
if sub_str_exp:
msg += " ==> {0} ==> {1} 保存为变量 {2}".format(
original_value + sub_str_exp, value, key)
else:
msg += " ==> {0} 保存为变量 {1}".format(value, key)
else:
msg += " ==> {0}".format(value)
logger.log_info(msg)
return value
def override_mapping_list(variables, new_mapping):
""" override variables with new mapping.
Args:
variables (list): variables list
[
{"var_a": 1},
{"var_b": "world"}
]
new_mapping (dict): overrided variables mapping
{
"var_a": "hello"
}
Returns:
OrderedDict: overrided variables mapping.
Examples:
>>> variables = [
{"var_a": 1},
{"var_b": "world"}
]
>>> new_mapping = {
"var_a": "hello"
}
>>> override_mapping_list(variables, new_mapping)
OrderedDict(
{
"var_a": "hello",
"var_b": "world"
}
)
"""
if isinstance(variables, list):
variables_ordered_dict = convert_mappinglist_to_orderdict(variables)
elif isinstance(variables, (OrderedDict, dict)):
variables_ordered_dict = variables
else:
raise exceptions.ParamsError("variables error!")
return update_ordered_dict(variables_ordered_dict, new_mapping)
def _get_block_by_name(ref_call, ref_type):
""" get test content by reference name.
Args:
ref_call (str): call function.
e.g. api_v1_Account_Login_POST($UserName, $Password)
ref_type (enum): "def-api" or "def-testcase"
Returns:
dict: api/testcase definition.
Raises:
exceptions.ParamsError: call args number is not equal to defined args number.
"""
function_meta = parser.parse_function(ref_call)
func_name = function_meta["func_name"]
call_args = function_meta["args"]
block = _get_test_definition(func_name, ref_type)
def_args = block.get("function_meta", {}).get("args", [])
if len(call_args) != len(def_args):
err_msg = "{}: call args number is not equal to defined args number!\n".format(
func_name)
err_msg += "defined args: {}\n".format(def_args)
err_msg += "reference args: {}".format(call_args)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
args_mapping = {}
for index, item in enumerate(def_args):
if call_args[index] == item:
continue
args_mapping[item] = call_args[index]
if args_mapping:
block = parser.substitute_variables(block, args_mapping)
return block
def parse_parameters(parameters, variables_mapping, functions_mapping):
""" parse parameters and generate cartesian product.
Args:
parameters (list) parameters: parameter name and value in list
parameter value may be in three types:
(1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"]
(2) call built-in parameterize function, "${parameterize(account.csv)}"
(3) call custom function in debugtalk.py, "${gen_app_version()}"
variables_mapping (dict): variables mapping loaded from debugtalk.py
functions_mapping (dict): functions mapping loaded from debugtalk.py
Returns:
list: cartesian product list
Examples:
>>> parameters = [
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": "******"},
{"app_version": "${gen_app_version()}"}
]
>>> parse_parameters(parameters)
"""
parsed_parameters_list = []
for parameter in parameters:
parameter_name, parameter_content = list(parameter.items())[0]
parameter_name_list = parameter_name.split("-")
if isinstance(parameter_content, list):
# (1) data list
# e.g. {"app_version": ["2.8.5", "2.8.6"]}
# => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
# e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
# => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
parameter_content_list = []
for parameter_item in parameter_content:
if not isinstance(parameter_item, (list, tuple)):
# "2.8.5" => ["2.8.5"]
parameter_item = [parameter_item]
# ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
# ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"}
parameter_content_dict = dict(
zip(parameter_name_list, parameter_item))
parameter_content_list.append(parameter_content_dict)
else:
# (2) & (3)
parsed_parameter_content = parse_data(parameter_content,
variables_mapping,
functions_mapping)
# e.g. [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# e.g. [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
if not isinstance(parsed_parameter_content, list):
raise exceptions.ParamsError("parameters syntax error!")
parameter_content_list = [
# get subset by parameter name
{key: parameter_item[key]
for key in parameter_name_list}
for parameter_item in parsed_parameter_content
]
parsed_parameters_list.append(parameter_content_list)
return utils.gen_cartesian_product(*parsed_parameters_list)
def run_test(self, teststep_dict):
""" run single teststep.
Args:
teststep_dict (dict): teststep info
{
"name": "teststep description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
Raises:
exceptions.ParamsError
exceptions.ValidationFailure
exceptions.ExtractFailure
"""
self.teststep_dict = teststep_dict
is_last = teststep_dict.get("is_last", None)
case_name = teststep_dict.get("name", None)
case_id = teststep_dict.get("case_id", None)
logger.log_info("【开始执行用例】: ID_{0}, {1}".format(case_id, case_name))
self.step_teardown_executed = False
# check skip
self._handle_skip_feature(teststep_dict)
# prepare
logger.log_info("-" * 12 + "【变量替换-开始】" + "-" * 12)
extractors = teststep_dict.get("extract", []) or teststep_dict.get(
"extractors", [])
validators = teststep_dict.get("validate", []) or teststep_dict.get(
"validators", [])
self.step_parse_variable_pass = True
self.running_hook = 'step_parse_variable'
parsed_request = self.init_config(teststep_dict, level="teststep")
self.context.update_teststep_variables_mapping("request",
parsed_request)
logger.log_info("-" * 12 + "【变量替换-结束】" + "-" * 12)
if self.variable_not_found:
self.handle_teardown(fail_type='变量替换')
raise exceptions.VariableNotFound
if not self.step_parse_variable_pass:
self.handle_teardown(fail_type='变量替换')
raise exceptions.CustomFuncRunError
# setup hooks
setup_hooks = teststep_dict.get("setup_hooks", [])
setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}")
logger.log_info("-" * 12 + "【请求前置-开始】" + "-" * 12)
self.step_setup_pass = True
self.running_hook = 'step_setup'
self.do_setup_hook_actions(setup_hooks)
logger.log_info("-" * 12 + "【请求前置-结束】" + "-" * 12)
if not self.step_setup_pass:
self.handle_teardown(fail_type='前置动作')
raise exceptions.SetupHooksFailure
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = [
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"
]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format(
"/".join(valid_methods))
logger.log_error(err_msg)
self.handle_teardown(fail_type='校验发送方式')
raise exceptions.ParamsError(err_msg)
# logger.log_info("{method} {url}".format(method=method, url=url))
# logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
# request
try:
resp = self.http_client_session.request(method,
url,
name=group_name,
**parsed_request)
except Exception as e:
self.handle_teardown(fail_type='接口请求')
raise exceptions.RequestFailure
resp_obj = response.ResponseObject(resp)
# # teardown hooks
# teardown_hooks = teststep_dict.get("teardown_hooks", [])
# if teardown_hooks:
# # logger.log_info("start to run teardown hooks")
# logger.log_info("【开始后置动作】...")
# self.context.update_teststep_variables_mapping("response", resp_obj)
# self.do_hook_actions(teardown_hooks)
# logger.log_info("【结束后置动作】")
# request teardown hooks 新增请求后置
request_teardown_hooks = teststep_dict.get("request_teardown_hooks",
[])
if request_teardown_hooks:
# logger.log_info("start to run teardown hooks")
logger.log_info("-" * 12 + "【请求后置-开始】" + "-" * 12)
self.step_request_teardown_pass = True
self.running_hook = 'step_request_teardown'
self.context.update_teststep_variables_mapping(
"response", resp_obj)
self.do_request_teardown_hook_actions(request_teardown_hooks)
logger.log_info("-" * 12 + "【请求后置-结束】" + "-" * 12)
if not self.step_request_teardown_pass:
self.handle_teardown(fail_type='请求后置动作')
raise exceptions.TeardownHooksFailure
# extract
logger.log_info("-" * 12 + "【提取变量-开始】" + "-" * 12)
try:
extracted_variables_mapping = resp_obj.extract_response(
extractors, self.context)
self.context.update_testcase_runtime_variables_mapping(
extracted_variables_mapping)
except Exception as err:
logger.log_error('提取变量失败：{0}'.format(err.args[0]))
self.handle_teardown(fail_type='提取变量')
raise exceptions.ExtractFailure
logger.log_info("-" * 12 + "【提取变量-结束】" + "-" * 12)
# validate
try:
logger.log_info("-" * 12 + "【结果校验-开始】" + "-" * 12)
self.evaluated_validators, validate_pass = self.context.validate(
validators, resp_obj)
logger.log_info("-" * 12 + "【结果校验-结束】" + "-" * 12)
if not validate_pass:
# self.handle_teardown(fail_type='结果校验')
raise exceptions.ValidationFailure
except (exceptions.ParamsError, exceptions.ValidationFailure,
exceptions.ExtractFailure, exceptions.VariableNotFound) as err:
# log request
# err_req_msg = "request: \n"
# err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {}))
# for k, v in parsed_request.items():
# err_req_msg += "{}: {}\n".format(k, repr(v))
# logger.log_error(err_req_msg)
#
# # log response
# err_resp_msg = "response: \n"
# err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
# err_resp_msg += "headers: {}\n".format(resp_obj.headers)
# err_resp_msg += "body: {}\n".format(repr(resp_obj.text))
# logger.log_error(err_resp_msg)
logger.log_error('结果校验失败')
self.handle_teardown(fail_type='结果校验')
raise exceptions.ValidationFailure
# teardown hooks
teardown_hooks = teststep_dict.get("teardown_hooks", [])
self.step_teardown_executed = True
if teardown_hooks:
# logger.log_info("start to run teardown hooks")
logger.log_info("-" * 12 + "【用例后置-开始】" + "-" * 12)
self.step_teardown_pass = True
self.running_hook = 'step_teardown'
self.context.update_teststep_variables_mapping(
"response", resp_obj)
self.do_teardown_hook_actions(teardown_hooks)
logger.log_info("-" * 12 + "【用例后置-结束】" + "-" * 12)
if not self.step_teardown_pass:
self.handle_teardown(fail_type='后置动作')
raise exceptions.TeardownHooksFailure
# total teardown hooks
if is_last:
if self.testcase_teardown_hooks and not self.testcase_teardown_hooks_executed:
logger.log_info("-" * 12 + "【全局后置-开始】" + "-" * 12)
self.testcase_teardown_hooks_executed = True
self.do_teardown_hook_actions(self.testcase_teardown_hooks)
logger.log_info("-" * 12 + "【全局后置-结束】" + "-" * 12)
logger.log_info("【结束执行用例】: ID_{0}, {1}".format(case_id, case_name))
def _extract_field_with_delimiter(self, field, context_obj=None):
""" response content could be json or html text.
@param (str) field should be string joined by delimiter.
e.g.
"status_code"
"headers"
"cookies"
"content"
"headers.content-type"
"content.person.name.first_name"
含用例内变量
"123$phoneNo"
查询SQL
"SELECT NEXT_VALUE FROM user_db.sequence WHERE SEQ_NAME='$MEMBER_ID';"
"""
# [:]
sub_str_exp = None
if field.endswith(']') and '[' in field and ':' in field.split(
'[')[-1]:
sub_str_exp = '[' + field.split('[')[-1]
field = field.strip(sub_str_exp)
# 支持提取变量步骤中写查询sql，查询结果保存为变量
if str(field).lower().startswith("select "):
db_connect_content = '$DB_CONNECT'
parsed_db_connect = context_obj.eval_content(db_connect_content)
if parser.extract_variables(field):
sql = context_obj.eval_content(field)
else:
sql = field
from atp.api.mysql_sql_executor import sql_execute, db_operation_to_json
from atp.utils.tools import convert_mysql_datatype_to_py
try:
res = sql_execute(sql, db_connect=parsed_db_connect)
except Exception as err:
raise
if res:
# 支持查询结果是多条数据的情况
if len(res) == 1:
if len(res[0]) == 1:
res_value = convert_mysql_datatype_to_py(res[0][0])
else:
res_value = db_operation_to_json(
sql, db_connect=parsed_db_connect, return_info=res)
else:
res_value = []
for res_item in res:
if len(res_item) == 1:
res_value.append(
convert_mysql_datatype_to_py(res_item[0]))
else:
res_value.append(
db_operation_to_json(
sql,
db_connect=parsed_db_connect,
return_info=res_item,
multi=True))
else:
res_value = 'variable sql return no result!'
# res_value = res[0][0] if res else "DB query returns EMPTY result!"
# if isinstance(res_value, decimal.Decimal):
# res_value = float(res_value)
return res_value, sub_str_exp
# string.split(sep=None, maxsplit=-1) -> list of strings
# e.g. "content.person.name" => ["content", "person.name"]
try:
top_query, sub_query = field.split('.', 1)
except ValueError:
top_query = field
sub_query = None
# status_code
if top_query in ["status_code", "encoding", "ok", "reason", "url"]:
if sub_query:
# status_code.XX
err_msg = u"Failed to extract: {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
return getattr(self, top_query), sub_str_exp
# cookies
elif top_query == "cookies":
cookies = self.cookies.get_dict()
if not sub_query:
# extract cookies
return cookies, sub_str_exp
try:
return cookies[sub_query], sub_str_exp
except KeyError:
err_msg = u"Failed to extract cookie! => {}\n".format(field)
err_msg += u"response cookies: {}\n".format(cookies)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# elapsed
elif top_query == "elapsed":
available_attributes = u"available attributes: days, seconds, microseconds, total_seconds"
if not sub_query:
err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n"
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
elif sub_query in ["days", "seconds", "microseconds"]:
return getattr(self.elapsed, sub_query), sub_str_exp
elif sub_query == "total_seconds":
return self.elapsed.total_seconds(), sub_str_exp
else:
err_msg = "{} is not valid datetime.timedelta attribute.\n".format(
sub_query)
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
# headers
elif top_query == "headers":
headers = self.headers
if not sub_query:
# extract headers
return headers, sub_str_exp
try:
return headers[sub_query], sub_str_exp
except KeyError:
err_msg = u"Failed to extract header! => {}\n".format(field)
err_msg += u"response headers: {}\n".format(headers)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# response body
elif top_query in ["content", "text", "json"]:
try:
body = self.json
except exceptions.JSONDecodeError:
body = self.text
if not sub_query:
# extract response body
return body, sub_str_exp
if isinstance(body, dict):
# content = {"xxx": 123}, content.xxx
'''如果body中content是字符串类型'content': "{'headImageUrl':'','isRegister':0,'nickName':''}"
转换成字典，然后'extract': [{'headImageUrl':"content.content.isRegister"}]可提取
'''
# if "content" in body.keys() and '{' in body["content"]:
# 修复bug：如果body["content"]是NoneType，报错“TypeError: argument of type 'NoneType' is not iterable ”
# if "content" in body.keys() and body["content"] and '{' in body["content"]:
# body_content_dict=json.loads(body["content"].replace("'", "\""))
# body["content"]=body_content_dict
# 修复bug："[]"未被json.loads
if "content" in body.keys() and body["content"] and isinstance(
body["content"], str):
try:
body_content_dict = json.loads(body["content"].replace(
' style="text-align: center;text-indent: 0;"',
'').replace("'", "\""))
body["content"] = body_content_dict
except (TypeError, json.decoder.JSONDecodeError) as e:
# logger.log_error(body["content"].replace("'", "\""))
logger.log_error('\n'.join([e,
traceback.format_exc()]))
return utils.query_json(body, sub_query), sub_str_exp
elif sub_query.isdigit():
# content = "abcdefg", content.3 => d
return utils.query_json(body, sub_query), sub_str_exp
else:
# content = "<html>abcdefg</html>", content.xxx
err_msg = u"Failed to extract attribute from response body! => {}\n".format(
field)
err_msg += u"response body: {}\n".format(body)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# new set response attributes in teardown_hooks
elif top_query in self.__dict__:
attributes = self.__dict__[top_query]
if not sub_query:
# extract response attributes
return attributes, sub_str_exp
if isinstance(attributes, (dict, list)):
# attributes = {"xxx": 123}, content.xxx
return utils.query_json(attributes, sub_query), sub_str_exp
elif sub_query.isdigit():
# attributes = "abcdefg", attributes.3 => d
return utils.query_json(attributes, sub_query), sub_str_exp
else:
# content = "attributes.new_attribute_not_exist"
err_msg = u"Failed to extract cumstom set attribute from teardown hooks! => {}\n".format(
field)
err_msg += u"response set attributes: {}\n".format(attributes)
logger.log_error(err_msg)
raise exceptions.TeardownHooksFailure(err_msg)
elif context_obj and parser.extract_variables(top_query):
# 表达式带已知变量，保存为新的变量
# ha$phone => ha18551602992
return context_obj.eval_content(top_query), sub_str_exp
# others
else:
err_msg = u"Failed to extract attribute from response! => {}\n".format(
field)
err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, text, json, encoding, ok, reason, url.\n\n"
err_msg += u"If you want to set attribute in teardown_hooks, take the following example as reference:\n"
err_msg += u"response.new_attribute = 'new_attribute_value'\n"
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
def _do_validation(self, validator_dict):
""" validate with functions
Args:
validator_dict (dict): validator dict
{
"check": "status_code",
"check_value": 200,
"expect": 201,
"comparator": "eq"
}
"""
# TODO: move comparator uniform to init_test_suites
comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = self.TESTCASE_SHARED_FUNCTIONS_MAPPING.get(comparator)
if not validate_func:
raise exceptions.FunctionNotFound(
"comparator not found: {}".format(comparator))
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in [
"is", "eq", "equals", "==", "json_contains", "json_same", "field_special_check", "db_validate",
"db_validate_cycle", "field_check_empty_list", "field_check_not_empty_list",
"field_check_not_in_list", "field_check_empty_json", "field_check_not_empty_json", "redis_validate", "mq_validate"]:
raise exceptions.ParamsError(
"Null value can only be compared with comparator: eq/equals/=="
)
# validate_msg = "validate: {} {} {}({})".format(
validate_msg = "【验证点】: 校验方法：{}, 待校验内容：{}, 期望结果：{}({})".format(
comparator, check_item, expect_value,
type(expect_value).__name__)
try:
validator_dict["check_result"] = "pass"
is_ok = validate_func(check_value, expect_value)
if is_ok is True:
validate_msg += "\t ....................PASS"
logger.log_info(validate_msg)
else:
validate_msg += "\t ....................FAIL"
if is_ok is not False:
validate_msg += "......原因: {}".format(is_ok[1])
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)
except (AssertionError, TypeError) as err:
validate_msg += "\t ....................FAIL"
validate_msg += "\t{}({}), {}, {}({})".format(
check_value,
type(check_value).__name__, comparator, expect_value,
type(expect_value).__name__)
validate_msg += "\t......原因: {}".format(err.args[0])
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)