def __get_real_value(self, variable):
_rs = re.compile(r"(\${([^{}]+)})") # to match ${VariableA}
_rf = re.compile(r"(\${{([^{}]+)}})") # to match ${{Function(A)}}
while re.search(_rs, format(variable)):
for _pair in re.findall(_rs, format(variable)):
_pair = list(_pair)
if _pair[1] in self.variables:
_pair[1] = self.variables[_pair[1]]
else:
logger.warning("Undefined variable: {}, set to ''".format(
_pair[1]))
_pair[1] = ''
if variable == _pair[0]:
variable = _pair[1]
break
variable = variable.replace(_pair[0], str(_pair[1]))
while re.search(_rf, format(variable)):
for _pair in re.findall(_rf, format(variable)):
_pair = list(_pair)
try:
_pair[1] = eval(_pair[1])
except SyntaxError or Exception as e:
logger.warning(
"Invalid function: {}: {}, set to ''".format(
_pair[0], e))
_pair[1] = ''
if variable == _pair[0]:
variable = _pair[1]
break
variable = variable.replace(_pair[0], str(_pair[1]))
return variable
def __har_times(entry, step_dict):
# startedDateTime:
# Chrome: 2019-07-24T03:42:07.867Z
# Fiddler: 2019-07-24T18:52:38.4367088+08:00
# Charles: 2019-07-31T14:21:35.033+08:00
# Other: Wed, 30 Jan 2019 07:56:42
if not ('startedDateTime' in entry.keys()
and entry['startedDateTime']):
logger.warning(
" * There is no startedDateTime in this entry: {}".format(
json.dumps(entry, ensure_ascii=False)))
return False
s_time = entry['startedDateTime']
if not ('time' in entry.keys() and entry['time']):
if not ('times' in entry.keys() and entry['times']):
logger.warning(" * There is no time in this entry: {}".format(
json.dumps(entry, ensure_ascii=False)))
return False
else:
i_time = int(round(entry['times']))
else:
i_time = int(round(entry['time']))
# convert startedDateTime to timestamp
step_dict['request']['time.start'] = har_time2timestamp(
har_time=s_time, ms=1)
step_dict['response']['time.spent'] = i_time
def load_test_case(self, suite_or_case, environment=None):
"""
:param suite_or_case: file or directory of test suites/cases/steps
:param environment: environment defined in test data, None - use defined in suites/cases/steps
:return: items of test suites/cases/steps
"""
logger.info(
"Start to load test suite or case: {}".format(suite_or_case))
self.environment = environment
items = []
if os.path.isdir(suite_or_case):
files = get_dir_files(suite_or_case)
else:
files = [
suite_or_case,
]
for _file in files:
logger.info("Load case from file: {}".format(_file))
if not (_file.endswith('yml') or _file.endswith('yaml')):
logger.warning("Not a yaml file, ignore: {}".format(_file))
continue
try:
_dict = yaml.full_load(stream=self.__read_file(_file))
except ScannerError as e:
logger.warning("Invalid yaml file: {}".format(e))
continue
logger.debug(" - yaml dict: {}".format(
json.dumps(_dict, ensure_ascii=False)))
_tmp_suite = copy.deepcopy(self.suite_tpl)
_tmp_case = copy.deepcopy(self.case_tpl)
_tmp_step = copy.deepcopy(self.step_tpl)
if 'test_cases' in _dict: # it's a test suite
_tmp_suite.update(_dict)
self.__parse_test_suite(the_dict=_tmp_suite,
base_path=get_file_path(_file))
elif 'test_steps' in _dict: # it's a test case
_tmp_case.update(_dict)
self.__parse_test_case(the_dict=_tmp_case,
base_path=get_file_path(_file))
_tmp_suite['test_cases'].append(_tmp_case)
else: # it's a test step
_tmp_step.update(_dict)
self.__parse_test_step(the_dict=_tmp_step,
base_path=get_file_path(_file))
_tmp_case['test_steps'].append(_tmp_step)
_tmp_suite['test_cases'].append(_tmp_case)
logger.debug(" - one suite: {}".format(
json.dumps(_tmp_suite, ensure_ascii=False)))
items.append(_tmp_suite)
logger.info("Done.")
logger.debug("The test suites are: {}".format(
json.dumps(items, ensure_ascii=False)))
return items
def __parse_test_suite(self, the_dict, base_path):
self.__parse_environments(the_dict, base_path)
for _idx, _case in enumerate(the_dict['test_cases']):
logger.debug(" - case {} in suite: {}".format(_idx, _case))
if _case.startswith('..'):
_case = "{}/{}".format(base_path, _case)
try:
the_dict['test_cases'][_idx] = copy.deepcopy(self.case_tpl)
the_dict['test_cases'][_idx].update(
yaml.full_load(self.__read_file(_case)))
logger.debug(" - case info: {}".format(
json.dumps(the_dict['test_cases'][_idx],
ensure_ascii=False)))
except ScannerError as e:
logger.warning("Invalid yaml file: {}".format(e))
continue
self.__parse_test_case(the_dict=the_dict['test_cases'][_idx],
base_path=get_file_path(_case))
def __parse_environments(self, the_dict, base_path):
if 'import' in the_dict['config'] and the_dict['config']['import']:
if the_dict['config']['import'].startswith('..'):
the_dict['config']['import'] = "{}/{}".format(
base_path, the_dict['config']['import'])
try:
_tmp_env = yaml.full_load(
stream=self.__read_file(the_dict['config']['import']))
the_dict['config']['import'] = _tmp_env
_variables = copy.deepcopy(
_tmp_env['global']) if 'global' in _tmp_env else {}
# env priority: argument > config
_env = self.environment if self.environment else the_dict[
'config']['environment']
if _env in _tmp_env:
_variables.update(_tmp_env[_env])
_variables.update(the_dict['config']['variables'])
the_dict['config']['variables'] = _variables
except ScannerError as e:
logger.warning("Invalid yaml file: {}".format(e))
logger.debug(" - config variables: {}".format(
json.dumps(the_dict['config']['variables'], ensure_ascii=False)))
def __har_response(self,
entry,
step_dict,
include,
exclude,
auto_extract=False):
if not ('response' in entry.keys() and entry['response']):
logger.warning(" * There is no response in this entry: {}".format(
json.dumps(entry, ensure_ascii=False)))
return False
_rsp = entry['response']
# get status
step_dict['response']['status'] = _rsp.get('status', 200)
step_dict['validations'].append(
{"eq": {
'status.code': step_dict['response']['status']
}})
# get headers
step_dict['response']['headers'] = {}
self.__har_headers(_rsp.get('headers'),
step_dict['response']['headers'], VALIDATE_HEADERS)
__headers = get_all_kv_pairs(item=step_dict['response']['headers'],
prefix='headers')
_vin = get_matched_keys(key=include,
keys=list(__headers.keys()),
fuzzy=1)
_vex = get_matched_keys(key=exclude,
keys=list(__headers.keys()),
fuzzy=1) if exclude else []
for _k, _v in step_dict['response']['headers'].items():
_k = "headers.{}".format(_k)
# Extracting temporary variables for automatic identification of interface dependencies
if auto_extract and isinstance(_v,
str) and len(_v) >= IDENTIFY_LEN:
if _v not in self.variables.keys():
self.variables[_v] = {'key': _k, 'flag': 0}
step_dict['response']['extract'][_v] = _k
if _k in _vin and _k not in _vex:
step_dict['validations'].append({"eq": {_k: _v}})
logger.debug(" - self.variables: {}".format(
json.dumps(self.variables, ensure_ascii=False)))
# get cookies
step_dict['response']['cookies'] = {}
self.__har_cookies(_rsp.get('cookies'),
step_dict['response']['cookies'])
# get content
try:
_text = _rsp.get('content').get('text', '')
_mime = _rsp.get('content').get('mimeType') or ''
_code = _rsp.get('content').get('encoding')
except AttributeError:
logger.warning(" * Invalid response content: {}".format(
_rsp.get('content')))
return False
if _code and _code == 'base64':
try:
_text = base64.b64decode(_text).decode('utf-8')
except UnicodeDecodeError as e:
logger.warning(" * Decode error: {}".format(e))
elif _code:
logger.warning(" * Unsupported encoding method: {}".format(_code))
return False
logger.debug(" - mimeType: {}, encoding: {}".format(_mime, _code))
logger.debug(" - content text: {}".format(_text))
if _mime.startswith('application/json'): # json => dict
try:
step_dict['response']['content'] = json.loads(_text)
# extract all content values into validations
logger.debug(" - validation include: {}, exclude: {}".format(
include, exclude))
_pairs = get_all_kv_pairs(item=json.loads(_text),
prefix='content')
_vin = get_matched_keys(key=include,
keys=list(_pairs.keys()),
fuzzy=1)
_vex = get_matched_keys(key=exclude,
keys=list(_pairs.keys()),
fuzzy=1) if exclude else []
for _k, _v in _pairs.items():
if isinstance(_v, str):
# _v = "__break_line__".join(_v.split("\n"))
_v = _v.replace("\r\n", '__break_line__').replace(
"\n", '__break_line__')
# Extracting temporary variables for automatic identification of interface dependencies
if auto_extract and isinstance(
_v, str) and len(_v) >= IDENTIFY_LEN:
if _v not in self.variables.keys():
self.variables[_v] = {'key': _k, 'flag': 0}
step_dict['response']['extract'][_v] = _k
if _k in _vin and _k not in _vex:
step_dict['validations'].append({"eq": {_k: _v}})
except json.decoder.JSONDecodeError:
logger.warning(
" * Invalid response content in json: {}".format(_text))
# sys.exit(-1)
elif _mime.startswith('text/html'): # TODO: html => dom tree, xpath
pass
else:
logger.warning(" * Unsupported mimeType: {}".format(_mime))
# step_dict['validations'].append({"eq": {'content': _text}})
logger.debug(" - validations: {}".format(
json.dumps(step_dict['validations'], ensure_ascii=False)))
logger.debug(" - self.variables: {}".format(
json.dumps(self.variables, ensure_ascii=False)))
return True
def __har_request(self,
entry,
step_dict,
include,
exclude,
auto_extract=False):
if not ('request' in entry.keys() and entry['request']):
logger.warning(" * There is no request in this entry: {}".format(
json.dumps(entry, ensure_ascii=False)))
return False
_req = entry['request']
# get method
step_dict['request']['method'] = _req.get('method', 'GET')
# get url: protocol, host, url path
_url = _req.get('url', "")
# logger.info(" Get a {} request: {}".format(step_dict['request']['method'], _url))
try:
(_whole_url, step_dict['request']['protocol'],
step_dict['request']['host'], step_dict['request']['url'],
_) = re.findall(r"((http\w*)://([\w.:]+)([^?]+))\??(.*)", _url)[0]
step_dict['config']['name'] = step_dict['request']['url']
logger.debug(" - protocol: {} host: {} url: {}".format(
step_dict['request']['protocol'], step_dict['request']['host'],
step_dict['request']['url']))
logger.info(" Get a {} request: {}".format(
step_dict['request']['method'], step_dict['request']['url']))
except IndexError:
logger.warning(" * Invalid url: {}".format(_url))
return False
# filter with include and exclude options
logger.debug(" - include: {} exclude: {}".format(include, exclude))
if not self.__if_include(_whole_url, include) or self.__if_exclude(
_whole_url, exclude):
logger.info(" According to include/exclude options, ignore it")
return False
# get parameters
# it may have both queryString and postData in an unusual post request
step_dict['request']['params'] = {}
step_dict['request']['data'] = {}
_param = _req.get('queryString', [])
_data = _req.get('postData', [])
if _data:
if 'params' in _req.get('postData'):
_data = _req.get('postData').get('params')
else:
_data = _req.get('postData').get('text')
# if 'mimeType' in _req.get('postData') and _req.get('postData').get('mimeType') == 'application/json':
# _tmp = json.loads(_data)
# _data = []
# for _tk, _tv in _tmp.items():
# _data.append({'name': _tk, 'value': _tv})
logger.debug(" - params: {}".format(_param))
logger.debug(" - data: {}".format(_data))
# extract all parameter values into variables, and keep {value} in parameters
if isinstance(_param, (list, tuple, set)):
for _item in _param:
self.__har_extract(step_dict, _item['name'], _item['value'],
'params', auto_extract)
else:
# step_dict['request']['params'] = _param
self.__har_extract(step_dict, '', _param, 'params', auto_extract)
if isinstance(_data, (list, tuple, set)):
for _item in _data:
self.__har_extract(step_dict, _item['name'], _item['value'],
'data', auto_extract)
else:
# step_dict['request']['data'] = _data
self.__har_extract(step_dict, '', _data, 'data', auto_extract)
logger.debug(" - self.variables: {}".format(
json.dumps(self.variables, ensure_ascii=False)))
# get headers
step_dict['request']['headers'] = {}
self.__har_headers(_req.get('headers'),
step_dict['request']['headers'], RECORD_HEADERS,
auto_extract)
logger.debug(" - headers: {}".format(
json.dumps(step_dict['request']['headers'], ensure_ascii=False)))
# get cookies
step_dict['request']['cookies'] = {}
self.__har_cookies(_req.get('cookies'),
step_dict['request']['cookies'], auto_extract)
logger.debug(" - cookies: {}".format(
json.dumps(step_dict['request']['cookies'], ensure_ascii=False)))
return True
def source_to_case(self,
source,
target="ParrotProject",
include=None,
exclude=None,
validate_include=None,
validate_exclude=None,
auto_extract=False):
"""
:param source: source file or direcotry
:param target: target directory for case output
:param include: list, not matched url would be ignored in recording
:param exclude: list, matched url would be ignored in recording
:param validate_include: list, not matched response would be ignored in validating
:param validate_exclude: list, matched response would be ignored in validating
:param auto_extract: bool, for automatic identification of interface dependencies
:return suite dict
"""
source = format(source).strip()
if not (source and os.path.exists(source)):
logger.error(
"Source file or directory does not exist: {}".format(source))
sys.exit(-1)
if source.endswith("/") or source.endswith("\\"):
suite_name = get_file_name(get_file_path(source))
else:
suite_name = get_file_name(source)
if os.path.isdir(source):
files = get_dir_files(source)
else:
files = [
source,
]
suite_dict = copy.deepcopy(self.suite_tpl)
suite_dict['config']['name'] = suite_name
logger.info(
"Start to parse cases from source files: {}".format(source))
for _file in files:
if _file.lower().endswith('.har'):
one_case = self.har_to_case(_file, target, include, exclude,
validate_include, validate_exclude,
auto_extract, suite_name)
# elif _file.lower().endswith('.trace'):
# self.charles_trace_to_case()
# elif _file.lower().endswith('.txt'):
# self.fiddler_txt_to_case()
else:
logger.warning(
"Unsupported file extension: {}, ignore".format(_file))
continue
# add case into suite
suite_dict['test_cases'].append(one_case)
logger.info("Parse finished.")
self.__generate_case(suite_dict, target)
return suite_dict
def case_replace(self, suite_or_case, rules, target="ParrotProjectNew"):
"""
:param suite_or_case: file or directory of test suites/cases/steps
:param rules: replace rule list, key=>value or value1=>value2
:param target: target directory for case output
:return: suite dict
"""
logger.info(
"Start to load test suite or case: {}".format(suite_or_case))
files = []
if isinstance(suite_or_case, (list, tuple, set)):
items = suite_or_case
else:
items = [
format(suite_or_case),
]
for item in items:
if not os.path.exists(item):
logger.warning("File {} does not exist, ignore.".format(item))
continue
if os.path.isdir(item):
files.extend(get_dir_files(item))
else:
files.append(item)
total = {}
for _file in files:
logger.info("Load case from file: {}".format(_file))
if not (_file.endswith('yml') or _file.endswith('yaml')):
logger.warning("Not a yaml file, ignore: {}".format(_file))
continue
try:
_dict = yaml.full_load(stream=self.__read_file(_file))
except ScannerError as e:
logger.warning("Invalid yaml file: {}".format(e))
continue
logger.debug(" - yaml dict: {}".format(
json.dumps(_dict, ensure_ascii=False)))
if 'test_cases' in _dict: # it's a test suite
_tmp_suite = copy.deepcopy(self.suite_tpl)
_tmp_suite.update(_dict)
self.__match_rule(_yaml=_file, _dict=_tmp_suite, rules=rules)
_path = re.findall(r"(.+)test_suites(.+)",
get_file_path(_file))
if _path:
total["{}/test_suites/{}/{}".format(
target, _path[0][1],
get_file_name(_file, ext=1))] = _tmp_suite
else:
total["{}/test_suites/{}".format(
target, get_file_name(_file, ext=1))] = _tmp_suite
_cases = []
for _case in _tmp_suite['test_cases']:
_cases.append("{}/{}".format(get_file_path(_file), _case))
if _tmp_suite['config']['import']:
_cases.append("{}/{}".format(
get_file_path(_file), _tmp_suite['config']['import']))
if _cases:
self.case_replace(suite_or_case=_cases,
target=target,
rules=rules)
elif 'test_steps' in _dict: # it's a test case
_tmp_case = copy.deepcopy(self.case_tpl)
_tmp_case.update(_dict)
self.__match_rule(_yaml=_file, _dict=_tmp_case, rules=rules)
_path = re.findall(r"(.+)test_cases(.+)", get_file_path(_file))
if _path:
total["{}/test_cases/{}/{}".format(
target, _path[0][1], get_file_name(_file,
ext=1))] = _tmp_case
else:
total["{}/test_cases/{}".format(
target, get_file_name(_file, ext=1))] = _tmp_case
_steps = []
for _step in _tmp_case['test_steps']:
_steps.append("{}/{}".format(get_file_path(_file), _step))
if _tmp_case['config']['import']:
_steps.append("{}/{}".format(
get_file_path(_file), _tmp_case['config']['import']))
if _steps:
self.case_replace(suite_or_case=_steps,
target=target,
rules=rules)
elif 'request' in _dict: # it's a test step
_tmp_step = copy.deepcopy(self.step_tpl)
_tmp_step.update(_dict)
self.__match_rule(_yaml=_file, _dict=_tmp_step, rules=rules)
_path = re.findall(r"(.+)test_steps(.+)", get_file_path(_file))
if _path:
total["{}/test_steps/{}/{}".format(
target, _path[0][1], get_file_name(_file,
ext=1))] = _tmp_step
else:
total["{}/test_steps/{}".format(
target, get_file_name(_file, ext=1))] = _tmp_step
if _tmp_step['config']['import']:
self.case_replace(suite_or_case="{}/{}".format(
get_file_path(_file), _tmp_step['config']['import']),
target=target,
rules=None)
else: # it's environment file
_path = re.findall(r"(.+)environments(.+)",
get_file_path(_file))
if _path:
total["{}/environments/{}/{}".format(
target, _path[0][1], get_file_name(_file,
ext=1))] = _dict
else:
total["{}/environments/{}".format(
target, get_file_name(_file, ext=1))] = _dict
for _k, _v in total.items():
make_dir(get_file_path(_k))
with open(file=format(_k), mode='w', encoding='utf-8') as f:
yaml.dump(data=_v,
stream=f,
encoding='utf-8',
allow_unicode=True)
logger.info("Write file after replace: {}".format(_k))
logger.info("Done. You could get them in {}".format(target))