def read_yaml_test(yaml_file):
with open(yaml_file) as data_file:
data = None
try:
data = yaml.load(data_file)
except yaml.YAMLError:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold(yaml_file) + " has invalid syntax")
return data
def read_json_test(json_file):
with open(json_file) as data_file:
data = None
try:
data = json.load(data_file)
except ValueError:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold(json_file) + " has invalid syntax")
return data
def load_config(ar, config_file=None):
"""
:param ar arguments class
:param config_file name of config file
"""
data = list()
if config_file is None:
config_file = config_yaml
my_file = Path(config_file)
if my_file.is_file():
if yaml_support:
data = load_yaml(config_file)
else:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT,
"Can't use " + config_yaml)
else:
config_file = config_json
my_file = Path(config_file)
if my_file.is_file():
data = load_json(config_file)
else:
error_handler.add_suspicion(
bcolors.bold(config_yaml) + " or " +
bcolors.bold(config_json) +
" is missing or can\'t be opened.")
ar.valid = False
return
else:
my_file = Path(config_file)
if my_file.is_file():
file_type = config_file.split('.')[-1]
if file_type == 'yaml':
if not yaml_support:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT)
else:
data = load_yaml(config_file)
elif file_type == 'json':
data = load_json(config_file)
else:
error_handler.add_suspicion(
bcolors.bold(config_file) +
" have to end with .json or .yaml")
ar.valid = False
return
else:
error_handler.add_suspicion(
bcolors.bold(config_file) + " is missing or can\'t be opened.")
ar.valid = False
return
parse_config(ar, config_file, data)
def test_set_missing(test, test_directory, test_json_file):
"""
try to recover from missing values
:param test test class
:param test_directory directory with test
:param test_json_file json file
"""
real_out = ""
if test.output is not None:
output = test_directory + test.output
output_path = Path(output)
if not output_path.is_file():
error_handler.call_error(Errors.MISSING_FILE,
bcolors.bold(output) + " is missing")
with open(output) as input_file:
real_out = input_file.read()
input_data = None
if test.input is not None:
input = test_directory + test.input
input_path = Path(input)
if not input_path.is_file():
error_handler.call_error(Errors.MISSING_FILE,
bcolors.bold(input) + " is missing")
with open(input) as input_file:
input_data = bytes(input_file.read(), "UTF-8")
if test.is_not_set():
error_handler.call_warning(
bcolors.bold(test_directory) + " is not valid test directory. " +
bcolors.bold("Skipping test!"))
score['SKIP'].append(test_directory)
return None, None, False
if test.code is None:
test.code = 0
error_handler.warning("key " + bcolors.bold("returnCode") + " in " +
test_json_file +
" is missing. Using default value: " +
bcolors.bold(str(test.code)))
error_handler.call_warning()
return input_data, real_out, True
def parse_command_line_init_test(arguments, ar=Test()):
i = 0
while i < len(arguments):
arg = arguments[i]
if i == len(arguments) - 1:
error_handler.call_error(
Errors.WRONG_PARAM,
"Wrong argument or incomplete argument " + bcolors.bold(arg))
break
next_arg = arguments[i + 1]
i += 1
if arg == '-i' or arg == '--input':
ar.input = next_arg
elif arg == '-o' or arg == '--output':
ar.output = next_arg
elif arg == '-n' or arg == '--name':
ar.name = next_arg
elif arg == '-r' or arg == '--return-code':
if is_int(next_arg):
ar.code = int(next_arg)
else:
error_handler.call_error(
Errors.WRONG_PARAM,
"Incompatible value for " + bcolors.bold(arg))
elif arg == '-c' or arg == '--comment':
ar.comment = next_arg
elif arg == '-T' or arg == '--timeout':
if is_int(next_arg):
ar.timeout = int(next_arg)
else:
error_handler.call_error(
Errors.WRONG_PARAM,
"Incompatible value for " + bcolors.bold(arg))
else:
error_handler.call_error(
Errors.WRONG_PARAM,
"Wrong argument or incomplete argument " + bcolors.bold(arg))
break
i += 1
return ar
def lets_test(arguments):
arguments = argstest.parse_args_test(arguments)
if not arguments.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arguments.hlp:
argstest.print_help()
exit(0)
elif arguments.version:
print("Tester version - " + version)
exit(0)
if arguments.executable is None:
error_handler.call_error(Errors.WRONG_PARAM, "Executable is not set")
# init test variables
my_test.executable = arguments.executable
my_test.only_failed = arguments.onlyFailed
my_test.resultDir = arguments.result_dir
my_test.timeout = arguments.timeout
my_test.exclude = arguments.exclude
error_handler.no_warnings = arguments.no_warnings
my_test.no_diff = arguments.no_diff
# clearing result directory
result_name = arguments.result_dir
result_path = Path(result_name)
if result_path.exists():
if not result_path.is_dir():
error_handler.call_error(
Errors.WRONG_PARAM,
bcolors.bold(result_name) + " is not a directory")
else:
rmtree(result_name)
if arguments.clean:
print(bcolors.note(arguments.result_dir) + " removed")
exit(0)
if arguments.tests_dir is None and arguments.test is None:
error_handler.call_error(Errors.WRONG_INPUT, "No tests")
if arguments.test is not None:
my_test.run(arguments.test)
elif arguments.tests_dir is not None:
my_test.run_all(arguments.tests_dir)
my_test.show_score()
def read_test_file(test, test_file, data):
"""
read test.json
:param test test class
:param test_file json file
:param data file data
"""
for key in data.keys():
if key == 'input':
if isinstance(data[key], str):
test.input = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('input') + " in " + bcolors.bold(test_file) +
" has to be string")
elif key == 'output':
if isinstance(data[key], str):
test.output = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('output') + " in " + bcolors.bold(test_file) +
" has to be string")
elif key == 'comment':
if isinstance(data[key], str):
test.comment = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('comment') + " in " +
bcolors.bold(test_file) + " has to be string")
elif key == 'returnCode':
if isinstance(data[key], int):
test.code = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('returnCode') + " in " +
bcolors.bold(test_file) + " has to be int")
elif key == 'timeout':
if isinstance(data[key], int) and data[key] > 0:
test.timeout = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('timeout') + " in " +
bcolors.bold(test_file) + " has to be int bigger than 0")
elif key == 'args':
if isinstance(data[key], list):
test.args = data[key]
elif isinstance(data[key], str):
test.args = [data[key]]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('args') + " in " + bcolors.bold(test_file) +
" has to be array, string")
elif key == 'expectedFiles':
if isinstance(data[key], list):
test.expected_files = data[key]
elif isinstance(data[key], str):
test.expected_files = [data[key]]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('expectedFiles') + " in " +
bcolors.bold(test_file) + " has to be array or string")
elif key == 'outputFiles':
if isinstance(data[key], list):
test.output_files = data[key]
elif isinstance(data[key], str):
test.output_files = [data[key]]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('output_files') + " in " +
bcolors.bold(test_file) + " has to be array or string")
else:
error_handler.warning("Unknown key \'" + bcolors.note(key) +
"\' in " + test_file)
if len(test.output_files) != len(test.expected_files):
error_handler.call_error(
Errors.FATAL_ERROR,
"Count of files in " + bcolors.bold('outputFiles') + " and " +
bcolors.bold('expectedFiles') + " is different")
def run(test_directory):
"""
run test and store results
:param test_directory test directory
"""
# exclude
if check_exclude(test_directory):
return
if test_directory[-1] != '/':
test_directory += '/'
test_d = Path(test_directory)
if not test_d.is_dir():
error_handler.call_error(
Errors.WRONG_INPUT,
bcolors.bold(test_directory) + " does not exists or is not folder")
test = Test()
test_json_file = test_directory + test_file_name + ".json"
test_yaml_file = test_directory + test_file_name + ".yaml"
test_json_file_path = Path(test_json_file)
test_yaml_file_path = Path(test_yaml_file)
test.timeout = timeout
yaml_exists = test_yaml_file_path.is_file()
if not (yaml_exists or test_json_file_path.is_file()):
missing_test(test, test_directory)
else:
test_file = None
data = None
if yaml_exists:
if yaml_support:
test_file = test_yaml_file
data = read_yaml_test(test_file)
else:
error_handler.call_error(
Errors.YAML_DOESNT_SUPPORT,
"Can not read " + bcolors.note(test_file_name + ".yaml"))
else:
test_file = test_json_file
data = read_json_test(test_file)
read_test_file(test, test_file, data)
input_data, real_out, valid = test_set_missing(test, test_directory,
test_json_file)
if not valid:
return
cm = ""
if test.args is not None and len(test.args) > 0:
cm = ' '.join(test.args)
cm = ' ' + cm
cmd = (executable + cm)
process = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE, stdin=PIPE)
out = None
err = None
runout = False
if input_data is None:
try:
out, err = process.communicate(timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
else:
try:
out, err = process.communicate(input=input_data,
timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
proc_out = ""
if out is not None:
proc_out = out.decode('utf-8')
proc_err = ""
if err is not None:
proc_err = err.decode('utf-8')
if runout:
score["FAIL"].append(test_directory)
print(bcolors.note(test_directory) + ":")
if test.comment != "":
print("\t" + bcolors.warning(test.comment))
print(bcolors.warning("\tRequest timed out"))
print(bcolors.fail('\tTest FAILED\n'))
return
proc_rc = -9
if process.returncode is not None:
proc_rc = int(process.returncode)
result_code = test.code == proc_rc
result_out = real_out == proc_out
result_files = None
missing_files = list()
wrong_files = list()
if len(test.output_files) > 0:
result_files = True
i = 0
while i < len(test.expected_files):
exp = test_directory + test.expected_files[i]
exp_path = Path(exp)
p_out = test.output_files[i]
p_out_path = Path(p_out)
v = True
if not exp_path.is_file():
missing_files.append(exp)
result_files = False
v = False
if not p_out_path.is_file():
missing_files.append(p_out)
result_files = False
v = False
if v:
with open(exp, "r") as myfile:
exp_data = myfile.read()
with open(p_out, "r") as myfile:
out_data = myfile.read()
if exp_data == out_data:
result_files = result_files and True
else:
result_files = False
d_out = out_data.strip().splitlines()
d_exp = exp_data.strip().splitlines()
diff = list()
for line in difflib.unified_diff(d_exp,
d_out,
test.expected_files[i],
p_out,
n=0,
lineterm=''):
diff.append(line)
diff = "\n".join(diff) + "\n"
wrong_files.append([exp, p_out, diff])
i += 1
result = result_code and result_out
if result_files is not None:
result = result and result_files
if result:
score["OK"].append(test_directory)
if not only_failed:
print(bcolors.note(test_directory) + ":")
print(bcolors.success('\tTest OK\n'))
else:
test_failed(proc_err, proc_out, proc_rc, real_out, result_code,
result_out, result_files, test, test_directory,
missing_files, wrong_files)
def parse_config(ar, config_file, data):
for key in data.keys():
if key == 'onlyFailed':
if isinstance(data[key], bool):
ar.onlyFailed = data[key]
else:
error_handler.call_error(Errors.INVALID_JSON,
"OnlyFailed has to be bool")
elif key == 'noWarnings':
if isinstance(data[key], bool):
ar.no_warnings = data[key]
else:
error_handler.call_error(Errors.INVALID_JSON,
"noWarnings has to be bool")
elif key == 'noDiff':
if isinstance(data[key], bool):
ar.no_diff = data[key]
else:
error_handler.call_error(Errors.INVALID_JSON,
"noDiff has to be bool")
elif key == "testsDir":
if isinstance(data[key], str):
ar.tests_dir = data[key]
if ar.tests_dir[-1] != '/':
ar.tests_dir += '/'
else:
error_handler.call_error(Errors.INVALID_JSON,
"testsDirectory has to be string")
elif key == "resultDir":
if isinstance(data[key], str):
ar.result_dir = data[key]
if ar.result_dir[-1] != '/':
ar.result_dir += '/'
else:
error_handler.call_error(Errors.INVALID_JSON,
"resultDirectory has to be string")
elif key == "timeout":
if isinstance(data[key], int) and data[key] > 0:
ar.timeout = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
"timeout has to be int and greater than 0")
elif key == "exclude":
if isinstance(data[key], str):
if data[key] != "":
ar.exclude = [data[key]]
elif isinstance(data[key], list):
ar.exclude = data[key]
else:
error_handler.call_error(Errors.INVALID_JSON,
"exclude has to be string or array")
elif key == "executable":
if isinstance(data[key], str):
ar.executable = data[key]
else:
error_handler.call_error(Errors.INVALID_JSON,
"executable has to be string path")
else:
error_handler.warning("Unknown key \'" + bcolors.note(key) +
"\' in " + config_file)
def lets_init(arguments):
if arguments[0] == 'init':
if len(arguments) > 1 and arguments[1] == '--json':
arg = arguments.copy()
arg.pop(0)
arg.pop(0)
use_all = False
force = False
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
json_file_path = Path(argstest.config_json)
arg = argstest.parse_command_line_init(arg)
if not arg.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arg.hlp:
argstest.print_init_help()
exit(0)
arg = arg.export(use_all)
json_data = json.dumps(arg, indent=4)
if json_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(argstest.config_json))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(argstest.config_json) +
" already exists. If you want to rewrite it use --force option")
with open(argstest.config_json, "w") as json_file:
print(json_data, file=json_file)
print(bcolors.success("Success: ") + argstest.config_json + ' created')
else:
if not yaml_support:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can not export to yaml")
arg = arguments.copy()
arg.pop(0)
use_all = False
force = False
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
arg = argstest.parse_command_line_init(arg)
if not arg.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arg.hlp:
argstest.print_init_help()
exit(0)
arg = arg.export(use_all)
yaml_data = yaml.dump(arg, explicit_start=True, default_flow_style=False)
yaml_file_path = Path(argstest.config_yaml)
if yaml_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(argstest.config_yaml))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(argstest.config_yaml) +
" already exists. If you want to rewrite it use --force option")
with open(argstest.config_yaml, "w") as yaml_file:
print(yaml_data, file=yaml_file)
print(bcolors.success("Success: ") + argstest.config_yaml + ' created')
elif arguments[0] == 'testinit':
if len(arguments) > 1 and arguments[1] == '--json':
arg = arguments.copy()
arg.pop(0)
arg.pop(0)
use_all = False
force = False
if '-h' in arg:
argstest.print_testinit_help()
exit(0)
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
arg = argstest.parse_command_line_init_test(arg)
name = my_test.test_file_name_json
if arg.name is not None:
if arg.name[-1] != '/':
arg.name += '/'
makedirs(arg.name)
name = arg.name + name
arg = arg.export(use_all)
json_data = json.dumps(arg, indent=4)
json_file_path = Path(name)
if json_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(name))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(name) +
" already exists. If you want to rewrite it use --force option")
with open(name, "w") as json_file:
print(json_data, file=json_file)
print(bcolors.success("Success: ") + name + ' created')
else:
if not yaml_support:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can not export to yaml")
arg = arguments.copy()
arg.pop(0)
use_all = False
force = False
if '-h' in arg:
argstest.print_testinit_help()
exit(0)
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
arg = argstest.parse_command_line_init_test(arg)
name = my_test.test_file_name_yaml
if arg.name is not None:
if arg.name[-1] != '/':
arg.name += '/'
makedirs(arg.name)
name = arg.name + name
arg = arg.export(use_all)
yaml_data = yaml.dump(arg, explicit_start=True, default_flow_style=False)
json_file_path = Path(name)
if json_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(name))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(name) +
" already exists. If you want to rewrite it use --force option")
with open(name, "w") as yaml_file:
print(yaml_data, file=yaml_file)
print(bcolors.success("Success: ") + name + ' created')