def show_score():
"""
show score
"""
poc_ok = len(score['OK'])
poc_fail = len(score['FAIL'])
poc_skip = len(score['SKIP'])
if poc_fail == 0 and poc_ok == 0 and poc_skip == 0:
return
print()
print(bcolors.note("###############"))
print(bcolors.note(" SCORE "))
print(bcolors.note("###############"))
if poc_ok > 0 and not only_failed:
print()
print(bcolors.success("Successful:"))
for t in score['OK']:
print("\t" + bcolors.success(t))
if poc_fail > 0:
print()
print(bcolors.fail("Failed:"))
for t in score['FAIL']:
print("\t" + bcolors.fail(t))
if poc_skip > 0 and not only_failed:
print()
print(bcolors.warning("Skipped:"))
for t in score['SKIP']:
print("\t" + bcolors.warning(t))
print()
print(bcolors.success("SUCC:\t"), poc_ok)
print(bcolors.fail("FAIL:\t"), poc_fail)
if poc_skip > 0:
print(bcolors.warning("SKIP:\t"), poc_skip)
if not only_failed:
percents = 100 * poc_ok // (poc_ok + poc_fail + poc_skip) if (
poc_ok + poc_fail + poc_skip > 0) else 0
if percents < 60:
print(bcolors.fail("Result with skip: " + str(percents) + "%"))
elif percents < 80:
print(
bcolors.warning("Result with skip: " + str(percents) +
"%"))
elif percents < 91:
print(bcolors.note("Result with skip: " + str(percents) + "%"))
else:
print(
bcolors.success("Result with skip: " + str(percents) +
"%"))
percents = 100 * poc_ok // (poc_ok + poc_fail) if (
poc_ok + poc_fail > 0) else 0
if percents < 60:
print(bcolors.fail("Final result: " + str(percents) + "%"))
elif percents < 80:
print(bcolors.warning("Final result: " + str(percents) + "%"))
elif percents < 91:
print(bcolors.note("Final result: " + str(percents) + "%"))
else:
print(bcolors.success("Final result: " + str(percents) + "%"))
return poc_fail == 0
def test_failed(proc_err, proc_out, proc_rc, real_out, result_code, result_out,
result_files, test, test_directory, missing_files,
wrong_files):
score["FAIL"].append(test_directory)
print(bcolors.note(test_directory) + ":")
if test.comment != "":
print("\t" + bcolors.warning(test.comment))
print('\tReturn code: ' +
(bcolors.success("OK") if result_code else bcolors.fail("FAIL")))
if not result_code:
print('\t\tYour return code:', proc_rc)
print('\t\tReal return code:', test.code)
sn, sm = get_signal(proc_rc)
if sn != "" and sm != "":
print("\t" + bcolors.warning(sn) + " " + sm)
print('\tStdout: ' +
(bcolors.success("OK") if result_out else bcolors.fail("FAIL")))
a = test_directory.split('/')
a.pop(0)
c = "/".join(a)
name = c
if not result_out:
# if or proc_err != "":
print('\t\tstdout saved: ' + str(resultDir) + name + "stdout.out")
if no_diff:
store_test(name, proc_out, None, proc_err)
else:
print('\t\t\tdiff saved: ' + str(resultDir) + name + "diff")
store_test(name, proc_out, real_out, proc_err)
if proc_err != "":
print('\t\tstderr saved: ' + str(resultDir) + name + "stderr.out")
if result_files is not None:
print('\tFiles: ' + (
bcolors.success("OK") if result_files else bcolors.fail("FAIL")))
if not result_files:
if len(missing_files) > 0:
print('\tMissing files:')
for mf in missing_files:
print('\t\t' + mf)
if len(wrong_files) > 0:
print('\tWrong files:')
for f in wrong_files:
exp, out, diff = tuple(f)
exp = exp.split('/')[-1]
out = out.split('/')[-1]
store_diff(name, exp, out, diff)
print('\t\t' + exp + " x " + out + " is different")
print('\t\tdiff saved: ' + str(resultDir) + name + exp +
'_x_' + out + '.diff')
print(bcolors.fail('\tTest FAILED\n'))
def run(test_directory):
"""
run test and store results
:param test_directory test directory
"""
# exclude
if check_exclude(test_directory):
return
if test_directory[-1] != '/':
test_directory += '/'
test_d = Path(test_directory)
if not test_d.is_dir():
error_handler.call_error(
Errors.WRONG_INPUT,
bcolors.bold(test_directory) + " does not exists or is not folder")
test = Test()
test_json_file = test_directory + test_file_name + ".json"
test_yaml_file = test_directory + test_file_name + ".yaml"
test_json_file_path = Path(test_json_file)
test_yaml_file_path = Path(test_yaml_file)
test.timeout = timeout
yaml_exists = test_yaml_file_path.is_file()
if not (yaml_exists or test_json_file_path.is_file()):
missing_test(test, test_directory)
else:
test_file = None
data = None
if yaml_exists:
if yaml_support:
test_file = test_yaml_file
data = read_yaml_test(test_file)
else:
error_handler.call_error(
Errors.YAML_DOESNT_SUPPORT,
"Can not read " + bcolors.note(test_file_name + ".yaml"))
else:
test_file = test_json_file
data = read_json_test(test_file)
read_test_file(test, test_file, data)
input_data, real_out, valid = test_set_missing(test, test_directory,
test_json_file)
if not valid:
return
cm = ""
if test.args is not None and len(test.args) > 0:
cm = ' '.join(test.args)
cm = ' ' + cm
cmd = (executable + cm)
process = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE, stdin=PIPE)
out = None
err = None
runout = False
if input_data is None:
try:
out, err = process.communicate(timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
else:
try:
out, err = process.communicate(input=input_data,
timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
proc_out = ""
if out is not None:
proc_out = out.decode('utf-8')
proc_err = ""
if err is not None:
proc_err = err.decode('utf-8')
if runout:
score["FAIL"].append(test_directory)
print(bcolors.note(test_directory) + ":")
if test.comment != "":
print("\t" + bcolors.warning(test.comment))
print(bcolors.warning("\tRequest timed out"))
print(bcolors.fail('\tTest FAILED\n'))
return
proc_rc = -9
if process.returncode is not None:
proc_rc = int(process.returncode)
result_code = test.code == proc_rc
result_out = real_out == proc_out
result_files = None
missing_files = list()
wrong_files = list()
if len(test.output_files) > 0:
result_files = True
i = 0
while i < len(test.expected_files):
exp = test_directory + test.expected_files[i]
exp_path = Path(exp)
p_out = test.output_files[i]
p_out_path = Path(p_out)
v = True
if not exp_path.is_file():
missing_files.append(exp)
result_files = False
v = False
if not p_out_path.is_file():
missing_files.append(p_out)
result_files = False
v = False
if v:
with open(exp, "r") as myfile:
exp_data = myfile.read()
with open(p_out, "r") as myfile:
out_data = myfile.read()
if exp_data == out_data:
result_files = result_files and True
else:
result_files = False
d_out = out_data.strip().splitlines()
d_exp = exp_data.strip().splitlines()
diff = list()
for line in difflib.unified_diff(d_exp,
d_out,
test.expected_files[i],
p_out,
n=0,
lineterm=''):
diff.append(line)
diff = "\n".join(diff) + "\n"
wrong_files.append([exp, p_out, diff])
i += 1
result = result_code and result_out
if result_files is not None:
result = result and result_files
if result:
score["OK"].append(test_directory)
if not only_failed:
print(bcolors.note(test_directory) + ":")
print(bcolors.success('\tTest OK\n'))
else:
test_failed(proc_err, proc_out, proc_rc, real_out, result_code,
result_out, result_files, test, test_directory,
missing_files, wrong_files)
def lets_init(arguments):
if arguments[0] == 'init':
if len(arguments) > 1 and arguments[1] == '--json':
arg = arguments.copy()
arg.pop(0)
arg.pop(0)
use_all = False
force = False
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
json_file_path = Path(argstest.config_json)
arg = argstest.parse_command_line_init(arg)
if not arg.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arg.hlp:
argstest.print_init_help()
exit(0)
arg = arg.export(use_all)
json_data = json.dumps(arg, indent=4)
if json_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(argstest.config_json))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(argstest.config_json) +
" already exists. If you want to rewrite it use --force option")
with open(argstest.config_json, "w") as json_file:
print(json_data, file=json_file)
print(bcolors.success("Success: ") + argstest.config_json + ' created')
else:
if not yaml_support:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can not export to yaml")
arg = arguments.copy()
arg.pop(0)
use_all = False
force = False
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
arg = argstest.parse_command_line_init(arg)
if not arg.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arg.hlp:
argstest.print_init_help()
exit(0)
arg = arg.export(use_all)
yaml_data = yaml.dump(arg, explicit_start=True, default_flow_style=False)
yaml_file_path = Path(argstest.config_yaml)
if yaml_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(argstest.config_yaml))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(argstest.config_yaml) +
" already exists. If you want to rewrite it use --force option")
with open(argstest.config_yaml, "w") as yaml_file:
print(yaml_data, file=yaml_file)
print(bcolors.success("Success: ") + argstest.config_yaml + ' created')
elif arguments[0] == 'testinit':
if len(arguments) > 1 and arguments[1] == '--json':
arg = arguments.copy()
arg.pop(0)
arg.pop(0)
use_all = False
force = False
if '-h' in arg:
argstest.print_testinit_help()
exit(0)
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
arg = argstest.parse_command_line_init_test(arg)
name = my_test.test_file_name_json
if arg.name is not None:
if arg.name[-1] != '/':
arg.name += '/'
makedirs(arg.name)
name = arg.name + name
arg = arg.export(use_all)
json_data = json.dumps(arg, indent=4)
json_file_path = Path(name)
if json_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(name))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(name) +
" already exists. If you want to rewrite it use --force option")
with open(name, "w") as json_file:
print(json_data, file=json_file)
print(bcolors.success("Success: ") + name + ' created')
else:
if not yaml_support:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can not export to yaml")
arg = arguments.copy()
arg.pop(0)
use_all = False
force = False
if '-h' in arg:
argstest.print_testinit_help()
exit(0)
if len(arg) > 0:
if arg[0] == '--all':
arg.pop(0)
use_all = True
if len(arg) > 0 and arg[0] == '--force':
arg.pop(0)
force = True
arg = argstest.parse_command_line_init_test(arg)
name = my_test.test_file_name_yaml
if arg.name is not None:
if arg.name[-1] != '/':
arg.name += '/'
makedirs(arg.name)
name = arg.name + name
arg = arg.export(use_all)
yaml_data = yaml.dump(arg, explicit_start=True, default_flow_style=False)
json_file_path = Path(name)
if json_file_path.exists():
if force:
error_handler.call_warning("Rewriting " + bcolors.note(name))
else:
error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(name) +
" already exists. If you want to rewrite it use --force option")
with open(name, "w") as yaml_file:
print(yaml_data, file=yaml_file)
print(bcolors.success("Success: ") + name + ' created')