def call_error(self, error_code, msg=None):
error_name = "Undefined error"
pom_msg = "use -h or --help to print help"
more_msg = None
if msg is not None:
pom_msg = msg
if error_code == Errors.WRONG_PARAM:
error_name = "Invalid arguments!"
elif error_code == Errors.INVALID_JSON:
error_name = "Invalid input json"
elif error_code == Errors.WRONG_INPUT:
error_name = "Invalid input"
elif error_code == Errors.MISSING_FILE:
error_name = "Missing file"
elif error_code == Errors.FATAL_ERROR:
error_name = "Fatal error"
elif error_code == Errors.YAML_DOESNT_SUPPORT:
error_name = "Yaml does not supported"
more_msg = "Please install " + bcolors.note(bcolors.bold("pyyaml")) + " package to use yaml."
if more_msg is not None:
stderr.write(bcolors.fail(error_name) + "\n\t" + bcolors.warning(more_msg) + "\n\t" + pom_msg + "\n")
else:
stderr.write(bcolors.fail(error_name) + "\n\t" + pom_msg + "\n")
if len(self.suspicions) > 0:
stderr.write(bcolors.note("\nNOTE:\n"))
for m in self.suspicions:
stderr.write("\t" + m + "\n")
self.clear_suspicions()
exit(error_code.value)
def show_score():
"""
show score
"""
poc_ok = len(score['OK'])
poc_fail = len(score['FAIL'])
poc_skip = len(score['SKIP'])
if poc_fail == 0 and poc_ok == 0 and poc_skip == 0:
return
print()
print(bcolors.note("###############"))
print(bcolors.note(" SCORE "))
print(bcolors.note("###############"))
if poc_ok > 0 and not only_failed:
print()
print(bcolors.success("Successful:"))
for t in score['OK']:
print("\t" + bcolors.success(t))
if poc_fail > 0:
print()
print(bcolors.fail("Failed:"))
for t in score['FAIL']:
print("\t" + bcolors.fail(t))
if poc_skip > 0 and not only_failed:
print()
print(bcolors.warning("Skipped:"))
for t in score['SKIP']:
print("\t" + bcolors.warning(t))
print()
print(bcolors.success("SUCC:\t"), poc_ok)
print(bcolors.fail("FAIL:\t"), poc_fail)
if poc_skip > 0:
print(bcolors.warning("SKIP:\t"), poc_skip)
if not only_failed:
percents = 100 * poc_ok // (poc_ok + poc_fail + poc_skip) if (
poc_ok + poc_fail + poc_skip > 0) else 0
if percents < 60:
print(bcolors.fail("Result with skip: " + str(percents) + "%"))
elif percents < 80:
print(
bcolors.warning("Result with skip: " + str(percents) +
"%"))
elif percents < 91:
print(bcolors.note("Result with skip: " + str(percents) + "%"))
else:
print(
bcolors.success("Result with skip: " + str(percents) +
"%"))
percents = 100 * poc_ok // (poc_ok + poc_fail) if (
poc_ok + poc_fail > 0) else 0
if percents < 60:
print(bcolors.fail("Final result: " + str(percents) + "%"))
elif percents < 80:
print(bcolors.warning("Final result: " + str(percents) + "%"))
elif percents < 91:
print(bcolors.note("Final result: " + str(percents) + "%"))
else:
print(bcolors.success("Final result: " + str(percents) + "%"))
return poc_fail == 0
def test_failed(proc_err, proc_out, proc_rc, real_out, result_code, result_out,
result_files, test, test_directory, missing_files,
wrong_files):
score["FAIL"].append(test_directory)
print(bcolors.note(test_directory) + ":")
if test.comment != "":
print("\t" + bcolors.warning(test.comment))
print('\tReturn code: ' +
(bcolors.success("OK") if result_code else bcolors.fail("FAIL")))
if not result_code:
print('\t\tYour return code:', proc_rc)
print('\t\tReal return code:', test.code)
sn, sm = get_signal(proc_rc)
if sn != "" and sm != "":
print("\t" + bcolors.warning(sn) + " " + sm)
print('\tStdout: ' +
(bcolors.success("OK") if result_out else bcolors.fail("FAIL")))
a = test_directory.split('/')
a.pop(0)
c = "/".join(a)
name = c
if not result_out:
# if or proc_err != "":
print('\t\tstdout saved: ' + str(resultDir) + name + "stdout.out")
if no_diff:
store_test(name, proc_out, None, proc_err)
else:
print('\t\t\tdiff saved: ' + str(resultDir) + name + "diff")
store_test(name, proc_out, real_out, proc_err)
if proc_err != "":
print('\t\tstderr saved: ' + str(resultDir) + name + "stderr.out")
if result_files is not None:
print('\tFiles: ' + (
bcolors.success("OK") if result_files else bcolors.fail("FAIL")))
if not result_files:
if len(missing_files) > 0:
print('\tMissing files:')
for mf in missing_files:
print('\t\t' + mf)
if len(wrong_files) > 0:
print('\tWrong files:')
for f in wrong_files:
exp, out, diff = tuple(f)
exp = exp.split('/')[-1]
out = out.split('/')[-1]
store_diff(name, exp, out, diff)
print('\t\t' + exp + " x " + out + " is different")
print('\t\tdiff saved: ' + str(resultDir) + name + exp +
'_x_' + out + '.diff')
print(bcolors.fail('\tTest FAILED\n'))
def run(test_directory):
"""
run test and store results
:param test_directory test directory
"""
# exclude
if check_exclude(test_directory):
return
if test_directory[-1] != '/':
test_directory += '/'
test_d = Path(test_directory)
if not test_d.is_dir():
error_handler.call_error(
Errors.WRONG_INPUT,
bcolors.bold(test_directory) + " does not exists or is not folder")
test = Test()
test_json_file = test_directory + test_file_name + ".json"
test_yaml_file = test_directory + test_file_name + ".yaml"
test_json_file_path = Path(test_json_file)
test_yaml_file_path = Path(test_yaml_file)
test.timeout = timeout
yaml_exists = test_yaml_file_path.is_file()
if not (yaml_exists or test_json_file_path.is_file()):
missing_test(test, test_directory)
else:
test_file = None
data = None
if yaml_exists:
if yaml_support:
test_file = test_yaml_file
data = read_yaml_test(test_file)
else:
error_handler.call_error(
Errors.YAML_DOESNT_SUPPORT,
"Can not read " + bcolors.note(test_file_name + ".yaml"))
else:
test_file = test_json_file
data = read_json_test(test_file)
read_test_file(test, test_file, data)
input_data, real_out, valid = test_set_missing(test, test_directory,
test_json_file)
if not valid:
return
cm = ""
if test.args is not None and len(test.args) > 0:
cm = ' '.join(test.args)
cm = ' ' + cm
cmd = (executable + cm)
process = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE, stdin=PIPE)
out = None
err = None
runout = False
if input_data is None:
try:
out, err = process.communicate(timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
else:
try:
out, err = process.communicate(input=input_data,
timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
proc_out = ""
if out is not None:
proc_out = out.decode('utf-8')
proc_err = ""
if err is not None:
proc_err = err.decode('utf-8')
if runout:
score["FAIL"].append(test_directory)
print(bcolors.note(test_directory) + ":")
if test.comment != "":
print("\t" + bcolors.warning(test.comment))
print(bcolors.warning("\tRequest timed out"))
print(bcolors.fail('\tTest FAILED\n'))
return
proc_rc = -9
if process.returncode is not None:
proc_rc = int(process.returncode)
result_code = test.code == proc_rc
result_out = real_out == proc_out
result_files = None
missing_files = list()
wrong_files = list()
if len(test.output_files) > 0:
result_files = True
i = 0
while i < len(test.expected_files):
exp = test_directory + test.expected_files[i]
exp_path = Path(exp)
p_out = test.output_files[i]
p_out_path = Path(p_out)
v = True
if not exp_path.is_file():
missing_files.append(exp)
result_files = False
v = False
if not p_out_path.is_file():
missing_files.append(p_out)
result_files = False
v = False
if v:
with open(exp, "r") as myfile:
exp_data = myfile.read()
with open(p_out, "r") as myfile:
out_data = myfile.read()
if exp_data == out_data:
result_files = result_files and True
else:
result_files = False
d_out = out_data.strip().splitlines()
d_exp = exp_data.strip().splitlines()
diff = list()
for line in difflib.unified_diff(d_exp,
d_out,
test.expected_files[i],
p_out,
n=0,
lineterm=''):
diff.append(line)
diff = "\n".join(diff) + "\n"
wrong_files.append([exp, p_out, diff])
i += 1
result = result_code and result_out
if result_files is not None:
result = result and result_files
if result:
score["OK"].append(test_directory)
if not only_failed:
print(bcolors.note(test_directory) + ":")
print(bcolors.success('\tTest OK\n'))
else:
test_failed(proc_err, proc_out, proc_rc, real_out, result_code,
result_out, result_files, test, test_directory,
missing_files, wrong_files)