def show_score(): """ show score """ poc_ok = len(score['OK']) poc_fail = len(score['FAIL']) poc_skip = len(score['SKIP']) if poc_fail == 0 and poc_ok == 0 and poc_skip == 0: return print() print(bcolors.note("###############")) print(bcolors.note(" SCORE ")) print(bcolors.note("###############")) if poc_ok > 0 and not only_failed: print() print(bcolors.success("Successful:")) for t in score['OK']: print("\t" + bcolors.success(t)) if poc_fail > 0: print() print(bcolors.fail("Failed:")) for t in score['FAIL']: print("\t" + bcolors.fail(t)) if poc_skip > 0 and not only_failed: print() print(bcolors.warning("Skipped:")) for t in score['SKIP']: print("\t" + bcolors.warning(t)) print() print(bcolors.success("SUCC:\t"), poc_ok) print(bcolors.fail("FAIL:\t"), poc_fail) if poc_skip > 0: print(bcolors.warning("SKIP:\t"), poc_skip) if not only_failed: percents = 100 * poc_ok // (poc_ok + poc_fail + poc_skip) if ( poc_ok + poc_fail + poc_skip > 0) else 0 if percents < 60: print(bcolors.fail("Result with skip: " + str(percents) + "%")) elif percents < 80: print( bcolors.warning("Result with skip: " + str(percents) + "%")) elif percents < 91: print(bcolors.note("Result with skip: " + str(percents) + "%")) else: print( bcolors.success("Result with skip: " + str(percents) + "%")) percents = 100 * poc_ok // (poc_ok + poc_fail) if ( poc_ok + poc_fail > 0) else 0 if percents < 60: print(bcolors.fail("Final result: " + str(percents) + "%")) elif percents < 80: print(bcolors.warning("Final result: " + str(percents) + "%")) elif percents < 91: print(bcolors.note("Final result: " + str(percents) + "%")) else: print(bcolors.success("Final result: " + str(percents) + "%")) return poc_fail == 0
def test_failed(proc_err, proc_out, proc_rc, real_out, result_code, result_out, result_files, test, test_directory, missing_files, wrong_files): score["FAIL"].append(test_directory) print(bcolors.note(test_directory) + ":") if test.comment != "": print("\t" + bcolors.warning(test.comment)) print('\tReturn code: ' + (bcolors.success("OK") if result_code else bcolors.fail("FAIL"))) if not result_code: print('\t\tYour return code:', proc_rc) print('\t\tReal return code:', test.code) sn, sm = get_signal(proc_rc) if sn != "" and sm != "": print("\t" + bcolors.warning(sn) + " " + sm) print('\tStdout: ' + (bcolors.success("OK") if result_out else bcolors.fail("FAIL"))) a = test_directory.split('/') a.pop(0) c = "/".join(a) name = c if not result_out: # if or proc_err != "": print('\t\tstdout saved: ' + str(resultDir) + name + "stdout.out") if no_diff: store_test(name, proc_out, None, proc_err) else: print('\t\t\tdiff saved: ' + str(resultDir) + name + "diff") store_test(name, proc_out, real_out, proc_err) if proc_err != "": print('\t\tstderr saved: ' + str(resultDir) + name + "stderr.out") if result_files is not None: print('\tFiles: ' + ( bcolors.success("OK") if result_files else bcolors.fail("FAIL"))) if not result_files: if len(missing_files) > 0: print('\tMissing files:') for mf in missing_files: print('\t\t' + mf) if len(wrong_files) > 0: print('\tWrong files:') for f in wrong_files: exp, out, diff = tuple(f) exp = exp.split('/')[-1] out = out.split('/')[-1] store_diff(name, exp, out, diff) print('\t\t' + exp + " x " + out + " is different") print('\t\tdiff saved: ' + str(resultDir) + name + exp + '_x_' + out + '.diff') print(bcolors.fail('\tTest FAILED\n'))
def run(test_directory): """ run test and store results :param test_directory test directory """ # exclude if check_exclude(test_directory): return if test_directory[-1] != '/': test_directory += '/' test_d = Path(test_directory) if not test_d.is_dir(): error_handler.call_error( Errors.WRONG_INPUT, bcolors.bold(test_directory) + " does not exists or is not folder") test = Test() test_json_file = test_directory + test_file_name + ".json" test_yaml_file = test_directory + test_file_name + ".yaml" test_json_file_path = Path(test_json_file) test_yaml_file_path = Path(test_yaml_file) test.timeout = timeout yaml_exists = test_yaml_file_path.is_file() if not (yaml_exists or test_json_file_path.is_file()): missing_test(test, test_directory) else: test_file = None data = None if yaml_exists: if yaml_support: test_file = test_yaml_file data = read_yaml_test(test_file) else: error_handler.call_error( Errors.YAML_DOESNT_SUPPORT, "Can not read " + bcolors.note(test_file_name + ".yaml")) else: test_file = test_json_file data = read_json_test(test_file) read_test_file(test, test_file, data) input_data, real_out, valid = test_set_missing(test, test_directory, test_json_file) if not valid: return cm = "" if test.args is not None and len(test.args) > 0: cm = ' '.join(test.args) cm = ' ' + cm cmd = (executable + cm) process = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE, stdin=PIPE) out = None err = None runout = False if input_data is None: try: out, err = process.communicate(timeout=test.timeout) except TimeoutExpired: process.kill() runout = True else: try: out, err = process.communicate(input=input_data, timeout=test.timeout) except TimeoutExpired: process.kill() runout = True proc_out = "" if out is not None: proc_out = out.decode('utf-8') proc_err = "" if err is not None: proc_err = err.decode('utf-8') if runout: score["FAIL"].append(test_directory) print(bcolors.note(test_directory) + ":") if test.comment != "": print("\t" + bcolors.warning(test.comment)) print(bcolors.warning("\tRequest timed out")) print(bcolors.fail('\tTest FAILED\n')) return proc_rc = -9 if process.returncode is not None: proc_rc = int(process.returncode) result_code = test.code == proc_rc result_out = real_out == proc_out result_files = None missing_files = list() wrong_files = list() if len(test.output_files) > 0: result_files = True i = 0 while i < len(test.expected_files): exp = test_directory + test.expected_files[i] exp_path = Path(exp) p_out = test.output_files[i] p_out_path = Path(p_out) v = True if not exp_path.is_file(): missing_files.append(exp) result_files = False v = False if not p_out_path.is_file(): missing_files.append(p_out) result_files = False v = False if v: with open(exp, "r") as myfile: exp_data = myfile.read() with open(p_out, "r") as myfile: out_data = myfile.read() if exp_data == out_data: result_files = result_files and True else: result_files = False d_out = out_data.strip().splitlines() d_exp = exp_data.strip().splitlines() diff = list() for line in difflib.unified_diff(d_exp, d_out, test.expected_files[i], p_out, n=0, lineterm=''): diff.append(line) diff = "\n".join(diff) + "\n" wrong_files.append([exp, p_out, diff]) i += 1 result = result_code and result_out if result_files is not None: result = result and result_files if result: score["OK"].append(test_directory) if not only_failed: print(bcolors.note(test_directory) + ":") print(bcolors.success('\tTest OK\n')) else: test_failed(proc_err, proc_out, proc_rc, real_out, result_code, result_out, result_files, test, test_directory, missing_files, wrong_files)
def lets_init(arguments): if arguments[0] == 'init': if len(arguments) > 1 and arguments[1] == '--json': arg = arguments.copy() arg.pop(0) arg.pop(0) use_all = False force = False if len(arg) > 0: if arg[0] == '--all': arg.pop(0) use_all = True if len(arg) > 0 and arg[0] == '--force': arg.pop(0) force = True json_file_path = Path(argstest.config_json) arg = argstest.parse_command_line_init(arg) if not arg.valid: error_handler.call_error(Errors.WRONG_PARAM) if arg.hlp: argstest.print_init_help() exit(0) arg = arg.export(use_all) json_data = json.dumps(arg, indent=4) if json_file_path.exists(): if force: error_handler.call_warning("Rewriting " + bcolors.note(argstest.config_json)) else: error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(argstest.config_json) + " already exists. If you want to rewrite it use --force option") with open(argstest.config_json, "w") as json_file: print(json_data, file=json_file) print(bcolors.success("Success: ") + argstest.config_json + ' created') else: if not yaml_support: error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can not export to yaml") arg = arguments.copy() arg.pop(0) use_all = False force = False if len(arg) > 0: if arg[0] == '--all': arg.pop(0) use_all = True if len(arg) > 0 and arg[0] == '--force': arg.pop(0) force = True arg = argstest.parse_command_line_init(arg) if not arg.valid: error_handler.call_error(Errors.WRONG_PARAM) if arg.hlp: argstest.print_init_help() exit(0) arg = arg.export(use_all) yaml_data = yaml.dump(arg, explicit_start=True, default_flow_style=False) yaml_file_path = Path(argstest.config_yaml) if yaml_file_path.exists(): if force: error_handler.call_warning("Rewriting " + bcolors.note(argstest.config_yaml)) else: error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(argstest.config_yaml) + " already exists. If you want to rewrite it use --force option") with open(argstest.config_yaml, "w") as yaml_file: print(yaml_data, file=yaml_file) print(bcolors.success("Success: ") + argstest.config_yaml + ' created') elif arguments[0] == 'testinit': if len(arguments) > 1 and arguments[1] == '--json': arg = arguments.copy() arg.pop(0) arg.pop(0) use_all = False force = False if '-h' in arg: argstest.print_testinit_help() exit(0) if len(arg) > 0: if arg[0] == '--all': arg.pop(0) use_all = True if len(arg) > 0 and arg[0] == '--force': arg.pop(0) force = True arg = argstest.parse_command_line_init_test(arg) name = my_test.test_file_name_json if arg.name is not None: if arg.name[-1] != '/': arg.name += '/' makedirs(arg.name) name = arg.name + name arg = arg.export(use_all) json_data = json.dumps(arg, indent=4) json_file_path = Path(name) if json_file_path.exists(): if force: error_handler.call_warning("Rewriting " + bcolors.note(name)) else: error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(name) + " already exists. If you want to rewrite it use --force option") with open(name, "w") as json_file: print(json_data, file=json_file) print(bcolors.success("Success: ") + name + ' created') else: if not yaml_support: error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can not export to yaml") arg = arguments.copy() arg.pop(0) use_all = False force = False if '-h' in arg: argstest.print_testinit_help() exit(0) if len(arg) > 0: if arg[0] == '--all': arg.pop(0) use_all = True if len(arg) > 0 and arg[0] == '--force': arg.pop(0) force = True arg = argstest.parse_command_line_init_test(arg) name = my_test.test_file_name_yaml if arg.name is not None: if arg.name[-1] != '/': arg.name += '/' makedirs(arg.name) name = arg.name + name arg = arg.export(use_all) yaml_data = yaml.dump(arg, explicit_start=True, default_flow_style=False) json_file_path = Path(name) if json_file_path.exists(): if force: error_handler.call_warning("Rewriting " + bcolors.note(name)) else: error_handler.call_error(Errors.FATAL_ERROR, bcolors.note(name) + " already exists. If you want to rewrite it use --force option") with open(name, "w") as yaml_file: print(yaml_data, file=yaml_file) print(bcolors.success("Success: ") + name + ' created')