def missing_test(test, test_directory): """ recovery from missing json :param test test class :param test_directory test directory """ error_handler.add_suspicion( bcolors.bold(test_file_name + '.json or ' + test_file_name + '.yaml') + " is missing in " + bcolors.bold(test_directory) + ". Trying to recover with default values") default_input_name = "input.in" input = test_directory + default_input_name input_path = Path(input) if not input_path.is_file(): error_handler.add_suspicion(bcolors.bold(input) + " is missing") else: test.input = default_input_name default_output_name = "output.out" output = test_directory + default_output_name input_path = Path(output) if not input_path.is_file(): error_handler.add_suspicion(bcolors.bold(output) + " is missing") else: test.output = default_output_name
def print_testinit_help(): """ print help """ print(bcolors.OKGREEN + "Tester init" + bcolors.ENDC) print( "Generates test config file template in yaml format. For json format use " + bcolors.bold("testinit --json")) print_option("testinit -h", "Print help") print_option("testinit --json", "Generates micro-template in json format") print_option("testinit --all, testinit --json --all", "Generates template") print_option( "testinit --force", "Generates template even if file exists\n" + bcolors.note("NOTE:") + " If you want to use it with " + bcolors.header("--all") + " write " + bcolors.header("--all --force")) print( bcolors.note("Note:") + "\n\t You can use following options at the end of command to fill template" ) print_option( "-n, --name FOLDER", "Create " + bcolors.bold("FOLDER") + " and generated file save in it") print_option("-i, --input FILE", "File with stdin") print_option("-o, --output FILE", "File with stdout") print_option("-r, --return-code NUMBER", "Expected return code") print_option("-c, --comment", "Test comment") print_option("-T, --timeout", "Maximum time for test")
def parse_command_line(ar, arguments): i = 0 while i < len(arguments): arg = arguments[i] if arg == '-h' or arg == '--help': ar.hlp = True break if arg == '-V' or arg == '--version': ar.version = True break elif arg == '-f' or arg == '--only-failed': ar.onlyFailed = True elif arg == '-W' or arg == '--no-warnings': ar.no_warnings = True elif arg == '-D' or arg == '--no-diff': ar.no_diff = True elif arg == '-C' or arg == '--clean': ar.clean = True else: if i == len(arguments) - 1: error_handler.add_suspicion( "Wrong argument or incomplete argument " + bcolors.bold(arg)) ar.valid = False break next_arg = arguments[i + 1] i += 1 if arg == '-d' or arg == '--test-dir': ar.tests_dir = next_arg if ar.tests_dir[-1] != '/': ar.tests_dir += '/' elif arg == '-t' or arg == '--test': ar.test = next_arg elif arg == '-T' or arg == '--timeout': if is_int(next_arg): ar.timeout = int(next_arg) else: error_handler.add_suspicion("Wrong argument type after " + bcolors.bold(arg)) elif arg == '-R' or arg == '--result-dir': ar.result_dir = next_arg if ar.result_dir[-1] != '/': ar.result_dir += '/' elif arg == '-e' or arg == '--executable': ar.executable = next_arg elif arg == '-E' or arg == '--exclude': if next_arg != "": ar.exclude = [next_arg] else: error_handler.add_suspicion( "Wrong argument or incomplete argument " + bcolors.bold(arg)) ar.valid = False break i += 1
def load_config(ar, config_file=None): """ :param ar arguments class :param config_file name of config file """ data = list() if config_file is None: config_file = config_yaml my_file = Path(config_file) if my_file.is_file(): if yaml_support: data = load_yaml(config_file) else: error_handler.call_error(Errors.YAML_DOESNT_SUPPORT, "Can't use " + config_yaml) else: config_file = config_json my_file = Path(config_file) if my_file.is_file(): data = load_json(config_file) else: error_handler.add_suspicion( bcolors.bold(config_yaml) + " or " + bcolors.bold(config_json) + " is missing or can\'t be opened.") ar.valid = False return else: my_file = Path(config_file) if my_file.is_file(): file_type = config_file.split('.')[-1] if file_type == 'yaml': if not yaml_support: error_handler.call_error(Errors.YAML_DOESNT_SUPPORT) else: data = load_yaml(config_file) elif file_type == 'json': data = load_json(config_file) else: error_handler.add_suspicion( bcolors.bold(config_file) + " have to end with .json or .yaml") ar.valid = False return else: error_handler.add_suspicion( bcolors.bold(config_file) + " is missing or can\'t be opened.") ar.valid = False return parse_config(ar, config_file, data)
def print_init_help(): """ print help """ print(bcolors.OKGREEN + "Tester init" + bcolors.ENDC) print( "Generates config file template in yaml format. For json format use " + bcolors.bold("init --json")) print_option("init -h, init --help", "Print help") print_option("init --json", "Generates micro-template in json format") print_option("init --all, init --json --all", "Generates template") print_option( "init --force", "Generates template even if file exists\n" + bcolors.note("NOTE:") + " If you want to use it with " + bcolors.header("--all") + " write " + bcolors.header("--all --force")) print( bcolors.note("Note:") + "\n\t You can use following options at the end of command to fill template" ) print_option("-W, --no-warnings", "Silence all test warnings") print_option("-D, --no-diff", "Stop storing stdout diff") print_option("-e, --executable " + bcolors.bold("EXECUTABLE"), "set executable") print_option("-f, --only-failed", "Print only failed tests") print_option("-d, --test-dir " + bcolors.bold("TESTS_DIRECTORY"), "Tests will run from " + bcolors.bold("TESTS_DIRECTORY")) print_option("-R, --result-dir " + bcolors.bold("RESULT_DIRECTORY"), "Output directory") print_option("-T, --timeout " + bcolors.bold("SECONDS"), "Maximum time for test\n" + bcolors.note("NOTE:")) print_option( "-E, --exclude " + bcolors.bold("REGEX"), "Directories matching " + bcolors.bold("REGEX") + " will be excluded\n" + bcolors.note("NOTE:") + " When using command line arguments you can use only one regex patter. If you want to use more, " "you have to use it in configuration file.")
def check_exclude(test_directory): """ check if directory is in excluded :param test_directory test directory """ for r in exclude: m = re.compile(r) if m.search(test_directory) is not None: error_handler.call_warning( bcolors.bold(test_directory) + " is excluded. " + bcolors.bold("Skipping test!")) score['SKIP'].append(test_directory) return True return False
def call_error(self, error_code, msg=None): error_name = "Undefined error" pom_msg = "use -h or --help to print help" more_msg = None if msg is not None: pom_msg = msg if error_code == Errors.WRONG_PARAM: error_name = "Invalid arguments!" elif error_code == Errors.INVALID_JSON: error_name = "Invalid input json" elif error_code == Errors.WRONG_INPUT: error_name = "Invalid input" elif error_code == Errors.MISSING_FILE: error_name = "Missing file" elif error_code == Errors.FATAL_ERROR: error_name = "Fatal error" elif error_code == Errors.YAML_DOESNT_SUPPORT: error_name = "Yaml does not supported" more_msg = "Please install " + bcolors.note(bcolors.bold("pyyaml")) + " package to use yaml." if more_msg is not None: stderr.write(bcolors.fail(error_name) + "\n\t" + bcolors.warning(more_msg) + "\n\t" + pom_msg + "\n") else: stderr.write(bcolors.fail(error_name) + "\n\t" + pom_msg + "\n") if len(self.suspicions) > 0: stderr.write(bcolors.note("\nNOTE:\n")) for m in self.suspicions: stderr.write("\t" + m + "\n") self.clear_suspicions() exit(error_code.value)
def read_yaml_test(yaml_file): with open(yaml_file) as data_file: data = None try: data = yaml.load(data_file) except yaml.YAMLError: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold(yaml_file) + " has invalid syntax") return data
def read_json_test(json_file): with open(json_file) as data_file: data = None try: data = json.load(data_file) except ValueError: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold(json_file) + " has invalid syntax") return data
def parse_command_line_init_test(arguments, ar=Test()): i = 0 while i < len(arguments): arg = arguments[i] if i == len(arguments) - 1: error_handler.call_error( Errors.WRONG_PARAM, "Wrong argument or incomplete argument " + bcolors.bold(arg)) break next_arg = arguments[i + 1] i += 1 if arg == '-i' or arg == '--input': ar.input = next_arg elif arg == '-o' or arg == '--output': ar.output = next_arg elif arg == '-n' or arg == '--name': ar.name = next_arg elif arg == '-r' or arg == '--return-code': if is_int(next_arg): ar.code = int(next_arg) else: error_handler.call_error( Errors.WRONG_PARAM, "Incompatible value for " + bcolors.bold(arg)) elif arg == '-c' or arg == '--comment': ar.comment = next_arg elif arg == '-T' or arg == '--timeout': if is_int(next_arg): ar.timeout = int(next_arg) else: error_handler.call_error( Errors.WRONG_PARAM, "Incompatible value for " + bcolors.bold(arg)) else: error_handler.call_error( Errors.WRONG_PARAM, "Wrong argument or incomplete argument " + bcolors.bold(arg)) break i += 1 return ar
def lets_test(arguments): arguments = argstest.parse_args_test(arguments) if not arguments.valid: error_handler.call_error(Errors.WRONG_PARAM) if arguments.hlp: argstest.print_help() exit(0) elif arguments.version: print("Tester version - " + version) exit(0) if arguments.executable is None: error_handler.call_error(Errors.WRONG_PARAM, "Executable is not set") # init test variables my_test.executable = arguments.executable my_test.only_failed = arguments.onlyFailed my_test.resultDir = arguments.result_dir my_test.timeout = arguments.timeout my_test.exclude = arguments.exclude error_handler.no_warnings = arguments.no_warnings my_test.no_diff = arguments.no_diff # clearing result directory result_name = arguments.result_dir result_path = Path(result_name) if result_path.exists(): if not result_path.is_dir(): error_handler.call_error( Errors.WRONG_PARAM, bcolors.bold(result_name) + " is not a directory") else: rmtree(result_name) if arguments.clean: print(bcolors.note(arguments.result_dir) + " removed") exit(0) if arguments.tests_dir is None and arguments.test is None: error_handler.call_error(Errors.WRONG_INPUT, "No tests") if arguments.test is not None: my_test.run(arguments.test) elif arguments.tests_dir is not None: my_test.run_all(arguments.tests_dir) my_test.show_score()
def parse_args_test(arguments): """ :param arguments arguments from sys.argv """ ar = ArgsTest() if len(arguments) > 0: i = 0 # check for config file while i < len(arguments): if arguments[i] == '-c' or arguments[i] == '--config': i += 1 if i < len(arguments): ar.config = arguments[i] else: error_handler.add_suspicion("Incomplete argument " + bcolors.bold(arguments[i - 1])) ar.valid = False return ar i -= 2 arguments.pop(i + 1) arguments.pop(i + 1) i += 1 # load config if ar.config is not None: load_config(ar, ar.config) else: json_file = Path(config_json) yaml_file = Path(config_yaml) if json_file.is_file() or yaml_file.is_file(): load_config(ar) parse_command_line(ar, arguments) else: load_config(ar) return ar
def test_set_missing(test, test_directory, test_json_file): """ try to recover from missing values :param test test class :param test_directory directory with test :param test_json_file json file """ real_out = "" if test.output is not None: output = test_directory + test.output output_path = Path(output) if not output_path.is_file(): error_handler.call_error(Errors.MISSING_FILE, bcolors.bold(output) + " is missing") with open(output) as input_file: real_out = input_file.read() input_data = None if test.input is not None: input = test_directory + test.input input_path = Path(input) if not input_path.is_file(): error_handler.call_error(Errors.MISSING_FILE, bcolors.bold(input) + " is missing") with open(input) as input_file: input_data = bytes(input_file.read(), "UTF-8") if test.is_not_set(): error_handler.call_warning( bcolors.bold(test_directory) + " is not valid test directory. " + bcolors.bold("Skipping test!")) score['SKIP'].append(test_directory) return None, None, False if test.code is None: test.code = 0 error_handler.warning("key " + bcolors.bold("returnCode") + " in " + test_json_file + " is missing. Using default value: " + bcolors.bold(str(test.code))) error_handler.call_warning() return input_data, real_out, True
import my_test import json from bColors import bcolors from errors import error_handler from errors import Errors from shutil import rmtree from pathlib import Path from importlib import util from os import makedirs spam_spec = util.find_spec("yaml") found = spam_spec is not None yaml_support = True if not found: error_handler.warning("yaml module does not exists. Please install " + bcolors.note(bcolors.bold("pyyaml")) + " package for using yaml module") yaml_support = False else: import yaml version = "2.3.0.2" def lets_test(arguments): arguments = argstest.parse_args_test(arguments) if not arguments.valid: error_handler.call_error(Errors.WRONG_PARAM) if arguments.hlp: argstest.print_help() exit(0)
def read_test_file(test, test_file, data): """ read test.json :param test test class :param test_file json file :param data file data """ for key in data.keys(): if key == 'input': if isinstance(data[key], str): test.input = data[key] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('input') + " in " + bcolors.bold(test_file) + " has to be string") elif key == 'output': if isinstance(data[key], str): test.output = data[key] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('output') + " in " + bcolors.bold(test_file) + " has to be string") elif key == 'comment': if isinstance(data[key], str): test.comment = data[key] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('comment') + " in " + bcolors.bold(test_file) + " has to be string") elif key == 'returnCode': if isinstance(data[key], int): test.code = data[key] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('returnCode') + " in " + bcolors.bold(test_file) + " has to be int") elif key == 'timeout': if isinstance(data[key], int) and data[key] > 0: test.timeout = data[key] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('timeout') + " in " + bcolors.bold(test_file) + " has to be int bigger than 0") elif key == 'args': if isinstance(data[key], list): test.args = data[key] elif isinstance(data[key], str): test.args = [data[key]] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('args') + " in " + bcolors.bold(test_file) + " has to be array, string") elif key == 'expectedFiles': if isinstance(data[key], list): test.expected_files = data[key] elif isinstance(data[key], str): test.expected_files = [data[key]] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('expectedFiles') + " in " + bcolors.bold(test_file) + " has to be array or string") elif key == 'outputFiles': if isinstance(data[key], list): test.output_files = data[key] elif isinstance(data[key], str): test.output_files = [data[key]] else: error_handler.call_error( Errors.INVALID_JSON, bcolors.bold('output_files') + " in " + bcolors.bold(test_file) + " has to be array or string") else: error_handler.warning("Unknown key \'" + bcolors.note(key) + "\' in " + test_file) if len(test.output_files) != len(test.expected_files): error_handler.call_error( Errors.FATAL_ERROR, "Count of files in " + bcolors.bold('outputFiles') + " and " + bcolors.bold('expectedFiles') + " is different")
def run(test_directory): """ run test and store results :param test_directory test directory """ # exclude if check_exclude(test_directory): return if test_directory[-1] != '/': test_directory += '/' test_d = Path(test_directory) if not test_d.is_dir(): error_handler.call_error( Errors.WRONG_INPUT, bcolors.bold(test_directory) + " does not exists or is not folder") test = Test() test_json_file = test_directory + test_file_name + ".json" test_yaml_file = test_directory + test_file_name + ".yaml" test_json_file_path = Path(test_json_file) test_yaml_file_path = Path(test_yaml_file) test.timeout = timeout yaml_exists = test_yaml_file_path.is_file() if not (yaml_exists or test_json_file_path.is_file()): missing_test(test, test_directory) else: test_file = None data = None if yaml_exists: if yaml_support: test_file = test_yaml_file data = read_yaml_test(test_file) else: error_handler.call_error( Errors.YAML_DOESNT_SUPPORT, "Can not read " + bcolors.note(test_file_name + ".yaml")) else: test_file = test_json_file data = read_json_test(test_file) read_test_file(test, test_file, data) input_data, real_out, valid = test_set_missing(test, test_directory, test_json_file) if not valid: return cm = "" if test.args is not None and len(test.args) > 0: cm = ' '.join(test.args) cm = ' ' + cm cmd = (executable + cm) process = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE, stdin=PIPE) out = None err = None runout = False if input_data is None: try: out, err = process.communicate(timeout=test.timeout) except TimeoutExpired: process.kill() runout = True else: try: out, err = process.communicate(input=input_data, timeout=test.timeout) except TimeoutExpired: process.kill() runout = True proc_out = "" if out is not None: proc_out = out.decode('utf-8') proc_err = "" if err is not None: proc_err = err.decode('utf-8') if runout: score["FAIL"].append(test_directory) print(bcolors.note(test_directory) + ":") if test.comment != "": print("\t" + bcolors.warning(test.comment)) print(bcolors.warning("\tRequest timed out")) print(bcolors.fail('\tTest FAILED\n')) return proc_rc = -9 if process.returncode is not None: proc_rc = int(process.returncode) result_code = test.code == proc_rc result_out = real_out == proc_out result_files = None missing_files = list() wrong_files = list() if len(test.output_files) > 0: result_files = True i = 0 while i < len(test.expected_files): exp = test_directory + test.expected_files[i] exp_path = Path(exp) p_out = test.output_files[i] p_out_path = Path(p_out) v = True if not exp_path.is_file(): missing_files.append(exp) result_files = False v = False if not p_out_path.is_file(): missing_files.append(p_out) result_files = False v = False if v: with open(exp, "r") as myfile: exp_data = myfile.read() with open(p_out, "r") as myfile: out_data = myfile.read() if exp_data == out_data: result_files = result_files and True else: result_files = False d_out = out_data.strip().splitlines() d_exp = exp_data.strip().splitlines() diff = list() for line in difflib.unified_diff(d_exp, d_out, test.expected_files[i], p_out, n=0, lineterm=''): diff.append(line) diff = "\n".join(diff) + "\n" wrong_files.append([exp, p_out, diff]) i += 1 result = result_code and result_out if result_files is not None: result = result and result_files if result: score["OK"].append(test_directory) if not only_failed: print(bcolors.note(test_directory) + ":") print(bcolors.success('\tTest OK\n')) else: test_failed(proc_err, proc_out, proc_rc, real_out, result_code, result_out, result_files, test, test_directory, missing_files, wrong_files)
import my_test import json from bColors import bcolors from errors import error_handler from errors import Errors from shutil import rmtree from pathlib import Path from importlib import util from os import makedirs spam_spec = util.find_spec("yaml") found = spam_spec is not None yaml_support = True if not found: error_handler.warning("yaml module does not exists. Please install " + bcolors.note(bcolors.bold("pyyaml")) + " package for using yaml module") yaml_support = False else: import yaml version = "2.3.0.2" def lets_test(arguments): arguments = argstest.parse_args_test(arguments) if not arguments.valid: error_handler.call_error(Errors.WRONG_PARAM) if arguments.hlp: argstest.print_help() exit(0) elif arguments.version:
def print_help(): """ print help """ print(bcolors.OKGREEN + "Tester" + bcolors.ENDC) print_option("-h, --help", "Print help") print_option("-V, --version", "Print version") print_option("-W, --no-warnings", "Silence all test warnings") print_option("-D, --no-diff", "Stop storing stdout diff") print_option("-e, --executable " + bcolors.bold("EXECUTABLE"), "set executable") print_option("-f, --only-failed", "Print only failed tests") print_option("-C, --clean", "Remove result directory and do not run tests") print_option( "-d, --test-dir " + bcolors.bold("TESTS_DIRECTORY"), "Tests will run from " + bcolors.bold("TESTS_DIRECTORY") + "\n" + bcolors.note("NOTE:") + " If " + bcolors.header("--test_dir") + " is set, this will have no effect") print_option( "-t, --test " + bcolors.bold("TEST_DIRECTORY"), "Only test from " + bcolors.bold("TEST_DIRECTORY") + " will run") print_option( "-R, --result-dir " + bcolors.bold("RESULT_DIRECTORY"), "Output directory\n" + bcolors.note("NOTE:") + " Default value is " + bcolors.bold("result")) print_option( "-T, --timeout " + bcolors.bold("SECONDS"), "Maximum time for test\n" + bcolors.note("NOTE:") + " Default value is " + bcolors.bold("15")) print_option( "-E, --exclude " + bcolors.bold("REGEX"), "Directories matching " + bcolors.bold("REGEX") + " will be excluded\n" + bcolors.note("NOTE:") + " When using command line arguments you can use only one regex patter. If you want to use more, " "you have to use it in configuration file.") print_option( "-c, --config " + bcolors.bold("CONFIG"), "will use " + bcolors.bold("CONFIG") + " file as configuration\n" + bcolors.note("NOTE:") + " configuration from json file can be overwritten with command line arguments" ) print_option( "init " + bcolors.bold("OPTIONS"), "Generate " + bcolors.bold("config file") + " template\n" + bcolors.note("NOTE:") + " for more info use " + bcolors.bold("tester init -h")) print_option( "testinit " + bcolors.bold("OPTIONS"), "Generate " + bcolors.bold("test config file") + " template\n" + bcolors.note("NOTE:") + " for more info use " + bcolors.bold("tester testinit -h"))