def missing_test(test, test_directory):
"""
recovery from missing json
:param test test class
:param test_directory test directory
"""
error_handler.add_suspicion(
bcolors.bold(test_file_name + '.json or ' + test_file_name + '.yaml') +
" is missing in " + bcolors.bold(test_directory) +
". Trying to recover with default values")
default_input_name = "input.in"
input = test_directory + default_input_name
input_path = Path(input)
if not input_path.is_file():
error_handler.add_suspicion(bcolors.bold(input) + " is missing")
else:
test.input = default_input_name
default_output_name = "output.out"
output = test_directory + default_output_name
input_path = Path(output)
if not input_path.is_file():
error_handler.add_suspicion(bcolors.bold(output) + " is missing")
else:
test.output = default_output_name
def print_testinit_help():
"""
print help
"""
print(bcolors.OKGREEN + "Tester init" + bcolors.ENDC)
print(
"Generates test config file template in yaml format. For json format use "
+ bcolors.bold("testinit --json"))
print_option("testinit -h", "Print help")
print_option("testinit --json", "Generates micro-template in json format")
print_option("testinit --all, testinit --json --all", "Generates template")
print_option(
"testinit --force", "Generates template even if file exists\n" +
bcolors.note("NOTE:") + " If you want to use it with " +
bcolors.header("--all") + " write " + bcolors.header("--all --force"))
print(
bcolors.note("Note:") +
"\n\t You can use following options at the end of command to fill template"
)
print_option(
"-n, --name FOLDER",
"Create " + bcolors.bold("FOLDER") + " and generated file save in it")
print_option("-i, --input FILE", "File with stdin")
print_option("-o, --output FILE", "File with stdout")
print_option("-r, --return-code NUMBER", "Expected return code")
print_option("-c, --comment", "Test comment")
print_option("-T, --timeout", "Maximum time for test")
def parse_command_line(ar, arguments):
i = 0
while i < len(arguments):
arg = arguments[i]
if arg == '-h' or arg == '--help':
ar.hlp = True
break
if arg == '-V' or arg == '--version':
ar.version = True
break
elif arg == '-f' or arg == '--only-failed':
ar.onlyFailed = True
elif arg == '-W' or arg == '--no-warnings':
ar.no_warnings = True
elif arg == '-D' or arg == '--no-diff':
ar.no_diff = True
elif arg == '-C' or arg == '--clean':
ar.clean = True
else:
if i == len(arguments) - 1:
error_handler.add_suspicion(
"Wrong argument or incomplete argument " +
bcolors.bold(arg))
ar.valid = False
break
next_arg = arguments[i + 1]
i += 1
if arg == '-d' or arg == '--test-dir':
ar.tests_dir = next_arg
if ar.tests_dir[-1] != '/':
ar.tests_dir += '/'
elif arg == '-t' or arg == '--test':
ar.test = next_arg
elif arg == '-T' or arg == '--timeout':
if is_int(next_arg):
ar.timeout = int(next_arg)
else:
error_handler.add_suspicion("Wrong argument type after " +
bcolors.bold(arg))
elif arg == '-R' or arg == '--result-dir':
ar.result_dir = next_arg
if ar.result_dir[-1] != '/':
ar.result_dir += '/'
elif arg == '-e' or arg == '--executable':
ar.executable = next_arg
elif arg == '-E' or arg == '--exclude':
if next_arg != "":
ar.exclude = [next_arg]
else:
error_handler.add_suspicion(
"Wrong argument or incomplete argument " +
bcolors.bold(arg))
ar.valid = False
break
i += 1
def load_config(ar, config_file=None):
"""
:param ar arguments class
:param config_file name of config file
"""
data = list()
if config_file is None:
config_file = config_yaml
my_file = Path(config_file)
if my_file.is_file():
if yaml_support:
data = load_yaml(config_file)
else:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT,
"Can't use " + config_yaml)
else:
config_file = config_json
my_file = Path(config_file)
if my_file.is_file():
data = load_json(config_file)
else:
error_handler.add_suspicion(
bcolors.bold(config_yaml) + " or " +
bcolors.bold(config_json) +
" is missing or can\'t be opened.")
ar.valid = False
return
else:
my_file = Path(config_file)
if my_file.is_file():
file_type = config_file.split('.')[-1]
if file_type == 'yaml':
if not yaml_support:
error_handler.call_error(Errors.YAML_DOESNT_SUPPORT)
else:
data = load_yaml(config_file)
elif file_type == 'json':
data = load_json(config_file)
else:
error_handler.add_suspicion(
bcolors.bold(config_file) +
" have to end with .json or .yaml")
ar.valid = False
return
else:
error_handler.add_suspicion(
bcolors.bold(config_file) + " is missing or can\'t be opened.")
ar.valid = False
return
parse_config(ar, config_file, data)
def print_init_help():
"""
print help
"""
print(bcolors.OKGREEN + "Tester init" + bcolors.ENDC)
print(
"Generates config file template in yaml format. For json format use " +
bcolors.bold("init --json"))
print_option("init -h, init --help", "Print help")
print_option("init --json", "Generates micro-template in json format")
print_option("init --all, init --json --all", "Generates template")
print_option(
"init --force", "Generates template even if file exists\n" +
bcolors.note("NOTE:") + " If you want to use it with " +
bcolors.header("--all") + " write " + bcolors.header("--all --force"))
print(
bcolors.note("Note:") +
"\n\t You can use following options at the end of command to fill template"
)
print_option("-W, --no-warnings", "Silence all test warnings")
print_option("-D, --no-diff", "Stop storing stdout diff")
print_option("-e, --executable " + bcolors.bold("EXECUTABLE"),
"set executable")
print_option("-f, --only-failed", "Print only failed tests")
print_option("-d, --test-dir " + bcolors.bold("TESTS_DIRECTORY"),
"Tests will run from " + bcolors.bold("TESTS_DIRECTORY"))
print_option("-R, --result-dir " + bcolors.bold("RESULT_DIRECTORY"),
"Output directory")
print_option("-T, --timeout " + bcolors.bold("SECONDS"),
"Maximum time for test\n" + bcolors.note("NOTE:"))
print_option(
"-E, --exclude " + bcolors.bold("REGEX"), "Directories matching " +
bcolors.bold("REGEX") + " will be excluded\n" + bcolors.note("NOTE:") +
" When using command line arguments you can use only one regex patter. If you want to use more, "
"you have to use it in configuration file.")
def check_exclude(test_directory):
"""
check if directory is in excluded
:param test_directory test directory
"""
for r in exclude:
m = re.compile(r)
if m.search(test_directory) is not None:
error_handler.call_warning(
bcolors.bold(test_directory) + " is excluded. " +
bcolors.bold("Skipping test!"))
score['SKIP'].append(test_directory)
return True
return False
def call_error(self, error_code, msg=None):
error_name = "Undefined error"
pom_msg = "use -h or --help to print help"
more_msg = None
if msg is not None:
pom_msg = msg
if error_code == Errors.WRONG_PARAM:
error_name = "Invalid arguments!"
elif error_code == Errors.INVALID_JSON:
error_name = "Invalid input json"
elif error_code == Errors.WRONG_INPUT:
error_name = "Invalid input"
elif error_code == Errors.MISSING_FILE:
error_name = "Missing file"
elif error_code == Errors.FATAL_ERROR:
error_name = "Fatal error"
elif error_code == Errors.YAML_DOESNT_SUPPORT:
error_name = "Yaml does not supported"
more_msg = "Please install " + bcolors.note(bcolors.bold("pyyaml")) + " package to use yaml."
if more_msg is not None:
stderr.write(bcolors.fail(error_name) + "\n\t" + bcolors.warning(more_msg) + "\n\t" + pom_msg + "\n")
else:
stderr.write(bcolors.fail(error_name) + "\n\t" + pom_msg + "\n")
if len(self.suspicions) > 0:
stderr.write(bcolors.note("\nNOTE:\n"))
for m in self.suspicions:
stderr.write("\t" + m + "\n")
self.clear_suspicions()
exit(error_code.value)
def read_yaml_test(yaml_file):
with open(yaml_file) as data_file:
data = None
try:
data = yaml.load(data_file)
except yaml.YAMLError:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold(yaml_file) + " has invalid syntax")
return data
def read_json_test(json_file):
with open(json_file) as data_file:
data = None
try:
data = json.load(data_file)
except ValueError:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold(json_file) + " has invalid syntax")
return data
def parse_command_line_init_test(arguments, ar=Test()):
i = 0
while i < len(arguments):
arg = arguments[i]
if i == len(arguments) - 1:
error_handler.call_error(
Errors.WRONG_PARAM,
"Wrong argument or incomplete argument " + bcolors.bold(arg))
break
next_arg = arguments[i + 1]
i += 1
if arg == '-i' or arg == '--input':
ar.input = next_arg
elif arg == '-o' or arg == '--output':
ar.output = next_arg
elif arg == '-n' or arg == '--name':
ar.name = next_arg
elif arg == '-r' or arg == '--return-code':
if is_int(next_arg):
ar.code = int(next_arg)
else:
error_handler.call_error(
Errors.WRONG_PARAM,
"Incompatible value for " + bcolors.bold(arg))
elif arg == '-c' or arg == '--comment':
ar.comment = next_arg
elif arg == '-T' or arg == '--timeout':
if is_int(next_arg):
ar.timeout = int(next_arg)
else:
error_handler.call_error(
Errors.WRONG_PARAM,
"Incompatible value for " + bcolors.bold(arg))
else:
error_handler.call_error(
Errors.WRONG_PARAM,
"Wrong argument or incomplete argument " + bcolors.bold(arg))
break
i += 1
return ar
def lets_test(arguments):
arguments = argstest.parse_args_test(arguments)
if not arguments.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arguments.hlp:
argstest.print_help()
exit(0)
elif arguments.version:
print("Tester version - " + version)
exit(0)
if arguments.executable is None:
error_handler.call_error(Errors.WRONG_PARAM, "Executable is not set")
# init test variables
my_test.executable = arguments.executable
my_test.only_failed = arguments.onlyFailed
my_test.resultDir = arguments.result_dir
my_test.timeout = arguments.timeout
my_test.exclude = arguments.exclude
error_handler.no_warnings = arguments.no_warnings
my_test.no_diff = arguments.no_diff
# clearing result directory
result_name = arguments.result_dir
result_path = Path(result_name)
if result_path.exists():
if not result_path.is_dir():
error_handler.call_error(
Errors.WRONG_PARAM,
bcolors.bold(result_name) + " is not a directory")
else:
rmtree(result_name)
if arguments.clean:
print(bcolors.note(arguments.result_dir) + " removed")
exit(0)
if arguments.tests_dir is None and arguments.test is None:
error_handler.call_error(Errors.WRONG_INPUT, "No tests")
if arguments.test is not None:
my_test.run(arguments.test)
elif arguments.tests_dir is not None:
my_test.run_all(arguments.tests_dir)
my_test.show_score()
def parse_args_test(arguments):
"""
:param arguments arguments from sys.argv
"""
ar = ArgsTest()
if len(arguments) > 0:
i = 0
# check for config file
while i < len(arguments):
if arguments[i] == '-c' or arguments[i] == '--config':
i += 1
if i < len(arguments):
ar.config = arguments[i]
else:
error_handler.add_suspicion("Incomplete argument " +
bcolors.bold(arguments[i - 1]))
ar.valid = False
return ar
i -= 2
arguments.pop(i + 1)
arguments.pop(i + 1)
i += 1
# load config
if ar.config is not None:
load_config(ar, ar.config)
else:
json_file = Path(config_json)
yaml_file = Path(config_yaml)
if json_file.is_file() or yaml_file.is_file():
load_config(ar)
parse_command_line(ar, arguments)
else:
load_config(ar)
return ar
def test_set_missing(test, test_directory, test_json_file):
"""
try to recover from missing values
:param test test class
:param test_directory directory with test
:param test_json_file json file
"""
real_out = ""
if test.output is not None:
output = test_directory + test.output
output_path = Path(output)
if not output_path.is_file():
error_handler.call_error(Errors.MISSING_FILE,
bcolors.bold(output) + " is missing")
with open(output) as input_file:
real_out = input_file.read()
input_data = None
if test.input is not None:
input = test_directory + test.input
input_path = Path(input)
if not input_path.is_file():
error_handler.call_error(Errors.MISSING_FILE,
bcolors.bold(input) + " is missing")
with open(input) as input_file:
input_data = bytes(input_file.read(), "UTF-8")
if test.is_not_set():
error_handler.call_warning(
bcolors.bold(test_directory) + " is not valid test directory. " +
bcolors.bold("Skipping test!"))
score['SKIP'].append(test_directory)
return None, None, False
if test.code is None:
test.code = 0
error_handler.warning("key " + bcolors.bold("returnCode") + " in " +
test_json_file +
" is missing. Using default value: " +
bcolors.bold(str(test.code)))
error_handler.call_warning()
return input_data, real_out, True
import my_test
import json
from bColors import bcolors
from errors import error_handler
from errors import Errors
from shutil import rmtree
from pathlib import Path
from importlib import util
from os import makedirs
spam_spec = util.find_spec("yaml")
found = spam_spec is not None
yaml_support = True
if not found:
error_handler.warning("yaml module does not exists. Please install " +
bcolors.note(bcolors.bold("pyyaml")) +
" package for using yaml module")
yaml_support = False
else:
import yaml
version = "2.3.0.2"
def lets_test(arguments):
arguments = argstest.parse_args_test(arguments)
if not arguments.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arguments.hlp:
argstest.print_help()
exit(0)
def read_test_file(test, test_file, data):
"""
read test.json
:param test test class
:param test_file json file
:param data file data
"""
for key in data.keys():
if key == 'input':
if isinstance(data[key], str):
test.input = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('input') + " in " + bcolors.bold(test_file) +
" has to be string")
elif key == 'output':
if isinstance(data[key], str):
test.output = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('output') + " in " + bcolors.bold(test_file) +
" has to be string")
elif key == 'comment':
if isinstance(data[key], str):
test.comment = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('comment') + " in " +
bcolors.bold(test_file) + " has to be string")
elif key == 'returnCode':
if isinstance(data[key], int):
test.code = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('returnCode') + " in " +
bcolors.bold(test_file) + " has to be int")
elif key == 'timeout':
if isinstance(data[key], int) and data[key] > 0:
test.timeout = data[key]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('timeout') + " in " +
bcolors.bold(test_file) + " has to be int bigger than 0")
elif key == 'args':
if isinstance(data[key], list):
test.args = data[key]
elif isinstance(data[key], str):
test.args = [data[key]]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('args') + " in " + bcolors.bold(test_file) +
" has to be array, string")
elif key == 'expectedFiles':
if isinstance(data[key], list):
test.expected_files = data[key]
elif isinstance(data[key], str):
test.expected_files = [data[key]]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('expectedFiles') + " in " +
bcolors.bold(test_file) + " has to be array or string")
elif key == 'outputFiles':
if isinstance(data[key], list):
test.output_files = data[key]
elif isinstance(data[key], str):
test.output_files = [data[key]]
else:
error_handler.call_error(
Errors.INVALID_JSON,
bcolors.bold('output_files') + " in " +
bcolors.bold(test_file) + " has to be array or string")
else:
error_handler.warning("Unknown key \'" + bcolors.note(key) +
"\' in " + test_file)
if len(test.output_files) != len(test.expected_files):
error_handler.call_error(
Errors.FATAL_ERROR,
"Count of files in " + bcolors.bold('outputFiles') + " and " +
bcolors.bold('expectedFiles') + " is different")
def run(test_directory):
"""
run test and store results
:param test_directory test directory
"""
# exclude
if check_exclude(test_directory):
return
if test_directory[-1] != '/':
test_directory += '/'
test_d = Path(test_directory)
if not test_d.is_dir():
error_handler.call_error(
Errors.WRONG_INPUT,
bcolors.bold(test_directory) + " does not exists or is not folder")
test = Test()
test_json_file = test_directory + test_file_name + ".json"
test_yaml_file = test_directory + test_file_name + ".yaml"
test_json_file_path = Path(test_json_file)
test_yaml_file_path = Path(test_yaml_file)
test.timeout = timeout
yaml_exists = test_yaml_file_path.is_file()
if not (yaml_exists or test_json_file_path.is_file()):
missing_test(test, test_directory)
else:
test_file = None
data = None
if yaml_exists:
if yaml_support:
test_file = test_yaml_file
data = read_yaml_test(test_file)
else:
error_handler.call_error(
Errors.YAML_DOESNT_SUPPORT,
"Can not read " + bcolors.note(test_file_name + ".yaml"))
else:
test_file = test_json_file
data = read_json_test(test_file)
read_test_file(test, test_file, data)
input_data, real_out, valid = test_set_missing(test, test_directory,
test_json_file)
if not valid:
return
cm = ""
if test.args is not None and len(test.args) > 0:
cm = ' '.join(test.args)
cm = ' ' + cm
cmd = (executable + cm)
process = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE, stdin=PIPE)
out = None
err = None
runout = False
if input_data is None:
try:
out, err = process.communicate(timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
else:
try:
out, err = process.communicate(input=input_data,
timeout=test.timeout)
except TimeoutExpired:
process.kill()
runout = True
proc_out = ""
if out is not None:
proc_out = out.decode('utf-8')
proc_err = ""
if err is not None:
proc_err = err.decode('utf-8')
if runout:
score["FAIL"].append(test_directory)
print(bcolors.note(test_directory) + ":")
if test.comment != "":
print("\t" + bcolors.warning(test.comment))
print(bcolors.warning("\tRequest timed out"))
print(bcolors.fail('\tTest FAILED\n'))
return
proc_rc = -9
if process.returncode is not None:
proc_rc = int(process.returncode)
result_code = test.code == proc_rc
result_out = real_out == proc_out
result_files = None
missing_files = list()
wrong_files = list()
if len(test.output_files) > 0:
result_files = True
i = 0
while i < len(test.expected_files):
exp = test_directory + test.expected_files[i]
exp_path = Path(exp)
p_out = test.output_files[i]
p_out_path = Path(p_out)
v = True
if not exp_path.is_file():
missing_files.append(exp)
result_files = False
v = False
if not p_out_path.is_file():
missing_files.append(p_out)
result_files = False
v = False
if v:
with open(exp, "r") as myfile:
exp_data = myfile.read()
with open(p_out, "r") as myfile:
out_data = myfile.read()
if exp_data == out_data:
result_files = result_files and True
else:
result_files = False
d_out = out_data.strip().splitlines()
d_exp = exp_data.strip().splitlines()
diff = list()
for line in difflib.unified_diff(d_exp,
d_out,
test.expected_files[i],
p_out,
n=0,
lineterm=''):
diff.append(line)
diff = "\n".join(diff) + "\n"
wrong_files.append([exp, p_out, diff])
i += 1
result = result_code and result_out
if result_files is not None:
result = result and result_files
if result:
score["OK"].append(test_directory)
if not only_failed:
print(bcolors.note(test_directory) + ":")
print(bcolors.success('\tTest OK\n'))
else:
test_failed(proc_err, proc_out, proc_rc, real_out, result_code,
result_out, result_files, test, test_directory,
missing_files, wrong_files)
import my_test
import json
from bColors import bcolors
from errors import error_handler
from errors import Errors
from shutil import rmtree
from pathlib import Path
from importlib import util
from os import makedirs
spam_spec = util.find_spec("yaml")
found = spam_spec is not None
yaml_support = True
if not found:
error_handler.warning("yaml module does not exists. Please install " +
bcolors.note(bcolors.bold("pyyaml")) + " package for using yaml module")
yaml_support = False
else:
import yaml
version = "2.3.0.2"
def lets_test(arguments):
arguments = argstest.parse_args_test(arguments)
if not arguments.valid:
error_handler.call_error(Errors.WRONG_PARAM)
if arguments.hlp:
argstest.print_help()
exit(0)
elif arguments.version:
def print_help():
"""
print help
"""
print(bcolors.OKGREEN + "Tester" + bcolors.ENDC)
print_option("-h, --help", "Print help")
print_option("-V, --version", "Print version")
print_option("-W, --no-warnings", "Silence all test warnings")
print_option("-D, --no-diff", "Stop storing stdout diff")
print_option("-e, --executable " + bcolors.bold("EXECUTABLE"),
"set executable")
print_option("-f, --only-failed", "Print only failed tests")
print_option("-C, --clean", "Remove result directory and do not run tests")
print_option(
"-d, --test-dir " + bcolors.bold("TESTS_DIRECTORY"),
"Tests will run from " + bcolors.bold("TESTS_DIRECTORY") + "\n" +
bcolors.note("NOTE:") + " If " + bcolors.header("--test_dir") +
" is set, this will have no effect")
print_option(
"-t, --test " + bcolors.bold("TEST_DIRECTORY"),
"Only test from " + bcolors.bold("TEST_DIRECTORY") + " will run")
print_option(
"-R, --result-dir " + bcolors.bold("RESULT_DIRECTORY"),
"Output directory\n" + bcolors.note("NOTE:") + " Default value is " +
bcolors.bold("result"))
print_option(
"-T, --timeout " + bcolors.bold("SECONDS"), "Maximum time for test\n" +
bcolors.note("NOTE:") + " Default value is " + bcolors.bold("15"))
print_option(
"-E, --exclude " + bcolors.bold("REGEX"), "Directories matching " +
bcolors.bold("REGEX") + " will be excluded\n" + bcolors.note("NOTE:") +
" When using command line arguments you can use only one regex patter. If you want to use more, "
"you have to use it in configuration file.")
print_option(
"-c, --config " + bcolors.bold("CONFIG"),
"will use " + bcolors.bold("CONFIG") + " file as configuration\n" +
bcolors.note("NOTE:") +
" configuration from json file can be overwritten with command line arguments"
)
print_option(
"init " + bcolors.bold("OPTIONS"), "Generate " +
bcolors.bold("config file") + " template\n" + bcolors.note("NOTE:") +
" for more info use " + bcolors.bold("tester init -h"))
print_option(
"testinit " + bcolors.bold("OPTIONS"),
"Generate " + bcolors.bold("test config file") + " template\n" +
bcolors.note("NOTE:") + " for more info use " +
bcolors.bold("tester testinit -h"))
