def __init__(self, yaml_test, settings, filename, title):
self.settings = settings
self.filename = filename
self.name = yaml_test.get('name', title)
self.engine = yaml_test.get('engine', 'engine.py:ExecutionEngine')
self.description = yaml_test.get('description')
self.preconditions = yaml_test.get('preconditions', {})
self.tags = yaml_test.get('tags')
self.features = yaml_test.get('features')
self.scenario = Scenario(yaml_test.get('scenario', []), underscore_case_steps=self.settings.get("underscore case steps", False))
if re.compile("^(.*?)\:(.*?)$").match(self.engine) is None:
raise RuntimeError("ERROR : engine should be of the form 'engine_filename.py:ClassName'")
else:
module_source = path.abspath(path.join(self.settings['engine_folder'], self.engine.split(":")[0]))
if not path.exists(module_source):
raise RuntimeError("ERROR : engine filename '{}' not found.".format(self.engine.split(':')[0]))
engine_class_name = self.engine.split(":")[1]
engine_module = imp.load_source("engine", module_source)
if len([x for x in inspect.getmembers(engine_module) if x[0] == engine_class_name]) == 0:
raise RuntimeError("ERROR : Class {} not found in engine.".format(engine_class_name))
self.engine_class = [x for x in inspect.getmembers(engine_module) if x[0] == engine_class_name][0][1]
if self.name[0].isdigit():
raise RuntimeError("ERROR : Test names cannot start with a digit.")
if "test" in self.name.lower():
warn("WARNING: The word 'test' should not appear in your test name - it is redundant.\n")
def _overwrite_settings_with_yaml(self, settings_filename):
"""Load YAML and overwrite """
if exists(settings_filename):
with open(settings_filename) as settingsfile_handle:
settingsfile_contents = settingsfile_handle.read()
try:
self.settings_dict.update(pyyaml.load(settingsfile_contents))
except pyyaml.parser.MarkedYAMLError as error:
warn("YAML parser error in {}:\n{}\nError:{}\n".format(
settings_filename, settingsfile_contents, str(error),
))
exit(1)
except ValueError:
warn("YAML parser error in {}. Should be associative array not list:\n\n{}\n".format(
settings_filename, settingsfile_contents,
))
exit(1)
def __init__(self, engine_directory, override_settings_filename, extra_string):
if exists(join(engine_directory, "settings.yml")):
warn((
"settings.yml as a 'default settings' file has been deprecated. "
"Rename to all.settings instead:\n\n"
"See here for more details on the change : \n"
"https://hitchtest.readthedocs.org/en/latest/faq/why_was_hitch_behavior_changed.html\n"
))
exit(1)
self.settings_dict = {}
full_override_settings_filename = None if override_settings_filename is None else join(engine_directory, override_settings_filename)
if exists(join(engine_directory, "all.settings")):
self._overwrite_settings_with_yaml(join(engine_directory, "all.settings"))
if override_settings_filename is not None:
if not exists(override_settings_filename):
warn("Settings file '{}' could not be found!\n".format(override_settings_filename))
exit(1)
else:
# Load settings from specified file, if it exists
self._overwrite_settings_with_yaml(override_settings_filename)
# Load extra settings from command line JSON and overwrite what's already set
if extra_string is not None:
try:
self.settings_dict.update(json.loads(extra_string).items())
except ValueError as error:
warn("""{} in:\n ==> --extra '{}' (must be valid JSON)\n""".format(str(error), extra_string))
exit(1)
except AttributeError:
warn("""Error in:\n ==> --extra '{}' (must be valid JSON and not a list)\n""".format(extra_string))
exit(1)
def cli(filenames, yaml, quiet, tags, settings, extra):
"""Run test files or entire directories containing .test files."""
# .hitch/virtualenv/bin/python <- 4 directories up from where the python exec resides
engine_folder = path.abspath(path.join(executable, "..", "..", "..", ".."))
filenames = [path.abspath(path.relpath(filename, engine_folder)) for filename in filenames]
chdir(engine_folder)
if quiet:
warn((
"The --quiet switch has been deprecated. You can make your tests run quietly "
"by setting the property quiet to True via --extra or in a settings file.\n\n"
"See here for more details on the change : \n"
"https://hitchtest.readthedocs.org/en/latest/faq/why_was_hitch_behavior_changed.html\n"
))
exit(1)
#new_default_settings_filename = path.join(engine_folder, "all.settings")
#settings_filename = None if settings is None else path.join(engine_folder, settings)
#if path.exists(new_default_settings_filename):
#_overwrite_settings_with_yaml(new_default_settings_filename, settings_dict)
#if settings_filename is not None:
#if not path.exists(settings_filename):
#warn("Settings file '{}' could not be found!\n".format(settings_filename))
#exit(1)
#else:
## Load settings from specified file, if it exists
#_overwrite_settings_with_yaml(settings_filename, settings_dict)
## Load extra settings from command line JSON and overwrite what's already set
#if extra is not None:
#try:
#settings_dict.update(json.loads(extra).items())
#except ValueError as error:
#warn("""{} in:\n ==> --extra '{}' (must be valid JSON)\n""".format(str(error), extra))
#exit(1)
#except AttributeError:
#warn("""Error in:\n ==> --extra '{}' (must be valid JSON and not a list)\n""".format(extra))
#exit(1)
settings_dict = Settings(engine_folder, settings, extra)
settings_dict['engine_folder'] = engine_folder
if 'quiet' not in settings_dict:
settings_dict['quiet'] = False
if len(filenames) == 0:
warn("No tests specified.\n")
exit(1)
# Get list of files from specified files/directories
matches = []
test_not_found = False
for filename in filenames:
if not path.exists(filename):
warn("Test '{}' not found.\n".format(filename))
test_not_found = True
if path.isdir(filename):
for root, dirnames, filenames_in_dir in walk(filename):
for filename_in_dir in fnmatch.filter(filenames_in_dir, '*.test'):
if ".hitch" not in root: # Ignore everything in .hitch
matches.append(path.join(root, filename_in_dir))
else:
matches.append(filename)
if test_not_found:
exit(1)
# Get list of modules from matching directly specified files from command line
# and indirectly (in the directories of) directories specified from cmd line
test_modules = []
for filename in matches:
if filename.endswith(".test"):
test_modules.append(module.Module(filename, settings_dict))
else:
warn(
"Tests must have the extension .test"
"- '{}' doesn't have that extension\n".format(filename)
)
exit(1)
test_suite = suite.Suite(test_modules, settings_dict, tags)
if yaml:
test_suite.printyaml()
else:
returned_results = test_suite.run(quiet=quiet)
# Lines must be split to prevent stdout blocking
result_lines = returned_results.to_template(
template=settings_dict.get('results_template', None)
).split('\n')
for line in result_lines:
log("{}\n".format(line))
exit(1 if len(returned_results.failures()) > 0 else 0)
def __init__(self, filename, settings):
self.filename = path.realpath(filename)
self.engine_folder = settings['engine_folder']
self.name = path.split(self.filename)[1].replace(".test", "")
self.title = self.name.replace("_", " ").title()
self.dirname = path.dirname(self.filename)
if settings is None:
self.settings = {}
else:
self.settings = settings
with open(filename, "r") as file_handle:
if file_handle.read() == "":
warn("{0} is an empty file.\n".format(filename))
sys.exit(1)
env = Environment()
env.loader = FileSystemLoader(path.split(filename)[0])
try:
tmpl = env.get_template(path.split(filename)[1])
except exceptions.TemplateError as error:
warn("Jinja2 template error in '{}' on line {}:\n==> {}\n".format(
error.filename, error.lineno, str(error)
))
sys.exit(1)
self.test_yaml_text = tmpl.render(**self.settings)
if self.test_yaml_text == "":
warn((
"{0} rendered as an empty file. "
"There is probably a problem with your jinja2 template.\n"
).format(filename))
sys.exit(1)
self.tests = []
try:
module_yaml_as_dict = yaml.load(self.test_yaml_text)
except yaml.parser.MarkedYAMLError as error:
warn("YAML parser error in {}:\n".format(filename))
warn(str(error))
warn("\n")
sys.exit(1)
try:
core = Core(source_data=module_yaml_as_dict, schema_files=[HITCHSCHEMA])
core.validate(raise_exception=True)
except PyKwalifyException as error:
warn("YAML validation error in {}:\n==> {}\n".format(filename, error.msg))
sys.exit(1)
if len(module_yaml_as_dict) == 1:
self.multiple_tests = False
self.tests = [Test(module_yaml_as_dict[0], self.settings, self.filename, self.title)]
else:
self.multiple_tests = True
for i, test_yaml in enumerate(module_yaml_as_dict, 1):
self.tests.append(
Test(test_yaml, self.settings, self.filename, "{} {}".format(self.title, i))
)
def run(self, quiet=False):
"""Run all tests in the defined suite of modules."""
tests = self.tests()
failedfast = False
result_list = []
for test in tests:
if quiet:
hijacked_stdout = sys.stdout
hijacked_stderr = sys.stderr
sys.stdout = open(path.join(self.settings['engine_folder'], ".hitch", "test.out"), "ab", 0)
sys.stderr = open(path.join(self.settings['engine_folder'], ".hitch", "test.err"), "ab", 0)
def run_test_in_separate_process(file_descriptor_stdin, result_queue):
"""Change process group, run test and return result via a queue."""
orig_pgid = os.getpgrp()
os.setpgrp()
result_queue.put("pgrp")
if not quiet:
sys.stdin = os.fdopen(file_descriptor_stdin)
result = test.run()
result_queue.put(result)
if not quiet:
try:
os.tcsetpgrp(file_descriptor_stdin, orig_pgid)
except OSError as error:
if error.args[0] == 25:
pass
if not quiet:
try:
orig_stdin_termios = termios.tcgetattr(sys.stdin.fileno())
except termios.error:
orig_stdin_termios = None
orig_stdin_fileno = sys.stdin.fileno()
orig_pgid = os.getpgrp()
file_descriptor_stdin = sys.stdin.fileno()
result_queue = multiprocessing.Queue()
# Start new process to run test in, to isolate it from future test runs
test_process = multiprocessing.Process(
target=run_test_in_separate_process,
args=(file_descriptor_stdin, result_queue)
)
test_timed_out = False
test_process.start()
# Ignore all exit signals but pass them on
signal_pass_on_to_separate_process_group(test_process.pid)
# Wait until PGRP is changed
result_queue.get()
# Make stdin go to the test process so that you can use ipython, etc.
if not quiet:
try:
os.tcsetpgrp(file_descriptor_stdin, os.getpgid(test_process.pid))
except OSError as error:
if error.args[0] == 25:
pass
# Wait until process has finished
proc = psutil.Process(test_process.pid)
test_timeout = self.settings.get("test_timeout", None)
test_shutdown_timeout = self.settings.get("test_shutdown_timeout", 10)
try:
proc.wait(timeout=test_timeout)
except psutil.TimeoutExpired:
test_timed_out = True
proc.send_signal(signal.SIGTERM)
try:
proc.wait(timeout=test_shutdown_timeout)
except psutil.TimeoutExpired:
for child in proc.get_children(recursive=True):
child.send_signal(signal.SIGKILL)
proc.send_signal(signal.SIGKILL)
# Take back signal handling from test running code
signals_trigger_exit()
try:
result = result_queue.get_nowait()
except multiprocessing.queues.Empty:
result = Result(test, True, 0.0)
if test_timed_out:
result.aborted = False
result_list.append(result)
if not quiet and orig_stdin_termios is not None:
try:
termios.tcsetattr(orig_stdin_fileno, termios.TCSANOW, orig_stdin_termios)
except termios.error as err:
# I/O error caused by another test stopping this one
if err[0] == 5:
pass
if quiet:
sys.stdout = hijacked_stdout
sys.stderr = hijacked_stderr
if quiet and result is not None:
if result.failure:
warn("X")
else:
warn(".")
if result.aborted:
warn("Aborted\n")
sys.exit(1)
if self.settings.get('failfast', False) and result.failure:
failedfast = True
break
return Results(result_list, failedfast, self.settings.get('colorless', False))
