def __init__(self, yaml_test, settings, filename, title): self.settings = settings self.filename = filename self.name = yaml_test.get('name', title) self.engine = yaml_test.get('engine', 'engine.py:ExecutionEngine') self.description = yaml_test.get('description') self.preconditions = yaml_test.get('preconditions', {}) self.tags = yaml_test.get('tags') self.features = yaml_test.get('features') self.scenario = Scenario(yaml_test.get('scenario', []), underscore_case_steps=self.settings.get("underscore case steps", False)) if re.compile("^(.*?)\:(.*?)$").match(self.engine) is None: raise RuntimeError("ERROR : engine should be of the form 'engine_filename.py:ClassName'") else: module_source = path.abspath(path.join(self.settings['engine_folder'], self.engine.split(":")[0])) if not path.exists(module_source): raise RuntimeError("ERROR : engine filename '{}' not found.".format(self.engine.split(':')[0])) engine_class_name = self.engine.split(":")[1] engine_module = imp.load_source("engine", module_source) if len([x for x in inspect.getmembers(engine_module) if x[0] == engine_class_name]) == 0: raise RuntimeError("ERROR : Class {} not found in engine.".format(engine_class_name)) self.engine_class = [x for x in inspect.getmembers(engine_module) if x[0] == engine_class_name][0][1] if self.name[0].isdigit(): raise RuntimeError("ERROR : Test names cannot start with a digit.") if "test" in self.name.lower(): warn("WARNING: The word 'test' should not appear in your test name - it is redundant.\n")
def _overwrite_settings_with_yaml(self, settings_filename): """Load YAML and overwrite """ if exists(settings_filename): with open(settings_filename) as settingsfile_handle: settingsfile_contents = settingsfile_handle.read() try: self.settings_dict.update(pyyaml.load(settingsfile_contents)) except pyyaml.parser.MarkedYAMLError as error: warn("YAML parser error in {}:\n{}\nError:{}\n".format( settings_filename, settingsfile_contents, str(error), )) exit(1) except ValueError: warn("YAML parser error in {}. Should be associative array not list:\n\n{}\n".format( settings_filename, settingsfile_contents, )) exit(1)
def __init__(self, engine_directory, override_settings_filename, extra_string): if exists(join(engine_directory, "settings.yml")): warn(( "settings.yml as a 'default settings' file has been deprecated. " "Rename to all.settings instead:\n\n" "See here for more details on the change : \n" "https://hitchtest.readthedocs.org/en/latest/faq/why_was_hitch_behavior_changed.html\n" )) exit(1) self.settings_dict = {} full_override_settings_filename = None if override_settings_filename is None else join(engine_directory, override_settings_filename) if exists(join(engine_directory, "all.settings")): self._overwrite_settings_with_yaml(join(engine_directory, "all.settings")) if override_settings_filename is not None: if not exists(override_settings_filename): warn("Settings file '{}' could not be found!\n".format(override_settings_filename)) exit(1) else: # Load settings from specified file, if it exists self._overwrite_settings_with_yaml(override_settings_filename) # Load extra settings from command line JSON and overwrite what's already set if extra_string is not None: try: self.settings_dict.update(json.loads(extra_string).items()) except ValueError as error: warn("""{} in:\n ==> --extra '{}' (must be valid JSON)\n""".format(str(error), extra_string)) exit(1) except AttributeError: warn("""Error in:\n ==> --extra '{}' (must be valid JSON and not a list)\n""".format(extra_string)) exit(1)
def cli(filenames, yaml, quiet, tags, settings, extra): """Run test files or entire directories containing .test files.""" # .hitch/virtualenv/bin/python <- 4 directories up from where the python exec resides engine_folder = path.abspath(path.join(executable, "..", "..", "..", "..")) filenames = [path.abspath(path.relpath(filename, engine_folder)) for filename in filenames] chdir(engine_folder) if quiet: warn(( "The --quiet switch has been deprecated. You can make your tests run quietly " "by setting the property quiet to True via --extra or in a settings file.\n\n" "See here for more details on the change : \n" "https://hitchtest.readthedocs.org/en/latest/faq/why_was_hitch_behavior_changed.html\n" )) exit(1) #new_default_settings_filename = path.join(engine_folder, "all.settings") #settings_filename = None if settings is None else path.join(engine_folder, settings) #if path.exists(new_default_settings_filename): #_overwrite_settings_with_yaml(new_default_settings_filename, settings_dict) #if settings_filename is not None: #if not path.exists(settings_filename): #warn("Settings file '{}' could not be found!\n".format(settings_filename)) #exit(1) #else: ## Load settings from specified file, if it exists #_overwrite_settings_with_yaml(settings_filename, settings_dict) ## Load extra settings from command line JSON and overwrite what's already set #if extra is not None: #try: #settings_dict.update(json.loads(extra).items()) #except ValueError as error: #warn("""{} in:\n ==> --extra '{}' (must be valid JSON)\n""".format(str(error), extra)) #exit(1) #except AttributeError: #warn("""Error in:\n ==> --extra '{}' (must be valid JSON and not a list)\n""".format(extra)) #exit(1) settings_dict = Settings(engine_folder, settings, extra) settings_dict['engine_folder'] = engine_folder if 'quiet' not in settings_dict: settings_dict['quiet'] = False if len(filenames) == 0: warn("No tests specified.\n") exit(1) # Get list of files from specified files/directories matches = [] test_not_found = False for filename in filenames: if not path.exists(filename): warn("Test '{}' not found.\n".format(filename)) test_not_found = True if path.isdir(filename): for root, dirnames, filenames_in_dir in walk(filename): for filename_in_dir in fnmatch.filter(filenames_in_dir, '*.test'): if ".hitch" not in root: # Ignore everything in .hitch matches.append(path.join(root, filename_in_dir)) else: matches.append(filename) if test_not_found: exit(1) # Get list of modules from matching directly specified files from command line # and indirectly (in the directories of) directories specified from cmd line test_modules = [] for filename in matches: if filename.endswith(".test"): test_modules.append(module.Module(filename, settings_dict)) else: warn( "Tests must have the extension .test" "- '{}' doesn't have that extension\n".format(filename) ) exit(1) test_suite = suite.Suite(test_modules, settings_dict, tags) if yaml: test_suite.printyaml() else: returned_results = test_suite.run(quiet=quiet) # Lines must be split to prevent stdout blocking result_lines = returned_results.to_template( template=settings_dict.get('results_template', None) ).split('\n') for line in result_lines: log("{}\n".format(line)) exit(1 if len(returned_results.failures()) > 0 else 0)
def __init__(self, filename, settings): self.filename = path.realpath(filename) self.engine_folder = settings['engine_folder'] self.name = path.split(self.filename)[1].replace(".test", "") self.title = self.name.replace("_", " ").title() self.dirname = path.dirname(self.filename) if settings is None: self.settings = {} else: self.settings = settings with open(filename, "r") as file_handle: if file_handle.read() == "": warn("{0} is an empty file.\n".format(filename)) sys.exit(1) env = Environment() env.loader = FileSystemLoader(path.split(filename)[0]) try: tmpl = env.get_template(path.split(filename)[1]) except exceptions.TemplateError as error: warn("Jinja2 template error in '{}' on line {}:\n==> {}\n".format( error.filename, error.lineno, str(error) )) sys.exit(1) self.test_yaml_text = tmpl.render(**self.settings) if self.test_yaml_text == "": warn(( "{0} rendered as an empty file. " "There is probably a problem with your jinja2 template.\n" ).format(filename)) sys.exit(1) self.tests = [] try: module_yaml_as_dict = yaml.load(self.test_yaml_text) except yaml.parser.MarkedYAMLError as error: warn("YAML parser error in {}:\n".format(filename)) warn(str(error)) warn("\n") sys.exit(1) try: core = Core(source_data=module_yaml_as_dict, schema_files=[HITCHSCHEMA]) core.validate(raise_exception=True) except PyKwalifyException as error: warn("YAML validation error in {}:\n==> {}\n".format(filename, error.msg)) sys.exit(1) if len(module_yaml_as_dict) == 1: self.multiple_tests = False self.tests = [Test(module_yaml_as_dict[0], self.settings, self.filename, self.title)] else: self.multiple_tests = True for i, test_yaml in enumerate(module_yaml_as_dict, 1): self.tests.append( Test(test_yaml, self.settings, self.filename, "{} {}".format(self.title, i)) )
def run(self, quiet=False): """Run all tests in the defined suite of modules.""" tests = self.tests() failedfast = False result_list = [] for test in tests: if quiet: hijacked_stdout = sys.stdout hijacked_stderr = sys.stderr sys.stdout = open(path.join(self.settings['engine_folder'], ".hitch", "test.out"), "ab", 0) sys.stderr = open(path.join(self.settings['engine_folder'], ".hitch", "test.err"), "ab", 0) def run_test_in_separate_process(file_descriptor_stdin, result_queue): """Change process group, run test and return result via a queue.""" orig_pgid = os.getpgrp() os.setpgrp() result_queue.put("pgrp") if not quiet: sys.stdin = os.fdopen(file_descriptor_stdin) result = test.run() result_queue.put(result) if not quiet: try: os.tcsetpgrp(file_descriptor_stdin, orig_pgid) except OSError as error: if error.args[0] == 25: pass if not quiet: try: orig_stdin_termios = termios.tcgetattr(sys.stdin.fileno()) except termios.error: orig_stdin_termios = None orig_stdin_fileno = sys.stdin.fileno() orig_pgid = os.getpgrp() file_descriptor_stdin = sys.stdin.fileno() result_queue = multiprocessing.Queue() # Start new process to run test in, to isolate it from future test runs test_process = multiprocessing.Process( target=run_test_in_separate_process, args=(file_descriptor_stdin, result_queue) ) test_timed_out = False test_process.start() # Ignore all exit signals but pass them on signal_pass_on_to_separate_process_group(test_process.pid) # Wait until PGRP is changed result_queue.get() # Make stdin go to the test process so that you can use ipython, etc. if not quiet: try: os.tcsetpgrp(file_descriptor_stdin, os.getpgid(test_process.pid)) except OSError as error: if error.args[0] == 25: pass # Wait until process has finished proc = psutil.Process(test_process.pid) test_timeout = self.settings.get("test_timeout", None) test_shutdown_timeout = self.settings.get("test_shutdown_timeout", 10) try: proc.wait(timeout=test_timeout) except psutil.TimeoutExpired: test_timed_out = True proc.send_signal(signal.SIGTERM) try: proc.wait(timeout=test_shutdown_timeout) except psutil.TimeoutExpired: for child in proc.get_children(recursive=True): child.send_signal(signal.SIGKILL) proc.send_signal(signal.SIGKILL) # Take back signal handling from test running code signals_trigger_exit() try: result = result_queue.get_nowait() except multiprocessing.queues.Empty: result = Result(test, True, 0.0) if test_timed_out: result.aborted = False result_list.append(result) if not quiet and orig_stdin_termios is not None: try: termios.tcsetattr(orig_stdin_fileno, termios.TCSANOW, orig_stdin_termios) except termios.error as err: # I/O error caused by another test stopping this one if err[0] == 5: pass if quiet: sys.stdout = hijacked_stdout sys.stderr = hijacked_stderr if quiet and result is not None: if result.failure: warn("X") else: warn(".") if result.aborted: warn("Aborted\n") sys.exit(1) if self.settings.get('failfast', False) and result.failure: failedfast = True break return Results(result_list, failedfast, self.settings.get('colorless', False))