def run_given_tests(cls, list_of_tests, file_path, req_str, meta_vars=None): """Loads all the templates and runs all the given tests This method calls run_test method to run each of the tests one by one. :param list list_of_tests: A list of all the loaded tests :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ pool = ThreadPool(CONF.syntribos.threads) try: template_start_time = time.time() failures = 0 errors = 0 print("\n ID \t\tTest Name \t\t\t\t\t\t Progress") for test_name, test_class in list_of_tests: test_class.test_id = cls.current_test_id cls.current_test_id += 5 result_string = "[{test_id}] : {name}".format( test_id=cli.colorize( test_class.test_id, color="green"), name=test_name.replace("_", " ").capitalize()) if CONF.no_colorize: result_string = result_string.ljust(55) else: result_string = result_string.ljust(60) try: test_class.create_init_request(file_path, req_str, meta_vars) except Exception: print(_( "Error in parsing template:\n %s\n" ) % traceback.format_exc()) LOG.error("Error in parsing template:") break test_cases = list( test_class.get_test_cases(file_path, req_str)) total_tests = len(test_cases) if total_tests > 0: log_string = "[{test_id}] : {name}".format( test_id=test_class.test_id, name=test_name) LOG.debug(log_string) last_failures = result.stats['unique_failures'] last_errors = result.stats['errors'] p_bar = cli.ProgressBar( message=result_string, total_len=total_tests) test_class.send_init_request(file_path, req_str, meta_vars) # This line runs the tests pool.map(lambda t: cls.run_test(t, p_bar), test_cases) failures = result.stats['unique_failures'] - last_failures errors = result.stats['errors'] - last_errors failures_str = cli.colorize_by_percent( failures, total_tests, "red") if errors: errors_str = cli.colorize(errors, "red") print(_( " : %(fail)s Failure(s), %(err)s Error(s)\r") % { "fail": failures_str, "err": errors_str}) else: print(_( " : %s Failure(s), 0 Error(s)\r") % failures_str) run_time = time.time() - template_start_time LOG.info(_("Run time: %s sec."), run_time) if hasattr(result, "testsRun"): num_tests = result.testsRun - result.testsRunSinceLastPrint print(_("\nRan %(num)s test(s) in %(time).3f s\n") % {"num": num_tests, "time": run_time}) result.testsRunSinceLastPrint = result.testsRun except KeyboardInterrupt: print(_( '\n\nPausing...Hit ENTER to continue, type quit to exit.')) try: response = input() if response.lower() == "quit": result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) pool.close() pool.join() exit(0) print(_('Resuming...')) except KeyboardInterrupt: result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) pool.close() pool.join() exit(0)
def run(cls): """Method sets up logger and decides on Syntribos control flow This is the method where control flow of Syntribos is decided based on the commands entered. Depending upon commands such as ```list_tests``` or ```run``` the respective method is called. """ global result cli.print_symbol() # If we are initializing, don't look for a default config file if "init" in sys.argv: cls.setup_config() else: cls.setup_config(use_file=True) try: if CONF.sub_command.name == "init": ENV.initialize_syntribos_env() exit(0) elif CONF.sub_command.name == "list_tests": cls.list_tests() exit(0) elif CONF.sub_command.name == "download": ENV.download_wrapper() exit(0) except AttributeError: print( _( "Not able to run the requested sub command, please check " "the debug logs for more information, exiting...")) exit(1) if not ENV.is_syntribos_initialized(): print(_("Syntribos was not initialized. Please run the 'init'" " command or set it up manually. See the README for" " more information about the installation process.")) exit(1) cls.setup_runtime_env() decorator = unittest.runner._WritelnDecorator(cls.output) result = syntribos.result.IssueTestResult(decorator, True, verbosity=1) cls.start_time = time.time() if CONF.sub_command.name == "run": list_of_tests = list( cls.get_tests(CONF.test_types, CONF.excluded_types)) elif CONF.sub_command.name == "dry_run": dry_run_output = {"failures": [], "successes": []} list_of_tests = list(cls.get_tests(dry_run=True)) print(_("\nRunning Tests...:")) templates_dir = CONF.syntribos.templates if templates_dir is None: print(_("Attempting to download templates from {}").format( CONF.remote.templates_uri)) templates_path = remotes.get(CONF.remote.templates_uri) try: templates_dir = ContentType("r", 0)(templates_path) except IOError: print(_("Not able to open `%s`; please verify path, " "exiting...") % templates_path) exit(1) print(_("\nPress Ctrl-C to pause or exit...\n")) meta_vars = None templates_dir = list(templates_dir) cls.meta_dir_dict = {} for file_path, file_content in templates_dir: if os.path.basename(file_path) == "meta.json": meta_path = os.path.dirname(file_path) try: cls.meta_dir_dict[meta_path] = json.loads(file_content) except Exception: print("Unable to parse %s, skipping..." % file_path) for file_path, req_str in templates_dir: if "meta.json" in file_path: continue meta_vars = cls.get_meta_vars(file_path) LOG = cls.get_logger(file_path) CONF.log_opt_values(LOG, logging.DEBUG) if not file_path.endswith(".template"): LOG.warning('file.....:%s (SKIPPED - not a .template file)', file_path) continue test_names = [t for (t, i) in list_of_tests] # noqa log_string = ''.join([ '\n{0}\nTEMPLATE FILE\n{0}\n'.format('-' * 12), 'file.......: {0}\n'.format(file_path), 'tests......: {0}\n'.format(test_names) ]) LOG.debug(log_string) print(syntribos.SEP) print("Template File...: {}".format(file_path)) print(syntribos.SEP) if CONF.sub_command.name == "run": cls.run_given_tests(list_of_tests, file_path, req_str, meta_vars) elif CONF.sub_command.name == "dry_run": cls.dry_run(list_of_tests, file_path, req_str, dry_run_output, meta_vars) if CONF.sub_command.name == "run": result.print_result(cls.start_time) cleanup.delete_temps() elif CONF.sub_command.name == "dry_run": cls.dry_run_report(dry_run_output)
def run_given_tests(cls, list_of_tests, file_path, req_str): """Loads all the templates and runs all the given tests This method calls run_test method to run each of the tests one by one. :param list list_of_tests: A list of all the loaded tests :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ try: template_start_time = time.time() failures = 0 errors = 0 print("\n ID \t\tTest Name \t\t\t\t\t\t Progress") for test_name, test_class in list_of_tests: test_class.test_id = cls.current_test_id cls.current_test_id += 5 log_string = "[{test_id}] : {name}".format( test_id=test_class.test_id, name=test_name) result_string = "[{test_id}] : {name}".format( test_id=cli.colorize(test_class.test_id, color="green"), name=test_name.replace("_", " ").capitalize()) if not CONF.colorize: result_string = result_string.ljust(55) else: result_string = result_string.ljust(60) LOG.debug(log_string) test_class.send_init_request(file_path, req_str) test_cases = list(test_class.get_test_cases( file_path, req_str)) if len(test_cases) > 0: bar = cli.ProgressBar(message=result_string, total_len=len(test_cases)) last_failures = result.stats["failures"] last_errors = result.stats["errors"] for test in test_cases: if test: cls.run_test(test, result) bar.increment(1) bar.print_bar() failures = result.stats["failures"] - last_failures errors = result.stats["errors"] - last_errors total_tests = len(test_cases) if failures > total_tests * 0.90: # More than 90 percent failure failures = cli.colorize(failures, "red") elif failures > total_tests * 0.45: # More than 45 percent failure failures = cli.colorize(failures, "yellow") elif failures > total_tests * 0.15: # More than 15 percent failure failures = cli.colorize(failures, "blue") if errors: last_failures = result.stats["failures"] last_errors = result.stats["errors"] errors = cli.colorize(errors, "red") print(" : {0} Failure(s), {1} Error(s)\r".format( failures, errors)) else: last_failures = result.stats["failures"] print(" : {} Failure(s), 0 Error(s)\r".format( failures)) run_time = time.time() - template_start_time LOG.debug("Run time: %s sec.", run_time) if hasattr(result, "testsRun"): num_tests = result.testsRun - result.testsRunSinceLastPrint print("\nRan {num} test(s) in {time:.3f}s\n".format( num=num_tests, time=run_time)) result.testsRunSinceLastPrint = result.testsRun except KeyboardInterrupt: print('\n\nPausing... Hit ENTER to continue, type quit to exit.') try: response = input() if response.lower() == "quit": result.print_result(cls.start_time) cleanup.delete_temps() print("Exiting...") exit(0) print('Resuming...') except KeyboardInterrupt: result.print_result(cls.start_time) cleanup.delete_temps() print("Exiting...") exit(0)
def run_given_tests(cls, list_of_tests, file_path, req_str, meta_vars=None): """Loads all the templates and runs all the given tests This method calls run_test method to run each of the tests one by one. :param list list_of_tests: A list of all the loaded tests :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ try: template_start_time = time.time() failures = 0 errors = 0 print("\n ID \t\tTest Name \t\t\t\t\t\t Progress") for test_name, test_class in list_of_tests: test_class.test_id = cls.current_test_id cls.current_test_id += 5 log_string = "[{test_id}] : {name}".format( test_id=test_class.test_id, name=test_name) result_string = "[{test_id}] : {name}".format( test_id=cli.colorize( test_class.test_id, color="green"), name=test_name.replace("_", " ").capitalize()) if not CONF.colorize: result_string = result_string.ljust(55) else: result_string = result_string.ljust(60) LOG.debug(log_string) try: test_class.send_init_request(file_path, req_str, meta_vars) except Exception: print(_( "Error in parsing template:\n %s\n" ) % traceback.format_exc()) LOG.error(_LE("Error in parsing template:")) break test_cases = list( test_class.get_test_cases(file_path, req_str)) if len(test_cases) > 0: p_bar = cli.ProgressBar( message=result_string, total_len=len(test_cases)) last_failures = result.stats["failures"] last_errors = result.stats["errors"] for test in test_cases: if test: cls.run_test(test, result) p_bar.increment(1) p_bar.print_bar() failures = result.stats["failures"] - last_failures errors = result.stats["errors"] - last_errors total_tests = len(test_cases) if failures > total_tests * 0.90: # More than 90 percent failure failures = cli.colorize(failures, "red") elif failures > total_tests * 0.45: # More than 45 percent failure failures = cli.colorize(failures, "yellow") elif failures > total_tests * 0.15: # More than 15 percent failure failures = cli.colorize(failures, "blue") if errors: last_failures = result.stats["failures"] last_errors = result.stats["errors"] errors = cli.colorize(errors, "red") print(_( " : %(fail)s Failure(s), %(err)s Error(s)\r") % { "fail": failures, "err": errors}) else: last_failures = result.stats["failures"] print( _( " : %s Failure(s), 0 Error(s)\r") % failures) run_time = time.time() - template_start_time LOG.info(_("Run time: %s sec."), run_time) if hasattr(result, "testsRun"): num_tests = result.testsRun - result.testsRunSinceLastPrint print(_("\nRan %(num)s test(s) in %(time).3f s\n") % {"num": num_tests, "time": run_time}) result.testsRunSinceLastPrint = result.testsRun except KeyboardInterrupt: print(_( '\n\nPausing...Hit ENTER to continue, type quit to exit.')) try: response = input() if response.lower() == "quit": result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) exit(0) print(_('Resuming...')) except KeyboardInterrupt: result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) exit(0)
def run(cls): """Method sets up logger and decides on Syntribos control flow This is the method where control flow of Syntribos is decided based on the commands entered. Depending upon commands such as ```list_tests``` or ```run``` the respective method is called. """ global result cli.print_symbol() # If we are initializing, don't look for a default config file if "init" in sys.argv: cls.setup_config() else: cls.setup_config(use_file=True) try: if CONF.sub_command.name == "init": ENV.initialize_syntribos_env() exit(0) elif CONF.sub_command.name == "list_tests": cls.list_tests() exit(0) elif CONF.sub_command.name == "download": ENV.download_wrapper() exit(0) except AttributeError: print( _( "Not able to run the requested sub command, please check " "the debug logs for more information, exiting...")) exit(1) if not ENV.is_syntribos_initialized(): print(_("Syntribos was not initialized. Please run the 'init'" " command or set it up manually. See the README for" " more information about the installation process.")) exit(1) cls.setup_runtime_env() decorator = unittest.runner._WritelnDecorator(cls.output) result = syntribos.result.IssueTestResult(decorator, True, verbosity=1) cls.start_time = time.time() if CONF.sub_command.name == "run": list_of_tests = list( cls.get_tests(CONF.test_types, CONF.excluded_types)) elif CONF.sub_command.name == "dry_run": dry_run_output = {"failures": [], "successes": []} list_of_tests = list(cls.get_tests(dry_run=True)) print(_("\nRunning Tests...:")) templates_dir = CONF.syntribos.templates if templates_dir is None: print(_("Attempting to download templates from {}").format( CONF.remote.templates_uri)) templates_path = remotes.get(CONF.remote.templates_uri) try: templates_dir = ContentType("r", 0)(templates_path) except IOError: print(_("Not able to open `%s`; please verify path, " "exiting...") % templates_path) exit(1) print(_("\nPress Ctrl-C to pause or exit...\n")) meta_vars = None templates_dir = list(templates_dir) cls.meta_dir_dict = {} for file_path, file_content in templates_dir: if os.path.basename(file_path) == "meta.json": meta_path = os.path.dirname(file_path) try: cls.meta_dir_dict[meta_path] = json.loads(file_content) except Exception: print("Unable to parse %s, skipping..." % file_path) for file_path, req_str in templates_dir: if "meta.json" in file_path: continue meta_vars = cls.get_meta_vars(file_path) LOG = cls.get_logger(file_path) CONF.log_opt_values(LOG, logging.DEBUG) if not file_path.endswith(".template"): LOG.warning( _LW('file.....:%s (SKIPPED - not a .template file)'), file_path) continue test_names = [t for (t, i) in list_of_tests] # noqa log_string = ''.join([ '\n{0}\nTEMPLATE FILE\n{0}\n'.format('-' * 12), 'file.......: {0}\n'.format(file_path), 'tests......: {0}\n'.format(test_names) ]) LOG.debug(log_string) print(syntribos.SEP) print("Template File...: {}".format(file_path)) print(syntribos.SEP) if CONF.sub_command.name == "run": cls.run_given_tests(list_of_tests, file_path, req_str, meta_vars) elif CONF.sub_command.name == "dry_run": cls.dry_run(list_of_tests, file_path, req_str, dry_run_output, meta_vars) if CONF.sub_command.name == "run": result.print_result(cls.start_time) cleanup.delete_temps() elif CONF.sub_command.name == "dry_run": cls.dry_run_report(dry_run_output)
def run_given_tests(cls, list_of_tests, file_path, req_str, meta_vars=None): """Loads all the templates and runs all the given tests This method calls run_test method to run each of the tests one by one. :param list list_of_tests: A list of all the loaded tests :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ pool = ThreadPool(CONF.syntribos.threads) try: template_start_time = time.time() failures = 0 errors = 0 print("\n ID \t\tTest Name \t\t\t\t\t\t Progress") for test_name, test_class in list_of_tests: test_class.test_id = cls.current_test_id cls.current_test_id += 5 result_string = "[{test_id}] : {name}".format( test_id=cli.colorize( test_class.test_id, color="green"), name=test_name.replace("_", " ").capitalize()) if not CONF.colorize: result_string = result_string.ljust(55) else: result_string = result_string.ljust(60) try: test_class.create_init_request(file_path, req_str, meta_vars) except Exception: print(_( "Error in parsing template:\n %s\n" ) % traceback.format_exc()) LOG.error("Error in parsing template:") break test_cases = list( test_class.get_test_cases(file_path, req_str, meta_vars)) total_tests = len(test_cases) if total_tests > 0: log_string = "[{test_id}] : {name}".format( test_id=test_class.test_id, name=test_name) LOG.debug(log_string) last_failures = result.stats['unique_failures'] last_errors = result.stats['errors'] p_bar = cli.ProgressBar( message=result_string, total_len=total_tests) test_class.send_init_request(file_path, req_str, meta_vars) # This line runs the tests pool.map(lambda t: cls.run_test(t, p_bar), test_cases) failures = result.stats['unique_failures'] - last_failures errors = result.stats['errors'] - last_errors failures_str = cli.colorize_by_percent( failures, total_tests) if errors: errors_str = cli.colorize(errors, "red") print(_( " : %(fail)s Failure(s), %(err)s Error(s)\r") % { "fail": failures_str, "err": errors_str}) else: print(_( " : %s Failure(s), 0 Error(s)\r") % failures_str) run_time = time.time() - template_start_time LOG.info(_("Run time: %s sec."), run_time) if hasattr(result, "testsRun"): num_tests = result.testsRun - result.testsRunSinceLastPrint print(_("\nRan %(num)s test(s) in %(time).3f s\n") % {"num": num_tests, "time": run_time}) result.testsRunSinceLastPrint = result.testsRun except KeyboardInterrupt: print(_( '\n\nPausing...Hit ENTER to continue, type quit to exit.')) try: response = input() if response.lower() == "quit": result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) pool.close() pool.join() exit(0) print(_('Resuming...')) except KeyboardInterrupt: result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) pool.close() pool.join() exit(0)