def __init__(self, report_name, output_dir, logs_dir, naarad_dir): """ :param report_name: used in the title of the front-end :param output_dir: directory where the report will be generated :param logs_dir: directory of where the logs will be collected :param naarad_dir: directory containing the naarad reports """ self.name = report_name self.env = Environment(loader=FileSystemLoader(constants.WEB_RESOURCE_DIR)) # used to load html pages for Jinja2 self.data_source = runtime.get_collector() self.report_info = _ReportInfo(output_dir, logs_dir, naarad_dir)
def run(self): """ This is the main executable function that will run the test """ self._setup() failure_handler = FailureHandler( self.master_config.mapping.get("max_suite_failures_before_abort")) naarad_obj = Naarad() for config in self.configs: self._reset_tests() if not failure_handler.get_abort_status(): config.result = constants.SKIPPED config.message += error_messages.CONFIG_ABORT self._skip_all_tests() logger.debug( "Skipping " + config.name + "due to too many setup_suite/teardown_suite failures") else: runtime.set_active_config(config) setup_fail = False if not self.master_config.mapping.get("no_perf", False): config.naarad_id = naarad_obj.signal_start( self.dynamic_config_module.naarad_config( config.mapping)) config.start_time = time.time() logger.info("Setting up configuration: " + config.name) try: if hasattr(self.deployment_module, 'setup_suite'): self.deployment_module.setup_suite() except BaseException: config.result = constants.SKIPPED config.message += error_messages.SETUP_SUITE_FAILED + traceback.format_exc( ) self._skip_all_tests() setup_fail = True failure_handler.notify_failure() logger.error( "Aborting {0} due to setup_suite failure:\n{1}".format( config.name, traceback.format_exc())) else: logger.debug("Running tests for configuration: " + config.name) self._execute_run(config, naarad_obj) self._copy_logs() if not self.master_config.mapping.get("no_perf", False): naarad_obj.signal_stop(config.naarad_id) self._execute_performance(naarad_obj) self._execute_verification() logger.debug("Tearing down configuration: " + config.name) try: if hasattr(self.deployment_module, 'teardown_suite'): self.deployment_module.teardown_suite() if not setup_fail: failure_handler.notify_success() except BaseException: config.message += error_messages.TEARDOWN_SUITE_FAILED + traceback.format_exc( ) if not setup_fail: failure_handler.notify_failure() logger.error("{0} failed teardown_suite(). {1}".format( config.name, traceback.format_exc())) config.end_time = time.time() logger.info("Execution of configuration: {0} complete".format( config.name)) tests = [test for test in self.tests if not isinstance(test, list)] +\ [individual_test for test in self.tests if isinstance(test, list) for individual_test in test] runtime.get_collector().collect(config, tests) # prints result to standard out - delete/comment out when things are working # self._print_debug() # analysis.generate_diff_reports() self.reporter.generate() if not self.master_config.mapping.get("no-display", False): self._display_results()
def run(self): """ This is the main executable function that will run the test """ self._setup() failure_handler = FailureHandler(self.master_config.mapping.get("max_suite_failures_before_abort")) naarad_obj = Naarad() for config in self.configs: config.mapping.iterkeys() self._reset_tests() if not failure_handler.get_abort_status(): config.result = constants.SKIPPED config.message += error_messages.CONFIG_ABORT self._skip_all_tests() logger.debug("Skipping " + config.name + "due to too many setup_suite/teardown_suite failures") else: runtime.set_active_config(config) setup_fail = False if not self.master_config.mapping.get("no_perf", False): try: naarad_config_file = self.dynamic_config_module.naarad_config() except TypeError: # Support backwards compatability naarad_config_file = self.dynamic_config_module.naarad_config(config.mapping) config.naarad_id = naarad_obj.signal_start(naarad_config_file) config.start_time = time.time() logger.info("Setting up configuration: " + config.name) try: if hasattr(self.deployment_module, 'setup_suite'): self.deployment_module.setup_suite() except BaseException: config.result = constants.SKIPPED config.message += error_messages.SETUP_SUITE_FAILED + traceback.format_exc() self._skip_all_tests() setup_fail = True failure_handler.notify_failure() logger.error("Aborting {0} due to setup_suite failure:\n{1}".format(config.name, traceback.format_exc())) else: try: logger.debug("Running tests for configuration: " + config.name) self._execute_run(config, naarad_obj) logger.debug("Tearing down configuration: " + config.name) finally: try: if hasattr(self.deployment_module, 'teardown_suite'): self.deployment_module.teardown_suite() if not setup_fail: failure_handler.notify_success() except BaseException: config.message += error_messages.TEARDOWN_SUITE_FAILED + traceback.format_exc() if not setup_fail: failure_handler.notify_failure() logger.error("{0} failed teardown_suite(). {1}".format(config.name, traceback.format_exc())) finally: # kill all orphaned process for deployer in runtime.get_deployers(): deployer.kill_all_process() config.end_time = time.time() logger.info("Execution of configuration: {0} complete".format(config.name)) tests = [test for test in self.tests if not isinstance(test, list)] +\ [individual_test for test in self.tests if isinstance(test, list) for individual_test in test] runtime.get_collector().collect(config, tests) # log results of tests so that it can be used easily via command-line self._log_results(tests) # analysis.generate_diff_reports() self.reporter.data_source.end_time = time.time() self.reporter.generate() if not self.master_config.mapping.get("no-display", False): self._display_results()
def run(self): """ This is the main executable function that will run the test """ self._setup() failure_handler = FailureHandler(self.master_config.mapping.get("max_suite_failures_before_abort")) naarad_obj = Naarad() for config in self.configs: config.mapping.iterkeys() self._reset_tests() if not failure_handler.get_abort_status(): config.result = constants.SKIPPED config.message += error_messages.CONFIG_ABORT self._skip_all_tests() logger.debug("Skipping " + config.name + "due to too many setup_suite/teardown_suite failures") else: runtime.set_active_config(config) setup_fail = False if not self.master_config.mapping.get("no_perf", False): try: naarad_config_file = self.dynamic_config_module.naarad_config() except TypeError: # Support backwards compatability naarad_config_file = self.dynamic_config_module.naarad_config(config.mapping) config.naarad_id = naarad_obj.signal_start(naarad_config_file) config.start_time = time.time() logger.info("Setting up configuration: " + config.name) try: if hasattr(self.deployment_module, 'setup_suite'): self.deployment_module.setup_suite() except BaseException: config.result = constants.SKIPPED config.message += error_messages.SETUP_SUITE_FAILED + traceback.format_exc() self._skip_all_tests() setup_fail = True failure_handler.notify_failure() logger.error("Aborting {0} due to setup_suite failure:\n{1}".format(config.name, traceback.format_exc())) else: try: logger.debug("Running tests for configuration: " + config.name) self._execute_run(config, naarad_obj) logger.debug("Tearing down configuration: " + config.name) finally: try: if hasattr(self.deployment_module, 'teardown_suite'): self.deployment_module.teardown_suite() if not setup_fail: failure_handler.notify_success() except BaseException: config.message += error_messages.TEARDOWN_SUITE_FAILED + traceback.format_exc() if not setup_fail: failure_handler.notify_failure() logger.error("{0} failed teardown_suite(). {1}".format(config.name, traceback.format_exc())) finally: # kill all orphaned process for deployer in runtime.get_deployers(): deployer.kill_all_process() config.end_time = time.time() logger.info("Execution of configuration: {0} complete".format(config.name)) tests = [test for test in self.tests if not isinstance(test, list)] +\ [individual_test for test in self.tests if isinstance(test, list) for individual_test in test] runtime.get_collector().collect(config, tests) # log results of tests so that it can be used easily via command-line self._log_results(tests) # analysis.generate_diff_reports() self.reporter.data_source.end_time = time.time() self.reporter.generate() if self.master_config.mapping.get("display", False) and not self.master_config.mapping.get("junit_reporter", False): self._display_results()
def run(self): """ This is the main executable function that will run the test """ self._setup() failure_handler = FailureHandler(self.master_config.mapping.get("max_suite_failures_before_abort")) naarad_obj = Naarad() for config in self.configs: self._reset_tests() if not failure_handler.get_abort_status(): config.result = constants.SKIPPED config.message += error_messages.CONFIG_ABORT self._skip_all_tests() logger.debug("Skipping " + config.name + "due to too many setup_suite/teardown_suite failures") else: runtime.set_active_config(config) setup_fail = False if not self.master_config.mapping.get("no_perf", False): config.naarad_id = naarad_obj.signal_start(self.dynamic_config_module.naarad_config(config.mapping)) config.start_time = time.time() logger.info("Setting up configuration: " + config.name) try: if hasattr(self.deployment_module, 'setup_suite'): self.deployment_module.setup_suite() except BaseException: config.result = constants.SKIPPED config.message += error_messages.SETUP_SUITE_FAILED + traceback.format_exc() self._skip_all_tests() setup_fail = True failure_handler.notify_failure() logger.error("Aborting {0} due to setup_suite failure:\n{1}".format(config.name, traceback.format_exc())) else: logger.debug("Running tests for configuration: " + config.name) self._execute_run(config, naarad_obj) self._copy_logs() if not self.master_config.mapping.get("no_perf", False): naarad_obj.signal_stop(config.naarad_id) self._execute_performance(naarad_obj) self._execute_verification() logger.debug("Tearing down configuration: " + config.name) try: if hasattr(self.deployment_module, 'teardown_suite'): self.deployment_module.teardown_suite() if not setup_fail: failure_handler.notify_success() except BaseException: config.message += error_messages.TEARDOWN_SUITE_FAILED + traceback.format_exc() if not setup_fail: failure_handler.notify_failure() logger.error("{0} failed teardown_suite(). {1}".format(config.name, traceback.format_exc())) config.end_time = time.time() logger.info("Execution of configuration: {0} complete".format(config.name)) tests = [test for test in self.tests if not isinstance(test, list)] +\ [individual_test for test in self.tests if isinstance(test, list) for individual_test in test] runtime.get_collector().collect(config, tests) # prints result to standard out - delete/comment out when things are working # self._print_debug() # analysis.generate_diff_reports() self.reporter.generate() if not self.master_config.mapping.get("no-display", False): self._display_results()