def run_suite(self, suite): """Run a suite. `suite` can be a path to a Python file or an import path. Both relative to the suites folder. Example: test = 'folder/suite.py' test = 'folder.suite' """ # TODO if suite.endswith('.py'): filename, _ = os.path.splitext(suite) suite = '.'.join(os.path.normpath(filename).split(os.sep)) self.tests = suite_module.get_suite_test_cases(self.project, suite) if len(self.tests) == 0: print('No tests found for suite {}'.format(suite)) suite_amount_processes = suite_module.get_suite_amount_of_processes(self.project, suite) self.suite.processes = suite_amount_processes self.suite.browsers = suite_module.get_suite_browsers(self.project, suite) self.suite.envs = suite_module.get_suite_environments(self.project, suite) suite_imported_module = suite_module.get_suite_module(self.project, suite) self.suite.before = getattr(suite_imported_module, 'before', None) self.suite.after = getattr(suite_imported_module, 'after', None) self.suite.tags = getattr(suite_imported_module, 'tags', None) self.suite_name = suite self.is_suite = True self._prepare()
def suite_view(project, suite): if not user.has_permissions_to_project(g.user.id, project, root_path, 'gui'): return render_template('not_permission.html') all_test_cases = utils.get_test_cases(root_path, project) selected_tests = suite_module.get_suite_test_cases(root_path, project, suite) worker_amount = suite_module.get_suite_amount_of_workers( root_path, project, suite) browsers = suite_module.get_suite_browsers(root_path, project, suite) browsers = ', '.join(browsers) default_browser = test_execution.settings['default_browser'] environments = suite_module.get_suite_environments(root_path, project, suite) environments = ', '.join(environments) return render_template('suite.html', project=project, all_test_cases=all_test_cases['sub_elements'], selected_tests=selected_tests, suite=suite, worker_amount=worker_amount, browsers=browsers, default_browser=default_browser, environments=environments)
def test_get_suite_environments(self, project_function): _, project = project_function.activate() suite_name = 'test_suite_002' suite.new_suite(project, [], suite_name) environments = ['env1', 'env2'] suite.save_suite(project, suite_name, [], 1, [], environments, []) result = suite.get_suite_environments(project, suite_name) assert result == environments
def test_get_suite_environments(self, project_function): testdir = project_function['testdir'] project = project_function['name'] suite_name = 'test_suite_002' suite.new_suite(testdir, project, [], suite_name) environments = ['env1', 'env2'] suite.save_suite(testdir, project, suite_name, [], 1, [], environments) result = suite.get_suite_environments(testdir, project, suite_name) assert result == environments
def suite_view(project, suite): if not suite_module.suite_exists(project, suite): abort(404, 'The suite {} does not exist'.format(suite)) all_test_cases = utils.get_test_cases(project) selected_tests = suite_module.get_suite_test_cases(project, suite) processes = suite_module.get_suite_amount_of_processes(project, suite) browsers = suite_module.get_suite_browsers(project, suite) default_browser = session.settings['default_browser'] environments = suite_module.get_suite_environments(project, suite) tags = suite_module.get_tags(project, suite) return render_template('suite.html', project=project, all_test_cases=all_test_cases['sub_elements'], selected_tests=selected_tests, suite=suite, processes=processes, browsers=browsers, default_browser=default_browser, environments=environments, tags=tags)
def suite_view(project, suite): suite_exists = suite_module.suite_exists(test_execution.root_path, project, suite) if not suite_exists: abort(404, 'The suite {} does not exist'.format(suite)) all_test_cases = utils.get_test_cases(root_path, project) selected_tests = suite_module.get_suite_test_cases(root_path, project, suite) worker_amount = suite_module.get_suite_amount_of_workers(root_path, project, suite) browsers = suite_module.get_suite_browsers(root_path, project, suite) browsers = ', '.join(browsers) default_browser = test_execution.settings['default_browser'] environments = suite_module.get_suite_environments(root_path, project, suite) environments = ', '.join(environments) return render_template('suite.html', project=project, all_test_cases=all_test_cases['sub_elements'], selected_tests=selected_tests, suite=suite, worker_amount=worker_amount, browsers=browsers, default_browser=default_browser, environments=environments)
def suite_view(project, suite): if not user.has_permissions_to_project(g.user.id, project, root_path, 'gui'): return render_template('not_permission.html') all_test_cases = utils.get_test_cases(root_path, project) selected_tests = suite_module.get_suite_test_cases(root_path, project, suite) worker_amount = suite_module.get_suite_amount_of_workers(root_path, project, suite) browsers = suite_module.get_suite_browsers(root_path, project, suite) browsers = ', '.join(browsers) default_browser = test_execution.settings['default_browser'] environments = suite_module.get_suite_environments(root_path, project, suite) environments = ', '.join(environments) return render_template('suite.html', project=project, all_test_cases=all_test_cases['sub_elements'], selected_tests=selected_tests, suite=suite, worker_amount=worker_amount, browsers=browsers, default_browser=default_browser, environments=environments)
def run_test_or_suite(workspace, project, test=None, suite=None, directory=None): """Run a suite or test or directory containing tests.""" execution = { 'tests': [], 'workers': 1, 'drivers': [], 'environments': [], 'suite_before': None, 'suite_after': None } suite_amount_workers = None suite_drivers = None suite_envs = [] suite_name = None is_suite = False if test: execution['tests'] = [test] suite_name = 'single_tests' elif suite: execution['tests'] = suite_module.get_suite_test_cases( workspace, project, suite) suite_amount_workers = suite_module.get_suite_amount_of_workers( workspace, project, suite) suite_drivers = suite_module.get_suite_browsers( workspace, project, suite) suite_envs = suite_module.get_suite_environments( workspace, project, suite) suite_imported_module = suite_module.get_suite_module( workspace, project, suite) execution['suite_before'] = getattr(suite_imported_module, 'before', None) execution['suite_after'] = getattr(suite_imported_module, 'after', None) suite_name = suite is_suite = True elif directory: execution['tests'] = utils.get_directory_test_cases( workspace, project, directory) suite_name = directory is_suite = True else: sys.exit("ERROR: invalid arguments for run_test_or_suite()") # warn if no tests were found if len(execution['tests']) == 0: print('Warning: no tests were found') # get amount of workers (parallel executions), default is 1 if test_execution.thread_amount: # the thread count passed through cli has higher priority execution['workers'] = test_execution.thread_amount elif suite_amount_workers: execution['workers'] = suite_amount_workers # select the drivers to use in this execution # the order of precedence is: # 1. drivers defined by CLI # 2. drivers defined inside a suite # 3. 'default_driver' setting # 4. default default is 'chrome' settings_default_driver = test_execution.settings['default_browser'] selected_drivers = utils.choose_driver_by_precedence( cli_drivers=test_execution.cli_drivers, suite_drivers=suite_drivers, settings_default_driver=settings_default_driver) # Define the attributes for each driver # # A driver can be predefined ('chrome, 'chrome-headless', 'firefox', etc) # or it can be defined by the user with the 'remote_browsers' setting. # Remote browsers have extra details such as capabilities # # Each driver must have the following attributes: # 'name': real name, # 'full_name': the remote_browser name defined by the user, # 'remote': is this a remote_browser or not # 'capabilities': full capabilities defined in the remote_browsers setting remote_browsers = settings_manager.get_remote_browsers( test_execution.settings) default_browsers = gui_utils.get_supported_browsers_suggestions() execution['drivers'] = _define_drivers(selected_drivers, remote_browsers, default_browsers) # Generate timestamp if needed # A timestamp is passed when the test is executed from the GUI. # The gui uses this timestamp to fetch the test execution status later on. # Otherwise, a new timestamp should be generated at this point if not test_execution.timestamp: test_execution.timestamp = utils.get_timestamp() # Select which envs to use # The user can define environments in the environments.json file. # The suite/test can be executed in one or more of these environments. # Which environments to use is defined by this order of preference: # 1. envs passed by CLI # 2. envs defined inside the suite # 3. The first env defined # 4. no envs at all # # Note, in the case of 4, the test might fail if it tries # to use env variables cli_envs = test_execution.cli_environments project_envs = environment_manager.get_envs(workspace, project) execution['environments'] = _select_environments(cli_envs, suite_envs, project_envs) # Generate the execution list # # Each test must be executed for each: # * data set # * environment # * driver # # The result is a list that contains all the requested combinations execution_list = _define_execution_list(workspace, project, execution) # create the execution directory # # if this is a suite, the directory takes this structure # reports/<suite_name>/<timestamp>/ # # if this is a single test, the directory takes this structure: # reports/single_tests/<test_name>/<timestamp>/ execution_directory = _create_execution_directory(workspace, project, test_execution.timestamp, test_name=test, suite_name=suite_name, is_suite=is_suite) # for each test, create the test directory # for example, in a suite 'suite1' with a 'test1': # reports/suite1/2017.07.02.19.22.20.001/test1/set_00001/ for test in execution_list: report_directory = report.create_report_directory( execution_directory, test['test_name'], is_suite) test['report_directory'] = report_directory # EXECUTION start_time = time.time() suite_error = False # run suite `before` function if execution['suite_before']: try: execution['suite_before'].__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) if not suite_error: if test_execution.interactive and execution['workers'] != 1: print('WARNING: to run in debug mode, threads must equal one') if execution['workers'] == 1: # run tests serially for test in execution_list: run_test(workspace, project, test['test_name'], test['data_set'], test['driver'], test_execution.settings, test['report_directory']) else: # run tests using multiprocessing multiprocess_executor(execution_list, execution['workers']) # run suite `after` function if execution['suite_after']: try: execution['suite_after'].__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) # generate execution_result.json elapsed_time = round(time.time() - start_time, 2) report_parser.generate_execution_report(execution_directory, elapsed_time)