def test_run_test__exception_in_test(self, project_function_clean, caplog, test_utils): """test() throws exception""" testdir = project_function_clean.testdir project = project_function_clean.name test_name = test_utils.random_numeric_string(10) content = """ description = 'desc' def setup(data): step('setup step') def test(data): foo = bar def teardown(data): step('teardown step') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(testdir, project) browser = _define_browsers_mock(['chrome'])[0] test_runner.run_test(workspace=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[5].message == 'Test Result: CODE ERROR' # verify report.json report = self._read_report_json(report_directory) assert report['result'] == 'code error' assert len(report['steps']) == 3 assert len(report['errors']) == 1 assert report['errors'][0]['message'] == "NameError: name 'bar' is not defined"
def test_run_test__failure_in_test(self, project_function_clean, caplog, test_utils): """test() throws AssertionError teardown() is run """ testdir, project = project_function_clean.activate() test_name = test_utils.random_numeric_string(10) content = """ description = 'desc' def setup(data): step('setup step') def test(data): step('test step') fail('test fail') def teardown(data): step('teardown step') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(project) browser = _define_browsers_mock(['chrome'])[0] test_runner.run_test(testdir=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[6].message == 'Test Result: FAILURE' # verify report.json report = self._read_report_json(report_directory) assert report['result'] == 'failure' assert len(report['steps']) == 4 assert len(report['errors']) == 1 assert report['errors'][0]['message'] == 'AssertionError: test fail'
def test_run_test__success_with_data(self, project_function_clean, caplog, test_utils): """Test runs successfully with test data""" testdir = project_function_clean.testdir project = project_function_clean.name test_name = test_utils.random_numeric_string(10) content = """ description = 'some description' def setup(data): step('setup step') def test(data): step('test step') def teardown(data): step('teardown step') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(testdir, project) browser = _define_browsers_mock(['chrome'])[0] test_data = dict(username='******', password='******') secrets = dict(very='secret') # run test test_runner.run_test(workspace=testdir, project=project, test_name=test_name, test_data=test_data, secrets=secrets, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[0].message == 'Test execution started: {}'.format( test_name) assert records[1].message == 'Browser: chrome' # Python 3.4 results not in order TODO value_a = 'Using data:\n username: username1\n password: password1\n' value_b = 'Using data:\n password: password1\n username: username1\n' assert records[2].message in [value_a, value_b] assert records[3].message == 'setup step' assert records[4].message == 'test step' assert records[5].message == 'teardown step' assert records[6].message == 'Test Result: SUCCESS' # verify report.json report = self._read_report_json(report_directory) assert report['browser'] == 'chrome' assert report['description'] == 'some description' assert report['environment'] == '' assert report['errors'] == [] assert report['result'] == 'success' # Python 3.4 TODO assert report['set_name'] in ['username1', 'password1'] assert report['steps'] == [ {'message': 'setup step', 'screenshot': None, 'error': None}, {'message': 'test step', 'screenshot': None, 'error': None}, {'message': 'teardown step', 'screenshot': None, 'error': None}, ] assert report['test_case'] == test_name assert report['test_data'] == {'username': "******", 'password': "******"} assert 'test_elapsed_time' in report assert 'test_timestamp' in report assert len(report.keys()) == 11
def test_run_test__import_error_page_object(self, project_function_clean, caplog, test_utils): """The test fails with 'code error' when an imported page has a syntax error""" testdir = project_function_clean.testdir project = project_function_clean.name test_name = test_utils.random_numeric_string(10) content = """ pages = ['page1'] def setup(data): step('this step wont be run') def test(data): step('this step wont be run') def teardown(data): step('this step wont be run') """ self._create_test(testdir, project, test_name, content) page_content = """ element1 = ('id', 'someId' element2 = ('css', '.oh.no') """ self._create_page(testdir, project, 'page1', page_content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(testdir, project) browser = _define_browsers_mock(['chrome'])[0] test_runner.run_test(workspace=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[0].message == 'Test execution started: {}'.format(test_name) assert records[1].message == 'Browser: chrome' assert records[2].levelname == 'ERROR' error_contains = "element2 = ('css', '.oh.no')\n ^\nSyntaxError: invalid syntax" assert error_contains in records[2].message assert records[3].message == 'Test Result: CODE ERROR' # verify report.json report = self._read_report_json(report_directory) assert report['browser'] == 'chrome' assert report['description'] is None # description could not be read assert report['environment'] == '' assert len(report['errors']) == 1 assert 'SyntaxError: invalid syntax' in report['errors'][0]['message'] assert error_contains in report['errors'][0]['description'] assert report['result'] == 'code error' assert report['set_name'] == '' assert report['steps'] == [] assert report['test_case'] == test_name assert report['test_data'] == {} assert 'test_elapsed_time' in report assert 'test_timestamp' in report assert len(report.keys()) == 11
def run_test(code, test_data={}, secrets={}, from_suite=False): test_module.edit_test_code(project, test_name, code, []) test_runner.run_test(testdir, project, test_name, test_data, secrets, browser, settings, report_directory, from_suite=from_suite)
def test_run_test__success(self, project_function_clean, caplog, test_utils): """Test runs successfully""" testdir = project_function_clean.testdir project = project_function_clean.name test_name = test_utils.random_numeric_string(10) content = """ description = 'some description' def setup(data): step('setup step') def test(data): step('test step') def teardown(data): step('teardown step') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(testdir, project) browser = _define_browsers_mock(['chrome'])[0] # run test test_runner.run_test(workspace=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[0].message == 'Test execution started: {}'.format(test_name) assert records[1].message == 'Browser: chrome' assert records[2].message == 'setup step' assert records[3].message == 'test step' assert records[4].message == 'teardown step' assert records[5].message == 'Test Result: SUCCESS' # verify report.json report = self._read_report_json(report_directory) assert report['browser'] == 'chrome' assert report['description'] == 'some description' assert report['environment'] == '' assert report['errors'] == [] assert report['result'] == 'success' assert report['set_name'] == '' assert report['steps'] == [ {'message': 'setup step', 'screenshot': None, 'error': None}, {'message': 'test step', 'screenshot': None, 'error': None}, {'message': 'teardown step', 'screenshot': None, 'error': None}, ] assert report['test_case'] == test_name assert report['test_data'] == {} assert 'test_elapsed_time' in report assert 'test_timestamp' in report assert len(report.keys()) == 11
def test_run_test__AssertionError_in_setup(self, project_function_clean, caplog, test_utils): """The test ends with 'failure' when the setup function throws AssertionError. Test is not run Teardown is run """ testdir = project_function_clean.testdir project = project_function_clean.name test_name = test_utils.random_numeric_string(10) content = """ description = 'desc' def setup(data): fail('setup step fail') def test(data): step('test step') def teardown(data): step('teardown step') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(testdir, project) browser = _define_browsers_mock(['chrome'])[0] test_runner.run_test(workspace=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[0].message == 'Test execution started: {}'.format(test_name) assert records[1].message == 'Browser: chrome' assert records[2].levelname == 'ERROR' assert 'setup step fail' in records[2].message assert 'AssertionError: setup step fail' in records[2].message assert records[3].message == 'teardown step' assert records[4].message == 'Test Result: FAILURE' # verify report.json report = self._read_report_json(report_directory) assert report['description'] == 'desc' assert len(report['errors']) == 1 assert 'setup step fail' in report['errors'][0]['message'] assert report['result'] == 'failure' assert report['steps'][0]['message'] == 'Failure' assert 'AssertionError: setup step fail' in report['steps'][0]['error']['description'] assert report['steps'][1]['message'] == 'teardown step'
def run_test(code, test_data={}, secrets={}, from_suite=False, set_name=''): set_content(code) test_runner.run_test(testdir, project, test_name, test_data, secrets, browser, env_name, settings, exec_dir, set_name=set_name, test_functions=[], from_suite=from_suite)
def test_run_test__import_error_on_test(self, project_function_clean, caplog, test_utils): """The test fails with 'code error' when it has a syntax error Test result is code error""" testdir = project_function_clean.testdir project = project_function_clean.name test_name = test_utils.random_numeric_string(10) content = """ description = 'some description' # missing colon def test(data) step('this step wont be run') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(testdir, project) browser = _define_browsers_mock(['chrome'])[0] test_runner.run_test(workspace=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[0].message == 'Test execution started: {}'.format( test_name) assert records[1].message == 'Browser: chrome' assert records[2].levelname == 'ERROR' error_contains = 'def test(data)\n ^\nSyntaxError: invalid syntax' assert error_contains in records[2].message assert records[3].message == 'Test Result: CODE ERROR' # verify report.json report = self._read_report_json(report_directory) assert report['browser'] == 'chrome' assert report['description'] is None # description could not be read assert report['environment'] == '' assert len(report['errors']) == 1 assert report['errors'][0]['message'] == 'SyntaxError: invalid syntax' assert error_contains in report['errors'][0]['description'] assert report['result'] == 'code error' assert report['set_name'] == '' assert report['steps'] == [] assert report['test_case'] == test_name assert report['test_data'] == {}
def _execute(self): self.start_time = time.time() suite_error = False # run suite `before` function if self.suite.before: try: self.suite.before.__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) if not suite_error: if self.interactive and self.execution.processes != 1: print( 'WARNING: to run in debug mode, processes must equal one') if self.execution.processes == 1: # run tests serially for test in self.execution.tests: run_test(session.testdir, self.project.name, test.name, test.data_set, test.secrets, test.browser, test.env, session.settings, test.reportdir, test.set_name, self.test_functions, self.execution.has_failed_tests, self.execution.tags, self.is_suite) else: # run tests using multiprocessing multiprocess_executor(self.project.name, self.execution.tests, self.execution.has_failed_tests, self.test_functions, self.execution.processes, self.execution.tags, self.is_suite) # run suite `after` function if self.suite.after: try: self.suite.after.__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) self._finalize()
def execute(self): start_time = time.time() suite_error = False # run suite `before` function if self.suite.before: try: self.suite.before.__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) if not suite_error: if self.interactive and self.execution.processes != 1: print('WARNING: to run in debug mode, threads must equal one') if self.execution.processes == 1: # run tests serially for test in self.execution.tests: run_test(test_execution.root_path, self.project, test.name, test.data_set, test.browser, test_execution.settings, test.reportdir) else: # run tests using multiprocessing multiprocess_executor(self.project, self.execution.tests, self.execution.processes) # run suite `after` function if self.suite.after: try: self.suite.after.__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) # generate execution_result.json elapsed_time = round(time.time() - start_time, 2) report_parser.generate_execution_report(self.execution.reportdir, elapsed_time)
def test_run_test__error_in_setup_test_and_teardown(self, project_function_clean, caplog, test_utils): """setup(), test() and teardown() have errors """ testdir, project = project_function_clean.activate() test_name = test_utils.random_numeric_string(10) content = """ description = 'desc' def setup(data): error('setup error') def test(data): error('test error') def teardown(data): error('teardown error') """ self._create_test(testdir, project, test_name, content) report_directory = self._mock_report_directory(testdir, project, test_name) settings = settings_manager.get_project_settings(project) browser = _define_browsers_mock(['chrome'])[0] test_runner.run_test(testdir=testdir, project=project, test_name=test_name, test_data={}, secrets={}, browser=browser, settings=settings, report_directory=report_directory) # verify console logs records = caplog.records assert records[5].message == 'Test Result: ERROR' # verify report.json report = self._read_report_json(report_directory) assert report['result'] == 'error' assert len(report['steps']) == 3 assert len(report['errors']) == 3 assert report['errors'][0]['message'] == 'setup error' assert report['errors'][1]['message'] == 'test error' assert report['errors'][2]['message'] == 'teardown error'
def _execute(self): start_time = time.time() suite_error = False # run suite `before` function if self.suite.before: try: self.suite.before.__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) if not suite_error: if self.interactive and self.execution.processes != 1: print('WARNING: to run in debug mode, threads must equal one') if self.execution.processes == 1: # run tests serially for test in self.execution.tests: run_test(test_execution.root_path, self.project, test.name, test.data_set, test.secrets, test.browser, test_execution.settings, test.reportdir, self.execution.has_failed_tests) else: # run tests using multiprocessing multiprocess_executor(self.project, self.execution.tests, self.execution.has_failed_tests, self.execution.processes) # run suite `after` function if self.suite.after: try: self.suite.after.__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) # generate report.json elapsed_time = round(time.time() - start_time, 2) self.report = report_parser.generate_execution_report(self.execution.reportdir, elapsed_time) if self.is_suite: self._print_results() # generate requested reports if self.is_suite: report_name = self.report_name or 'report' report_folder = self.report_folder or self.execution.reportdir if 'junit' in self.reports: report_parser.generate_junit_report(self.execution.reportdir, self.suite_name, self.timestamp, self.report_folder, report_name) if 'json' in self.reports and (self.report_folder or self.report_name): report_parser.save_execution_json_report(self.report, report_folder, report_name) if 'html' in self.reports: gui_utils.generate_html_report(self.project, self.suite_name, self.timestamp, self.report_folder, report_name) if 'html-no-images' in self.reports: if 'html' in self.reports: report_name = report_name + '-no-images' gui_utils.generate_html_report(self.project, self.suite_name, self.timestamp, self.report_folder, report_name, no_images=True) # exit to the console with exit status code 1 in case a test fails if self.execution.has_failed_tests.value: sys.exit(1)
def run_test_or_suite(workspace, project, test=None, suite=None, directory=None): """Run a suite or test or directory containing tests.""" execution = { 'tests': [], 'workers': 1, 'drivers': [], 'environments': [], 'suite_before': None, 'suite_after': None } suite_amount_workers = None suite_drivers = None suite_envs = [] suite_name = None is_suite = False if test: execution['tests'] = [test] suite_name = 'single_tests' elif suite: execution['tests'] = suite_module.get_suite_test_cases( workspace, project, suite) suite_amount_workers = suite_module.get_suite_amount_of_workers( workspace, project, suite) suite_drivers = suite_module.get_suite_browsers( workspace, project, suite) suite_envs = suite_module.get_suite_environments( workspace, project, suite) suite_imported_module = suite_module.get_suite_module( workspace, project, suite) execution['suite_before'] = getattr(suite_imported_module, 'before', None) execution['suite_after'] = getattr(suite_imported_module, 'after', None) suite_name = suite is_suite = True elif directory: execution['tests'] = utils.get_directory_test_cases( workspace, project, directory) suite_name = directory is_suite = True else: sys.exit("ERROR: invalid arguments for run_test_or_suite()") # warn if no tests were found if len(execution['tests']) == 0: print('Warning: no tests were found') # get amount of workers (parallel executions), default is 1 if test_execution.thread_amount: # the thread count passed through cli has higher priority execution['workers'] = test_execution.thread_amount elif suite_amount_workers: execution['workers'] = suite_amount_workers # select the drivers to use in this execution # the order of precedence is: # 1. drivers defined by CLI # 2. drivers defined inside a suite # 3. 'default_driver' setting # 4. default default is 'chrome' settings_default_driver = test_execution.settings['default_browser'] selected_drivers = utils.choose_driver_by_precedence( cli_drivers=test_execution.cli_drivers, suite_drivers=suite_drivers, settings_default_driver=settings_default_driver) # Define the attributes for each driver # # A driver can be predefined ('chrome, 'chrome-headless', 'firefox', etc) # or it can be defined by the user with the 'remote_browsers' setting. # Remote browsers have extra details such as capabilities # # Each driver must have the following attributes: # 'name': real name, # 'full_name': the remote_browser name defined by the user, # 'remote': is this a remote_browser or not # 'capabilities': full capabilities defined in the remote_browsers setting remote_browsers = settings_manager.get_remote_browsers( test_execution.settings) default_browsers = gui_utils.get_supported_browsers_suggestions() execution['drivers'] = _define_drivers(selected_drivers, remote_browsers, default_browsers) # Generate timestamp if needed # A timestamp is passed when the test is executed from the GUI. # The gui uses this timestamp to fetch the test execution status later on. # Otherwise, a new timestamp should be generated at this point if not test_execution.timestamp: test_execution.timestamp = utils.get_timestamp() # Select which envs to use # The user can define environments in the environments.json file. # The suite/test can be executed in one or more of these environments. # Which environments to use is defined by this order of preference: # 1. envs passed by CLI # 2. envs defined inside the suite # 3. The first env defined # 4. no envs at all # # Note, in the case of 4, the test might fail if it tries # to use env variables cli_envs = test_execution.cli_environments project_envs = environment_manager.get_envs(workspace, project) execution['environments'] = _select_environments(cli_envs, suite_envs, project_envs) # Generate the execution list # # Each test must be executed for each: # * data set # * environment # * driver # # The result is a list that contains all the requested combinations execution_list = _define_execution_list(workspace, project, execution) # create the execution directory # # if this is a suite, the directory takes this structure # reports/<suite_name>/<timestamp>/ # # if this is a single test, the directory takes this structure: # reports/single_tests/<test_name>/<timestamp>/ execution_directory = _create_execution_directory(workspace, project, test_execution.timestamp, test_name=test, suite_name=suite_name, is_suite=is_suite) # for each test, create the test directory # for example, in a suite 'suite1' with a 'test1': # reports/suite1/2017.07.02.19.22.20.001/test1/set_00001/ for test in execution_list: report_directory = report.create_report_directory( execution_directory, test['test_name'], is_suite) test['report_directory'] = report_directory # EXECUTION start_time = time.time() suite_error = False # run suite `before` function if execution['suite_before']: try: execution['suite_before'].__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) if not suite_error: if test_execution.interactive and execution['workers'] != 1: print('WARNING: to run in debug mode, threads must equal one') if execution['workers'] == 1: # run tests serially for test in execution_list: run_test(workspace, project, test['test_name'], test['data_set'], test['driver'], test_execution.settings, test['report_directory']) else: # run tests using multiprocessing multiprocess_executor(execution_list, execution['workers']) # run suite `after` function if execution['suite_after']: try: execution['suite_after'].__call__() except: print('ERROR: suite before function failed') print(traceback.format_exc()) # generate execution_result.json elapsed_time = round(time.time() - start_time, 2) report_parser.generate_execution_report(execution_directory, elapsed_time)