def finalize(self): test_end_time = time.time() test_elapsed_time = round(test_end_time - self.test_start_time, 2) if self.result['result'] not in [ ResultsEnum.CODE_ERROR, ResultsEnum.FAILURE ]: if execution.errors: self.result['result'] = ResultsEnum.ERROR if self.result['result'] is None: self.result['result'] = ResultsEnum.SUCCESS execution.logger.info('Test Result: {}'.format( self.result['result'].upper())) self.result['description'] = execution.description self.result['steps'] = execution.steps self.result['errors'] = execution.errors self.result['test_elapsed_time'] = test_elapsed_time self.result['test_timestamp'] = self.test_timestamp self.result['browser'] = execution.browser_definition['name'] self.result['browser_full_name'] = execution.browser_definition[ 'full_name'] # Report a test has failed in the test execution, this will later determine the exit status _error_codes = [ ResultsEnum.CODE_ERROR, ResultsEnum.ERROR, ResultsEnum.FAILURE ] if self.execution_has_failed_tests is not None and self.result[ 'result'] in _error_codes: self.execution_has_failed_tests.value = True report.generate_report(self.report_directory, self.test.name, execution.data, self.result) execution_logger.reset_logger(execution.logger)
def finalize(self): test_end_time = time.time() test_elapsed_time = round(test_end_time - self.test_start_time, 2) if self.result['result'] not in [ ResultsEnum.CODE_ERROR, ResultsEnum.FAILURE ]: if execution.errors: self.result['result'] = ResultsEnum.ERROR else: self.result['result'] = ResultsEnum.SUCCESS execution.logger.info('Test Result: {}'.format( self.result['result'].upper())) self.result['description'] = execution.description self.result['steps'] = execution.steps self.result['errors'] = execution.errors self.result['test_elapsed_time'] = test_elapsed_time self.result['test_timestamp'] = self.test_timestamp self.result['browser'] = execution.browser_definition['name'] self.result['browser_full_name'] = execution.browser_definition[ 'full_name'] report.generate_report(self.report_directory, self.test_name, execution.data, self.result) execution_logger.reset_logger(execution.logger) execution._reset()
def test_generate_report_with_env(self, permanent_project_fixture): project = permanent_project_fixture['name'] testdir = permanent_project_fixture['testdir'] timestamp = utils.get_timestamp() test_name = 'testing_report_003' exec_dir = report.create_execution_directory(testdir, project, timestamp, test_name=test_name) report_dir = report.create_report_directory(exec_dir, test_name, is_suite=True) test_data = { 'env': { 'name': 'env01', 'url': '1.1.1.1' }, 'var2': 'value2' } test_data = test_runner.Data(test_data) result = { 'result': 'pass', 'error': '', 'description': 'description of the test', 'steps': ['step1', 'step2'], 'test_elapsed_time': 22.22, 'test_timestamp': '2018.02.04.02.16.42.729', 'browser': 'chrome', 'browser_full_name': '', 'set_name': 'set_001', } report.generate_report(report_dir, test_name, test_data, result) expected = { 'test_case': test_name, 'result': 'pass', 'steps': ['step1', 'step2'], 'description': 'description of the test', 'error': '', 'short_error': '', 'test_elapsed_time': 22.22, 'test_timestamp': '2018.02.04.02.16.42.729', 'browser': 'chrome', 'environment': 'env01', 'set_name': 'set_001', 'test_data': { 'env': "{'name': 'env01', 'url': '1.1.1.1'}", 'var2': "'value2'" } } path = os.path.join(report_dir, 'report.json') with open(path) as report_file: actual = json.load(report_file) assert actual == expected
def test_generate_report(self, project_session): project = project_session['name'] testdir = project_session['testdir'] timestamp = utils.get_timestamp() test_name = 'testing_report_002' exec_dir = report.create_execution_directory(testdir, project, timestamp, test_name=test_name) report_dir = report.create_report_directory(exec_dir, test_name, is_suite=True) test_data = { 'var1': 'value1', 'var2': 'value2' } result = { 'result': 'pass', 'error': '', 'description': 'description of the test', 'steps': ['step1', 'step2'], 'test_elapsed_time': 22.22, 'test_timestamp': '2018.02.04.02.16.42.729', 'browser': 'chrome', 'browser_full_name': '', 'set_name': 'set_001', } report.generate_report(report_dir, test_name, test_data, result, timestamp, hash(timestamp + str(test_data))) expected = { 'test_case': test_name, 'result': 'pass', 'steps': ['step1', 'step2'], 'description': 'description of the test', 'error': '', 'short_error': '', 'test_elapsed_time': 22.22, 'test_timestamp': '2018.02.04.02.16.42.729', 'browser': 'chrome', 'environment': '', 'set_name': 'set_001', 'suite_timestamp': timestamp, 'test_id': hash(timestamp + str(test_data)), 'user': getpass.getuser(), 'hostname': socket.gethostname(), 'test_data': { 'var1': "'value1'", 'var2': "'value2'" } } path = os.path.join(report_dir, 'report.json') with open(path) as report_file: actual = json.load(report_file) assert actual == expected
def test_generate_report_with_env(self, project_session): timestamp = utils.get_timestamp() test_name = 'testing_report_003' exec_dir = report.create_execution_directory(project_session.testdir, project_session.name, timestamp, test_name=test_name) report_dir = report.create_report_directory(exec_dir, test_name, is_suite=True) test_data = { 'env': { 'name': 'env01', 'url': '1.1.1.1' }, 'var2': 'value2' } test_data = test_runner.Data(test_data) result = { 'result': 'success', 'errors': [], 'description': 'description of the test', 'steps': [ {'message': 'step1', 'screenshot': None, 'error': None}, {'message': 'step2', 'screenshot': None, 'error': None} ], 'test_elapsed_time': 22.22, 'test_timestamp': '2018.02.04.02.16.42.729', 'browser': 'chrome', 'browser_full_name': '', 'set_name': 'set_001', } report.generate_report(report_dir, test_name, test_data, result) path = os.path.join(report_dir, 'report.json') with open(path) as report_file: actual = json.load(report_file) assert len(actual.items()) == 11 assert actual['test_case'] == test_name assert actual['result'] == 'success' assert actual['steps'][0]['message'] == 'step1' assert actual['steps'][1]['message'] == 'step2' assert actual['description'] == 'description of the test' assert actual['errors'] == [] assert actual['test_elapsed_time'] == 22.22 assert actual['test_timestamp'] == '2018.02.04.02.16.42.729' assert actual['browser'] == 'chrome' assert actual['environment'] == 'env01' assert actual['set_name'] == 'set_001' test_data_a = "{'url': '1.1.1.1', 'name': 'env01'}" test_data_b = "{'name': 'env01', 'url': '1.1.1.1'}" assert actual['test_data']['env'] in [test_data_a, test_data_b] assert actual['test_data']['var2'] == "'value2'"
def run_test(workspace, project, test_name, test_data, browser, settings, report_directory): ''' runs a single test case by name''' result = { 'result': 'pass', 'error': '', 'description': '', 'steps': [], 'test_elapsed_time': None, 'test_timestamp': None, 'browser': '', 'browser_full_name': '', 'set_name': '', } from golem.core import execution_logger from golem import actions from golem import execution # convert test_data to data obj class Data(dict): """dot notation access to dictionary attributes""" def __getattr__(*args): val = dict.get(*args) return Data(val) if type(val) is dict else val __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ execution.data = Data(test_data) # set set_name # set name is the value of 'set_name' if present in the data table # if it is not present, use the value of the first column in the data table # if there's no data in the data table, leave set_name as '' if 'set_name' in test_data: result['set_name'] = test_data['set_name'] elif test_data: result['set_name'] = test_data[next(iter(test_data))] logger = execution_logger.get_logger(report_directory, settings['console_log_level'], settings['file_log_level'], settings['log_all_events']) execution.logger = logger # Print execution info to console logger.info('Test execution started: {}'.format(test_name)) logger.info('Browser: {}'.format(browser['name'])) if 'env' in test_data: if 'name' in test_data['env']: logger.info('Environment: {}'.format(test_data['env']['name'])) if test_data: data_string = '\n' for key, value in test_data.items(): if key == 'env': if 'url' in value: data_string += ' {}: {}\n'.format('url', value['url']) else: data_string += ' {}: {}\n'.format(key, value) logger.info('Using data: {}'.format(data_string)) test_timestamp = utils.get_timestamp() test_start_time = time.time() execution.project = project execution.workspace = workspace execution.browser_definition = browser execution.settings = settings execution.report_directory = report_directory # add the 'project' directory to python path # so it's possible to make relative imports from the test # example, some_test.py # from pages import some_page sys.path.append(os.path.join(workspace, 'projects', project)) test_module = None try: if '/' in test_name: test_name = test_name.replace('/', '.') test_module = importlib.import_module('projects.{0}.tests.{1}'.format( project, test_name)) # import the page objects into the test module if hasattr(test_module, 'pages'): for page in test_module.pages: test_module = utils.generate_page_object_module( project, test_module, page, page.split('.')) # import logger into the test module setattr(test_module, 'logger', execution.logger) # import actions into the test module for action in dir(actions): setattr(test_module, action, getattr(actions, action)) # log description if hasattr(test_module, 'description'): execution.description = test_module.description else: logger.info('Test does not have description') # run setup method if hasattr(test_module, 'setup'): test_module.setup(execution.data) else: logger.info('Test does not have setup function') if hasattr(test_module, 'test'): test_module.test(execution.data) else: raise Exception('Test does not have test function') if settings['screenshot_on_end'] and execution.browser: actions.capture('test end') except: result['result'] = 'fail' result['error'] = traceback.format_exc() try: if settings['screenshot_on_error'] and execution.browser: actions.capture('error') except: # if the test failed and driver is not available # capture screenshot is not possible, continue pass logger.error('An error ocurred:', exc_info=True) try: if hasattr(test_module, 'teardown'): test_module.teardown(execution.data) else: logger.info('Test does not have a teardown function') except: result['result'] = 'fail' result['error'] += '\n\nteardown failed' result['error'] += '\n' + traceback.format_exc() logger.error('An error ocurred in the teardown:', exc_info=True) # if there is no teardown or teardown failed or it did not close the driver, # let's try to close the driver manually if execution.browser: try: execution.browser.quit() except: # if this fails, we have lost control over the webdriver window # and we are not going to be able to close it logger.error('There was an error closing the driver') logger.error(traceback.format_exc()) finally: execution.browser = None test_end_time = time.time() test_elapsed_time = round(test_end_time - test_start_time, 2) if not result['error']: logger.info('Test passed') result['description'] = execution.description result['steps'] = execution.steps result['test_elapsed_time'] = test_elapsed_time result['test_timestamp'] = test_timestamp result['browser'] = execution.browser_definition['name'] result['browser_full_name'] = execution.browser_definition['full_name'] # remove golem.execution from sys.modules to guarantee thread safety #sys.modules['golem.execution'] = None report.generate_report(report_directory, test_name, execution.data, result) del execution return
def run_test(workspace, project, test_name, test_data, browser, settings, report_directory): ''' runs a single test case by name''' result = { 'result': 'pass', 'error': '', 'description': '', 'steps': [], 'test_elapsed_time': None, 'test_timestamp': None, 'browser': '', 'browser_full_name': '', 'set_name': '', } from golem.core import execution_logger from golem import actions from golem import execution # convert test_data to data obj class Data(dict): """dot notation access to dictionary attributes""" def __getattr__(*args): val = dict.get(*args) return Data(val) if type(val) is dict else val __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ execution.data = Data(test_data) # set set_name # set name is the value of 'set_name' if present in the data table # if it is not present, use the value of the first column in the data table # if there's no data in the data table, leave set_name as '' _set_name = '' if 'set_name' in test_data: _set_name = test_data['set_name'] elif test_data: data_without_env = dict(test_data) data_without_env.pop('env', None) if data_without_env: _set_name = test_data[next(iter(data_without_env))] result['set_name'] = _set_name logger = execution_logger.get_logger(report_directory, settings['console_log_level'], settings['log_all_events']) execution.logger = logger # Print execution info to console logger.info('Test execution started: {}'.format(test_name)) logger.info('Browser: {}'.format(browser['name'])) if 'env' in test_data: if 'name' in test_data['env']: logger.info('Environment: {}'.format(test_data['env']['name'])) if test_data: data_string = '\n' for key, value in test_data.items(): if key == 'env': if 'url' in value: data_string += ' {}: {}\n'.format('url', value['url']) else: data_string += ' {}: {}\n'.format(key, value) logger.info('Using data: {}'.format(data_string)) test_timestamp = utils.get_timestamp() test_start_time = time.time() execution.project = project execution.workspace = workspace execution.browser_definition = browser execution.settings = settings execution.report_directory = report_directory # add the 'project' directory to python path # so it's possible to make relative imports from the test # example, some_test.py # from pages import some_page sys.path.append(os.path.join(workspace, 'projects', project)) test_module = None try: if '/' in test_name: test_name = test_name.replace('/', '.') test_module = importlib.import_module( 'projects.{0}.tests.{1}'.format(project, test_name)) # import the page objects into the test module if hasattr(test_module, 'pages'): for page in test_module.pages: test_module = utils.generate_page_object_module(project, test_module, page, page.split('.')) # import logger into the test module setattr(test_module, 'logger', execution.logger) # import actions into the test module for action in dir(actions): setattr(test_module, action, getattr(actions, action)) # log description if hasattr(test_module, 'description'): execution.description = test_module.description else: logger.info('Test does not have description') # run setup method if hasattr(test_module, 'setup'): test_module.setup(execution.data) else: logger.info('Test does not have setup function') if hasattr(test_module, 'test'): test_module.test(execution.data) else: raise Exception('Test does not have test function') if settings['screenshot_on_end'] and execution.browser: actions.capture('test end') except: result['result'] = 'fail' result['error'] = traceback.format_exc() try: if settings['screenshot_on_error'] and execution.browser: actions.capture('error') except: # if the test failed and driver is not available # capture screenshot is not possible, continue pass logger.error('An error ocurred:', exc_info=True) try: if hasattr(test_module, 'teardown'): test_module.teardown(execution.data) else: logger.info('Test does not have a teardown function') except: result['result'] = 'fail' result['error'] += '\n\nteardown failed' result['error'] += '\n' + traceback.format_exc() logger.error('An error ocurred in the teardown:', exc_info=True) # if there is no teardown or teardown failed or it did not close the driver, # let's try to close the driver manually if execution.browser: try: for browser, driver in execution.browsers.items(): driver.quit() except: # if this fails, we have lost control over the webdriver window # and we are not going to be able to close it logger.error('There was an error closing the driver') logger.error(traceback.format_exc()) finally: execution.browser = None test_end_time = time.time() test_elapsed_time = round(test_end_time - test_start_time, 2) if not result['error']: logger.info('Test passed') result['description'] = execution.description result['steps'] = execution.steps result['test_elapsed_time'] = test_elapsed_time result['test_timestamp'] = test_timestamp result['browser'] = execution.browser_definition['name'] result['browser_full_name'] = execution.browser_definition['full_name'] # remove golem.execution from sys.modules to guarantee thread safety #sys.modules['golem.execution'] = None report.generate_report(report_directory, test_name, execution.data, result) # del execution sys.modules['golem.execution'] = None return
def run_test(workspace, project, test_name, test_data, driver, settings, report_directory): ''' runs a single test case by name''' result = { 'result': 'pass', 'error': '', 'description': '', 'steps': [], 'test_elapsed_time': None, 'test_timestamp': None, 'browser': driver } from golem.core import execution_logger from golem import actions # convert test_data to data obj # TODO convert data dict to data obj class data: pass new_data_class = data() for key, value in test_data.items(): setattr(new_data_class, key, value) execution_logger.get_logger(report_directory, settings['console_log_level'], settings['file_log_level'], settings['log_all_events']) execution_logger.logger.info( 'Test execution started: {}'.format(test_name)) execution_logger.logger.info('Driver: {}'.format(driver)) if test_data: data_string = '\n' for key, value in test_data.items(): data_string += ' {}: {}\n'.format(key, value) execution_logger.logger.info('Using data: {}'.format(data_string)) test_timestamp = utils.get_timestamp() test_start_time = time.time() golem.core.project = project golem.core.workspace = workspace golem.core.test_data = new_data_class golem.core.driver_name = driver golem.core.set_settings(settings) golem.core.report_directory = report_directory test_module = None try: test_module = importlib.import_module('projects.{0}.tests.{1}'.format( project, test_name)) # import the page objects into the test module if hasattr(test_module, 'pages'): for page in test_module.pages: test_module = utils.generate_page_object_module( project, test_module, page, page.split('.')) # import logger into the test module setattr(test_module, 'logger', golem.core.execution_logger) # import actions into the test module for action in dir(golem.actions): setattr(test_module, action, getattr(golem.actions, action)) # log description if hasattr(test_module, 'description'): golem.core.execution_logger.description = test_module.description else: execution_logger.logger.info('Test does not have description') # run setup method if hasattr(test_module, 'setup'): test_module.setup(golem.core.test_data) else: execution_logger.logger.info('Test does not have setup function') if hasattr(test_module, 'test'): test_module.test(golem.core.test_data) else: raise Exception('Test does not have test function') except: result['result'] = 'fail' result['error'] = traceback.format_exc() try: if settings['screenshot_on_error'] and golem.core.driver: actions.capture('error') except: # if the test failed and driver is not available # capture screenshot is not possible, continue pass execution_logger.logger.error('An error ocurred:', exc_info=True) try: if hasattr(test_module, 'teardown'): test_module.teardown(golem.core.test_data) else: execution_logger.logger.info( 'Test does not have a teardown function') except: result['result'] = 'fail' result['error'] += '\n\nteardown failed' result['error'] += '\n' + traceback.format_exc() execution_logger.logger.error('An error ocurred in the teardown:', exc_info=True) # if there is no teardown or teardown failed or it did not close the driver, # let's try to close the driver manually if golem.core.driver: try: golem.core.driver.quit() except: # if this fails, we have lost control over the webdriver window # and we are not going to be able to close it execution_logger.logger.error( 'There was an error closing the driver') execution_logger.logger.error(traceback.format_exc()) finally: golem.core.driver = None test_end_time = time.time() test_elapsed_time = round(test_end_time - test_start_time, 2) if not result['error']: execution_logger.logger.info('Test passed') result['description'] = execution_logger.description result['steps'] = execution_logger.steps result['test_elapsed_time'] = test_elapsed_time result['test_timestamp'] = test_timestamp result['browser'] = golem.core.get_selected_driver() execution_logger.description = None execution_logger.steps = [] execution_logger.screenshots = {} report.generate_report(report_directory, test_name, golem.core.test_data, result) return
def test_runner(workspace, project, test_case_name, test_data, suite_name, suite_data, suite_timestamp, settings): ''' runs a single test case by name''' result = { 'result': 'pass', 'error': None, 'description': None, 'steps': None, 'test_elapsed_time': None, 'test_timestamp': None} import execution_logger instance = None test_timestamp = utils.get_timestamp() test_start_time = time.time() golem.core.set_settings(settings) # create a directory to store report.json and screenshots report_directory = report.create_report_directory(workspace, project, test_case_name, suite_name, suite_timestamp) try: test_class = utils.get_test_case_class( project, test_case_name) instance = test_class() if hasattr(instance, 'setup'): instance.setup() else: raise Exception if hasattr(instance, 'test'): instance.test(test_data) else: raise Exception except: result['result'] = 'fail' result['error'] = traceback.format_exc() if settings['screenshot_on_error']: actions.capture('error') print dir(traceback) print traceback.print_exc() try: if hasattr(instance, 'teardown'): instance.teardown() else: raise Exception except: result['result'] = 'fail' result['error'] = 'teardown failed' test_end_time = time.time() test_elapsed_time = round(test_end_time - test_start_time, 2) result['description'] = execution_logger.description result['steps'] = execution_logger.steps result['test_elapsed_time'] = test_elapsed_time result['test_timestamp'] = test_timestamp result['screenshots'] = execution_logger.screenshots report.generate_report(report_directory, test_case_name, test_data, result) return result