def report_execution_json_download(project, execution, timestamp): report_data = exec_report.get_execution_data(project=project, execution=execution, timestamp=timestamp) json_report = json.dumps(report_data, indent=4) headers = {'Content-disposition': 'attachment; filename=report.json'} return Response(json_report, mimetype='application/json', headers=headers)
def test_run_suite_filter_by_invalid_tag_expression( self, _project_with_tags, test_utils, capsys): """When a invalid tag expression is used a message is displayed to the console, no tests are run, the report is generated, and the execution exists with status code 1 """ _, project = _project_with_tags.activate() suite_name = test_utils.random_numeric_string(10, 'suite') tests = [ _project_with_tags.t.test_alfa_bravo, _project_with_tags.t.test_bravo_charlie ] test_utils.create_suite(project, suite_name, tests=tests) timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp, tags=['sierra = tango']) execution_runner.project = project with pytest.raises(SystemExit): execution_runner.run_suite(suite_name) out, err = capsys.readouterr() expected = ( "InvalidTagExpression: unknown expression <class '_ast.Assign'>, the " "only valid operators for tag expressions are: 'and', 'or' & 'not'" ) assert expected in out data = exec_report.get_execution_data(project=project, suite=suite_name, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def report_get_reports(): user = _get_user_api_or_session() user_projects = user.project_list project = request.args['project'] execution = request.args['execution'] last_days = int(request.args['lastDays']) if project is None or project == 'null': project_list = user_projects else: _verify_permissions(Permissions.REPORTS_ONLY, project) project_list = [project] last_timestamps = report.get_last_execution_timestamps(project_list, execution=execution, last_days=last_days) result = [] for proj, executions in last_timestamps.items(): for exec_, timestamps in executions.items(): for timestamp in timestamps: execution_data = exec_report.get_execution_data(project=proj, execution=exec_, timestamp=timestamp) result.append({ 'project': proj, 'execution': exec_, 'timestamp': timestamp, 'report': execution_data }) # re-sort result = sorted(result, key=lambda x: x['timestamp'], reverse=True) return jsonify(result)
def test_run_with_not_existing_environments(self, project_function, test_utils, capsys): """Run tests with a not existing environment. It should throw an error and finish with status code 1 """ _, project = project_function.activate() test_utils.create_test(project, 'test01') timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner( browsers=['chrome'], timestamp=timestamp, environments=['not_existing']) execution_runner.project = project with pytest.raises(SystemExit) as wrapped_execution: execution_runner.run_directory('') assert wrapped_execution.value.code == 1 out, err = capsys.readouterr() msg = ( 'ERROR: the following environments do not exist for project {}: ' 'not_existing'.format(project)) assert msg in out data = exec_report.get_execution_data(project=project, suite='all', execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def report_execution_json_download(project, suite, execution): report_data = exec_report.get_execution_data(project=project, suite=suite, execution=execution) json_report = json.dumps(report_data, indent=4) headers = { 'Content-disposition': 'attachment; filename={}'.format('report.json') } return Response(json_report, mimetype='application/json', headers=headers)
def report_execution(): project = request.args['project'] execution = request.args['execution'] timestamp = request.args['timestamp'] _verify_permissions(Permissions.REPORTS_ONLY, project) execution_data = exec_report.get_execution_data(project=project, execution=execution, timestamp=timestamp) response = jsonify(execution_data) if execution_data['has_finished']: response.cache_control.max_age = 60 * 60 * 24 * 7 response.cache_control.public = True return response
def test_get_execution_data_from_report_json(self, project_function, test_utils): _, project = project_function.activate() suite_name = 'suite1' test_utils.create_test(project, name='test1') test_utils.create_suite(project, name=suite_name, tests=['test1']) timestamp = test_utils.run_suite(project, suite_name) exec_data = get_execution_data(project=project, suite=suite_name, execution=timestamp) assert len(exec_data['tests']) == 1 assert exec_data['tests'][0]['name'] == 'test1' assert exec_data['total_tests'] == 1 assert exec_data['has_finished'] is True
def project_health(): project = request.args['project'] _verify_permissions(Permissions.REPORTS_ONLY, project) project_data = report.get_last_executions(projects=[project], suite=None, limit=1) health_data = {} for suite, executions in project_data[project].items(): execution_data = exec_report.get_execution_data(project=project, suite=suite, execution=executions[0]) health_data[suite] = { 'execution': executions[0], 'total': execution_data['total_tests'], 'totals_by_result': execution_data['totals_by_result'] } return jsonify(health_data)
def test_run_directory(self, _project_with_tags, capsys): _, project = _project_with_tags.activate() timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp) execution_runner.project = project execution_runner.run_directory('foo') out, err = capsys.readouterr() assert 'Tests found: 2' in out data = exec_report.get_execution_data(project=project, suite='foo', execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 2
def test_get_execution_data_unfinished_execution(self, project_function, test_utils): _, project = project_function.activate() suite_name = 'suite1' test_utils.create_test(project, name='test1') test_utils.create_suite(project, name=suite_name, tests=['test1']) timestamp = test_utils.run_suite(project, suite_name) exec_dir = suite_execution_path(project, suite_name, timestamp) report_path = os.path.join(exec_dir, 'report.json') os.remove(report_path) exec_data = get_execution_data(project=project, suite=suite_name, execution=timestamp) assert len(exec_data['tests']) == 1 assert exec_data['tests'][0]['name'] == 'test1' assert exec_data['total_tests'] == 1 assert exec_data['has_finished'] is False
def test_run_directory_without_tests(self, _project_with_tags, capsys): _, project = _project_with_tags.activate() timestamp = utils.get_timestamp() dirname = 'empty' execution_runner = exc_runner.ExecutionRunner(project, browsers=['chrome'], timestamp=timestamp) execution_runner.run_directory(dirname) out, err = capsys.readouterr() expected = f"No tests were found in {os.path.join('tests', dirname)}" assert expected in out data = exec_report.get_execution_data(project=project, execution=dirname, timestamp=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def test_run_directory_filter_by_tags(self, _project_with_tags, test_utils, capsys): _, project = _project_with_tags.activate() timestamp = utils.get_timestamp() dirname = 'foo' execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp, tags=['alfa', 'bravo']) execution_runner.project = project execution_runner.run_directory(dirname) out, err = capsys.readouterr() assert 'Tests found: 1' in out data = exec_report.get_execution_data(project=project, suite=dirname, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 1
def test_run_suite_without_tests(self, _project_with_tags, test_utils, capsys): _, project = _project_with_tags.activate() suite_name = test_utils.random_numeric_string(10, 'suite') test_utils.create_suite(project, suite_name, tests=[]) timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp) execution_runner.project = project execution_runner.run_suite(suite_name) out, err = capsys.readouterr() assert 'No tests found for suite {}'.format(suite_name) in out data = exec_report.get_execution_data(project=project, suite=suite_name, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def test_run_with_environments(self, project_function, test_utils, capsys): _, project = project_function.activate() environments = json.dumps({'test': {}, 'stage': {}}) environment_manager.save_environments(project, environments) test_utils.create_test(project, 'test01') timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner( browsers=['chrome'], timestamp=timestamp, environments=['test', 'stage']) execution_runner.project = project execution_runner.run_directory('') out, err = capsys.readouterr() assert 'Tests found: 1 (2 sets)' in out data = exec_report.get_execution_data(project=project, suite='all', execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 2
def test_run_suite(self, _project_with_tags, test_utils, capsys): _, project = _project_with_tags.activate() suite_name = test_utils.random_numeric_string(10, 'suite') tests = [ _project_with_tags.t.test_alfa_bravo, _project_with_tags.t.test_bravo_charlie ] test_utils.create_suite(project, suite_name, tests=tests) timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(project, browsers=['chrome'], timestamp=timestamp) execution_runner.run_suite(suite_name) out, err = capsys.readouterr() assert 'Tests found: 2' in out data = exec_report.get_execution_data(project=project, execution=suite_name, timestamp=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 2
def execute_suite(project, suite_name, timestamp=None, ignore_sys_exit=False): if not timestamp: timestamp = utils.get_timestamp() try: timestamp = TestUtils.run_suite(project, suite_name, timestamp) except SystemExit as e: if not ignore_sys_exit: raise e exec_data = get_execution_data(project=project, suite=suite_name, execution=timestamp) exec_dir = suite_execution_path(project, suite_name, timestamp) return { 'exec_dir': exec_dir, 'report_path': os.path.join(exec_dir, 'report.json'), 'suite_name': suite_name, 'timestamp': timestamp, 'exec_data': exec_data }
def generate_html_report(project, execution, timestamp, destination_folder=None, report_name=None, no_images=False): """Generate static HTML report. Report is generated in <report_directory>/<report_name> By default it's generated in <testdir>/projects/<project>/reports/<suite>/<timestamp> Default name is 'report.html' and 'report-no-images.html' """ execution_directory = exec_report.execution_report_path(project, execution, timestamp) if destination_folder is None: destination_folder = execution_directory if not report_name: if no_images: report_name = 'report-no-images' else: report_name = 'report' formatted_date = utils.get_date_time_from_timestamp(timestamp) app = gui.create_app() static_folder = app.static_folder # css paths css_folder = os.path.join(static_folder, 'css') boostrap_css = os.path.join(css_folder, 'bootstrap', 'bootstrap.min.css') main_css = os.path.join(css_folder, 'main.css') report_css = os.path.join(css_folder, 'report.css') # js paths js_folder = os.path.join(static_folder, 'js') main_js = os.path.join(js_folder, 'main.js') jquery_js = os.path.join(js_folder, 'external', 'jquery.min.js') datatables_js = os.path.join(js_folder, 'external', 'datatable', 'datatables.min.js') bootstrap_js = os.path.join(js_folder, 'external', 'bootstrap.min.js') report_execution_js = os.path.join(js_folder, 'report_execution.js') css = { 'bootstrap': open(boostrap_css, encoding='utf-8').read(), 'main': open(main_css, encoding='utf-8').read(), 'report': open(report_css, encoding='utf-8').read() } js = { 'jquery': open(jquery_js, encoding='utf-8').read(), 'datatables': open(datatables_js, encoding='utf-8').read(), 'bootstrap': open(bootstrap_js, encoding='utf-8').read(), 'main': open(main_js, encoding='utf-8').read(), 'report_execution': open(report_execution_js).read() } execution_data = exec_report.get_execution_data(execution_directory) detail_test_data = {} for test in execution_data['tests']: test_detail = exec_report.function_test_execution_result( project, execution, timestamp, test['test_file'], test['test'], test['set_name'], no_screenshots=no_images, encode_screenshots=True ) # testId is test_file + test + set_name test_id = f"{test['test_file']}.{test['test']}" if test['set_name']: test_id = f"{test_id}.{test['set_name']}" detail_test_data[test_id] = test_detail with app.app_context(): html_string = render_template( 'report/report_execution_static.html', project=project, execution=execution, timestamp=timestamp, execution_data=execution_data, detail_test_data=detail_test_data, formatted_date=formatted_date, css=css, js=js, static=True ) _, file_extension = os.path.splitext(report_name) if not file_extension: report_name = f'{report_name}.html' destination = os.path.join(destination_folder, report_name) if not os.path.exists(os.path.dirname(destination)): os.makedirs(os.path.dirname(destination), exist_ok=True) try: with open(destination, 'w', encoding='utf-8') as f: f.write(html_string) except IOError as e: if e.errno == errno.EACCES: print(f'ERROR: cannot write to {destination}, PermissionError (Errno 13)') else: print(f'ERROR: There was an error writing to {destination}') return html_string
def report_execution_json(project, execution, timestamp): json_report = exec_report.get_execution_data(project=project, execution=execution, timestamp=timestamp) return json_report
def generate_junit_report(project_name, execution, timestamp, report_folder=None, report_name=None): """Generate a report in JUnit XML format. Output conforms to https://github.com/jenkinsci/xunit-plugin/blob/master/ src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd """ data = get_execution_data(project=project_name, execution=execution, timestamp=timestamp) totals = data['totals_by_result'] errors = totals.get(Results.CODE_ERROR, 0) failures = totals.get(Results.FAILURE, 0) + totals.get(Results.ERROR, 0) skipped = totals.get(Results.SKIPPED, 0) testsuites_attrs = { 'name': execution, 'errors': str(errors), 'failures': str(failures), 'tests': str(data['total_tests']), 'time': str(data['net_elapsed_time']) } testsuites = ET.Element('testsuites', testsuites_attrs) testsuites_attrs['timestamp'] = timestamp testsuites_attrs['skipped'] = str(skipped) testsuite = ET.SubElement(testsuites, 'testsuite', testsuites_attrs) for test in data['tests']: class_name = test['test_file'] if test['set_name']: class_name = f"{class_name}_{test['set_name']}" test_attrs = { 'name': test['test'], 'classname': class_name, 'time': str(test['elapsed_time']) } testcase = ET.SubElement(testsuite, 'testcase', test_attrs) # testcase nodes can contain 'failure', 'error', and 'skipped' sub-nodes # matching Golem 'error' and 'failure' to JUnit 'failure', # 'code error' to 'error', and 'skipped' to 'skipped' # # A Golem test can have 0 or more errors and 0 or 1 failures. # Correct mapping would be one sub-node for each of these. # The list of errors for a test is currently not available in the # execution json. if test['result'] in (Results.CODE_ERROR, Results.FAILURE, Results.ERROR, Results.SKIPPED): if test['result'] in [Results.ERROR, Results.FAILURE]: error_type = 'failure' elif test['result'] == Results.CODE_ERROR: error_type = 'error' else: error_type = 'skipped' error_data = { 'type': test['result'], 'message': str(test['test_data']) } error_message = ET.SubElement(testcase, error_type, error_data) # add debug log to /test/system-out node log_lines = get_test_debug_log(project_name, execution, timestamp, test['test_file'], test['set_name']) log_string = '\n'.join(log_lines) system_out = ET.SubElement(testcase, 'system-out') system_out.text = _clean_illegal_xml_chars(log_string) xmlstring = ET.tostring(testsuites) doc = minidom.parseString(xmlstring).toprettyxml(indent=' ' * 4, encoding='UTF-8') if not report_folder: report_folder = execution_report_path(project_name, execution, timestamp) if not report_name: report_name = 'report' report_path = os.path.join(report_folder, report_name + '.xml') if not os.path.exists(os.path.dirname(report_path)): os.makedirs(os.path.dirname(report_path), exist_ok=True) try: with open(report_path, 'w', encoding='utf-8') as f: f.write(doc.decode('UTF-8')) except IOError as e: if e.errno == errno.EACCES: print(f'ERROR: cannot write to {report_path}, PermissionError (Errno 13)') else: print(f'ERROR: There was an error writing to {report_path}') return doc
def generate_html_report(project, suite, execution, report_directory=None, report_name=None, no_images=False): """Generate static HTML report. Report is generated in <report_directory>/<report_name> By default it's generated in <testdir>/projects/<project>/reports/<suite>/<timestamp> Default name is 'report.html' and 'report-no-images.html' """ if not report_directory: report_directory = os.path.join(session.testdir, 'projects', project, 'reports', suite, execution) if not report_name: if no_images: report_name = 'report-no-images' else: report_name = 'report' formatted_date = utils.get_date_time_from_timestamp(execution) app = gui.create_app() static_folder = app.static_folder # css paths css_folder = os.path.join(static_folder, 'css') boostrap_css = os.path.join(css_folder, 'bootstrap', 'bootstrap.min.css') main_css = os.path.join(css_folder, 'main.css') report_css = os.path.join(css_folder, 'report.css') # js paths js_folder = os.path.join(static_folder, 'js') main_js = os.path.join(js_folder, 'main.js') jquery_js = os.path.join(js_folder, 'external', 'jquery.min.js') datatables_js = os.path.join(js_folder, 'external', 'datatable', 'datatables.min.js') bootstrap_js = os.path.join(js_folder, 'external', 'bootstrap.min.js') report_execution_js = os.path.join(js_folder, 'report_execution.js') css = { 'bootstrap': open(boostrap_css, encoding='utf-8').read(), 'main': open(main_css, encoding='utf-8').read(), 'report': open(report_css, encoding='utf-8').read() } js = { 'jquery': open(jquery_js, encoding='utf-8').read(), 'datatables': open(datatables_js, encoding='utf-8').read(), 'bootstrap': open(bootstrap_js, encoding='utf-8').read(), 'main': open(main_js, encoding='utf-8').read(), 'report_execution': open(report_execution_js).read() } execution_data = exec_report.get_execution_data(project=project, suite=suite, execution=execution) detail_test_data = {} for test in execution_data['tests']: test_detail = test_report.get_test_case_data(project, test['full_name'], suite=suite, execution=execution, test_set=test['test_set'], is_single=False, encode_screenshots=True, no_screenshots=no_images) detail_test_data[test['test_set']] = test_detail with app.app_context(): html_string = render_template('report/report_execution_static.html', project=project, suite=suite, execution=execution, execution_data=execution_data, detail_test_data=detail_test_data, formatted_date=formatted_date, css=css, js=js, static=True) _, file_extension = os.path.splitext(report_name) if not file_extension: report_name = '{}.html'.format(report_name) destination = os.path.join(report_directory, report_name) if not os.path.exists(os.path.dirname(destination)): os.makedirs(os.path.dirname(destination), exist_ok=True) try: with open(destination, 'w', encoding='utf-8') as f: f.write(html_string) except IOError as e: if e.errno == errno.EACCES: print('ERROR: cannot write to {}, PermissionError (Errno 13)' .format(destination)) else: print('ERROR: There was an error writing to {}'.format(destination)) return html_string
def report_execution_json(project, suite, execution): json_report = exec_report.get_execution_data(project=project, suite=suite, execution=execution) return jsonify(json_report)