def test_run_with_not_existing_environments(self, project_function, test_utils, capsys): """Run tests with a not existing environment. It should throw an error and finish with status code 1 """ _, project = project_function.activate() test_utils.create_test(project, 'test01') timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner( browsers=['chrome'], timestamp=timestamp, environments=['not_existing']) execution_runner.project = project with pytest.raises(SystemExit) as wrapped_execution: execution_runner.run_directory('') assert wrapped_execution.value.code == 1 out, err = capsys.readouterr() msg = ( 'ERROR: the following environments do not exist for project {}: ' 'not_existing'.format(project)) assert msg in out data = report_parser.get_execution_data(project=project, suite='all', execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def test_run_suite_filter_by_invalid_tag_expression( self, _project_with_tags, test_utils, capsys): """When a invalid tag expression is used a message is displayed to the console, no tests are run, the report is generated, and the execution exists with status code 1 """ _, project = _project_with_tags.activate() suite_name = test_utils.random_numeric_string(10, 'suite') tests = [ _project_with_tags.t.test_alfa_bravo, _project_with_tags.t.test_bravo_charlie ] test_utils.create_suite(project, suite_name, tests=tests) timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp, tags=['sierra = tango']) execution_runner.project = project with pytest.raises(SystemExit): execution_runner.run_suite(suite_name) out, err = capsys.readouterr() expected = ( "InvalidTagExpression: unknown expression <class '_ast.Assign'>, the " "only valid operators for tag expressions are: 'and', 'or' & 'not'" ) assert expected in out data = report_parser.get_execution_data(project=project, suite=suite_name, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def get_execution_data(): if request.method == 'POST': project = request.form['project'] suite = request.form['suite'] execution = request.form['execution'] execution_data = report_parser.get_execution_data(workspace=root_path, project=project, suite=suite, execution=execution) return jsonify(execution_data)
def test_run_directory(self, _project_with_tags, capsys): _, project = _project_with_tags.activate() timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp) execution_runner.project = project execution_runner.run_directory('foo') out, err = capsys.readouterr() assert 'Tests found: 2' in out data = report_parser.get_execution_data(project=project, suite='foo', execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 2
def report_suite_execution(): project = request.args['project'] suite = request.args['suite'] execution = request.args['execution'] _verify_permissions(Permissions.REPORTS_ONLY, project) execution_data = report_parser.get_execution_data(project=project, suite=suite, execution=execution) response = jsonify(execution_data) if execution_data['has_finished']: response.cache_control.max_age = 60 * 60 * 24 * 7 response.cache_control.public = True return response
def test_run_directory_without_tests(self, _project_with_tags, capsys): _, project = _project_with_tags.activate() timestamp = utils.get_timestamp() dirname = 'empty' execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp) execution_runner.project = project execution_runner.run_directory(dirname) out, err = capsys.readouterr() expected = 'No tests were found in {}'.format(os.path.join('tests', dirname)) assert expected in out data = report_parser.get_execution_data(project=project, suite=dirname, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def test_run_suite_without_tests(self, _project_with_tags, test_utils, capsys): _, project = _project_with_tags.activate() suite_name = test_utils.random_numeric_string(10, 'suite') test_utils.create_suite(project, suite_name, tests=[]) timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp) execution_runner.project = project execution_runner.run_suite(suite_name) out, err = capsys.readouterr() assert 'No tests found for suite {}'.format(suite_name) in out data = report_parser.get_execution_data(project=project, suite=suite_name, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def test_run_directory_filter_by_tags(self, _project_with_tags, test_utils, capsys): _, project = _project_with_tags.activate() timestamp = utils.get_timestamp() dirname = 'foo' execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp, tags=['alfa', 'bravo']) execution_runner.project = project execution_runner.run_directory(dirname) out, err = capsys.readouterr() assert 'Tests found: 1' in out data = report_parser.get_execution_data(project=project, suite=dirname, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 1
def project_health(): project = request.args['project'] _verify_permissions(Permissions.REPORTS_ONLY, project) project_data = report_parser.get_last_executions(projects=[project], suite=None, limit=1) health_data = {} for suite, executions in project_data[project].items(): execution_data = report_parser.get_execution_data(project=project, suite=suite, execution=executions[0]) health_data[suite] = { 'execution': executions[0], 'total': execution_data['total_tests'], 'totals_by_result': execution_data['totals_by_result'] } return jsonify(health_data)
def test_run_with_environments(self, project_function, test_utils, capsys): _, project = project_function.activate() environments = json.dumps({'test': {}, 'stage': {}}) environment_manager.save_environments(project, environments) test_utils.create_test(project, [], 'test01') timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp, environments=['test', 'stage']) execution_runner.project = project execution_runner.run_directory('') out, err = capsys.readouterr() assert 'Tests found: 1 (2 sets)' in out data = report_parser.get_execution_data(project=project, suite='all', execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 2
def test_run_suite_filter_by_invalid_tags(self, _project_with_tags, test_utils, capsys): _, project = _project_with_tags.activate() suite_name = test_utils.random_numeric_string(10, 'suite') tests = [_project_with_tags.t.test_alfa_bravo, _project_with_tags.t.test_bravo_charlie] test_utils.create_suite(project, suite_name, tests=tests) timestamp = utils.get_timestamp() execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp, tags=['sierra', 'tango']) execution_runner.project = project execution_runner.run_suite(suite_name) out, err = capsys.readouterr() assert 'No tests found with tag(s): sierra, tango' in out data = report_parser.get_execution_data(project=project, suite=suite_name, execution=timestamp) assert data['has_finished'] is True assert data['total_tests'] == 0
def get_project_health_data(): if request.method == 'POST': project = request.form['project'] project_data = report_parser.get_last_executions(root_path, project=project, suite=None, limit=1) health_data = {} for suite, executions in project_data[project].items(): execution_data = report_parser.get_execution_data(workspace=root_path, project=project, suite=suite, execution=executions[0]) health_data[suite] = { 'execution': executions[0], 'total': execution_data['total_cases'], 'total_ok': execution_data['total_cases_ok'], 'total_fail': execution_data['total_cases_fail'], 'total_pending': execution_data['total_pending'] } return jsonify(health_data)
def generate_html_report(project, suite, execution, report_directory=None, report_name=None, no_images=False): """Generate static HTML report. Report is generated in <report_directory>/<report_name> By default it's generated in <testdir>/projects/<project>/reports/<suite>/<timestamp> Default name is 'report.html' and 'report-no-images.html' """ from golem import gui app = gui.app if not report_directory: report_directory = os.path.join(session.testdir, 'projects', project, 'reports', suite, execution) if not report_name: if no_images: report_name = 'report-no-images' else: report_name = 'report' formatted_date = report_parser.get_date_time_from_timestamp(execution) css = {} js = {} boostrap_path = os.path.join(app.static_folder, 'css', 'bootstrap', 'bootstrap.min.css') datatables_js = os.path.join(app.static_folder, 'js', 'external', 'datatable', 'datatables.min.js') css['bootstrap'] = open(boostrap_path).read() css['main'] = open(os.path.join(app.static_folder, 'css', 'main.css')).read() css['report'] = open(os.path.join(app.static_folder, 'css', 'report.css')).read() js['jquery'] = open(os.path.join(app.static_folder, 'js', 'external', 'jquery.min.js')).read() js['datatables'] = open(datatables_js).read() js['bootstrap'] = open(os.path.join(app.static_folder, 'js', 'external', 'bootstrap.min.js')).read() js['main'] = open(os.path.join(app.static_folder, 'js', 'main.js')).read() js['report_execution'] = open(os.path.join(app.static_folder, 'js', 'report_execution.js')).read() execution_data = report_parser.get_execution_data(project=project, suite=suite, execution=execution) detail_test_data = {} for test in execution_data['tests']: test_detail = report_parser.get_test_case_data(project, test['full_name'], suite=suite, execution=execution, test_set=test['test_set'], is_single=False, encode_screenshots=True, no_screenshots=no_images) detail_test_data[test['test_set']] = test_detail with app.app_context(): html_string = render_template('report/report_execution_static.html', project=project, suite=suite, execution=execution, execution_data=execution_data, detail_test_data=detail_test_data, formatted_date=formatted_date, css=css, js=js, static=True) _, file_extension = os.path.splitext(report_name) if not file_extension: report_name = '{}.html'.format(report_name) destination = os.path.join(report_directory, report_name) if not os.path.exists(os.path.dirname(destination)): os.makedirs(os.path.dirname(destination), exist_ok=True) try: with open(destination, 'w') as f: f.write(html_string) except IOError as e: if e.errno == errno.EACCES: print('ERROR: cannot write to {}, PermissionError (Errno 13)' .format(destination)) else: print('ERROR: There was an error writing to {}'.format(destination)) return html_string