def test_run_with_not_existing_environments(self, project_function,
test_utils, capsys):
"""Run tests with a not existing environment.
It should throw an error and finish with status code 1
"""
_, project = project_function.activate()
test_utils.create_test(project, 'test01')
timestamp = utils.get_timestamp()
execution_runner = exc_runner.ExecutionRunner(
browsers=['chrome'],
timestamp=timestamp,
environments=['not_existing'])
execution_runner.project = project
with pytest.raises(SystemExit) as wrapped_execution:
execution_runner.run_directory('')
assert wrapped_execution.value.code == 1
out, err = capsys.readouterr()
msg = (
'ERROR: the following environments do not exist for project {}: '
'not_existing'.format(project))
assert msg in out
data = report_parser.get_execution_data(project=project,
suite='all',
execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 0
def test_run_suite_filter_by_invalid_tag_expression(
self, _project_with_tags, test_utils, capsys):
"""When a invalid tag expression is used a message is displayed
to the console, no tests are run, the report is generated,
and the execution exists with status code 1
"""
_, project = _project_with_tags.activate()
suite_name = test_utils.random_numeric_string(10, 'suite')
tests = [
_project_with_tags.t.test_alfa_bravo,
_project_with_tags.t.test_bravo_charlie
]
test_utils.create_suite(project, suite_name, tests=tests)
timestamp = utils.get_timestamp()
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'],
timestamp=timestamp,
tags=['sierra = tango'])
execution_runner.project = project
with pytest.raises(SystemExit):
execution_runner.run_suite(suite_name)
out, err = capsys.readouterr()
expected = (
"InvalidTagExpression: unknown expression <class '_ast.Assign'>, the "
"only valid operators for tag expressions are: 'and', 'or' & 'not'"
)
assert expected in out
data = report_parser.get_execution_data(project=project,
suite=suite_name,
execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 0
def get_execution_data():
if request.method == 'POST':
project = request.form['project']
suite = request.form['suite']
execution = request.form['execution']
execution_data = report_parser.get_execution_data(workspace=root_path,
project=project,
suite=suite,
execution=execution)
return jsonify(execution_data)
def test_run_directory(self, _project_with_tags, capsys):
_, project = _project_with_tags.activate()
timestamp = utils.get_timestamp()
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp)
execution_runner.project = project
execution_runner.run_directory('foo')
out, err = capsys.readouterr()
assert 'Tests found: 2' in out
data = report_parser.get_execution_data(project=project, suite='foo', execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 2
def report_suite_execution():
project = request.args['project']
suite = request.args['suite']
execution = request.args['execution']
_verify_permissions(Permissions.REPORTS_ONLY, project)
execution_data = report_parser.get_execution_data(project=project, suite=suite,
execution=execution)
response = jsonify(execution_data)
if execution_data['has_finished']:
response.cache_control.max_age = 60 * 60 * 24 * 7
response.cache_control.public = True
return response
def test_run_directory_without_tests(self, _project_with_tags, capsys):
_, project = _project_with_tags.activate()
timestamp = utils.get_timestamp()
dirname = 'empty'
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp)
execution_runner.project = project
execution_runner.run_directory(dirname)
out, err = capsys.readouterr()
expected = 'No tests were found in {}'.format(os.path.join('tests', dirname))
assert expected in out
data = report_parser.get_execution_data(project=project, suite=dirname, execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 0
def test_run_suite_without_tests(self, _project_with_tags, test_utils, capsys):
_, project = _project_with_tags.activate()
suite_name = test_utils.random_numeric_string(10, 'suite')
test_utils.create_suite(project, suite_name, tests=[])
timestamp = utils.get_timestamp()
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'], timestamp=timestamp)
execution_runner.project = project
execution_runner.run_suite(suite_name)
out, err = capsys.readouterr()
assert 'No tests found for suite {}'.format(suite_name) in out
data = report_parser.get_execution_data(project=project, suite=suite_name, execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 0
def test_run_directory_filter_by_tags(self, _project_with_tags, test_utils, capsys):
_, project = _project_with_tags.activate()
timestamp = utils.get_timestamp()
dirname = 'foo'
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'],
timestamp=timestamp,
tags=['alfa', 'bravo'])
execution_runner.project = project
execution_runner.run_directory(dirname)
out, err = capsys.readouterr()
assert 'Tests found: 1' in out
data = report_parser.get_execution_data(project=project, suite=dirname, execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 1
def project_health():
project = request.args['project']
_verify_permissions(Permissions.REPORTS_ONLY, project)
project_data = report_parser.get_last_executions(projects=[project], suite=None,
limit=1)
health_data = {}
for suite, executions in project_data[project].items():
execution_data = report_parser.get_execution_data(project=project,
suite=suite,
execution=executions[0])
health_data[suite] = {
'execution': executions[0],
'total': execution_data['total_tests'],
'totals_by_result': execution_data['totals_by_result']
}
return jsonify(health_data)
def test_run_with_environments(self, project_function, test_utils, capsys):
_, project = project_function.activate()
environments = json.dumps({'test': {}, 'stage': {}})
environment_manager.save_environments(project, environments)
test_utils.create_test(project, [], 'test01')
timestamp = utils.get_timestamp()
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'],
timestamp=timestamp,
environments=['test', 'stage'])
execution_runner.project = project
execution_runner.run_directory('')
out, err = capsys.readouterr()
assert 'Tests found: 1 (2 sets)' in out
data = report_parser.get_execution_data(project=project, suite='all', execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 2
def test_run_suite_filter_by_invalid_tags(self, _project_with_tags, test_utils, capsys):
_, project = _project_with_tags.activate()
suite_name = test_utils.random_numeric_string(10, 'suite')
tests = [_project_with_tags.t.test_alfa_bravo,
_project_with_tags.t.test_bravo_charlie]
test_utils.create_suite(project, suite_name, tests=tests)
timestamp = utils.get_timestamp()
execution_runner = exc_runner.ExecutionRunner(browsers=['chrome'],
timestamp=timestamp,
tags=['sierra', 'tango'])
execution_runner.project = project
execution_runner.run_suite(suite_name)
out, err = capsys.readouterr()
assert 'No tests found with tag(s): sierra, tango' in out
data = report_parser.get_execution_data(project=project, suite=suite_name, execution=timestamp)
assert data['has_finished'] is True
assert data['total_tests'] == 0
def get_project_health_data():
if request.method == 'POST':
project = request.form['project']
project_data = report_parser.get_last_executions(root_path, project=project,
suite=None, limit=1)
health_data = {}
for suite, executions in project_data[project].items():
execution_data = report_parser.get_execution_data(workspace=root_path,
project=project,
suite=suite,
execution=executions[0])
health_data[suite] = {
'execution': executions[0],
'total': execution_data['total_cases'],
'total_ok': execution_data['total_cases_ok'],
'total_fail': execution_data['total_cases_fail'],
'total_pending': execution_data['total_pending']
}
return jsonify(health_data)
def generate_html_report(project, suite, execution, report_directory=None,
report_name=None, no_images=False):
"""Generate static HTML report.
Report is generated in <report_directory>/<report_name>
By default it's generated in <testdir>/projects/<project>/reports/<suite>/<timestamp>
Default name is 'report.html' and 'report-no-images.html'
"""
from golem import gui
app = gui.app
if not report_directory:
report_directory = os.path.join(session.testdir, 'projects', project,
'reports', suite, execution)
if not report_name:
if no_images:
report_name = 'report-no-images'
else:
report_name = 'report'
formatted_date = report_parser.get_date_time_from_timestamp(execution)
css = {}
js = {}
boostrap_path = os.path.join(app.static_folder, 'css', 'bootstrap', 'bootstrap.min.css')
datatables_js = os.path.join(app.static_folder, 'js', 'external', 'datatable', 'datatables.min.js')
css['bootstrap'] = open(boostrap_path).read()
css['main'] = open(os.path.join(app.static_folder, 'css', 'main.css')).read()
css['report'] = open(os.path.join(app.static_folder, 'css', 'report.css')).read()
js['jquery'] = open(os.path.join(app.static_folder, 'js', 'external', 'jquery.min.js')).read()
js['datatables'] = open(datatables_js).read()
js['bootstrap'] = open(os.path.join(app.static_folder, 'js', 'external', 'bootstrap.min.js')).read()
js['main'] = open(os.path.join(app.static_folder, 'js', 'main.js')).read()
js['report_execution'] = open(os.path.join(app.static_folder, 'js', 'report_execution.js')).read()
execution_data = report_parser.get_execution_data(project=project, suite=suite, execution=execution)
detail_test_data = {}
for test in execution_data['tests']:
test_detail = report_parser.get_test_case_data(project, test['full_name'],
suite=suite, execution=execution,
test_set=test['test_set'],
is_single=False,
encode_screenshots=True,
no_screenshots=no_images)
detail_test_data[test['test_set']] = test_detail
with app.app_context():
html_string = render_template('report/report_execution_static.html', project=project,
suite=suite, execution=execution, execution_data=execution_data,
detail_test_data=detail_test_data, formatted_date=formatted_date,
css=css, js=js, static=True)
_, file_extension = os.path.splitext(report_name)
if not file_extension:
report_name = '{}.html'.format(report_name)
destination = os.path.join(report_directory, report_name)
if not os.path.exists(os.path.dirname(destination)):
os.makedirs(os.path.dirname(destination), exist_ok=True)
try:
with open(destination, 'w') as f:
f.write(html_string)
except IOError as e:
if e.errno == errno.EACCES:
print('ERROR: cannot write to {}, PermissionError (Errno 13)'
.format(destination))
else:
print('ERROR: There was an error writing to {}'.format(destination))
return html_string
