def delete_execution_timestamp(project, execution, timestamp):
errors = []
path = execution_report_path(project, execution, timestamp)
if os.path.isdir(path):
try:
shutil.rmtree(path)
except Exception as e:
errors.append(repr(e))
else:
errors.append(
f'Execution for {project} {execution} {timestamp} does not exist')
return errors
def test_parse_execution_data_exec_dir(self, project_function, test_utils):
_, project = project_function.activate()
suite_name = 'suite1'
test_utils.create_test(project, name='test1')
test_utils.create_suite(project, name=suite_name, tests=['test1'])
timestamp = test_utils.run_suite(project, suite_name)
exec_dir = execution_report_path(project, suite_name, timestamp)
exec_data = _parse_execution_data(execution_directory=exec_dir)
assert len(exec_data['tests']) == 1
assert exec_data['tests'][0]['test_file'] == 'test1'
assert exec_data['total_tests'] == 1
def test_file_report_dir(test_name,
project=None,
execution=None,
timestamp=None,
set_name='',
execdir=None):
if execdir is None:
execdir = execution_report.execution_report_path(
project, execution, timestamp)
if set_name:
folder_name = '{}.{}'.format(test_name, set_name)
else:
folder_name = test_name
return os.path.join(execdir, folder_name)
def get_or_generate_junit_report(project, execution, timestamp):
"""Get the JUnit XML report as a string.
If it does not exist, generate it first.
Report is generated at:
<testdir>/projects/<project>/reports/<execution>/<timestamp>/report.xml
"""
report_filename = 'report'
report_directory = execution_report_path(project, execution, timestamp)
report_filepath = os.path.join(report_directory, report_filename + '.xml')
if os.path.isfile(report_filepath):
xml_string = open(report_filepath, encoding='utf-8').read()
else:
xml_string = generate_junit_report(project, execution, timestamp)
return xml_string
def test_get_execution_data_unfinished_execution(self, project_function, test_utils):
_, project = project_function.activate()
suite_name = 'suite1'
test_utils.create_test(project, name='test1')
test_utils.create_suite(project, name=suite_name, tests=['test1'])
timestamp = test_utils.run_suite(project, suite_name)
exec_dir = execution_report_path(project, suite_name, timestamp)
report_path = os.path.join(exec_dir, 'report.json')
os.remove(report_path)
exec_data = get_execution_data(project=project, execution=suite_name, timestamp=timestamp)
assert len(exec_data['tests']) == 1
assert exec_data['tests'][0]['test_file'] == 'test1'
assert exec_data['total_tests'] == 1
assert exec_data['has_finished'] is False
def test_golem_run_suite(self, project_session, test_utils, caplog):
_, project = project_session.activate()
test_name = test_utils.create_random_test(project)
suite_name = test_utils.random_string()
test_utils.create_suite(project, suite_name, tests=[test_name])
timestamp = utils.get_timestamp()
commands.run_command(project=project,
test_query=suite_name,
timestamp=timestamp)
records = caplog.records
assert records[0].message == f'Test execution started: {test_name}'
assert records[3].message == 'Test Result: SUCCESS'
# the execution report is created for suite
path = os.path.join(
execution_report.execution_report_path(project, suite_name,
timestamp), 'report.json')
assert os.path.isfile(path)
def get_or_generate_html_report(project, execution, timestamp, no_images=False):
"""Get the HTML report as a string.
If it does not exist, generate it:
Report is generated at
<testdir>/projects/<project>/reports/<execution>/<timestamp>/report.html|report-no-images.html
"""
if no_images:
report_filename = 'report-no-images'
else:
report_filename = 'report'
report_directory = exec_report.execution_report_path(project, execution, timestamp)
report_filepath = os.path.join(report_directory, report_filename + '.html')
if os.path.isfile(report_filepath):
html_string = open(report_filepath, encoding='utf-8').read()
else:
html_string = generate_html_report(project, execution, timestamp,
report_name=report_filename,
no_images=no_images)
return html_string
def execute_suite(project,
suite_name,
timestamp=None,
ignore_sys_exit=False):
if not timestamp:
timestamp = utils.get_timestamp()
try:
timestamp = TestUtils.run_suite(project, suite_name, timestamp)
except SystemExit as e:
if not ignore_sys_exit:
raise e
exec_data = get_execution_data(project=project,
execution=suite_name,
timestamp=timestamp)
exec_dir = execution_report_path(project, suite_name, timestamp)
return {
'exec_dir': exec_dir,
'report_path': os.path.join(exec_dir, 'report.json'),
'suite_name': suite_name,
'timestamp': timestamp,
'exec_data': exec_data
}
def generate_html_report(project, execution, timestamp, destination_folder=None,
report_name=None, no_images=False):
"""Generate static HTML report.
Report is generated in <report_directory>/<report_name>
By default it's generated in <testdir>/projects/<project>/reports/<suite>/<timestamp>
Default name is 'report.html' and 'report-no-images.html'
"""
execution_directory = exec_report.execution_report_path(project, execution, timestamp)
if destination_folder is None:
destination_folder = execution_directory
if not report_name:
if no_images:
report_name = 'report-no-images'
else:
report_name = 'report'
formatted_date = utils.get_date_time_from_timestamp(timestamp)
app = gui.create_app()
static_folder = app.static_folder
# css paths
css_folder = os.path.join(static_folder, 'css')
boostrap_css = os.path.join(css_folder, 'bootstrap', 'bootstrap.min.css')
main_css = os.path.join(css_folder, 'main.css')
report_css = os.path.join(css_folder, 'report.css')
# js paths
js_folder = os.path.join(static_folder, 'js')
main_js = os.path.join(js_folder, 'main.js')
jquery_js = os.path.join(js_folder, 'external', 'jquery.min.js')
datatables_js = os.path.join(js_folder, 'external', 'datatable', 'datatables.min.js')
bootstrap_js = os.path.join(js_folder, 'external', 'bootstrap.min.js')
report_execution_js = os.path.join(js_folder, 'report_execution.js')
css = {
'bootstrap': open(boostrap_css, encoding='utf-8').read(),
'main': open(main_css, encoding='utf-8').read(),
'report': open(report_css, encoding='utf-8').read()
}
js = {
'jquery': open(jquery_js, encoding='utf-8').read(),
'datatables': open(datatables_js, encoding='utf-8').read(),
'bootstrap': open(bootstrap_js, encoding='utf-8').read(),
'main': open(main_js, encoding='utf-8').read(),
'report_execution': open(report_execution_js).read()
}
execution_data = exec_report.get_execution_data(execution_directory)
detail_test_data = {}
for test in execution_data['tests']:
test_detail = exec_report.function_test_execution_result(
project, execution, timestamp, test['test_file'], test['test'], test['set_name'],
no_screenshots=no_images, encode_screenshots=True
)
# testId is test_file + test + set_name
test_id = f"{test['test_file']}.{test['test']}"
if test['set_name']:
test_id = f"{test_id}.{test['set_name']}"
detail_test_data[test_id] = test_detail
with app.app_context():
html_string = render_template(
'report/report_execution_static.html', project=project, execution=execution,
timestamp=timestamp, execution_data=execution_data,
detail_test_data=detail_test_data, formatted_date=formatted_date,
css=css, js=js, static=True
)
_, file_extension = os.path.splitext(report_name)
if not file_extension:
report_name = f'{report_name}.html'
destination = os.path.join(destination_folder, report_name)
if not os.path.exists(os.path.dirname(destination)):
os.makedirs(os.path.dirname(destination), exist_ok=True)
try:
with open(destination, 'w', encoding='utf-8') as f:
f.write(html_string)
except IOError as e:
if e.errno == errno.EACCES:
print(f'ERROR: cannot write to {destination}, PermissionError (Errno 13)')
else:
print(f'ERROR: There was an error writing to {destination}')
return html_string
def generate_junit_report(project_name, execution, timestamp, report_folder=None,
report_name=None):
"""Generate a report in JUnit XML format.
Output conforms to https://github.com/jenkinsci/xunit-plugin/blob/master/
src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd
"""
data = get_execution_data(project=project_name, execution=execution, timestamp=timestamp)
totals = data['totals_by_result']
errors = totals.get(Results.CODE_ERROR, 0)
failures = totals.get(Results.FAILURE, 0) + totals.get(Results.ERROR, 0)
skipped = totals.get(Results.SKIPPED, 0)
testsuites_attrs = {
'name': execution,
'errors': str(errors),
'failures': str(failures),
'tests': str(data['total_tests']),
'time': str(data['net_elapsed_time'])
}
testsuites = ET.Element('testsuites', testsuites_attrs)
testsuites_attrs['timestamp'] = timestamp
testsuites_attrs['skipped'] = str(skipped)
testsuite = ET.SubElement(testsuites, 'testsuite', testsuites_attrs)
for test in data['tests']:
class_name = test['test_file']
if test['set_name']:
class_name = f"{class_name}_{test['set_name']}"
test_attrs = {
'name': test['test'],
'classname': class_name,
'time': str(test['elapsed_time'])
}
testcase = ET.SubElement(testsuite, 'testcase', test_attrs)
# testcase nodes can contain 'failure', 'error', and 'skipped' sub-nodes
# matching Golem 'error' and 'failure' to JUnit 'failure',
# 'code error' to 'error', and 'skipped' to 'skipped'
#
# A Golem test can have 0 or more errors and 0 or 1 failures.
# Correct mapping would be one sub-node for each of these.
# The list of errors for a test is currently not available in the
# execution json.
if test['result'] in (Results.CODE_ERROR, Results.FAILURE, Results.ERROR, Results.SKIPPED):
if test['result'] in [Results.ERROR, Results.FAILURE]:
error_type = 'failure'
elif test['result'] == Results.CODE_ERROR:
error_type = 'error'
else:
error_type = 'skipped'
error_data = {
'type': test['result'],
'message': str(test['test_data'])
}
error_message = ET.SubElement(testcase, error_type, error_data)
# add debug log to /test/system-out node
log_lines = get_test_debug_log(project_name, execution, timestamp,
test['test_file'], test['set_name'])
log_string = '\n'.join(log_lines)
system_out = ET.SubElement(testcase, 'system-out')
system_out.text = _clean_illegal_xml_chars(log_string)
xmlstring = ET.tostring(testsuites)
doc = minidom.parseString(xmlstring).toprettyxml(indent=' ' * 4, encoding='UTF-8')
if not report_folder:
report_folder = execution_report_path(project_name, execution, timestamp)
if not report_name:
report_name = 'report'
report_path = os.path.join(report_folder, report_name + '.xml')
if not os.path.exists(os.path.dirname(report_path)):
os.makedirs(os.path.dirname(report_path), exist_ok=True)
try:
with open(report_path, 'w', encoding='utf-8') as f:
f.write(doc.decode('UTF-8'))
except IOError as e:
if e.errno == errno.EACCES:
print(f'ERROR: cannot write to {report_path}, PermissionError (Errno 13)')
else:
print(f'ERROR: There was an error writing to {report_path}')
return doc
