def format_script_results(self, script_execution_id):
script_execution = ScriptDAO.get_script_execution_details(
script_execution_id)
status, passed, total = ScriptDAO.get_script_execution_statistics(
script_execution_id)
overall_status = ResultStatus.PASSED if status else ResultStatus.FAILED
results_header = "Test script{horizontal_separator}{test_script_path}{vertical_separator}".format(
horizontal_separator=HORIZONTAL_SEPARATOR,
test_script_path=script_execution.script_path,
vertical_separator=VERTICAL_SEPARATOR)
results_header += "Overall status{horizontal_separator}{overall_status}{vertical_separator}".format(
horizontal_separator=HORIZONTAL_SEPARATOR,
overall_status=overall_status,
vertical_separator=VERTICAL_SEPARATOR)
results_header += "Pass rate{horizontal_separator}{passed}/{total}{vertical_separator}".format(
horizontal_separator=HORIZONTAL_SEPARATOR,
passed=passed,
total=total,
vertical_separator=VERTICAL_SEPARATOR)
results_header += "configuration:{vertical_separator}".format(
vertical_separator=VERTICAL_SEPARATOR)
for param in script_execution.configuration.split("--"):
if not param:
continue
param_name = param.split(" ")[0]
if param_name in [PARAM_USER, PARAM_PASSWORD]:
param_value = "**********"
else:
param_value = "".join(param.split(" ")[1:])
results_header += \
"{horizontal_separator}" \
"{param_name:10}{horizontal_separator}" \
"{param_value}{vertical_separator}".format(
vertical_separator=VERTICAL_SEPARATOR,
param_name=param_name,
param_value=param_value,
horizontal_separator=HORIZONTAL_SEPARATOR)
results_header += "Reported cases"
yield results_header
cases_results = (
self.format_case_results(case_execution_id)
for case_execution_id in CaseDAO.
get_cases_execution_ids_for_script_execution(script_execution_id))
yield VERTICAL_SEPARATOR
for case_result in cases_results:
for line in case_result:
yield line
yield VERTICAL_SEPARATOR
def save_results(self, run_id):
status, passed, total = RunDAO.get_overall_run_result(run_id)
text_status = HtmlFormatter.PASSED if status else HtmlFormatter.FAILED
script_execution_ids = ScriptDAO.get_scripts_execution_ids_for_run(
run_id)
test_scripts_result = ""
for script_execution_id in script_execution_ids:
script_execution_details = ScriptDAO.get_script_execution_details(
script_execution_id)
self.save_script_results(script_execution_id, run_id)
overall, passed, total = ScriptDAO.get_script_execution_statistics(
script_execution_id)
if passed == total:
li_style = "success"
glyphicon = "ok"
elif overall:
li_style = "warning"
glyphicon = "exclamation-sign"
else:
li_style = "danger"
glyphicon = "remove"
test_scripts_result += "<li class='list-group-item list-item-%s'><a class='report' " \
"href='%s_script_report/%s_script_report.html'><span class='report glyphicon " \
"glyphicon-%s'></span> %s</a></li>" % (
li_style, script_execution_details.id, script_execution_details.id, glyphicon,
script_execution_details.script_path)
result_report = HtmlFormatter.RESULT_RUN_REPORT.format(
status=text_status,
passed=passed,
total=total,
test_script_result=test_scripts_result)
case_results_dir = '/'.join(
(HtmlFormatter.REPORT_FOLDER, "%s_run_report" % run_id))
case_results_html = '/'.join(
(case_results_dir, "%s_run_report.html" % run_id))
self._check_and_make_path(case_results_dir)
parent_lvl = self._parent_level(1)
with open(case_results_html, "w") as report:
report.write("{header}{resultRunReport}{footer}".format(
header=HtmlFormatter.HEADER.format(parent_level=parent_lvl),
resultRunReport=result_report,
footer=HtmlFormatter.FOOTER.format(
version=BuildInformation.BUILD_VERSION,
parent_level=parent_lvl)))
def format_script_results(self, script_execution_id):
script_execution = ScriptDAO.get_script_execution_details(script_execution_id)
status, passed, total = ScriptDAO.get_script_execution_statistics(script_execution_id)
overall_status = ResultStatus.PASSED if status else ResultStatus.FAILED
results_header = "Test script{horizontal_separator}{test_script_path}{vertical_separator}".format(
horizontal_separator=HORIZONTAL_SEPARATOR, test_script_path=script_execution.script_path,
vertical_separator=VERTICAL_SEPARATOR)
results_header += "Overall status{horizontal_separator}{overall_status}{vertical_separator}".format(
horizontal_separator=HORIZONTAL_SEPARATOR, overall_status=ColorPrinter.format_status(overall_status),
vertical_separator=VERTICAL_SEPARATOR)
results_header += "Pass rate{horizontal_separator}{passed}/{total}{vertical_separator}".format(
horizontal_separator=HORIZONTAL_SEPARATOR, passed=passed, total=total,
vertical_separator=VERTICAL_SEPARATOR)
results_header += "configuration:{vertical_separator}".format(vertical_separator=VERTICAL_SEPARATOR)
for param in script_execution.configuration.split("--"):
if not param:
continue
param_name = param.split(" ")[0]
if param_name in [PARAM_USER, PARAM_PASSWORD]:
param_value = "**********"
else:
param_value = "".join(param.split(" ")[1:])
results_header += \
"{horizontal_separator}" \
"{param_name:10}{horizontal_separator}" \
"{param_value}{vertical_separator}" \
.format(vertical_separator=VERTICAL_SEPARATOR,
param_name=param_name,
param_value=param_value,
horizontal_separator=HORIZONTAL_SEPARATOR)
results_header += "Level of details -d{level}\n".format(level=self.message_level)
results_header += "Reported cases"
yield results_header
cases_results = (self.format_case_results(case_execution_id) for case_execution_id in
CaseDAO.get_cases_execution_ids_for_script_execution(script_execution_id))
yield VERTICAL_SEPARATOR
for case_result in cases_results:
for line in case_result:
yield line
yield VERTICAL_SEPARATOR
def process_action(self, configuration):
test_runs = RunDAO.list_all_runs()
print "Run ID\tDate\n"
for test_run in test_runs:
print "%s\n[%s]\t%s\n\n\t%s\n" % ("-"*89, test_run.id, test_run.run_datetime,
to_multiline(test_run.cmd, 80, '\\\n\t'))
for script_execution_id in ScriptDAO.get_scripts_execution_ids_for_run(test_run.id):
script_execution = ScriptDAO.get_script_execution_details(script_execution_id)
cmd = "\t[id=%d]\n\t%s execute %s" % (script_execution_id,
script_execution.script_path,
script_execution.configuration)
print to_multiline(cmd, 80, ' \\\n\t')
print '\n\n'
def save_script_results(self, script_execution_id, run_id):
cases_execution_ids = CaseDAO.get_cases_execution_ids_for_script_execution(
script_execution_id)
status, passed, total = ScriptDAO.get_script_execution_statistics(
script_execution_id)
text_status = HtmlFormatter.PASSED if status else HtmlFormatter.FAILED
test_script_result = ""
for case_execution_id in cases_execution_ids:
case_execution_details = CaseDAO.get_case_execution_details(
case_execution_id)
if case_execution_details.status == ResultStatus.PASSED:
li_style = "success"
glyphicon = "ok"
elif case_execution_details.status in [
ResultStatus.FAILED, ResultStatus.TIMEOUT
]:
li_style = "warning"
glyphicon = "exclamation-sign"
elif case_execution_details.status in [
ResultStatus.BLOCKED, ResultStatus.UNKNOWN
]:
li_style = "danger"
glyphicon = "remove"
else:
li_style = "neutral"
glyphicon = "minus"
test_script_result += "<li class='list-group-item list-item-%s'><a class='report' " \
"href='case_results/%s_case_results.html'><span class='report glyphicon " \
"glyphicon-%s'></span> %s</a></li>" % (li_style,
case_execution_details.id,
glyphicon,
case_execution_details.name)
self.save_case_result(case_execution_id, script_execution_id,
run_id)
result_report = HtmlFormatter.RESULT_SCRIPT_REPORT.format(
status=text_status,
passed=passed,
total=total,
test_script_result=test_script_result)
script_path = '/'.join(
(HtmlFormatter.REPORT_FOLDER, "%s_run_report" % run_id,
"%s_script_report" % script_execution_id))
script_html_path = '/'.join(
(script_path, "%s_script_report.html" % script_execution_id))
self._check_and_make_path(script_path)
parent_lvl = self._parent_level(2)
with open(script_html_path, "w") as report:
report.write("{header}{resultRunReport}{footer}".format(
header=HtmlFormatter.HEADER.format(parent_level=parent_lvl),
resultRunReport=result_report,
footer=HtmlFormatter.FOOTER.format(
version=BuildInformation.BUILD_VERSION,
parent_level=parent_lvl)))
def get_overall_run_result(run_id, database_session):
"""
:type run_id: int
:rtype: bool
"""
overall_status = True
total = 0
passed = 0
for script_id in ScriptDAO.get_scripts_execution_ids_for_run(run_id):
total += 1
status, _, _ = ScriptDAO.get_script_execution_statistics(script_id)
overall_status = overall_status and status
if status:
passed += 1
return overall_status, passed, total
def process_action(self, configuration):
replay_id = configuration.replay_id[0]
print "Using CTS in version %s to replay execution %s" \
% (ColorPrinter.format_text(BuildInformation.BUILD_VERSION, bold=True), replay_id)
error, script_execution_id = split_replay_id(replay_id)
if error:
return
# TODO: warn user when he tries to replay using newer CTS
script_execution = ScriptDAO.get_script_execution_details(
script_execution_id)
if script_execution is None:
cts_error(
"Recording for script execution id={id:ignore} not found",
id=script_execution_id)
return
script_path = script_execution.script_path
configuration = self._configuration_from_string(
script_execution.configuration)
test_plan = self._prepare_test_plan(script_path)
environ[ReplayController.CTS_REPLAY_SCRIPT_EXECUTION_ID] = str(
script_execution_id)
self._execute(configuration, test_plan)
def get_overall_run_result(run_id, database_session):
"""
:type run_id: int
:rtype: bool
"""
overall_status = True
total = 0
passed = 0
for script_id in ScriptDAO.get_scripts_execution_ids_for_run(run_id):
total += 1
status, _, _ = ScriptDAO.get_script_execution_statistics(script_id)
overall_status = overall_status and status
if status:
passed += 1
return overall_status, passed, total
def process_action(self, configuration):
test_runs = RunDAO.list_all_runs()
print "Run ID\tDate\n"
for test_run in test_runs:
print "%s\n[%s]\t%s\n\n\t%s\n" % (
"-" * 89, test_run.id, test_run.run_datetime,
to_multiline(test_run.cmd, 80, '\\\n\t'))
for script_execution_id in ScriptDAO.get_scripts_execution_ids_for_run(
test_run.id):
script_execution = ScriptDAO.get_script_execution_details(
script_execution_id)
cmd = "\t[id=%d]\n\t%s execute %s" % (
script_execution_id, script_execution.script_path,
script_execution.configuration)
print to_multiline(cmd, 80, ' \\\n\t')
print '\n\n'
def save_results(self, run_id):
status, passed, total = RunDAO.get_overall_run_result(run_id)
text_status = HtmlFormatter.PASSED if status else HtmlFormatter.FAILED
script_execution_ids = ScriptDAO.get_scripts_execution_ids_for_run(run_id)
test_scripts_result = ""
for script_execution_id in script_execution_ids:
script_execution_details = ScriptDAO.get_script_execution_details(script_execution_id)
self.save_script_results(script_execution_id, run_id)
overall, passed, total = ScriptDAO.get_script_execution_statistics(script_execution_id)
if passed == total:
li_style = "success"
glyphicon = "ok"
elif overall:
li_style = "warning"
glyphicon = "exclamation-sign"
else:
li_style = "danger"
glyphicon = "remove"
test_scripts_result += "<li class='list-group-item list-item-%s'><a class='report' " \
"href='%s_script_report/%s_script_report.html'><span class='report glyphicon " \
"glyphicon-%s'></span> %s</a></li>" % (
li_style, script_execution_details.id, script_execution_details.id, glyphicon,
script_execution_details.script_path)
result_report = HtmlFormatter.RESULT_RUN_REPORT.format(status=text_status, passed=passed, total=total,
test_script_result=test_scripts_result)
case_results_dir = '/'.join((HtmlFormatter.REPORT_FOLDER,
"%s_run_report" % run_id))
case_results_html = '/'.join((case_results_dir,
"%s_run_report.html" % run_id))
self._check_and_make_path(case_results_dir)
parent_lvl = self._parent_level(1)
with open(case_results_html, "w") as report:
report.write("{header}{resultRunReport}{footer}".format(
header=HtmlFormatter.HEADER.format(parent_level=parent_lvl),
resultRunReport=result_report,
footer=HtmlFormatter.FOOTER.format(version=BuildInformation.BUILD_VERSION, parent_level=parent_lvl)))
def __init__(self, configuration):
try:
ssl = strtobool(configuration.UseSSL)
except ValueNotFound:
ssl = False
self._links_factory = LinksFactory(configuration.ApiEndpoint, ssl=ssl)
self._config_property_reader = configuration
try:
self.script_execution_id = ScriptDAO.get_last_script_execution_id()
if self.script_execution_id != None:
script_execution = ScriptDAO.get_script_execution_details(self.script_execution_id)
if getppid() != script_execution.pid:
# this is probably replay or running test script without framework
# do not register requests
self.script_execution_id = None
except:
self.script_execution_id = None
self.request_registration = self.script_execution_id is not None
def _prepare_test_case_list(self, configuration, run_id, test_case_list):
for script in ScriptDAO.get_scripts_execution_ids_for_run(run_id):
Case = collections.namedtuple('Case', 'status message response')
cases_execution_ids = CaseDAO.get_cases_execution_ids_for_script_execution(script)
status, _, _ = ScriptDAO.get_script_execution_statistics(script)
configuration = MetadataReportAction.get_configuration_for_test_case(run_id)
if 'RedfishMetadata' not in configuration:
print('Test report {} is not compatible with Interoperability Report operation.\n'
'You can only include a test case done with Redfish Metadata {}'
.format(run_id,
MetadataReportAction.REDFISH_CHOICES))
continue
self.__get_messages_for_case_execution(Case, cases_execution_ids,
self.__get_raw_responses_from_db(),
status,
test_case_list)
return configuration
def _prepare_test_case_list(self, configuration, run_id, test_case_list):
for script in ScriptDAO.get_scripts_execution_ids_for_run(run_id):
Case = collections.namedtuple('Case', 'status message response')
cases_execution_ids = CaseDAO.get_cases_execution_ids_for_script_execution(script)
status, _, _ = ScriptDAO.get_script_execution_statistics(script)
configuration = MetadataReportAction.get_configuration_for_test_case(run_id)
if 'RedfishMetadata' not in configuration:
print('Test report {} is not compatible with Interoperability Report operation.\n'
'You can only include a test case done with Redfish Metadata {}'
.format(run_id,
MetadataReportAction.REDFISH_CHOICES))
continue
self.__get_messages_for_case_execution(Case, cases_execution_ids,
self.__get_raw_responses_from_db(),
status,
test_case_list)
return configuration
def __init__(self, configuration):
try:
ssl = strtobool(configuration.UseSSL)
except ValueNotFound:
ssl = False
self._links_factory = LinksFactory(configuration.ApiEndpoint, ssl=ssl)
self._config_property_reader = configuration
try:
self.script_execution_id = ScriptDAO.get_last_script_execution_id()
if self.script_execution_id != None:
script_execution = ScriptDAO.get_script_execution_details(self.script_execution_id)
if getppid() != script_execution.pid:
# this is probably replay or running test script without framework
# do not register requests
self.script_execution_id = None
except:
self.script_execution_id = None
self.request_registration = self.script_execution_id is not None
def save_script_results(self, script_execution_id, run_id):
cases_execution_ids = CaseDAO.get_cases_execution_ids_for_script_execution(script_execution_id)
status, passed, total = ScriptDAO.get_script_execution_statistics(script_execution_id)
text_status = HtmlFormatter.PASSED if status else HtmlFormatter.FAILED
test_script_result = ""
for case_execution_id in cases_execution_ids:
case_execution_details = CaseDAO.get_case_execution_details(case_execution_id)
if case_execution_details.status == ResultStatus.PASSED:
li_style = "success"
glyphicon = "ok"
elif case_execution_details.status in [ResultStatus.FAILED, ResultStatus.TIMEOUT]:
li_style = "warning"
glyphicon = "exclamation-sign"
elif case_execution_details.status in [ResultStatus.BLOCKED, ResultStatus.UNKNOWN]:
li_style = "danger"
glyphicon = "remove"
else:
li_style = "neutral"
glyphicon = "minus"
test_script_result += "<li class='list-group-item list-item-%s'><a class='report' " \
"href='case_results/%s_case_results.html'><span class='report glyphicon " \
"glyphicon-%s'></span> %s</a></li>" % (li_style,
case_execution_details.id,
glyphicon,
case_execution_details.name)
self.save_case_result(case_execution_id, script_execution_id, run_id)
result_report = HtmlFormatter.RESULT_SCRIPT_REPORT.format(status=text_status, passed=passed, total=total,
test_script_result=test_script_result)
script_path = '/'.join((HtmlFormatter.REPORT_FOLDER,
"%s_run_report" % run_id,
"%s_script_report" % script_execution_id))
script_html_path = '/'.join((script_path,
"%s_script_report.html" % script_execution_id))
self._check_and_make_path(script_path)
parent_lvl = self._parent_level(2)
with open(script_html_path, "w") as report:
report.write(
"{header}{resultRunReport}{footer}".format(header=HtmlFormatter.HEADER.format(parent_level=parent_lvl),
resultRunReport=result_report,
footer=HtmlFormatter.FOOTER.format(
version=BuildInformation.BUILD_VERSION,
parent_level=parent_lvl)))
def get_configuration_for_test_case(run_id):
script_execution = ScriptDAO.get_script_execution_details(run_id)
param_dict = {'script_path': script_execution.script_path.split('/')[-5:]}
for param in script_execution.configuration.split("--"):
if not param:
continue
param_name = param.split(" ")[0]
if param_name in [PARAM_USER, PARAM_PASSWORD]:
param_value = "**********"
else:
param_value = "".join(param.split(" ")[1:])
param_dict[param_name] = param_value
return param_dict
def get_configuration_for_test_case(run_id):
script_execution = ScriptDAO.get_script_execution_details(run_id)
param_dict = {'script_path': script_execution.script_path.split('/')[-5:]}
for param in script_execution.configuration.split("--"):
if not param:
continue
param_name = param.split(" ")[0]
if param_name in [PARAM_USER, PARAM_PASSWORD]:
param_value = "**********"
else:
param_value = "".join(param.split(" ")[1:])
param_dict[param_name] = param_value
return param_dict
def print_results(self, run_id):
run_details = RunDAO.get_run_details(run_id)
results_header = "Test run ID{horizontal_separator} #{run_id}""".format(
horizontal_separator=HORIZONTAL_SEPARATOR,
run_id=run_details.id)
print "CTS version\t%s\n%s%s" % (BuildInformation.BUILD_VERSION, results_header, VERTICAL_SEPARATOR)
script_executions_results = (self.format_script_results(script_execution_id) for script_execution_id in
ScriptDAO.get_scripts_execution_ids_for_run(run_id))
for script_results in script_executions_results:
for line in script_results:
# replace comma to dot to avoid conflict in opening CSV format and drop empty line
if line is not VERTICAL_SEPARATOR:
sys.stdout.write(line.replace(',', '.'))
print VERTICAL_SEPARATOR
def print_results(self, run_id):
run_details = RunDAO.get_run_details(run_id)
results_header = "Test run ID{horizontal_separator} #{run_id}""".format(
horizontal_separator=HORIZONTAL_SEPARATOR,
run_id=run_details.id)
print "CTS version\t%s\n%s%s" % (BuildInformation.get_version(), results_header, VERTICAL_SEPARATOR)
script_executions_results = (self.format_script_results(script_execution_id) for script_execution_id in
ScriptDAO.get_scripts_execution_ids_for_run(run_id))
lines = []
for script_results in script_executions_results:
for line in script_results:
sys.stdout.write(line)
lines.append(line)
print VERTICAL_SEPARATOR
lines.append(VERTICAL_SEPARATOR)
return lines
def print_results(self, run_id):
run_details = RunDAO.get_run_details(run_id)
results_header = "Test run ID{horizontal_separator} #{run_id}""".format(
horizontal_separator=HORIZONTAL_SEPARATOR,
run_id=run_details.id)
print "CTS version\t%s\n%s%s" % (BuildInformation.BUILD_VERSION, results_header, VERTICAL_SEPARATOR)
script_executions_results = (self.format_script_results(script_execution_id) for script_execution_id in
ScriptDAO.get_scripts_execution_ids_for_run(run_id))
lines = []
for script_results in script_executions_results:
for line in script_results:
sys.stdout.write(line)
lines.append(line)
print VERTICAL_SEPARATOR
lines.append(VERTICAL_SEPARATOR)
return lines
def print_results(self, run_id):
run_details = RunDAO.get_run_details(run_id)
results_header = "Test run ID{horizontal_separator} #{run_id}" "".format(
horizontal_separator=HORIZONTAL_SEPARATOR, run_id=run_details.id)
print "CTS version\t%s\n%s%s" % (BuildInformation.get_version(),
results_header, VERTICAL_SEPARATOR)
script_executions_results = (
self.format_script_results(script_execution_id)
for script_execution_id in
ScriptDAO.get_scripts_execution_ids_for_run(run_id))
for script_results in script_executions_results:
for line in script_results:
# replace comma to dot to avoid conflict in opening CSV format and drop empty line
if line is not VERTICAL_SEPARATOR:
sys.stdout.write(line.replace(',', '.'))
print VERTICAL_SEPARATOR
def register_execution(self, test_run_id):
self.test_run_id = test_run_id
self.test_script_execution_id = ScriptDAO.register_script_execution(self.test_script.path, test_run_id,
self.configuration,
self.test_script.control_sum)